From 172755131bf9664b6f5b13622ea53a53bf399699 Mon Sep 17 00:00:00 2001 From: Tanvirul Date: Mon, 27 Jul 2026 01:37:56 -0400 Subject: [PATCH] Simplify results and installation --- README.md | 44 ++++++----------------------- docs/QUICKSTART.md | 27 ++++++++++++++++-- docs/README.md | 35 +++-------------------- tests/test_visibility_surfaces.py | 46 +++++++++++++++++++++++++++++-- 4 files changed, 79 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 240e2c64..2b13c528 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ points**, respectively. [See the transfer results](#external-benchmark-transfer)

Live Demo Paper + Hugging Face Collection Dataset - Reproduce

![Trace examples across charts, games, geometry, graphs, icons, illustrations, pages, physics, puzzles, symbolic reasoning, and 3D scenes](docs/assets/paper-domain-montage/trace-paper-domain-montage.png) @@ -70,19 +70,6 @@ aligned. Each finalized record contains the rendered problem, exact scoring contract, image-space annotation, and an execution-trace reference for inspection and replay. -## TRACE Validation - -On 2,000 previously unseen instances generated from the same 1,000 task -programs, TRACE training improves accuracy at both model scales. - -| Model scale | Base | TRACE | Change | -| --- | ---: | ---: | ---: | -| Qwen2.5-VL-3B | 24.45 | 41.05 | +16.60 | -| Qwen2.5-VL-7B | 34.25 | 51.55 | +17.30 | - -These results measure new realizations within the TRACE task distributions; -the evaluation below measures transfer to external benchmarks. - ## External Benchmark Transfer Qwen2.5-VL models trained on 64,000 Trace instances improve the unweighted @@ -96,6 +83,9 @@ macro-average across 24 external benchmarks at both evaluated model scales. Values are mean ± sample standard deviation across decoding seeds 42, 43, and 44. Paired changes compare matched benchmark and seed results. +On 2,000 unseen TRACE instances, accuracy improves from 24.45 to 41.05 at 3B +and from 34.25 to 51.55 at 7B. + ### Base versus TRACE by benchmark The table reports all 24 external benchmarks for the matched 3B and 7B @@ -202,30 +192,12 @@ for training, evaluation, progress reporting, and validation. ```bash git clone https://github.com/maveryn/trace.git cd trace -python3 -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip setuptools wheel -python -m pip install -e ".[test]" +python -m pip install -e . ``` -Trace supports Python 3.10 through 3.14. Use the reproducibility constraints on -Python 3.10-3.12 when generating datasets or documentation assets: - -```bash -python -m pip install -c constraints/release.txt -e ".[test]" -``` - -For Python 3.14 package and CLI development, use: - -```bash -python -m pip install -c constraints/compat-py314.txt -e ".[test]" -``` - -Install Parquet and Hugging Face export support with: - -```bash -python -m pip install -e ".[test,export]" -``` +See the [installation documentation](docs/QUICKSTART.md#install) for +reproducibility constraints, export dependencies, and Python 3.14 +compatibility. ## Generate diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 0396578b..c4713408 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -2,13 +2,34 @@ ## Install -From a clone of the repository: +For a standard installation from a clone: + +```bash +git clone https://github.com/maveryn/trace.git +cd trace +python -m pip install -e . +``` + +For development and reproducible generation on Python 3.10-3.12, create and +activate a virtual environment, then install the release constraints: ```bash python3 -m venv .venv source .venv/bin/activate -python -m pip install --upgrade pip setuptools wheel -python -m pip install -e ".[test]" +python -m pip install -c constraints/release.txt -e ".[test]" +``` + +Use the compatibility constraints for package and CLI development on Python +3.14: + +```bash +python -m pip install -c constraints/compat-py314.txt -e ".[test]" +``` + +Install Parquet and Hugging Face export support with: + +```bash +python -m pip install -e ".[export]" ``` ## Inspect The Registry diff --git a/docs/README.md b/docs/README.md index fb9c9629..4b265198 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,8 +37,8 @@ hide: Live Demo Code Paper + Collection Dataset - Reproduce @@ -141,36 +141,6 @@ aligned. Each finalized record contains the rendered problem, exact scoring contract, image-space annotation, and an execution-trace reference for inspection and replay. -## TRACE validation - -The released checkpoints improve accuracy on 2,000 previously unseen -instances generated from the same 1,000 task programs. Each model is evaluated -once with decoding seed 42. - -
-
- Qwen2.5-VL-3B -
- Base24.45 - - Trace41.05 -
- Improvement: +16.60 -
-
- Qwen2.5-VL-7B -
- Base34.25 - - Trace51.55 -
- Improvement: +17.30 -
-
- -These results measure new realizations within the TRACE task distributions. -The evaluation below measures transfer to external benchmarks. - ## External benchmark transfer Qwen2.5-VL models trained on 64,000 Trace instances improve the macro-average @@ -199,6 +169,9 @@ compare matched seeds. +On 2,000 unseen TRACE instances, accuracy improves from 24.45 to 41.05 at 3B +and from 34.25 to 51.55 at 7B. + ### Base versus TRACE by benchmark The table reports all 24 external benchmarks for the matched 3B and 7B diff --git a/tests/test_visibility_surfaces.py b/tests/test_visibility_surfaces.py index 69c1a984..e37cdfb3 100644 --- a/tests/test_visibility_surfaces.py +++ b/tests/test_visibility_surfaces.py @@ -17,6 +17,10 @@ "percentage points**" ) BENCHMARK_EXPANDER = "View all 24 benchmark results" +TRACE_VALIDATION_SUMMARY = ( + "On 2,000 unseen TRACE instances, accuracy improves from 24.45 to 41.05 " + "at 3B and from 34.25 to 51.55 at 7B." +) def test_landing_quickstarts_follow_the_domain_montage() -> None: @@ -94,9 +98,9 @@ def test_primary_resource_links_stay_focused() -> None: readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") badge_block = readme.split('

', 1)[1].split("

", 1)[0] assert badge_block.count(" None: 1, )[0] assert hero_links.count('class="md-button') == 5 - for label in ("Live Demo", "Code", "Paper", "Dataset", "Reproduce"): + for label in ("Live Demo", "Code", "Paper", "Collection", "Dataset"): assert f">{label}" in hero_links + assert ">Reproduce" not in hero_links + + +def test_landing_pages_keep_external_transfer_as_the_single_results_section() -> None: + pages = { + "README.md": "## External Benchmark Transfer", + "docs/README.md": "## External benchmark transfer", + } + for relative_path, transfer_heading in pages.items(): + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + normalized = " ".join(text.split()) + assert "## TRACE Validation" not in text + assert "## TRACE validation" not in text + assert normalized.count(TRACE_VALIDATION_SUMMARY) == 1 + assert text.index(transfer_heading) < text.index( + "" + ) + assert "Improvement: +16.60" not in text + + +def test_readme_keeps_installation_brief_and_links_the_full_guide() -> None: + readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") + section = readme.split("## Installation", 1)[1].split("## Generate", 1)[0] + assert section.count("```bash") == 1 + assert "python -m pip install -e ." in section + assert "docs/QUICKSTART.md#install" in section + for detail in (".venv", "constraints/release.txt", "compat-py314.txt", "[export]"): + assert detail not in section + + guide = (REPO_ROOT / "docs/QUICKSTART.md").read_text(encoding="utf-8") + for detail in ( + "constraints/release.txt", + "constraints/compat-py314.txt", + '".[export]"', + ): + assert detail in guide def test_public_contribution_surfaces_are_linked() -> None: