From 3fe3d38925f8bf855dfb9307560291c1f2db02f7 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Wed, 2 Sep 2026 15:33:50 +0200 Subject: [PATCH] Advance the real-project-v2 candidate registry to Bifrost v0.10.8 The v2 slice's execution identities are frozen in a versioned registry, and the registry the freeze selected still pinned Bifrost v0.10.2 -- so every v2 freeze re-measured the analyzer of a month ago by construction. The slice's own README states the doctrine for moving forward: a later Bifrost run is evidence for a subsequent release, never a replacement for v0.3.0 evidence. `candidates-v0.3.4.json` is that subsequent identity set. It advances the measured Bifrost to v0.10.8 at `8ddf1362`, the same identity the active development registry pins, and changes nothing else: every reference-server identity is byte-identical to the v0.3.0 registry, and the corpus, protocol, selection, and reviewed ground truth are untouched. `candidates-v0.3.0.json` stays byte-identical as the immutable record of what v0.3.0 measured. The freeze workflow selects the new registry for evaluation snapshots. Shard identities hash both registries and read candidate identity from the current one. The reproduction contract now asserts the superseding registry's Bifrost matches the active development pin and that no non-Bifrost identity moved, alongside its existing assertions that the historical registry is unchanged. Expected effect on the next v2 freeze, from the repository-tree measurement under v0.10.8: 12 of v0.3.0's 17 failures resolve, 5 remain (#181, #182, #183), none appear. The freeze that publishes this is the real acceptance test, and its aggregate counts -- not this prediction -- are what the release approval should be checked against. Requires the scan-budget capability fix (#191), without which a v0.10.8 evaluation freeze would error every usage scan. Closes #185 Co-Authored-By: Claude Opus 5 --- .github/workflows/freeze.yml | 2 +- CITATION.cff | 2 +- .../evaluation/real-project-v2/README.md | 9 ++++ .../real-project-v2/candidates-v0.3.4.json | 53 +++++++++++++++++++ scripts/freeze-shards.py | 3 +- scripts/validate-reproduction-contract.sh | 21 +++++++- src/freeze.rs | 22 ++++++++ tests/test_freeze_shards.py | 42 ++++++++++++++- 8 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json diff --git a/.github/workflows/freeze.yml b/.github/workflows/freeze.yml index 7e56245..0672714 100644 --- a/.github/workflows/freeze.yml +++ b/.github/workflows/freeze.yml @@ -115,7 +115,7 @@ jobs: case_path="$(jq -er .casePath <<< "$scope")" candidates="$(jq -er '.candidates | join(",")' <<< "$scope")" [[ "$REQUESTED_CANDIDATES" == "$candidates" ]] - candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json" + candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json" elif [[ "$SNAPSHOT_KIND" == "legacy-promoted" ]]; then scope="$(bash scripts/resolve-freeze-scope.sh legacy-promoted)" case_path="$(jq -er .casePath <<< "$scope")" diff --git a/CITATION.cff b/CITATION.cff index 48cfb2e..2696644 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ authors: contact: - name: "Brokk, Inc." website: "https://brokk.ai" -version: "0.3.3" +version: "0.3.4" date-released: "2026-09-02" repository-code: "https://github.com/BrokkAi/usagebench" repository-artifact: "https://github.com/BrokkAi/usagebench/releases" diff --git a/benchmarks/evaluation/real-project-v2/README.md b/benchmarks/evaluation/real-project-v2/README.md index 4b7a572..18f98f8 100644 --- a/benchmarks/evaluation/real-project-v2/README.md +++ b/benchmarks/evaluation/real-project-v2/README.md @@ -89,6 +89,15 @@ registered macOS host and has no canonical reference runner. Any later Bifrost run is evidence for a subsequent snapshot or release, never a replacement for v0.3.0 evidence. +`candidates-v0.3.4.json` is that subsequent identity set. It advances the +measured Bifrost to public release v0.10.8 at +`8ddf13625653caf927137a0de1966401055debad` -- the same identity the active +development registry pins -- and changes nothing else: every reference-server +identity is byte-identical to the v0.3.0 registry, and the corpus, protocol, +selection, and reviewed ground truth are untouched. Freezes run under it +publish as a new release; `candidates-v0.3.0.json` remains the immutable record +of what v0.3.0 measured. + The v1 selection is hash-linked by the v2 protocol. Every v1-selected repository is ineligible even if it appears in a v2 GitHub language frame. Capture, rank, and replacement decisions use public source metadata only. diff --git a/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json b/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json new file mode 100644 index 0000000..32fd086 --- /dev/null +++ b/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 3, + "candidates": [ + { + "id": "bifrost", + "runner": "bifrost", + "name": "Bifrost", + "requestedVersion": "v0.10.8", + "source": "https://github.com/BrokkAi/bifrost", + "revision": "8ddf13625653caf927137a0de1966401055debad", + "advertised": true, + "runtimeNetworking": "disabled", + "projectHydration": "fixture sources are staged in the released corpus" + }, + { + "id": "eclipse-jdtls", + "runner": "lsp", + "name": "Eclipse JDT LS", + "requestedVersion": "1.61.0-202607142124", + "source": "https://download.eclipse.org/jdtls/snapshots/", + "profile": "adapters/lsp/eclipse-jdtls.json", + "profileSha256": "a04864d9c9c03fda591e1929766a8cc698417da2b4495dce8285024b9766743e", + "advertised": true, + "runtimeNetworking": "not required after the JDT LS distribution is installed", + "projectHydration": "JDT LS imports each generated fixture project into isolated workspace data" + }, + { + "id": "rust-analyzer", + "runner": "lsp", + "name": "rust-analyzer", + "requestedVersion": "2026-07-13", + "source": "https://github.com/rust-lang/rust-analyzer/releases/tag/2026-07-13", + "profile": "adapters/lsp/rust-analyzer.json", + "profileSha256": "ff3d13a0530ca57c0710931e7e6bee6cde3e842c4a72b078f34e0e69f0c6673f", + "advertised": true, + "runtimeNetworking": "not required after the requested rust-analyzer binary and Rust toolchain are installed", + "projectHydration": "rust-analyzer loads the generated Cargo fixture workspace" + }, + { + "id": "apple-clangd-21", + "runner": "lsp", + "name": "Apple clangd", + "requestedVersion": "21.0.0", + "source": "https://developer.apple.com/xcode/", + "profile": "adapters/lsp/apple-clangd-21.json", + "profileSha256": "a386689abb1e83048204a1f23a7f0c9b39b575db1a548607ef119b647fa36651", + "resolvedVersionPrefix": "Apple clangd version 21.0.0", + "advertised": true, + "runtimeNetworking": "not required by clangd; fixture hydration is local", + "projectHydration": "UsageBench writes compile_commands.json, opens benchmark query documents, and waits for clangd fileStatus idle in each isolated workspace" + } + ] +} diff --git a/scripts/freeze-shards.py b/scripts/freeze-shards.py index a031253..e96fae9 100755 --- a/scripts/freeze-shards.py +++ b/scripts/freeze-shards.py @@ -25,6 +25,7 @@ "benchmarks/evaluation/real-project-v2/sources.json", "benchmarks/evaluation/real-project-v2/declarations.json", "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json", + "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json", ] @@ -76,7 +77,7 @@ def identity(root, version, revision, shard, corpus_hashes=None): raise ValueError(f"staged corpus hashes omit case file: {name}") hashes[name] = staged_hashes[name] registry = json.loads( - (root / "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json").read_text() + (root / "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json").read_text() ) profile = next(item for item in registry["candidates"] if item["id"] == candidate) result = { diff --git a/scripts/validate-reproduction-contract.sh b/scripts/validate-reproduction-contract.sh index 1b3dd30..d3a0572 100755 --- a/scripts/validate-reproduction-contract.sh +++ b/scripts/validate-reproduction-contract.sh @@ -10,6 +10,7 @@ scope_resolver="$repo_root/scripts/resolve-freeze-scope.sh" freeze_workflow="$repo_root/.github/workflows/freeze.yml" docs_workflow="$repo_root/.github/workflows/docs.yml" v030_registry="$repo_root/benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json" +v034_registry="$repo_root/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json" publication_validator="$repo_root/scripts/validate-publication-bundle.py" [[ -f "$publication_validator" ]] || { @@ -54,6 +55,22 @@ cargo run --locked -- validate-legacy-promotion "$repo_root/benchmarks/promotion jq -e '.schemaVersion == 3' "$registry" >/dev/null jq -e '.schemaVersion == 3' "$v030_registry" >/dev/null +jq -e '.schemaVersion == 3' "$v034_registry" >/dev/null +# The superseding evaluation registry advances only the measured Bifrost, to +# the same identity the active development registry pins; every other frozen +# candidate identity is byte-identical to the historical v0.3.0 registry. +jq -e ' + [.candidates[] | select( + .id == "bifrost" + and .requestedVersion == "v0.10.8" + and .revision == "8ddf13625653caf927137a0de1966401055debad" + )] | length == 1 +' "$v034_registry" >/dev/null +diff <(jq -S '[.candidates[] | select(.id != "bifrost")]' "$v030_registry") \ + <(jq -S '[.candidates[] | select(.id != "bifrost")]' "$v034_registry") >/dev/null || { + echo "superseding evaluation registry changes a non-Bifrost identity" >&2 + exit 1 +} jq -e ' [.candidates[] | select(.id == "bifrost")] == [{ @@ -204,8 +221,8 @@ grep -Fq 'python3 scripts/freeze-shards.py aggregate' "$freeze_workflow" || { echo "freeze workflow does not verify shard identity and coverage before aggregation" >&2 exit 1 } -grep -Fq 'benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json' "$freeze_workflow" || { - echo "v0.3.0 evaluation freeze is not bound to its historical candidate registry" >&2 +grep -Fq 'candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json"' "$freeze_workflow" || { + echo "evaluation freeze is not bound to the superseding candidate registry" >&2 exit 1 } grep -Fq "public_bifrost_source='https://github.com/BrokkAi/bifrost'" "$freeze_workflow" || { diff --git a/src/freeze.rs b/src/freeze.rs index de30504..2fb793b 100644 --- a/src/freeze.rs +++ b/src/freeze.rs @@ -1258,6 +1258,28 @@ mod tests { } } + #[test] + fn v034_evaluation_registry_advances_the_measured_bifrost() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let registry = load_registry( + &root.join("benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json"), + ) + .unwrap(); + let bifrost = registry + .candidates + .iter() + .find(|candidate| candidate.id == "bifrost") + .unwrap(); + + assert_eq!(bifrost.requested_version, "v0.10.8"); + assert_eq!(bifrost.source, "https://github.com/BrokkAi/bifrost"); + assert_eq!( + bifrost.revision.as_deref(), + Some("8ddf13625653caf927137a0de1966401055debad") + ); + assert!(bifrost.reference_runner.is_none()); + } + #[test] fn v030_evaluation_registry_uses_public_native_bifrost_identity() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/test_freeze_shards.py b/tests/test_freeze_shards.py index 8ca1f5b..bdb85b8 100644 --- a/tests/test_freeze_shards.py +++ b/tests/test_freeze_shards.py @@ -71,12 +71,52 @@ def test_v030_registry_uses_public_native_bifrost_identity(self): ) self.assertNotIn("referenceRunner", frozen_bifrost) + def test_v034_registry_advances_only_the_bifrost_pin(self): + """The superseding registry moves the measured Bifrost and nothing else. + + candidates-v0.3.0.json stays byte-identical as the historical record of + what v0.3.0 measured; the v0.3.4 registry is a new frozen identity set, + not an edit. + """ + previous = json.loads( + (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json").read_text() + ) + current = json.loads( + (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json").read_text() + ) + bifrost = next(item for item in current["candidates"] if item["id"] == "bifrost") + self.assertEqual(bifrost["requestedVersion"], "v0.10.8") + self.assertEqual(bifrost["revision"], "8ddf13625653caf927137a0de1966401055debad") + self.assertNotIn("referenceRunner", bifrost) + # The active development pin and the evaluation pin now agree. + active = json.loads((ROOT / "adapters/candidates.json").read_text()) + active_bifrost = next(item for item in active["candidates"] if item["id"] == "bifrost") + self.assertEqual(bifrost["revision"], active_bifrost["revision"]) + # Every non-Bifrost identity is unchanged from the superseded registry. + strip = lambda registry: [ + item for item in registry["candidates"] if item["id"] != "bifrost" + ] + self.assertEqual(strip(current), strip(previous)) + + def test_freeze_workflow_selects_the_v034_evaluation_registry(self): + workflow = (ROOT / ".github/workflows/freeze.yml").read_text() + self.assertIn( + "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json", workflow + ) + self.assertNotIn( + 'candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json"', + workflow, + ) + def test_apple_clangd_registries_use_exact_reported_banner_prefix(self): active = json.loads((ROOT / "adapters/candidates.json").read_text()) frozen = json.loads( (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json").read_text() ) - for registry in (active, frozen): + superseding = json.loads( + (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json").read_text() + ) + for registry in (active, frozen, superseding): apple_clangd = next( item for item in registry["candidates"] if item["id"] == "apple-clangd-21" )