diff --git a/.github/workflows/freeze.yml b/.github/workflows/freeze.yml index 7e56245..0672714 100644 --- a/.github/workflows/freeze.yml +++ b/.github/workflows/freeze.yml @@ -115,7 +115,7 @@ jobs: case_path="$(jq -er .casePath <<< "$scope")" candidates="$(jq -er '.candidates | join(",")' <<< "$scope")" [[ "$REQUESTED_CANDIDATES" == "$candidates" ]] - candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json" + candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json" elif [[ "$SNAPSHOT_KIND" == "legacy-promoted" ]]; then scope="$(bash scripts/resolve-freeze-scope.sh legacy-promoted)" case_path="$(jq -er .casePath <<< "$scope")" diff --git a/CITATION.cff b/CITATION.cff index 48cfb2e..2696644 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ authors: contact: - name: "Brokk, Inc." website: "https://brokk.ai" -version: "0.3.3" +version: "0.3.4" date-released: "2026-09-02" repository-code: "https://github.com/BrokkAi/usagebench" repository-artifact: "https://github.com/BrokkAi/usagebench/releases" diff --git a/benchmarks/evaluation/real-project-v2/README.md b/benchmarks/evaluation/real-project-v2/README.md index 4b7a572..18f98f8 100644 --- a/benchmarks/evaluation/real-project-v2/README.md +++ b/benchmarks/evaluation/real-project-v2/README.md @@ -89,6 +89,15 @@ registered macOS host and has no canonical reference runner. Any later Bifrost run is evidence for a subsequent snapshot or release, never a replacement for v0.3.0 evidence. +`candidates-v0.3.4.json` is that subsequent identity set. It advances the +measured Bifrost to public release v0.10.8 at +`8ddf13625653caf927137a0de1966401055debad` -- the same identity the active +development registry pins -- and changes nothing else: every reference-server +identity is byte-identical to the v0.3.0 registry, and the corpus, protocol, +selection, and reviewed ground truth are untouched. Freezes run under it +publish as a new release; `candidates-v0.3.0.json` remains the immutable record +of what v0.3.0 measured. + The v1 selection is hash-linked by the v2 protocol. Every v1-selected repository is ineligible even if it appears in a v2 GitHub language frame. Capture, rank, and replacement decisions use public source metadata only. diff --git a/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json b/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json new file mode 100644 index 0000000..32fd086 --- /dev/null +++ b/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 3, + "candidates": [ + { + "id": "bifrost", + "runner": "bifrost", + "name": "Bifrost", + "requestedVersion": "v0.10.8", + "source": "https://github.com/BrokkAi/bifrost", + "revision": "8ddf13625653caf927137a0de1966401055debad", + "advertised": true, + "runtimeNetworking": "disabled", + "projectHydration": "fixture sources are staged in the released corpus" + }, + { + "id": "eclipse-jdtls", + "runner": "lsp", + "name": "Eclipse JDT LS", + "requestedVersion": "1.61.0-202607142124", + "source": "https://download.eclipse.org/jdtls/snapshots/", + "profile": "adapters/lsp/eclipse-jdtls.json", + "profileSha256": "a04864d9c9c03fda591e1929766a8cc698417da2b4495dce8285024b9766743e", + "advertised": true, + "runtimeNetworking": "not required after the JDT LS distribution is installed", + "projectHydration": "JDT LS imports each generated fixture project into isolated workspace data" + }, + { + "id": "rust-analyzer", + "runner": "lsp", + "name": "rust-analyzer", + "requestedVersion": "2026-07-13", + "source": "https://github.com/rust-lang/rust-analyzer/releases/tag/2026-07-13", + "profile": "adapters/lsp/rust-analyzer.json", + "profileSha256": "ff3d13a0530ca57c0710931e7e6bee6cde3e842c4a72b078f34e0e69f0c6673f", + "advertised": true, + "runtimeNetworking": "not required after the requested rust-analyzer binary and Rust toolchain are installed", + "projectHydration": "rust-analyzer loads the generated Cargo fixture workspace" + }, + { + "id": "apple-clangd-21", + "runner": "lsp", + "name": "Apple clangd", + "requestedVersion": "21.0.0", + "source": "https://developer.apple.com/xcode/", + "profile": "adapters/lsp/apple-clangd-21.json", + "profileSha256": "a386689abb1e83048204a1f23a7f0c9b39b575db1a548607ef119b647fa36651", + "resolvedVersionPrefix": "Apple clangd version 21.0.0", + "advertised": true, + "runtimeNetworking": "not required by clangd; fixture hydration is local", + "projectHydration": "UsageBench writes compile_commands.json, opens benchmark query documents, and waits for clangd fileStatus idle in each isolated workspace" + } + ] +} diff --git a/scripts/freeze-shards.py b/scripts/freeze-shards.py index a031253..e96fae9 100755 --- a/scripts/freeze-shards.py +++ b/scripts/freeze-shards.py @@ -25,6 +25,7 @@ "benchmarks/evaluation/real-project-v2/sources.json", "benchmarks/evaluation/real-project-v2/declarations.json", "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json", + "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json", ] @@ -76,7 +77,7 @@ def identity(root, version, revision, shard, corpus_hashes=None): raise ValueError(f"staged corpus hashes omit case file: {name}") hashes[name] = staged_hashes[name] registry = json.loads( - (root / "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json").read_text() + (root / "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json").read_text() ) profile = next(item for item in registry["candidates"] if item["id"] == candidate) result = { diff --git a/scripts/validate-reproduction-contract.sh b/scripts/validate-reproduction-contract.sh index 1b3dd30..d3a0572 100755 --- a/scripts/validate-reproduction-contract.sh +++ b/scripts/validate-reproduction-contract.sh @@ -10,6 +10,7 @@ scope_resolver="$repo_root/scripts/resolve-freeze-scope.sh" freeze_workflow="$repo_root/.github/workflows/freeze.yml" docs_workflow="$repo_root/.github/workflows/docs.yml" v030_registry="$repo_root/benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json" +v034_registry="$repo_root/benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json" publication_validator="$repo_root/scripts/validate-publication-bundle.py" [[ -f "$publication_validator" ]] || { @@ -54,6 +55,22 @@ cargo run --locked -- validate-legacy-promotion "$repo_root/benchmarks/promotion jq -e '.schemaVersion == 3' "$registry" >/dev/null jq -e '.schemaVersion == 3' "$v030_registry" >/dev/null +jq -e '.schemaVersion == 3' "$v034_registry" >/dev/null +# The superseding evaluation registry advances only the measured Bifrost, to +# the same identity the active development registry pins; every other frozen +# candidate identity is byte-identical to the historical v0.3.0 registry. +jq -e ' + [.candidates[] | select( + .id == "bifrost" + and .requestedVersion == "v0.10.8" + and .revision == "8ddf13625653caf927137a0de1966401055debad" + )] | length == 1 +' "$v034_registry" >/dev/null +diff <(jq -S '[.candidates[] | select(.id != "bifrost")]' "$v030_registry") \ + <(jq -S '[.candidates[] | select(.id != "bifrost")]' "$v034_registry") >/dev/null || { + echo "superseding evaluation registry changes a non-Bifrost identity" >&2 + exit 1 +} jq -e ' [.candidates[] | select(.id == "bifrost")] == [{ @@ -204,8 +221,8 @@ grep -Fq 'python3 scripts/freeze-shards.py aggregate' "$freeze_workflow" || { echo "freeze workflow does not verify shard identity and coverage before aggregation" >&2 exit 1 } -grep -Fq 'benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json' "$freeze_workflow" || { - echo "v0.3.0 evaluation freeze is not bound to its historical candidate registry" >&2 +grep -Fq 'candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json"' "$freeze_workflow" || { + echo "evaluation freeze is not bound to the superseding candidate registry" >&2 exit 1 } grep -Fq "public_bifrost_source='https://github.com/BrokkAi/bifrost'" "$freeze_workflow" || { diff --git a/src/freeze.rs b/src/freeze.rs index de30504..2fb793b 100644 --- a/src/freeze.rs +++ b/src/freeze.rs @@ -1258,6 +1258,28 @@ mod tests { } } + #[test] + fn v034_evaluation_registry_advances_the_measured_bifrost() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let registry = load_registry( + &root.join("benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json"), + ) + .unwrap(); + let bifrost = registry + .candidates + .iter() + .find(|candidate| candidate.id == "bifrost") + .unwrap(); + + assert_eq!(bifrost.requested_version, "v0.10.8"); + assert_eq!(bifrost.source, "https://github.com/BrokkAi/bifrost"); + assert_eq!( + bifrost.revision.as_deref(), + Some("8ddf13625653caf927137a0de1966401055debad") + ); + assert!(bifrost.reference_runner.is_none()); + } + #[test] fn v030_evaluation_registry_uses_public_native_bifrost_identity() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/test_freeze_shards.py b/tests/test_freeze_shards.py index 8ca1f5b..bdb85b8 100644 --- a/tests/test_freeze_shards.py +++ b/tests/test_freeze_shards.py @@ -71,12 +71,52 @@ def test_v030_registry_uses_public_native_bifrost_identity(self): ) self.assertNotIn("referenceRunner", frozen_bifrost) + def test_v034_registry_advances_only_the_bifrost_pin(self): + """The superseding registry moves the measured Bifrost and nothing else. + + candidates-v0.3.0.json stays byte-identical as the historical record of + what v0.3.0 measured; the v0.3.4 registry is a new frozen identity set, + not an edit. + """ + previous = json.loads( + (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json").read_text() + ) + current = json.loads( + (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json").read_text() + ) + bifrost = next(item for item in current["candidates"] if item["id"] == "bifrost") + self.assertEqual(bifrost["requestedVersion"], "v0.10.8") + self.assertEqual(bifrost["revision"], "8ddf13625653caf927137a0de1966401055debad") + self.assertNotIn("referenceRunner", bifrost) + # The active development pin and the evaluation pin now agree. + active = json.loads((ROOT / "adapters/candidates.json").read_text()) + active_bifrost = next(item for item in active["candidates"] if item["id"] == "bifrost") + self.assertEqual(bifrost["revision"], active_bifrost["revision"]) + # Every non-Bifrost identity is unchanged from the superseded registry. + strip = lambda registry: [ + item for item in registry["candidates"] if item["id"] != "bifrost" + ] + self.assertEqual(strip(current), strip(previous)) + + def test_freeze_workflow_selects_the_v034_evaluation_registry(self): + workflow = (ROOT / ".github/workflows/freeze.yml").read_text() + self.assertIn( + "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json", workflow + ) + self.assertNotIn( + 'candidate_registry="benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json"', + workflow, + ) + def test_apple_clangd_registries_use_exact_reported_banner_prefix(self): active = json.loads((ROOT / "adapters/candidates.json").read_text()) frozen = json.loads( (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.0.json").read_text() ) - for registry in (active, frozen): + superseding = json.loads( + (ROOT / "benchmarks/evaluation/real-project-v2/candidates-v0.3.4.json").read_text() + ) + for registry in (active, frozen, superseding): apple_clangd = next( item for item in registry["candidates"] if item["id"] == "apple-clangd-21" )