diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b7fa9b5..6cac766 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -105,6 +105,7 @@ jobs: --bifrost-working-tree --work-dir target/usagebench --scan-usages-max-duration-secs 300 + --expected-failures benchmarks/expectations/bifrost-expected-failures.yaml --expected-passes benchmarks/expectations/bifrost-expected-passes.yaml --output "benchmark-output/run-${GITHUB_RUN_ID}.json" ) diff --git a/benchmarks/README.md b/benchmarks/README.md index 1950740..89150dc 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -218,6 +218,9 @@ Each case supports both benchmark directions: - `expectedFailure.reason` keeps a known analyzer gap in the baseline while still running the case and reporting it as improved if it unexpectedly starts passing. +- `benchmarks/expectations/bifrost-expected-failures.yaml` records current + Bifrost regressions without changing content-addressed legacy or evaluation + case documents. The scheduled benchmark applies this overlay before scoring. - `benchmarks/expectations/bifrost-expected-passes.yaml` can promote a frozen historical `expectedFailure` to a current Bifrost required pass without mutating content-addressed legacy benchmark documents. A matching case must diff --git a/benchmarks/expectations/bifrost-expected-failures.yaml b/benchmarks/expectations/bifrost-expected-failures.yaml new file mode 100644 index 0000000..bc95d06 --- /dev/null +++ b/benchmarks/expectations/bifrost-expected-failures.yaml @@ -0,0 +1,40 @@ +schemaVersion: 1 +expectedFailures: + - caseId: real-project-v2-cpp-01-2 + reason: > + Bifrost cannot resolve the ImGui_ImplSDL3_CloseGamepads declaration symbol, + although forward definition lookup succeeds. + - caseId: real-project-v2-cpp-02-1 + reason: > + Bifrost does not navigate from the TessPageIteratorDelete header declaration + to its implementation body. + - caseId: real-project-v2-cpp-02-3 + reason: > + Bifrost does not navigate from the TessResultIteratorSymbolIsSubscript header + declaration to its implementation body. + - caseId: real-project-v2-cpp-04-3 + reason: > + Bifrost cannot resolve the MountFilesystem declaration symbol, although + forward definition lookup succeeds. + - caseId: real-project-v2-rust-04-3 + reason: > + Bifrost cannot resolve the ServerEvents usage across the indexed Rust + crate boundary. + - caseId: rust-struct-construction + reason: > + Bifrost declaration-to-usages misses the two Self references. + - caseId: rust-parity-module-declaration-definition + reason: > + Bifrost module declaration lookup returns a zero-width range instead of + the authored module token. + - caseId: rust-parity-macro-generated-function-reference + reason: > + Bifrost does not index or resolve the macro-generated function. + - caseId: scala-parity-case-class-generated-construction-and-copy + reason: > + Bifrost reverse usage includes the synthetic copy call as an unexpected + extra location. + - caseId: ts-parity-interface-property-access + reason: > + Bifrost cannot resolve the user.name interface property usage to its + declaration and omits it from reverse usages. diff --git a/src/main.rs b/src/main.rs index 3ecbcd4..0487291 100644 --- a/src/main.rs +++ b/src/main.rs @@ -197,6 +197,9 @@ enum Command { /// Run only benchmark documents for this language. #[arg(long)] language: Option, + /// Current Bifrost failures applied without mutating frozen benchmark documents. + #[arg(long)] + expected_failures: Option, /// Versioned overlay that promotes historical expected failures to current required passes. #[arg(long)] expected_passes: Option, @@ -457,6 +460,7 @@ fn main() -> Result<()> { keep_worktrees, case_id, language, + expected_failures, expected_passes, } => { let mut options = RunBifrostOptions::with_defaults(path); @@ -472,6 +476,7 @@ fn main() -> Result<()> { options.keep_worktrees = keep_worktrees; options.case_id = case_id; options.language = language; + options.expected_failures = expected_failures; options.expected_passes = expected_passes; let report = run_bifrost(options)?; println!( diff --git a/src/runners/bifrost.rs b/src/runners/bifrost.rs index 6e7e011..b7b339c 100644 --- a/src/runners/bifrost.rs +++ b/src/runners/bifrost.rs @@ -108,6 +108,7 @@ pub struct RunBifrostOptions { pub keep_worktrees: bool, pub case_id: Option, pub language: Option, + pub expected_failures: Option, pub expected_passes: Option, } @@ -128,6 +129,7 @@ impl RunBifrostOptions { keep_worktrees: false, case_id: None, language: None, + expected_failures: None, expected_passes: None, } } @@ -147,6 +149,20 @@ struct ExpectedPass { reason: String, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ExpectedFailureManifest { + schema_version: u32, + expected_failures: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ExpectedFailureOverride { + case_id: String, + reason: String, +} + pub type BifrostRunReport = RunReport; pub fn generated_bifrost_report_schema_json() -> Result { @@ -185,6 +201,11 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result { &repo_root, options.expected_passes.as_deref(), )?; + apply_expected_failures( + &mut benchmark_documents, + &repo_root, + options.expected_failures.as_deref(), + )?; let requested_totals = requested_run_totals( benchmark_documents .iter() @@ -452,6 +473,90 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result { Ok(report) } +fn apply_expected_failures( + documents: &mut [(PathBuf, BenchmarkDocument)], + repo_root: &Path, + manifest_path: Option<&Path>, +) -> Result<()> { + let Some(manifest_path) = manifest_path else { + return Ok(()); + }; + let manifest_path = if manifest_path.is_absolute() { + manifest_path.to_path_buf() + } else { + repo_root.join(manifest_path) + }; + let source = fs::read_to_string(&manifest_path) + .with_context(|| format!("read expected-failure overlay {}", manifest_path.display()))?; + let manifest = serde_yaml::from_str::(&source) + .with_context(|| format!("parse expected-failure overlay {}", manifest_path.display()))?; + if manifest.schema_version != 1 { + bail!( + "expected-failure overlay {} has unsupported schemaVersion {}", + manifest_path.display(), + manifest.schema_version + ); + } + + let mut expected_failures = BTreeMap::new(); + for expected_failure in manifest.expected_failures { + if expected_failure.case_id.trim().is_empty() || expected_failure.reason.trim().is_empty() { + bail!( + "expected-failure overlay {} requires non-empty caseId and reason", + manifest_path.display() + ); + } + if expected_failures + .insert(expected_failure.case_id.clone(), expected_failure.reason) + .is_some() + { + bail!( + "expected-failure overlay {} repeats caseId {}", + manifest_path.display(), + expected_failure.case_id + ); + } + } + + let mut applied = BTreeSet::new(); + for (_, document) in documents { + for case in &mut document.cases { + let Some(reason) = expected_failures.get(&case.id) else { + continue; + }; + if case.expected_failure.is_some() + || case.expected_pass_reason.is_some() + || case.not_planned.is_some() + || case.unsupported.is_some() + { + bail!( + "expected-failure overlay {} may only mark an unclassified case: {}", + manifest_path.display(), + case.id + ); + } + case.expected_failure = Some(crate::ExpectedFailure { + reason: reason.clone(), + }); + applied.insert(case.id.clone()); + } + } + + let missing = expected_failures + .keys() + .filter(|case_id| !applied.contains(*case_id)) + .cloned() + .collect::>(); + if !missing.is_empty() { + bail!( + "expected-failure overlay {} references case IDs absent from this run: {}", + manifest_path.display(), + missing.join(", ") + ); + } + Ok(()) +} + fn apply_expected_passes( documents: &mut [(PathBuf, BenchmarkDocument)], repo_root: &Path, @@ -4618,6 +4723,99 @@ for line in sys.stdin: assert_eq!(report.diagnostics[0].kind, "expected_pass_override"); } + #[test] + fn expected_failure_overlay_marks_an_unclassified_case() { + let tempdir = tempfile::tempdir().unwrap(); + let manifest_path = tempdir.path().join("expected-failures.yaml"); + fs::write( + &manifest_path, + "schemaVersion: 1\nexpectedFailures:\n - caseId: current-gap\n reason: current regression\n", + ) + .unwrap(); + let mut documents = vec![( + PathBuf::from("benchmarks/cases/sample.yaml"), + serde_yaml::from_str::( + "schemaVersion: 2\ncorpus:\n partition: development\n selection: analyzer_informed\ngroundTruth:\n status: legacy_unattributed\n reviewers: []\nreferencePolicy: bindings_optional\nsource:\n kind: fixture\n path: fixtures/sample\nlanguage: rust\ncases:\n - id: current-gap\n", + ) + .unwrap(), + )]; + + apply_expected_failures(&mut documents, tempdir.path(), Some(&manifest_path)).unwrap(); + + assert_eq!( + documents[0].1.cases[0] + .expected_failure + .as_ref() + .map(|expected| expected.reason.as_str()), + Some("current regression") + ); + } + + #[test] + fn expected_failure_overlay_does_not_replace_an_authored_marker() { + let tempdir = tempfile::tempdir().unwrap(); + let manifest_path = tempdir.path().join("expected-failures.yaml"); + fs::write( + &manifest_path, + "schemaVersion: 1\nexpectedFailures:\n - caseId: existing-gap\n reason: replacement\n", + ) + .unwrap(); + let mut documents = vec![( + PathBuf::from("benchmarks/cases/sample.yaml"), + serde_yaml::from_str::( + "schemaVersion: 2\ncorpus:\n partition: development\n selection: analyzer_informed\ngroundTruth:\n status: legacy_unattributed\n reviewers: []\nreferencePolicy: bindings_optional\nsource:\n kind: fixture\n path: fixtures/sample\nlanguage: rust\ncases:\n - id: existing-gap\n expectedFailure:\n reason: authored gap\n", + ) + .unwrap(), + )]; + + let error = apply_expected_failures(&mut documents, tempdir.path(), Some(&manifest_path)) + .unwrap_err(); + + assert!(error + .to_string() + .contains("may only mark an unclassified case")); + } + + #[test] + fn checked_in_expected_failure_overlay_marks_all_current_regressions() { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let case_files = [ + "benchmarks/cases/evaluation/real-project-v2/cpp-01.yaml", + "benchmarks/cases/evaluation/real-project-v2/cpp-02.yaml", + "benchmarks/cases/evaluation/real-project-v2/cpp-04.yaml", + "benchmarks/cases/evaluation/real-project-v2/rust-04.yaml", + "benchmarks/cases/rust-baseline.yaml", + "benchmarks/cases/rust-lsp-parity.yaml", + "benchmarks/cases/scala-lsp-parity.yaml", + "benchmarks/cases/typescript-lsp-parity.yaml", + ]; + let mut documents = case_files + .iter() + .map(|case_file| { + let path = repo_root.join(case_file); + let source = fs::read_to_string(&path).unwrap(); + let document = serde_yaml::from_str::(&source).unwrap(); + (path, document) + }) + .collect::>(); + + apply_expected_failures( + &mut documents, + &repo_root, + Some(Path::new( + "benchmarks/expectations/bifrost-expected-failures.yaml", + )), + ) + .unwrap(); + + let overlaid = documents + .iter() + .flat_map(|(_, document)| &document.cases) + .filter(|case| case.expected_failure.is_some()) + .count(); + assert_eq!(overlaid, 10); + } + #[test] fn expected_pass_overlay_promotes_only_authored_expected_failures() { let tempdir = tempfile::tempdir().unwrap();