From ba55618c4865256bc72ef948767eec717871d97f Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 5 Oct 2026 10:36:46 +0200 Subject: [PATCH 1/3] Preserve Bifrost scan failure diagnostics --- src/runners/bifrost.rs | 109 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/src/runners/bifrost.rs b/src/runners/bifrost.rs index f8bc9f3..31f2819 100644 --- a/src/runners/bifrost.rs +++ b/src/runners/bifrost.rs @@ -2038,6 +2038,9 @@ fn parse_scan_usages(value: &Value) -> ParsedScanUsages { ); if let Some(status) = result.get("status").and_then(Value::as_str) { raw_statuses.push(status.to_string()); + if status == "failure" { + diagnostics.push(scan_usages_failure_diagnostic(result)); + } } if result .get("complete") @@ -2077,12 +2080,14 @@ fn parse_scan_usages(value: &Value) -> ParsedScanUsages { } for key in ["not_found", "ambiguous", "failures", "too_many_callsites"] { - if value + if let Some(items) = value .get(key) .and_then(Value::as_array) - .map(|items| !items.is_empty()) - .unwrap_or(false) + .filter(|items| !items.is_empty()) { + if key == "failures" { + diagnostics.extend(items.iter().map(scan_usages_failure_diagnostic)); + } raw_statuses.push( match key { "failures" => "failure", @@ -2139,6 +2144,40 @@ fn scan_usages_incomplete_diagnostic(result: &Value) -> RunDiagnostic { } } +fn scan_usages_failure_diagnostic(result: &Value) -> RunDiagnostic { + let mut details = Vec::new(); + if let Some(reason_kind) = result.get("reason_kind").and_then(Value::as_str) { + details.push(format!("reason_kind={reason_kind}")); + } + if let Some(reason) = result.get("reason").and_then(Value::as_str) { + details.push(format!("reason={reason}")); + } + if let Some(message) = result.get("message").and_then(Value::as_str) { + details.push(format!("message={message}")); + } + if let Some(input_kind) = result.get("input_kind").and_then(Value::as_str) { + details.push(format!("input_kind={input_kind}")); + } + if let Some(input) = result.get("input") { + details.push(format!("input={input}")); + } + if let Some(symbol) = result.get("symbol").and_then(Value::as_str) { + details.push(format!("symbol={symbol}")); + } + if let Some(fq_name) = result.get("fq_name").and_then(Value::as_str) { + details.push(format!("fq_name={fq_name}")); + } + let message = if details.is_empty() { + "Bifrost reported a scan failure without structured details".to_string() + } else { + format!("Bifrost scan usages failure: {}", details.join("; ")) + }; + RunDiagnostic { + kind: "scan_usages_failure".to_string(), + message, + } +} + fn collect_scan_usage_locations( value: &Value, group: &str, @@ -5299,6 +5338,38 @@ for line in sys.stdin: assert!(parsed.has_failure_status()); } + #[test] + fn parse_scan_usages_preserves_current_failure_details() { + let parsed = parse_scan_usages(&json!({ + "summary": {"partial": false}, + "results": [{ + "fq_name": "github.com/gohugoio/hugo/hugolib.shortcodeParseInfo.addName", + "input": {"column": 30, "line": 296, "path": "hugolib/shortcode.go"}, + "input_kind": "target", + "message": "GoUsageGraphStrategy: Go graph indexed source was unavailable", + "reason_kind": "unavailable_canonical_facts", + "status": "failure", + "symbol": "github.com/gohugoio/hugo/hugolib.shortcodeParseInfo.addName" + }] + })); + + assert_eq!(parsed.raw_statuses, vec!["failure".to_string()]); + assert!(parsed.has_failure_status()); + assert!(!parsed.partial); + assert_eq!(parsed.diagnostics.len(), 1); + assert_eq!(parsed.diagnostics[0].kind, "scan_usages_failure"); + assert!(parsed.diagnostics[0] + .message + .contains("reason_kind=unavailable_canonical_facts")); + assert!(parsed.diagnostics[0] + .message + .contains("message=GoUsageGraphStrategy: Go graph indexed source was unavailable")); + assert!(parsed.diagnostics[0].message.contains("input_kind=target")); + assert!(parsed.diagnostics[0] + .message + .contains(r#"input={"column":30,"line":296,"path":"hugolib/shortcode.go"}"#)); + } + #[test] fn parse_scan_usages_marks_incomplete_results_partial() { let parsed = @@ -5386,6 +5457,38 @@ for line in sys.stdin: assert!(!parsed.partial); } + #[test] + fn parse_scan_usages_preserves_legacy_failure_details() { + let parsed = parse_scan_usages(&json!({ + "summary": {"partial": false}, + "failures": [{ + "fq_name": "example.build_service", + "reason": "legacy usage graph unavailable", + "reason_kind": "unavailable_canonical_facts", + "symbol": "example.build_service" + }], + "usages": [] + })); + + assert_eq!(parsed.raw_statuses, vec!["failure".to_string()]); + assert!(parsed.has_failure_status()); + assert!(!parsed.partial); + assert_eq!(parsed.diagnostics.len(), 1); + assert_eq!(parsed.diagnostics[0].kind, "scan_usages_failure"); + assert!(parsed.diagnostics[0] + .message + .contains("reason_kind=unavailable_canonical_facts")); + assert!(parsed.diagnostics[0] + .message + .contains("reason=legacy usage graph unavailable")); + assert!(parsed.diagnostics[0] + .message + .contains("symbol=example.build_service")); + assert!(parsed.diagnostics[0] + .message + .contains("fq_name=example.build_service")); + } + #[test] fn constructor_declarations_resolve_from_functions_bucket() { let mut case = benchmark_case(); From 0a0e1a5d2c0fe10001a13de2baa62891d74b3f23 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 5 Oct 2026 10:59:45 +0200 Subject: [PATCH 2/3] Fix equivalent container references in report reproduction --- src/runners/report_compare.rs | 232 ++++++++++++++++++++++++++++++++-- 1 file changed, 222 insertions(+), 10 deletions(-) diff --git a/src/runners/report_compare.rs b/src/runners/report_compare.rs index 52f067f..0eba418 100644 --- a/src/runners/report_compare.rs +++ b/src/runners/report_compare.rs @@ -58,8 +58,9 @@ pub fn write_differences(path: &Path, differences: &[ReportDifference]) -> Resul } fn compare_values(expected: Value, actual: Value) -> Vec { - let expected = semantic_value(expected); - let actual = semantic_value(actual); + let mut expected = semantic_value(expected); + let mut actual = semantic_value(actual); + normalize_equivalent_image_references(&mut expected, &mut actual); let mut differences = Vec::new(); collect_differences("$", &expected, &actual, &mut differences); differences @@ -101,12 +102,6 @@ fn semantic_value(mut value: Value) -> Value { } if let Some(Value::Object(environment)) = root.get_mut("environment") { - if let Some(Value::Object(container)) = environment.get_mut("container") { - container.insert( - "imageDigest".to_string(), - Value::String("".to_string()), - ); - } if let Some(Value::Object(executable)) = environment.get_mut("analyzerExecutable") { executable.remove("resolvedPath"); } @@ -120,6 +115,89 @@ fn semantic_value(mut value: Value) -> Value { value } +fn normalize_equivalent_image_references(expected: &mut Value, actual: &mut Value) { + let Some(expected_container) = container_object(expected) else { + return; + }; + let Some(actual_container) = container_object(actual) else { + return; + }; + + let Some(expected_digest) = expected_container + .get("imageDigest") + .and_then(Value::as_str) + else { + return; + }; + let Some(actual_digest) = actual_container.get("imageDigest").and_then(Value::as_str) else { + return; + }; + if expected_digest != actual_digest + || !is_sha256_digest(expected_digest) + || !is_sha256_digest(actual_digest) + { + return; + } + + let Some(expected_reference) = expected_container + .get("imageReference") + .and_then(Value::as_str) + else { + return; + }; + let Some(actual_reference) = actual_container + .get("imageReference") + .and_then(Value::as_str) + else { + return; + }; + if expected_reference == actual_reference + || (is_immutable_image_reference(expected_reference) + && is_immutable_image_reference(actual_reference)) + { + return; + } + + let canonical_reference = Value::String("".to_string()); + if let Some(container) = container_object_mut(expected) { + container.insert("imageReference".to_string(), canonical_reference.clone()); + } + if let Some(container) = container_object_mut(actual) { + container.insert("imageReference".to_string(), canonical_reference); + } +} + +fn container_object(value: &Value) -> Option<&Map> { + value + .get("environment") + .and_then(Value::as_object) + .and_then(|environment| environment.get("container")) + .and_then(Value::as_object) +} + +fn container_object_mut(value: &mut Value) -> Option<&mut Map> { + value + .get_mut("environment") + .and_then(Value::as_object_mut) + .and_then(|environment| environment.get_mut("container")) + .and_then(Value::as_object_mut) +} + +fn is_immutable_image_reference(value: &str) -> bool { + value + .rsplit_once('@') + .is_some_and(|(_, digest)| is_sha256_digest(digest)) +} + +fn is_sha256_digest(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|hex| { + hex.len() == 64 + && hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + fn keyed_documents(documents: Vec) -> Value { let mut grouped = BTreeMap::>::new(); for mut document in documents { @@ -316,8 +394,6 @@ mod tests { actual.started_at_unix_seconds = 99; actual.finished_at_unix_seconds = 100; actual.documents[0].source_root = "/another/run/source".to_string(); - actual.environment.container.as_mut().unwrap().image_digest = - format!("sha256:{}", "d".repeat(64)); actual.environment.analyzer_executable.resolved_path = Some("/another/bin/bifrost".to_string()); actual.timings.analyzer_query_millis = 42; @@ -326,6 +402,142 @@ mod tests { assert!(compare_reports(&expected, &actual).is_empty()); } + #[test] + fn compares_image_digest_while_allowing_local_and_registry_references() { + let mut expected = report(); + expected + .environment + .container + .as_mut() + .unwrap() + .image_reference = format!( + "ghcr.io/brokkai/usagebench-reference@sha256:{}", + "f".repeat(64) + ); + let actual = report(); + + assert!(compare_reports(&expected, &actual).is_empty()); + + let mut different_image = actual.clone(); + different_image + .environment + .container + .as_mut() + .unwrap() + .image_digest = format!("sha256:{}", "e".repeat(64)); + let differences = compare_reports(&expected, &different_image); + assert!(differences + .iter() + .any(|difference| difference.path == "$.environment.container.imageDigest")); + } + + #[test] + fn compares_different_immutable_image_references() { + let mut expected = report(); + expected + .environment + .container + .as_mut() + .unwrap() + .image_reference = format!( + "ghcr.io/brokkai/usagebench-reference@sha256:{}", + "f".repeat(64) + ); + let mut actual = expected.clone(); + actual + .environment + .container + .as_mut() + .unwrap() + .image_reference = format!( + "ghcr.io/brokkai/usagebench-reference@sha256:{}", + "e".repeat(64) + ); + + let differences = compare_reports(&expected, &actual); + + assert_eq!(differences.len(), 1); + assert_eq!( + differences[0].path, + "$.environment.container.imageReference" + ); + } + + #[test] + fn compares_same_image_reference_when_the_digest_changes() { + let expected = report(); + let mut actual = expected.clone(); + actual.environment.container.as_mut().unwrap().image_digest = + format!("sha256:{}", "e".repeat(64)); + + let differences = compare_reports(&expected, &actual); + + assert_eq!(differences.len(), 1); + assert_eq!(differences[0].path, "$.environment.container.imageDigest"); + } + + #[test] + fn compares_different_local_image_references_when_the_digest_changes() { + let expected = report(); + let mut actual = expected.clone(); + actual + .environment + .container + .as_mut() + .unwrap() + .image_reference = "usagebench-reference:another-image".to_string(); + actual.environment.container.as_mut().unwrap().image_digest = + format!("sha256:{}", "e".repeat(64)); + + let differences = compare_reports(&expected, &actual); + + assert!(differences + .iter() + .any(|difference| difference.path == "$.environment.container.imageDigest")); + assert!(differences + .iter() + .any(|difference| difference.path == "$.environment.container.imageReference")); + } + + #[test] + fn does_not_relax_references_without_matching_valid_digests() { + let expected_reference = "ghcr.io/brokkai/usagebench-reference@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let actual_reference = "usagebench-reference:v0.1.0-env1-gopls"; + for (label, expected_digest, actual_digest) in [ + ("missing", None, None), + ("null", Some(Value::Null), Some(Value::Null)), + ( + "malformed", + Some(Value::String("not-a-digest".to_string())), + Some(Value::String("not-a-digest".to_string())), + ), + ] { + let mut expected_container = serde_json::json!({ + "imageReference": expected_reference, + }); + let mut actual_container = serde_json::json!({ + "imageReference": actual_reference, + }); + if let Some(digest) = expected_digest { + expected_container["imageDigest"] = digest; + } + if let Some(digest) = actual_digest { + actual_container["imageDigest"] = digest; + } + + let differences = compare_values( + serde_json::json!({"environment": {"container": expected_container}}), + serde_json::json!({"environment": {"container": actual_container}}), + ); + assert!( + differences + .iter() + .any(|difference| difference.path == "$.environment.container.imageReference"), + "{label} digest must not relax image references" + ); + } + } + #[test] fn reports_case_level_semantic_differences() { let expected = report(); From 9c78a1baf7ac25ee14176e4018cbec6e02ab5f4a Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 5 Oct 2026 15:07:16 +0200 Subject: [PATCH 3/3] Preserve Bifrost incomplete scan notes --- src/runners/bifrost.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/runners/bifrost.rs b/src/runners/bifrost.rs index 31f2819..6aa8fca 100644 --- a/src/runners/bifrost.rs +++ b/src/runners/bifrost.rs @@ -2138,6 +2138,10 @@ fn scan_usages_incomplete_diagnostic(result: &Value) -> RunDiagnostic { message.push_str("; candidate_files_sample="); message.push_str(&sample.to_string()); } + if let Some(notes) = result.get("notes") { + message.push_str("; notes="); + message.push_str(¬es.to_string()); + } RunDiagnostic { kind: "scan_usages_incomplete".to_string(), message, @@ -5413,6 +5417,34 @@ for line in sys.stdin: .contains(r#"candidate_files_sample={"omitted":["src/c.py"],"omitted_count":7,"scanned":["src/a.py","src/b.py"]}"#)); } + #[test] + fn parse_scan_usages_preserves_semantic_incompleteness_notes() { + let note = + "selected reference enumeration is incomplete: [InverseIndexResolutionIncomplete]"; + let parsed = parse_scan_usages(&json!({ + "summary": {"partial": true}, + "results": [{ + "status": "found", + "complete": false, + "incomplete_reason": "semantic_analysis", + "notes": [note], + "files": [{ + "path": "src/lib.rs", + "hits": [{"line": 3, "column": 24}] + }] + }] + })); + + assert!(parsed.partial); + assert_eq!(parsed.locations.len(), 1); + assert_eq!(parsed.diagnostics.len(), 1); + assert!(parsed.diagnostics[0].message.contains(note)); + assert!(parsed.diagnostics[0] + .message + .contains("reason=semantic_analysis")); + assert!(parsed.raw_statuses.iter().any(|status| status == "partial")); + } + #[test] fn parse_scan_usages_keeps_legacy_usages_shape() { let parsed = parse_scan_usages(&json!({