From 21ebc770a3f0718ea88bfd801b03291175a77b0a Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 16 Jun 2026 17:32:50 +0530 Subject: [PATCH] Improve verify summary diagnostics Signed-off-by: docushell-admin --- crates/ethos-cli/src/cmd/verify.rs | 66 ++++++++++++++++++++++++++- crates/ethos-cli/tests/verify.rs | 3 ++ examples/verify/README.md | 4 +- examples/verify/check_verify_alpha.py | 7 +++ 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/crates/ethos-cli/src/cmd/verify.rs b/crates/ethos-cli/src/cmd/verify.rs index 52952d9..a5e8368 100644 --- a/crates/ethos-cli/src/cmd/verify.rs +++ b/crates/ethos-cli/src/cmd/verify.rs @@ -27,7 +27,7 @@ use ethos_core::grounding::{ }; use ethos_core::model::Document; use ethos_core::verify_types::{ - CapabilityLimit, CheckReason, CheckStatus, ClaimKind, EvidenceOptions, MatchMethod, + CapabilityLimit, Check, CheckReason, CheckStatus, ClaimKind, EvidenceOptions, MatchMethod, VerificationConfig, VerificationReport, }; use ethos_grounding_opendataloader_json::OdlJsonSource; @@ -233,6 +233,7 @@ fn verification_report_summary_bytes(report: &VerificationReport) -> Result &'static str { } } +fn check_diagnostic(check: &Check) -> String { + match check.reason { + Some(CheckReason::MissingLocator) => { + "citation has no locator; provide page, element_id, span_id, table_id/cell, or bbox" + .to_string() + } + Some(CheckReason::MissingRequiredText) => { + "textual claim is missing required text".to_string() + } + Some(CheckReason::UnsupportedClaimKind) => { + "claim kind is outside the active verifier scope".to_string() + } + Some(CheckReason::StaleFingerprint) => { + "citation fingerprint does not match grounding source fingerprint".to_string() + } + Some(CheckReason::MissingSourceFingerprint) => { + "citations are fingerprint-pinned, but the grounding source did not declare a fingerprint" + .to_string() + } + Some(CheckReason::MissingCitationFingerprint) => { + "active staleness policy requires a citation fingerprint".to_string() + } + Some(CheckReason::MissingSpanCapability) => { + "span locator requires a source with span capability".to_string() + } + Some(CheckReason::MissingTableCapability) => { + "table_cell lookup requires a source with table capability".to_string() + } + Some(CheckReason::UnknownCoordinateOrigin) => { + "bbox locator requires a known coordinate origin".to_string() + } + Some(CheckReason::ElementNotFound) => { + "element_id locator did not resolve in the grounding source".to_string() + } + Some(CheckReason::SpanNotFound) => { + "span_id locator did not resolve in the grounding source".to_string() + } + Some(CheckReason::PageNotFound) => { + "page locator did not resolve in the grounding source".to_string() + } + Some(CheckReason::BboxNotFound) => { + "bbox locator did not resolve to a containing grounding element".to_string() + } + Some(CheckReason::MissingPageForBbox) => { + "bbox locator requires page unless another target locator is present".to_string() + } + Some(CheckReason::MissingTableCellLocator) => { + "table_cell claim requires both table_id and cell locator".to_string() + } + Some(CheckReason::TableNotFound) => { + "table_id locator did not resolve in the grounding source".to_string() + } + Some(CheckReason::TableCellNotFound) => { + "table resolved, but the cited cell address was not found".to_string() + } + Some(CheckReason::TextMismatch) => format!( + "target resolved, but target text did not match claimed text under {}; no semantic inference was attempted", + match_method_label(check.match_method) + ), + None => "check did not ground and no stable reason was reported".to_string(), + } +} + fn capability_limit_label(limit: CapabilityLimit) -> &'static str { match limit { CapabilityLimit::MissingSpans => "missing_spans", diff --git a/crates/ethos-cli/tests/verify.rs b/crates/ethos-cli/tests/verify.rs index 710394b..870854e 100644 --- a/crates/ethos-cli/tests/verify.rs +++ b/crates/ethos-cli/tests/verify.rs @@ -404,6 +404,8 @@ fn summary_format_reports_reason_before_fail_on_ungrounded_exit() { assert!(summary.contains("checks_capability_blocked: 1\n")); assert!(summary.contains("capability_limits: missing_fingerprint,missing_spans,missing_char_offsets,missing_tables,unknown_coordinate_origin\n")); assert!(summary.contains("- v0001 status=capability_blocked reason=missing_table_capability kind=table_cell locator=table_id:odl-t1;cell:1,1 match_method=none\n")); + assert!(summary + .contains(" diagnostic: table_cell lookup requires a source with table capability\n")); } #[test] @@ -436,6 +438,7 @@ fn summary_format_reports_no_non_grounded_checks_for_grounded_input() { assert!(summary.contains("capability_limits: none\n")); assert!(summary.contains("warnings: none\n")); assert!(summary.contains("non_grounded_checks:\n- none\n")); + assert!(!summary.contains(" diagnostic:")); } #[test] diff --git a/examples/verify/README.md b/examples/verify/README.md index 06867f6..4bcb11b 100644 --- a/examples/verify/README.md +++ b/examples/verify/README.md @@ -23,10 +23,10 @@ ethos verify schemas/examples/document.example.json \ ``` The summary includes the verification config hash, declared grounding capabilities, check counts, -report warnings, and non-grounded check reasons. +report warnings, non-grounded check reasons, and deterministic diagnostic lines. `make verify-alpha` also checks summary output for a native ungrounded citation set, including -the non-grounded reason lines. +the non-grounded reason and diagnostic lines. ## Native Ethos Ungrounded Citations diff --git a/examples/verify/check_verify_alpha.py b/examples/verify/check_verify_alpha.py index 642d026..ceba5bb 100644 --- a/examples/verify/check_verify_alpha.py +++ b/examples/verify/check_verify_alpha.py @@ -137,10 +137,17 @@ "locator=page:p0001;element_id:e000002 " "match_method=normalized_text_contains\n" ), + ( + " diagnostic: target resolved, but target text did not match " + "claimed text under normalized_text_contains; no semantic inference was attempted\n" + ), ( "- v0002 status=not_found reason=element_not_found kind=presence " "locator=element_id:missing-element match_method=none\n" ), + ( + " diagnostic: element_id locator did not resolve in the grounding source\n" + ), ], }, ]