Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 65 additions & 1 deletion crates/ethos-cli/src/cmd/verify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ use ethos_core::grounding::{
};
use ethos_core::model::Document;
use ethos_core::verify_types::{
CapabilityLimit, CheckReason, CheckStatus, ClaimKind, EvidenceOptions, MatchMethod,
CapabilityLimit, Check, CheckReason, CheckStatus, ClaimKind, EvidenceOptions, MatchMethod,
VerificationConfig, VerificationReport,
};
use ethos_grounding_opendataloader_json::OdlJsonSource;
Expand Down Expand Up @@ -233,6 +233,7 @@ fn verification_report_summary_bytes(report: &VerificationReport) -> Result<Vec<
citation_locator_label(&check.claim.citation),
match_method_label(check.match_method)
));
out.push_str(&format!(" diagnostic: {}\n", check_diagnostic(check)));
}
}
Ok(out.into_bytes())
Expand Down Expand Up @@ -273,6 +274,69 @@ fn check_reason_label(reason: CheckReason) -> &'static str {
}
}

fn check_diagnostic(check: &Check) -> String {
match check.reason {
Some(CheckReason::MissingLocator) => {
"citation has no locator; provide page, element_id, span_id, table_id/cell, or bbox"
.to_string()
}
Some(CheckReason::MissingRequiredText) => {
"textual claim is missing required text".to_string()
}
Some(CheckReason::UnsupportedClaimKind) => {
"claim kind is outside the active verifier scope".to_string()
}
Some(CheckReason::StaleFingerprint) => {
"citation fingerprint does not match grounding source fingerprint".to_string()
}
Some(CheckReason::MissingSourceFingerprint) => {
"citations are fingerprint-pinned, but the grounding source did not declare a fingerprint"
.to_string()
}
Some(CheckReason::MissingCitationFingerprint) => {
"active staleness policy requires a citation fingerprint".to_string()
}
Some(CheckReason::MissingSpanCapability) => {
"span locator requires a source with span capability".to_string()
}
Some(CheckReason::MissingTableCapability) => {
"table_cell lookup requires a source with table capability".to_string()
}
Some(CheckReason::UnknownCoordinateOrigin) => {
"bbox locator requires a known coordinate origin".to_string()
}
Some(CheckReason::ElementNotFound) => {
"element_id locator did not resolve in the grounding source".to_string()
}
Some(CheckReason::SpanNotFound) => {
"span_id locator did not resolve in the grounding source".to_string()
}
Some(CheckReason::PageNotFound) => {
"page locator did not resolve in the grounding source".to_string()
}
Some(CheckReason::BboxNotFound) => {
"bbox locator did not resolve to a containing grounding element".to_string()
}
Some(CheckReason::MissingPageForBbox) => {
"bbox locator requires page unless another target locator is present".to_string()
}
Some(CheckReason::MissingTableCellLocator) => {
"table_cell claim requires both table_id and cell locator".to_string()
}
Some(CheckReason::TableNotFound) => {
"table_id locator did not resolve in the grounding source".to_string()
}
Some(CheckReason::TableCellNotFound) => {
"table resolved, but the cited cell address was not found".to_string()
}
Some(CheckReason::TextMismatch) => format!(
"target resolved, but target text did not match claimed text under {}; no semantic inference was attempted",
match_method_label(check.match_method)
),
None => "check did not ground and no stable reason was reported".to_string(),
}
}

fn capability_limit_label(limit: CapabilityLimit) -> &'static str {
match limit {
CapabilityLimit::MissingSpans => "missing_spans",
Expand Down
3 changes: 3 additions & 0 deletions crates/ethos-cli/tests/verify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,8 @@ fn summary_format_reports_reason_before_fail_on_ungrounded_exit() {
assert!(summary.contains("checks_capability_blocked: 1\n"));
assert!(summary.contains("capability_limits: missing_fingerprint,missing_spans,missing_char_offsets,missing_tables,unknown_coordinate_origin\n"));
assert!(summary.contains("- v0001 status=capability_blocked reason=missing_table_capability kind=table_cell locator=table_id:odl-t1;cell:1,1 match_method=none\n"));
assert!(summary
.contains(" diagnostic: table_cell lookup requires a source with table capability\n"));
}

#[test]
Expand Down Expand Up @@ -436,6 +438,7 @@ fn summary_format_reports_no_non_grounded_checks_for_grounded_input() {
assert!(summary.contains("capability_limits: none\n"));
assert!(summary.contains("warnings: none\n"));
assert!(summary.contains("non_grounded_checks:\n- none\n"));
assert!(!summary.contains(" diagnostic:"));
}

#[test]
Expand Down
4 changes: 2 additions & 2 deletions examples/verify/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ ethos verify schemas/examples/document.example.json \
```

The summary includes the verification config hash, declared grounding capabilities, check counts,
report warnings, and non-grounded check reasons.
report warnings, non-grounded check reasons, and deterministic diagnostic lines.

`make verify-alpha` also checks summary output for a native ungrounded citation set, including
the non-grounded reason lines.
the non-grounded reason and diagnostic lines.

## Native Ethos Ungrounded Citations

Expand Down
7 changes: 7 additions & 0 deletions examples/verify/check_verify_alpha.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,17 @@
"locator=page:p0001;element_id:e000002 "
"match_method=normalized_text_contains\n"
),
(
" diagnostic: target resolved, but target text did not match "
"claimed text under normalized_text_contains; no semantic inference was attempted\n"
),
(
"- v0002 status=not_found reason=element_not_found kind=presence "
"locator=element_id:missing-element match_method=none\n"
),
(
" diagnostic: element_id locator did not resolve in the grounding source\n"
),
],
},
]
Expand Down
Loading