Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions crates/ethos-cli/src/cmd/verify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1009,9 +1009,10 @@ fn validate_verification_config(config: &VerificationConfig) -> Result<(), Failu
}
let mut seen = HashSet::new();
for kind in &config.claim_kinds {
if *kind == ClaimKind::Other {
if matches!(kind, ClaimKind::Region | ClaimKind::Other) {
return Err(Failure::Usage(
"verification config claim_kinds must not include other".to_string(),
"verification config claim_kinds must include only quote, value, presence, and table_cell"
.to_string(),
));
}
if !seen.insert(*kind) {
Expand Down
41 changes: 39 additions & 2 deletions crates/ethos-cli/tests/verify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ fn verify_alpha_schema_report_example_matches_cli_output() {
#[test]
fn verify_alpha_demo_reports_match_goldens() {
let root = repo_root();
let cases: [(&str, Vec<String>, PathBuf); 5] = [
let cases: [(&str, Vec<String>, PathBuf); 6] = [
(
"native-grounded",
vec![
Expand Down Expand Up @@ -193,6 +193,20 @@ fn verify_alpha_demo_reports_match_goldens() {
],
root.join("examples/verify/goldens/native_split_quote_report.json"),
),
(
"native-non-v1-claims",
vec![
"verify".to_string(),
root.join("schemas/examples/document.example.json")
.display()
.to_string(),
"--citations".to_string(),
root.join("examples/verify/native_non_v1_claims_citations.json")
.display()
.to_string(),
],
root.join("examples/verify/goldens/native_non_v1_claims_report.json"),
),
(
"native-stale",
vec![
Expand Down Expand Up @@ -1494,7 +1508,27 @@ fn invalid_config_constraints_are_usage_errors() {
"max_checks": 256
}
}"#,
"verification config claim_kinds must not include other",
"verification config claim_kinds must include only quote, value, presence, and table_cell",
),
(
"region-claim-kind-config",
r#"{
"schema_version": "1.0.0",
"config_version": "region-kind",
"claim_kinds": ["region"],
"matching": {
"text_normalization": "collapse_whitespace",
"case_sensitive": true,
"bbox_containment_tolerance_q": 50
},
"staleness": {
"require_fingerprint_match": true
},
"limits": {
"max_checks": 256
}
}"#,
"verification config claim_kinds must include only quote, value, presence, and table_cell",
),
(
"negative-bbox-tolerance-config",
Expand Down Expand Up @@ -2033,6 +2067,9 @@ fn config_excluded_value_claim_is_unsupported() {

assert_eq!(report["checks"][0]["status"], "unsupported_claim_kind");
assert_eq!(report["checks"][0]["reason"], "unsupported_claim_kind");
assert_eq!(report["checks"][0]["match_method"], "none");
assert_eq!(report["checks"][0]["semantic_unverified"], false);
assert!(report["checks"][0].get("evidence").is_none());
assert_eq!(
report["unsupported_claim_kinds"],
serde_json::json!(["value"])
Expand Down
6 changes: 4 additions & 2 deletions crates/ethos-core/src/verify_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ pub enum CapabilityLimit {
MissingCropSupport,
}

/// Claim kinds. `Other` appears only in reports (as an unsupported kind), never in configs.
/// Claim kinds. `Region` and `Other` appear only in citations/reports as unsupported non-v1
/// kinds, never in configs.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ClaimKind {
Expand Down Expand Up @@ -368,7 +369,8 @@ pub struct VerificationConfig {
pub schema_version: String,
/// User-facing config label, e.g. `"default-v1"`.
pub config_version: String,
/// Supported claim kinds for this run (never `Other`).
/// Supported claim kinds for this run. Configs accept only the four v1 literal kinds:
/// quote, value, presence, and table_cell.
pub claim_kinds: Vec<ClaimKind>,
/// Matching parameters.
pub matching: Matching,
Expand Down
73 changes: 73 additions & 0 deletions crates/ethos-verify/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1881,6 +1881,79 @@ mod tests {
assert_eq!(report.unsupported_claim_kinds, vec!["region"]);
}

#[test]
fn non_v1_claim_kinds_are_deduped_and_keep_gate_false() {
let source = TestSource::default();
let report = verify(
&source,
vec![
claim(
ClaimKind::Presence,
None,
Citation {
page: Some("p0001".into()),
..Default::default()
},
),
claim(
ClaimKind::Region,
None,
Citation {
element_id: Some("e000002".into()),
..Default::default()
},
),
claim(
ClaimKind::Other,
Some("$12.4M equals 12400000"),
Citation {
element_id: Some("e000002".into()),
..Default::default()
},
),
claim(
ClaimKind::Region,
None,
Citation {
page: Some("p0001".into()),
..Default::default()
},
),
],
);

assert!(!report.all_evidence_grounded);
assert_eq!(report.checks[0].status, CheckStatus::Grounded);
assert_eq!(report.checks[1].status, CheckStatus::UnsupportedClaimKind);
assert_eq!(report.checks[2].status, CheckStatus::UnsupportedClaimKind);
assert_eq!(report.checks[3].status, CheckStatus::UnsupportedClaimKind);
assert_eq!(report.checks[1].match_method, MatchMethod::None);
assert_eq!(report.checks[2].match_method, MatchMethod::None);
assert_eq!(report.checks[3].match_method, MatchMethod::None);
assert_eq!(
report.checks[1].reason,
Some(CheckReason::UnsupportedClaimKind)
);
assert_eq!(
report.checks[2].reason,
Some(CheckReason::UnsupportedClaimKind)
);
assert_eq!(
report.checks[3].reason,
Some(CheckReason::UnsupportedClaimKind)
);
assert!(report.checks[1].evidence.is_none());
assert!(report.checks[2].evidence.is_none());
assert!(report.checks[3].evidence.is_none());
assert!(report.checks[1].warnings.is_empty());
assert!(report.checks[2].warnings.is_empty());
assert!(report.checks[3].warnings.is_empty());
assert!(!report.checks[1].semantic_unverified);
assert!(!report.checks[2].semantic_unverified);
assert!(!report.checks[3].semantic_unverified);
assert_eq!(report.unsupported_claim_kinds, vec!["region", "other"]);
}

#[test]
fn missing_span_capability_blocks_span_locator() {
let source = TestSource {
Expand Down
22 changes: 22 additions & 0 deletions docs/demos/verify-alpha.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ repeatable `make verify-alpha` path:
- OpenDataLoader-style JSON can enter the same verification loop through a grounding adapter
- real pinned OpenDataLoader 2.4.7 output has both grounded and ungrounded citation cases
- native and synthetic OpenDataLoader fixtures cover missing cited elements
- native fixtures cover unsupported non-v1 claim kinds
- malformed citation inputs and malformed OpenDataLoader-style grounding inputs return usage
diagnostics with exit code `2`
- `--fail-on-ungrounded` turns the report into a CI/agent gate with exit code `1` when evidence is not fully grounded
Expand Down Expand Up @@ -71,6 +72,27 @@ Golden report:
examples/verify/goldens/native_ungrounded_report.json
```

## Native Non-v1 Claim Policy

```bash
ethos verify schemas/examples/document.example.json \
--citations examples/verify/native_non_v1_claims_citations.json \
--out /tmp/ethos-native-non-v1-report.json
```

Expected outcome:

- the presence check is grounded
- `region` and `other` checks have status `unsupported_claim_kind`
- `unsupported_claim_kinds` lists `region` and `other`
- `all_evidence_grounded` is `false`

Golden report:

```text
examples/verify/goldens/native_non_v1_claims_report.json
```

## OpenDataLoader-Style JSON

```bash
Expand Down
11 changes: 11 additions & 0 deletions examples/verify/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,17 @@ ethos verify examples/verify/native_split_quote_document.json \
Expected result: `all_evidence_grounded: true`. The quote locator points at the second adjacent
text element, and the verifier grounds the claim only through the explicit adjacent-element rule.

## Native Non-v1 Claim Policy

```bash
ethos verify schemas/examples/document.example.json \
--citations examples/verify/native_non_v1_claims_citations.json \
--out verification_report.json
```

Expected result: `all_evidence_grounded: false`. The page presence check can ground, but
`region` and `other` remain unsupported non-v1 claim kinds and are reported explicitly.

## Native Ethos Ungrounded Citations

```bash
Expand Down
6 changes: 6 additions & 0 deletions examples/verify/check_verify_alpha.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@
"citations": "examples/verify/native_split_quote_citations.json",
"golden": "examples/verify/goldens/native_split_quote_report.json",
},
{
"name": "native-non-v1-claims",
"input": "schemas/examples/document.example.json",
"citations": "examples/verify/native_non_v1_claims_citations.json",
"golden": "examples/verify/goldens/native_non_v1_claims_report.json",
},
{
"name": "native-ungrounded",
"input": "schemas/examples/document.example.json",
Expand Down
80 changes: 80 additions & 0 deletions examples/verify/goldens/native_non_v1_claims_report.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
{
"all_evidence_grounded": false,
"capability_limits": [],
"checks": [
{
"claim": {
"citation": {
"page": "p0001"
},
"kind": "presence"
},
"evidence": {
"bbox": [
0,
0,
61200,
79200
],
"page": "p0001"
},
"id": "v0001",
"match_method": "presence_only",
"semantic_unverified": false,
"status": "grounded",
"warnings": []
},
{
"claim": {
"citation": {
"element_id": "e000002"
},
"kind": "region"
},
"id": "v0002",
"match_method": "none",
"reason": "unsupported_claim_kind",
"semantic_unverified": false,
"status": "unsupported_claim_kind",
"warnings": []
},
{
"claim": {
"citation": {
"element_id": "e000002"
},
"kind": "other",
"text": "$12.4M equals 12400000"
},
"id": "v0003",
"match_method": "none",
"reason": "unsupported_claim_kind",
"semantic_unverified": false,
"status": "unsupported_claim_kind",
"warnings": []
}
],
"document_fingerprint": "sha256:b5d30710d0c25cc38d8dec924ecaf57ae4f81276dd5dc14d75cb3b5b6bde62d3",
"fingerprint_stale": false,
"grounding": {
"capabilities": {
"char_offsets": true,
"coordinate_origin": "top-left",
"crop_support": false,
"fingerprint": true,
"spans": true,
"tables": true
},
"parser": {
"name": "ethos",
"version": "0.1.0"
}
},
"schema_version": "1.0.0",
"unsupported_claim_kinds": [
"region",
"other"
],
"verification_config_sha256": "4bb224166a04a25fed2dd3ecdb9638ddcc5b398658532b73f1c0547e4983d0b0",
"warnings": []
}
24 changes: 24 additions & 0 deletions examples/verify/native_non_v1_claims_citations.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"document_fingerprint": "sha256:b5d30710d0c25cc38d8dec924ecaf57ae4f81276dd5dc14d75cb3b5b6bde62d3",
"claims": [
{
"kind": "presence",
"citation": {
"page": "p0001"
}
},
{
"kind": "region",
"citation": {
"element_id": "e000002"
}
},
{
"kind": "other",
"text": "$12.4M equals 12400000",
"citation": {
"element_id": "e000002"
}
}
]
}
4 changes: 2 additions & 2 deletions schemas/ethos-verification-config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
"config_version": { "type": "string", "description": "User-facing label for this config (e.g. 'default-v1')." },
"claim_kinds": {
"type": "array",
"description": "Claim kinds this run supports; kinds in the input but not here are reported under unsupported_claim_kinds.",
"items": { "enum": ["quote", "value", "presence", "table_cell", "region"] },
"description": "Claim kinds this run supports. Configs accept only the four v1 literal kinds: quote, value, presence, and table_cell. Non-v1 citation kinds are reported under unsupported_claim_kinds rather than enabled here.",
"items": { "enum": ["quote", "value", "presence", "table_cell"] },
"minItems": 1,
"uniqueItems": true
},
Expand Down
Loading