From a099a337ba5bf9d99922d7e89a0df873657e5bd6 Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 16 Jun 2026 18:28:49 +0530 Subject: [PATCH] Lock non-v1 claim policy Signed-off-by: docushell-admin --- crates/ethos-cli/src/cmd/verify.rs | 5 +- crates/ethos-cli/tests/verify.rs | 41 +++++++++- crates/ethos-core/src/verify_types.rs | 6 +- crates/ethos-verify/src/lib.rs | 73 +++++++++++++++++ docs/demos/verify-alpha.md | 22 +++++ examples/verify/README.md | 11 +++ examples/verify/check_verify_alpha.py | 6 ++ .../goldens/native_non_v1_claims_report.json | 80 +++++++++++++++++++ .../native_non_v1_claims_citations.json | 24 ++++++ schemas/ethos-verification-config.schema.json | 4 +- 10 files changed, 264 insertions(+), 8 deletions(-) create mode 100644 examples/verify/goldens/native_non_v1_claims_report.json create mode 100644 examples/verify/native_non_v1_claims_citations.json diff --git a/crates/ethos-cli/src/cmd/verify.rs b/crates/ethos-cli/src/cmd/verify.rs index a5e8368..57def53 100644 --- a/crates/ethos-cli/src/cmd/verify.rs +++ b/crates/ethos-cli/src/cmd/verify.rs @@ -1009,9 +1009,10 @@ fn validate_verification_config(config: &VerificationConfig) -> Result<(), Failu } let mut seen = HashSet::new(); for kind in &config.claim_kinds { - if *kind == ClaimKind::Other { + if matches!(kind, ClaimKind::Region | ClaimKind::Other) { return Err(Failure::Usage( - "verification config claim_kinds must not include other".to_string(), + "verification config claim_kinds must include only quote, value, presence, and table_cell" + .to_string(), )); } if !seen.insert(*kind) { diff --git a/crates/ethos-cli/tests/verify.rs b/crates/ethos-cli/tests/verify.rs index 2ed574c..c270e3c 100644 --- a/crates/ethos-cli/tests/verify.rs +++ b/crates/ethos-cli/tests/verify.rs @@ -148,7 +148,7 @@ fn verify_alpha_schema_report_example_matches_cli_output() { #[test] fn verify_alpha_demo_reports_match_goldens() { let root = repo_root(); - let cases: [(&str, Vec, PathBuf); 5] = [ + let cases: [(&str, Vec, PathBuf); 6] = [ ( "native-grounded", vec![ @@ -193,6 +193,20 @@ fn verify_alpha_demo_reports_match_goldens() { ], root.join("examples/verify/goldens/native_split_quote_report.json"), ), + ( + "native-non-v1-claims", + vec![ + "verify".to_string(), + root.join("schemas/examples/document.example.json") + .display() + .to_string(), + "--citations".to_string(), + root.join("examples/verify/native_non_v1_claims_citations.json") + .display() + .to_string(), + ], + root.join("examples/verify/goldens/native_non_v1_claims_report.json"), + ), ( "native-stale", vec![ @@ -1494,7 +1508,27 @@ fn invalid_config_constraints_are_usage_errors() { "max_checks": 256 } }"#, - "verification config claim_kinds must not include other", + "verification config claim_kinds must include only quote, value, presence, and table_cell", + ), + ( + "region-claim-kind-config", + r#"{ + "schema_version": "1.0.0", + "config_version": "region-kind", + "claim_kinds": ["region"], + "matching": { + "text_normalization": "collapse_whitespace", + "case_sensitive": true, + "bbox_containment_tolerance_q": 50 + }, + "staleness": { + "require_fingerprint_match": true + }, + "limits": { + "max_checks": 256 + } + }"#, + "verification config claim_kinds must include only quote, value, presence, and table_cell", ), ( "negative-bbox-tolerance-config", @@ -2033,6 +2067,9 @@ fn config_excluded_value_claim_is_unsupported() { assert_eq!(report["checks"][0]["status"], "unsupported_claim_kind"); assert_eq!(report["checks"][0]["reason"], "unsupported_claim_kind"); + assert_eq!(report["checks"][0]["match_method"], "none"); + assert_eq!(report["checks"][0]["semantic_unverified"], false); + assert!(report["checks"][0].get("evidence").is_none()); assert_eq!( report["unsupported_claim_kinds"], serde_json::json!(["value"]) diff --git a/crates/ethos-core/src/verify_types.rs b/crates/ethos-core/src/verify_types.rs index 3426a8f..ba471e7 100644 --- a/crates/ethos-core/src/verify_types.rs +++ b/crates/ethos-core/src/verify_types.rs @@ -43,7 +43,8 @@ pub enum CapabilityLimit { MissingCropSupport, } -/// Claim kinds. `Other` appears only in reports (as an unsupported kind), never in configs. +/// Claim kinds. `Region` and `Other` appear only in citations/reports as unsupported non-v1 +/// kinds, never in configs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ClaimKind { @@ -368,7 +369,8 @@ pub struct VerificationConfig { pub schema_version: String, /// User-facing config label, e.g. `"default-v1"`. pub config_version: String, - /// Supported claim kinds for this run (never `Other`). + /// Supported claim kinds for this run. Configs accept only the four v1 literal kinds: + /// quote, value, presence, and table_cell. pub claim_kinds: Vec, /// Matching parameters. pub matching: Matching, diff --git a/crates/ethos-verify/src/lib.rs b/crates/ethos-verify/src/lib.rs index 8a4bb3e..ae52125 100644 --- a/crates/ethos-verify/src/lib.rs +++ b/crates/ethos-verify/src/lib.rs @@ -1881,6 +1881,79 @@ mod tests { assert_eq!(report.unsupported_claim_kinds, vec!["region"]); } + #[test] + fn non_v1_claim_kinds_are_deduped_and_keep_gate_false() { + let source = TestSource::default(); + let report = verify( + &source, + vec![ + claim( + ClaimKind::Presence, + None, + Citation { + page: Some("p0001".into()), + ..Default::default() + }, + ), + claim( + ClaimKind::Region, + None, + Citation { + element_id: Some("e000002".into()), + ..Default::default() + }, + ), + claim( + ClaimKind::Other, + Some("$12.4M equals 12400000"), + Citation { + element_id: Some("e000002".into()), + ..Default::default() + }, + ), + claim( + ClaimKind::Region, + None, + Citation { + page: Some("p0001".into()), + ..Default::default() + }, + ), + ], + ); + + assert!(!report.all_evidence_grounded); + assert_eq!(report.checks[0].status, CheckStatus::Grounded); + assert_eq!(report.checks[1].status, CheckStatus::UnsupportedClaimKind); + assert_eq!(report.checks[2].status, CheckStatus::UnsupportedClaimKind); + assert_eq!(report.checks[3].status, CheckStatus::UnsupportedClaimKind); + assert_eq!(report.checks[1].match_method, MatchMethod::None); + assert_eq!(report.checks[2].match_method, MatchMethod::None); + assert_eq!(report.checks[3].match_method, MatchMethod::None); + assert_eq!( + report.checks[1].reason, + Some(CheckReason::UnsupportedClaimKind) + ); + assert_eq!( + report.checks[2].reason, + Some(CheckReason::UnsupportedClaimKind) + ); + assert_eq!( + report.checks[3].reason, + Some(CheckReason::UnsupportedClaimKind) + ); + assert!(report.checks[1].evidence.is_none()); + assert!(report.checks[2].evidence.is_none()); + assert!(report.checks[3].evidence.is_none()); + assert!(report.checks[1].warnings.is_empty()); + assert!(report.checks[2].warnings.is_empty()); + assert!(report.checks[3].warnings.is_empty()); + assert!(!report.checks[1].semantic_unverified); + assert!(!report.checks[2].semantic_unverified); + assert!(!report.checks[3].semantic_unverified); + assert_eq!(report.unsupported_claim_kinds, vec!["region", "other"]); + } + #[test] fn missing_span_capability_blocks_span_locator() { let source = TestSource { diff --git a/docs/demos/verify-alpha.md b/docs/demos/verify-alpha.md index 3a33e8b..7400397 100644 --- a/docs/demos/verify-alpha.md +++ b/docs/demos/verify-alpha.md @@ -12,6 +12,7 @@ repeatable `make verify-alpha` path: - OpenDataLoader-style JSON can enter the same verification loop through a grounding adapter - real pinned OpenDataLoader 2.4.7 output has both grounded and ungrounded citation cases - native and synthetic OpenDataLoader fixtures cover missing cited elements +- native fixtures cover unsupported non-v1 claim kinds - malformed citation inputs and malformed OpenDataLoader-style grounding inputs return usage diagnostics with exit code `2` - `--fail-on-ungrounded` turns the report into a CI/agent gate with exit code `1` when evidence is not fully grounded @@ -71,6 +72,27 @@ Golden report: examples/verify/goldens/native_ungrounded_report.json ``` +## Native Non-v1 Claim Policy + +```bash +ethos verify schemas/examples/document.example.json \ + --citations examples/verify/native_non_v1_claims_citations.json \ + --out /tmp/ethos-native-non-v1-report.json +``` + +Expected outcome: + +- the presence check is grounded +- `region` and `other` checks have status `unsupported_claim_kind` +- `unsupported_claim_kinds` lists `region` and `other` +- `all_evidence_grounded` is `false` + +Golden report: + +```text +examples/verify/goldens/native_non_v1_claims_report.json +``` + ## OpenDataLoader-Style JSON ```bash diff --git a/examples/verify/README.md b/examples/verify/README.md index 53ef4c3..46ce0b0 100644 --- a/examples/verify/README.md +++ b/examples/verify/README.md @@ -39,6 +39,17 @@ ethos verify examples/verify/native_split_quote_document.json \ Expected result: `all_evidence_grounded: true`. The quote locator points at the second adjacent text element, and the verifier grounds the claim only through the explicit adjacent-element rule. +## Native Non-v1 Claim Policy + +```bash +ethos verify schemas/examples/document.example.json \ + --citations examples/verify/native_non_v1_claims_citations.json \ + --out verification_report.json +``` + +Expected result: `all_evidence_grounded: false`. The page presence check can ground, but +`region` and `other` remain unsupported non-v1 claim kinds and are reported explicitly. + ## Native Ethos Ungrounded Citations ```bash diff --git a/examples/verify/check_verify_alpha.py b/examples/verify/check_verify_alpha.py index 262bdfd..d7e4735 100644 --- a/examples/verify/check_verify_alpha.py +++ b/examples/verify/check_verify_alpha.py @@ -46,6 +46,12 @@ "citations": "examples/verify/native_split_quote_citations.json", "golden": "examples/verify/goldens/native_split_quote_report.json", }, + { + "name": "native-non-v1-claims", + "input": "schemas/examples/document.example.json", + "citations": "examples/verify/native_non_v1_claims_citations.json", + "golden": "examples/verify/goldens/native_non_v1_claims_report.json", + }, { "name": "native-ungrounded", "input": "schemas/examples/document.example.json", diff --git a/examples/verify/goldens/native_non_v1_claims_report.json b/examples/verify/goldens/native_non_v1_claims_report.json new file mode 100644 index 0000000..8577e14 --- /dev/null +++ b/examples/verify/goldens/native_non_v1_claims_report.json @@ -0,0 +1,80 @@ +{ + "all_evidence_grounded": false, + "capability_limits": [], + "checks": [ + { + "claim": { + "citation": { + "page": "p0001" + }, + "kind": "presence" + }, + "evidence": { + "bbox": [ + 0, + 0, + 61200, + 79200 + ], + "page": "p0001" + }, + "id": "v0001", + "match_method": "presence_only", + "semantic_unverified": false, + "status": "grounded", + "warnings": [] + }, + { + "claim": { + "citation": { + "element_id": "e000002" + }, + "kind": "region" + }, + "id": "v0002", + "match_method": "none", + "reason": "unsupported_claim_kind", + "semantic_unverified": false, + "status": "unsupported_claim_kind", + "warnings": [] + }, + { + "claim": { + "citation": { + "element_id": "e000002" + }, + "kind": "other", + "text": "$12.4M equals 12400000" + }, + "id": "v0003", + "match_method": "none", + "reason": "unsupported_claim_kind", + "semantic_unverified": false, + "status": "unsupported_claim_kind", + "warnings": [] + } + ], + "document_fingerprint": "sha256:b5d30710d0c25cc38d8dec924ecaf57ae4f81276dd5dc14d75cb3b5b6bde62d3", + "fingerprint_stale": false, + "grounding": { + "capabilities": { + "char_offsets": true, + "coordinate_origin": "top-left", + "crop_support": false, + "fingerprint": true, + "spans": true, + "tables": true + }, + "parser": { + "name": "ethos", + "version": "0.1.0" + } + }, + "schema_version": "1.0.0", + "unsupported_claim_kinds": [ + "region", + "other" + ], + "verification_config_sha256": "4bb224166a04a25fed2dd3ecdb9638ddcc5b398658532b73f1c0547e4983d0b0", + "warnings": [] +} diff --git a/examples/verify/native_non_v1_claims_citations.json b/examples/verify/native_non_v1_claims_citations.json new file mode 100644 index 0000000..ef41f5f --- /dev/null +++ b/examples/verify/native_non_v1_claims_citations.json @@ -0,0 +1,24 @@ +{ + "document_fingerprint": "sha256:b5d30710d0c25cc38d8dec924ecaf57ae4f81276dd5dc14d75cb3b5b6bde62d3", + "claims": [ + { + "kind": "presence", + "citation": { + "page": "p0001" + } + }, + { + "kind": "region", + "citation": { + "element_id": "e000002" + } + }, + { + "kind": "other", + "text": "$12.4M equals 12400000", + "citation": { + "element_id": "e000002" + } + } + ] +} diff --git a/schemas/ethos-verification-config.schema.json b/schemas/ethos-verification-config.schema.json index 68d1748..e316b56 100644 --- a/schemas/ethos-verification-config.schema.json +++ b/schemas/ethos-verification-config.schema.json @@ -11,8 +11,8 @@ "config_version": { "type": "string", "description": "User-facing label for this config (e.g. 'default-v1')." }, "claim_kinds": { "type": "array", - "description": "Claim kinds this run supports; kinds in the input but not here are reported under unsupported_claim_kinds.", - "items": { "enum": ["quote", "value", "presence", "table_cell", "region"] }, + "description": "Claim kinds this run supports. Configs accept only the four v1 literal kinds: quote, value, presence, and table_cell. Non-v1 citation kinds are reported under unsupported_claim_kinds rather than enabled here.", + "items": { "enum": ["quote", "value", "presence", "table_cell"] }, "minItems": 1, "uniqueItems": true },