From 625375e9d1072e1b2d589fee3414c79171786435 Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 16 Jun 2026 13:09:32 +0530 Subject: [PATCH] Cover verify summary in alpha demo Signed-off-by: docushell-admin --- README.md | 1 + docs/demos/verify-alpha.md | 1 + examples/verify/README.md | 3 + examples/verify/check_verify_alpha.py | 82 ++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3cdb18d..04f4aeb 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ ok real-opendataloader-grounded matches fixtures/foreign/opendataloader/real/ ok real-opendataloader-ungrounded matches fixtures/foreign/opendataloader/real/expected.ungrounded.verification_report.json ok invalid-table-cell-citation exits 2 with expected usage diagnostic ok invalid-bbox-citation exits 2 with expected usage diagnostic +ok native-ungrounded-summary summary includes expected diagnostics ok native-grounded-crops crop descriptors validate against schemas/ethos-crop-descriptor.schema.json verify-alpha demo checks passed diff --git a/docs/demos/verify-alpha.md b/docs/demos/verify-alpha.md index b688a61..505c804 100644 --- a/docs/demos/verify-alpha.md +++ b/docs/demos/verify-alpha.md @@ -46,6 +46,7 @@ examples/verify/goldens/native_grounded_report.json For terminal review, `--format summary` emits a compact text view with the verification config hash, declared grounding capabilities, check counts, warnings, and non-grounded check reasons. The JSON report remains the canonical artifact. +`make verify-alpha` checks this summary path for a native ungrounded citation set. ## Native Ethos Ungrounded Citations diff --git a/examples/verify/README.md b/examples/verify/README.md index 2db496a..d656afd 100644 --- a/examples/verify/README.md +++ b/examples/verify/README.md @@ -25,6 +25,9 @@ ethos verify schemas/examples/document.example.json \ The summary includes the verification config hash, declared grounding capabilities, check counts, report warnings, and non-grounded check reasons. +`make verify-alpha` also checks summary output for a native ungrounded citation set, including +the non-grounded reason lines. + ## Native Ethos Ungrounded Citations ```bash diff --git a/examples/verify/check_verify_alpha.py b/examples/verify/check_verify_alpha.py index 2458c76..80f5ae7 100644 --- a/examples/verify/check_verify_alpha.py +++ b/examples/verify/check_verify_alpha.py @@ -81,6 +81,40 @@ }, ] +SUMMARY_CASES = [ + { + "name": "native-ungrounded-summary", + "input": "schemas/examples/document.example.json", + "citations": "examples/verify/native_ungrounded_citations.json", + "expected_exit": 1, + "fail_on_ungrounded": True, + "stdout_contains": [ + "ethos verify summary\n", + ( + "verification_config_sha256: " + "4bb224166a04a25fed2dd3ecdb9638ddcc5b398658532b73f1c0547e4983d0b0\n" + ), + "all_evidence_grounded: false\n", + ( + "grounding_capabilities: " + "spans=true,char_offsets=true,tables=true,fingerprint=true," + "coordinate_origin=top-left,crop_support=false\n" + ), + "checks_not_found: 1\n", + "checks_mismatch: 1\n", + ( + "- v0001 status=mismatch reason=text_mismatch kind=quote " + "locator=page:p0001;element_id:e000002 " + "match_method=normalized_text_contains\n" + ), + ( + "- v0002 status=not_found reason=element_not_found kind=presence " + "locator=element_id:missing-element match_method=none\n" + ), + ], + }, +] + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) @@ -146,7 +180,9 @@ def compare_json(actual_path, expected_path, repo_root, name): def list_crop_descriptors(crop_dir): return sorted( - path for path in crop_dir.iterdir() if path.name.startswith("crop-") and path.suffix == ".json" + path + for path in crop_dir.iterdir() + if path.name.startswith("crop-") and path.suffix == ".json" ) @@ -313,6 +349,48 @@ def verify_usage_error_case(case, args): print(f"ok {case['name']} exits 2 with expected usage diagnostic") +def verify_summary_case(case, args): + command = [ + str(args.ethos_bin), + "verify", + str(args.repo_root / case["input"]), + "--citations", + str(args.repo_root / case["citations"]), + "--format", + "summary", + ] + if "grounding" in case: + command.extend(["--grounding", case["grounding"]]) + if case.get("fail_on_ungrounded", False): + command.append("--fail-on-ungrounded") + + print("$ " + " ".join(str(part) for part in command), flush=True) + result = subprocess.run(command, cwd=args.repo_root, capture_output=True, check=False) + expected_exit = case["expected_exit"] + if result.returncode != expected_exit: + sys.stderr.write( + f"{case['name']} exited {result.returncode}, expected {expected_exit}\n" + ) + sys.stderr.write(result.stderr.decode("utf-8", errors="replace")) + sys.stderr.write(result.stdout.decode("utf-8", errors="replace")) + raise SystemExit(1) + if result.stderr: + raise SystemExit(f"{case['name']} wrote unexpected stderr") + stdout = result.stdout.decode("utf-8", errors="replace") + try: + json.loads(stdout) + except json.JSONDecodeError: + pass + else: + raise SystemExit(f"{case['name']} summary output unexpectedly parsed as JSON") + for expected in case["stdout_contains"]: + if expected not in stdout: + raise SystemExit( + f"{case['name']} stdout did not contain {expected!r}\n{stdout}" + ) + print(f"ok {case['name']} summary includes expected diagnostics") + + def main(): args = parse_args() args.repo_root = args.repo_root.resolve() @@ -328,6 +406,8 @@ def main(): verify_case(case, args) for case in USAGE_ERROR_CASES: verify_usage_error_case(case, args) + for case in SUMMARY_CASES: + verify_summary_case(case, args) verify_crop_descriptor_case(args) print("\nverify-alpha demo checks passed")