From b37644d3d2319e243936fc5dda01742b6d2e93d2 Mon Sep 17 00:00:00 2001 From: claude-bot-go Date: Mon, 27 Jul 2026 14:04:04 +0800 Subject: [PATCH] Fix #27: [milestone Milestone 2 ] Update `fixtures/golden/*.jsonl` and `tests/test_schemas*.py` to validate agains... --- fixtures/fixtures.lock.json | 8 +-- .../golden/compliance-direct-fail.v1.jsonl | 2 +- .../golden/compliance-direct-pass.v1.jsonl | 2 +- .../golden/compliance-repair-pass.v1.jsonl | 2 +- tests/test_schemas_json.py | 55 +++++++++++++++++++ 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/fixtures/fixtures.lock.json b/fixtures/fixtures.lock.json index 9db8bc5..2bcb30f 100644 --- a/fixtures/fixtures.lock.json +++ b/fixtures/fixtures.lock.json @@ -1,11 +1,11 @@ { "version": "1", - "updated": "2026-06-25", + "updated": "2026-07-27", "fixtures": [ { "fixture": "fixtures/golden/compliance-direct-fail.v1.jsonl", "schema_version": "compliance-eval-record/v1", - "sha256": "654c8ed5f92ee12023c3449ba75091ac815373f9f7d6d27a8ae9daca47ec752b", + "sha256": "b35c9802381be3ff95111bf6f4e56f0cb6fbd00a0bfcd2563bca573869bb90bf", "n_records": 1, "generated_by": "trace-pipeline/scripts", "generated_at": "2026-06-25" @@ -13,7 +13,7 @@ { "fixture": "fixtures/golden/compliance-direct-pass.v1.jsonl", "schema_version": "compliance-eval-record/v1", - "sha256": "a80a9bc0face93d7880a02584663825929c5529968497b45fafc17dc1eeaad3b", + "sha256": "3112665d1862cfe238f8c740a1fc1efb14d00254e83fa22762933901a357a828", "n_records": 1, "generated_by": "trace-pipeline/scripts", "generated_at": "2026-06-25" @@ -21,7 +21,7 @@ { "fixture": "fixtures/golden/compliance-repair-pass.v1.jsonl", "schema_version": "compliance-eval-record/v1", - "sha256": "20d5d1b5ad3875b5b6a74edc36c2d13782fafcceeba5931307c1a9a75a2aab0f", + "sha256": "c9998885aa3f177d71a1808d79896145ecd6b0223503f95c148d550b927932b9", "n_records": 1, "generated_by": "trace-pipeline/scripts", "generated_at": "2026-06-25" diff --git a/fixtures/golden/compliance-direct-fail.v1.jsonl b/fixtures/golden/compliance-direct-fail.v1.jsonl index a98ee83..fce9e2f 100644 --- a/fixtures/golden/compliance-direct-fail.v1.jsonl +++ b/fixtures/golden/compliance-direct-fail.v1.jsonl @@ -1 +1 @@ -{"schema_version":"compliance-eval-record/v1","task_id":"golden-task-01","task_spec_hash":"1ce85aeef714574b6692d80b006bd8b7cf2d514a71cac6797ef4c7cd832a4171","model":"qwen2.5-1.5b","mode":"direct","violations":[{"constraint_id":"c_title","level":"hard","category":"format","hint":"response is missing a title wrapped in <<...>>","evidence_span":null,"detected_at":"post_decode"}],"repair_trace":[],"repair_rounds":0,"final_pass":false,"token_cost":{"prompt":42,"generation":18,"repair":null},"latency_ms":221.0,"artifact":"Water evaporates and forms clouds. It then falls as precipitation.","error":null} +{"schema_version":"compliance-eval-record/v1","task_id":"golden-task-01","task_spec_hash":"1ce85aeef714574b6692d80b006bd8b7cf2d514a71cac6797ef4c7cd832a4171","model":"qwen2.5-1.5b","mode":"direct","violations":[{"constraint_id":"c_title","level":"hard","category":"format","hint":"response is missing a title wrapped in <<...>>","detected_at":"post_decode"}],"repair_trace":[],"repair_rounds":0,"final_pass":false,"token_cost":{"prompt":42,"generation":18},"latency_ms":221.0,"artifact":"Water evaporates and forms clouds. It then falls as precipitation."} diff --git a/fixtures/golden/compliance-direct-pass.v1.jsonl b/fixtures/golden/compliance-direct-pass.v1.jsonl index 4efa9a7..ee3f871 100644 --- a/fixtures/golden/compliance-direct-pass.v1.jsonl +++ b/fixtures/golden/compliance-direct-pass.v1.jsonl @@ -1 +1 @@ -{"schema_version":"compliance-eval-record/v1","task_id":"golden-task-01","task_spec_hash":"1ce85aeef714574b6692d80b006bd8b7cf2d514a71cac6797ef4c7cd832a4171","model":"qwen2.5-1.5b","mode":"direct","violations":[],"repair_trace":[],"repair_rounds":0,"final_pass":true,"token_cost":{"prompt":42,"generation":38,"repair":null},"latency_ms":348.0,"artifact":"<>\nWater evaporates and forms clouds. It then falls as precipitation.\nAction List:\n- Monitor precipitation levels daily","error":null} +{"schema_version":"compliance-eval-record/v1","task_id":"golden-task-01","task_spec_hash":"1ce85aeef714574b6692d80b006bd8b7cf2d514a71cac6797ef4c7cd832a4171","model":"qwen2.5-1.5b","mode":"direct","violations":[],"repair_trace":[],"repair_rounds":0,"final_pass":true,"token_cost":{"prompt":42,"generation":38},"latency_ms":348.0,"artifact":"<>\nWater evaporates and forms clouds. It then falls as precipitation.\nAction List:\n- Monitor precipitation levels daily"} diff --git a/fixtures/golden/compliance-repair-pass.v1.jsonl b/fixtures/golden/compliance-repair-pass.v1.jsonl index 94b8a75..313c5f3 100644 --- a/fixtures/golden/compliance-repair-pass.v1.jsonl +++ b/fixtures/golden/compliance-repair-pass.v1.jsonl @@ -1 +1 @@ -{"schema_version":"compliance-eval-record/v1","task_id":"golden-task-01","task_spec_hash":"1ce85aeef714574b6692d80b006bd8b7cf2d514a71cac6797ef4c7cd832a4171","model":"qwen2.5-1.5b","mode":"full_pcl","violations":[{"constraint_id":"c_title","level":"hard","category":"format","hint":"response is missing a title wrapped in <<...>>","evidence_span":null,"detected_at":"post_decode"}],"repair_trace":[{"round":1,"violation_ids":["c_title"],"strategy":"insert_section","target_region":null,"ok":true,"rolled_back":false,"remaining_violation_ids":[],"token_cost":null,"latency_ms":312.0}],"repair_rounds":1,"final_pass":true,"token_cost":{"prompt":42,"generation":38,"repair":55},"latency_ms":504.0,"artifact":"<>\nWater evaporates and forms clouds. It then falls as precipitation.\nAction List:\n- Monitor precipitation levels daily","error":null} +{"schema_version":"compliance-eval-record/v1","task_id":"golden-task-01","task_spec_hash":"1ce85aeef714574b6692d80b006bd8b7cf2d514a71cac6797ef4c7cd832a4171","model":"qwen2.5-1.5b","mode":"full_pcl","violations":[{"constraint_id":"c_title","level":"hard","category":"format","hint":"response is missing a title wrapped in <<...>>","detected_at":"post_decode"}],"repair_trace":[{"round":1,"violation_ids":["c_title"],"strategy":"insert_section","ok":true,"rolled_back":false,"remaining_violation_ids":[],"latency_ms":312.0}],"repair_rounds":1,"final_pass":true,"token_cost":{"prompt":42,"generation":38,"repair":55},"latency_ms":504.0,"artifact":"<>\nWater evaporates and forms clouds. It then falls as precipitation.\nAction List:\n- Monitor precipitation levels daily"} diff --git a/tests/test_schemas_json.py b/tests/test_schemas_json.py index 991e8ee..9791130 100644 --- a/tests/test_schemas_json.py +++ b/tests/test_schemas_json.py @@ -7,6 +7,10 @@ import tempfile from pathlib import Path +import wasmagent_protocol as wp +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + REPO_ROOT = Path(__file__).parent.parent SCHEMAS_DIR = REPO_ROOT / "schemas" @@ -196,3 +200,54 @@ def test_fixture_fields_present_in_rollout_schema(self): for key in record: assert key in props, \ f"fixture field '{key}' not in rollout-wire schema properties" + + +class TestGoldenFixturesValidateAgainstPackageSchema: + """Golden fixtures must validate against the canonical wasmagent-protocol + package schema (the SSOT), not the local fork — Milestone 2, issue #27.""" + + GOLDEN_DIR = REPO_ROOT / "fixtures" / "golden" + + # record schema_version -> wasmagent-protocol schema id + SCHEMA_FOR_VERSION = { + "compliance-eval-record/v1": "compliance-eval-record", + "rollout-wire/v1": "rollout-wire", + } + + @staticmethod + def _package_registry(): + """Register every canonical package schema by $id so $ref siblings + (e.g. constraint-violation, repair-trace referenced from + compliance-eval-record) resolve without network access.""" + docs = [wp.get_schema(sid) for sid in wp.schema_ids()] + return Registry().with_resources( + [(d["$id"], Resource.from_contents(d)) for d in docs] + ) + + def test_every_golden_record_validates(self): + golden = sorted(self.GOLDEN_DIR.glob("*.jsonl")) + assert golden, "no golden fixtures found under fixtures/golden/" + registry = self._package_registry() + failures = [] + for path in golden: + for lineno, raw in enumerate(path.read_text().splitlines(), 1): + raw = raw.strip() + if not raw: + continue + record = json.loads(raw) + version = record.get("schema_version") + assert version in self.SCHEMA_FOR_VERSION, ( + f"{path.name}:{lineno} unmapped schema_version={version!r}" + ) + schema = wp.get_schema(self.SCHEMA_FOR_VERSION[version]) + errors = sorted( + Draft202012Validator(schema, registry=registry).iter_errors(record), + key=lambda e: list(e.path), + ) + for err in errors: + loc = "/".join(str(p) for p in err.path) or "" + failures.append(f"{path.name}:{lineno} {loc}: {err.message}") + assert not failures, ( + "golden fixtures fail the canonical package schema:\n " + + "\n ".join(failures) + )