From 733647838da41eda592fb33611da70c922387b54 Mon Sep 17 00:00:00 2001 From: matchalatteguy <45487848+matchalatteguy@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:33:17 -0300 Subject: [PATCH] Bind pipeline exports to producer digests and add pinned replay review --- .github/workflows/ci.yml | 16 ++ .gitignore | 1 + CHANGELOG.md | 20 +++ README.md | 38 ++++- docs/contract-schema.md | 31 +++- docs/python-api.md | 3 + examples/replay-pipeline/.gitignore | 4 + examples/replay-pipeline/README.md | 111 ++++++++++++++ examples/replay-pipeline/events.jsonl | 13 ++ examples/replay-pipeline/evidence.yaml | 27 ++++ examples/replay-pipeline/manifest.json | 9 ++ examples/replay-pipeline/pyproject.toml | 13 ++ examples/replay-pipeline/run.py | 159 ++++++++++++++++++++ examples/replay-pipeline/scenarios.yaml | 55 +++++++ examples/replay-pipeline/test_pipeline.py | 97 ++++++++++++ examples/replay-pipeline/uv.lock | 174 ++++++++++++++++++++++ pyproject.toml | 2 +- src/evidence_gate/models.py | 1 + src/evidence_gate/specs.py | 14 +- src/evidence_gate/validators.py | 61 ++++++++ tests/test_artifact_integrity.py | 142 ++++++++++++++++++ uv.lock | 2 +- 22 files changed, 985 insertions(+), 8 deletions(-) create mode 100644 examples/replay-pipeline/.gitignore create mode 100644 examples/replay-pipeline/README.md create mode 100644 examples/replay-pipeline/events.jsonl create mode 100644 examples/replay-pipeline/evidence.yaml create mode 100644 examples/replay-pipeline/manifest.json create mode 100644 examples/replay-pipeline/pyproject.toml create mode 100644 examples/replay-pipeline/run.py create mode 100644 examples/replay-pipeline/scenarios.yaml create mode 100644 examples/replay-pipeline/test_pipeline.py create mode 100644 examples/replay-pipeline/uv.lock create mode 100644 tests/test_artifact_integrity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d349333..7c25ac8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,19 @@ jobs: "$RUNNER_TEMP/wheel-env/bin/evidence-gate" init-example wheel-example "$RUNNER_TEMP/wheel-env/bin/evidence-gate" validate --spec wheel-example/evidence-gate.yaml --run wheel-example/runs/demo-run --json-out status.json "$RUNNER_TEMP/wheel-env/bin/evidence-gate" packet --status status.json --md-out review-packet.md + + replay-pipeline: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - name: Verify the pinned companion tools and failure paths + working-directory: examples/replay-pipeline + run: | + uv sync --locked --group dev + uv run --locked pytest -q test_pipeline.py + uv run --locked python run.py --output .runs/ci --demonstrate-failure diff --git a/.gitignore b/.gitignore index 5969d20..191c6fb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ __pycache__/ dist/ build/ *.egg-info/ +examples/replay-pipeline/.runs/ reports/evidence-status.json reports/review-packet.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5feda5b..e021f21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 0.3.0 — 2026-10-02 + +- Add a reproducible input-contract → replay-comparison → evidence-review example + with companion repositories pinned to immutable commits. +- Exercise incomplete exports, duplicate input events, missing future markouts, + and refusal to overwrite earlier runs in a separate integration CI job. +- Add optional output `sha256_field` checks, using chunked file hashing. Missing or + malformed expected digests, mismatches, and read errors fail validation. +- Bind all four replay exports to their producer-recorded hashes, with regressions + for truncated CSVs and empty/tampered JSON, Markdown, and input-contract reports. +- Keep the standalone checker's runtime dependencies unchanged. + +Compatibility: contract schema remains `1`. Outputs without `sha256_field` retain +their existing presence/header behavior. Absent optional outputs still warn; when +present, a configured digest must validate. An explicitly configured field must +be a non-empty dotted path, and its report value must be 64 hexadecimal characters. +Older package versions reject the new contract key; use 0.3+ for digest contracts. +Matching bytes trust the producer's report and do not authenticate source data or +validate artifact content/schema. CSV checks continue to inspect headers, not rows. + ## 0.2.0 ### Contract and status validation diff --git a/README.md b/README.md index bb7f73d..1837a14 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,28 @@ After export: exit 0; approved (10 checks) The example data and metrics are synthetic; the demonstration is of the gate's behavior. +## Review a complete replay pipeline + +The [replay review example](examples/replay-pipeline/README.md) connects three tools: +input ordering and identity checks, execution-assumption comparisons, and this evidence gate. +It runs three profiles over two instruments with multiple book levels, produces JSON/CSV/Markdown +artifacts, then demonstrates a missing export that blocks review despite passing metrics. +The example pins its companion tools and checks the actual emitted reports. + +```bash +cd examples/replay-pipeline +uv run --locked python run.py --output .runs/complete --demonstrate-failure +``` + +Separate failure cases show duplicate inputs stopping before simulation and missing future +evidence preventing promotion. The prices and decisions are synthetic; this is a pipeline +verification example, not a strategy-performance estimate. + ## Use it in your pipeline ```bash -# Install from this checkout; uv users can keep using uv run instead. -python -m pip install . +# Install the versioned CLI. From a checkout, uv run or pip install . also works. +uv tool install 'git+https://github.com/matchalatteguy/evidence-gate.git@v0.3.0' evidence-gate init-example scratch/toy-run evidence-gate validate \ @@ -71,6 +88,19 @@ reports: The report is a JSON object containing the named `status`, `summary`, `counts`, `metrics`, and `outputs` fields. Output paths are relative to the run root. Contracts reject empty report lists, unknown fields, and contradictory thresholds; paths that escape the run root, including escaping symlinks, fail validation. +## Detect changed or truncated exports + +Optionally bind an output to the SHA-256 digest recorded in its producer's report: + +```yaml +outputs: + predictions: + path_field: outputs.predictions + sha256_field: sha256.predictions +``` + +After finishing the export, the producer writes its 64-hex digest at `sha256.predictions` in the report. Validation hashes the artifact in chunks and fails for missing/malformed digests, changed bytes, or read errors. The [replay example](examples/replay-pipeline/README.md) applies this to all four exports, so a header-only CSV or emptied report cannot retain approval. [Digest contract details](docs/contract-schema.md#output-byte-integrity). + ## Python API ```python @@ -84,9 +114,9 @@ for failure in result.failures: ## What the decision means -`approved` means the configured file and value checks passed. It does not verify that a report is truthful, that an experiment is sound, or that a model is useful. CSV checks inspect headers, not rows. Contracts and files should come from trusted local pipelines; containment checks are not a sandbox for hostile processes changing files during validation. +`approved` means the configured file and value checks passed. It does not verify that a report is truthful, that an experiment is sound, or that a model is useful. CSV checks inspect headers, not rows. A matching digest establishes byte integrity against the trusted producer's report; it does not authenticate sources or validate artifact content/schema. Contracts and files should come from trusted local pipelines; containment checks are not a sandbox for hostile processes changing files during validation. -The tool supports JSON reports, finite numeric thresholds, local file artifacts, and CSV headers. It has no scheduler, hosted state, network calls, or plugin execution. Version 0.2 tightens contract and status validation; see the [migration notes](CHANGELOG.md). +The tool supports JSON reports, finite numeric thresholds, local file artifacts, optional SHA-256 integrity checks, and CSV headers. It has no scheduler, hosted state, network calls, or plugin execution. Version 0.3 adds opt-in artifact digests while preserving unconfigured contracts; see the [migration notes](CHANGELOG.md). ## Documentation and development diff --git a/docs/contract-schema.md b/docs/contract-schema.md index a6d0050..9a1f400 100644 --- a/docs/contract-schema.md +++ b/docs/contract-schema.md @@ -55,7 +55,7 @@ The report list must be non-empty and report names must be unique. Unknown or du ## Dot paths -`required_fields` and output `path_field` values use simple dot paths through JSON objects: +`required_fields`, output `path_field`, and output `sha256_field` values use simple dot paths through JSON objects: ```yaml required_fields: @@ -110,6 +110,35 @@ outputs: The referenced report value must be a relative path string under the run root. Required missing artifacts fail; optional missing artifacts warn. Each output spec must be a mapping with a `path_field`; malformed output specs raise a clear `ValueError` during spec loading. When `csv_columns` is set, Evidence Gate reads only the CSV header and checks that every listed column is present. Missing columns fail with `table.column_missing`; unreadable or non-UTF-8 CSV files fail with `table.invalid_csv` instead of crashing validation. The old `columns` spelling is accepted as a backward-compatible alias, but new contracts should use `csv_columns` so the CSV-specific behavior is explicit. +### Output byte integrity + +Set `sha256_field` to the dotted field containing the producer's expected file digest: + +```yaml +outputs: + predictions: + path_field: outputs.predictions + sha256_field: sha256.predictions + csv_columns: [id, label, score] +``` + +The field name must be a non-empty string with non-empty dotted components and no surrounding component whitespace. The expected report value must contain exactly 64 hexadecimal characters; uppercase and lowercase are accepted. A producer can record the digest after completing its output: + +```python +import hashlib + +# report already contains outputs.predictions; run_root is the run directory. +with (run_root / report["outputs"]["predictions"]).open("rb") as source: + report["sha256"] = { + "predictions": hashlib.file_digest(source, "sha256").hexdigest() + } +# Write this report after the export is closed and the digest is recorded. +``` + +Validation reads the artifact in 1 MiB chunks. A matching digest adds `artifact.sha256_match`. Missing fields, malformed digests, changed bytes, and read errors fail with `artifact.sha256_missing`, `artifact.sha256_invalid`, `artifact.sha256_mismatch`, and `artifact.sha256_read_error`, respectively. + +Without this option, existing presence/header checks are unchanged. A missing optional artifact still warns without requiring a digest; a present optional artifact must satisfy its configured digest. The hash binds bytes to a trusted producer report. It does not authenticate that report or its sources, establish semantic correctness, or validate the artifact's content/schema. Preserve exports and their producer report together, and stop writers before validating. + ## Path policy Paths must be non-empty strings. Output artifacts must be files; a directory fails with `artifact.not_file` even when no CSV columns are requested. Unreadable or non-UTF-8 reports fail with `report.invalid_json`. diff --git a/docs/python-api.md b/docs/python-api.md index 7c12103..be590c0 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -6,6 +6,7 @@ Evidence Gate exposes a small API for scripts, notebooks, and LLM agents that ne - `EvidenceSpec`: normalized contract containing report specifications and `schema_version`. - `ReportSpec`: expected JSON report, required fields, thresholds, and output artifacts. +- `OutputSpec`: artifact path field, optional CSV columns, and optional producer digest field. - `RunBundle`: safe wrapper around a run root directory. - `ValidationResult`: pass/fail result, recommendation, counts, checks, warnings, and failures. - `Check`: one machine-readable check with a stable code and severity. @@ -53,6 +54,8 @@ Common failure codes include: - `metric.invalid`, `metric.below_min`, `metric.above_max` - `numeric.invalid`, `numeric.below_min`, `numeric.above_max` - `artifact.missing` +- `artifact.sha256_missing`, `artifact.sha256_invalid` +- `artifact.sha256_mismatch`, `artifact.sha256_read_error` - `path.absolute`, `path.escape` - `table.column_missing` diff --git a/examples/replay-pipeline/.gitignore b/examples/replay-pipeline/.gitignore new file mode 100644 index 0000000..d8df0f5 --- /dev/null +++ b/examples/replay-pipeline/.gitignore @@ -0,0 +1,4 @@ +.venv/ +.runs/ +__pycache__/ +.pytest_cache/ diff --git a/examples/replay-pipeline/README.md b/examples/replay-pipeline/README.md new file mode 100644 index 0000000..c5dbfc8 --- /dev/null +++ b/examples/replay-pipeline/README.md @@ -0,0 +1,111 @@ +# Review an execution-assumption pipeline + +Check that a replay run has valid input ordering, internally consistent comparisons, +and all its promised output files before handing it to a reviewer. + +This example connects: + +1. **Replay Contract Kit**: check event identities and ordering before simulation. +2. **Replay Realism Kit**: compare named fee, latency, and queue assumptions. +3. **Evidence Gate**: require the resulting report bundle and complete markout evidence. + +The thirteen input events describe two invented instruments and multiple price levels. +Three profiles change the simulated fills and costs. All amounts use the same declared +synthetic quote unit; no exchange, external data service, credentials, or live orders are used. +Replay `timestamp` values use milliseconds. The auxiliary ISO-8601 `observed_at` field +expresses those same offsets from a fixed UTC epoch for the dataset contract's timestamp parser. + +## Run it + +Requires Python 3.11+ and [uv](https://docs.astral.sh/uv/). From the repository root: + +```bash +cd examples/replay-pipeline +uv run --locked python run.py --output .runs/complete --demonstrate-failure +``` + +The example's lock file pins the two companion repositories to immutable commits. +The first installation downloads those public sources and dependencies. The pipeline +itself subsequently reads local files and makes no network calls. + +Expected result: + +```text +Input contract: passed (13 events) +Scenario comparison: 3 profiles; 0 gate failures +Missing export: needs_work; artifact.missing +Complete bundle: approved +Review packet: /review.md +``` + +The missing-export step removes a generated CSV, verifies failure, restores that CSV, +and verifies the same contract again. Each invocation requires a new output directory. +It refuses to overwrite existing results. + +## Inspect the evidence + +| Output under the run directory | Purpose | +| --- | --- | +| `input/` | Exact local event and scenario inputs used for this run | +| `reports/input-contract.json` | Input validation results and stable failure codes | +| `artifacts/comparison.json` | Profiles, quantities, costs, fill traces, and gate results | +| `artifacts/comparison.csv` | Scenario summary for spreadsheet inspection | +| `artifacts/comparison.md` | Human-readable comparison and model limits | +| `reports/pipeline.json` | Small consumer envelope connecting metrics to output paths | +| `reports/incomplete-status.json` | The deliberate missing-export failure | +| `reports/evidence-status.json` | Final evidence decision | +| `review.md` | Review packet listing each configured check | + +The envelope's values are read from the comparison's emitted reports. They are not +hardcoded success labels. It records SHA-256 digests of all four completed exports. +The contract in `evidence.yaml` requires at least three scenarios, zero replay-gate +failures, complete markout coverage among filled decisions, all four declared files, +and matching export bytes. A truncated CSV with its header intact or an emptied +JSON/Markdown/input-contract report fails with `artifact.sha256_mismatch`. + +## Deliberately fail each stage + +```bash +# Exit 1: duplicate event identity and sequence; replay never runs. +uv run --locked python run.py --output .runs/duplicate --case duplicate-event + +# Exit 1: a complete metrics report cannot compensate for an unfinished CSV export. +uv run --locked python run.py --output .runs/missing-export --case missing-export + +# Exit 1: input ordering passes, but filled ALPHA decisions lack future markout evidence. +uv run --locked python run.py --output .runs/missing-markout --case missing-markout +``` + +Output bundles from failed runs remain available for inspection. Exit `2` means that +the command could not process its inputs or create its output directory. + +## Adapt the workflow + +- Map your event fields in `manifest.json`; keep input validation before simulation. +- Describe explicit assumptions in `scenarios.yaml`. The `reference` profile is a + comparison baseline, not a selected profitable strategy. +- Change evidence requirements in `evidence.yaml` to match the outputs you promise. +- Preserve both failed and completed run directories so a reviewer can reproduce them. + +This consumer is an example, not a new orchestration framework. Its Python subprocess +executes the installed replay CLI with explicit arguments and inspects the actual files. + +## Verify the example + +```bash +uv sync --locked --group dev +uv run --locked pytest -q test_pipeline.py +``` + +Tests exercise the complete run, missing export, duplicate inputs, missing future evidence, +truncated/tampered exports, and refusal to overwrite earlier results. The repository's CI runs this example separately +from the standalone evidence checker, preserving its small dependency surface. + +## What approval establishes + +Approval establishes that this fixture's configured input and output checks passed. +Matching hashes establish byte integrity against the producer's envelope; they trust +that envelope and do not authenticate sources or validate artifact content/schema. +It does not authenticate source prices, establish statistical adequacy, validate venue +matching behavior, or demonstrate trading profitability. Execution decisions remain +independent, and the synthetic scenarios do not describe observed market conditions. diff --git a/examples/replay-pipeline/events.jsonl b/examples/replay-pipeline/events.jsonl new file mode 100644 index 0000000..71f56cb --- /dev/null +++ b/examples/replay-pipeline/events.jsonl @@ -0,0 +1,13 @@ +{"event_type":"book","timestamp":0,"sequence":1,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","bids":[["99.90","2"],["99.80","4"]],"asks":[["100.10","1"],["100.20","3"]],"observed_at":"2026-01-01T00:00:00.000Z"} +{"event_type":"book","timestamp":0,"sequence":2,"instrument_id":"BETA","venue_id":"fixture","source":"synthetic","bids":[["49.90","1"],["49.80","4"]],"asks":[["50.10","2"],["50.20","4"]],"observed_at":"2026-01-01T00:00:00.000Z"} +{"event_type":"decision","timestamp":10,"sequence":3,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","side":"buy","size":"2","limit_price":"100.20","order_type":"taker","observed_at":"2026-01-01T00:00:00.010Z"} +{"event_type":"decision","timestamp":10,"sequence":4,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","side":"buy","size":"1","limit_price":"99.90","order_type":"maker","observed_at":"2026-01-01T00:00:00.010Z"} +{"event_type":"decision","timestamp":12,"sequence":5,"instrument_id":"BETA","venue_id":"fixture","source":"synthetic","side":"sell","size":"2","limit_price":"49.80","order_type":"taker","observed_at":"2026-01-01T00:00:00.012Z"} +{"event_type":"trade","timestamp":30,"sequence":6,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","side":"sell","price":"99.90","size":"1","observed_at":"2026-01-01T00:00:00.030Z"} +{"event_type":"trade","timestamp":60,"sequence":7,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","side":"sell","price":"99.90","size":"2","observed_at":"2026-01-01T00:00:00.060Z"} +{"event_type":"book","timestamp":170,"sequence":8,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","bids":[["100.50","2"],["100.40","4"]],"asks":[["100.70","2"],["100.80","4"]],"observed_at":"2026-01-01T00:00:00.170Z"} +{"event_type":"book","timestamp":180,"sequence":9,"instrument_id":"BETA","venue_id":"fixture","source":"synthetic","bids":[["49.10","2"],["49.00","4"]],"asks":[["49.30","2"],["49.40","4"]],"observed_at":"2026-01-01T00:00:00.180Z"} +{"event_type":"book","timestamp":250,"sequence":10,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","bids":[["100.40","2"],["100.30","4"]],"asks":[["100.60","2"],["100.70","4"]],"observed_at":"2026-01-01T00:00:00.250Z"} +{"event_type":"book","timestamp":250,"sequence":11,"instrument_id":"BETA","venue_id":"fixture","source":"synthetic","bids":[["49.00","2"],["48.90","4"]],"asks":[["49.20","2"],["49.30","4"]],"observed_at":"2026-01-01T00:00:00.250Z"} +{"event_type":"book","timestamp":500,"sequence":12,"instrument_id":"ALPHA","venue_id":"fixture","source":"synthetic","bids":[["100.40","2"],["100.30","4"]],"asks":[["100.60","2"],["100.70","4"]],"observed_at":"2026-01-01T00:00:00.500Z"} +{"event_type":"book","timestamp":510,"sequence":13,"instrument_id":"BETA","venue_id":"fixture","source":"synthetic","bids":[["49.00","2"],["48.90","4"]],"asks":[["49.20","2"],["49.30","4"]],"observed_at":"2026-01-01T00:00:00.510Z"} diff --git a/examples/replay-pipeline/evidence.yaml b/examples/replay-pipeline/evidence.yaml new file mode 100644 index 0000000..a355c9a --- /dev/null +++ b/examples/replay-pipeline/evidence.yaml @@ -0,0 +1,27 @@ +schema_version: 1 +reports: + - name: replay-review + path: reports/pipeline.json + expected_status: passed + required_fields: [comparison_schema, counts, metrics, outputs] + counts: + input_events: {min: 1} + scenarios: {min: 3} + gate_failures: {max: 0} + missing_markouts: {min: 0, max: 0} + metrics: + minimum_markout_coverage: {min: 1, max: 1} + outputs: + input_contract: + path_field: outputs.input_contract + sha256_field: sha256.input_contract + comparison_json: + path_field: outputs.comparison_json + sha256_field: sha256.comparison_json + comparison_csv: + path_field: outputs.comparison_csv + sha256_field: sha256.comparison_csv + csv_columns: [name, decision_count, filled_count, markout_count, markout_coverage, fee_total, edge_after_fees, gate_failure_count] + comparison_review: + path_field: outputs.comparison_review + sha256_field: sha256.comparison_review diff --git a/examples/replay-pipeline/manifest.json b/examples/replay-pipeline/manifest.json new file mode 100644 index 0000000..45b6a38 --- /dev/null +++ b/examples/replay-pipeline/manifest.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dataset_id": "synthetic-execution-review", + "event_file": "events.jsonl", + "event_format": "jsonl", + "event_time_field": "observed_at", + "sequence_field": "sequence", + "entity_keys": ["instrument_id", "venue_id"] +} diff --git a/examples/replay-pipeline/pyproject.toml b/examples/replay-pipeline/pyproject.toml new file mode 100644 index 0000000..1fc9a7c --- /dev/null +++ b/examples/replay-pipeline/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "replay-evidence-example" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = ["evidence-gate", "replay-contract-kit", "replay-realism-kit"] + +[dependency-groups] +dev = ["pytest>=8"] + +[tool.uv.sources] +evidence-gate = {path = "../.."} +replay-contract-kit = {git = "https://github.com/matchalatteguy/replay-contract-kit.git", rev = "6b9d31a8140ba5992812d749fb47be1af2c26fef"} +replay-realism-kit = {git = "https://github.com/matchalatteguy/replay-realism-kit.git", rev = "0039cbcd02c06410881948a716ae6567f7220ac7"} diff --git a/examples/replay-pipeline/run.py b/examples/replay-pipeline/run.py new file mode 100644 index 0000000..7b2642c --- /dev/null +++ b/examples/replay-pipeline/run.py @@ -0,0 +1,159 @@ +"""Validate inputs, compare execution assumptions, then check the output bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +from decimal import Decimal +from pathlib import Path + +from replay_contract_kit import load_manifest, validate_dataset + +from evidence_gate import RunBundle, load_spec, validate_run, write_review_packet_file + +FIXTURE = Path(__file__).resolve().parent + + +def write_json(path: Path, data: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1048576), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_pipeline(output: Path, case: str = "complete", demonstrate_failure: bool = False) -> int: + output = output.resolve() + output.mkdir(parents=True, exist_ok=False) + source = output / "input" + source.mkdir() + for name in ("events.jsonl", "manifest.json", "scenarios.yaml"): + shutil.copyfile(FIXTURE / name, source / name) + events_path = source / "events.jsonl" + rows = [json.loads(line) for line in events_path.read_text(encoding="utf-8").splitlines()] + if case == "duplicate-event": + rows.insert(1, rows[0]) + elif case == "missing-markout": + rows = [ + row + for row in rows + if not ( + row["event_type"] == "book" + and row["instrument_id"] == "ALPHA" + and row["timestamp"] > 100 + ) + ] + events_path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + contract = validate_dataset(load_manifest(source / "manifest.json")) + write_json(output / "reports/input-contract.json", contract.to_dict()) + if not contract.passed: + print("Input contract failed: " + ", ".join(issue.code for issue in contract.failures)) + return 1 + print(f"Input contract: passed ({contract.rows_read} events)") + artifacts = output / "artifacts" + artifacts.mkdir() + command = [ + sys.executable, + "-m", + "replay_realism.cli", + "compare", + "--events", + str(events_path), + "--scenarios", + str(source / "scenarios.yaml"), + "--json-out", + str(artifacts / "comparison.json"), + "--csv-out", + str(artifacts / "comparison.csv"), + "--md-out", + str(artifacts / "comparison.md"), + ] + replay = subprocess.run( + command, capture_output=True, text=True, encoding="utf-8", errors="replace" + ) + if replay.returncode not in (0, 1): + raise RuntimeError("Replay could not run: " + replay.stderr.strip()) + comparison = json.loads((artifacts / "comparison.json").read_text(encoding="utf-8")) + coverage = [ + Decimal(scenario["summary"]["markout_coverage"]) + for scenario in comparison["scenarios"] + if scenario["summary"]["markout_coverage"] is not None + ] + output_paths = { + "input_contract": "reports/input-contract.json", + "comparison_json": "artifacts/comparison.json", + "comparison_csv": "artifacts/comparison.csv", + "comparison_review": "artifacts/comparison.md", + } + envelope = { + "status": "passed" if replay.returncode == 0 else "failed", + "comparison_schema": comparison["schema_version"], + "counts": { + "input_events": contract.rows_read, + "scenarios": comparison["scenario_count"], + "gate_failures": comparison["gate_failure_count"], + "missing_markouts": sum( + scenario["summary"]["filled_count"] - scenario["summary"]["markout_count"] + for scenario in comparison["scenarios"] + ), + }, + "metrics": {"minimum_markout_coverage": float(min(coverage)) if coverage else 0}, + "outputs": output_paths, + "sha256": {name: file_sha256(output / path) for name, path in output_paths.items()}, + } + write_json(output / "reports/pipeline.json", envelope) + print( + f"Scenario comparison: {comparison['scenario_count']} profiles; " + f"{comparison['gate_failure_count']} gate failures" + ) + spec = load_spec(FIXTURE / "evidence.yaml") + csv_path = artifacts / "comparison.csv" + saved_csv = csv_path.read_bytes() + if demonstrate_failure or case == "missing-export": + csv_path.unlink() + incomplete = validate_run(RunBundle(output), spec) + write_json(output / "reports/incomplete-status.json", incomplete.to_dict()) + assert not incomplete.passed and any( + c.code == "artifact.missing" for c in incomplete.failures + ) + print("Missing export: needs_work; artifact.missing") + if demonstrate_failure: + csv_path.write_bytes(saved_csv) + result = validate_run(RunBundle(output), spec) + write_json(output / "reports/evidence-status.json", result.to_dict()) + write_review_packet_file(result, output / "review.md") + print(f"Complete bundle: {result.recommendation}") + print("Review packet: " + str(output / "review.md")) + return 0 if result.passed else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="A new output directory") + parser.add_argument( + "--case", + choices=["complete", "missing-export", "duplicate-event", "missing-markout"], + default="complete", + ) + parser.add_argument("--demonstrate-failure", action="store_true") + args = parser.parse_args(argv) + if args.demonstrate_failure and args.case != "complete": + parser.error("--demonstrate-failure is for the complete case") + try: + return run_pipeline(args.output, args.case, args.demonstrate_failure) + except (OSError, ValueError, RuntimeError) as exc: + print("error: " + str(exc), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/replay-pipeline/scenarios.yaml b/examples/replay-pipeline/scenarios.yaml new file mode 100644 index 0000000..df31377 --- /dev/null +++ b/examples/replay-pipeline/scenarios.yaml @@ -0,0 +1,55 @@ +schema_version: replay-realism-scenarios/v1 +baseline: reference +quote_unit: synthetic USD +profiles: + - name: reference + assumptions: + latency_ms: 10 + stale_book_ms: 250 + tick_size: "0.01" + min_size: "0.0001" + allow_partial_fills: true + require_future_markout: true + fee_model: + name: fixture-fees + maker_bps: "1" + taker_bps: "5" + explicit_zero_fees: false + simulation: + maker_queue_ahead: "1" + markout_horizon_ms: 100 + maker_lifetime_ms: 300 + - name: congestion + assumptions: + latency_ms: 30 + stale_book_ms: 250 + tick_size: "0.01" + min_size: "0.0001" + allow_partial_fills: true + require_future_markout: true + fee_model: + name: fixture-fees + maker_bps: "2" + taker_bps: "8" + explicit_zero_fees: false + simulation: + maker_queue_ahead: "2" + markout_horizon_ms: 100 + maker_lifetime_ms: 300 + - name: delayed + assumptions: + latency_ms: 80 + stale_book_ms: 250 + tick_size: "0.01" + min_size: "0.0001" + allow_partial_fills: true + require_future_markout: true + fee_model: + name: fixture-fees + maker_bps: "2" + taker_bps: "8" + explicit_zero_fees: false + simulation: + maker_queue_ahead: "2" + markout_horizon_ms: 100 + maker_lifetime_ms: 300 diff --git a/examples/replay-pipeline/test_pipeline.py b/examples/replay-pipeline/test_pipeline.py new file mode 100644 index 0000000..13610c7 --- /dev/null +++ b/examples/replay-pipeline/test_pipeline.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json + +import pytest +from run import FIXTURE, run_pipeline + +from evidence_gate import RunBundle, load_spec, validate_run + + +def test_complete_run_checks_real_outputs_and_missing_export(tmp_path): + output = tmp_path / "run" + assert run_pipeline(output, demonstrate_failure=True) == 0 + incomplete = json.loads((output / "reports/incomplete-status.json").read_text()) + assert incomplete["passed"] is False + assert any(c["code"] == "artifact.missing" for c in incomplete["checks"]) + final = json.loads((output / "reports/evidence-status.json").read_text()) + assert final["passed"] is True + assert sum(check["code"] == "artifact.sha256_match" for check in final["checks"]) == 4 + comparison = json.loads((output / "artifacts/comparison.json").read_text()) + assert comparison["scenario_count"] == 3 + assert comparison["decision_count"] == 3 + assert comparison["gate_failure_count"] == 0 + # The trial changes actual execution, rather than just renaming the profile. + assert ( + comparison["scenarios"][0]["summary"]["filled_count"] + > comparison["scenarios"][2]["summary"]["filled_count"] + ) + assert "approved" in (output / "review.md").read_text() + + +def test_duplicate_event_stops_before_replay(tmp_path): + output = tmp_path / "run" + assert run_pipeline(output, "duplicate-event") == 1 + report = json.loads((output / "reports/input-contract.json").read_text()) + assert report["passed"] is False + assert any(issue["code"] == "duplicate_event" for issue in report["failures"]) + assert not (output / "artifacts/comparison.json").exists() + + +def test_missing_export_fails_even_when_metrics_pass(tmp_path): + output = tmp_path / "run" + assert run_pipeline(output, "missing-export") == 1 + envelope = json.loads((output / "reports/pipeline.json").read_text()) + assert envelope["status"] == "passed" + status = json.loads((output / "reports/evidence-status.json").read_text()) + assert status["passed"] is False + assert any(c["code"] == "artifact.missing" for c in status["checks"]) + + +def test_missing_future_evidence_cannot_be_promoted(tmp_path): + output = tmp_path / "run" + assert run_pipeline(output, "missing-markout") == 1 + input_report = json.loads((output / "reports/input-contract.json").read_text()) + assert input_report["passed"] is True + comparison = json.loads((output / "artifacts/comparison.json").read_text()) + assert comparison["gate_failure_count"] > 0 + status = json.loads((output / "reports/evidence-status.json").read_text()) + assert status["passed"] is False + + +def test_existing_output_directory_is_not_overwritten(tmp_path): + output = tmp_path / "run" + output.mkdir() + marker = output / "keep.txt" + marker.write_text("keep") + try: + run_pipeline(output) + except FileExistsError: + pass + else: + raise AssertionError("existing output directory was accepted") + assert marker.read_text() == "keep" + + +def test_header_only_csv_cannot_be_promoted(tmp_path): + output = tmp_path / "run" + assert run_pipeline(output) == 0 + path = output / "artifacts/comparison.csv" + path.write_text(path.read_text().splitlines()[0] + "\n") + result = validate_run(RunBundle(output), load_spec(FIXTURE / "evidence.yaml")) + assert not result.passed + assert [check.code for check in result.failures] == ["artifact.sha256_mismatch"] + + +@pytest.mark.parametrize( + "relative", + ["artifacts/comparison.json", "artifacts/comparison.md", "reports/input-contract.json"], +) +@pytest.mark.parametrize("contents", [b"", b"tampered evidence\n"]) +def test_empty_or_tampered_evidence_cannot_be_promoted(tmp_path, relative, contents): + output = tmp_path / "run" + assert run_pipeline(output) == 0 + (output / relative).write_bytes(contents) + result = validate_run(RunBundle(output), load_spec(FIXTURE / "evidence.yaml")) + assert not result.passed + assert [check.code for check in result.failures] == ["artifact.sha256_mismatch"] diff --git a/examples/replay-pipeline/uv.lock b/examples/replay-pipeline/uv.lock new file mode 100644 index 0000000..25a879e --- /dev/null +++ b/examples/replay-pipeline/uv.lock @@ -0,0 +1,174 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "evidence-gate" +version = "0.3.0" +source = { directory = "../../" } +dependencies = [ + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "ruff", specifier = ">=0.6" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "replay-contract-kit" +version = "0.2.0" +source = { git = "https://github.com/matchalatteguy/replay-contract-kit.git?rev=6b9d31a8140ba5992812d749fb47be1af2c26fef#6b9d31a8140ba5992812d749fb47be1af2c26fef" } + +[[package]] +name = "replay-evidence-example" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "evidence-gate" }, + { name = "replay-contract-kit" }, + { name = "replay-realism-kit" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "evidence-gate", directory = "../../" }, + { name = "replay-contract-kit", git = "https://github.com/matchalatteguy/replay-contract-kit.git?rev=6b9d31a8140ba5992812d749fb47be1af2c26fef" }, + { name = "replay-realism-kit", git = "https://github.com/matchalatteguy/replay-realism-kit.git?rev=0039cbcd02c06410881948a716ae6567f7220ac7" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8" }] + +[[package]] +name = "replay-realism-kit" +version = "0.3.0" +source = { git = "https://github.com/matchalatteguy/replay-realism-kit.git?rev=0039cbcd02c06410881948a716ae6567f7220ac7#0039cbcd02c06410881948a716ae6567f7220ac7" } +dependencies = [ + { name = "pyyaml" }, +] diff --git a/pyproject.toml b/pyproject.toml index caf4c7b..a70e7d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "evidence-gate" -version = "0.2.0" +version = "0.3.0" description = "Validate local run artifacts against review-ready evidence contracts." readme = "README.md" requires-python = ">=3.11" diff --git a/src/evidence_gate/models.py b/src/evidence_gate/models.py index d7f196f..cb99809 100644 --- a/src/evidence_gate/models.py +++ b/src/evidence_gate/models.py @@ -50,6 +50,7 @@ class OutputSpec: path_field: str required: bool = True csv_columns: list[str] = field(default_factory=list) + sha256_field: str | None = None @property def columns(self) -> list[str]: diff --git a/src/evidence_gate/specs.py b/src/evidence_gate/specs.py index 39cf6bb..00c8f9a 100644 --- a/src/evidence_gate/specs.py +++ b/src/evidence_gate/specs.py @@ -128,7 +128,9 @@ def _parse_report(payload: Any, location: str) -> ReportSpec: def _parse_output(payload: Any, location: str) -> OutputSpec: if not isinstance(payload, dict): raise SpecValidationError(f"{location} must be a mapping") - _reject_unknown(payload, {"path_field", "required", "columns", "csv_columns"}, location) + _reject_unknown( + payload, {"path_field", "required", "columns", "csv_columns", "sha256_field"}, location + ) if "columns" in payload and "csv_columns" in payload: raise SpecValidationError(f"{location} must use only one of columns or csv_columns") column_key = "csv_columns" if "csv_columns" in payload else "columns" @@ -136,9 +138,19 @@ def _parse_output(payload: Any, location: str) -> OutputSpec: path_field=_required_str(payload, "path_field", location), required=_optional_bool(payload, "required", True, f"{location}.required"), csv_columns=_string_list(payload.get(column_key, []), f"{location}.{column_key}"), + sha256_field=_optional_field_path(payload, "sha256_field", location), ) +def _optional_field_path(payload: dict[str, Any], key: str, location: str) -> str | None: + if key not in payload: + return None + value = _required_str(payload, key, location) + if any(not part or part.strip() != part for part in value.split(".")): + raise SpecValidationError(f"{location}.{key} must name non-empty dotted field components") + return value + + def _reject_unknown(payload: dict, allowed: set[str], location: str) -> None: unknown = set(payload) - allowed if unknown: diff --git a/src/evidence_gate/validators.py b/src/evidence_gate/validators.py index 93cbdd9..d212f95 100644 --- a/src/evidence_gate/validators.py +++ b/src/evidence_gate/validators.py @@ -1,7 +1,9 @@ from __future__ import annotations import csv +import hashlib import math +import re from pathlib import Path from typing import Any @@ -297,9 +299,68 @@ def _validate_output( path, output_spec.csv_columns, report.name, bundle.display_path(path) ) ) + if output_spec.sha256_field is not None: + checks.extend( + _validate_sha256( + path, + _get_path(payload, output_spec.sha256_field), + output_spec.sha256_field, + report.name, + bundle.display_path(path), + ) + ) return checks +def _validate_sha256( + path: Path, expected: Any, field: str, report_name: str, display_path: str +) -> list[Check]: + if expected is _MISSING: + return [ + _failure( + "artifact.sha256_missing", + f"Expected SHA-256 field {field!r} is missing", + report_name, + display_path, + ) + ] + if not isinstance(expected, str) or re.fullmatch(r"[0-9a-fA-F]{64}", expected) is None: + return [ + _failure( + "artifact.sha256_invalid", + f"Expected SHA-256 field {field!r} must contain exactly 64 hexadecimal characters", + report_name, + display_path, + ) + ] + digest = hashlib.sha256() + try: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1048576), b""): + digest.update(chunk) + except OSError as exc: + return [ + _failure( + "artifact.sha256_read_error", + f"Could not hash output: {exc}", + report_name, + display_path, + ) + ] + if digest.hexdigest() != expected.lower(): + return [ + _failure( + "artifact.sha256_mismatch", + f"Output bytes do not match SHA-256 field {field!r}", + report_name, + display_path, + ) + ] + return [ + Check("artifact.sha256_match", "Output SHA-256 matched", "pass", report_name, display_path) + ] + + def _validate_csv_columns( path: Path, expected_columns: list[str], report_name: str, display_path: str ) -> list[Check]: diff --git a/tests/test_artifact_integrity.py b/tests/test_artifact_integrity.py new file mode 100644 index 0000000..f2105c0 --- /dev/null +++ b/tests/test_artifact_integrity.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest +import yaml + +from evidence_gate import RunBundle, SpecValidationError, load_spec, validate_run + + +def _configure_digest(demo_run: tuple[Path, Path], *, optional: bool = False) -> Path: + spec_path, run = demo_run + contract = yaml.safe_load(spec_path.read_text()) + config = contract["reports"][0]["outputs"]["predictions"] + config["sha256_field"] = "sha256.predictions" + config["required"] = not optional + spec_path.write_text(yaml.safe_dump(contract)) + artifact = run / "artifacts/predictions.csv" + _set_expected(run, hashlib.sha256(artifact.read_bytes()).hexdigest()) + return artifact + + +def _set_expected(run: Path, expected: object) -> None: + report = run / "reports/metrics.json" + payload = json.loads(report.read_text()) + payload["sha256"] = {"predictions": expected} + report.write_text(json.dumps(payload)) + + +def test_digest_matches_multiple_chunks_and_accepts_uppercase(demo_run: tuple[Path, Path]) -> None: + artifact = _configure_digest(demo_run) + artifact.write_bytes(b"id,label,score\n" + b"1,cat,0.98\n" * 300000) + spec_path, run = demo_run + _set_expected(run, hashlib.sha256(artifact.read_bytes()).hexdigest().upper()) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert result.passed + assert "artifact.sha256_match" in [check.code for check in result.checks] + + +@pytest.mark.parametrize("contents", [b"", b"id,label,score\n", b"id,label,score\n2,dog,0.1\n"]) +def test_changed_export_bytes_fail(demo_run: tuple[Path, Path], contents: bytes) -> None: + artifact = _configure_digest(demo_run) + artifact.write_bytes(contents) + spec_path, run = demo_run + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert not result.passed + assert "artifact.sha256_mismatch" in [check.code for check in result.failures] + + +def test_missing_expected_digest_fails(demo_run: tuple[Path, Path]) -> None: + _configure_digest(demo_run) + spec_path, run = demo_run + report = run / "reports/metrics.json" + payload = json.loads(report.read_text()) + payload.pop("sha256") + report.write_text(json.dumps(payload)) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert not result.passed + assert "artifact.sha256_missing" in [check.code for check in result.failures] + + +@pytest.mark.parametrize( + "expected", [None, True, 4, [], "a" * 63, "a" * 65, "g" * 64, " " + "a" * 64] +) +def test_malformed_expected_digest_fails(demo_run: tuple[Path, Path], expected: object) -> None: + _configure_digest(demo_run) + spec_path, run = demo_run + _set_expected(run, expected) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert not result.passed + assert "artifact.sha256_invalid" in [check.code for check in result.failures] + + +def test_digest_read_error_fails(demo_run: tuple[Path, Path], monkeypatch) -> None: + artifact = _configure_digest(demo_run) + spec_path, run = demo_run + original_open = Path.open + + def denied_binary_read(path, mode="r", *args, **kwargs): + if path == artifact and mode == "rb": + raise PermissionError("test denied artifact read") + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", denied_binary_read) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert not result.passed + assert "artifact.sha256_read_error" in [check.code for check in result.failures] + + +def test_absent_optional_output_keeps_warning_semantics(demo_run: tuple[Path, Path]) -> None: + artifact = _configure_digest(demo_run, optional=True) + artifact.unlink() + spec_path, run = demo_run + _set_expected(run, None) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert result.passed + assert [check.code for check in result.warnings] == ["artifact.missing"] + assert not any("sha256" in check.code for check in result.checks) + + +def test_present_optional_output_still_requires_configured_digest( + demo_run: tuple[Path, Path], +) -> None: + _configure_digest(demo_run, optional=True) + spec_path, run = demo_run + _set_expected(run, None) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert not result.passed + assert "artifact.sha256_invalid" in [check.code for check in result.failures] + + +def test_unconfigured_digest_does_not_change_existing_contract(demo_run: tuple[Path, Path]) -> None: + spec_path, run = demo_run + _set_expected(run, None) + result = validate_run(RunBundle(run), load_spec(spec_path)) + assert result.passed + assert not any("sha256" in check.code for check in result.checks) + + +@pytest.mark.parametrize( + "value", + [None, True, 4, [], "", ".", ".digest", "digest.", "sha..file", "sha. file", " sha.file"], +) +def test_digest_field_spec_must_be_a_nonempty_dotted_path(tmp_path: Path, value: object) -> None: + spec_path = tmp_path / "spec.json" + spec_path.write_text( + json.dumps( + { + "reports": [ + { + "name": "metrics", + "path": "metrics.json", + "outputs": {"data": {"path_field": "outputs.data", "sha256_field": value}}, + } + ] + } + ) + ) + with pytest.raises(SpecValidationError, match="sha256_field"): + load_spec(spec_path) diff --git a/uv.lock b/uv.lock index ade29d7..be433ab 100644 --- a/uv.lock +++ b/uv.lock @@ -13,7 +13,7 @@ wheels = [ [[package]] name = "evidence-gate" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "pyyaml" },