diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09a141c..e8529ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: pull_request: branches: ["main", "master"] +permissions: + contents: read + jobs: test: name: Tests + Lint @@ -37,10 +40,12 @@ jobs: - 6379:6379 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" cache: "pip" @@ -56,6 +61,9 @@ jobs: - name: Format check (ruff format) run: ruff format --check src tests + - name: Verify committed release evidence + run: agent-runtime-grid verify-committed-evidence + - name: Run tests env: DATABASE_URL: postgresql+asyncpg://testuser:testpassword@localhost:5432/agent_runtime_grid_test diff --git a/CHANGELOG.md b/CHANGELOG.md index 073a3ce..4b6c24f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Versioning after its first tagged release. ## Unreleased +- Bind the committed v0.1.0 evidence content address and release semantics to an + executable verifier and regression tests. +- Keep release reproduction outputs outside the tracked canonical evidence + directory. + ## 0.1.0 - 2026-07-13 - Narrowed the supported surface to the local CLI/library runtime. diff --git a/README.md b/README.md index 0b87ea9..88cb0fa 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,16 @@ content-addressed exception: it records a clean source revision, machine-readabl run data, the human-readable report, and a verifier manifest for a 20-job local stub smoke run. +After local setup, verify that immutable release boundary directly: + +```bash +PATH=.venv/bin:$PATH agent-runtime-grid verify-committed-evidence +``` + +This command checks the exact manifest content address and the published source, +lifecycle, cost, and recorded artifact-integrity semantics. Reproduction writes +new evidence outside the tracked release directory. + Important evidence terms: - `idempotency replay` means the same submission key returned the existing job; diff --git a/docs/evidence/releases/v0.1.0/README.md b/docs/evidence/releases/v0.1.0/README.md index a78610b..47336e0 100644 --- a/docs/evidence/releases/v0.1.0/README.md +++ b/docs/evidence/releases/v0.1.0/README.md @@ -29,10 +29,16 @@ not be interpreted as capacity or SLO evidence. Verify the committed bytes from the repository root: ```bash +PATH=.venv/bin:$PATH agent-runtime-grid verify-committed-evidence PATH=.venv/bin:$PATH agent-runtime-grid verify-evidence \ --manifest docs/evidence/releases/v0.1.0/runtime-smoke.manifest.json ``` +The committed-evidence command additionally binds the exact manifest content +address, clean source revision, 20-job lifecycle result, zero-cost stub boundary, +and 20/20 recorded artifact-integrity result. The generic command remains useful +for newly generated bundles. + Expected manifest entries: ```text @@ -49,14 +55,22 @@ The original run used Python 3.12.3 on Linux x86_64 and the following command shape after starting the pinned Postgres and Redis Compose services: ```bash +REPRO_DIR="$(mktemp -d)" PATH=.venv/bin:$PATH agent-runtime-grid smoke \ --jobs 20 \ --workers 4 \ --mode stub \ --reset-local-database \ - --report docs/evidence/releases/v0.1.0/runtime-smoke.md + --artifact-root "$REPRO_DIR/artifacts" \ + --report "$REPRO_DIR/runtime-smoke.md" +PATH=.venv/bin:$PATH agent-runtime-grid verify-evidence \ + --manifest "$REPRO_DIR/runtime-smoke.manifest.json" ``` -Run identifiers, timestamps, job identifiers, and timings are expected to -change on reproduction. The verifier proves the committed run has not changed; -the test suite and the runtime invariants determine whether a new run is valid. +This reproduction path never overwrites the committed release directory. Remove +`$REPRO_DIR` after review; run identifiers, timestamps, job identifiers, and +timings are intentionally expected to differ. + +The committed verifier proves the published run has not changed; the generic +verifier, test suite, and runtime invariants determine whether a new run is +internally valid. diff --git a/src/agent_runtime_grid/cli/main.py b/src/agent_runtime_grid/cli/main.py index d3c5edd..10cb8ac 100644 --- a/src/agent_runtime_grid/cli/main.py +++ b/src/agent_runtime_grid/cli/main.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import shutil from dataclasses import dataclass from pathlib import Path @@ -13,7 +14,11 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from agent_runtime_grid.domain.jobs import JobSubmission -from agent_runtime_grid.evidence import EvidenceVerificationError, verify_evidence_manifest +from agent_runtime_grid.evidence import ( + EvidenceVerificationError, + verify_committed_release_evidence, + verify_evidence_manifest, +) from agent_runtime_grid.queue.redis_streams import RedisStreamsQueue from agent_runtime_grid.queue.types import QueueJobMessage from agent_runtime_grid.storage.models import job_events_table, jobs_table @@ -216,6 +221,30 @@ def verify_evidence_command( typer.echo(f"verified: {manifest}") +@app.command("verify-committed-evidence") +def verify_committed_evidence_command( + repository_root: Annotated[Path, typer.Option("--repository-root")] = Path("."), +) -> None: + try: + result = verify_committed_release_evidence(repository_root) + except EvidenceVerificationError as exc: + typer.echo(f"committed evidence verification failed: {exc}", err=True) + raise typer.Exit(code=1) from exc + typer.echo( + json.dumps( + { + "content_address": result.content_address, + "manifest_path": result.manifest_path, + "source_revision": result.source_revision, + "submitted_jobs": result.submitted_jobs, + "valid_artifacts": result.valid_artifacts, + "verified": True, + }, + sort_keys=True, + ) + ) + + from agent_runtime_grid.cli.benchmark import benchmark_app # noqa: E402 from agent_runtime_grid.cli.cost import app as cost_app # noqa: E402 from agent_runtime_grid.cli.failure_reports import app as failure_reports_app # noqa: E402 diff --git a/src/agent_runtime_grid/evidence.py b/src/agent_runtime_grid/evidence.py index 754eba2..c085de1 100644 --- a/src/agent_runtime_grid/evidence.py +++ b/src/agent_runtime_grid/evidence.py @@ -15,6 +15,11 @@ RUN_EVIDENCE_SCHEMA = "agent-runtime-grid.run-evidence.v1" MANIFEST_SCHEMA = "agent-runtime-grid.evidence-manifest.v1" +COMMITTED_RELEASE_MANIFEST = Path("docs/evidence/releases/v0.1.0/runtime-smoke.manifest.json") +COMMITTED_RELEASE_MANIFEST_SHA256 = ( + "07442b769fad42a1664cc3adc7a11d73cb2041ceb61f65e402087e096d5b12ed" +) +COMMITTED_RELEASE_SOURCE_REVISION = "ddf533b36bbca0fd90a3093984d0b3f36e8ebeab" class EvidenceVerificationError(RuntimeError): @@ -28,6 +33,15 @@ class EvidenceBundle: manifest_path: Path +@dataclass(frozen=True) +class CommittedEvidenceVerification: + manifest_path: str + content_address: str + source_revision: str + submitted_jobs: int + valid_artifacts: int + + def portable_path(value: str | Path, *, namespace: str = "artifact") -> str: path = Path(value) if not path.is_absolute(): @@ -150,6 +164,190 @@ def verify_evidence_manifest(manifest_path: Path) -> None: ) +def verify_committed_release_evidence(repository_root: Path) -> CommittedEvidenceVerification: + root = repository_root.resolve() + manifest_path = root / COMMITTED_RELEASE_MANIFEST + if manifest_path.is_symlink() or not manifest_path.is_file(): + raise EvidenceVerificationError( + f"committed release manifest is missing or unsafe: {COMMITTED_RELEASE_MANIFEST}" + ) + + manifest_sha256 = _sha256(manifest_path) + if manifest_sha256 != COMMITTED_RELEASE_MANIFEST_SHA256: + raise EvidenceVerificationError( + "committed release manifest content address mismatch: " + f"expected {COMMITTED_RELEASE_MANIFEST_SHA256}, got {manifest_sha256}" + ) + verify_evidence_manifest(manifest_path) + + data_path = manifest_path.with_name("runtime-smoke.json") + try: + payload = json.loads(data_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise EvidenceVerificationError(f"cannot read committed release data: {data_path}") from exc + _verify_committed_release_semantics(payload) + + report = payload["report"] + artifact_integrity = report["artifact_integrity"] + return CommittedEvidenceVerification( + manifest_path=COMMITTED_RELEASE_MANIFEST.as_posix(), + content_address=f"sha256:{COMMITTED_RELEASE_MANIFEST_SHA256}", + source_revision=COMMITTED_RELEASE_SOURCE_REVISION, + submitted_jobs=report["submitted_jobs"], + valid_artifacts=artifact_integrity["valid_count"], + ) + + +def _verify_committed_release_semantics(payload: Any) -> None: + if not isinstance(payload, dict): + raise EvidenceVerificationError("committed release data must be a JSON object") + + expected_top_level = { + "schema_version": RUN_EVIDENCE_SCHEMA, + "command": "smoke", + "seed": None, + "source_revision": { + "commit": COMMITTED_RELEASE_SOURCE_REVISION, + "dirty": False, + }, + "config": { + "artifact_root": "artifacts", + "failure_rate": 0.0, + "jobs": 20, + "mode": "stub", + "workers": 4, + }, + "environment": { + "implementation": "CPython", + "machine": "x86_64", + "platform": "Linux", + "python": "3.12.3", + }, + } + for field, expected in expected_top_level.items(): + _require_committed_value(payload.get(field), expected, field) + + report = payload.get("report") + if not isinstance(report, dict): + raise EvidenceVerificationError("committed release report must be a JSON object") + expected_report = { + "title": "Load Smoke Report", + "source": "runtime_state:artifacts", + "submitted_jobs": 20, + "lifecycle_counts": { + "cancelled": 0, + "completed": 20, + "dlq": 0, + "failed": 0, + "queued": 0, + "running": 0, + "timed_out": 0, + }, + "completion_rate": 1.0, + "duplicate_terminal_event_count": 0, + "finalization_conflict_attempt_count": 0, + "retry_count": 0, + "artifact_completeness": 1.0, + "failure_classification": {}, + "estimated_cost_usd": "0", + "idempotency_replay_count": 0, + "injected_failure_count": 0, + } + for field, expected in expected_report.items(): + _require_committed_value(report.get(field), expected, f"report.{field}") + + backpressure = report.get("backpressure") + if not isinstance(backpressure, dict): + raise EvidenceVerificationError("committed release backpressure must be a JSON object") + for field in ( + "consumer_lag", + "dlq_count", + "leased_jobs", + "oldest_pending_age_seconds", + "queue_depth", + "retry_rate", + "running_jobs", + "worker_utilization", + ): + _require_committed_value(backpressure.get(field), 0, f"report.backpressure.{field}") + _require_committed_value( + report.get("queue_lag_seconds"), + backpressure.get("p95_queue_wait_seconds"), + "report.queue_lag_seconds", + ) + _require_committed_value( + report.get("p95_duration_seconds"), + backpressure.get("p95_execution_seconds"), + "report.p95_duration_seconds", + ) + + artifact_integrity = report.get("artifact_integrity") + if not isinstance(artifact_integrity, dict): + raise EvidenceVerificationError("committed release artifact integrity must be an object") + _require_committed_value( + artifact_integrity.get("checked_count"), 20, "report.artifact_integrity.checked_count" + ) + _require_committed_value( + artifact_integrity.get("valid_count"), 20, "report.artifact_integrity.valid_count" + ) + artifacts = artifact_integrity.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) != 20: + raise EvidenceVerificationError( + "committed release artifact integrity must contain exactly 20 rows" + ) + + run_id = report.get("run_id") + if not isinstance(run_id, str) or not run_id: + raise EvidenceVerificationError("committed release run_id must be a non-empty string") + job_ids: set[str] = set() + for index, artifact in enumerate(artifacts): + if not isinstance(artifact, dict): + raise EvidenceVerificationError(f"committed artifact row {index} must be an object") + job_id = artifact.get("job_id") + if not isinstance(job_id, str) or not job_id or job_id in job_ids: + raise EvidenceVerificationError( + f"committed artifact row {index} has a missing or duplicate job_id" + ) + job_ids.add(job_id) + _require_committed_value(artifact.get("run_id"), run_id, f"artifact[{index}].run_id") + _require_committed_value( + artifact.get("path"), + f"artifact://{job_id}/attempt-1.json", + f"artifact[{index}].path", + ) + _require_committed_value( + artifact.get("attempt_number"), 1, f"artifact[{index}].attempt_number" + ) + _require_committed_value( + artifact.get("eval_result_path"), None, f"artifact[{index}].eval_result_path" + ) + if not _is_sha256(artifact.get("sha256")): + raise EvidenceVerificationError(f"artifact[{index}].sha256 must be lowercase SHA-256") + if not _is_sha256(artifact.get("input_digest")): + raise EvidenceVerificationError( + f"artifact[{index}].input_digest must be lowercase SHA-256" + ) + size_bytes = artifact.get("size_bytes") + if not isinstance(size_bytes, int) or size_bytes <= 0: + raise EvidenceVerificationError(f"artifact[{index}].size_bytes must be positive") + + +def _require_committed_value(actual: Any, expected: Any, field: str) -> None: + if actual != expected: + raise EvidenceVerificationError( + "committed release semantic mismatch for " + f"{field}: expected {expected!r}, got {actual!r}" + ) + + +def _is_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + def _report_payload(report: Any) -> dict[str, Any]: artifact_integrity = getattr(report, "artifact_integrity", None) artifact_rows: list[dict[str, Any]] = [] diff --git a/tests/test_ci_contract.py b/tests/test_ci_contract.py index bda49fb..25103d9 100644 --- a/tests/test_ci_contract.py +++ b/tests/test_ci_contract.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from typing import Any @@ -5,6 +6,8 @@ REPO_ROOT = Path(__file__).resolve().parents[1] CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" +CHECKOUT_ACTION = "actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd" +SETUP_PYTHON_ACTION = "actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1" def _load_ci_workflow() -> dict[str, Any]: @@ -25,14 +28,21 @@ def _step_named(steps: list[dict[str, Any]], name_fragment: str) -> dict[str, An def test_ci_has_required_gates() -> None: + workflow = _load_ci_workflow() job = _test_job() steps = job["steps"] - assert any(step.get("uses") == "actions/checkout@v5" for step in steps) + assert workflow["permissions"] == {"contents": "read"} + checkout = next(step for step in steps if step.get("uses") == CHECKOUT_ACTION) + assert checkout["with"]["persist-credentials"] is False - setup_python = next(step for step in steps if step.get("uses") == "actions/setup-python@v6") + setup_python = next(step for step in steps if step.get("uses") == SETUP_PYTHON_ACTION) assert setup_python["with"]["python-version"] == "3.12" + action_references = [step["uses"] for step in steps if "uses" in step] + assert action_references + assert all(re.fullmatch(r"[^@]+@[0-9a-f]{40}", action) for action in action_references) + install = _step_named(steps, "Install dependencies") assert "requirements-dev.txt" in install["run"] @@ -42,6 +52,9 @@ def test_ci_has_required_gates() -> None: format_check = _step_named(steps, "ruff format") assert "ruff format --check" in format_check["run"] + committed_evidence = _step_named(steps, "committed release evidence") + assert committed_evidence["run"] == "agent-runtime-grid verify-committed-evidence" + test = _step_named(steps, "Run tests") assert "python -m pytest" in test["run"] diff --git a/tests/test_committed_release_evidence.py b/tests/test_committed_release_evidence.py new file mode 100644 index 0000000..c82a11f --- /dev/null +++ b/tests/test_committed_release_evidence.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from agent_runtime_grid.cli.main import app +from agent_runtime_grid.evidence import ( + COMMITTED_RELEASE_MANIFEST, + COMMITTED_RELEASE_MANIFEST_SHA256, + EvidenceVerificationError, + verify_committed_release_evidence, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_committed_release_evidence_binds_content_and_semantics() -> None: + result = verify_committed_release_evidence(REPO_ROOT) + + assert result.manifest_path == COMMITTED_RELEASE_MANIFEST.as_posix() + assert result.content_address == f"sha256:{COMMITTED_RELEASE_MANIFEST_SHA256}" + assert result.source_revision == "ddf533b36bbca0fd90a3093984d0b3f36e8ebeab" + assert result.submitted_jobs == 20 + assert result.valid_artifacts == 20 + + +def test_committed_release_verifier_rejects_resealed_semantic_changes(tmp_path: Path) -> None: + release_source = REPO_ROOT / COMMITTED_RELEASE_MANIFEST.parent + release_copy = tmp_path / COMMITTED_RELEASE_MANIFEST.parent + shutil.copytree(release_source, release_copy) + + data_path = release_copy / "runtime-smoke.json" + payload = json.loads(data_path.read_text(encoding="utf-8")) + payload["report"]["submitted_jobs"] = 19 + data_path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n", encoding="utf-8") + manifest_path = release_copy / "runtime-smoke.manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for entry in manifest["files"]: + if entry["path"] == data_path.name: + entry["sha256"] = hashlib.sha256(data_path.read_bytes()).hexdigest() + manifest_path.write_text( + json.dumps(manifest, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + + with pytest.raises(EvidenceVerificationError, match="content address mismatch"): + verify_committed_release_evidence(tmp_path) + + +def test_committed_release_cli_reports_machine_readable_receipt() -> None: + result = CliRunner().invoke( + app, + ["verify-committed-evidence", "--repository-root", str(REPO_ROOT)], + ) + + assert result.exit_code == 0, result.output + receipt = json.loads(result.stdout) + assert receipt == { + "content_address": f"sha256:{COMMITTED_RELEASE_MANIFEST_SHA256}", + "manifest_path": COMMITTED_RELEASE_MANIFEST.as_posix(), + "source_revision": "ddf533b36bbca0fd90a3093984d0b3f36e8ebeab", + "submitted_jobs": 20, + "valid_artifacts": 20, + "verified": True, + }