From 9d92145ce8d3aadb23f9319359deb278f7e6fc16 Mon Sep 17 00:00:00 2001 From: yui-stingray Date: Thu, 13 Aug 2026 18:52:53 +0900 Subject: [PATCH 1/3] fix: restore v1 evidence compatibility --- CHANGELOG.md | 12 +- README.md | 18 +- docs/compatibility.md | 51 +-- docs/evidence-contracts.md | 23 +- scripts/check_wheel_contract.py | 34 +- src/agent_guard/cli/common.py | 1 + src/agent_guard/cli/evidence_pack.py | 38 +- src/agent_guard/cli/report.py | 8 +- src/agent_guard/consumer/__init__.py | 12 + src/agent_guard/consumer/_bindings.py | 40 +- src/agent_guard/consumer/_bundle.py | 10 +- src/agent_guard/consumer/_cli.py | 4 +- src/agent_guard/consumer/_report.py | 3 +- src/agent_guard/consumer/_schema.py | 62 ++- src/agent_guard/consumer/_sections.py | 20 + src/agent_guard/evidence_pack.py | 36 +- ...uard.evidence_pack_manifest.v1.schema.json | 36 +- ...uard.evidence_pack_manifest.v2.schema.json | 129 ++++++ ...agent-guard.report_evidence.v1.schema.json | 36 +- ...agent-guard.report_evidence.v2.schema.json | 381 ++++++++++++++++++ tests/cli/test_evidence_pack.py | 172 ++++++-- tests/cli/test_report_output.py | 40 +- tests/test_contract_stability.py | 15 +- tests/test_evidence_consumer.py | 298 ++++++++++++-- tests/test_package_contract.py | 2 + tests/test_schemas.py | 70 +++- 26 files changed, 1337 insertions(+), 214 deletions(-) create mode 100644 src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v2.schema.json create mode 100644 src/agent_guard/schemas/agent-guard.report_evidence.v2.schema.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c35b4e7..f2db67d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,14 @@ Why: keep static guard releases auditable while the package is still alpha. pattern-length limits. Timeout and limit failures remain deterministic, sanitized configuration errors; no raw pattern or context text is emitted. - Content-bound optional `agent-policy` audit-event references with a - canonical-JSON, profile-bound, public-safe digest. Producers now require a - reviewed repo-local event file and explicit profile, while consumers require - the separately supplied event and reject missing, malformed, or replaced - content. The event body remains outside the fixed seven-file public bundle. + canonical-JSON, profile-bound, public-safe digest. Producers require a + caller-designated repo-local JSON event and explicit profile; maintainer review + and event-schema validation remain external. Consumers require the separately + supplied event and reject missing, malformed, or replaced content. Audit-event + binding uses report and manifest v2; the released v1 + schemas remain unchanged and their path-and-role references remain readable + as explicitly unbound legacy metadata. The event body remains outside the + fixed seven-file public bundle. - Defined a bounded demand-validation window through 2026-09-20 and froze feature releases pending an explicit maintainer decision after the 2026-09-21 review. Marketplace publication remains separately prohibited without diff --git a/README.md b/README.md index fda5e7f..444c042 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,7 @@ event, add the same `--agent-policy-audit-event ` and both artifacts again after the producer has written the repo-local JSON event and a maintainer has reviewed it. The manifest records only a sanitized repository-relative path and a profile-bound canonical-content digest, not the -event body. If the standalone manifest is present, the public bundle consumer +event body. Audit events select evidence v2; event-free reports stay v1. If the standalone manifest is present, the public bundle consumer requires it to match the manifest embedded in the report. It also requires the event path and profile again to verify the binding. Keep the event outside `.agent-guard/evidence`: the seven-file public bundle allow-list rejects it. @@ -694,9 +694,9 @@ For `report`, it returns: or context-lock coverage, digest, workflow, or policy/spec drift - exit `2` on configuration/runtime error -Report output follows `agent-guard.report_evidence.v1`: the evidence payload is -limited to deterministic scanner metadata and sanitized findings. The shared -scanner JSON envelope remains `agent-guard.result.v1`. +Event-free output follows `agent-guard.report_evidence.v1`; attaching a reviewed +audit event selects `agent-guard.report_evidence.v2` with a bound v2 manifest. +Both remain sanitized, inside the shared `agent-guard.result.v1` envelope. #### Packaged JSON schemas @@ -709,13 +709,13 @@ files from the source tree: inventory evidence. - `agent-guard.context_lock_coverage.v1.schema.json`: hash-free context lock coverage evidence, including covered context files. -- `agent-guard.report_evidence.v1.schema.json`: sanitized report evidence - payload for Markdown, JSON, GitHub annotation, and SARIF rendering, including - surface inventory and evidence coverage on success/violation payloads. +- `agent-guard.report_evidence.v1.schema.json` (event-free) and + `agent-guard.report_evidence.v2.schema.json` (bound audit event): sanitized + report evidence, including surface inventory and evidence coverage. - `agent-guard.conformance.v1.schema.json`: profile evidence for `minimal`, `recommended`, and `strict` adoption levels. -- `agent-guard.evidence_pack_manifest.v1.schema.json`: sanitized evidence - artifact manifest for reviewer handoff. +- `agent-guard.evidence_pack_manifest.v1.schema.json` (legacy unbound) and + `agent-guard.evidence_pack_manifest.v2.schema.json` (bound): sanitized manifests. Installed wheels also include `agent-guard.surface_delta.v1.schema.json` for sanitized PR base/head agent surface delta evidence. diff --git a/docs/compatibility.md b/docs/compatibility.md index 24065e7..777fb07 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -40,16 +40,18 @@ Installed wheels package these JSON Schema resources under | Shared result envelope | `agent-guard.result.v1` | `agent-guard.result.v1.schema.json` | none | `schema_version`, `tool`, `scanner`, `status`, `exit_code`, `policy`, `summary`, `finding_count`, `findings`, and optional `error`. | | Context inventory | `agent-guard.context_inventory.v1` | `agent-guard.context_inventory.v1.schema.json` | none | Repository-relative context file metadata and permission-boundary status. | | Context lock coverage | `agent-guard.context_lock_coverage.v1` | `agent-guard.context_lock_coverage.v1.schema.json` | none | Hash-free coverage counts, covered context files, and digest coverage findings. | -| Sanitized report evidence | `agent-guard.report_evidence.v1` | `agent-guard.report_evidence.v1.schema.json` | none | Public-safe report JSON, including the embedded `report.schema_version` contract marker. | +| Sanitized report evidence | `agent-guard.report_evidence.v1` | `agent-guard.report_evidence.v1.schema.json` | none | Public-safe report JSON without content-bound audit-event references. Existing v1 consumers remain supported. | +| Sanitized report evidence | `agent-guard.report_evidence.v2` | `agent-guard.report_evidence.v2.schema.json` | none | Public-safe report JSON whose embedded evidence-pack manifest can bind separately supplied `agent-policy` audit events. | | Conformance evidence | `agent-guard.conformance.v1` | `agent-guard.conformance.v1.schema.json` | none | Profile, status, required gates/surfaces, counts, and conformance findings. | -| Evidence pack manifest | `agent-guard.evidence_pack_manifest.v1` | `agent-guard.evidence_pack_manifest.v1.schema.json` | none | Sanitized artifact manifest, gate summary, and optional conformance summary. | +| Evidence pack manifest | `agent-guard.evidence_pack_manifest.v1` | `agent-guard.evidence_pack_manifest.v1.schema.json` | none | Sanitized artifact manifest, including legacy path-and-role audit-event references that are not content-bound. | +| Evidence pack manifest | `agent-guard.evidence_pack_manifest.v2` | `agent-guard.evidence_pack_manifest.v2.schema.json` | none | Sanitized artifact manifest whose audit-event entries require a controlled profile and canonical-content digest. | | Surface delta evidence | `agent-guard.surface_delta.v1` | `agent-guard.surface_delta.v1.schema.json` | none | Sanitized PR base/head agent-surface delta: added/removed/modified surface entries with controlled-vocabulary `changed_fields` names and risk labels; no raw diffs, base ref names, or instruction/description bodies. | Standalone conformance and evidence-pack files use the `agent-guard.result.v1` outer envelope. The listed -`agent-guard.conformance.v1` and `agent-guard.evidence_pack_manifest.v1` -schemas apply to their nested `conformance` and `evidence_pack_manifest` -members, respectively. +`agent-guard.conformance.v1` and the selected evidence-pack manifest schema +apply to their nested `conformance` and `evidence_pack_manifest` members, +respectively. These schema file names and `$id` values are frozen by tests. A schema version will not change without a new schema identifier and a documented compatibility @@ -79,20 +81,23 @@ with exit `2`; raw patterns, paths, or file contents are not included in the error. These ceilings are implementation safety limits, not evidence-schema fields or a promise of generic content/credential scanning. -Optional `agent-policy` audit-event entries are an additive v1 manifest field: -the existing required `path` and `role` fields retain their meaning, while new -producer output adds `content_binding`. The binding uses canonical JSON, an -explicit expected event profile, and a domain-separated SHA-256 digest encoded -as lowercase base32 with a controlled `b` prefix. This controlled digest is not -a raw hexadecimal hash and is the only hash-like value admitted on this field. -Current packaged consumers fail closed when an audit-event role lacks the -binding or when the separately supplied event does not match. Older v1 -consumers may ignore the optional field, so callers that require content -binding must use a consumer version that documents this contract. The event -profile identifies the caller-selected contract but does not itself validate -the event against an `agent-policy` JSON Schema; that remains producer-owned. -The event body remains outside the fixed public bundle. Replacing both a -trusted manifest and its event is outside this binding's threat model. +Optional `agent-policy` audit-event content binding uses report and manifest v2. +The producer emits v2 only when at least one caller-designated event is attached; reports +without an event continue to use v1. Released v1 path-and-role references remain +structurally valid legacy metadata, but they are explicitly unbound. A current +consumer can read that v1 evidence, but fails closed if a caller asks it to +verify event content against the unbound reference. + +V2 requires `content_binding` on every audit-event entry. The binding uses +canonical JSON, an explicit expected event profile, and a domain-separated +SHA-256 digest encoded as lowercase base32 with a controlled `b` prefix. This +controlled digest is not a raw hexadecimal hash and is the only hash-like value +admitted on this field. Current consumers fail closed when the separately +supplied event or expected profile does not match. The profile identifies the +caller-selected contract but does not itself validate the event against an +`agent-policy` JSON Schema; that remains producer-owned. The event body remains +outside the fixed public bundle. Replacing both a trusted manifest and its event +is outside this binding's threat model. Workflow inputs use the same fail-closed approach. Workflow policies are capped at 256 KiB, individual policy strings at 4 KiB, and workflow files at 1 MiB. @@ -223,7 +228,7 @@ future `0.x` releases: requires a new schema identifier and an explicit release-note compatibility decision. -Consumers should fail closed on unknown top-level schema versions, but they -should tolerate additional optional properties allowed by the schema. The -packaged `agent_guard.consumer` module demonstrates that policy for sanitized -report evidence. +`load_report_schema()` remains an explicit v1 pin. Callers opting into supported +version dispatch pass `select_report_schema(payload)` to `validate_report`. +Unknown versions fail closed; allowed optional properties remain tolerated. The +packaged consumer CLI applies that policy to sanitized report evidence. diff --git a/docs/evidence-contracts.md b/docs/evidence-contracts.md index ea2591c..c0be359 100644 --- a/docs/evidence-contracts.md +++ b/docs/evidence-contracts.md @@ -18,16 +18,16 @@ Installed wheels package these JSON Schema resources under discovered agent context files. - `agent-guard.context_lock_coverage.v1.schema.json`: hash-free evidence that discovered agent context files are pinned by digest policy. -- `agent-guard.report_evidence.v1.schema.json`: the sanitized report payload - used by Markdown, JSON, and GitHub annotation output. Successful and - violation reports include agent surface inventory and evidence coverage. +- `agent-guard.report_evidence.v1.schema.json` (event-free) and + `agent-guard.report_evidence.v2.schema.json` (bound audit event): sanitized + report payloads with agent surface inventory and evidence coverage. - `agent-guard.conformance.v1.schema.json`: profile evidence for `minimal`, `recommended`, and `strict` adoption levels. The `recommended` profile requires the first-class `mcp_config` gate; the `strict` profile can also fail on deterministic malformed MCP config or risk metadata emitted by the v2 surface inventory. -- `agent-guard.evidence_pack_manifest.v1.schema.json`: a sanitized manifest of - report artifacts and evidence counts for pull request review. +- `agent-guard.evidence_pack_manifest.v1.schema.json` (legacy unbound) and + `agent-guard.evidence_pack_manifest.v2.schema.json` (bound): sanitized manifests. Installed wheels also include `agent-guard.surface_delta.v1.schema.json`. The schema covers sanitized PR base/head agent surface delta evidence emitted by @@ -112,8 +112,8 @@ Move to recommended evidence after the starter files are reviewed: require those repository-specific gates unless supplied. 4. Pair the static report with a runtime admission event from `agent-policy` when the repository uses an agent hook or wrapper before side effects. Pass - that event only as an artifact reference; `agent-guard` does not read or - embed the event body. + that event only as an artifact reference; `agent-guard` reads and + canonicalizes it locally, but does not embed or publish the event body. 5. Review the evidence as a maintainer aid, not as a model-generated verdict. Example commands for a new repository. Review the `init --print` plan before @@ -165,7 +165,8 @@ python examples/evidence_consumer.py \ The referenced event must already be produced, reviewed, and stored as a repo-local regular JSON file. Pass the identical path and explicit expected -profile to both producers. The manifest records a sanitized repository-relative +profile to both producers. Events select v2; event-free reports stay v1. The +manifest records a sanitized repository-relative path and a profile-bound digest. `agent-guard` reads and canonicalizes the bounded event JSON locally to compute that binding, but never embeds the event body. The consumer requires the event separately and fails closed when the @@ -229,9 +230,9 @@ The JSON report is a compact statement of what `agent-guard` checked: evidence-pack expectations part of conformance. - Optional `evidence_pack_manifest` records the sanitized artifact manifest for reviewer handoff. Artifact roles are limited to `report` and - `agent-policy-audit-event`. Audit-event entries include a controlled binding - profile and public-safe canonical-content digest; consumers verify the event - supplied outside the public bundle. + `agent-policy-audit-event`. V2 entries include a controlled binding profile + and public-safe canonical-content digest for the separately supplied event; + legacy v1 references remain readable but cannot satisfy content verification. - `context_lock` records whether discovered context files are covered by digest policy, without emitting hash values. - Optional `path`, `content`, `api`, `digest`, and `workflow` sections summarize diff --git a/scripts/check_wheel_contract.py b/scripts/check_wheel_contract.py index c4132a8..2d603e6 100644 --- a/scripts/check_wheel_contract.py +++ b/scripts/check_wheel_contract.py @@ -1917,6 +1917,38 @@ def main() -> int: json.dumps(audit_event_payload, sort_keys=True), encoding="utf-8", ) + bound_report_cli = run( + isolated_module_command( + python, + "agent_guard.cli", + "report", + "--root", + str(repo), + "--context-policy", + str(context_policy), + "--digest-policy", + str(digest_policy), + "--workflow-policy", + str(workflow_policy), + "--drift-check", + "--agent-policy-audit-event", + str(audit_event_path), + "--agent-policy-audit-event-profile", + audit_event_profile, + "--format", + "json", + "--output", + str(report_output), + ), + cwd=temp, + ) + assert bound_report_cli.stdout == "" + bound_report_payload = json.loads(report_output.read_text(encoding="utf-8")) + assert bound_report_payload["report"]["schema_version"] == "agent-guard.report_evidence.v2" + assert ( + bound_report_payload["evidence_pack_manifest"]["schema_version"] + == "agent-guard.evidence_pack_manifest.v2" + ) manifest_cli = run( isolated_module_command( python, @@ -1941,7 +1973,7 @@ def main() -> int: ) manifest_payload = json.loads(manifest_cli.stdout) assert manifest_payload["status"] == "ok" - assert manifest_payload["evidence_pack_manifest"]["schema_version"] == "agent-guard.evidence_pack_manifest.v1" + assert manifest_payload["evidence_pack_manifest"]["schema_version"] == "agent-guard.evidence_pack_manifest.v2" manifest_artifacts = manifest_payload["evidence_pack_manifest"]["artifacts"] assert manifest_artifacts[:2] == [ {"path": ".agent-guard/evidence/agent-guard-report.json", "role": "report"}, diff --git a/src/agent_guard/cli/common.py b/src/agent_guard/cli/common.py index 9d2a5a9..4def23c 100644 --- a/src/agent_guard/cli/common.py +++ b/src/agent_guard/cli/common.py @@ -20,6 +20,7 @@ RESULT_SCHEMA_VERSION = "agent-guard.result.v1" REPORT_EVIDENCE_SCHEMA_VERSION = "agent-guard.report_evidence.v1" +REPORT_EVIDENCE_SCHEMA_VERSION_V2 = "agent-guard.report_evidence.v2" TOOL_NAME = "agent-guard" RECOMMENDED_EVIDENCE_PRESET = "recommended" URL_LIKE_POLICY_ARG_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://") diff --git a/src/agent_guard/cli/evidence_pack.py b/src/agent_guard/cli/evidence_pack.py index 48419d7..677e593 100644 --- a/src/agent_guard/cli/evidence_pack.py +++ b/src/agent_guard/cli/evidence_pack.py @@ -8,7 +8,15 @@ import json from pathlib import Path -from ..evidence_pack import build_evidence_pack_manifest +from ..consumer import ( + select_report_schema, + validate_agent_policy_audit_event_files, + validate_report, +) +from ..evidence_pack import ( + build_agent_policy_audit_event_artifacts, + build_evidence_pack_manifest, +) from .common import load_json_file, result_payload @@ -38,10 +46,36 @@ def run_evidence_pack_manifest(args: argparse.Namespace) -> int: report_path = Path(args.report).resolve() try: payload = load_json_file(report_path) + audit_event_paths = list(args.agent_policy_audit_event or []) + audit_event_artifacts = None + if audit_event_paths: + audit_event_artifacts = build_agent_policy_audit_event_artifacts( + audit_event_paths, + event_profile=str(args.agent_policy_audit_event_profile), + root=root, + ) + report_metadata = payload.get("report") + is_v2_report = ( + isinstance(report_metadata, dict) + and report_metadata.get("schema_version") + == "agent-guard.report_evidence.v2" + ) + if audit_event_paths or is_v2_report: + validate_report(payload, select_report_schema(payload)) + if is_v2_report: + verification_paths = tuple( + path if path.is_absolute() else root / path + for path in (Path(raw_path) for raw_path in audit_event_paths) + ) + validate_agent_policy_audit_event_files( + payload, + verification_paths, + event_profile=str(args.agent_policy_audit_event_profile), + ) manifest = build_evidence_pack_manifest( report_payload=payload, artifact_paths=list(args.artifact or []), - agent_policy_audit_event_paths=list(args.agent_policy_audit_event or []), + agent_policy_audit_event_artifacts=audit_event_artifacts, agent_policy_audit_event_profile=str(args.agent_policy_audit_event_profile), root=root, ) diff --git a/src/agent_guard/cli/report.py b/src/agent_guard/cli/report.py index f5457f9..60b1588 100644 --- a/src/agent_guard/cli/report.py +++ b/src/agent_guard/cli/report.py @@ -26,6 +26,7 @@ from .common import ( RECOMMENDED_EVIDENCE_PRESET, REPORT_EVIDENCE_SCHEMA_VERSION, + REPORT_EVIDENCE_SCHEMA_VERSION_V2, result_payload, resolve_policy_arg, safe_policy_path, @@ -207,6 +208,11 @@ def run_report(args: argparse.Namespace) -> int: workflow_policy_arg = str(args.workflow_policy).strip() audit_event_paths = list(args.agent_policy_audit_event or []) audit_event_profile = str(args.agent_policy_audit_event_profile).strip() + report_schema_version = ( + REPORT_EVIDENCE_SCHEMA_VERSION_V2 + if audit_event_paths + else REPORT_EVIDENCE_SCHEMA_VERSION + ) if audit_event_paths: args.evidence_pack_manifest = True safe_context_policy_path = safe_policy_path(args.context_policy, root) @@ -585,7 +591,7 @@ def run_report(args: argparse.Namespace) -> int: extra={ "command": "report", "report": { - "schema_version": REPORT_EVIDENCE_SCHEMA_VERSION, + "schema_version": report_schema_version, "format": args.format, "scope": report_scope( path_enabled=path_report is not None, diff --git a/src/agent_guard/consumer/__init__.py b/src/agent_guard/consumer/__init__.py index 2e83e26..0c0b185 100644 --- a/src/agent_guard/consumer/__init__.py +++ b/src/agent_guard/consumer/__init__.py @@ -22,7 +22,11 @@ ) from ._report import validate_report from ._schema import ( + EVIDENCE_PACK_SCHEMA, + EVIDENCE_PACK_SCHEMA_V2, REPORT_SCHEMA, + REPORT_SCHEMA_V2, + load_evidence_pack_schema, load_payload, load_report_schema, require, @@ -30,6 +34,8 @@ require_mapping, require_sequence, schema_condition_matches, + select_evidence_pack_schema, + select_report_schema, validate_against_schema, value_has_json_type, ) @@ -46,15 +52,19 @@ __all__ = [ "ALLOWED_EVIDENCE_ARTIFACT_NAMES", + "EVIDENCE_PACK_SCHEMA", + "EVIDENCE_PACK_SCHEMA_V2", "FORBIDDEN_PUBLIC_KEYS", "LOCAL_PATH_RE", "MCP_POLICY_CONFORMANCE_RULES", "RAW_URL_RE", "REPORT_SCHEMA", + "REPORT_SCHEMA_V2", "REQUIRED_MCP_RISK_LABELS", "REVIEWED_MCP_POLICY_PATH", "SECRET_VALUE_RE", "SHA256_VALUE_RE", + "load_evidence_pack_schema", "load_payload", "load_report_schema", "main", @@ -64,6 +74,8 @@ "require_mapping", "require_sequence", "schema_condition_matches", + "select_evidence_pack_schema", + "select_report_schema", "validate_against_schema", "validate_agent_policy_audit_event_files", "validate_conformance", diff --git a/src/agent_guard/consumer/_bindings.py b/src/agent_guard/consumer/_bindings.py index 601e255..3c76078 100644 --- a/src/agent_guard/consumer/_bindings.py +++ b/src/agent_guard/consumer/_bindings.py @@ -17,19 +17,31 @@ ERROR_AUDIT_EVENT_BINDING_INVALID = "agent-policy audit event binding is invalid" -def _bound_audit_event_artifacts(report: Mapping[str, Any]) -> list[Mapping[str, Any]]: +def _audit_event_artifacts( + report: Mapping[str, Any], +) -> tuple[str, list[Mapping[str, Any]]]: manifest = report.get("evidence_pack_manifest") if manifest is None: - return [] + return "", [] manifest_obj = require_mapping(manifest, ERROR_AUDIT_EVENT_BINDING_INVALID) + manifest_version = manifest_obj.get("schema_version") + require( + manifest_version + in { + "agent-guard.evidence_pack_manifest.v1", + "agent-guard.evidence_pack_manifest.v2", + }, + ERROR_AUDIT_EVENT_BINDING_INVALID, + ) artifacts = require_sequence( manifest_obj.get("artifacts"), ERROR_AUDIT_EVENT_BINDING_INVALID, ) - return [ + return str(manifest_version), [ require_mapping(item, ERROR_AUDIT_EVENT_BINDING_INVALID) for item in artifacts - if isinstance(item, Mapping) and item.get("role") == "agent-policy-audit-event" + if isinstance(item, Mapping) + and item.get("role") == "agent-policy-audit-event" ] @@ -41,10 +53,26 @@ def validate_agent_policy_audit_event_files( ) -> None: """Pair each path positionally with the same-index bound audit artifact.""" - artifacts = _bound_audit_event_artifacts(report) - if not artifacts: + manifest_version, artifacts = _audit_event_artifacts(report) + if manifest_version == "agent-guard.evidence_pack_manifest.v1": require(not paths and not event_profile, ERROR_AUDIT_EVENT_BINDING_INVALID) return + if not manifest_version: + report_metadata = report.get("report") + is_v2_report = ( + isinstance(report_metadata, Mapping) + and report_metadata.get("schema_version") + == "agent-guard.report_evidence.v2" + ) + require( + not is_v2_report and not paths and not event_profile, + ERROR_AUDIT_EVENT_BINDING_INVALID, + ) + return + require( + manifest_version == "agent-guard.evidence_pack_manifest.v2" and bool(artifacts), + ERROR_AUDIT_EVENT_BINDING_INVALID, + ) try: profile = validate_agent_policy_audit_event_profile(event_profile) except ValueError: diff --git a/src/agent_guard/consumer/_bundle.py b/src/agent_guard/consumer/_bundle.py index b87aa8c..fcd728f 100644 --- a/src/agent_guard/consumer/_bundle.py +++ b/src/agent_guard/consumer/_bundle.py @@ -21,11 +21,12 @@ from ._schema import ( DuplicateJSONKeyError, load_json_text, - load_report_schema, require, require_int, require_mapping, require_sequence, + select_evidence_pack_schema, + select_report_schema, validate_against_schema, ) from ._sections import ( @@ -48,7 +49,6 @@ ) RESULT_SCHEMA = "agent-guard.result.v1.schema.json" CONFORMANCE_SCHEMA = "agent-guard.conformance.v1.schema.json" -EVIDENCE_PACK_SCHEMA = "agent-guard.evidence_pack_manifest.v1.schema.json" ERROR_PUBLIC_BUNDLE_INVALID = "public evidence bundle is invalid" ERROR_PUBLIC_BUNDLE_LIMIT = "public evidence bundle exceeds configured limits" @@ -327,7 +327,7 @@ def _validate_evidence_bundle( require(report_path.is_file() and not report_path.is_symlink(), "report artifact is invalid") report = _load_limited_payload(report_path, limit=MAX_REPORT_JSON_BYTES) - summary = validate_report(report, load_report_schema()) + summary = validate_report(report, select_report_schema(report)) validate_agent_policy_audit_event_files( report, agent_policy_audit_event_paths, @@ -337,7 +337,7 @@ def _validate_evidence_bundle( bundle_report_path = evidence_dir / "agent-guard-report.json" if bundle_report_path.is_file(): bundle_report = _load_limited_payload(bundle_report_path, limit=MAX_REPORT_JSON_BYTES) - validate_report(bundle_report, load_report_schema()) + validate_report(bundle_report, select_report_schema(bundle_report)) require(bundle_report == report, "bundle report does not match selected report") markdown_path = evidence_dir / "agent-guard-report.md" @@ -435,7 +435,7 @@ def _validate_evidence_bundle( "$.artifact.evidence_pack_manifest", ) validate_against_schema( - _load_packaged_schema(EVIDENCE_PACK_SCHEMA), + select_evidence_pack_schema(manifest), manifest, path="$.artifact.evidence_pack_manifest", ) diff --git a/src/agent_guard/consumer/_cli.py b/src/agent_guard/consumer/_cli.py index 1254b7f..a558e7f 100644 --- a/src/agent_guard/consumer/_cli.py +++ b/src/agent_guard/consumer/_cli.py @@ -13,7 +13,7 @@ from ._bindings import validate_agent_policy_audit_event_files from ._bundle import _validate_evidence_bundle from ._report import validate_report -from ._schema import load_payload, load_report_schema +from ._schema import load_payload, select_report_schema BUNDLE_VALIDATION_ERROR = "agent-guard evidence bundle invalid" @@ -79,7 +79,7 @@ def main(argv: list[str] | None = None) -> int: try: report = load_payload(args.report) - summary = validate_report(report, load_report_schema()) + summary = validate_report(report, select_report_schema(report)) validate_agent_policy_audit_event_files( report, event_paths, diff --git a/src/agent_guard/consumer/_report.py b/src/agent_guard/consumer/_report.py index d130022..7803fa7 100644 --- a/src/agent_guard/consumer/_report.py +++ b/src/agent_guard/consumer/_report.py @@ -77,8 +77,9 @@ def validate_report(payload: dict[str, Any], schema: dict[str, Any]) -> dict[str manifest = payload.get("evidence_pack_manifest") if manifest is not None: require(isinstance(manifest, dict), "evidence_pack_manifest must be an object") + manifest_properties = properties["evidence_pack_manifest"]["properties"] require( - manifest.get("schema_version") == "agent-guard.evidence_pack_manifest.v1", + manifest.get("schema_version") == manifest_properties["schema_version"]["const"], "evidence_pack_manifest.schema_version mismatch", ) require(manifest.get("sanitized") is True, "evidence_pack_manifest.sanitized must be true") diff --git a/src/agent_guard/consumer/_schema.py b/src/agent_guard/consumer/_schema.py index 2811742..b9fb4a0 100644 --- a/src/agent_guard/consumer/_schema.py +++ b/src/agent_guard/consumer/_schema.py @@ -12,9 +12,26 @@ from typing import Any +REPORT_SCHEMA_VERSION = "agent-guard.report_evidence.v1" +REPORT_SCHEMA_VERSION_V2 = "agent-guard.report_evidence.v2" REPORT_SCHEMA = "agent-guard.report_evidence.v1.schema.json" +REPORT_SCHEMA_V2 = "agent-guard.report_evidence.v2.schema.json" +EVIDENCE_PACK_SCHEMA_VERSION = "agent-guard.evidence_pack_manifest.v1" +EVIDENCE_PACK_SCHEMA_VERSION_V2 = "agent-guard.evidence_pack_manifest.v2" +EVIDENCE_PACK_SCHEMA = "agent-guard.evidence_pack_manifest.v1.schema.json" +EVIDENCE_PACK_SCHEMA_V2 = "agent-guard.evidence_pack_manifest.v2.schema.json" +_REPORT_SCHEMAS = { + REPORT_SCHEMA_VERSION: REPORT_SCHEMA, + REPORT_SCHEMA_VERSION_V2: REPORT_SCHEMA_V2, +} +_EVIDENCE_PACK_SCHEMAS = { + EVIDENCE_PACK_SCHEMA_VERSION: EVIDENCE_PACK_SCHEMA, + EVIDENCE_PACK_SCHEMA_VERSION_V2: EVIDENCE_PACK_SCHEMA_V2, +} ERROR_DUPLICATE_JSON_KEYS = "public evidence JSON contains duplicate object keys" ERROR_PUBLIC_EVIDENCE_READ = "public evidence could not be read" +ERROR_REPORT_SCHEMA_UNSUPPORTED = "report evidence schema version is not supported" +ERROR_EVIDENCE_PACK_SCHEMA_UNSUPPORTED = "evidence-pack schema version is not supported" class DuplicateJSONKeyError(ValueError): @@ -34,9 +51,48 @@ def load_json_text(text: str) -> Any: return json.loads(text, object_pairs_hook=_object_without_duplicate_keys) -def load_report_schema() -> dict[str, Any]: - schema_path = resources.files("agent_guard.schemas").joinpath(REPORT_SCHEMA) - return json.loads(schema_path.read_text(encoding="utf-8")) +def _load_packaged_schema(name: str) -> dict[str, Any]: + schema_path = resources.files("agent_guard.schemas").joinpath(name) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + require(isinstance(schema, dict), "packaged schema must be an object") + return schema + + +def load_report_schema( + schema_version: str = REPORT_SCHEMA_VERSION, +) -> dict[str, Any]: + """Load one of the explicitly supported report schemas; default remains v1.""" + + if not isinstance(schema_version, str) or schema_version not in _REPORT_SCHEMAS: + raise ValueError(ERROR_REPORT_SCHEMA_UNSUPPORTED) + return _load_packaged_schema(_REPORT_SCHEMAS[schema_version]) + + +def select_report_schema(payload: Mapping[str, Any]) -> dict[str, Any]: + """Select a packaged report schema from the payload's bounded version marker.""" + + report = payload.get("report") + if not isinstance(report, Mapping) or "schema_version" not in report: + return load_report_schema() + version = report.get("schema_version") + if not isinstance(version, str): + raise ValueError(ERROR_REPORT_SCHEMA_UNSUPPORTED) + return load_report_schema(version) + + +def load_evidence_pack_schema(schema_version: str) -> dict[str, Any]: + """Load one of the explicitly supported evidence-pack manifest schemas.""" + + if not isinstance(schema_version, str) or schema_version not in _EVIDENCE_PACK_SCHEMAS: + raise ValueError(ERROR_EVIDENCE_PACK_SCHEMA_UNSUPPORTED) + return _load_packaged_schema(_EVIDENCE_PACK_SCHEMAS[schema_version]) + + +def select_evidence_pack_schema(manifest: Mapping[str, Any]) -> dict[str, Any]: + version = manifest.get("schema_version") + if not isinstance(version, str): + raise ValueError(ERROR_EVIDENCE_PACK_SCHEMA_UNSUPPORTED) + return load_evidence_pack_schema(version) def load_payload(path: Path) -> dict[str, Any]: diff --git a/src/agent_guard/consumer/_sections.py b/src/agent_guard/consumer/_sections.py index 6e43a3c..c352803 100644 --- a/src/agent_guard/consumer/_sections.py +++ b/src/agent_guard/consumer/_sections.py @@ -26,6 +26,12 @@ } ) MCP_POLICY_CONFORMANCE_RULES = frozenset({"required_mcp_policy_not_reviewed", "mcp_policy_weakened"}) +EVIDENCE_PACK_SCHEMA_VERSIONS = frozenset( + { + "agent-guard.evidence_pack_manifest.v1", + "agent-guard.evidence_pack_manifest.v2", + } +) def validate_gate_counts(evidence_coverage: Mapping[str, Any], *, report_status: str) -> None: @@ -162,6 +168,11 @@ def _validate_mcp_policy_conformance( def validate_evidence_pack_manifest(manifest: Mapping[str, Any], payload: Mapping[str, Any]) -> None: + manifest_version = manifest.get("schema_version") + require( + manifest_version in EVIDENCE_PACK_SCHEMA_VERSIONS, + "$.evidence_pack_manifest.schema_version mismatch", + ) manifest_report = require_mapping(manifest.get("report"), "$.evidence_pack_manifest.report") report = require_mapping(payload.get("report"), "$.report") for key in ("schema_version", "format", "scope"): @@ -191,6 +202,7 @@ def validate_evidence_pack_manifest(manifest: Mapping[str, Any], payload: Mappin def _validate_manifest_artifacts(manifest: Mapping[str, Any]) -> None: artifacts = require_sequence(manifest.get("artifacts"), "$.evidence_pack_manifest.artifacts") + audit_event_count = 0 for index, raw_artifact in enumerate(artifacts): artifact = require_mapping(raw_artifact, f"$.evidence_pack_manifest.artifacts[{index}]") role = artifact.get("role") @@ -204,6 +216,9 @@ def _validate_manifest_artifacts(manifest: Mapping[str, Any]) -> None: ) if role != "agent-policy-audit-event": continue + audit_event_count += 1 + if manifest.get("schema_version") == "agent-guard.evidence_pack_manifest.v1": + continue require( set(artifact) == {"path", "role", "content_binding"}, f"$.evidence_pack_manifest.artifacts[{index}] has invalid fields", @@ -214,6 +229,11 @@ def _validate_manifest_artifacts(manifest: Mapping[str, Any]) -> None: raise ValueError( f"$.evidence_pack_manifest.artifacts[{index}].content_binding is invalid" ) from None + if manifest.get("schema_version") == "agent-guard.evidence_pack_manifest.v2": + require( + audit_event_count > 0, + "$.evidence_pack_manifest.artifacts must include a bound audit event", + ) def _manifest_gate_map(gates: Sequence[Any]) -> tuple[dict[str, Mapping[str, Any]], int, int, int]: diff --git a/src/agent_guard/evidence_pack.py b/src/agent_guard/evidence_pack.py index b27cf9b..005fa30 100644 --- a/src/agent_guard/evidence_pack.py +++ b/src/agent_guard/evidence_pack.py @@ -17,6 +17,8 @@ from .public_redaction import contains_raw_url, sanitize_public_mapping EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION = "agent-guard.evidence_pack_manifest.v1" +EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION_V2 = "agent-guard.evidence_pack_manifest.v2" +REPORT_EVIDENCE_SCHEMA_VERSION_V2 = "agent-guard.report_evidence.v2" AGENT_POLICY_AUDIT_EVENT_BINDING_SCHEMA_VERSION = ( "agent-guard.agent_policy_audit_event_binding.v1" ) @@ -27,6 +29,9 @@ ERROR_AUDIT_EVENT_INVALID = "agent-policy audit event is not valid bounded JSON" ERROR_AUDIT_EVENT_PATH = "agent-policy audit event must be a repository file" ERROR_AUDIT_EVENT_PROFILE = "agent-policy audit event profile is invalid" +ERROR_AUDIT_EVENT_REPORT_VERSION = ( + "bound agent-policy audit events require report evidence v2" +) _AUDIT_EVENT_PROFILE_RE = re.compile(r"^[a-z][a-z0-9._-]{0,127}$") _AUDIT_EVENT_DIGEST_RE = re.compile(r"^b[a-z2-7]{52}$") @@ -477,33 +482,44 @@ def build_evidence_pack_manifest( } ) - artifacts = [] + artifacts: list[dict[str, object]] = [] for path in artifact_paths or []: safe_path = safe_artifact_path(path, root=root) if safe_path: artifacts.append({"path": safe_path, "role": "report"}) + audit_event_artifacts: list[dict[str, object]] = [] if agent_policy_audit_event_artifacts is not None: if agent_policy_audit_event_paths: raise ValueError(ERROR_AUDIT_EVENT_INVALID) - artifacts.extend( + audit_event_artifacts = [ _validate_agent_policy_audit_event_artifact_shape(artifact, root=root) for artifact in agent_policy_audit_event_artifacts - ) + ] elif agent_policy_audit_event_paths: if root is None: raise ValueError(ERROR_AUDIT_EVENT_PATH) - artifacts.extend( - build_agent_policy_audit_event_artifacts( - list(agent_policy_audit_event_paths), - event_profile=agent_policy_audit_event_profile, - root=root, - ) + audit_event_artifacts = build_agent_policy_audit_event_artifacts( + list(agent_policy_audit_event_paths), + event_profile=agent_policy_audit_event_profile, + root=root, ) elif str(agent_policy_audit_event_profile).strip(): raise ValueError(ERROR_AUDIT_EVENT_PROFILE) + artifacts.extend(audit_event_artifacts) + if audit_event_artifacts: + report_metadata = report_payload.get("report") + if ( + not isinstance(report_metadata, dict) + or report_metadata.get("schema_version") != REPORT_EVIDENCE_SCHEMA_VERSION_V2 + ): + raise ValueError(ERROR_AUDIT_EVENT_REPORT_VERSION) manifest: dict[str, object] = { - "schema_version": EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION, + "schema_version": ( + EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION_V2 + if audit_event_artifacts + else EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION + ), "tool": report_payload.get("tool", {}), "sanitized": True, "report": { diff --git a/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v1.schema.json b/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v1.schema.json index 70c91e3..3450359 100644 --- a/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v1.schema.json +++ b/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v1.schema.json @@ -77,42 +77,8 @@ "required": ["path", "role"], "properties": { "path": {"type": "string"}, - "role": {"type": "string", "enum": ["report", "agent-policy-audit-event"]}, - "content_binding": { - "type": "object", - "required": [ - "schema_version", - "event_profile", - "canonicalization", - "digest_algorithm", - "digest_encoding", - "digest" - ], - "properties": { - "schema_version": {"const": "agent-guard.agent_policy_audit_event_binding.v1"}, - "event_profile": { - "type": "string", - "pattern": "^[a-z][a-z0-9._-]{0,127}$" - }, - "canonicalization": {"const": "canonical-json-v1"}, - "digest_algorithm": {"const": "sha256"}, - "digest_encoding": {"const": "base32-lower-no-padding"}, - "digest": {"type": "string", "pattern": "^b[a-z2-7]{52}$"} - }, - "additionalProperties": false - } + "role": {"type": "string", "enum": ["report", "agent-policy-audit-event"]} }, - "allOf": [ - { - "if": {"properties": {"role": {"const": "agent-policy-audit-event"}}}, - "then": { - "required": ["content_binding"], - "propertyNames": { - "enum": ["path", "role", "content_binding"] - } - } - } - ], "additionalProperties": true } } diff --git a/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v2.schema.json b/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v2.schema.json new file mode 100644 index 0000000..04ba693 --- /dev/null +++ b/src/agent_guard/schemas/agent-guard.evidence_pack_manifest.v2.schema.json @@ -0,0 +1,129 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.evidence_pack_manifest.v2.schema.json", + "title": "agent-guard evidence pack manifest v2", + "type": "object", + "required": [ + "schema_version", + "tool", + "sanitized", + "report", + "summary", + "gates", + "artifacts" + ], + "properties": { + "schema_version": {"const": "agent-guard.evidence_pack_manifest.v2"}, + "tool": {"type": "object"}, + "sanitized": {"const": true}, + "report": { + "type": "object", + "required": ["schema_version", "format", "scope", "status", "finding_count"], + "properties": { + "schema_version": {"const": "agent-guard.report_evidence.v2"}, + "format": {"type": "string"}, + "scope": {"type": "string"}, + "status": {"type": "string"}, + "finding_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + }, + "summary": { + "type": "object", + "required": [ + "gate_count", + "enabled_gate_count", + "missing_gate_count", + "failing_gate_count", + "surface_count" + ], + "properties": { + "gate_count": {"type": "integer", "minimum": 0}, + "enabled_gate_count": {"type": "integer", "minimum": 0}, + "missing_gate_count": {"type": "integer", "minimum": 0}, + "failing_gate_count": {"type": "integer", "minimum": 0}, + "surface_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + }, + "gates": { + "type": "array", + "items": { + "type": "object", + "required": ["gate", "status", "finding_count"], + "properties": { + "gate": {"type": "string"}, + "status": {"type": "string"}, + "finding_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + } + }, + "conformance": { + "type": "object", + "required": ["schema_version", "profile", "status", "finding_count"], + "properties": { + "schema_version": {"const": "agent-guard.conformance.v1"}, + "profile": {"type": "string", "enum": ["minimal", "recommended", "strict"]}, + "status": {"type": "string", "enum": ["ok", "violation"]}, + "finding_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + }, + "artifacts": { + "type": "array", + "contains": { + "type": "object", + "required": ["role"], + "properties": { + "role": {"const": "agent-policy-audit-event"} + } + }, + "minContains": 1, + "items": { + "type": "object", + "required": ["path", "role"], + "properties": { + "path": {"type": "string"}, + "role": {"type": "string", "enum": ["report", "agent-policy-audit-event"]}, + "content_binding": { + "type": "object", + "required": [ + "schema_version", + "event_profile", + "canonicalization", + "digest_algorithm", + "digest_encoding", + "digest" + ], + "properties": { + "schema_version": {"const": "agent-guard.agent_policy_audit_event_binding.v1"}, + "event_profile": { + "type": "string", + "pattern": "^[a-z][a-z0-9._-]{0,127}$" + }, + "canonicalization": {"const": "canonical-json-v1"}, + "digest_algorithm": {"const": "sha256"}, + "digest_encoding": {"const": "base32-lower-no-padding"}, + "digest": {"type": "string", "pattern": "^b[a-z2-7]{52}$"} + }, + "additionalProperties": false + } + }, + "allOf": [ + { + "if": {"properties": {"role": {"const": "agent-policy-audit-event"}}}, + "then": { + "required": ["content_binding"], + "propertyNames": { + "enum": ["path", "role", "content_binding"] + } + } + } + ], + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/src/agent_guard/schemas/agent-guard.report_evidence.v1.schema.json b/src/agent_guard/schemas/agent-guard.report_evidence.v1.schema.json index 6d248af..30cff3a 100644 --- a/src/agent_guard/schemas/agent-guard.report_evidence.v1.schema.json +++ b/src/agent_guard/schemas/agent-guard.report_evidence.v1.schema.json @@ -310,42 +310,8 @@ "required": ["path", "role"], "properties": { "path": {"type": "string"}, - "role": {"type": "string", "enum": ["report", "agent-policy-audit-event"]}, - "content_binding": { - "type": "object", - "required": [ - "schema_version", - "event_profile", - "canonicalization", - "digest_algorithm", - "digest_encoding", - "digest" - ], - "properties": { - "schema_version": {"const": "agent-guard.agent_policy_audit_event_binding.v1"}, - "event_profile": { - "type": "string", - "pattern": "^[a-z][a-z0-9._-]{0,127}$" - }, - "canonicalization": {"const": "canonical-json-v1"}, - "digest_algorithm": {"const": "sha256"}, - "digest_encoding": {"const": "base32-lower-no-padding"}, - "digest": {"type": "string", "pattern": "^b[a-z2-7]{52}$"} - }, - "additionalProperties": false - } + "role": {"type": "string", "enum": ["report", "agent-policy-audit-event"]} }, - "allOf": [ - { - "if": {"properties": {"role": {"const": "agent-policy-audit-event"}}}, - "then": { - "required": ["content_binding"], - "propertyNames": { - "enum": ["path", "role", "content_binding"] - } - } - } - ], "additionalProperties": true } } diff --git a/src/agent_guard/schemas/agent-guard.report_evidence.v2.schema.json b/src/agent_guard/schemas/agent-guard.report_evidence.v2.schema.json new file mode 100644 index 0000000..40cb2c5 --- /dev/null +++ b/src/agent_guard/schemas/agent-guard.report_evidence.v2.schema.json @@ -0,0 +1,381 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.report_evidence.v2.schema.json", + "title": "agent-guard sanitized report evidence v2", + "type": "object", + "required": [ + "schema_version", + "tool", + "scanner", + "status", + "exit_code", + "policy", + "summary", + "finding_count", + "findings", + "command", + "report", + "evidence_pack_manifest" + ], + "allOf": [ + { + "if": { + "properties": { + "status": {"enum": ["ok", "violation"]} + } + }, + "then": { + "required": ["inventory", "surface_inventory", "evidence_coverage"] + } + } + ], + "properties": { + "schema_version": {"const": "agent-guard.result.v1"}, + "tool": { + "type": "object", + "required": ["name", "version"], + "properties": { + "name": {"const": "agent-guard"}, + "version": {"type": "string"} + }, + "additionalProperties": false + }, + "scanner": {"const": "context"}, + "status": {"type": "string", "enum": ["ok", "violation", "error"]}, + "exit_code": {"type": "integer"}, + "policy": { + "type": "object", + "required": ["path"], + "properties": {"path": {"type": "string"}}, + "additionalProperties": false + }, + "summary": { + "type": "object", + "required": ["finding_count"], + "properties": { + "finding_count": {"type": "integer", "minimum": 0}, + "scanned_count": {"type": "integer", "minimum": 0}, + "scanned_unit": {"type": "string"} + }, + "additionalProperties": true + }, + "finding_count": {"type": "integer", "minimum": 0}, + "findings": { + "type": "array", + "items": {"type": "object", "additionalProperties": true} + }, + "command": {"const": "report"}, + "report": { + "type": "object", + "required": ["schema_version", "format", "scope", "sanitized"], + "properties": { + "schema_version": {"const": "agent-guard.report_evidence.v2"}, + "format": { + "type": "string", + "enum": ["markdown", "json", "github-annotations", "sarif"] + }, + "scope": {"type": "string"}, + "sanitized": {"const": true} + }, + "additionalProperties": false + }, + "inventory": { + "type": "object", + "required": ["schema_version", "context_files", "permission_boundaries"], + "properties": { + "schema_version": {"const": "agent-guard.context_inventory.v1"}, + "context_files": {"type": "array"}, + "permission_boundaries": {"type": "array"} + }, + "additionalProperties": true + }, + "surface_inventory": { + "type": "object", + "required": ["schema_version", "summary", "surfaces"], + "properties": { + "schema_version": { + "type": "string", + "enum": ["agent-guard.agent_surface_inventory.v1", "agent-guard.agent_surface_inventory.v2"] + }, + "summary": {"type": "object"}, + "surfaces": { + "type": "array", + "items": {"type": "object", "additionalProperties": true} + } + }, + "additionalProperties": true + }, + "evidence_coverage": { + "type": "object", + "required": [ + "schema_version", + "gate_count", + "enabled_count", + "missing_count", + "failing_count", + "gates" + ], + "properties": { + "schema_version": {"const": "agent-guard.evidence_coverage.v1"}, + "gate_count": {"type": "integer", "minimum": 0}, + "enabled_count": {"type": "integer", "minimum": 0}, + "missing_count": {"type": "integer", "minimum": 0}, + "failing_count": {"type": "integer", "minimum": 0}, + "gates": { + "type": "array", + "items": { + "type": "object", + "required": ["gate", "status", "checked_count", "finding_count"], + "properties": { + "gate": {"type": "string"}, + "status": {"type": "string", "enum": ["ok", "violation", "missing", "error"]}, + "checked_count": {"type": "integer", "minimum": 0}, + "finding_count": {"type": "integer", "minimum": 0}, + "policy": {"type": "object"} + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true + }, + "surface_delta": { + "type": "object", + "required": ["schema_version", "base_resolved"], + "allOf": [ + { + "if": { + "properties": {"base_resolved": {"const": true}} + }, + "then": { + "required": ["summary", "entries"] + } + } + ], + "properties": { + "schema_version": {"const": "agent-guard.surface_delta.v1"}, + "base_resolved": {"type": "boolean"}, + "summary": { + "type": "object", + "required": ["added", "removed", "modified", "unchanged"], + "properties": { + "added": {"type": "integer", "minimum": 0}, + "removed": {"type": "integer", "minimum": 0}, + "modified": {"type": "integer", "minimum": 0}, + "unchanged": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "path", "name", "status", "changed_fields"], + "properties": { + "kind": {"type": "string"}, + "path": {"type": "string"}, + "name": {"type": "string"}, + "status": {"type": "string", "enum": ["added", "removed", "modified"]}, + "risk_labels": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "broad_authorization_scope", + "filesystem_root_reference", + "inline_authorization_value", + "inline_env_value", + "instruction_like_description", + "latest_package", + "secret_shaped_inline_value", + "unsafe_url_scheme", + "unpinned_package" + ] + }, + "uniqueItems": true + }, + "changed_fields": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "artifact_path", + "command", + "command_basename", + "content", + "env_vars", + "file_count", + "filesystem_root", + "job_id", + "kind", + "line_count", + "package_manager", + "remote_host", + "risky_patterns", + "size_bytes", + "status", + "transport", + "truncated", + "version_pinned" + ] + }, + "uniqueItems": true + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "context_lock": { + "type": "object", + "required": ["policy", "status", "checked_count", "covered_count", "covered", "finding_count", "findings"], + "properties": { + "policy": {"type": "object"}, + "status": {"type": "string", "enum": ["ok", "violation"]}, + "checked_count": {"type": "integer", "minimum": 0}, + "covered_count": {"type": "integer", "minimum": 0}, + "covered": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "kind", "status", "check_id"], + "properties": { + "path": {"type": "string"}, + "kind": {"type": "string"}, + "status": {"const": "covered"}, + "check_id": {"type": "string"} + }, + "additionalProperties": false + } + }, + "finding_count": {"type": "integer", "minimum": 0}, + "findings": {"type": "array"} + }, + "additionalProperties": true + }, + "digest": {"type": "object"}, + "mcp_config": { + "type": "object", + "required": ["status", "checked_count", "finding_count", "findings"], + "properties": { + "status": {"type": "string", "enum": ["ok", "violation"]}, + "checked_count": {"type": "integer", "minimum": 0}, + "finding_count": {"type": "integer", "minimum": 0}, + "findings": {"type": "array"}, + "surfaces": {"type": "array"} + }, + "additionalProperties": true + }, + "workflow": {"type": "object"}, + "policy_spec_drift": {"type": "object"}, + "conformance": { + "type": "object", + "required": ["schema_version", "profile", "status", "checked_count", "finding_count", "findings"], + "properties": { + "schema_version": {"const": "agent-guard.conformance.v1"}, + "profile": {"type": "string", "enum": ["minimal", "recommended", "strict"]}, + "status": {"type": "string", "enum": ["ok", "violation"]}, + "checked_count": {"type": "integer", "minimum": 0}, + "finding_count": {"type": "integer", "minimum": 0}, + "mcp_config_checked_count": {"type": "integer", "minimum": 0}, + "findings": {"type": "array"} + }, + "additionalProperties": true + }, + "evidence_pack_manifest": { + "type": "object", + "required": ["schema_version", "sanitized", "report", "summary", "gates", "artifacts"], + "properties": { + "schema_version": {"const": "agent-guard.evidence_pack_manifest.v2"}, + "sanitized": {"const": true}, + "report": { + "type": "object", + "required": ["schema_version", "format", "scope", "status", "finding_count"], + "properties": { + "schema_version": {"const": "agent-guard.report_evidence.v2"}, + "format": {"type": "string"}, + "scope": {"type": "string"}, + "status": {"type": "string"}, + "finding_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + }, + "summary": {"type": "object"}, + "gates": {"type": "array"}, + "conformance": { + "type": "object", + "required": ["schema_version", "profile", "status", "finding_count"], + "properties": { + "schema_version": {"const": "agent-guard.conformance.v1"}, + "profile": {"type": "string", "enum": ["minimal", "recommended", "strict"]}, + "status": {"type": "string", "enum": ["ok", "violation"]}, + "finding_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + }, + "artifacts": { + "type": "array", + "contains": { + "type": "object", + "required": ["role"], + "properties": { + "role": {"const": "agent-policy-audit-event"} + } + }, + "minContains": 1, + "items": { + "type": "object", + "required": ["path", "role"], + "properties": { + "path": {"type": "string"}, + "role": {"type": "string", "enum": ["report", "agent-policy-audit-event"]}, + "content_binding": { + "type": "object", + "required": [ + "schema_version", + "event_profile", + "canonicalization", + "digest_algorithm", + "digest_encoding", + "digest" + ], + "properties": { + "schema_version": {"const": "agent-guard.agent_policy_audit_event_binding.v1"}, + "event_profile": { + "type": "string", + "pattern": "^[a-z][a-z0-9._-]{0,127}$" + }, + "canonicalization": {"const": "canonical-json-v1"}, + "digest_algorithm": {"const": "sha256"}, + "digest_encoding": {"const": "base32-lower-no-padding"}, + "digest": {"type": "string", "pattern": "^b[a-z2-7]{52}$"} + }, + "additionalProperties": false + } + }, + "allOf": [ + { + "if": {"properties": {"role": {"const": "agent-policy-audit-event"}}}, + "then": { + "required": ["content_binding"], + "propertyNames": { + "enum": ["path", "role", "content_binding"] + } + } + } + ], + "additionalProperties": true + } + } + }, + "additionalProperties": true + }, + "path": {"type": "object"}, + "content": {"type": "object"}, + "api": {"type": "object"}, + "error": {"type": "string"} + }, + "additionalProperties": true +} diff --git a/tests/cli/test_evidence_pack.py b/tests/cli/test_evidence_pack.py index 9deeaef..521be0d 100644 --- a/tests/cli/test_evidence_pack.py +++ b/tests/cli/test_evidence_pack.py @@ -20,36 +20,34 @@ from tests.cli.helpers import run_cli AUDIT_EVENT_PROFILE = "agent-policy.audit_event.v1.1" +REPO_ROOT = Path(__file__).resolve().parents[2] +EVIDENCE_SAMPLE_REPORT = REPO_ROOT / "docs" / "evidence-samples" / "agent-guard-report.json" +V2_REPORT_PAYLOAD = { + "report": {"schema_version": "agent-guard.report_evidence.v2"}, +} def test_evidence_pack_manifest_cli_is_sanitized(tmp_path: Path) -> None: report = tmp_path / "report.json" - report.write_text( - json.dumps( - { - "tool": {"name": "agent-guard", "version": "0.1.7"}, - "status": "ok", - "finding_count": 0, - "summary": {"surface_count": 2}, - "report": { - "schema_version": "agent-guard.report_evidence.v1", - "format": "json", - "scope": "context", - }, - "evidence_coverage": { - "gate_count": 1, - "enabled_count": 1, - "missing_count": 0, - "failing_count": 0, - "gates": [{"gate": "context", "status": "ok", "finding_count": 0}], - }, - } - ), - encoding="utf-8", - ) event = tmp_path / ".agent-guard" / "evidence" / "policy-admission-event.json" event.parent.mkdir(parents=True) event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + report_payload = json.loads(EVIDENCE_SAMPLE_REPORT.read_text(encoding="utf-8")) + report_payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + embedded_manifest = report_payload["evidence_pack_manifest"] + embedded_manifest["schema_version"] = "agent-guard.evidence_pack_manifest.v2" + embedded_manifest["report"]["schema_version"] = "agent-guard.report_evidence.v2" + embedded_manifest["artifacts"].append( + { + "path": ".agent-guard/evidence/policy-admission-event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + ) + report.write_text(json.dumps(report_payload), encoding="utf-8") result = run_cli( "evidence-pack", @@ -69,7 +67,7 @@ def test_evidence_pack_manifest_cli_is_sanitized(tmp_path: Path) -> None: "--artifact", "file://localhost/home/synthetic/private/report.json", "--agent-policy-audit-event", - str(event), + event.relative_to(tmp_path).as_posix(), "--agent-policy-audit-event-profile", AUDIT_EVENT_PROFILE, "--json", @@ -78,7 +76,7 @@ def test_evidence_pack_manifest_cli_is_sanitized(tmp_path: Path) -> None: assert result.returncode == 0 payload = json.loads(result.stdout) manifest = payload["evidence_pack_manifest"] - assert manifest["schema_version"] == "agent-guard.evidence_pack_manifest.v1" + assert manifest["schema_version"] == "agent-guard.evidence_pack_manifest.v2" assert manifest["sanitized"] is True assert manifest["artifacts"][:-1] == [ {"path": ".agent-guard/evidence/report.json", "role": "report"}, @@ -103,6 +101,95 @@ def test_evidence_pack_manifest_cli_is_sanitized(tmp_path: Path) -> None: assert r"\\server\share" not in result.stdout +def test_evidence_pack_manifest_cli_rejects_unverified_bound_v2_report( + tmp_path: Path, +) -> None: + report = tmp_path / "report.json" + event = tmp_path / "reviewed" / "event.json" + event.parent.mkdir() + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + report_payload = json.loads(EVIDENCE_SAMPLE_REPORT.read_text(encoding="utf-8")) + report_payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest = report_payload["evidence_pack_manifest"] + manifest["schema_version"] = "agent-guard.evidence_pack_manifest.v2" + manifest["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + ) + report.write_text(json.dumps(report_payload), encoding="utf-8") + + result = run_cli( + "evidence-pack", + "manifest", + "--root", + str(tmp_path), + "--report", + str(report), + "--json", + ) + + assert result.returncode == 2 + payload = json.loads(result.stdout) + assert payload["error"] == "agent-policy audit event binding is invalid" + assert str(event) not in result.stdout + result.stderr + assert AUDIT_EVENT_PROFILE not in result.stdout + result.stderr + + +def test_evidence_pack_manifest_cli_rejects_mismatched_bound_v2_event( + tmp_path: Path, +) -> None: + report = tmp_path / "report.json" + event = tmp_path / "reviewed" / "event.json" + event.parent.mkdir() + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + report_payload = json.loads(EVIDENCE_SAMPLE_REPORT.read_text(encoding="utf-8")) + report_payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest = report_payload["evidence_pack_manifest"] + manifest["schema_version"] = "agent-guard.evidence_pack_manifest.v2" + manifest["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + ) + report.write_text(json.dumps(report_payload), encoding="utf-8") + private_marker = "synthetic-unreviewed-event-body" + event.write_text(json.dumps({"marker": private_marker}), encoding="utf-8") + + result = run_cli( + "evidence-pack", + "manifest", + "--root", + str(tmp_path), + "--report", + str(report), + "--agent-policy-audit-event", + str(event), + "--agent-policy-audit-event-profile", + AUDIT_EVENT_PROFILE, + "--json", + ) + + assert result.returncode == 2 + payload = json.loads(result.stdout) + assert payload["error"] == "agent-policy audit event binding is invalid" + assert private_marker not in result.stdout + result.stderr + assert str(event) not in result.stdout + result.stderr + assert AUDIT_EVENT_PROFILE not in result.stdout + result.stderr + + def test_evidence_pack_manifest_cli_sanitizes_copied_report_metadata(tmp_path: Path) -> None: secret_shaped = "AKIA" + ("A" * 16) raw_url = "HtTpS:/example.invalid/private's/synthetic-tail" @@ -163,6 +250,7 @@ def test_evidence_pack_manifest_cli_sanitizes_copied_report_metadata(tmp_path: P assert result.returncode == 0 payload = json.loads(result.stdout) manifest = payload["evidence_pack_manifest"] + assert manifest["schema_version"] == "agent-guard.evidence_pack_manifest.v1" assert manifest["sanitized"] is True assert manifest["tool"]["name"] == "" assert manifest["tool"]["version"] == "" @@ -312,7 +400,7 @@ def test_manifest_rejects_extra_fields_in_prebuilt_audit_event_artifact( match="^agent-policy audit event is not valid bounded JSON$", ) as exc_info: build_evidence_pack_manifest( - report_payload={}, + report_payload=V2_REPORT_PAYLOAD, agent_policy_audit_event_artifacts=[ { "path": "reviewed/event.json", @@ -342,10 +430,11 @@ def test_manifest_rejects_invalid_prebuilt_audit_event_artifact_shape( "content_binding": binding, } manifest = build_evidence_pack_manifest( - report_payload={}, + report_payload=V2_REPORT_PAYLOAD, agent_policy_audit_event_artifacts=[valid_artifact], root=tmp_path, ) + assert manifest["schema_version"] == "agent-guard.evidence_pack_manifest.v2" assert manifest["artifacts"] == [valid_artifact] invalid_artifacts = [ @@ -359,12 +448,39 @@ def test_manifest_rejects_invalid_prebuilt_audit_event_artifact_shape( match="^agent-policy audit event is not valid bounded JSON$", ): build_evidence_pack_manifest( - report_payload={}, + report_payload=V2_REPORT_PAYLOAD, agent_policy_audit_event_artifacts=[artifact], root=tmp_path, ) +def test_bound_manifest_rejects_v1_report_without_leak(tmp_path: Path) -> None: + event = tmp_path / "event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + artifact = { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + + with pytest.raises( + ValueError, + match=r"^bound agent-policy audit events require report evidence v2$", + ) as exc_info: + build_evidence_pack_manifest( + report_payload={ + "report": {"schema_version": "agent-guard.report_evidence.v1"}, + }, + agent_policy_audit_event_artifacts=[artifact], + root=tmp_path, + ) + + assert str(event) not in str(exc_info.value) + + @pytest.mark.parametrize( "event_profile", ( diff --git a/tests/cli/test_report_output.py b/tests/cli/test_report_output.py index 10d0057..fd5effd 100644 --- a/tests/cli/test_report_output.py +++ b/tests/cli/test_report_output.py @@ -227,9 +227,12 @@ def test_report_cli_json_output_writes_file_and_suppresses_stdout(tmp_path: Path payload = json.loads(output.read_text(encoding="utf-8")) assert_shared_envelope(payload, scanner="context", status="ok", exit_code=0, finding_count=0) assert payload["command"] == "report" - assert payload["report"]["schema_version"] == "agent-guard.report_evidence.v1" + assert payload["report"]["schema_version"] == "agent-guard.report_evidence.v2" assert payload["report"]["format"] == "json" assert payload["report"]["sanitized"] is True + assert payload["evidence_pack_manifest"]["schema_version"] == ( + "agent-guard.evidence_pack_manifest.v2" + ) artifacts = payload["evidence_pack_manifest"]["artifacts"] assert artifacts[0] == {"path": "evidence/agent-guard-report.json", "role": "report"} assert artifacts[1]["path"] == "evidence/policy-admission-event.json" @@ -270,6 +273,10 @@ def test_report_cli_audit_event_implies_evidence_pack_manifest(tmp_path: Path) - assert result.returncode == 0 payload = json.loads(result.stdout) + assert payload["report"]["schema_version"] == "agent-guard.report_evidence.v2" + assert payload["evidence_pack_manifest"]["schema_version"] == ( + "agent-guard.evidence_pack_manifest.v2" + ) artifact = payload["evidence_pack_manifest"]["artifacts"][0] assert artifact["path"] == "evidence/nested/policy-admission-event.json" assert artifact["role"] == "agent-policy-audit-event" @@ -300,6 +307,37 @@ def test_report_cli_rejects_audit_event_profile_without_path(tmp_path: Path) -> assert str(tmp_path) not in result.stdout +def test_report_cli_error_after_valid_audit_event_remains_v1(tmp_path: Path) -> None: + policy = tmp_path / "context_policy.yaml" + policy.write_text("patterns: [\n", encoding="utf-8") + event_marker = "synthetic-reviewed-event-marker" + event = tmp_path / "evidence" / "policy-admission-event.json" + event.parent.mkdir(parents=True) + event.write_text(json.dumps({"marker": event_marker}), encoding="utf-8") + + result = run_cli( + "report", + "--root", + str(tmp_path), + "--context-policy", + str(policy), + "--agent-policy-audit-event", + str(event), + "--agent-policy-audit-event-profile", + "agent-policy.audit_event.v1.1", + "--format", + "json", + ) + + assert result.returncode == 2 + payload = json.loads(result.stdout) + assert payload["status"] == "error" + assert payload["report"]["schema_version"] == "agent-guard.report_evidence.v1" + assert "evidence_pack_manifest" not in payload + assert event_marker not in result.stdout + assert str(tmp_path) not in result.stdout + + def test_report_cli_stderr_summary_ok_after_output_write_is_sanitized(tmp_path: Path) -> None: policy = tmp_path / "context_policy.yaml" policy.write_text("{}\n", encoding="utf-8") diff --git a/tests/test_contract_stability.py b/tests/test_contract_stability.py index 1966670..087931c 100644 --- a/tests/test_contract_stability.py +++ b/tests/test_contract_stability.py @@ -32,6 +32,10 @@ "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.evidence_pack_manifest.v1.schema.json", {"schema_version": "agent-guard.evidence_pack_manifest.v1"}, ), + "agent-guard.evidence_pack_manifest.v2.schema.json": ( + "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.evidence_pack_manifest.v2.schema.json", + {"schema_version": "agent-guard.evidence_pack_manifest.v2"}, + ), "agent-guard.report_evidence.v1.schema.json": ( "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.report_evidence.v1.schema.json", { @@ -39,6 +43,13 @@ "report.schema_version": "agent-guard.report_evidence.v1", }, ), + "agent-guard.report_evidence.v2.schema.json": ( + "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.report_evidence.v2.schema.json", + { + "schema_version": "agent-guard.result.v1", + "report.schema_version": "agent-guard.report_evidence.v2", + }, + ), "agent-guard.result.v1.schema.json": ( "https://github.com/yui-stingray/agent-guard/schemas/agent-guard.result.v1.schema.json", {"schema_version": "agent-guard.result.v1"}, @@ -54,8 +65,10 @@ "agent-guard.context_inventory.v1", "agent-guard.context_lock_coverage.v1", "agent-guard.report_evidence.v1", + "agent-guard.report_evidence.v2", "agent-guard.conformance.v1", "agent-guard.evidence_pack_manifest.v1", + "agent-guard.evidence_pack_manifest.v2", "agent-guard.surface_delta.v1", "agent-guard.agb_results.v1", "agent-guard.evidence_results.v1", @@ -157,7 +170,7 @@ def test_changelog_records_latest_release_entry() -> None: assert normalized_unreleased == " ".join( [ "- Isolated repository-controlled context-policy regular-expression matching behind the existing bounded scanner worker and added fixed pattern-count and pattern-length limits. Timeout and limit failures remain deterministic, sanitized configuration errors; no raw pattern or context text is emitted.", - "- Content-bound optional `agent-policy` audit-event references with a canonical-JSON, profile-bound, public-safe digest. Producers now require a reviewed repo-local event file and explicit profile, while consumers require the separately supplied event and reject missing, malformed, or replaced content. The event body remains outside the fixed seven-file public bundle.", + "- Content-bound optional `agent-policy` audit-event references with a canonical-JSON, profile-bound, public-safe digest. Producers require a caller-designated repo-local JSON event and explicit profile; maintainer review and event-schema validation remain external. Consumers require the separately supplied event and reject missing, malformed, or replaced content. Audit-event binding uses report and manifest v2; the released v1 schemas remain unchanged and their path-and-role references remain readable as explicitly unbound legacy metadata. The event body remains outside the fixed seven-file public bundle.", "- Defined a bounded demand-validation window through 2026-09-20 and froze feature releases pending an explicit maintainer decision after the 2026-09-21 review. Marketplace publication remains separately prohibited without explicit authorization.", "- Locked the release build toolchain, pinned copyable GitHub Action examples to the immutable v0.3.4 release commit, and documented the post-release pin refresh contract.", "- Simplified reviewed bootstrap and monorepo onboarding, added explicit Python interpreter checks, and tightened guidance for copying public-safe evidence.", diff --git a/tests/test_evidence_consumer.py b/tests/test_evidence_consumer.py index ff256c9..cacddf0 100644 --- a/tests/test_evidence_consumer.py +++ b/tests/test_evidence_consumer.py @@ -20,6 +20,7 @@ RAW_URL_RE, load_payload, load_report_schema, + select_report_schema, validate_agent_policy_audit_event_files, validate_report, ) @@ -37,6 +38,25 @@ AUDIT_EVENT_PROFILE = "agent-policy.audit_event.v1.1" +def _bound_v2_report(event: Path) -> dict[str, object]: + payload = json.loads(SAMPLE.read_text(encoding="utf-8")) + payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest = payload["evidence_pack_manifest"] + manifest["schema_version"] = "agent-guard.evidence_pack_manifest.v2" + manifest["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + ) + return payload + + def run_consumer(path: Path) -> subprocess.CompletedProcess[str]: env = os.environ.copy() env["PYTHONPATH"] = f"{SRC}{os.pathsep}{env['PYTHONPATH']}" if env.get("PYTHONPATH") else str(SRC) @@ -63,7 +83,9 @@ def run_packaged_consumer_cli(*args: str) -> subprocess.CompletedProcess[str]: ) -def test_packaged_consumer_rejects_unbound_legacy_audit_event_reference() -> None: +def test_packaged_consumer_accepts_legacy_unbound_audit_event_reference( + tmp_path: Path, +) -> None: payload = json.loads(SAMPLE.read_text(encoding="utf-8")) payload["evidence_pack_manifest"]["artifacts"].append( { @@ -72,8 +94,53 @@ def test_packaged_consumer_rejects_unbound_legacy_audit_event_reference() -> Non } ) - with pytest.raises(ValueError, match="content_binding is required"): - validate_report(payload, load_report_schema()) + summary = validate_report(payload, load_report_schema()) + validate_agent_policy_audit_event_files(payload, (), event_profile="") + + assert summary["report_schema_version"] == "agent-guard.report_evidence.v1" + + event = tmp_path / "synthetic-legacy-event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + with pytest.raises( + ValueError, + match=r"^agent-policy audit event binding is invalid$", + ) as exc_info: + validate_agent_policy_audit_event_files( + payload, + (event,), + event_profile=AUDIT_EVENT_PROFILE, + ) + + assert str(event) not in str(exc_info.value) + assert AUDIT_EVENT_PROFILE not in str(exc_info.value) + + +def test_v1_content_binding_field_never_counts_as_bound(tmp_path: Path) -> None: + event = tmp_path / "synthetic-v1-event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + payload = json.loads(SAMPLE.read_text(encoding="utf-8")) + payload["evidence_pack_manifest"]["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + ) + + validate_report(payload, load_report_schema()) + validate_agent_policy_audit_event_files(payload, (), event_profile="") + with pytest.raises( + ValueError, + match=r"^agent-policy audit event binding is invalid$", + ): + validate_agent_policy_audit_event_files( + payload, + (event,), + event_profile=AUDIT_EVENT_PROFILE, + ) def test_packaged_consumer_rejects_extra_audit_event_artifact_fields_without_leak( @@ -87,6 +154,13 @@ def test_packaged_consumer_rejects_extra_audit_event_artifact_fields_without_lea event_profile=AUDIT_EVENT_PROFILE, ) payload = json.loads(SAMPLE.read_text(encoding="utf-8")) + payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + payload["evidence_pack_manifest"]["schema_version"] = ( + "agent-guard.evidence_pack_manifest.v2" + ) + payload["evidence_pack_manifest"]["report"]["schema_version"] = ( + "agent-guard.report_evidence.v2" + ) payload["evidence_pack_manifest"]["artifacts"].append( { "path": "reviewed/event.json", @@ -97,7 +171,7 @@ def test_packaged_consumer_rejects_extra_audit_event_artifact_fields_without_lea ) with pytest.raises(ValueError, match="invalid fields") as exc_info: - validate_report(payload, load_report_schema()) + validate_report(payload, select_report_schema(payload)) assert marker not in str(exc_info.value) @@ -110,18 +184,7 @@ def test_packaged_consumer_rejects_final_audit_event_symlink_without_leak( external = tmp_path / "external.json" external_marker = "synthetic-consumer-external-marker" event.write_text('{"status":"reviewed"}\n', encoding="utf-8") - binding = build_agent_policy_audit_event_binding( - event, - event_profile=AUDIT_EVENT_PROFILE, - ) - report = json.loads(SAMPLE.read_text(encoding="utf-8")) - report["evidence_pack_manifest"]["artifacts"].append( - { - "path": "reviewed/event.json", - "role": "agent-policy-audit-event", - "content_binding": binding, - } - ) + report = _bound_v2_report(event) external.write_text(json.dumps({"marker": external_marker}), encoding="utf-8") event.unlink() event.symlink_to(external) @@ -145,18 +208,7 @@ def test_packaged_consumer_rejects_large_number_audit_event_substitution( ) -> None: event = tmp_path / "event.json" event.write_text('{"sequence":9007199254740992.0}\n', encoding="utf-8") - binding = build_agent_policy_audit_event_binding( - event, - event_profile=AUDIT_EVENT_PROFILE, - ) - report = json.loads(SAMPLE.read_text(encoding="utf-8")) - report["evidence_pack_manifest"]["artifacts"].append( - { - "path": "reviewed/event.json", - "role": "agent-policy-audit-event", - "content_binding": binding, - } - ) + report = _bound_v2_report(event) event.write_text('{"sequence":9007199254740993.0}\n', encoding="utf-8") @@ -171,6 +223,157 @@ def test_packaged_consumer_rejects_large_number_audit_event_substitution( ) +def test_current_consumer_accepts_matching_bound_v2_evidence(tmp_path: Path) -> None: + event = tmp_path / "synthetic-reviewed-event.json" + event.write_text('{"status":"reviewed","sequence":7}\n', encoding="utf-8") + payload = _bound_v2_report(event) + + summary = validate_report(payload, select_report_schema(payload)) + validate_agent_policy_audit_event_files( + payload, + (event,), + event_profile=AUDIT_EVENT_PROFILE, + ) + + assert summary["report_schema_version"] == "agent-guard.report_evidence.v2" + report_path = tmp_path / "synthetic-report.json" + report_path.write_text(json.dumps(payload), encoding="utf-8") + result = run_packaged_consumer_cli( + "--agent-policy-audit-event", + str(event), + "--agent-policy-audit-event-profile", + AUDIT_EVENT_PROFILE, + str(report_path), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert json.loads(result.stdout)["report_schema_version"] == ( + "agent-guard.report_evidence.v2" + ) + + +def test_bound_v2_verification_failures_are_sanitized(tmp_path: Path) -> None: + event_marker = "synthetic-event-body-marker" + event = tmp_path / "synthetic-sensitive-event-name.json" + event.write_text(json.dumps({"marker": event_marker}), encoding="utf-8") + payload = _bound_v2_report(event) + mismatched_profile = "agent-policy.audit_event.v1.2" + + failures: list[ValueError] = [] + for paths, profile in ( + ((), ""), + ((event,), mismatched_profile), + ): + with pytest.raises( + ValueError, + match=r"^agent-policy audit event binding is invalid$", + ) as exc_info: + validate_agent_policy_audit_event_files( + payload, + paths, + event_profile=profile, + ) + failures.append(exc_info.value) + + event.write_text('{"marker":', encoding="utf-8") + with pytest.raises( + ValueError, + match=r"^agent-policy audit event binding is invalid$", + ) as exc_info: + validate_agent_policy_audit_event_files( + payload, + (event,), + event_profile=AUDIT_EVENT_PROFILE, + ) + failures.append(exc_info.value) + + for failure in failures: + rendered = str(failure) + assert event_marker not in rendered + assert str(event) not in rendered + assert mismatched_profile not in rendered + + +def test_current_consumer_rejects_mixed_unbound_v2_entries_without_leak( + tmp_path: Path, +) -> None: + event = tmp_path / "synthetic-reviewed-event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + payload = _bound_v2_report(event) + marker = "synthetic-private-unbound-path.json" + payload["evidence_pack_manifest"]["artifacts"].append( + { + "path": marker, + "role": "agent-policy-audit-event", + } + ) + + with pytest.raises(ValueError, match="content_binding is required") as exc_info: + validate_report(payload, select_report_schema(payload)) + + assert marker not in str(exc_info.value) + assert str(event) not in str(exc_info.value) + + +def test_current_consumer_rejects_v2_without_bound_audit_event() -> None: + payload = json.loads(SAMPLE.read_text(encoding="utf-8")) + payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + payload["evidence_pack_manifest"]["schema_version"] = ( + "agent-guard.evidence_pack_manifest.v2" + ) + payload["evidence_pack_manifest"]["report"]["schema_version"] = ( + "agent-guard.report_evidence.v2" + ) + + with pytest.raises( + ValueError, + match=r"^\$\.evidence_pack_manifest\.artifacts must include a bound audit event$", + ): + validate_report(payload, select_report_schema(payload)) + with pytest.raises( + ValueError, + match=r"^agent-policy audit event binding is invalid$", + ): + validate_agent_policy_audit_event_files(payload, (), event_profile="") + + payload.pop("evidence_pack_manifest") + with pytest.raises( + ValueError, + match=r"^\$\.evidence_pack_manifest is required$", + ): + validate_report(payload, select_report_schema(payload)) + with pytest.raises( + ValueError, + match=r"^agent-policy audit event binding is invalid$", + ): + validate_agent_policy_audit_event_files(payload, (), event_profile="") + + +def test_report_schema_loader_is_bounded_and_v1_default_is_preserved() -> None: + v1_schema = load_report_schema() + v2_schema = load_report_schema("agent-guard.report_evidence.v2") + + assert v1_schema["$id"].endswith("agent-guard.report_evidence.v1.schema.json") + assert v2_schema["$id"].endswith("agent-guard.report_evidence.v2.schema.json") + with pytest.raises( + ValueError, + match=r"^report evidence schema version is not supported$", + ): + load_report_schema("synthetic-untrusted.schema.json") + + +def test_v1_schema_consumer_fails_closed_on_v2_report(tmp_path: Path) -> None: + event = tmp_path / "synthetic-reviewed-event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + payload = _bound_v2_report(event) + + with pytest.raises( + ValueError, + match=r"must equal 'agent-guard\.(?:evidence_pack_manifest|report_evidence)\.v1'", + ): + validate_report(payload, load_report_schema()) + + def _prepend_duplicate_json_member(text: str, *, key: str, value: object) -> str: assert text.startswith("{") member = f"{json.dumps(key, ensure_ascii=False)}:{json.dumps(value, ensure_ascii=False)}" @@ -463,6 +666,41 @@ def test_packaged_consumer_cli_accepts_fixture_bundle(tmp_path: Path) -> None: assert json.loads(result.stdout)["status"] == "ok" +def test_packaged_consumer_accepts_bound_v2_standalone_manifest_bundle( + tmp_path: Path, +) -> None: + evidence_dir = tmp_path / "evidence" + evidence_dir.mkdir() + event = tmp_path / "synthetic-reviewed-event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + payload = _bound_v2_report(event) + report = evidence_dir / "agent-guard-report.json" + report.write_text(json.dumps(payload), encoding="utf-8") + envelope = _canonical_standalone_envelopes(payload)[ + "agent-guard-evidence-pack.json" + ] + (evidence_dir / "agent-guard-evidence-pack.json").write_text( + json.dumps(envelope), + encoding="utf-8", + ) + + result = run_packaged_consumer_cli( + "--evidence-dir", + str(evidence_dir), + "--agent-policy-audit-event", + str(event), + "--agent-policy-audit-event-profile", + AUDIT_EVENT_PROFILE, + str(report), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert json.loads(result.stdout)["report_schema_version"] == ( + "agent-guard.report_evidence.v2" + ) + assert event.name not in result.stdout + result.stderr + + def test_bundle_directory_enumeration_stops_at_configured_cap( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1144,14 +1382,14 @@ def test_packaged_consumer_report_read_failure_does_not_echo_path(tmp_path: Path def test_evidence_consumer_fails_closed_on_schema_drift(tmp_path: Path) -> None: payload = json.loads(SAMPLE.read_text(encoding="utf-8")) - payload["report"]["schema_version"] = "agent-guard.report_evidence.v2" + payload["report"]["schema_version"] = "agent-guard.report_evidence.v3" report = tmp_path / "report.json" report.write_text(json.dumps(payload), encoding="utf-8") result = run_consumer(report) assert result.returncode == 1 - assert "$.report.schema_version must equal" in result.stderr + assert "report evidence schema version is not supported" in result.stderr def test_evidence_consumer_rejects_invalid_conformance_profile(tmp_path: Path) -> None: diff --git a/tests/test_package_contract.py b/tests/test_package_contract.py index 8687758..596d2c0 100644 --- a/tests/test_package_contract.py +++ b/tests/test_package_contract.py @@ -1240,8 +1240,10 @@ def test_schema_resources_are_present_in_package_tree() -> None: "agent-guard.context_inventory.v1.schema.json", "agent-guard.context_lock_coverage.v1.schema.json", "agent-guard.report_evidence.v1.schema.json", + "agent-guard.report_evidence.v2.schema.json", "agent-guard.conformance.v1.schema.json", "agent-guard.evidence_pack_manifest.v1.schema.json", + "agent-guard.evidence_pack_manifest.v2.schema.json", "agent-guard.surface_delta.v1.schema.json", } diff --git a/tests/test_schemas.py b/tests/test_schemas.py index cde970d..abfd799 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -22,10 +22,12 @@ EXPECTED_SCHEMAS = { "agent-guard.conformance.v1.schema.json": "agent-guard.conformance.v1", "agent-guard.evidence_pack_manifest.v1.schema.json": "agent-guard.evidence_pack_manifest.v1", + "agent-guard.evidence_pack_manifest.v2.schema.json": "agent-guard.evidence_pack_manifest.v2", "agent-guard.result.v1.schema.json": "agent-guard.result.v1", "agent-guard.context_inventory.v1.schema.json": "agent-guard.context_inventory.v1", "agent-guard.context_lock_coverage.v1.schema.json": "agent-guard.context_lock_coverage.v1", "agent-guard.report_evidence.v1.schema.json": "agent-guard.report_evidence.v1", + "agent-guard.report_evidence.v2.schema.json": "agent-guard.report_evidence.v2", "agent-guard.surface_delta.v1.schema.json": "agent-guard.surface_delta.v1", } @@ -133,7 +135,7 @@ def test_schema_versions_are_pinned() -> None: for name, schema_version in EXPECTED_SCHEMAS.items(): schema = load_schema(name) properties = schema["properties"] - if name == "agent-guard.report_evidence.v1.schema.json": + if name.startswith("agent-guard.report_evidence."): assert properties["schema_version"]["const"] == "agent-guard.result.v1" assert properties["report"]["properties"]["schema_version"]["const"] == schema_version else: @@ -188,9 +190,29 @@ def test_report_schema_allows_conformance_and_evidence_pack_manifest() -> None: schema["properties"]["evidence_pack_manifest"]["properties"]["artifacts"]["items"]["properties"]["role"] ) assert artifact_role["enum"] == ["report", "agent-policy-audit-event"] + artifact = schema["properties"]["evidence_pack_manifest"]["properties"]["artifacts"]["items"] + assert "content_binding" not in artifact["properties"] -def test_evidence_schemas_reject_extra_audit_fields_but_allow_report_metadata() -> None: +def test_v1_evidence_schemas_accept_released_unbound_audit_event_references() -> None: + for schema_name in ( + "agent-guard.evidence_pack_manifest.v1.schema.json", + "agent-guard.report_evidence.v1.schema.json", + ): + report = json.loads(EVIDENCE_SAMPLE_REPORT.read_text(encoding="utf-8")) + value = report["evidence_pack_manifest"] if "manifest" in schema_name else report + manifest = value if "manifest" in schema_name else value["evidence_pack_manifest"] + manifest["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + } + ) + + assert Draft202012Validator(load_schema(schema_name)).is_valid(value) + + +def test_v2_evidence_schemas_require_exact_bound_audit_event_entries() -> None: binding = { "schema_version": "agent-guard.agent_policy_audit_event_binding.v1", "event_profile": "agent-policy.audit_event.v1.1", @@ -200,25 +222,61 @@ def test_evidence_schemas_reject_extra_audit_fields_but_allow_report_metadata() "digest": "b" + ("a" * 52), } for schema_name in ( - "agent-guard.evidence_pack_manifest.v1.schema.json", - "agent-guard.report_evidence.v1.schema.json", + "agent-guard.evidence_pack_manifest.v2.schema.json", + "agent-guard.report_evidence.v2.schema.json", ): report = json.loads(EVIDENCE_SAMPLE_REPORT.read_text(encoding="utf-8")) + report["report"]["schema_version"] = "agent-guard.report_evidence.v2" + report["evidence_pack_manifest"]["schema_version"] = ( + "agent-guard.evidence_pack_manifest.v2" + ) value = report["evidence_pack_manifest"] if "manifest" in schema_name else report manifest = value if "manifest" in schema_name else value["evidence_pack_manifest"] validator = Draft202012Validator(load_schema(schema_name)) + assert not validator.is_valid(value) + if schema_name == "agent-guard.report_evidence.v2.schema.json": + without_manifest = dict(value) + without_manifest.pop("evidence_pack_manifest") + assert not validator.is_valid(without_manifest) + + manifest["report"]["schema_version"] = "agent-guard.report_evidence.v1" + assert not validator.is_valid(value) + manifest["report"]["schema_version"] = "agent-guard.report_evidence.v2" + manifest["artifacts"][0]["review_metadata"] = "synthetic-public-metadata" - assert validator.is_valid(value) + assert not validator.is_valid(value) + + manifest["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + } + ) + assert not validator.is_valid(value) + manifest["artifacts"].pop() manifest["artifacts"].append( { "path": "reviewed/event.json", "role": "agent-policy-audit-event", "content_binding": binding, - "event_body": {"passphrase": "synthetic-private-passphrase"}, } ) + assert validator.is_valid(value) + + manifest["artifacts"][-1]["event_body"] = { + "passphrase": "synthetic-private-passphrase" + } + assert not validator.is_valid(value) + + manifest["artifacts"][-1] = { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": binding | { + "schema_version": "agent-guard.agent_policy_audit_event_binding.v2" + }, + } assert not validator.is_valid(value) From 7e32824c5ebc3fba12eda343769a9b093ef91ff5 Mon Sep 17 00:00:00 2001 From: yui-stingray Date: Thu, 13 Aug 2026 19:05:30 +0900 Subject: [PATCH 2/3] test: align Windows audit fixture with v2 --- tests/test_windows_file_boundaries.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_windows_file_boundaries.py b/tests/test_windows_file_boundaries.py index 04700bd..748b1da 100644 --- a/tests/test_windows_file_boundaries.py +++ b/tests/test_windows_file_boundaries.py @@ -46,7 +46,12 @@ def test_windows_repo_bound_readers_accept_in_root_regular_files(tmp_path: Path) root=repo, ) validate_agent_policy_audit_event_files( - {"evidence_pack_manifest": {"artifacts": artifacts}}, + { + "evidence_pack_manifest": { + "schema_version": "agent-guard.evidence_pack_manifest.v2", + "artifacts": artifacts, + } + }, (audit_event_path,), event_profile=AUDIT_EVENT_PROFILE, ) From 8f9a78cf76fc0a6195147f6a13eb372f8bb08e4d Mon Sep 17 00:00:00 2001 From: yui-stingray Date: Thu, 13 Aug 2026 19:53:02 +0900 Subject: [PATCH 3/3] fix: enforce v2 evidence invariants --- docs/compatibility.md | 6 +-- scripts/check_wheel_contract.py | 58 +++++++++++++++++++-------- src/agent_guard/consumer/_sections.py | 8 ++++ src/agent_guard/evidence_pack.py | 22 +++++++--- tests/cli/test_evidence_pack.py | 34 ++++++++++++++++ tests/test_docs_contract.py | 1 + tests/test_evidence_consumer.py | 48 ++++++++++++++++++++++ tests/test_schemas.py | 19 +++++++-- 8 files changed, 168 insertions(+), 28 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index 777fb07..2de34f5 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -41,7 +41,7 @@ Installed wheels package these JSON Schema resources under | Context inventory | `agent-guard.context_inventory.v1` | `agent-guard.context_inventory.v1.schema.json` | none | Repository-relative context file metadata and permission-boundary status. | | Context lock coverage | `agent-guard.context_lock_coverage.v1` | `agent-guard.context_lock_coverage.v1.schema.json` | none | Hash-free coverage counts, covered context files, and digest coverage findings. | | Sanitized report evidence | `agent-guard.report_evidence.v1` | `agent-guard.report_evidence.v1.schema.json` | none | Public-safe report JSON without content-bound audit-event references. Existing v1 consumers remain supported. | -| Sanitized report evidence | `agent-guard.report_evidence.v2` | `agent-guard.report_evidence.v2.schema.json` | none | Public-safe report JSON whose embedded evidence-pack manifest can bind separately supplied `agent-policy` audit events. | +| Sanitized report evidence | `agent-guard.report_evidence.v2` | `agent-guard.report_evidence.v2.schema.json` | none | Public-safe report JSON whose embedded evidence-pack manifest requires at least one bound `agent-policy` audit-event artifact. | | Conformance evidence | `agent-guard.conformance.v1` | `agent-guard.conformance.v1.schema.json` | none | Profile, status, required gates/surfaces, counts, and conformance findings. | | Evidence pack manifest | `agent-guard.evidence_pack_manifest.v1` | `agent-guard.evidence_pack_manifest.v1.schema.json` | none | Sanitized artifact manifest, including legacy path-and-role audit-event references that are not content-bound. | | Evidence pack manifest | `agent-guard.evidence_pack_manifest.v2` | `agent-guard.evidence_pack_manifest.v2.schema.json` | none | Sanitized artifact manifest whose audit-event entries require a controlled profile and canonical-content digest. | @@ -88,8 +88,8 @@ structurally valid legacy metadata, but they are explicitly unbound. A current consumer can read that v1 evidence, but fails closed if a caller asks it to verify event content against the unbound reference. -V2 requires `content_binding` on every audit-event entry. The binding uses -canonical JSON, an explicit expected event profile, and a domain-separated +V2 requires at least one audit-event artifact and `content_binding` on every +audit-event entry. The binding uses canonical JSON, an explicit expected event profile, and a domain-separated SHA-256 digest encoded as lowercase base32 with a controlled `b` prefix. This controlled digest is not a raw hexadecimal hash and is the only hash-like value admitted on this field. Current consumers fail closed when the separately diff --git a/scripts/check_wheel_contract.py b/scripts/check_wheel_contract.py index 2d603e6..726bc1e 100644 --- a/scripts/check_wheel_contract.py +++ b/scripts/check_wheel_contract.py @@ -1323,32 +1323,38 @@ def main() -> int: "agent-guard.context_inventory.v1.schema.json": "agent-guard.context_inventory.v1", "agent-guard.context_lock_coverage.v1.schema.json": "agent-guard.context_lock_coverage.v1", "agent-guard.report_evidence.v1.schema.json": "agent-guard.report_evidence.v1", + "agent-guard.report_evidence.v2.schema.json": "agent-guard.report_evidence.v2", "agent-guard.conformance.v1.schema.json": "agent-guard.conformance.v1", "agent-guard.evidence_pack_manifest.v1.schema.json": "agent-guard.evidence_pack_manifest.v1", + "agent-guard.evidence_pack_manifest.v2.schema.json": "agent-guard.evidence_pack_manifest.v2", "agent-guard.surface_delta.v1.schema.json": "agent-guard.surface_delta.v1", }} schema_dir = resources.files("agent_guard.schemas") for filename, schema_version in schema_names.items(): schema = json.loads((schema_dir / filename).read_text(encoding="utf-8")) assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" - if filename == "agent-guard.report_evidence.v1.schema.json": + if filename in {{ + "agent-guard.report_evidence.v1.schema.json", + "agent-guard.report_evidence.v2.schema.json", + }}: assert schema["properties"]["report"]["properties"]["schema_version"]["const"] == schema_version - assert "surface_inventory" in schema["allOf"][0]["then"]["required"] - assert "evidence_coverage" in schema["allOf"][0]["then"]["required"] - assert "conformance" in schema["properties"] - assert "evidence_pack_manifest" in schema["properties"] - assert schema["properties"]["conformance"]["properties"]["profile"]["enum"] == [ - "minimal", - "recommended", - "strict", - ] - artifact_role = schema["properties"]["evidence_pack_manifest"]["properties"]["artifacts"]["items"]["properties"]["role"] - assert "agent-policy-audit-event" in artifact_role["enum"] - surface_schema = schema["properties"]["surface_inventory"]["properties"]["schema_version"] - assert "agent-guard.agent_surface_inventory.v2" in surface_schema["enum"] - assert schema["properties"]["surface_delta"]["properties"]["schema_version"]["const"] == ( - "agent-guard.surface_delta.v1" - ) + if filename == "agent-guard.report_evidence.v1.schema.json": + assert "surface_inventory" in schema["allOf"][0]["then"]["required"] + assert "evidence_coverage" in schema["allOf"][0]["then"]["required"] + assert "conformance" in schema["properties"] + assert "evidence_pack_manifest" in schema["properties"] + assert schema["properties"]["conformance"]["properties"]["profile"]["enum"] == [ + "minimal", + "recommended", + "strict", + ] + artifact_role = schema["properties"]["evidence_pack_manifest"]["properties"]["artifacts"]["items"]["properties"]["role"] + assert "agent-policy-audit-event" in artifact_role["enum"] + surface_schema = schema["properties"]["surface_inventory"]["properties"]["schema_version"] + assert "agent-guard.agent_surface_inventory.v2" in surface_schema["enum"] + assert schema["properties"]["surface_delta"]["properties"]["schema_version"]["const"] == ( + "agent-guard.surface_delta.v1" + ) else: assert schema["properties"]["schema_version"]["const"] == schema_version """ @@ -1949,6 +1955,24 @@ def main() -> int: bound_report_payload["evidence_pack_manifest"]["schema_version"] == "agent-guard.evidence_pack_manifest.v2" ) + bound_consumer_cli = run( + isolated_module_command( + python, + "agent_guard.consumer", + "--agent-policy-audit-event", + str(audit_event_path), + "--agent-policy-audit-event-profile", + audit_event_profile, + str(report_output), + ), + cwd=temp, + ) + bound_consumer_summary = json.loads(bound_consumer_cli.stdout) + assert bound_consumer_summary["report_schema_version"] == ( + "agent-guard.report_evidence.v2" + ) + assert audit_event_marker not in bound_consumer_cli.stdout + assert str(temp) not in bound_consumer_cli.stdout manifest_cli = run( isolated_module_command( python, diff --git a/src/agent_guard/consumer/_sections.py b/src/agent_guard/consumer/_sections.py index c352803..72debfc 100644 --- a/src/agent_guard/consumer/_sections.py +++ b/src/agent_guard/consumer/_sections.py @@ -177,6 +177,14 @@ def validate_evidence_pack_manifest(manifest: Mapping[str, Any], payload: Mappin report = require_mapping(payload.get("report"), "$.report") for key in ("schema_version", "format", "scope"): require(manifest_report.get(key) == report.get(key), f"$.evidence_pack_manifest.report.{key} must match $.report.{key}") + expected_report_version = { + "agent-guard.evidence_pack_manifest.v1": "agent-guard.report_evidence.v1", + "agent-guard.evidence_pack_manifest.v2": "agent-guard.report_evidence.v2", + }[manifest_version] + require( + report.get("schema_version") == expected_report_version, + "$.evidence_pack_manifest.schema_version does not match $.report.schema_version", + ) require(manifest_report.get("status") == payload.get("status"), "$.evidence_pack_manifest.report.status must match $.status") require( manifest_report.get("finding_count") == payload.get("finding_count"), diff --git a/src/agent_guard/evidence_pack.py b/src/agent_guard/evidence_pack.py index 005fa30..53d96d6 100644 --- a/src/agent_guard/evidence_pack.py +++ b/src/agent_guard/evidence_pack.py @@ -32,6 +32,9 @@ ERROR_AUDIT_EVENT_REPORT_VERSION = ( "bound agent-policy audit events require report evidence v2" ) +ERROR_AUDIT_EVENT_BINDING_REQUIRED = ( + "report evidence v2 requires bound agent-policy audit events" +) _AUDIT_EVENT_PROFILE_RE = re.compile(r"^[a-z][a-z0-9._-]{0,127}$") _AUDIT_EVENT_DIGEST_RE = re.compile(r"^b[a-z2-7]{52}$") @@ -495,6 +498,11 @@ def build_evidence_pack_manifest( _validate_agent_policy_audit_event_artifact_shape(artifact, root=root) for artifact in agent_policy_audit_event_artifacts ] + if ( + not audit_event_artifacts + and str(agent_policy_audit_event_profile).strip() + ): + raise ValueError(ERROR_AUDIT_EVENT_PROFILE) elif agent_policy_audit_event_paths: if root is None: raise ValueError(ERROR_AUDIT_EVENT_PATH) @@ -506,13 +514,17 @@ def build_evidence_pack_manifest( elif str(agent_policy_audit_event_profile).strip(): raise ValueError(ERROR_AUDIT_EVENT_PROFILE) artifacts.extend(audit_event_artifacts) + report_metadata = report_payload.get("report") + report_schema_version = ( + report_metadata.get("schema_version") + if isinstance(report_metadata, dict) + else "" + ) if audit_event_artifacts: - report_metadata = report_payload.get("report") - if ( - not isinstance(report_metadata, dict) - or report_metadata.get("schema_version") != REPORT_EVIDENCE_SCHEMA_VERSION_V2 - ): + if report_schema_version != REPORT_EVIDENCE_SCHEMA_VERSION_V2: raise ValueError(ERROR_AUDIT_EVENT_REPORT_VERSION) + elif report_schema_version == REPORT_EVIDENCE_SCHEMA_VERSION_V2: + raise ValueError(ERROR_AUDIT_EVENT_BINDING_REQUIRED) manifest: dict[str, object] = { "schema_version": ( diff --git a/tests/cli/test_evidence_pack.py b/tests/cli/test_evidence_pack.py index 521be0d..bb197f6 100644 --- a/tests/cli/test_evidence_pack.py +++ b/tests/cli/test_evidence_pack.py @@ -481,6 +481,40 @@ def test_bound_manifest_rejects_v1_report_without_leak(tmp_path: Path) -> None: assert str(event) not in str(exc_info.value) +def test_manifest_rejects_v2_report_without_bound_audit_event() -> None: + for prebuilt_artifacts in (None, []): + with pytest.raises( + ValueError, + match=( + r"^report evidence v2 requires bound agent-policy audit events$" + ), + ): + build_evidence_pack_manifest( + report_payload=V2_REPORT_PAYLOAD, + agent_policy_audit_event_artifacts=prebuilt_artifacts, + ) + + manifest = build_evidence_pack_manifest( + report_payload={ + "report": {"schema_version": "agent-guard.report_evidence.v1"}, + }, + agent_policy_audit_event_artifacts=[], + ) + assert manifest["schema_version"] == "agent-guard.evidence_pack_manifest.v1" + + +def test_manifest_rejects_profile_with_explicitly_empty_prebuilt_artifacts() -> None: + with pytest.raises( + ValueError, + match=r"^agent-policy audit event profile is invalid$", + ): + build_evidence_pack_manifest( + report_payload=V2_REPORT_PAYLOAD, + agent_policy_audit_event_artifacts=[], + agent_policy_audit_event_profile=AUDIT_EVENT_PROFILE, + ) + + @pytest.mark.parametrize( "event_profile", ( diff --git a/tests/test_docs_contract.py b/tests/test_docs_contract.py index 69d59f1..bd52040 100644 --- a/tests/test_docs_contract.py +++ b/tests/test_docs_contract.py @@ -1011,6 +1011,7 @@ def test_compatibility_doc_keeps_public_safe_contract_bounded() -> None: assert "without a new bundle version or explicit" in docs assert "`agent-guard.result.v1` outer envelope" in docs assert "nested `conformance` and `evidence_pack_manifest`" in " ".join(docs.split()) + assert "requires at least one bound `agent-policy` audit-event artifact" in docs def test_release_criteria_keep_patch_releases_bounded() -> None: diff --git a/tests/test_evidence_consumer.py b/tests/test_evidence_consumer.py index cacddf0..d180c98 100644 --- a/tests/test_evidence_consumer.py +++ b/tests/test_evidence_consumer.py @@ -22,6 +22,7 @@ load_report_schema, select_report_schema, validate_agent_policy_audit_event_files, + validate_evidence_pack_manifest, validate_report, ) from agent_guard.consumer import ( @@ -374,6 +375,53 @@ def test_v1_schema_consumer_fails_closed_on_v2_report(tmp_path: Path) -> None: validate_report(payload, load_report_schema()) +@pytest.mark.parametrize( + ("manifest_version", "report_version"), + ( + ( + "agent-guard.evidence_pack_manifest.v1", + "agent-guard.report_evidence.v2", + ), + ( + "agent-guard.evidence_pack_manifest.v2", + "agent-guard.report_evidence.v1", + ), + ), +) +def test_evidence_pack_manifest_rejects_incompatible_version_pair( + tmp_path: Path, + manifest_version: str, + report_version: str, +) -> None: + payload = json.loads(SAMPLE.read_text(encoding="utf-8")) + manifest = payload["evidence_pack_manifest"] + payload["report"]["schema_version"] = report_version + manifest["schema_version"] = manifest_version + manifest["report"]["schema_version"] = report_version + if manifest_version == "agent-guard.evidence_pack_manifest.v2": + event = tmp_path / "synthetic-reviewed-event.json" + event.write_text('{"status":"reviewed"}\n', encoding="utf-8") + manifest["artifacts"].append( + { + "path": "reviewed/event.json", + "role": "agent-policy-audit-event", + "content_binding": build_agent_policy_audit_event_binding( + event, + event_profile=AUDIT_EVENT_PROFILE, + ), + } + ) + + with pytest.raises( + ValueError, + match=( + r"^\$\.evidence_pack_manifest\.schema_version does not match " + r"\$\.report\.schema_version$" + ), + ): + validate_evidence_pack_manifest(manifest, payload) + + def _prepend_duplicate_json_member(text: str, *, key: str, value: object) -> str: assert text.startswith("{") member = f"{json.dumps(key, ensure_ascii=False)}:{json.dumps(value, ensure_ascii=False)}" diff --git a/tests/test_schemas.py b/tests/test_schemas.py index abfd799..05b0827 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -244,9 +244,6 @@ def test_v2_evidence_schemas_require_exact_bound_audit_event_entries() -> None: assert not validator.is_valid(value) manifest["report"]["schema_version"] = "agent-guard.report_evidence.v2" - manifest["artifacts"][0]["review_metadata"] = "synthetic-public-metadata" - assert not validator.is_valid(value) - manifest["artifacts"].append( { "path": "reviewed/event.json", @@ -265,6 +262,11 @@ def test_v2_evidence_schemas_require_exact_bound_audit_event_entries() -> None: ) assert validator.is_valid(value) + manifest["artifacts"][-1]["review_metadata"] = "synthetic-public-metadata" + assert not validator.is_valid(value) + del manifest["artifacts"][-1]["review_metadata"] + assert validator.is_valid(value) + manifest["artifacts"][-1]["event_body"] = { "passphrase": "synthetic-private-passphrase" } @@ -280,6 +282,17 @@ def test_v2_evidence_schemas_require_exact_bound_audit_event_entries() -> None: assert not validator.is_valid(value) +def test_v2_embedded_manifest_artifacts_match_standalone_contract() -> None: + standalone = load_schema( + "agent-guard.evidence_pack_manifest.v2.schema.json" + ) + report = load_schema("agent-guard.report_evidence.v2.schema.json") + + assert standalone["properties"]["artifacts"] == ( + report["properties"]["evidence_pack_manifest"]["properties"]["artifacts"] + ) + + def test_surface_delta_schema_requires_details_only_when_base_resolves() -> None: unresolved = { "schema_version": "agent-guard.surface_delta.v1",