diff --git a/claude-code/engine/capture.py b/claude-code/engine/capture.py index dbfa6d2..f5e76a9 100644 --- a/claude-code/engine/capture.py +++ b/claude-code/engine/capture.py @@ -437,31 +437,84 @@ def _load_or_create_manifest_keypair(): # --------------------------------------------------------------------------- # # report rendering # --------------------------------------------------------------------------- # +#: Shown wherever a category was not measured. An integrity report must not let +#: "we did not check" read like "we checked and there is nothing", because a +#: reader who cannot tell them apart will treat an absent measurement as a pass. +_UNMEASURED = "not measured this run" + +#: How a reader gets the categories a shell hook cannot see. Rendered next to the +#: unmeasured lines so the report says what to do rather than only what is absent. +_ENRICH_HINT = "run /manifest verify to include model, tools and MCP" + + def render_report(cur: dict, changes: list[dict] | None, signed: bool) -> str: m = cur["model"] + observed = set(cur.get("observed", [])) + # The hook runs in a shell and cannot introspect the live tool roster or the + # model, so those arrive only via --live-context. `observed` records what this + # snapshot actually measured; anything outside it is reported as unmeasured + # rather than as a count of zero. + tools_seen = "tools" in observed + mcp_seen = "mcp" in observed + model_seen = m.get("model_id") not in (None, "", "unknown") + n_builtin = len([t for t in cur["tools"] if not t.startswith("mcp:")]) + model_line = ( + f"{m['provider']}/{m['model_id']} {m['version']}" if model_seen + else f"{_UNMEASURED} ({_ENRICH_HINT})" + ) + if tools_seen or mcp_seen: + builtin_part = f"{n_builtin} built-in" if tools_seen else f"built-in {_UNMEASURED}" + mcp_part = ( + f"{len(cur['mcp_servers'])} MCP server(s)" if mcp_seen + else f"MCP {_UNMEASURED}" + ) + tools_line = f"{builtin_part} + {mcp_part}" + else: + tools_line = f"{_UNMEASURED} ({_ENRICH_HINT})" + mcp_line = ( + (", ".join(cur["mcp_servers"]) or "none connected") if mcp_seen + else f"{_UNMEASURED}. Servers found on disk: " + f"{', '.join(cur['mcp_servers']) or 'none'}" + ) + + # A tool-catalog hash over an unmeasured roster is just the hash of an empty + # list, the same value on every run. Printing it as a fingerprint invites a + # reader to treat a constant as evidence, so name it for what it is. + catalog_line = ( + f"{cur['hashes']['tool_catalog'][:23]}..." if tools_seen or mcp_seen + else _UNMEASURED + ) + L = ["=" * 66, " AGENT INTEGRITY REPORT -- your Claude Code session", "=" * 66, "", f" Agent identity : {cur['agent_id']}", - f" Model : {m['provider']}/{m['model_id']} {m['version']}", + f" Model : {model_line}", f" Captured : {cur['captured_at']}", "", " WHAT THIS AGENT IS (agent-manifest -- signed composition)", " " + "-" * 62, f" Skills loaded : {len(cur['skills'])} ({', '.join(cur['skills']) or 'none'})", - f" Tools exposed : {n_builtin} built-in + {len(cur['mcp_servers'])} MCP server(s)", - f" MCP servers : {', '.join(cur['mcp_servers']) or 'none on disk'}", + f" Tools exposed : {tools_line}", + f" MCP servers : {mcp_line}", f" Permissions : {len(cur['allow_rules'])} allow-rule(s), enforce mode", "", " Fingerprints (change here == your agent changed):", f" instruction layer : {cur['hashes']['system_prompt'][:23]}...", f" permissions : {cur['hashes']['policy_bundle'][:23]}...", f" skills set : {cur['hashes']['skills_set'][:23]}...", - f" tool catalog : {cur['hashes']['tool_catalog'][:23]}...", ""] + f" tool catalog : {catalog_line}", ""] + if not (tools_seen and mcp_seen and model_seen): + L += [" Categories marked \"not measured\" are NOT part of this comparison.", + " They are unchecked, not verified as empty.", ""] if changes is not None: L += [" NOTHING ADDED, NOTHING SUBTRACTED? (vs approved baseline)", " " + "-" * 62] if not changes: - L.append(" >> Verified: nothing added, nothing subtracted.") + # "Nothing added, nothing subtracted" is only true of what was + # compared. Qualifying it keeps a partial check from reading as a + # clean bill of health. + scope = "" if (tools_seen and mcp_seen) else " in the categories checked" + L.append(f" >> Verified: nothing added, nothing subtracted{scope}.") else: sym = {"added": "+", "removed": "-", "changed": "~"} for c in changes: @@ -502,9 +555,29 @@ def _load(path: Path) -> dict | None: def _live_from(args) -> dict | None: - if args.live_context: - return json.loads(Path(args.live_context).read_text(encoding="utf-8")) - return None + """Load the live-context file, or exit with a clear message. + + The file is written by the agent, so a missing path or malformed JSON is a + realistic failure. Reporting it plainly beats a traceback, and silently + treating it as absent would be worse than both: the run would proceed with + model, tools, and MCP unmeasured while the caller believed it had supplied + them, which is exactly the false-assurance this tool exists to prevent. + """ + if not args.live_context: + return None + path = Path(args.live_context) + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + raise SystemExit(f"live-context file could not be read: {path}: {exc}") from exc + except json.JSONDecodeError as exc: + raise SystemExit(f"live-context file is not valid JSON: {path}: {exc}") from exc + if not isinstance(loaded, dict): + raise SystemExit( + f"live-context file must contain a JSON object, got " + f"{type(loaded).__name__}: {path}" + ) + return loaded # --------------------------------------------------------------------------- # diff --git a/claude-code/tests/test_capture.py b/claude-code/tests/test_capture.py index 9986eea..b563142 100644 --- a/claude-code/tests/test_capture.py +++ b/claude-code/tests/test_capture.py @@ -28,6 +28,118 @@ def _base(**over): return snap +def _report_snap(observed, **over): + """A snapshot shaped for render_report, with `observed` under test.""" + snap = { + "observed": observed, + "agent_id": "spiffe://claude-code.local/u/h", + "captured_at": "2026-07-31T00:00:00Z", + "model": {"provider": "anthropic", "model_id": "unknown", "version": "unknown"}, + "skills": {"trace": "sha256:" + "a" * 64}, + "allow_rules": [], + "mcp_servers": [], + "tools": [], + "hashes": { + "system_prompt": "sha256:" + "1" * 64, + "policy_bundle": "sha256:" + "2" * 64, + "skills_set": "sha256:" + "3" * 64, + "tool_catalog": "sha256:" + "4" * 64, + }, + } + snap.update(over) + return snap + + +class TestUnmeasuredIsNotReportedAsEmpty: + """ + An integrity report must not let "we did not check" read like "we checked + and there is nothing". A shell hook cannot see the model or the live tool + roster, so those categories must be labelled unmeasured rather than + rendered as zero. + """ + + def test_hook_snapshot_labels_model_and_tools_unmeasured(self): + out = capture.render_report(_report_snap(["skills", "policy", "prompt"]), None, False) + assert "0 built-in" not in out + assert "anthropic/unknown" not in out + assert out.count(capture._UNMEASURED) >= 3 # model, tools, tool catalog + assert "unchecked, not verified as empty" in out + + def test_unmeasured_tool_catalog_hash_is_not_shown_as_a_fingerprint(self): + """The hash of an empty roster is a constant; showing it invites a + reader to treat it as evidence.""" + out = capture.render_report(_report_snap(["skills", "policy", "prompt"]), None, False) + assert "sha256:" + "4" * 23 not in out + assert f"tool catalog : {capture._UNMEASURED}" in out + + def test_disk_found_servers_are_shown_without_claiming_measurement(self): + snap = _report_snap(["skills", "policy", "prompt"], mcp_servers=["Slack"]) + out = capture.render_report(snap, None, False) + assert "Slack" in out # do not hide what was found + assert capture._UNMEASURED in out # but do not call it measured + + def test_fully_measured_snapshot_reports_real_values(self): + snap = _report_snap( + ["skills", "policy", "prompt", "mcp", "tools"], + model={"provider": "anthropic", "model_id": "claude-opus-5", "version": "1m"}, + mcp_servers=["Slack"], + tools=["Bash", "mcp:Slack"], + ) + out = capture.render_report(snap, None, False) + assert "anthropic/claude-opus-5 1m" in out + assert "1 built-in + 1 MCP server(s)" in out + assert capture._UNMEASURED not in out + assert "unchecked, not verified as empty" not in out + + def test_clean_verdict_is_qualified_when_coverage_is_partial(self): + partial = capture.render_report(_report_snap(["skills", "policy", "prompt"]), [], False) + assert "nothing subtracted in the categories checked." in partial + + def test_clean_verdict_is_unqualified_when_coverage_is_complete(self): + full = capture.render_report( + _report_snap(["skills", "policy", "prompt", "mcp", "tools"]), [], False + ) + assert "nothing subtracted." in full + assert "in the categories checked" not in full + + +class TestLiveContextLoading: + """A live-context file is written by the agent, so bad input is realistic. + It must fail loudly: silently treating it as absent would leave the caller + believing it supplied a measurement it did not.""" + + def test_missing_file_exits_with_a_message(self, tmp_path): + with pytest.raises(SystemExit, match="could not be read"): + capture._live_from(_Args(live_context=str(tmp_path / "nope.json"))) + + def test_malformed_json_exits_with_a_message(self, tmp_path): + bad = tmp_path / "live.json" + bad.write_text("{not json", encoding="utf-8") + with pytest.raises(SystemExit, match="not valid JSON"): + capture._live_from(_Args(live_context=str(bad))) + + def test_non_object_json_exits_with_a_message(self, tmp_path): + bad = tmp_path / "live.json" + bad.write_text('["a", "list"]', encoding="utf-8") + with pytest.raises(SystemExit, match="must contain a JSON object"): + capture._live_from(_Args(live_context=str(bad))) + + def test_absent_flag_is_not_an_error(self): + assert capture._live_from(_Args(live_context=None)) is None + + def test_valid_file_marks_tools_and_mcp_observed(self, tmp_path): + good = tmp_path / "live.json" + good.write_text( + json.dumps({"model_id": "claude-opus-5", "builtin_tools": ["Bash"], + "mcp_servers": ["Slack"]}), + encoding="utf-8", + ) + live = capture._live_from(_Args(live_context=str(good))) + snap = capture.snapshot(live) + assert "tools" in snap["observed"] and "mcp" in snap["observed"] + assert snap["model"]["model_id"] == "claude-opus-5" + + def test_identical_snapshots_have_no_diff(): assert capture.diff(_base(), _base()) == [] @@ -87,6 +199,10 @@ class _Args: json = False sign = False + def __init__(self, **over): + for key, value in over.items(): + setattr(self, key, value) + def test_verify_detects_drift_introduced_after_baseline(tmp_path, monkeypatch, capsys): """verify must re-snapshot, not trust a stale session-latest.json. diff --git a/plugins/agentrust-codex/engine/capture.py b/plugins/agentrust-codex/engine/capture.py index 7463884..2662d36 100644 --- a/plugins/agentrust-codex/engine/capture.py +++ b/plugins/agentrust-codex/engine/capture.py @@ -855,12 +855,34 @@ def sign_all( return manifest, trace +#: Shown wherever a category was not measured. An integrity report must not let +#: "we did not check" read like "we checked and there is nothing", because a +#: reader who cannot tell them apart will treat an absent measurement as a pass. +UNMEASURED = "not measured this run" + + def render_report( current: Mapping[str, Any], changes: Optional[Sequence[Mapping[str, str]]], signed: bool, ) -> str: model = current["model"] + # The model and the live tool roster are runtime facts a shell hook cannot + # see; they enter only via a live context. `observed` records what this + # snapshot actually measured, so anything outside it is labelled rather than + # rendered as a real value. + observed = set(current.get("observed", [])) + model_line = ( + "%s/%s" % (model["provider"], model["model_id"]) if "model" in observed + else UNMEASURED + ) + # A tool-catalog hash over an unmeasured roster is the hash of an empty list, + # identical on every run. Presenting it as a fingerprint invites a reader to + # treat a constant as evidence. + catalog_line = ( + "%s..." % current["hashes"]["tool_catalog"][:23] if "tools" in observed + else UNMEASURED + ) lines = [ "=" * 68, " AGENTRUST CODEX INTEGRITY REPORT", @@ -868,7 +890,7 @@ def render_report( "", " Workspace : %s" % current["workspace_id"][:16], " Agent identity : %s" % current["agent_id"], - " Model : %s/%s" % (model["provider"], model["model_id"]), + " Model : %s" % model_line, " Permission mode: %s" % current["permission_mode"], " Captured : %s" % current["captured_at"], "", @@ -884,13 +906,22 @@ def render_report( " policy bundle : %s..." % current["hashes"]["policy_bundle"][:23], " skills set : %s..." % current["hashes"]["skills_set"][:23], " plugin set : %s..." % current["hashes"]["plugin_set"][:23], - " tool catalog : %s..." % current["hashes"]["tool_catalog"][:23], + " tool catalog : %s" % catalog_line, "", ] + if not {"model", "tools"} <= observed: + lines += [ + " Categories marked \"%s\" are NOT part of this comparison." % UNMEASURED, + " They are unchecked, not verified as empty.", + "", + ] if changes is not None: lines += [" Baseline comparison", " " + "-" * 64] if not changes: - lines.append(" Verified: no composition changes.") + # "No changes" is only true of what was compared. Qualifying it keeps + # a partial check from reading as a clean bill of health. + scope = "" if {"model", "tools"} <= observed else " in the categories checked" + lines.append(" Verified: no composition changes%s." % scope) else: symbols = {"added": "+", "removed": "-", "changed": "~"} for change in changes: