From d26a3af59ab1d5b946a7d20f4e24dcc353b00631 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Thu, 30 Jul 2026 21:20:07 -0700 Subject: [PATCH] fix(report): never render an unmeasured category as a measured zero The integrity report presented categories it had not measured as if it had. A SessionStart hook runs in a shell and cannot introspect the model or the live tool roster; those arrive only via --live-context. With none supplied the report printed: Model : anthropic/unknown unknown Tools exposed : 0 built-in + 0 MCP server(s) tool catalog : sha256:4f53cda18c2baa0c... all three of which read as measurements. The catalog fingerprint is the hash of an empty list, identical on every run, presented in a section headed "change here == your agent changed". A reader cannot tell "we did not check" from "we checked and there is nothing", and for an integrity tool that distinction is the product. The data model already recorded this correctly: `observed` marks which categories a snapshot measured, and diff() has always scoped comparisons to it, so the diffing was never wrong. Only the rendering was. render_report now consults `observed` and labels unmeasured categories, keeps the tool-catalog line out of the fingerprint list when the roster was not measured, and qualifies a clean verdict as "nothing added, nothing subtracted in the categories checked" when coverage is partial. Servers found on disk are still shown, without being described as measured. Same fix in the Codex plugin engine, which had the same defect against its own observed categories. Also makes a bad --live-context fail loudly. It was an unhandled traceback, and the tempting alternative of treating it as absent is worse than both: the run would proceed with model, tools and MCP unmeasured while the caller believed it had supplied them, which is the false assurance this tool exists to prevent. 11 new tests. claude-code: 26 passed, up from 15. Note for reviewers: plugins/agentrust-codex/tests/test_capture.py has one pre-existing failure on main (test_signed_outputs_verify_and_pass_trace_level_zero, a TRACE conformance check on generated records). Verified failing on main with this change stashed, so it is untouched by this PR and not addressed here. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Imran Siddique --- claude-code/engine/capture.py | 89 +++++++++++++++-- claude-code/tests/test_capture.py | 116 ++++++++++++++++++++++ plugins/agentrust-codex/engine/capture.py | 37 ++++++- 3 files changed, 231 insertions(+), 11 deletions(-) diff --git a/claude-code/engine/capture.py b/claude-code/engine/capture.py index dbfa6d2..f5e76a9 100644 --- a/claude-code/engine/capture.py +++ b/claude-code/engine/capture.py @@ -437,31 +437,84 @@ def _load_or_create_manifest_keypair(): # --------------------------------------------------------------------------- # # report rendering # --------------------------------------------------------------------------- # +#: Shown wherever a category was not measured. An integrity report must not let +#: "we did not check" read like "we checked and there is nothing", because a +#: reader who cannot tell them apart will treat an absent measurement as a pass. +_UNMEASURED = "not measured this run" + +#: How a reader gets the categories a shell hook cannot see. Rendered next to the +#: unmeasured lines so the report says what to do rather than only what is absent. +_ENRICH_HINT = "run /manifest verify to include model, tools and MCP" + + def render_report(cur: dict, changes: list[dict] | None, signed: bool) -> str: m = cur["model"] + observed = set(cur.get("observed", [])) + # The hook runs in a shell and cannot introspect the live tool roster or the + # model, so those arrive only via --live-context. `observed` records what this + # snapshot actually measured; anything outside it is reported as unmeasured + # rather than as a count of zero. + tools_seen = "tools" in observed + mcp_seen = "mcp" in observed + model_seen = m.get("model_id") not in (None, "", "unknown") + n_builtin = len([t for t in cur["tools"] if not t.startswith("mcp:")]) + model_line = ( + f"{m['provider']}/{m['model_id']} {m['version']}" if model_seen + else f"{_UNMEASURED} ({_ENRICH_HINT})" + ) + if tools_seen or mcp_seen: + builtin_part = f"{n_builtin} built-in" if tools_seen else f"built-in {_UNMEASURED}" + mcp_part = ( + f"{len(cur['mcp_servers'])} MCP server(s)" if mcp_seen + else f"MCP {_UNMEASURED}" + ) + tools_line = f"{builtin_part} + {mcp_part}" + else: + tools_line = f"{_UNMEASURED} ({_ENRICH_HINT})" + mcp_line = ( + (", ".join(cur["mcp_servers"]) or "none connected") if mcp_seen + else f"{_UNMEASURED}. Servers found on disk: " + f"{', '.join(cur['mcp_servers']) or 'none'}" + ) + + # A tool-catalog hash over an unmeasured roster is just the hash of an empty + # list, the same value on every run. Printing it as a fingerprint invites a + # reader to treat a constant as evidence, so name it for what it is. + catalog_line = ( + f"{cur['hashes']['tool_catalog'][:23]}..." if tools_seen or mcp_seen + else _UNMEASURED + ) + L = ["=" * 66, " AGENT INTEGRITY REPORT -- your Claude Code session", "=" * 66, "", f" Agent identity : {cur['agent_id']}", - f" Model : {m['provider']}/{m['model_id']} {m['version']}", + f" Model : {model_line}", f" Captured : {cur['captured_at']}", "", " WHAT THIS AGENT IS (agent-manifest -- signed composition)", " " + "-" * 62, f" Skills loaded : {len(cur['skills'])} ({', '.join(cur['skills']) or 'none'})", - f" Tools exposed : {n_builtin} built-in + {len(cur['mcp_servers'])} MCP server(s)", - f" MCP servers : {', '.join(cur['mcp_servers']) or 'none on disk'}", + f" Tools exposed : {tools_line}", + f" MCP servers : {mcp_line}", f" Permissions : {len(cur['allow_rules'])} allow-rule(s), enforce mode", "", " Fingerprints (change here == your agent changed):", f" instruction layer : {cur['hashes']['system_prompt'][:23]}...", f" permissions : {cur['hashes']['policy_bundle'][:23]}...", f" skills set : {cur['hashes']['skills_set'][:23]}...", - f" tool catalog : {cur['hashes']['tool_catalog'][:23]}...", ""] + f" tool catalog : {catalog_line}", ""] + if not (tools_seen and mcp_seen and model_seen): + L += [" Categories marked \"not measured\" are NOT part of this comparison.", + " They are unchecked, not verified as empty.", ""] if changes is not None: L += [" NOTHING ADDED, NOTHING SUBTRACTED? (vs approved baseline)", " " + "-" * 62] if not changes: - L.append(" >> Verified: nothing added, nothing subtracted.") + # "Nothing added, nothing subtracted" is only true of what was + # compared. Qualifying it keeps a partial check from reading as a + # clean bill of health. + scope = "" if (tools_seen and mcp_seen) else " in the categories checked" + L.append(f" >> Verified: nothing added, nothing subtracted{scope}.") else: sym = {"added": "+", "removed": "-", "changed": "~"} for c in changes: @@ -502,9 +555,29 @@ def _load(path: Path) -> dict | None: def _live_from(args) -> dict | None: - if args.live_context: - return json.loads(Path(args.live_context).read_text(encoding="utf-8")) - return None + """Load the live-context file, or exit with a clear message. + + The file is written by the agent, so a missing path or malformed JSON is a + realistic failure. Reporting it plainly beats a traceback, and silently + treating it as absent would be worse than both: the run would proceed with + model, tools, and MCP unmeasured while the caller believed it had supplied + them, which is exactly the false-assurance this tool exists to prevent. + """ + if not args.live_context: + return None + path = Path(args.live_context) + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + raise SystemExit(f"live-context file could not be read: {path}: {exc}") from exc + except json.JSONDecodeError as exc: + raise SystemExit(f"live-context file is not valid JSON: {path}: {exc}") from exc + if not isinstance(loaded, dict): + raise SystemExit( + f"live-context file must contain a JSON object, got " + f"{type(loaded).__name__}: {path}" + ) + return loaded # --------------------------------------------------------------------------- # diff --git a/claude-code/tests/test_capture.py b/claude-code/tests/test_capture.py index 9986eea..b563142 100644 --- a/claude-code/tests/test_capture.py +++ b/claude-code/tests/test_capture.py @@ -28,6 +28,118 @@ def _base(**over): return snap +def _report_snap(observed, **over): + """A snapshot shaped for render_report, with `observed` under test.""" + snap = { + "observed": observed, + "agent_id": "spiffe://claude-code.local/u/h", + "captured_at": "2026-07-31T00:00:00Z", + "model": {"provider": "anthropic", "model_id": "unknown", "version": "unknown"}, + "skills": {"trace": "sha256:" + "a" * 64}, + "allow_rules": [], + "mcp_servers": [], + "tools": [], + "hashes": { + "system_prompt": "sha256:" + "1" * 64, + "policy_bundle": "sha256:" + "2" * 64, + "skills_set": "sha256:" + "3" * 64, + "tool_catalog": "sha256:" + "4" * 64, + }, + } + snap.update(over) + return snap + + +class TestUnmeasuredIsNotReportedAsEmpty: + """ + An integrity report must not let "we did not check" read like "we checked + and there is nothing". A shell hook cannot see the model or the live tool + roster, so those categories must be labelled unmeasured rather than + rendered as zero. + """ + + def test_hook_snapshot_labels_model_and_tools_unmeasured(self): + out = capture.render_report(_report_snap(["skills", "policy", "prompt"]), None, False) + assert "0 built-in" not in out + assert "anthropic/unknown" not in out + assert out.count(capture._UNMEASURED) >= 3 # model, tools, tool catalog + assert "unchecked, not verified as empty" in out + + def test_unmeasured_tool_catalog_hash_is_not_shown_as_a_fingerprint(self): + """The hash of an empty roster is a constant; showing it invites a + reader to treat it as evidence.""" + out = capture.render_report(_report_snap(["skills", "policy", "prompt"]), None, False) + assert "sha256:" + "4" * 23 not in out + assert f"tool catalog : {capture._UNMEASURED}" in out + + def test_disk_found_servers_are_shown_without_claiming_measurement(self): + snap = _report_snap(["skills", "policy", "prompt"], mcp_servers=["Slack"]) + out = capture.render_report(snap, None, False) + assert "Slack" in out # do not hide what was found + assert capture._UNMEASURED in out # but do not call it measured + + def test_fully_measured_snapshot_reports_real_values(self): + snap = _report_snap( + ["skills", "policy", "prompt", "mcp", "tools"], + model={"provider": "anthropic", "model_id": "claude-opus-5", "version": "1m"}, + mcp_servers=["Slack"], + tools=["Bash", "mcp:Slack"], + ) + out = capture.render_report(snap, None, False) + assert "anthropic/claude-opus-5 1m" in out + assert "1 built-in + 1 MCP server(s)" in out + assert capture._UNMEASURED not in out + assert "unchecked, not verified as empty" not in out + + def test_clean_verdict_is_qualified_when_coverage_is_partial(self): + partial = capture.render_report(_report_snap(["skills", "policy", "prompt"]), [], False) + assert "nothing subtracted in the categories checked." in partial + + def test_clean_verdict_is_unqualified_when_coverage_is_complete(self): + full = capture.render_report( + _report_snap(["skills", "policy", "prompt", "mcp", "tools"]), [], False + ) + assert "nothing subtracted." in full + assert "in the categories checked" not in full + + +class TestLiveContextLoading: + """A live-context file is written by the agent, so bad input is realistic. + It must fail loudly: silently treating it as absent would leave the caller + believing it supplied a measurement it did not.""" + + def test_missing_file_exits_with_a_message(self, tmp_path): + with pytest.raises(SystemExit, match="could not be read"): + capture._live_from(_Args(live_context=str(tmp_path / "nope.json"))) + + def test_malformed_json_exits_with_a_message(self, tmp_path): + bad = tmp_path / "live.json" + bad.write_text("{not json", encoding="utf-8") + with pytest.raises(SystemExit, match="not valid JSON"): + capture._live_from(_Args(live_context=str(bad))) + + def test_non_object_json_exits_with_a_message(self, tmp_path): + bad = tmp_path / "live.json" + bad.write_text('["a", "list"]', encoding="utf-8") + with pytest.raises(SystemExit, match="must contain a JSON object"): + capture._live_from(_Args(live_context=str(bad))) + + def test_absent_flag_is_not_an_error(self): + assert capture._live_from(_Args(live_context=None)) is None + + def test_valid_file_marks_tools_and_mcp_observed(self, tmp_path): + good = tmp_path / "live.json" + good.write_text( + json.dumps({"model_id": "claude-opus-5", "builtin_tools": ["Bash"], + "mcp_servers": ["Slack"]}), + encoding="utf-8", + ) + live = capture._live_from(_Args(live_context=str(good))) + snap = capture.snapshot(live) + assert "tools" in snap["observed"] and "mcp" in snap["observed"] + assert snap["model"]["model_id"] == "claude-opus-5" + + def test_identical_snapshots_have_no_diff(): assert capture.diff(_base(), _base()) == [] @@ -87,6 +199,10 @@ class _Args: json = False sign = False + def __init__(self, **over): + for key, value in over.items(): + setattr(self, key, value) + def test_verify_detects_drift_introduced_after_baseline(tmp_path, monkeypatch, capsys): """verify must re-snapshot, not trust a stale session-latest.json. diff --git a/plugins/agentrust-codex/engine/capture.py b/plugins/agentrust-codex/engine/capture.py index 7463884..2662d36 100644 --- a/plugins/agentrust-codex/engine/capture.py +++ b/plugins/agentrust-codex/engine/capture.py @@ -855,12 +855,34 @@ def sign_all( return manifest, trace +#: Shown wherever a category was not measured. An integrity report must not let +#: "we did not check" read like "we checked and there is nothing", because a +#: reader who cannot tell them apart will treat an absent measurement as a pass. +UNMEASURED = "not measured this run" + + def render_report( current: Mapping[str, Any], changes: Optional[Sequence[Mapping[str, str]]], signed: bool, ) -> str: model = current["model"] + # The model and the live tool roster are runtime facts a shell hook cannot + # see; they enter only via a live context. `observed` records what this + # snapshot actually measured, so anything outside it is labelled rather than + # rendered as a real value. + observed = set(current.get("observed", [])) + model_line = ( + "%s/%s" % (model["provider"], model["model_id"]) if "model" in observed + else UNMEASURED + ) + # A tool-catalog hash over an unmeasured roster is the hash of an empty list, + # identical on every run. Presenting it as a fingerprint invites a reader to + # treat a constant as evidence. + catalog_line = ( + "%s..." % current["hashes"]["tool_catalog"][:23] if "tools" in observed + else UNMEASURED + ) lines = [ "=" * 68, " AGENTRUST CODEX INTEGRITY REPORT", @@ -868,7 +890,7 @@ def render_report( "", " Workspace : %s" % current["workspace_id"][:16], " Agent identity : %s" % current["agent_id"], - " Model : %s/%s" % (model["provider"], model["model_id"]), + " Model : %s" % model_line, " Permission mode: %s" % current["permission_mode"], " Captured : %s" % current["captured_at"], "", @@ -884,13 +906,22 @@ def render_report( " policy bundle : %s..." % current["hashes"]["policy_bundle"][:23], " skills set : %s..." % current["hashes"]["skills_set"][:23], " plugin set : %s..." % current["hashes"]["plugin_set"][:23], - " tool catalog : %s..." % current["hashes"]["tool_catalog"][:23], + " tool catalog : %s" % catalog_line, "", ] + if not {"model", "tools"} <= observed: + lines += [ + " Categories marked \"%s\" are NOT part of this comparison." % UNMEASURED, + " They are unchecked, not verified as empty.", + "", + ] if changes is not None: lines += [" Baseline comparison", " " + "-" * 64] if not changes: - lines.append(" Verified: no composition changes.") + # "No changes" is only true of what was compared. Qualifying it keeps + # a partial check from reading as a clean bill of health. + scope = "" if {"model", "tools"} <= observed else " in the categories checked" + lines.append(" Verified: no composition changes%s." % scope) else: symbols = {"added": "+", "removed": "-", "changed": "~"} for change in changes: