From 53aa7bfc546ddc55ba3aeee0a77071ffe22a938a Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:24:31 +0000 Subject: [PATCH 1/9] CLI: `tracekit help` shows every command, grouped, and the main features `tracekit --help` printed every subcommand as one flat list in the order they were registered, and `tracekit` alone was an argparse error. - `tracekit help` and a bare `tracekit` print an overview: each command under what a person runs it for, one entry each from its own help text (up to where its usage starts), experimental ones marked as their help marks them, then the features that are not commands (witnesses and anchoring, Tracekit Box, the SDKs and integrations) with their pages. - `tracekit help NAME` is `tracekit NAME --help`; an unknown name lists the commands and exits 2. - The parser is built in `_parser()` so the overview and the tests read the registered commands; tests/test_cli.py checks every command is in exactly one group. - docs/quickstart-v2.md names `tracekit help`. --- docs/quickstart-v2.md | 3 ++ tests/test_cli.py | 62 +++++++++++++++++++++++++++ tracekit/cli.py | 98 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 tests/test_cli.py diff --git a/docs/quickstart-v2.md b/docs/quickstart-v2.md index 2e8cf4cf..1b638670 100644 --- a/docs/quickstart-v2.md +++ b/docs/quickstart-v2.md @@ -31,6 +31,9 @@ venv the agent already has. pipx and `uv tool` give you the `tracekit` command then run from that install), but nothing your agent can import. The command and the SDK can come from two installs: both use the same dev signer. Keep them at the same version. +`tracekit help` lists every command, grouped by what you'd use it for, and the features that aren't commands of their +own; `tracekit help ` shows a command's options. + ### Upgrading `pip install -U tracekit-ai` upgrades the environment you run it in. Two other places can keep the old version, and diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..64209092 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,62 @@ +"""`tracekit help` and a bare `tracekit`: every command in one group, and `help NAME` as NAME's own --help. +python3 -m pytest tests/test_cli.py -q""" +import contextlib +import io +import unittest + +from tracekit import cli + + +def run(argv): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + code = cli.main(argv) + except SystemExit as e: + code = e.code + return code, out.getvalue(), err.getvalue() + + +COMMANDS = sorted(cli._parser()[1].choices) + + +class Help(unittest.TestCase): + def test_every_command_is_in_exactly_one_group(self): + grouped = sorted(c for _, names in cli._GROUPS for c in names.split()) + self.assertEqual(grouped, sorted(set(COMMANDS) - {"help"})) + + def test_the_overview_names_every_group_and_command_within_80_columns(self): + code, out, err = run(["help"]) + self.assertEqual((code, err), (0, "")) + lines = out.splitlines() + self.assertLessEqual(max(map(len, lines)), 80) + for title, names in cli._GROUPS: + self.assertIn(title, lines) + for name in names.split(): + self.assertTrue(any(line.split()[:1] == [name] for line in lines), name) + self.assertIn(" verify verify a .tkb offline", lines) + self.assertIn(" report what the agents did this week, on one page, from the v2", lines) + self.assertIn(" why causal investigation and replay of multi-agent runs", lines) + self.assertIn(" (experimental)", lines) + self.assertEqual(lines[-1], "`tracekit help ` shows a command's options.") + + def test_bare_tracekit_prints_the_overview(self): + self.assertEqual(run([]), run(["help"])) + + def test_help_command_is_the_commands_own_help(self): + for name in COMMANDS: + with self.subTest(name=name): + code, out, err = run(["help", name]) + self.assertEqual(code, 0) + self.assertIn(f"usage: tracekit {name}", out) + self.assertEqual((code, out, err), run([name, "--help"])) + + def test_an_unknown_command_lists_the_names_and_exits_2(self): + code, out, err = run(["help", "nosuch"]) + self.assertEqual((code, out), (2, "")) + self.assertTrue(err.startswith("tracekit help: no command named 'nosuch'; the commands are:\n"), err) + self.assertEqual(err.replace("\n", " ").split(":", 2)[2].replace(",", " ").split(), COMMANDS) + + +if __name__ == "__main__": + unittest.main() diff --git a/tracekit/cli.py b/tracekit/cli.py index 6b8d19a6..ff786b6d 100644 --- a/tracekit/cli.py +++ b/tracekit/cli.py @@ -101,10 +101,89 @@ def main(argv=None): return run(args[1:]) finally: experimental(_EXPERIMENTAL[args[0]], help=True) + ap, sub = _parser() + a = ap.parse_args(argv) + try: + if a.cmd in (None, "help"): + return _help(ap, sub, getattr(a, "command", None)) + return _run(a) + except KeyboardInterrupt: + print("interrupted", file=sys.stderr) + return 130 + except BrokenPipeError: + return 0 + except SystemExit: + raise + except Exception as e: # a CLI should print a reason, not a traceback; TRACEKIT_DEBUG=1 restores it + if os.environ.get("TRACEKIT_DEBUG") == "1": + raise + print(f"tracekit: {type(e).__name__}: {e}", file=sys.stderr) + return 1 + + +# `tracekit help` and a bare `tracekit`: each command under what a person runs it for (tests/test_cli.py checks that +# every command is in exactly one group), then what Tracekit does that is not a command of its own +_GROUPS = ( + ("Start here", "init up down status doctor demo"), + ("See what agents did", "view last report audit why graph cost analyze observe"), + ("Approvals", "approvals pending approve reject"), + ("Evidence", "export verify verify-model pcm"), + ("Policy", "policy"), + ("Run it on a server", "signer deploy issuer monitor reviewer witness public-witness daemon ingest"), + ("Record model, tool and network traffic", "gateway executor mcp-gateway a2a-gateway egress proxy"), + ("Connect other tools", "follow srt-decider openshell-middleware ext-mcp sensor otel"), + ("Maintenance", "migrate uninstall"), +) +_FEATURES = ( + ("Witnesses and anchoring: other parties cosign the signer's log, and Rekor and RFC 3161 timestamps anchor it " + "(the free public witness is not deployed yet)", wiki_url("docs/witnesses.md")), + ("Tracekit Box, a separate program: runs an agent in its own micro-VM whose only ways out are decided and signed", + "https://github.com/Cygnux-Labs/Tracekit-box"), + ("SDKs and integrations: tracekit.instrument() and the Python SDK, the TypeScript SDK, framework integrations " + "and coding-agent hooks", wiki_url("docs/integrations.md")), +) + + +def _help(ap, sub, name): + """`tracekit help NAME` is `tracekit NAME --help`; without NAME, every command in its group, each with its own help + up to where its usage starts, then the features.""" + import textwrap + if name in sub.choices: + return main([name, "--help"]) + if name: + print(f"tracekit help: no command named {name!r}; the commands are:\n" + + textwrap.fill(", ".join(sorted(sub.choices)), 80, initial_indent=" ", subsequent_indent=" ", + break_on_hyphens=False), file=sys.stderr) + return 2 + helps = {x.dest: x.help for x in sub._choices_actions} + lines = [f"tracekit {__version__}: {ap.description}"] + for title, names in _GROUPS: + lines += ["", title] + for n in names.split(): + text = re.split(r"; |: `", helps[n])[0] + if n in _EXPERIMENTAL and "experimental" not in text: + text += " (experimental)" + first = f" {n:<14} " + if len(first) > 18: # a name longer than the column gets a line of its own + lines.append(first.rstrip()) + first = " " * 18 + lines.append(textwrap.fill(text, 80, initial_indent=first, subsequent_indent=" " * 18, + break_on_hyphens=False)) + lines += ["", "Also in Tracekit"] + for text, url in _FEATURES: + lines += [textwrap.fill(text, 80, initial_indent=" ", subsequent_indent=" "), f" {url}"] + print("\n".join(lines + ["", "`tracekit help ` shows a command's options."])) + return 0 + + +def _parser(): + """tracekit's argument parser and its subcommands.""" ap = argparse.ArgumentParser(prog="tracekit", description="Tamper-evident records of what AI agents did.") ap.add_argument("--version", action="version", version=f"tracekit {__version__}") - sub = ap.add_subparsers(dest="cmd", required=True, metavar="command") + sub = ap.add_subparsers(dest="cmd", metavar="command") + p = sub.add_parser("help", help="every command, grouped by what it is for; `help COMMAND` shows its options") + p.add_argument("command", nargs="?") p = sub.add_parser("init", help="install the signer and Claude Code hooks") p.add_argument("--remote", metavar="URL", help="agent on another machine: with --v2, send to this signer's HTTPS " f"transport; without, to the deprecated ingest gateway ({wiki_url('docs/remote-ingest.md')})") @@ -366,22 +445,7 @@ def main(argv=None): help="`tracekit why` on the signer: a 3-agent desk, a held email, its cause found and cited") p.add_argument("--agent", default="claude", choices=demo.AGENTS, help="send the scripted run as this coding agent's own hook payloads (default: claude)") - - a = ap.parse_args(argv) - try: - return _run(a) - except KeyboardInterrupt: - print("interrupted", file=sys.stderr) - return 130 - except BrokenPipeError: - return 0 - except SystemExit: - raise - except Exception as e: # a CLI should print a reason, not a traceback; TRACEKIT_DEBUG=1 restores it - if os.environ.get("TRACEKIT_DEBUG") == "1": - raise - print(f"tracekit: {type(e).__name__}: {e}", file=sys.stderr) - return 1 + return ap, sub def _init_remote(a): From dee5e3c70bfd534a24346866065d15753ca8c62a Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:56:13 +0000 Subject: [PATCH 2/9] First-day fixes: plainer messages, a Claude Code quickstart, and an uninstall that says what it keeps A new-user walkthrough of main in a clean container found these; each is fixed at its cause. - The dev-mode line says how to guard (system mode, Tracekit Box), with links. - doctor: no Claude Code and no witness are info in dev, not warnings; a healthy dev run ends with "dev setup OK"; the system-mode fix names the wiki and a Python init accepts. - init --dev --v2 ends with a next step, and points to lines that exist today. - tracekit last with no runs says instrument() records only the frameworks it lists. - Framework quickstarts lead with their no-key --scripted route and name what the live example needs (langchain-openai and OPENAI_API_KEY, ANTHROPIC_API_KEY). - docs/quickstarts/claude-code.md: init, a new session, a hold, answering with !, retry, last, view, uninstall. - tracekit pending lists the v2 signer's held calls when there is no v1 signer. - uninstall lists what it keeps with the commands that delete it, deletes a settings file init made that is now empty, and its help no longer says "ledger". - approvals show gives the cause the hook gives; an unclosed quote has its own cause; the Claude Code `!` hint is for Claude Code runs only; list and show give expiry as UTC time and minutes left. - view: --approval's help says it starts the viewer; plain start lines; a clickable 127.0.0.1 link for an any-address bind; --host and --port help. - verify: an empty run says "no tool calls recorded". - status prints a summary; --json keeps the JSON. - The hook command for a checkout or --user install is clear text, not base64. - quickstart-v2: a short step 2, the server pattern, the wait table and socket details under Deeper, a Remove it section, and the scripted snippet says when the wait ran out. - Help text for up, down, policy, uninstall, observe; ingest, otel and daemon say experimental. - Nits: demo --server's approval line, report's side_effect label, last says when it waits. --- docs/doctor.md | 8 +- docs/integrations.md | 2 +- docs/portability.md | 2 +- docs/quickstart-v2.md | 147 +++++++++++++++--------- docs/quickstarts/README.md | 3 +- docs/quickstarts/claude-agent-sdk.md | 16 ++- docs/quickstarts/claude-code.md | 96 ++++++++++++++++ docs/quickstarts/langchain.md | 20 +++- docs/quickstarts/mcp.md | 19 ++- docs/quickstarts/openai-agents.md | 16 ++- docs/walkthrough.md | 2 +- plugin/README.md | 4 +- tests/test_bundle_v2.py | 3 +- tests/test_dev_mode_line.py | 18 ++- tests/test_doctor_v2.py | 74 +++++++++++- tests/test_hook_invocation.py | 25 ++++ tests/test_hook_v2.py | 36 +++++- tests/test_install_files.py | 4 +- tests/test_instrument.py | 2 +- tests/test_policy2.py | 8 +- tests/test_quickstart.py | 2 +- tests/test_remote_tokens.py | 3 +- tests/test_sdk_client.py | 6 +- tests/test_signer_approvals.py | 50 +++++++- tests/test_signer_dev.py | 21 +++- tests/test_system_v2.py | 7 +- tests/test_verify_short.py | 12 +- tests/test_view.py | 54 +++++++-- tracekit/cli.py | 165 ++++++++++++++++++++++----- tracekit/demo_server.py | 5 +- tracekit/deploy/files.py | 6 + tracekit/doctor.py | 39 +++++-- tracekit/install.py | 61 ++++++---- tracekit/integrations/claude_code.py | 18 ++- tracekit/policy2/compile.py | 2 + tracekit/policy2/shell.py | 10 +- tracekit/sdk/autospawn.py | 15 +++ tracekit/verify/__init__.py | 3 +- tracekit/view.py | 32 +++--- 39 files changed, 822 insertions(+), 194 deletions(-) create mode 100644 docs/quickstarts/claude-code.md diff --git a/docs/doctor.md b/docs/doctor.md index e03fd69d..a037da69 100644 --- a/docs/doctor.md +++ b/docs/doctor.md @@ -4,7 +4,7 @@ sudo /opt/tracekit/bin/python -I -m tracekit doctor # system mode: run it as root, so it can probe as the agent's user tracekit doctor # the v2 dev signer tracekit doctor --config signer.yaml # one v2 signer, by its config -tracekit doctor --json # [{"id", "status": "ok" | "warn" | "fail", "detail", "fix"}] +tracekit doctor --json # [{"id", "status": "ok" | "info" | "warn" | "fail", "detail", "fix"}] tracekit doctor --k8s # inside a pod: the pod and its service account, read from the API server tracekit doctor --k8s --manifests DIR # rendered manifests (helm template, kustomize build) instead tracekit doctor --config signer.yaml --issuer-key alias/tracekit-ca # also: may this principal sign with the CA key? @@ -20,7 +20,7 @@ Doctor works out which setup it's looking at and runs the checks that apply: - **`--k8s`:** the Kubernetes checks below, of the pod doctor runs in (it needs `get` on its pod and service account) or of the manifests under `--manifests DIR`. With `--config` it runs that signer's checks too. -Exit codes: 1 when any check fails, else 0. A warning is advice: it doesn't fail doctor. +Exit codes: 1 when any check fails, else 0. A warning is advice: it doesn't fail doctor. `info` says what a setup doesn't have and needn't (the dev signer's witnesses, Claude Code's hooks where there is no Claude Code). A dev run whose only warning is `D-PROCESS-BOUNDARY` ends with `dev setup OK`. **Doctor output is advice, not evidence.** It describes the host at the moment you run it. Nothing it prints is signed, and a verifier never reads it. What a bundle proves comes from its signatures, its checkpoints and the verifier's own @@ -51,7 +51,7 @@ skipped with the reason) and checks that doctor flags each one. It also checks t | `D-KEY-HYGIENE` | `data_dir/hygiene.json`, written by the signer on start | the signer process may dump core (RLIMIT_CORE not 0), is dumpable (Linux `PR_SET_DUMPABLE`), or could not mlock its key files (warn; also when the signer never started) | run it with `tracekit signer serve`; raise `LimitMEMLOCK` | | `D-UNIT-HARDENING` | `tracekit-signer.service` (or the launchd plist) | a hardening directive that init writes is missing: the detail gives the score and lists the missing directives (warn) | re-run init to rewrite the service file | | `D-HARNESS-HELPER` | `tracekit-harness-helper.service` (or its launchd plist), when signer.yaml has `harness_binding` | the file is missing, or its capability bounding set is anything but `CAP_SYS_PTRACE CAP_DAC_READ_SEARCH` | re-run init with `--harness` | -| `D-HOOKS-PRESENT` | the agent's Claude Code settings, and whether they enable the Tracekit plugin (`enabledPlugins`, whose hooks count); Codex's, Gemini CLI's, Cursor's, goose's, Qwen Code's, Factory Droid's, Antigravity CLI's and Crush's v2 hooks and OpenCode's v2 plugin count too | a tool event (PreToolUse, PostToolUse, PostToolUseFailure) has no v2 hook, the plugin isn't enabled and no other coding agent has v2 hooks (not when `tracekit init --v2 --no-hooks`, dev or system mode, recorded that you chose none: the check passes and says how to add them); or the plugin and init's hooks both run, so every approved call is blocked; or (warn) with `init --v2 --managed`, the agent user's own `~/.claude/settings.json` has Tracekit's v2 hooks too, so every call is decided twice (not with `allowManagedHooksOnly` on, which stops them running: the check says so and passes) | re-run init; with both, keep one (`tracekit uninstall`, or `/plugin disable` with the plugin key the check names, e.g. `tracekit@tracekit`); with `--managed`, re-run init with `--managed`, which takes them out of the agent user's file | +| `D-HOOKS-PRESENT` | the agent's Claude Code settings, and whether they enable the Tracekit plugin (`enabledPlugins`, whose hooks count); Codex's, Gemini CLI's, Cursor's, goose's, Qwen Code's, Factory Droid's, Antigravity CLI's and Crush's v2 hooks and OpenCode's v2 plugin count too | a tool event (PreToolUse, PostToolUse, PostToolUseFailure) has no v2 hook, the plugin isn't enabled and no other coding agent has v2 hooks (not when `tracekit init --v2 --no-hooks`, dev or system mode, recorded that you chose none: the check passes and says how to add them; dev: info when there is no Claude Code, no `claude` on PATH and no `~/.claude`); or the plugin and init's hooks both run, so every approved call is blocked; or (warn) with `init --v2 --managed`, the agent user's own `~/.claude/settings.json` has Tracekit's v2 hooks too, so every call is decided twice (not with `allowManagedHooksOnly` on, which stops them running: the check says so and passes) | re-run init; with both, keep one (`tracekit uninstall`, or `/plugin disable` with the plugin key the check names, e.g. `tracekit@tracekit`); with `--managed`, re-run init with `--managed`, which takes them out of the agent user's file | | `D-HOOKS-MANAGED` | with `init --v2 --managed`: Claude Code's admin-managed settings file, which holds the hooks | the file, or a directory above it, isn't root-owned or is group- or world-writable, so the agent's user could replace it; or (as root) the agent's user can't read it, so its Claude Code starts without it | `sudo chown 0:0` and `sudo chmod 644` the file, `sudo chmod go-w` the directories above it; or re-run init with `--managed` (and `--managed-only` when an init with it turned `allowManagedHooksOnly` on) | | `D-HOOKS-EVENT` | Claude Code, Codex, Cursor, Gemini CLI, goose, Qwen Code, Factory Droid, Antigravity CLI and Crush hook commands (OpenCode's JavaScript plugin isn't checked); for Claude Code also each Tracekit hook's `matcher` and `disableAllHooks` in the settings files doctor reads (dev mode: the user file, the file init recorded, the working directory's project and local files, and managed settings; system mode: the agent user's file, the file init wrote and managed settings; with `--managed`, managed settings only, as only they can turn managed hooks off) | a hook command doesn't name its configured event (installed before 1.2), so a pre-tool payload the hook can't read lets the call through unrecorded; a hook other than the pre-tool one blocks its event when it can't run (a command init wrote before 1.3, or a Cursor hook with `failClosed` on that event, as init wrote up to 1.4.2), so a broken install blocks prompts and Stop too; or, for the other coding agents, a call can run that no hook decided: no Tracekit v2 hook is on the pre-tool event, Cursor's `preToolUse` hook has no `failClosed: true`, or goose's `PreToolUse` hook has no `on_failure: "block"`; or, for Claude Code, a call can run that no hook decided: `disableAllHooks` is set in one of those files, so no hook runs (fail in production), a Tracekit hook's matcher isn't `*`, empty or absent, so it skips the other tools, or the PreToolUse command isn't wrapped to block the call when the hook can't run (POSIX) (warn) | re-run init for the active mode: `tracekit init --dev --v2 [--agent NAME]` in dev, or `sudo /opt/tracekit/bin/python -I -m tracekit init --v2 --user AGENT [--agent NAME]` in system mode (with Python 3.9 or 3.10, [laptop system mode](deploy.md#laptop-system-mode)'s command); remove `disableAllHooks` from the file the check names | | `D-HOOKS-VENV` | the hook commands | a hook isn't the command init writes for the root-owned venv: another Python, or written by init before 1.3 | re-run init | @@ -69,7 +69,7 @@ skipped with the reason) and checks that doctor flags each one. It also checks t | `D-POLICY-ENGINE` | the policy, compiled by policy2 | it doesn't compile, or neither `google-re2` nor `regex` is installed | `tracekit policy lint FILE`; `pip install tracekit-ai` (a `--no-deps` install leaves the engine out) | | `D-DURABILITY` | `durability` | it's `ack-on-write` in a production profile, so a power loss can drop records that were already acknowledged (warn) | `durability: ack-on-fsync` | | `D-FAIL-MODES` | `fail_modes` and `client.json` `fail_mode` | a tool class, or the system config, fails open (warn) | set them to `closed` unless an outage must not stop the agent | -| `D-WITNESS-CONFIGURED` | `witnesses` | there are none, so checkpoints aren't cosigned and assurance stays `dev` (`local` when `anchors` is set) (dev: warn) | add a witness (`docs/witnesses.md`) | +| `D-WITNESS-CONFIGURED` | `witnesses` | there are none, so checkpoints aren't cosigned and assurance stays `dev` (`local` when `anchors` is set) (dev: info, as the dev signer takes none) | add a witness (`docs/witnesses.md`) | | `D-WITNESS-OWNERSHIP` | the witnesses' ownership classes | every witness is `operator`: the default production profile needs at least one `public`, `customer` or `tracekit` witness (dev: warn) | add a non-operator witness | | `D-WITNESS-FRESH` | `store/witness-queue.json` against the latest checkpoint | a witness has never cosigned the latest checkpoint, or has been failing for longer than the witness gap window (300 s) (warn) | check the witness is reachable and accepts this log | | `D-CLOCK-SKEW` | this clock against the latest cosignature timestamp | this clock is more than 300 s behind it (warn) | sync the clock (NTP) | diff --git a/docs/integrations.md b/docs/integrations.md index 37eae8d9..70485301 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -32,7 +32,7 @@ Another agent: record its calls with the SDK's run handle ([custom agent](quicks | Agent | Status | Guide | |---|---|---| -| Claude Code hooks (`tracekit init --dev --v2`) | Stable | [quickstart](quickstart-v2.md) | +| Claude Code hooks (`tracekit init --dev --v2`) | Stable | [quickstart](quickstarts/claude-code.md) | | The Claude Code plugin | Experimental | [plugin](../plugin/README.md) | | Codex CLI, Cursor, Gemini CLI, OpenCode | Experimental | [coding agents](coding-agents.md) | | goose (`tracekit init --dev --v2 --agent goose`) | Experimental | [coding agents](coding-agents.md#on-the-v2-signer) | diff --git a/docs/portability.md b/docs/portability.md index 17c04cd2..76c736c6 100644 --- a/docs/portability.md +++ b/docs/portability.md @@ -38,7 +38,7 @@ Things to check on first use, because they depend on the machine: - The Python interpreter and the `tracekit` package must be readable by `_tracekit`. A Python under your home directory is not, and the daemon will fail to start; look at `/var/lib/tracekit/tracekitd.log`. -- `tracekit status` should show `signer_isolation: separate-user` and the ledger owned by `_tracekit`. +- `tracekit status --json` should show `signer_isolation: separate-user` and the ledger owned by `_tracekit`. - A quick isolation test: `echo x >> /var/lib/tracekit/ledger/ledger.jsonl` as your own user must fail. If any of that is wrong, please open an issue with the output. Until it has been validated on hardware the diff --git a/docs/quickstart-v2.md b/docs/quickstart-v2.md index 1b638670..daa29c33 100644 --- a/docs/quickstart-v2.md +++ b/docs/quickstart-v2.md @@ -58,16 +58,86 @@ A hook whose signer runs another version says so too, once per session, with the import tracekit; tracekit.instrument() # the first line, before your agent builds its agents, tools or sessions ``` -Then run your agent as usual. `instrument()` registers a run with the signer and closes it when the process exits. It -wires every framework it finds installed: OpenAI Agents SDK tools, LangGraph / LangChain `ToolNode`s (which -`create_agent` builds), Claude Agent SDK hooks, MCP `ClientSession.call_tool`, Browser Use's `Tools` (which an `Agent` -builds), CrewAI's tool hooks, Pydantic AI `Agent`s (the `Tracekit` capability), and the OpenAI, Anthropic and Google Gen AI clients' model calls. Each tool call is decided by the signer -before it runs (`allow`, `deny`, `ask`) and recorded after. A denied call reaches the model as a refusal and the agent -goes on. Each model response is committed to and, with the signer's content vault on (the dev signer keeps it), kept -in the vault, so `tracekit analyze --v2` can read what the model said before and after each call ([findings](findings.md#v2-runs-what-was-asked-said-and-done-tracekit-analyze---v2)). -Calling it again returns the same run. With none of these installed it warns and does nothing. TypeScript: -`const tk = await instrument()` from `@cygnuxlabs/tracekit`, then pass `tk.ai` (Vercel AI SDK) or `tk.agents` (OpenAI Agents -JS) to the framework. +Then run your agent as usual. You don't start anything: the first call starts a dev signer in the background. +`instrument()` records only the frameworks it finds installed: OpenAI Agents SDK tools, LangGraph / LangChain +`ToolNode`s (which `create_agent` builds), Claude Agent SDK hooks, MCP `ClientSession.call_tool`, Browser Use's +`Tools`, CrewAI's tool hooks, Pydantic AI `Agent`s, and the OpenAI, Anthropic and Google Gen AI clients' model calls. +With none of them installed it warns and records nothing: record your agent's calls yourself +([a custom agent](quickstarts/custom.md)). + +Each tool call is decided before it runs (`allow`, `deny` or `ask`) and recorded after. A denied call reaches the model +as a refusal and the agent goes on. A held call (`ask`) waits for a person: run `tracekit approvals list` in another +terminal to see it, and `tracekit approvals approve ` to let it run once ([approve from another +terminal](#approve-from-another-terminal)). While it waits, an SDK agent prints nothing: it just seems to stop. + +Claude Code: `tracekit init --dev --v2` instead ([the Claude Code quickstart](quickstarts/claude-code.md)). +TypeScript: `const tk = await instrument()` from `@cygnuxlabs/tracekit`, then pass `tk.ai` (Vercel AI SDK) or +`tk.agents` (OpenAI Agents JS) to the framework. A server that handles many conversations, how each framework waits +for an approval and where the signer listens: [deeper](#deeper-drive-the-signer-yourself). + +## 3. See it verified + +```sh +tracekit last +``` + +```text +run 40c7ee89ae9eaec8f8eb8556b32fdc9d (my-agent) +VERIFIED: tool calls in signed records, none changed, removed or added after signing. +NOT PROOF AGAINST THE AGENT: in this setup the agent can reach the signing key, so it could have rewritten the record. +To make it proof, keep the key away from the agent: https://github.com/Cygnux-Labs/Tracekit/wiki/Deploy +bundle: /home/me/project/40c7ee89ae9eaec8f8eb8556b32fdc9d.tkb +check it again: tracekit verify 40c7....tkb --trust 40c7....trust.json +next: tracekit view --dev (every run, replayed and reviewed) +``` + +`tracekit last` finds the dev signer's most recent finished run, exports it, writes a trust config that pins the dev +signer's log key next to the bundle, and verifies the bundle. `-o PATH` writes the bundle elsewhere. The report is +explained [below](#export-and-verify-by-hand). `tracekit view --dev` lists each run with its verdict, and replays and +reviews one ([viewer.md](viewer.md#laptop-viewer)). `tracekit status` says what is set up and where. + +Per framework, the same three steps with a 10-line example: [LangChain / LangGraph](quickstarts/langchain.md), +[OpenAI Agents SDK](quickstarts/openai-agents.md), [Claude Agent SDK](quickstarts/claude-agent-sdk.md), +[MCP client](quickstarts/mcp.md), [Claude Code](quickstarts/claude-code.md), [a custom agent](quickstarts/custom.md). + +## First-run errors + +| You see | What to do | +|---|---| +| `tracekit.instrument(): no signer answering at ...; dev auto-spawn is off while TRACEKIT_SIGNER is set` | Start the signer `TRACEKIT_SIGNER` names, or unset it for a same-user dev signer. | +| `tracekit.instrument(): ...; system mode: start the system signer` | `sudo /opt/tracekit/bin/python -I -m tracekit doctor` says what is wrong with the system signer. | +| `the Tracekit signer X (pid N, protocol [a, b]) ... cannot serve this client` | A dev signer from a newer Tracekit version is running (an older one is replaced on its own): `tracekit up --replace` stops it and starts this one. A signer you run yourself (`TRACEKIT_SIGNER`, system mode) that is older than this client: upgrade the signer first. From the TypeScript SDK, an older `tracekit` on your PATH starts an older signer: `pip install -U tracekit-ai`. | +| `tracekit last: the newer run ID (agent) is still open` | The agent is still running, or was killed before it closed its run. End it and run `tracekit last` again; meanwhile it shows the run before. | +| `tracekit last: no runs in the dev signer's store` | `instrument()` records only the frameworks it lists, and warns when it finds none of them: for another agent, record its calls as [the custom quickstart](quickstarts/custom.md) shows. Claude Code: start a new session after `tracekit init --dev --v2`. | +| `tracekit.instrument(): not wired ... Tracekit is tested with ...` | Install the versions it names, or wire that adapter by hand as its quickstart shows. | +| `the v2 signer needs its policy engine ... pip install tracekit-ai` | A `--no-deps` install verifies but can't sign: install with its dependencies. | +| A path with spaces (Windows profiles, macOS `Application Support`) | Paste the commands `tracekit last` prints as they are: their paths are quoted for your shell. | + +## Remove it + +```sh +tracekit uninstall # Claude Code's hooks; --agent NAME for another coding agent's +``` + +It removes Tracekit's hooks and keeps your other settings (a settings file init made, with nothing else in it, is +deleted). Then it lists what it kept: the dev signer's keys and signed runs, Tracekit's client config and the dev +signer's runtime dir, with the commands that delete them and Tracekit too, for example on Linux: + +```text +kept: + /home/me/.local/share/tracekit (the dev signer's keys and signed runs: a bundle you exported verifies without them) + /home/me/.tracekit-client (Tracekit's client config) +to delete these and Tracekit too: + tracekit down + rm -rf /home/me/.local/share/tracekit /home/me/.tracekit-client + /home/me/.venv/bin/python -m pip uninstall tracekit-ai +``` + +## Deeper: drive the signer yourself + +What `instrument()` and `tracekit last` do, by hand and in more detail: a server with a run per conversation, how each +framework waits for an approval, where the signer listens, and a scripted agent with the approval step a framework's +own control flow needs. ### A server: a run per conversation @@ -124,6 +194,8 @@ agent's retry of the same call runs once you approved it ([held calls](coding-ag | Vercel AI SDK (TS, `tk.ai`) | the SDK's own tool approval | none of its own | `approvalResponses(parts)`, pushed to `messages`, resumes it; a call not approved throws | | `tracekit.why` runtime | timed wait in `act()` | `approval_wait_s`, 300 s | an error result; retrying the same call in the same run reuses the approval | +### Where the signer runs + You don't start anything yourself. The first client that needs a signer starts one in the background and later clients reuse it (`tracekit up` / `tracekit down` do the same by hand). The signer's keys and store live in your user's data dir, one per `HOME`. It listens on a Unix socket in a private runtime dir; where two of your `HOME`s share that @@ -134,49 +206,6 @@ names. On Windows it listens on a loopback TCP port instead, and publishes the port and a token in `%LOCALAPPDATA%\tracekit\run\endpoint.json`, which only you can read. Each side proves it holds the token before any request is answered. -## 3. See it verified - -```sh -tracekit last -``` - -```text -run 40c7ee89ae9eaec8f8eb8556b32fdc9d (my-agent) -VERIFIED: tool calls in signed records, none changed, removed or added after signing. -NOT PROOF AGAINST THE AGENT: in this setup the agent can reach the signing key, so it could have rewritten the record. -To make it proof, keep the key away from the agent: https://github.com/Cygnux-Labs/Tracekit/wiki/Deploy -bundle: /home/me/project/40c7ee89ae9eaec8f8eb8556b32fdc9d.tkb -check it again: tracekit verify 40c7....tkb --trust 40c7....trust.json -next: tracekit view --dev (every run, replayed and reviewed) -``` - -`tracekit last` finds the dev signer's most recent finished run, exports it, writes a trust config that pins the dev -signer's log key next to the bundle, and verifies the bundle. `-o PATH` writes the bundle elsewhere. The report is -explained [below](#export-and-verify-by-hand). `tracekit view --dev` lists each run with its verdict, and replays and -reviews one ([viewer.md](viewer.md#laptop-viewer)). - -Per framework, the same three steps with a 10-line example: [LangChain / LangGraph](quickstarts/langchain.md), -[OpenAI Agents SDK](quickstarts/openai-agents.md), [Claude Agent SDK](quickstarts/claude-agent-sdk.md), -[MCP client](quickstarts/mcp.md), [a custom agent](quickstarts/custom.md). - -## First-run errors - -| You see | What to do | -|---|---| -| `tracekit.instrument(): no signer answering at ...; dev auto-spawn is off while TRACEKIT_SIGNER is set` | Start the signer `TRACEKIT_SIGNER` names, or unset it for a same-user dev signer. | -| `tracekit.instrument(): ...; system mode: start the system signer` | `sudo /opt/tracekit/bin/python -I -m tracekit doctor` says what is wrong with the system signer. | -| `the Tracekit signer X (pid N, protocol [a, b]) ... cannot serve this client` | A dev signer from a newer Tracekit version is running (an older one is replaced on its own): `tracekit up --replace` stops it and starts this one. A signer you run yourself (`TRACEKIT_SIGNER`, system mode) that is older than this client: upgrade the signer first. From the TypeScript SDK, an older `tracekit` on your PATH starts an older signer: `pip install -U tracekit-ai`. | -| `tracekit last: the newer run ID (agent) is still open` | The agent is still running, or was killed before it closed its run. End it and run `tracekit last` again; meanwhile it shows the run before. | -| `tracekit last: no runs in the dev signer's store` | Add `import tracekit; tracekit.instrument()` to your agent and run it first. | -| `tracekit.instrument(): not wired ... Tracekit is tested with ...` | Install the versions it names, or wire that adapter by hand as its quickstart shows. | -| `the v2 signer needs its policy engine ... pip install tracekit-ai` | A `--no-deps` install verifies but can't sign: install with its dependencies. | -| A path with spaces (Windows profiles, macOS `Application Support`) | Paste the commands `tracekit last` prints as they are: their paths are quoted for your shell. | - -## Deeper: drive the signer yourself - -The rest of this page does by hand what `instrument()` and `tracekit last` do, with the approval step a framework's -own control flow needs. - ### A scripted agent Before each tool call runs, the agent asks the signer. The signer answers `allow`, `deny` or `ask` and signs that @@ -195,9 +224,11 @@ with Client().run(agent="quickstart") as run: d = run.decide("call-3", "Bash", held) # ask: a command the signer can't parse waits for a human aid = run.call("approval_request", tool_call_id="call-3")["approval_id"] print("approve with: tracekit approvals approve", aid) - if run.call("approval_wait", approval_id=aid, timeout_ms=300000)["state"] == "approved" and \ - run.approval_consume("call-3", "Bash", held, approval_id_hint=aid)["ok"]: + state = run.call("approval_wait", approval_id=aid, timeout_ms=300000)["state"] # waits up to 5 minutes + if state == "approved" and run.approval_consume("call-3", "Bash", held, approval_id_hint=aid)["ok"]: run.complete("call-3") # run it only now, with exactly those arguments + else: # still "requested": nobody answered in time + print("call-3 not run:", "no answer within 5 minutes" if state == "requested" else state) print("run", run.run_id) ``` @@ -210,7 +241,11 @@ decision then names the arguments whose values only untrusted content brought in ([provenance](policy-v2.md#provenance-untrusted-and-from-untrusted)). Send the task as `trusted` from your app, not from the agent. -Claude Code: `tracekit init --dev --v2` wires its hooks to the same signer (`python -I -m tracekit.integrations.claude_code`). +Claude Code: `tracekit init --dev --v2` wires its hooks to the same signer ([the Claude Code +quickstart](quickstarts/claude-code.md)). Each hook command is plain text in `~/.claude/settings.json`: the Python +init ran with, then `-I -m tracekit.integrations.claude_code `; from a source checkout or a `pip install +--user`, `-I -c` with a one-line import that puts that directory first on the path, then the directory (and a +checkout's branch) and the event as quoted arguments. `tracekit demo --server` runs the whole loop in a temp dir, with a test witness's cosignature and a tampered copy that fails. A deployment for server-hosted agents: [deploy.md](deploy.md). diff --git a/docs/quickstarts/README.md b/docs/quickstarts/README.md index e46d4d09..551737a5 100644 --- a/docs/quickstarts/README.md +++ b/docs/quickstarts/README.md @@ -6,7 +6,8 @@ reference page covers the rest. | Integration | Quickstart | Reference | |---|---|---| -| Coding agents | | [coding-agents.md](../coding-agents.md) | +| Claude Code | [claude-code.md](claude-code.md) | [the plugin](../../plugin/README.md) | +| Other coding agents (Codex, Cursor, Gemini CLI, ...) | | [coding-agents.md](../coding-agents.md) | | Hermes Agent | | [integrations/hermes.md](../integrations/hermes.md) | | OpenAI Agents SDK | [openai-agents.md](openai-agents.md) | [integrations/openai-agents.md](../integrations/openai-agents.md) | | Claude Agent SDK | [claude-agent-sdk.md](claude-agent-sdk.md) | [integrations/claude-agent-sdk.md](../integrations/claude-agent-sdk.md) | diff --git a/docs/quickstarts/claude-agent-sdk.md b/docs/quickstarts/claude-agent-sdk.md index 5f24d325..48180a11 100644 --- a/docs/quickstarts/claude-agent-sdk.md +++ b/docs/quickstarts/claude-agent-sdk.md @@ -4,10 +4,23 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no API key + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'claude-agent-sdk>=0.2.165,<0.3' +python examples/v2/claude_agent_sdk/agent.py --scripted +``` + +A mock CLI stands in for Claude: `ls` is allowed, `sudo rm -rf /` is denied and the agent goes on, and `echo +"unterminated` is held until a second process approves it. Then the run is exported and verified. + ## 1. Install ```sh pip install tracekit-ai 'claude-agent-sdk>=0.2.165,<0.3' +export ANTHROPIC_API_KEY=... # the SDK runs the Claude Code it bundles, which calls Claude ``` ## 2. One line in your agent @@ -55,8 +68,7 @@ async for message in query(prompt="Use the Bash tool to list the files here.", o The example runs a mock CLI (`transport=`) so it needs no `claude` binary and no API key. An ask holds the call inside the PreToolUse hook for up to nine minutes; SessionEnd closes the run. -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/claude_agent_sdk/agent.py ---scripted` from a checkout. +The no-key example at the top of this page wires it this way, with an approval. ## Read the verify report diff --git a/docs/quickstarts/claude-code.md b/docs/quickstarts/claude-code.md new file mode 100644 index 00000000..5b4f54e9 --- /dev/null +++ b/docs/quickstarts/claude-code.md @@ -0,0 +1,96 @@ +# Quickstart: Claude Code + +Every tool call Claude Code makes is decided by the v2 signer before it runs and recorded after, as a signed event. +This page sets it up on your own machine with the dev signer, which runs as your own user (see [what dev assurance +means](../quickstart-v2.md#what-dev-assurance-means)). No API key: Claude Code uses its own sign-in. + +## 1. Install and wire the hooks + +```sh +pip install tracekit-ai # or pipx install tracekit-ai, uv tool install tracekit-ai +tracekit init --dev --v2 +``` + +Init adds Tracekit's hooks to Claude Code's user settings (`~/.claude/settings.json`, or the `settings.json` in +`$CLAUDE_CONFIG_DIR`) and leaves your other settings and hooks alone. `--project` writes them to +`./.claude/settings.json` instead. The hooks run the Python you ran init with. Init also turns on Claude Code's +thinking summaries (`showThinkingSummaries`) unless your settings set them, so `tracekit analyze --v2` can read the +model's reasoning; `--no-thinking-summaries` leaves the setting alone. + +You don't start a signer: the first hook starts the dev signer in the background (`tracekit up` and `tracekit down` +do it by hand). `tracekit doctor` checks the setup and ends with `dev setup OK` when everything checks out. + +The [Claude Code plugin](../../plugin/README.md) is the other way in: the same hooks, from the plugin instead of +init's entries in your settings. Use one, not both: with the plugin enabled, init installs no hooks and says so. + +## 2. Start a new Claude Code session + +Hooks are read when a session starts, so a session that was already open records nothing: start a new one. From then +on each tool call is decided before it runs: most are allowed, some are denied, and some are held for you. + +A denied call never runs, and Claude sees why: + +```text +[tracekit] BLOCKED, nothing ran, and no one can approve it (TK-D001, TK-D004, TK-F003). +Why: privilege escalation; recursive delete of root or home +Instead: Run the command as the agent's own user, without sudo; ... +``` + +A held call doesn't run either, until you approve it. Claude Code shows the hold in the session: + +```text +[tracekit] HELD, nothing ran: apr-6f0c… (TK-SHELL-PARSE), expires 14:26 UTC. +Why: a quote on the line is never closed +Instead: Close the quote. +Person: look with `! tracekit approvals show apr-6f0c…`; allow once with `! tracekit approvals approve apr-6f0c…`; refuse with `! tracekit approvals reject apr-6f0c…`. +Agent: tell the person what this call was for and pass on these commands, then wait for their answer; ... +``` + +## 3. Answer it + +Paste the `! tracekit approvals approve ` line into the session. A line that starts with `!` runs as your own +user, not as a tool call of Claude's, so it is your answer. Then ask Claude to make the same call again: the retry, +with exactly the same arguments, runs once. `! tracekit approvals reject ` refuses it. `tracekit approvals show +` prints why it was held and the signer's copy of the arguments; from another terminal, run the same commands +without the `!`. `tracekit approvals list` (or `tracekit pending`) lists every held call. + +Claude itself can't approve its own calls: the policy denies a `tracekit approvals approve` that the agent runs +(`TK-D012`). In dev mode the signer can't tell you from your agent, so your approval is recorded as a self-approval and +the run's assurance stays `dev`. + +To have Claude follow the hold message every time, paste [these +lines](https://github.com/Cygnux-Labs/Tracekit/blob/main/plugin/skills/tracekit-holds/SKILL.md) into your +`CLAUDE.md` (the plugin ships them as a skill). + +## 4. See what it did + +```sh +tracekit last # the most recent finished run: exported, its key pinned, verified +tracekit last --open # the session still open, verified up to its last checkpointed record +tracekit view --dev # every run in the browser, replayed and reviewed +``` + +A session's run stays open until the session ends, so `tracekit last` shows the session before it until then. +`tracekit last` prints `VERIFIED` and where it wrote the bundle; [the v2 quickstart](../quickstart-v2.md#3-see-it-verified) +explains each line. `tracekit status` shows where the hooks and the signer are. + +## Remove it + +```sh +tracekit uninstall +``` + +It removes Tracekit's hooks from Claude Code's settings and keeps your other settings; a settings file init made +that holds nothing else is deleted. It then lists what it kept, the dev signer's keys and signed runs and Tracekit's +client config, with the commands that delete them and Tracekit too (`tracekit down`, `rm -rf …`, `pip uninstall +tracekit-ai`). A bundle you exported verifies without them. + +## Next steps + +- **Other coding agents**: Codex, Cursor, Gemini CLI, OpenCode and others take `--agent NAME` + ([coding agents](../coding-agents.md)). +- **System mode** runs the signer as its own OS user, so Claude Code can't reach its keys or answer its own holds: + [system mode](../quickstart-v2.md#system-mode-linux-macos-experimental). +- **Wait for the answer instead**: with `TRACEKIT_APPROVAL_WAIT` set (seconds, up to about 480), a held call's hook + waits for an answer from Slack, the viewer or an agent approver before it blocks the call + ([held calls](../coding-agents.md#held-calls-approve-then-retry)). diff --git a/docs/quickstarts/langchain.md b/docs/quickstarts/langchain.md index ce0c41d0..c5a3f713 100644 --- a/docs/quickstarts/langchain.md +++ b/docs/quickstarts/langchain.md @@ -4,12 +4,27 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no API key + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'langchain>=1.4,<1.5' 'langgraph>=1.2,<1.3' +python examples/v2/langchain/agent.py --scripted +``` + +A mock model drives a real graph: `ls` is allowed, `sudo rm -rf /` is denied and the agent goes on, and `echo +"unterminated` is held until a second process approves it. Then the run is exported and verified. + ## 1. Install ```sh -pip install tracekit-ai 'langchain>=1.4,<1.5' 'langgraph>=1.2,<1.3' +pip install tracekit-ai 'langchain>=1.4,<1.5' 'langgraph>=1.2,<1.3' langchain-openai +export OPENAI_API_KEY=... # the model below is OpenAI's ``` +Any model `create_agent` takes works: install its provider's package and set its key instead. + ## 2. One line in your agent ```python @@ -70,8 +85,7 @@ first error) and through `.with_fallbacks()`, as LangGraph's customer-support tu exceptions around the node ends the pause: the call doesn't run and its approval expires ([limits](../limits.md#approvals)). -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/langchain/agent.py ---scripted` from a checkout. +The no-key example at the top of this page wires it this way, with an approval. ## Routing between assistants (handoffs) diff --git a/docs/quickstarts/mcp.md b/docs/quickstarts/mcp.md index 76dd7195..d58c0caf 100644 --- a/docs/quickstarts/mcp.md +++ b/docs/quickstarts/mcp.md @@ -4,12 +4,28 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no MCP server + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'mcp>=2.3,<2.4' +python examples/v2/mcp/agent.py --scripted +``` + +A server runs in the same process: `list_files` is allowed, `tracekit_demo_denied` is denied (the dev policy's demo +rule) and the client goes on, and `tracekit_demo_ask` is held until a second process approves it. Then the run is +exported and verified. + ## 1. Install ```sh pip install tracekit-ai 'mcp>=2.3,<2.4' ``` +No API key: the MCP client calls your server's tools. Replace `my-mcp-server` below with the command that starts +yours. + ## 2. One line in your agent ```python @@ -54,8 +70,7 @@ A denied call comes back as a result with `isError: true` and the refusal as its calls; the example's deny and ask come from the dev policy's demo rules (TK-DEMO-DENY, TK-DEMO-ASK). Write your own policy for real tools. -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/mcp/agent.py ---scripted` from a checkout. +The example at the top of this page wires it this way, with an approval. ## Read the verify report diff --git a/docs/quickstarts/openai-agents.md b/docs/quickstarts/openai-agents.md index 7ee69167..2ed1cd03 100644 --- a/docs/quickstarts/openai-agents.md +++ b/docs/quickstarts/openai-agents.md @@ -4,10 +4,23 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no API key + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'openai-agents>=0.23.1,<0.24' +python examples/v2/openai_agents/agent.py --scripted +``` + +A mock model drives a real `Runner`: `ls` is allowed, `sudo rm -rf /` is denied and the agent goes on, and `echo +"unterminated` is held until a second process approves it. Then the run is exported and verified. + ## 1. Install ```sh pip install tracekit-ai 'openai-agents>=0.23.1,<0.24' +export OPENAI_API_KEY=... # the agent below calls OpenAI's gpt-5 ``` ## 2. One line in your agent @@ -58,8 +71,7 @@ while result.interruptions: # an ask: once a person decide Function tools, ShellTool with a local executor, LocalShellTool and ApplyPatchTool are gated. Hosted tools, ComputerTool and handoffs are recorded only inside the signed model responses. -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/openai_agents/agent.py ---scripted` from a checkout. +The no-key example at the top of this page wires it this way, with an approval. ## Read the verify report diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 1e7885dd..95bf7585 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -80,7 +80,7 @@ Integrity: FAILED. ```console $ tracekit view --dev -tracekit view on http://127.0.0.1:7778/?token= +tracekit view on http://127.0.0.1:7778/?token= (open it in a browser; Ctrl-C stops the viewer) ``` Every run, each verified, every call with its decision: diff --git a/plugin/README.md b/plugin/README.md index 689150b2..7bdce97d 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -59,7 +59,9 @@ without the `!`), Claude's retry of the same call runs. `TRACEKIT_SIGNER` is set. Before that, every event shows a notice and runs, unrecorded. The hooks `tracekit init --dev --v2` writes treat events the same way, but run the interpreter init ran with -(`python -I -m tracekit.integrations.claude_code `) instead of searching for one. +(`python -I -m tracekit.integrations.claude_code `; from a source checkout or a `pip install --user`, `python -I +-c` with a one-line import that puts that directory first on the path, then the directory and the event as quoted +arguments, all in plain text) instead of searching for one. ## Upgrading from the v1 plugin diff --git a/tests/test_bundle_v2.py b/tests/test_bundle_v2.py index a8c9d42d..eb2c3551 100644 --- a/tests/test_bundle_v2.py +++ b/tests/test_bundle_v2.py @@ -956,7 +956,8 @@ def test_verifiers_need_only_the_standard_library(self): "TAMPERED at event 6: "), "v1 sample, every check": (["verify", os.path.join(sample, "demo-run.tkb"), *key, "--detail"], 0, "Integrity: VERIFIED."), - "v2 bundle": (["verify", v2_bundle, "--trust", self.trust], 0, "VERIFIED: 5 signed records, ")} + "v2 bundle": (["verify", v2_bundle, "--trust", self.trust], 0, + "VERIFIED: no tool calls recorded; 5 signed records, ")} for why, (argv, want, verdict) in cases.items(): with self.subTest(why): p = subprocess.run([sys.executable, "-c", blocker, *argv], capture_output=True, text=True, diff --git a/tests/test_dev_mode_line.py b/tests/test_dev_mode_line.py index 936c9fd5..932964d0 100644 --- a/tests/test_dev_mode_line.py +++ b/tests/test_dev_mode_line.py @@ -13,8 +13,9 @@ from tracekit import cli, doctor, install from tracekit.sdk import autospawn -LINE = ("Dev mode records; it does not guard. The agent runs as you and can reach the key. " - "For protection, use system mode or Box.") +LINE = ("Dev mode records; it does not guard. The agent runs as you and can reach the key. To guard, run the signer " + "as another user (system mode: https://github.com/Cygnux-Labs/Tracekit/wiki/Deploy) or the agent in Tracekit " + "Box, a micro-VM (https://github.com/Cygnux-Labs/Tracekit-box).") def run(argv): @@ -81,7 +82,18 @@ def status(self, dev, system): with mock.patch.object(install, "status", return_value={"client_config": None}), \ mock.patch.object(autospawn, "status", return_value={"socket": "s", "dev": dev}), \ mock.patch("tracekit.client.system_config", return_value=system): - return json.loads(run(["status"])[1]).get("dev_mode") + return json.loads(run(["status", "--json"])[1]).get("dev_mode") + + def test_status_prints_a_summary_and_json_only_with_json(self): + """A new user's `tracekit status` printed a page of JSON; it prints what it found in lines, --json the JSON.""" + hello = {"socket": "s.sock", "dev": True, "running": True, "version": "9", "pid": 7, "data_dir": "/d"} + with mock.patch.object(install, "status", return_value={"client_config": None, "hooks": [], "signer": None}), \ + mock.patch.object(autospawn, "status", return_value=hello), \ + mock.patch("tracekit.client.system_config", return_value=None): + self.assertEqual(run(["status"]), (0, "\n".join([ + "signer dev signer 9, running (pid 7), socket s.sock", "data /d (its keys and signed runs)", + "hooks none in Claude Code's settings: tracekit init --dev --v2 adds them", LINE, + "(tracekit status --json prints all of it)", ""]))) def test_status_says_it_for_a_dev_signer_and_not_for_a_system_one(self): self.assertEqual(self.status(True, None), LINE) diff --git a/tests/test_doctor_v2.py b/tests/test_doctor_v2.py index b486eade..e6e65e91 100644 --- a/tests/test_doctor_v2.py +++ b/tests/test_doctor_v2.py @@ -716,7 +716,9 @@ def test_no_hooks_after_a_hooked_init_takes_the_hooks_out(self): self.assertTrue(install._v2_hooks_in(settings)) with contextlib.redirect_stdout(io.StringIO()) as out: self.assertEqual(cli.main(["init", "--dev", "--v2", "--no-hooks"]), 0) - self.assertEqual(out.getvalue(), f"Tracekit v2 hooks removed from {settings}\nv2 hooks: not installed\n{install.DEV_MODE_LINE}\n") + self.assertEqual(out.getvalue(), f"Tracekit v2 hooks removed from {settings}, and the file with them: init " + f"made it, and nothing else is in it\nv2 hooks: not installed\n" + f"{install.DEV_MODE_LINE}\n") self.assertFalse(install._v2_hooks_in(settings)) self.assertEqual(client.client_config(), {"no_hooks": True}) @@ -738,6 +740,51 @@ def test_system_mode_reads_the_choice_from_client_json(self): self.assertEqual((got["status"], got["detail"]), want) +class FirstDay(unittest.TestCase): + """A new user's healthy dev setup got three warnings: no hooks where there is no Claude Code, no witness (which the + dev signer can't have), and the dev profile's own process boundary. Only the last is a warning now, and a healthy + run ends saying so.""" + + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d, True) + + def rows(self, **kw): + return {r["id"]: r for r in doctor.v2_checks({"data_dir": self.d}, "dev", **kw)} + + def test_no_claude_code_and_no_witness_are_info_in_dev(self): + nowhere = os.path.join(self.d, "no-claude", "settings.json") + with mock.patch.object(doctor.shutil, "which", return_value=None), \ + mock.patch.object(install, "user_settings", return_value=nowhere): + got = self.rows(settings=nowhere) + self.assertEqual((got["D-HOOKS-PRESENT"]["status"], got["D-HOOKS-PRESENT"]["fix"]), + ("info", "once Claude Code is installed: tracekit init --dev --v2")) + self.assertTrue(got["D-HOOKS-PRESENT"]["detail"].startswith("no Claude Code here (no claude on PATH, no ")) + self.assertEqual(got["D-WITNESS-CONFIGURED"]["status"], "info") + with mock.patch.object(doctor.shutil, "which", return_value="/usr/local/bin/claude"): # Claude Code, no hooks + self.assertEqual(self.rows(settings=nowhere)["D-HOOKS-PRESENT"]["status"], "warn") + prod = {r["id"]: r for r in doctor.v2_checks({"data_dir": self.d})} + self.assertEqual(prod["D-WITNESS-CONFIGURED"]["status"], "fail") # production: unchanged + + def test_a_healthy_dev_run_ends_with_one_ok_line(self): + ok = [doctor.result("D-PROCESS-BOUNDARY", doctor.WARN, install.DEV_MODE_LINE, "system mode"), + doctor.result("D-WITNESS-CONFIGURED", doctor.INFO, "no witnesses", "a signer.yaml"), + doctor.result("D-KEYS-MODE", doctor.OK, "8 keys")] + + def run(rows, *argv): + out = io.StringIO() + with mock.patch.object(doctor, "collect", return_value=rows), contextlib.redirect_stdout(out): + self.assertEqual(doctor.main(*argv), 0) + return out.getvalue().splitlines() + lines = run(ok) + self.assertEqual(lines[-1], "dev setup OK: everything checks out for dev mode, which records but does not " + "guard (D-PROCESS-BOUNDARY)") + self.assertIn("info no witnesses (D-WITNESS-CONFIGURED)", lines) + self.assertFalse(any(x.startswith("dev setup OK") for x in run( + ok + [doctor.result("D-HOOKS-EVENT", doctor.WARN, "unwrapped", "re-run init")]))) + self.assertIsInstance(json.loads("\n".join(run(ok, None, True))), list) # --json: the rows alone + + class Reinit(unittest.TestCase): def test_the_reinit_fix_runs_on_this_platform(self): """On macOS init refuses system mode without --experimental-macos, which the fix left out.""" @@ -748,11 +795,26 @@ def test_the_reinit_fix_runs_init_where_it_works_on_every_python(self): and on a dev host with no /opt (D-PROCESS-BOUNDARY), and the version from doctor's own Python; with Tracekit only in /opt's venv the system Python's `-I` form found no module. Init from a root-owned clone works on every Python and with or without /opt, as the plugin and CI run it.""" - self.assertIn("from a root-owned clone of the release (docs/deploy.md#laptop-system-mode), " - "sudo /usr/bin/python3 -m tracekit init --v2 --user ", doctor.REINIT) + self.assertIn("from a root-owned clone of the release (https://github.com/Cygnux-Labs/Tracekit/wiki/" + f"Deploy#laptop-system-mode), sudo {shlex.quote(doctor._PY)} -m tracekit init --v2 --user " + "", doctor.REINIT) self.assertNotIn("/opt/tracekit", doctor.REINIT) self.assertNotIn(" -I ", doctor.REINIT) # -I leaves the clone out of the import path + def test_the_reinit_fix_names_the_wiki_and_a_python_init_accepts(self): + """A new user saw docs/deploy.md, a path in a checkout they don't have, and /usr/bin/python3, which a container + may not have: the fix names the wiki, and doctor's own Python when only root can change it (init refuses any + other), else the system one.""" + def reinit(python, trusted): + with mock.patch.object(sys, "executable", python), \ + mock.patch("tracekit.harness_helper.trusted_file", return_value=None if trusted else "not root's"): + return importlib.reload(doctor)._PY + self.addCleanup(importlib.reload, doctor) + self.assertEqual(reinit("/usr/local/bin/python3", True), "/usr/local/bin/python3") + self.assertEqual(reinit("/home/me/.venv/bin/python", False), "/usr/bin/python3") + self.assertEqual(reinit("/opt/tracekit/bin/python", True), "/usr/bin/python3") # 3.9, 3.10: init fails there + self.assertNotIn("docs/deploy.md", doctor.REINIT) + class K8s(unittest.TestCase): def test_clean_deployment_passes_every_check(self): @@ -1636,9 +1698,11 @@ def test_dev_signer_warns_instead_of_failing(self): d = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, d, True) os.chmod(d, 0o700) - got = {r["id"]: r["status"] for r in doctor.v2_checks({"data_dir": d}, "dev", settings=os.path.join(d, "none"))} + with mock.patch.object(doctor.shutil, "which", return_value="/usr/local/bin/claude"): # Claude Code is here + got = {r["id"]: r["status"] for r in doctor.v2_checks({"data_dir": d}, "dev", + settings=os.path.join(d, "none"))} self.assertEqual(got["D-PROCESS-BOUNDARY"], "warn") - self.assertEqual(got["D-WITNESS-CONFIGURED"], "warn") + self.assertEqual(got["D-WITNESS-CONFIGURED"], "info") # the dev signer can't have one self.assertEqual(got["D-HOOKS-PRESENT"], "warn") self.assertEqual(got["D-DURABILITY"], "ok") self.assertNotIn("D-AGENT-PRIV", got) diff --git a/tests/test_hook_invocation.py b/tests/test_hook_invocation.py index ab3d6fab..2c95bb53 100644 --- a/tests/test_hook_invocation.py +++ b/tests/test_hook_invocation.py @@ -47,6 +47,31 @@ def test_project_directory_cannot_shadow_the_hook(self): self.assertNotIn("Traceback", r.stderr, cmd) +@unittest.skipIf(os.name == "nt", "the fail-closed wrapper is POSIX shell") +class ClearText(unittest.TestCase): + """The hook command init wrote from a working tree (or a --user install) carried the tree's path and branch in + base64, which reads as malware to a security reviewer: they are plain, quoted arguments now, and the hook runs as + before from a path with spaces and quotes.""" + + def test_the_command_names_its_tree_in_clear_text_and_runs_from_any_path(self): + from tracekit import doctor + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + tree = os.path.join(d, "it's a \"tree\" $HOME") + os.makedirs(os.path.join(tree, "tracekit", "integrations")) + for name in ("__init__.py", os.path.join("integrations", "__init__.py")): + open(os.path.join(tree, "tracekit", name), "w").close() + with open(os.path.join(tree, "tracekit", "integrations", "claude_code.py"), "w") as f: + f.write("import sys\ndef _entry(event):\n print('ran', event, sys.argv[1:])\n return 0\n") + with mock.patch.object(install, "ROOT", tree), mock.patch("importlib.util.find_spec", return_value=None): + cmd = install._hook_command(install.V2_HOOK, args=("PreToolUse",)) + self.assertNotIn("base64", cmd) + self.assertIn("from tracekit.integrations.claude_code import _entry", cmd) + p = subprocess.run(cmd, shell=True, input="{}", capture_output=True, text=True, timeout=60) + self.assertEqual((p.returncode, p.stdout.strip()), (0, f"ran PreToolUse {[tree, 'PreToolUse']}"), p.stderr) + self.assertEqual(doctor._probe_target(doctor._hook_argv(cmd)), (sys.executable, tree)) # D-VERSIONS' probe + + @unittest.skipIf(os.name == "nt", "the fail-closed wrapper is POSIX shell") class WorkingTree(unittest.TestCase): """A dev hook imports Tracekit from the working tree init ran in (#427): a branch switch or rebase there that diff --git a/tests/test_hook_v2.py b/tests/test_hook_v2.py index 9300b778..70627ec4 100644 --- a/tests/test_hook_v2.py +++ b/tests/test_hook_v2.py @@ -23,6 +23,7 @@ from test_bundle_v2 import LOG_SECRET, ORIGIN, pub from tracekit import cli, install, privacy from tracekit.bundle_v2 import export +from tracekit.client import client_home from tracekit.format.canon import event_hash from tracekit.format import checkpoint from tracekit.format.checkpoint import ED25519 @@ -1492,6 +1493,34 @@ def test_init_turns_thinking_summaries_on_for_the_plugins_users_and_uninstall_of with open(settings) as f: self.assertEqual(json.load(f), {"enabledPlugins": {install.PLUGIN: True}}) + def test_uninstall_deletes_the_settings_file_init_made_and_says_what_it_keeps(self): + """A new user's uninstall left a settings.json holding `{}` that init had made, and didn't say that the dev + signer's keys and runs and the client config stay, nor how to delete them and Tracekit.""" + settings = os.path.join(self.home, ".claude", "settings.json") + self.assertEqual(self.init("--dev", "--v2")[0], 0) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(cli.main(["uninstall"]), 0) + self.assertFalse(os.path.exists(settings)) + lines = out.getvalue().splitlines() + data = autospawn.data_dir() + self.assertIn(f" {data} (the dev signer's keys and signed runs: a bundle you exported verifies without " + "them)", lines) + self.assertIn(f" {client_home()} (Tracekit's client config)", lines) + gone = [os.path.dirname(data) if sys.platform == "darwin" else data, client_home()] + self.assertEqual(lines[-4:-2], ["to delete these and Tracekit too:", " tracekit down"]) + if os.name != "nt": # the runtime dir too, where one exists + self.assertEqual(shlex.split(lines[-2])[:2 + len(gone)], ["rm", "-rf", *gone]) + self.assertTrue(lines[-1].endswith(" uninstall tracekit-ai"), lines[-1]) + os.makedirs(os.path.dirname(settings), exist_ok=True) # a file that was there before init stays + with open(settings, "w") as f: + f.write("{}") + self.assertEqual(self.init("--dev", "--v2", "--no-thinking-summaries")[0], 0) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(cli.main(["uninstall"]), 0) + with open(settings) as f: + self.assertEqual(json.load(f), {}) + def test_uninstall_leaves_a_persons_own_thinking_setting(self): settings = os.path.join(self.home, "settings.json") for before in ({}, {install.THINKING: True}, {install.THINKING: False}): @@ -1546,7 +1575,12 @@ def test_init_dev_v2_writes_the_hook(self): self.assertTrue(all(install._is_ours(g) for gs in hooks.values() for g in gs)) # #587: init gave the agent nothing on what to do when a call is stopped; it points to the lines that do self.assertIn("\n" + cli.guidance() + "\n", self.out) - self.assertIn("/wiki/Coding-Agents#when-tracekit-stops-a-call into its CLAUDE.md or AGENTS.md", cli.guidance()) + # the first-day walkthrough: that anchor wasn't on the published wiki yet; the plugin's skill file is on GitHub + self.assertIn(f"{cli.HOLDS_SKILL} into its CLAUDE.md or AGENTS.md", cli.guidance()) + self.assertTrue(os.path.isfile(os.path.join(ROOT, cli.HOLDS_SKILL.split("/blob/main/")[1]))) + # and what to do next: a new session, the `!` line a hold shows, the viewer + self.assertIn("\n" + cli.next_step() + "\n", self.out) + self.assertIn("! tracekit approvals approve ", cli.next_step()) def test_the_agent_guidance_init_points_to_is_the_plugins_skill(self): """#587: one wording of what an agent does when Tracekit stops its call, in the docs init points to and in the diff --git a/tests/test_install_files.py b/tests/test_install_files.py index d6bce9aa..41dabe40 100644 --- a/tests/test_install_files.py +++ b/tests/test_install_files.py @@ -523,7 +523,7 @@ def test_v2_init_records_where_it_put_the_hooks(self): with mock.patch.dict(os.environ, env), contextlib.redirect_stdout(io.StringIO()): self.assertEqual(doctor._dev_settings(), settings) self.assertEqual(install.uninstall(), settings) - self.assertEqual(install.read_json(settings).get("hooks"), None) + self.assertFalse(os.path.exists(settings)) # init made it, and nothing else is in it def test_hooks_are_found_and_removed_with_or_without_claude_config_dir(self): for installed_with, removed_with in ((True, False), (False, True)): @@ -545,7 +545,7 @@ def test_hooks_are_found_and_removed_with_or_without_claude_config_dir(self): os.environ.pop("CLAUDE_CONFIG_DIR", None) self.assertIn(path, [h["settings"] for h in install._installed_hooks()]) self.assertEqual(install.uninstall(), path) # it named files it never found - self.assertEqual(install._hook_summary(install.read_json(path)), install._hook_summary({})) + self.assertFalse(os.path.exists(path)) # install_hooks made it: no hooks, and no file if __name__ == "__main__": diff --git a/tests/test_instrument.py b/tests/test_instrument.py index 43c4aa51..34821e53 100644 --- a/tests/test_instrument.py +++ b/tests/test_instrument.py @@ -96,7 +96,7 @@ def test_each_framework(self): if name != "openai": self.assertIn(REFUSED, out[1]) p = self.sh("-m", "tracekit", "last") - self.assertRegex(p.stdout, r"\nVERIFIED: (\d+ tool calls? in )?\d+ signed records, none changed") + self.assertRegex(p.stdout, r"\nVERIFIED: (\d+ tool calls? in |no tool calls recorded; )?\d+ signed records, none changed") self.assertIn("\nNOT PROOF AGAINST THE AGENT: ", p.stdout) # a dev signer bundle = p.stdout.split("bundle: ", 1)[1].splitlines()[0] self.assertTrue(os.path.isfile(bundle) and os.path.isfile(bundle[:-4] + ".trust.json")) diff --git a/tests/test_policy2.py b/tests/test_policy2.py index 51aafd45..931a371c 100644 --- a/tests/test_policy2.py +++ b/tests/test_policy2.py @@ -2812,11 +2812,15 @@ def test_the_reason_names_why_the_policy_cant_read_a_command(self): e = Engine(pc.build(os.path.join(PACKS, "dev.yaml"))[0]) for cmd, cause in (("X=$(printf ls); $X", "name"), ("echo x | base64 -d | sh", "input"), ("fish -c 'ls'", "foreign"), ("l*s x", "pattern"), ("echo $((x[$(id)]))", "syntax"), - ("f(){ sh x; }; curl -s -o x https://example.com/x; f", "download")): + ("f(){ sh x; }; curl -s -o x https://example.com/x; f", "download"), + # a new user's first hold said "shell syntax the policy doesn't read" for a missing quote + ('echo "unterminated', "quote"), ("echo 'x", "quote"), ("echo $'x", "quote"), + ("echo ${x", "syntax")): with self.subTest(cmd=cmd): self.assertEqual(e.decide("Bash", {"command": cmd}).get("unread"), cause) + self.assertEqual(e.decide("Bash", {"command": cmd})["verdict"], "ask") # the cause is words only self.assertNotIn("unread", e.decide("Bash", {"command": "ls"})) - for cause in ("name", "pattern", "input", "foreign", "syntax", "function", "download"): + for cause in ("name", "pattern", "input", "foreign", "syntax", "function", "download", "quote"): entry = pc.ENGINE_REASONS["TK-SHELL-PARSE"]["causes"][cause] self.assertLessEqual(len(entry["remedy"]), pc.MAX_REMEDY, cause) diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index e03c4ce4..9136d48d 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -66,7 +66,7 @@ def last(self): """Step 3; its time.""" t = time.monotonic() out = self.sh(self.tracekit, "last").stdout - self.assertRegex(out, r"\nVERIFIED: (\d+ tool calls? in )?\d+ signed records, none changed") + self.assertRegex(out, r"\nVERIFIED: (\d+ tool calls? in |no tool calls recorded; )?\d+ signed records, none changed") self.assertIn("\nNOT PROOF AGAINST THE AGENT: ", out) # a dev signer self.assertIn("next: tracekit view --dev", out) return time.monotonic() - t diff --git a/tests/test_remote_tokens.py b/tests/test_remote_tokens.py index e66be53a..d8733ae6 100644 --- a/tests/test_remote_tokens.py +++ b/tests/test_remote_tokens.py @@ -259,8 +259,7 @@ def test_a_later_init_and_uninstall_drop_the_remote_signer(self): with open(settings) as f: self.assertNotIn("TRACEKIT_SIGNER_CA", json.load(f)["env"]) self.assertEqual(cli.main(["uninstall", "--project"]), 0) - with open(settings) as f: - self.assertNotIn("env", json.load(f)) + self.assertFalse(os.path.exists(settings)) # init made it: no env, and no file if __name__ == "__main__": diff --git a/tests/test_sdk_client.py b/tests/test_sdk_client.py index 58661d37..18d055f8 100644 --- a/tests/test_sdk_client.py +++ b/tests/test_sdk_client.py @@ -218,16 +218,16 @@ def cli(self, *argv): def test_cli_up_status_down(self): self.assertIn(f"dev signer {__version__} running", self.cli("up", "--wait")) - status = json.loads(self.cli("status")) + status = json.loads(self.cli("status", "--json")) self.assertIn("client_config", status) # the v1 sections are still there self.assertTrue(status["signer_v2"]["running"]) self.assertIn("dev signer stopped", self.cli("down")) - self.assertFalse(json.loads(self.cli("status"))["signer_v2"]["running"]) + self.assertFalse(json.loads(self.cli("status", "--json"))["signer_v2"]["running"]) self.assertIn("no dev signer running", self.cli("down")) def test_status_survives_an_unusable_runtime_dir(self): with mock.patch.object(autospawn, "runtime_dir", side_effect=PermissionError("read-only")): - status = json.loads(self.cli("status")) + status = json.loads(self.cli("status", "--json")) self.assertIn("client_config", status) self.assertFalse(status["signer_v2"]["running"]) diff --git a/tests/test_signer_approvals.py b/tests/test_signer_approvals.py index cbedb9a4..1df20553 100644 --- a/tests/test_signer_approvals.py +++ b/tests/test_signer_approvals.py @@ -716,7 +716,7 @@ def test_show_says_why_what_to_do_instead_and_how_to_answer_and_list_puts_the_li self.assertEqual(shown.returncode, 0, shown.stderr) self.assertIn("\nwhy moving money (a Stripe, PayPal or Square charge, payout, transfer or refund) " "(TK-A043)\ninstead Check the amount, currency and payee", shown.stdout) - self.assertRegex(shown.stdout, r"\nexpires \S+Z \(in 5\d min\)\n") + self.assertRegex(shown.stdout, r"\nexpires \d\d:\d\d UTC \(in 5\d min\)\n") self.assertIn(f"\nanswer `tracekit approvals --signer {sock} approve {aids[1]}` to allow it once, " f"`tracekit approvals --signer {sock} reject {aids[1]}` to refuse it\n", shown.stdout) self.assertIn(f"\nrecord tracekit view --approval {aids[1]} (with the --config or --data-dir of this " @@ -818,5 +818,53 @@ def test_approve_round_trip_with_the_dev_signer(self): self.assertIn(f"{aid} consumed", buf.getvalue()) +class TestFirstDayCli(unittest.TestCase): + """What a new user's walkthrough found: `approvals show` gave TK-SHELL-PARSE's general reason where the hook gave + the cause's own, offered Claude Code's `! ` to runs that aren't Claude Code's, and `tracekit pending` failed with + "signer unavailable" with held v2 calls and no v1 signer.""" + + def setUp(self): + self.run_dir = tempfile.mkdtemp(dir="/tmp" if os.path.isdir("/tmp") else None) + self.addCleanup(shutil.rmtree, self.run_dir, True) + os.chmod(self.run_dir, 0o700) + env = mock.patch.dict(os.environ, {"TRACEKIT_RUNTIME_DIR": self.run_dir, + "XDG_DATA_HOME": os.path.join(self.run_dir, "none"), + "HOME": os.path.join(self.run_dir, "home")}) + env.start() + self.addCleanup(env.stop) + + def lines(self, command, run_id="run-1"): + from tracekit.policy2 import compile as pc + pol, _ = pc.build(os.path.join(ts.ROOT, "tracekit", "policy2", "packs", "dev.yaml")) + x = {"approval_id": "apr-" + "1" * 32, "state": "requested", "run_id": run_id, "tool": "Bash", + "rule_ids": ["TK-SHELL-PARSE"], "policy_hash": pc.policy_hash(pol), "args_source": "parsed", + "args": {"command": command}, "expires_at": time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", + time.gmtime(time.time() + 3600))} + return cli._stop_lines(mock.Mock(signer=None), x) + + def test_show_gives_the_cause_the_hook_gives(self): + self.assertEqual(self.lines('echo "unterminated')[:2], [ + "why a quote on the line is never closed (TK-SHELL-PARSE)", "instead Close the quote."]) + self.assertTrue(self.lines("X=$(printf ls); $X")[0].startswith( + "why its program is a variable the line doesn't set to plain text (TK-SHELL-PARSE)")) + self.assertRegex(self.lines("ls x 'y")[2], r"^expires \d\d:\d\d UTC \(in (59|60) min\)$") + + def test_the_bang_hint_is_for_claude_code_runs_only(self): + answer = next(line for line in self.lines('echo "x') if line.startswith("answer")) + self.assertNotIn("! ", answer) # an SDK agent's run: no Claude Code session to paste it into + with open(os.path.join(self.run_dir, "claude-code-" + "a" * 64 + ".json"), "w") as f: + json.dump({"run_id": "run-1", "run_token": "t"}, f) # the v2 hook's session state + answer = next(line for line in self.lines('echo "x') if line.startswith("answer")) + self.assertTrue(answer.endswith("; in the agent's Claude Code session, with a leading `! `"), answer) + + def test_pending_without_a_v1_signer_lists_the_v2_signers_held_calls(self): + from tracekit import client + with mock.patch.object(client, "DEFAULT_SOCKET", os.path.join(self.run_dir, "no-tracekitd.sock")), \ + mock.patch.object(client, "socket_path", return_value=client.DEFAULT_SOCKET), \ + mock.patch.object(cli, "_approvals", return_value=0) as v2: + self.assertEqual(cli.main(["pending"]), 0) + self.assertEqual((v2.call_args[0][0].action, v2.call_args[0][0].signer), ("list", None)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_signer_dev.py b/tests/test_signer_dev.py index 796f6b41..09067a22 100644 --- a/tests/test_signer_dev.py +++ b/tests/test_signer_dev.py @@ -143,7 +143,7 @@ def test_a_runtime_dir_too_long_for_a_socket_path_gets_a_short_one_of_its_own(se again = Client() self.addCleanup(again.close) self.assertEqual(self.one_run(again), pid) - self.assertEqual(json.loads(self.cli("status"))["signer_v2"]["socket"], os.path.join(short, autospawn.SOCK)) + self.assertEqual(json.loads(self.cli("status", "--json"))["signer_v2"]["socket"], os.path.join(short, autospawn.SOCK)) self.assertNotEqual(autospawn.runtime_dir(), short) # another dir's is never this one near = "/tmp/" + "s" * (103 - len("/tmp/") - len("/" + autospawn.SOCK)) # its socket path fits, a side dir's not self.assertEqual(autospawn._short(near), near) @@ -322,7 +322,7 @@ def test_export_says_what_it_wrote_and_the_commands_it_gives_verify_it(self): r"trust\.json\n$") trust, verify = [shlex.split(x)[1:] for x in said.splitlines() if x.startswith(" tracekit ")] self.cli(*trust) - self.assertRegex(self.cli(*verify), r"^VERIFIED: (\d+ tool calls? in )?\d+ signed records, none changed") + self.assertRegex(self.cli(*verify), r"^VERIFIED: (\d+ tool calls? in |no tool calls recorded; )?\d+ signed records, none changed") info = json.loads(self.cli("export", "--v2", "--run", run.run_id, "-o", "again.tkb", "--json")) self.assertEqual((info["bundle"], info["verifier_min_version"] in said), ("again.tkb", True)) @@ -439,6 +439,21 @@ def setUp(self): self.addCleanup(env.stop) os.environ.pop("TRACEKIT_SIGNER", None) + def test_uninstall_names_only_this_homes_runtime_dirs(self): + """A first-day run of `tracekit uninstall` from a second HOME named the per-user runtime dir another HOME's + live signer used, in the `rm -rf` it prints: it names that dir only when it isn't another data dir's, and + this data dir's own sibling of it.""" + self.assertEqual(autospawn.own_runtime_dirs(), []) # none made yet + os.makedirs(self.run_dir, 0o700) + self.assertEqual(autospawn.own_runtime_dirs(), [self.run_dir]) + with open(os.path.join(self.run_dir, autospawn.ENDPOINT), "w") as f: + json.dump({"data_dir": os.path.join(self.dir, "another-home-data")}, f) + self.assertEqual(autospawn.own_runtime_dirs(), []) + side = autospawn._side(self.run_dir) + os.makedirs(side, 0o700) + self.addCleanup(shutil.rmtree, side, True) + self.assertEqual(autospawn.own_runtime_dirs(), [side]) + def test_signer_starts_in_the_private_runtime_dir_with_cwd_off_sys_path(self): os.makedirs(self.run_dir, 0o700) with mock.patch.object(autospawn.subprocess, "Popen") as popen: @@ -500,7 +515,7 @@ def replaced(pid, sig): # a signer started since publishes its own pid def test_status_creates_nothing(self): with contextlib.redirect_stdout(io.StringIO()) as out: - self.assertEqual(cli.main(["status"]), 0) + self.assertEqual(cli.main(["status", "--json"]), 0) self.assertFalse(json.loads(out.getvalue())["signer_v2"]["running"]) self.assertFalse(os.path.exists(self.run_dir)) diff --git a/tests/test_system_v2.py b/tests/test_system_v2.py index f9d42d45..cad301b7 100644 --- a/tests/test_system_v2.py +++ b/tests/test_system_v2.py @@ -398,6 +398,8 @@ def managed_setup(self, admin=None): json.dump(admin, f) def read(self, path): + if not os.path.exists(path): # init made it, and taking Tracekit's hooks out left nothing: deleted + return {} with open(path) as f: return json.load(f) @@ -537,7 +539,7 @@ def test_init_names_the_claude_code_file_it_takes_the_hooks_out_of(self): _, _, sc, _ = self.init_real({}) sc["hooks"]["user"] = "alice" if "project" in kw else "agent" out = self.init_real(sc, **kw)[3] - self.assertIn(f"Tracekit v2 hooks removed from {self.user}\n", out) + self.assertIn(f"Tracekit v2 hooks removed from {self.user}", out) self.assertEqual(install._hook_summary(self.read(self.user))["versions"], []) def test_uninstall_takes_tracekits_hooks_out_of_the_managed_settings_the_config_forgot(self): @@ -725,8 +727,7 @@ def test_signer_env_is_set_and_removed_only_while_unchanged(self): with open(path) as f: self.assertEqual(json.load(f)["env"], {"TRACEKIT_SIGNER": "/run/s.sock"}) install.install_hooks(path, uninstall=True, signer="/run/s.sock") - with open(path) as f: - self.assertNotIn("env", json.load(f)) + self.assertFalse(os.path.exists(path)) # install_hooks made it: no env, and no file with open(path, "w") as f: json.dump({"env": {"TRACEKIT_SIGNER": "/mine.sock"}}, f) install.install_hooks(path, uninstall=True, signer="/run/s.sock") diff --git a/tests/test_verify_short.py b/tests/test_verify_short.py index 7ef28f1e..02d52f13 100644 --- a/tests/test_verify_short.py +++ b/tests/test_verify_short.py @@ -71,9 +71,15 @@ def test_a_dev_bundle_is_not_proof_against_the_agent_however_it_was_witnessed(se (0, "VERIFIED: 2 tool calls in 4 signed records, none changed, removed or added after " f"signing.\n{DEV}")) - def test_a_run_with_no_tool_call_counts_its_signed_records(self): - self.assertEqual(verify(self.bundle("same-user", calls=()), "--trust", self.trust)[1].splitlines()[0], - "VERIFIED: 2 signed records, none changed, removed or added after signing.") + def test_a_run_with_no_tool_call_says_so_and_counts_its_signed_records(self): + """A new user's first run with nothing recorded read only "VERIFIED: 3 signed records", as if calls were in + them; the exit code and --json stay as they were.""" + bundle = self.bundle("same-user", calls=()) + self.assertEqual(verify(bundle, "--trust", self.trust)[1].splitlines()[0], + "VERIFIED: no tool calls recorded; 2 signed records, none changed, removed or added after " + "signing.") + code, out = verify(bundle, "--trust", self.trust, "--json") + self.assertEqual((code, json.loads(out)["integrity"]), (0, "VERIFIED")) def test_an_isolation_the_signer_could_not_tell_shows_nothing(self): code, out = verify(self.bundle("unknown"), "--trust", self.trust) diff --git a/tests/test_view.py b/tests/test_view.py index 1f45ccc2..c82d945e 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -42,6 +42,7 @@ from tracekit.why import cli as why_cli, import_v2 from tracekit.why.import_v2 import _target, store_records from tracekit.why.server import Feed as WhyFeed, IngestError, Store, tape +from tracekit.wiki import wiki_url POLICY = Engine({"deny": [{"id": "T-DENY", "tool": "rm", "pattern": "^"}], "ask": [{"id": "T-PAY", "tool": "pay", "pattern": "^"}]}) @@ -563,10 +564,12 @@ def test_policy_reasons_of_a_configured_policy_by_its_hash(self): path = os.path.join(ts.tmpdir(self), "p.yaml") with open(path, "w", encoding="utf-8") as f: f.write("deny:\n - id: X-1\n pattern: 'rm'\n reason: removing files\n remedy: Move it aside.\n" - "flag:\n - id: X-2\n pattern: 'ls'\n label: listing\n") + "flag:\n - id: X-2\n pattern: 'ls'\n label: listing\n" + " - id: X-3\n pattern: 'git push'\n label: side_effect\n") h = policy_compile.policy_hash(policy_compile.build(path)[0]) self.assertEqual(view.policy_reasons([path])[h], {"X-1": {"reason": "removing files", "remedy": "Move it aside."}, - "X-2": {"reason": "listing"}}) + "X-2": {"reason": "listing"}, + "X-3": {"reason": "side effect"}}) # report printed side_effect self.assertEqual(SERVER[SERVER_HASH]["TK-DB001"], {"reason": "dropping a database object"}) def test_a_calls_signed_provenance_rides_on_its_row(self): @@ -1141,7 +1144,7 @@ def test_dev_run_is_listed_with_its_counts_and_verified_while_the_signer_holds_t self.assertEqual(rep["title"], f"RUN {run_id} · Integrity VERIFIED · Assurance dev") self.assertIn("tenant default · agent e2e · final · decisions allow 1, ask 1, deny 1 · approvals 1 · gaps 0", rep["text"]) - self.assertIn("same user as the dev signer", rep["text"]) + self.assertIn("in dev mode it runs as you, as the signer does", rep["text"]) self.assertIn(view.OPERATOR_SIDE, rep["text"]) self.assertIn("Integrity: VERIFIED.\nAssurance: dev;", rep["text"]) # verify.v2's own report rows = self.rows(feed, run_id) @@ -1286,8 +1289,23 @@ def test_last_before_any_dev_signer_makes_nothing(self): contextlib.redirect_stderr(io.StringIO()) as err: self.assertEqual(cli.main(["last"]), 1) self.assertTrue(err.getvalue().startswith(f"tracekit last: no runs in the dev signer's store " - f"({os.path.join(nope, 'store')}) yet: "), err.getvalue()) + f"({os.path.join(nope, 'store')}) yet. "), err.getvalue()) self.assertFalse(os.path.exists(nope)) + # a new user who had added instrument() was told to add it: it records only the frameworks it lists + self.assertIn("`tracekit.instrument()` records only the frameworks it lists", err.getvalue()) + self.assertIn(f"as {wiki_url('docs/quickstarts/custom.md')} shows", err.getvalue()) + + def test_last_says_it_waits_for_a_run_that_just_closed(self): + """`tracekit last` waited up to LAST_WAIT_S for a closing run in silence; it says so, once.""" + closing = [v2rec(0, "run.registered", {"agent": {"name": "a"}}, "r1"), v2rec(1, "run.closing", {}, "r1")] + fake = mock.Mock(runs={("default", "r1"): {"seqs": [1]}}) + fake.iter_run.side_effect = lambda *key: iter(closing) + with mock.patch.object(svc, "reader", return_value=fake), \ + mock.patch.object(svc, "signer_config", return_value={"data_dir": self.d}), \ + mock.patch.object(cli, "LAST_WAIT_S", 0.6), contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(cli.main(["last"]), 1) + self.assertEqual(err.getvalue().count("tracekit last: run r1 (a) just closed; waiting up to 0.6 s while the " + "signer finalises it\n"), 1) @pytest.mark.unverified_bundles def test_tampered_record_shows_only_the_failure(self): @@ -1609,6 +1627,27 @@ def test_approval_prints_the_link_that_opens_the_held_call(self): self.assertEqual(view.main(["--data-dir", self.d, "--approval", aid + "&x="]), 2) self.assertIn("--approval takes an approval id", err.getvalue()) + def test_the_start_lines_are_plain_and_the_link_is_one_a_browser_opens(self): + """A new user's walkthrough: the viewer's lines said "trust pins this store's own log.vkey: dev assurance", + an any-address bind printed http://0.0.0.0:..., which a browser doesn't open, --host and --port had no help, and + --approval's help said it prints a link where it starts the viewer and prints one.""" + with mock.patch.object(view, "server") as server, mock.patch.dict(os.environ, {"TRACEKIT_VIEW_TOKEN": "t"}), \ + contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()): + server.return_value.serve_forever.side_effect = KeyboardInterrupt + server.return_value.server_address = ("0.0.0.0", 7778) + self.assertEqual(view.main(["--data-dir", self.d, "--host", "0.0.0.0", "--insecure-http"]), 0) + first, second = out.getvalue().splitlines()[:2] + self.assertEqual(first, "tracekit view on http://127.0.0.1:7778/ (open it in a browser; Ctrl-C stops the " + "viewer)") + self.assertIn("read-only; each run is checked against the key this store names", second) + self.assertNotIn("log.vkey", second) + with contextlib.redirect_stdout(io.StringIO()) as out, self.assertRaises(SystemExit): + view.main(["--help"]) + help_text = " ".join(out.getvalue().split()) + for words in ("--host HOST the address to serve on", "--port PORT the port to serve on", + "--approval ID start the viewer and print a link that opens the call approval ID held"): + self.assertIn(words, help_text) + def test_loopback_viewer_prints_a_token_url_and_needs_it(self): with mock.patch.object(view, "server") as server, mock.patch.dict(os.environ), \ contextlib.redirect_stdout(io.StringIO()) as out: @@ -1627,7 +1666,8 @@ def test_loopback_viewer_prints_a_token_url_and_needs_it(self): server.return_value.server_address = ("127.0.0.1", 7778) runs.return_value.logs = ["a.dsn", "b.dsn"] self.assertEqual(view.main(["--config", "signer.yaml"]), 0) - self.assertIn(f"http://127.0.0.1:7778/?token={server.call_args[0][3]} (2 Postgres logs", out.getvalue()) + self.assertIn(f"http://127.0.0.1:7778/?token={server.call_args[0][3]} (open it in a browser; Ctrl-C stops " + "the viewer)\n reading 2 Postgres logs, read-only\n", out.getvalue()) with self.assertRaises(ValueError): view.server(None, "127.0.0.1", 0, None) @@ -2257,7 +2297,7 @@ def test_v1_bundle_is_served_on_the_viewers_server_and_a_tampered_one_is_refused self.assertEqual(code, 0) self.assertIsInstance(feed, observe.Feed) # the bundle it shows, not the temp ledger it was unpacked to - self.assertIn(f"(bundle: {os.path.join(self.SAMPLE, 'demo-run.tkb')}, verified, read-only)", out) + self.assertIn(f"reading the bundle {os.path.join(self.SAMPLE, 'demo-run.tkb')}, verified, read-only", out) self.assertNotIn(feed.path, out) srv = view.server(feed, "127.0.0.1", 0, token) threading.Thread(target=srv.serve_forever, daemon=True).start() @@ -2293,7 +2333,7 @@ def test_why_alone_is_why_serve(self): self.assertEqual(code, 0) self.assertEqual((store.root, store.tape, store.allow, args[0]), (root, True, {"m:SYSTEM"}, store.out)) self.assertIsNotNone(store.webhook) - self.assertIn(f"(why logs alone: {root})", out) + self.assertIn(f"reading why logs alone: {root}", out) def test_one_nav_reaches_every_page_this_server_has(self): runs = mock.Mock() diff --git a/tracekit/cli.py b/tracekit/cli.py index ff786b6d..a6515b4d 100644 --- a/tracekit/cli.py +++ b/tracekit/cli.py @@ -26,7 +26,20 @@ def guidance(): """The line `tracekit init --v2` prints once it installs hooks: where the agent's instructions for a held or blocked call are (the Claude Code plugin's tracekit-holds skill holds the same lines).""" return ("when Tracekit holds or blocks a call, the agent should tell you and pass on the commands: paste the lines " - f"at {wiki_url('docs/coding-agents.md', 'when-tracekit-stops-a-call')} into its CLAUDE.md or AGENTS.md") + f"at {HOLDS_SKILL} into its CLAUDE.md or AGENTS.md") + + +# the lines guidance() names, in the Claude Code plugin's tracekit-holds skill +HOLDS_SKILL = "https://github.com/Cygnux-Labs/Tracekit/blob/main/plugin/skills/tracekit-holds/SKILL.md" + + +def next_step(): + """The line `tracekit init --dev --v2` ends with once it wired Claude Code's hooks: what to do now.""" + import shutil + return (("next: install Claude Code (no `claude` on PATH yet), then start a session" if not shutil.which("claude") + else "next: start a new Claude Code session (one already open doesn't load the hooks)") + + ". A call the policy holds shows a `! tracekit approvals approve ` line: paste it into that session " + "to allow the call once, then ask the agent to retry it. See every run: tracekit view --dev") def experimental(what, help=False): @@ -238,12 +251,13 @@ def _parser(): "enclave attestation, KMS key metadata); its hash is signed into checkpoints and " "exports carry it") - sub.add_parser("status", help="hooks, signer, witnesses, policy, fail mode, capture sources") - p = sub.add_parser("up", help="find or start the same-user v2 dev signer") + p = sub.add_parser("status", help="what is set up here: hooks, signer, policy, capture sources") + p.add_argument("--json", action="store_true", help="print everything it found as one JSON object") + p = sub.add_parser("up", help="start the local signer (dev mode), or find the running one") p.add_argument("--wait", action="store_true", help="return once it answers") p.add_argument("--json", action="store_true", help="wait, then print its hello as JSON") p.add_argument("--replace", action="store_true", help="stop the running dev signer first, even an incompatible one") - sub.add_parser("down", help="stop the same-user v2 dev signer") + sub.add_parser("down", help="stop the local signer (dev mode)") p = sub.add_parser("doctor", help=f"check this setup (v1/v2 system mode, the dev signer); {wiki_url('docs/doctor.md')}") p.add_argument("--json", action="store_true", help="print [{id, status, detail, fix}]") p.add_argument("--config", help="check the v2 signer of this signer.yaml") @@ -251,13 +265,15 @@ def _parser(): p.add_argument("--manifests", metavar="DIR", help="with --k8s: check these rendered manifests instead") p.add_argument("--issuer-key", metavar="KEY_ID", help="the record key issuer's KMS key: the signer must not sign " "with it") - p = sub.add_parser("uninstall", help="remove hooks (the ledger is kept)") + p = sub.add_parser("uninstall", help="remove Tracekit's hooks (your signed runs and keys stay: it says where, and " + "how to delete them too)") p.add_argument("--project", action="store_true") p.add_argument("--v2", action="store_true", help="as root: remove v2 system mode (service, configs, hooks)") p.add_argument("--purge", action="store_true", help="with --v2: also delete the signer's keys, log and user") p.add_argument("--agent", choices=("claude", *agent_hooks.HARNESSES), default="claude") - p = sub.add_parser("ingest", help="remote ingestion gateway: `ingest token NAME` / `ingest serve`", add_help=False) + p = sub.add_parser("ingest", help="remote ingestion gateway (experimental): `ingest token NAME` / `ingest serve`", + add_help=False) p.add_argument("rest", nargs=argparse.REMAINDER) p = sub.add_parser("analyze", help="run the detectors over a run and sign the findings into the ledger " "(--v2: check a v2 run's request, reasoning and actions)", add_help=False) @@ -311,9 +327,10 @@ def _parser(): p = sub.add_parser("follow", help="follow agentsh's audit log into the v2 signer as it is written: " "`follow agentsh --log PATH`", add_help=False) p.add_argument("rest", nargs=argparse.REMAINDER) - p = sub.add_parser("otel", help="OpenTelemetry receiver: `otel serve` records agent spans sent over OTLP/HTTP", add_help=False) + p = sub.add_parser("otel", help="OpenTelemetry receiver (experimental): `otel serve` records agent spans sent over " + "OTLP/HTTP", add_help=False) p.add_argument("rest", nargs=argparse.REMAINDER) - p = sub.add_parser("daemon", help="run tracekitd in the foreground") + p = sub.add_parser("daemon", help="run the v1 signer, tracekitd, in the foreground (experimental)") p.add_argument("--home") p = sub.add_parser("proxy", help="run the model proxy in the foreground (experimental)") p.add_argument("--home") @@ -321,7 +338,8 @@ def _parser(): p.add_argument("--port", type=int) p.add_argument("--upstream") - p = sub.add_parser("observe", help="now part of `tracekit view --ledger`: the v1 ledger or a bundle (read-only web UI)") + p = sub.add_parser("observe", help="the old name of `tracekit view --ledger`: the v1 ledger or a v1 bundle in a " + "read-only web page") p.add_argument("rest", nargs=argparse.REMAINDER) p = sub.add_parser("view", help="the viewer: the v2 signer's runs, each verified; --why DIR, --ledger, --bundle (read-only web UI)") p.add_argument("rest", nargs=argparse.REMAINDER) @@ -420,9 +438,9 @@ def _parser(): "(the full report; scripts that read the text use it, or --json)") p.add_argument("--json", action="store_true") - p = sub.add_parser("policy", help="policy v2: `policy compile FILE` prints canonical JSON and its hash; `policy lint " - "FILE`; `policy backtest --policy P (--store DIR | --bundle B...)` prints the " - "incidents its detection would open") + p = sub.add_parser("policy", help="check a v2 policy and try it on past runs: `policy compile FILE` prints " + "canonical JSON and its hash; `policy lint FILE`; `policy backtest --policy P " + "(--store DIR | --bundle B...)` prints the incidents its detection would open") p.add_argument("action", choices=("compile", "lint", "backtest")) p.add_argument("file", nargs="?") p.add_argument("--policy", help="backtest: the policy file") @@ -656,6 +674,7 @@ def _run(a): print(install.thinking_line(hooks, on)) if hooks: print(guidance()) + print(next_step()) return 0 try: cfg = install.init_dev(home, a.witness, a.checkpoint_every, hooks, proxy=a.proxy, proxy_port=a.proxy_port, @@ -695,7 +714,11 @@ def _run(a): from . import client system = client.system_config() # a root-owned config is system mode: the signer isn't the same user's dev = (st["client_config"] or {}).get("mode") == "dev" or v2.get("dev") and not system - print(json.dumps({**st, "signer_v2": v2, **({"dev_mode": install.DEV_MODE_LINE} if dev else {})}, indent=2)) + if a.json: + print(json.dumps({**st, "signer_v2": v2, **({"dev_mode": install.DEV_MODE_LINE} if dev else {})}, indent=2)) + return 0 + print(*_status_lines(st, v2), *([install.DEV_MODE_LINE] if dev else []), + "(tracekit status --json prints all of it)", sep="\n") return 0 if a.cmd == "up": from . import install @@ -736,6 +759,8 @@ def _run(a): return 0 if a.cmd == "uninstall" and a.agent != "claude": agent_hooks.install(a.agent, os.getcwd() if a.project else None, uninstall=True) # it says what it removed + for line in [] if a.project else _kept(): + print(line) return 0 if a.cmd == "uninstall": from . import install @@ -745,6 +770,8 @@ def _run(a): except install.SettingsError as e: print(f"tracekit: {e}", file=sys.stderr) return 1 + for line in [] if a.project else _kept(): + print(line) return 0 if a.cmd == "daemon": from . import daemon @@ -761,6 +788,9 @@ def _run(a): return _approvals(argparse.Namespace(action=a.cmd, approval_id=a.approval_id, signer=None, reason=None)) if a.cmd in ("pending", "approve", "reject"): from . import client + if a.cmd == "pending" and client.socket_path() == client.DEFAULT_SOCKET \ + and not os.path.exists(client.DEFAULT_SOCKET): # no v1 signer here: the v2 signer's held calls + return _approvals(argparse.Namespace(action="list", signer=None, json=False, all=False)) try: pend = client.rpc({"op": "approval_list"}).get("pending", []) if a.cmd == "pending": @@ -904,6 +934,49 @@ def _run(a): return 2 +def _kept(): + """What `tracekit uninstall` leaves in place (the dev signer's keys and signed runs, the client config, the dev + signer's runtime dir, a v1 dev signer's home), each that exists, and the commands that delete them and Tracekit.""" + from . import client + from .sdk import autospawn + data = autospawn.data_dir() + where = [(data, "the dev signer's keys and signed runs: a bundle you exported verifies without them"), + (client.client_home(), "Tracekit's client config"), + (client.client_config().get("signer_home"), "the v1 dev signer's home"), + *[(d, "the dev signer's runtime dir") for d in autospawn.own_runtime_dirs()]] + where = [(d, what) for d, what in where if d and os.path.isdir(d)] + if not where: + return [] + paths = [d for d, _ in where] + up = os.path.dirname(data) + if sys.platform == "darwin": # data/ and run/ in Tracekit's own Application Support dir (autospawn.data_dir) + paths = [up] + [d for d in paths if not d.startswith(up + os.sep)] + pip = ("uv tool uninstall" if re.search(r"[/\\]uv[/\\]tools[/\\]", sys.executable) else # as claude_code.upgrade + "pipx uninstall" if re.search(r"[/\\]pipx[/\\]venvs[/\\]", sys.executable) else + f"{_quote(sys.executable)} -m pip uninstall") + remove = (f"Remove-Item -Recurse -Force {', '.join(map(_quote, paths))}" if os.name == "nt" + else f"rm -rf {' '.join(map(_quote, paths))}") + return (["kept:"] + [f" {d} ({what})" for d, what in where] + + ["to delete these and Tracekit too:", " tracekit down", f" {remove}", f" {pip} tracekit-ai"]) + + +def _status_lines(st, v2): + """`tracekit status` for a person: the v2 signer (install.status's and autospawn.status's findings), where its + keys and runs are, the hooks, then a v1 signer and proxy when there is one.""" + name = "dev signer" if v2.get("dev") else "signer" + lines = [f"signer {name} {v2['version']}, running (pid {v2['pid']}), socket {v2['socket']}" if "version" in v2 + else f"signer {name}: {v2.get('error') or 'running'}"] + if v2.get("data_dir"): + lines.append(f"data {v2['data_dir']} (its keys and signed runs)") + lines += [f"hooks {' and '.join(h['versions'])} hooks in {h['settings']} ({len(h['events'])} events)" + for h in st["hooks"]] or ["hooks none in Claude Code's settings: tracekit init --dev --v2 adds them"] + if st.get("signer"): + lines.append(f"v1 signer {st['signer'].get('error') or 'answering'}") + if st.get("proxy"): + lines.append(f"proxy {st['proxy']['url']}: {'reachable' if st['proxy']['reachable'] else 'unreachable'}") + return lines + + def _signer_extra(): """True when the v2 signer's policy engine imports; else says how to install it.""" from .policy2 import engine @@ -1048,7 +1121,7 @@ def _last(a): from .verify import v2 from .view import write_trust from .sdk.autospawn import data_dir - deadline = time.monotonic() + LAST_WAIT_S + deadline, waiting = time.monotonic() + LAST_WAIT_S, False store = os.path.join(data_dir(), "store") # named when there is none yet try: cfg = signer_config(None) @@ -1067,6 +1140,10 @@ def _last(a): r.close() if not (runs and runs[0][1] == "run.closing" and time.monotonic() < deadline): break + if not waiting: + print(f"tracekit last: run {runs[0][0][1]} ({runs[0][2]}) just closed; waiting up to {LAST_WAIT_S:g} s " + "while the signer finalises it", file=sys.stderr) + waiting = True time.sleep(0.25) # the signer finalises a closed run once its grace window has passed except FileNotFoundError: runs = [] @@ -1084,8 +1161,11 @@ def _last(a): pick = runs if a.open else final if not pick: if not runs: - print(f"tracekit last: no runs in the dev signer's store ({store}) yet: add `import tracekit; " - "tracekit.instrument()` to your agent and run it", file=sys.stderr) + print(f"tracekit last: no runs in the dev signer's store ({store}) yet. `tracekit.instrument()` records " + "only the frameworks it lists, and says so when it finds none of them " + f"({wiki_url('docs/quickstart-v2.md', '2-one-line-in-your-agent')}); for any other agent, record " + f"its calls as {wiki_url('docs/quickstarts/custom.md')} shows. Claude Code: a session started after " + "`tracekit init --dev --v2` records its calls.", file=sys.stderr) return 1 (tenant, run_id), _, agent = pick[0] out = os.path.abspath(a.out or f"{run_id}.tkb") @@ -1171,11 +1251,10 @@ def _stop_lines(c, x): """What `approvals show` adds to approval `x`: why its call was held and what to do instead (the reasons and remedies of its rules, in the policy whose hash it records: a shipped pack's, or one the dev signer kept), when it expires, the commands that answer it and where the viewer opens it.""" - import datetime - from . import view - from .integrations.claude_code import approvals_cmd + from .integrations.claude_code import approvals_cmd, expires from .policy2.compile import ENGINE_REASONS + from .policy2.engine import Engine from .signer.service import dev_data_dir try: kept = None if c.signer else os.path.join(dev_data_dir(create=False), "policies") @@ -1183,34 +1262,60 @@ def _stop_lines(c, x): kept = None reasons = view.policy_reasons(kept=kept) known = reasons.get(x["policy_hash"], {}) - flags = {r["id"] for r in reasons.policies.get(x["policy_hash"], {}).get("flag", [])} # a flagged call runs - rules = [(i, known.get(i) or ENGINE_REASONS.get(i) or {"reason": view.ENGINE_RULES.get(i, "")}) + pol = reasons.policies.get(x["policy_hash"], {}) + flags = {r["id"] for r in pol.get("flag", [])} # a flagged call runs + engine = dict(ENGINE_REASONS) + if "TK-SHELL-PARSE" in x["rule_ids"] and pol and x["args"] is not None: + # the cause the signer gave the hook (its decision's `unread`), from the same engine on the signer's copy + try: + cause = Engine(pol).decide(x["tool"], _args(x)).get("unread") + except Exception: # as the signer's _evaluate: an engine error here leaves the general reason + cause = None + engine["TK-SHELL-PARSE"] = ENGINE_REASONS["TK-SHELL-PARSE"]["causes"].get(cause) or engine["TK-SHELL-PARSE"] + rules = [(i, known.get(i) or engine.get(i) or {"reason": view.ENGINE_RULES.get(i, "")}) for i in x["rule_ids"] if i != "TK-UNKNOWN-TOOL" and i not in flags] why = "; ".join(f"{r['reason']} ({i})" for i, r in rules if r.get("reason")) instead = " ".join(dict.fromkeys(r["remedy"] for _, r in rules if r.get("remedy"))) - try: - left = (datetime.datetime.strptime(x["expires_at"], "%Y-%m-%dT%H:%M:%S.%fZ") - .replace(tzinfo=datetime.timezone.utc) - datetime.datetime.now(datetime.timezone.utc)).total_seconds() - left = f" (in {left // 60:.0f} min)" if left >= 0 else f" ({-left // 60:.0f} min ago)" - except ValueError: # another signer's time format: as it is - left = "" cmd = approvals_cmd(c.signer) why = why or ("not on this machine: `tracekit policy compile ` prints the reasons of the signer's policy, " + x["policy_hash"]) + # Claude Code runs a `!` line in the session as its OS user: the person's, with the dev signer (as the hook says) + bang = not c.signer and _claude_code_run(x["run_id"]) return [f"why {why}", *([f"instead {instead}"] if instead else []), - f"expires {x['expires_at']}{left}", + f"expires {expires(x['expires_at']) or x['expires_at']}", *([f"answer `{cmd} approve {x['approval_id']}` to allow it once, `{cmd} reject {x['approval_id']}` " - "to refuse it" + ("" if c.signer else "; in the agent's Claude Code session, with a leading `! `")] + "to refuse it" + ("; in the agent's Claude Code session, with a leading `! `" if bang else "")] if x["state"] == "requested" else []), f"record tracekit view --approval {x['approval_id']}" + (" (with the --config or --data-dir of this signer)" if c.signer else "")] +def _claude_code_run(run_id): + """Whether run `run_id` is a Claude Code session's on this machine: the v2 hook keeps each session's run in its + state in the dev signer's runtime dir (integrations.claude_code._state).""" + import glob + + from .sdk import autospawn + try: + d = autospawn.runtime_dir(create=False) + except (OSError, autospawn.SignerUnavailable): + return False + for f in glob.glob(os.path.join(glob.escape(d), "claude-code-*.json")): + try: + with open(f, encoding="utf-8") as fh: + if json.load(fh).get("run_id") == run_id: + return True + except (OSError, ValueError, AttributeError): # one a hook is replacing, or not a state file + continue + return False + + def _approvals(a): """`tracekit approvals`. Strings an agent chose (tool, args, reason) are printed JSON-escaped, never raw.""" import pydoc + from .integrations.claude_code import expires from .sdk.client import Client, Incompatible, SignerUnavailable from .signer.rpc_schema import RPCError if a.action == "slack": @@ -1234,7 +1339,7 @@ def _approvals(a): for x in [] if a.json else shown: state = x["state"] + (" (run closing: expires with it)" if x.get("run_closing") else "") print(f"{x['approval_id']} {state} run={x['run_id']} tool={json.dumps(x['tool'])} " - f"rules={','.join(x['rule_ids'])} expires {x['expires_at']}" + ( + f"rules={','.join(x['rule_ids'])} expires {expires(x['expires_at']) or x['expires_at']}" + ( f" {json.dumps(_gist(c, x['approval_id']))}" if x["state"] in ("requested", "approved") else "")) if a.json: diff --git a/tracekit/demo_server.py b/tracekit/demo_server.py index aff22234..8415773c 100644 --- a/tracekit/demo_server.py +++ b/tracekit/demo_server.py @@ -69,12 +69,13 @@ def agent(client): for call_id, command in CALLS: args = {"command": command} d = run.decide(call_id, "Bash", args) - print(f" Bash {command!r}: {d['decision']} {' '.join(d['rule_ids'])}") + print(f" Bash {command!r}: {' '.join([d['decision'], *d['rule_ids']])}") hint = None if d["decision"] == "ask": hint = run.call("approval_request", tool_call_id=call_id)["approval_id"] a = client.approval_decide({"approval_id": hint, "decision": "approve"}) # `tracekit approvals approve` - print(f" approved {hint} (self-approved: {a['self_approved']})") + print(f" approved {hint}" + (", by the run's own user: a self-approval, so the run's assurance stays " + "dev" if a["self_approved"] else "")) if d["decision"] != "deny" and run.approval_consume(call_id, "Bash", args, approval_id_hint=hint)["ok"]: run.complete(call_id) # the pretend command ran return run.run_id diff --git a/tracekit/deploy/files.py b/tracekit/deploy/files.py index 2a2a2ae0..d1bdf1d6 100644 --- a/tracekit/deploy/files.py +++ b/tracekit/deploy/files.py @@ -161,6 +161,12 @@ def replace_retrying(src, dst, tries=20): time.sleep(0.05) +def remove(d, name): + """Delete `name` in d; a symlink itself, never what it points to.""" + path, at = _at(d, name) + os.unlink(path, **at) + + def backup(d, name, data): """Save data as a new `name.bak-