diff --git a/docs/doctor.md b/docs/doctor.md index e03fd69d9..e58b856f5 100644 --- a/docs/doctor.md +++ b/docs/doctor.md @@ -4,7 +4,7 @@ sudo /opt/tracekit/bin/python -I -m tracekit doctor # system mode: run it as root, so it can probe as the agent's user tracekit doctor # the v2 dev signer tracekit doctor --config signer.yaml # one v2 signer, by its config -tracekit doctor --json # [{"id", "status": "ok" | "warn" | "fail", "detail", "fix"}] +tracekit doctor --json # [{"id", "status": "ok" | "info" | "warn" | "fail", "detail", "fix"}] tracekit doctor --k8s # inside a pod: the pod and its service account, read from the API server tracekit doctor --k8s --manifests DIR # rendered manifests (helm template, kustomize build) instead tracekit doctor --config signer.yaml --issuer-key alias/tracekit-ca # also: may this principal sign with the CA key? @@ -20,7 +20,7 @@ Doctor works out which setup it's looking at and runs the checks that apply: - **`--k8s`:** the Kubernetes checks below, of the pod doctor runs in (it needs `get` on its pod and service account) or of the manifests under `--manifests DIR`. With `--config` it runs that signer's checks too. -Exit codes: 1 when any check fails, else 0. A warning is advice: it doesn't fail doctor. +Exit codes: 1 when any check fails, else 0. A warning is advice: it doesn't fail doctor. `info` says what a setup doesn't have and needn't (a witness for the dev signer, which uses the public witness by default and has none only until a release names it or after `--no-witness`; Claude Code's hooks where there is no Claude Code; before the dev signer's first start, the keys and report that start makes: `D-CONTENT-VAULT`, `D-KEYS-DISTINCT`, `D-KEY-HYGIENE`). A dev run whose only warning is `D-PROCESS-BOUNDARY` ends with `dev setup OK`. **Doctor output is advice, not evidence.** It describes the host at the moment you run it. Nothing it prints is signed, and a verifier never reads it. What a bundle proves comes from its signatures, its checkpoints and the verifier's own @@ -42,16 +42,16 @@ skipped with the reason) and checks that doctor flags each one. It also checks t | `D-AGENT-PROBE` | the agent's user's actual access | as that user (doctor forks and drops to its uid), it can read or write the data dir, the keys dir or a key, or write signer.yaml, the policy or `client.json` (warn: doctor isn't running as root or the agent) | `chmod o-rwx`, `go-w`; owner root or the signer's user | | `D-CODE-TRUST` | the root-owned venv `/opt/tracekit` | a file in it, or a directory above it, isn't root-owned or is group- or world-writable, or the venv is missing | re-run init to reinstall it | | `D-KEYS-MODE` | `data_dir` and `data_dir/keys` | either one isn't 0700, or a key isn't a 0600 file owned by the keys dir's owner (warn: doctor can't read them, so run it as root) | `chmod 700` the dirs, `chmod 600` the keys | -| `D-CONTENT-VAULT` | `content` in signer.yaml and `keys/content.key` | the `content` section isn't valid (as the signer checks it), or the vault is on and its key isn't a 0600 file of the signer's user (warn: no key yet, or doctor can't read it; the vault off while runs kept before are still retained: "off, N runs still retained"). The detail says whether the vault is on and its retention | fix `content`; `chmod 600`, chown to the signer's user; erase the retained runs or keep them | -| `D-KEYS-DISTINCT` | `keys/log.key` and `keys/record.key` | they're the same key (warn: none yet); ok when `log_key` names a KMS key | move both away and restart the signer: new keys start a new log | +| `D-CONTENT-VAULT` | `content` in signer.yaml and `keys/content.key` | the `content` section isn't valid (as the signer checks it), or the vault is on and its key isn't a 0600 file of the signer's user (warn: no key yet, or doctor can't read it; dev: info before the dev signer's first start; the vault off while runs kept before are still retained: "off, N runs still retained"). The detail says whether the vault is on and its retention | fix `content`; `chmod 600`, chown to the signer's user; erase the retained runs or keep them | +| `D-KEYS-DISTINCT` | `keys/log.key` and `keys/record.key` | they're the same key (warn: none yet; dev: info before the dev signer's first start); ok when `log_key` names a KMS key | move both away and restart the signer: new keys start a new log | | `D-KMS-LOG-KEY` | `log_key: {aws_kms}`, with this host's AWS credentials | the key isn't an `ECC_NIST_EDWARDS25519` `SIGN_VERIFY` key (warn: not checked, without boto3, credentials or access) | create the log key with that spec | | `D-KMS-ISSUER-SIGN` | `--issuer-key KEY_ID`, the record key issuer's CA key | this principal may call `kms:Sign` on it (a `DryRun` Sign succeeds); only the issuer may (warn: the answer was neither allowed nor denied) | remove `kms:Sign` on it from the signer's IAM role | | `D-PG-SIGNER-ROLE` | `storage: {postgres}`: the role of `dsn_file`'s DSN | it is superuser or has UPDATE, DELETE or TRUNCATE on `tracekit_records`, `_registry`, `_notes` or `_anchors` (warn: not checked, without psycopg or a connection) | grant it only `tracekit.storage.postgres.GRANTS` | | `D-PG-READER-ROLES` | every other role | one that is neither superuser nor the tables' owner can INSERT, UPDATE, DELETE or TRUNCATE a log table | give readers SELECT only | -| `D-KEY-HYGIENE` | `data_dir/hygiene.json`, written by the signer on start | the signer process may dump core (RLIMIT_CORE not 0), is dumpable (Linux `PR_SET_DUMPABLE`), or could not mlock its key files (warn; also when the signer never started) | run it with `tracekit signer serve`; raise `LimitMEMLOCK` | +| `D-KEY-HYGIENE` | `data_dir/hygiene.json`, written by the signer on start | the signer process may dump core (RLIMIT_CORE not 0), is dumpable (Linux `PR_SET_DUMPABLE`), or could not mlock its key files (warn; also when the signer never started, dev: info before its first start) | run it with `tracekit signer serve`; raise `LimitMEMLOCK` | | `D-UNIT-HARDENING` | `tracekit-signer.service` (or the launchd plist) | a hardening directive that init writes is missing: the detail gives the score and lists the missing directives (warn) | re-run init to rewrite the service file | | `D-HARNESS-HELPER` | `tracekit-harness-helper.service` (or its launchd plist), when signer.yaml has `harness_binding` | the file is missing, or its capability bounding set is anything but `CAP_SYS_PTRACE CAP_DAC_READ_SEARCH` | re-run init with `--harness` | -| `D-HOOKS-PRESENT` | the agent's Claude Code settings, and whether they enable the Tracekit plugin (`enabledPlugins`, whose hooks count); Codex's, Gemini CLI's, Cursor's, goose's, Qwen Code's, Factory Droid's, Antigravity CLI's and Crush's v2 hooks and OpenCode's v2 plugin count too | a tool event (PreToolUse, PostToolUse, PostToolUseFailure) has no v2 hook, the plugin isn't enabled and no other coding agent has v2 hooks (not when `tracekit init --v2 --no-hooks`, dev or system mode, recorded that you chose none: the check passes and says how to add them); or the plugin and init's hooks both run, so every approved call is blocked; or (warn) with `init --v2 --managed`, the agent user's own `~/.claude/settings.json` has Tracekit's v2 hooks too, so every call is decided twice (not with `allowManagedHooksOnly` on, which stops them running: the check says so and passes) | re-run init; with both, keep one (`tracekit uninstall`, or `/plugin disable` with the plugin key the check names, e.g. `tracekit@tracekit`); with `--managed`, re-run init with `--managed`, which takes them out of the agent user's file | +| `D-HOOKS-PRESENT` | the agent's Claude Code settings, and whether they enable the Tracekit plugin (`enabledPlugins`, whose hooks count); Codex's, Gemini CLI's, Cursor's, goose's, Qwen Code's, Factory Droid's, Antigravity CLI's and Crush's v2 hooks and OpenCode's v2 plugin count too | a tool event (PreToolUse, PostToolUse, PostToolUseFailure) has no v2 hook, the plugin isn't enabled and no other coding agent has v2 hooks (not when `tracekit init --v2 --no-hooks`, dev or system mode, recorded that you chose none: the check passes and says how to add them; dev: info when there is no Claude Code, no `claude` on PATH and no `~/.claude`); or the plugin and init's hooks both run, so every approved call is blocked; or (warn) with `init --v2 --managed`, the agent user's own `~/.claude/settings.json` has Tracekit's v2 hooks too, so every call is decided twice (not with `allowManagedHooksOnly` on, which stops them running: the check says so and passes) | re-run init; with both, keep one (`tracekit uninstall`, or `/plugin disable` with the plugin key the check names, e.g. `tracekit@tracekit`); with `--managed`, re-run init with `--managed`, which takes them out of the agent user's file | | `D-HOOKS-MANAGED` | with `init --v2 --managed`: Claude Code's admin-managed settings file, which holds the hooks | the file, or a directory above it, isn't root-owned or is group- or world-writable, so the agent's user could replace it; or (as root) the agent's user can't read it, so its Claude Code starts without it | `sudo chown 0:0` and `sudo chmod 644` the file, `sudo chmod go-w` the directories above it; or re-run init with `--managed` (and `--managed-only` when an init with it turned `allowManagedHooksOnly` on) | | `D-HOOKS-EVENT` | Claude Code, Codex, Cursor, Gemini CLI, goose, Qwen Code, Factory Droid, Antigravity CLI and Crush hook commands (OpenCode's JavaScript plugin isn't checked); for Claude Code also each Tracekit hook's `matcher` and `disableAllHooks` in the settings files doctor reads (dev mode: the user file, the file init recorded, the working directory's project and local files, and managed settings; system mode: the agent user's file, the file init wrote and managed settings; with `--managed`, managed settings only, as only they can turn managed hooks off) | a hook command doesn't name its configured event (installed before 1.2), so a pre-tool payload the hook can't read lets the call through unrecorded; a hook other than the pre-tool one blocks its event when it can't run (a command init wrote before 1.3, or a Cursor hook with `failClosed` on that event, as init wrote up to 1.4.2), so a broken install blocks prompts and Stop too; or, for the other coding agents, a call can run that no hook decided: no Tracekit v2 hook is on the pre-tool event, Cursor's `preToolUse` hook has no `failClosed: true`, or goose's `PreToolUse` hook has no `on_failure: "block"`; or, for Claude Code, a call can run that no hook decided: `disableAllHooks` is set in one of those files, so no hook runs (fail in production), a Tracekit hook's matcher isn't `*`, empty or absent, so it skips the other tools, or the PreToolUse command isn't wrapped to block the call when the hook can't run (POSIX) (warn) | re-run init for the active mode: `tracekit init --dev --v2 [--agent NAME]` in dev, or `sudo /opt/tracekit/bin/python -I -m tracekit init --v2 --user AGENT [--agent NAME]` in system mode (with Python 3.9 or 3.10, [laptop system mode](deploy.md#laptop-system-mode)'s command); remove `disableAllHooks` from the file the check names | | `D-HOOKS-VENV` | the hook commands | a hook isn't the command init writes for the root-owned venv: another Python, or written by init before 1.3 | re-run init | @@ -69,7 +69,7 @@ skipped with the reason) and checks that doctor flags each one. It also checks t | `D-POLICY-ENGINE` | the policy, compiled by policy2 | it doesn't compile, or neither `google-re2` nor `regex` is installed | `tracekit policy lint FILE`; `pip install tracekit-ai` (a `--no-deps` install leaves the engine out) | | `D-DURABILITY` | `durability` | it's `ack-on-write` in a production profile, so a power loss can drop records that were already acknowledged (warn) | `durability: ack-on-fsync` | | `D-FAIL-MODES` | `fail_modes` and `client.json` `fail_mode` | a tool class, or the system config, fails open (warn) | set them to `closed` unless an outage must not stop the agent | -| `D-WITNESS-CONFIGURED` | `witnesses` | there are none, so checkpoints aren't cosigned and assurance stays `dev` (`local` when `anchors` is set) (dev: warn) | add a witness (`docs/witnesses.md`) | +| `D-WITNESS-CONFIGURED` | `witnesses` | there are none, so checkpoints aren't cosigned and assurance stays `dev` (`local` when `anchors` is set) (dev: info: the dev signer uses the public witness by default, so it has none only until a release names it or after `init --dev --v2 --no-witness`) | add a witness (`docs/witnesses.md`) | | `D-WITNESS-OWNERSHIP` | the witnesses' ownership classes | every witness is `operator`: the default production profile needs at least one `public`, `customer` or `tracekit` witness (dev: warn) | add a non-operator witness | | `D-WITNESS-FRESH` | `store/witness-queue.json` against the latest checkpoint | a witness has never cosigned the latest checkpoint, or has been failing for longer than the witness gap window (300 s) (warn) | check the witness is reachable and accepts this log | | `D-CLOCK-SKEW` | this clock against the latest cosignature timestamp | this clock is more than 300 s behind it (warn) | sync the clock (NTP) | diff --git a/docs/integrations.md b/docs/integrations.md index 37eae8d97..70485301f 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -32,7 +32,7 @@ Another agent: record its calls with the SDK's run handle ([custom agent](quicks | Agent | Status | Guide | |---|---|---| -| Claude Code hooks (`tracekit init --dev --v2`) | Stable | [quickstart](quickstart-v2.md) | +| Claude Code hooks (`tracekit init --dev --v2`) | Stable | [quickstart](quickstarts/claude-code.md) | | The Claude Code plugin | Experimental | [plugin](../plugin/README.md) | | Codex CLI, Cursor, Gemini CLI, OpenCode | Experimental | [coding agents](coding-agents.md) | | goose (`tracekit init --dev --v2 --agent goose`) | Experimental | [coding agents](coding-agents.md#on-the-v2-signer) | diff --git a/docs/portability.md b/docs/portability.md index 17c04cd21..76c736c67 100644 --- a/docs/portability.md +++ b/docs/portability.md @@ -38,7 +38,7 @@ Things to check on first use, because they depend on the machine: - The Python interpreter and the `tracekit` package must be readable by `_tracekit`. A Python under your home directory is not, and the daemon will fail to start; look at `/var/lib/tracekit/tracekitd.log`. -- `tracekit status` should show `signer_isolation: separate-user` and the ledger owned by `_tracekit`. +- `tracekit status --json` should show `signer_isolation: separate-user` and the ledger owned by `_tracekit`. - A quick isolation test: `echo x >> /var/lib/tracekit/ledger/ledger.jsonl` as your own user must fail. If any of that is wrong, please open an issue with the output. Until it has been validated on hardware the diff --git a/docs/quickstart-v2.md b/docs/quickstart-v2.md index bd53ecea9..3e925db78 100644 --- a/docs/quickstart-v2.md +++ b/docs/quickstart-v2.md @@ -58,16 +58,88 @@ A hook whose signer runs another version says so too, once per session, with the import tracekit; tracekit.instrument() # the first line, before your agent builds its agents, tools or sessions ``` -Then run your agent as usual. `instrument()` registers a run with the signer and closes it when the process exits. It -wires every framework it finds installed: OpenAI Agents SDK tools, LangGraph / LangChain `ToolNode`s (which -`create_agent` builds), Claude Agent SDK hooks, MCP `ClientSession.call_tool`, Browser Use's `Tools` (which an `Agent` -builds), CrewAI's tool hooks, Pydantic AI `Agent`s (the `Tracekit` capability), and the OpenAI, Anthropic and Google Gen AI clients' model calls. Each tool call is decided by the signer -before it runs (`allow`, `deny`, `ask`) and recorded after. A denied call reaches the model as a refusal and the agent -goes on. Each model response is committed to and, with the signer's content vault on (the dev signer keeps it), kept -in the vault, so `tracekit analyze --v2` can read what the model said before and after each call ([findings](findings.md#v2-runs-what-was-asked-said-and-done-tracekit-analyze---v2)). -Calling it again returns the same run. With none of these installed it warns and does nothing. TypeScript: -`const tk = await instrument()` from `@cygnuxlabs/tracekit`, then pass `tk.ai` (Vercel AI SDK) or `tk.agents` (OpenAI Agents -JS) to the framework. +Then run your agent as usual. You don't start anything: the first call starts a dev signer in the background. +`instrument()` records only the frameworks it finds installed: OpenAI Agents SDK tools, LangGraph / LangChain +`ToolNode`s (which `create_agent` builds), Claude Agent SDK hooks, MCP `ClientSession.call_tool`, Browser Use's +`Tools`, CrewAI's tool hooks, Pydantic AI `Agent`s, and the OpenAI, Anthropic and Google Gen AI clients' model calls. +With none of them installed it warns and records nothing: record your agent's calls yourself +([a custom agent](quickstarts/custom.md)). + +Each tool call is decided before it runs (`allow`, `deny` or `ask`) and recorded after. A denied call reaches the model +as a refusal and the agent goes on. A held call (`ask`) waits for a person: run `tracekit approvals list` in another +terminal to see it, and `tracekit approvals approve ` to let it run once ([approve from another +terminal](#approve-from-another-terminal)). While it waits, an SDK agent prints nothing: it just seems to stop. + +Claude Code: `tracekit init --dev --v2` instead ([the Claude Code quickstart](quickstarts/claude-code.md)). +TypeScript: `const tk = await instrument()` from `@cygnuxlabs/tracekit`, then pass `tk.ai` (Vercel AI SDK) or +`tk.agents` (OpenAI Agents JS) to the framework. A server that handles many conversations, how each framework waits +for an approval and where the signer listens: [deeper](#deeper-drive-the-signer-yourself). + +## 3. See it verified + +```sh +tracekit last +``` + +```text +run 40c7ee89ae9eaec8f8eb8556b32fdc9d (my-agent) +VERIFIED: tool calls in signed records, none changed, removed or added after signing. +NOT PROOF AGAINST THE AGENT: in this setup the agent can reach the signing key, so it could have rewritten the record. +To make it proof, keep the key away from the agent: https://github.com/Cygnux-Labs/Tracekit/wiki/Deploy +NOT ANCHORED: no independent witness or anchor cosigned the log (https://github.com/Cygnux-Labs/Tracekit/wiki/Witnesses). +Pinned log key SHA-256 fingerprint: <64 hex digits>. +bundle: /home/me/project/40c7ee89ae9eaec8f8eb8556b32fdc9d.tkb +check it again: tracekit verify 40c7....tkb --trust 40c7....trust.json +next: tracekit view --dev (every run, replayed and reviewed) +``` + +`tracekit last` finds the dev signer's most recent finished run, exports it, writes a trust config that pins the dev +signer's log key next to the bundle, and verifies the bundle. `-o PATH` writes the bundle elsewhere. The report is +explained [below](#export-and-verify-by-hand). `tracekit view --dev` lists each run with its verdict, and replays and +reviews one ([viewer.md](viewer.md#laptop-viewer)). `tracekit status` says what is set up and where. + +Per framework, the same three steps with a 10-line example: [LangChain / LangGraph](quickstarts/langchain.md), +[OpenAI Agents SDK](quickstarts/openai-agents.md), [Claude Agent SDK](quickstarts/claude-agent-sdk.md), +[MCP client](quickstarts/mcp.md), [Claude Code](quickstarts/claude-code.md), [a custom agent](quickstarts/custom.md). + +## First-run errors + +| You see | What to do | +|---|---| +| `tracekit.instrument(): no signer answering at ...; dev auto-spawn is off while TRACEKIT_SIGNER is set` | Start the signer `TRACEKIT_SIGNER` names, or unset it for a same-user dev signer. | +| `tracekit.instrument(): ...; system mode: start the system signer` | `sudo /opt/tracekit/bin/python -I -m tracekit doctor` says what is wrong with the system signer. | +| `the Tracekit signer X (pid N, protocol [a, b]) ... cannot serve this client` | A dev signer from a newer Tracekit version is running (an older one is replaced on its own): `tracekit up --replace` stops it and starts this one. A signer you run yourself (`TRACEKIT_SIGNER`, system mode) that is older than this client: upgrade the signer first. From the TypeScript SDK, an older `tracekit` on your PATH starts an older signer: `pip install -U tracekit-ai`. | +| `tracekit last: the newer run ID (agent) is still open` | The agent is still running, or was killed before it closed its run. End it and run `tracekit last` again; meanwhile it shows the run before. | +| `tracekit last: no runs in the dev signer's store` | `instrument()` records only the frameworks it lists, and warns when it finds none of them: for another agent, record its calls as [the custom quickstart](quickstarts/custom.md) shows. Claude Code: start a new session after `tracekit init --dev --v2`. | +| `tracekit.instrument(): not wired ... Tracekit is tested with ...` | Install the versions it names, or wire that adapter by hand as its quickstart shows. | +| `the v2 signer needs its policy engine ... pip install tracekit-ai` | A `--no-deps` install verifies but can't sign: install with its dependencies. | +| A path with spaces (Windows profiles, macOS `Application Support`) | Paste the commands `tracekit last` prints as they are: their paths are quoted for your shell. | + +## Remove it + +```sh +tracekit uninstall # Claude Code's hooks; --agent NAME for another coding agent's +``` + +It removes Tracekit's hooks and keeps your other settings (a settings file init made, with nothing else in it, is +deleted). Then it lists what it kept: the dev signer's keys and signed runs, Tracekit's client config and the dev +signer's runtime dir, with the commands that delete them and Tracekit too, for example on Linux: + +```text +kept: + /home/me/.local/share/tracekit (the dev signer's keys and signed runs: a bundle you exported verifies without them) + /home/me/.tracekit-client (Tracekit's client config) +to delete these and Tracekit too: + tracekit down + rm -rf /home/me/.local/share/tracekit /home/me/.tracekit-client + /home/me/.venv/bin/python -m pip uninstall tracekit-ai +``` + +## Deeper: drive the signer yourself + +What `instrument()` and `tracekit last` do, by hand and in more detail: a server with a run per conversation, how each +framework waits for an approval, where the signer listens, and a scripted agent with the approval step a framework's +own control flow needs. ### A server: a run per conversation @@ -124,6 +196,8 @@ agent's retry of the same call runs once you approved it ([held calls](coding-ag | Vercel AI SDK (TS, `tk.ai`) | the SDK's own tool approval | none of its own | `approvalResponses(parts)`, pushed to `messages`, resumes it; a call not approved throws | | `tracekit.why` runtime | timed wait in `act()` | `approval_wait_s`, 300 s | an error result; retrying the same call in the same run reuses the approval | +### Where the signer runs + You don't start anything yourself. The first client that needs a signer starts one in the background and later clients reuse it (`tracekit up` / `tracekit down` do the same by hand). The signer's keys and store live in your user's data dir, one per `HOME`. It listens on a Unix socket in a private runtime dir; where two of your `HOME`s share that @@ -134,51 +208,6 @@ names. On Windows it listens on a loopback TCP port instead, and publishes the port and a token in `%LOCALAPPDATA%\tracekit\run\endpoint.json`, which only you can read. Each side proves it holds the token before any request is answered. -## 3. See it verified - -```sh -tracekit last -``` - -```text -run 40c7ee89ae9eaec8f8eb8556b32fdc9d (my-agent) -VERIFIED: tool calls in signed records, none changed, removed or added after signing. -NOT PROOF AGAINST THE AGENT: in this setup the agent can reach the signing key, so it could have rewritten the record. -To make it proof, keep the key away from the agent: https://github.com/Cygnux-Labs/Tracekit/wiki/Deploy -NOT ANCHORED: no independent witness or anchor cosigned the log (https://github.com/Cygnux-Labs/Tracekit/wiki/Witnesses). -Pinned log key SHA-256 fingerprint: <64 hex digits>. -bundle: /home/me/project/40c7ee89ae9eaec8f8eb8556b32fdc9d.tkb -check it again: tracekit verify 40c7....tkb --trust 40c7....trust.json -next: tracekit view --dev (every run, replayed and reviewed) -``` - -`tracekit last` finds the dev signer's most recent finished run, exports it, writes a trust config that pins the dev -signer's log key next to the bundle, and verifies the bundle. `-o PATH` writes the bundle elsewhere. The report is -explained [below](#export-and-verify-by-hand). `tracekit view --dev` lists each run with its verdict, and replays and -reviews one ([viewer.md](viewer.md#laptop-viewer)). - -Per framework, the same three steps with a 10-line example: [LangChain / LangGraph](quickstarts/langchain.md), -[OpenAI Agents SDK](quickstarts/openai-agents.md), [Claude Agent SDK](quickstarts/claude-agent-sdk.md), -[MCP client](quickstarts/mcp.md), [a custom agent](quickstarts/custom.md). - -## First-run errors - -| You see | What to do | -|---|---| -| `tracekit.instrument(): no signer answering at ...; dev auto-spawn is off while TRACEKIT_SIGNER is set` | Start the signer `TRACEKIT_SIGNER` names, or unset it for a same-user dev signer. | -| `tracekit.instrument(): ...; system mode: start the system signer` | `sudo /opt/tracekit/bin/python -I -m tracekit doctor` says what is wrong with the system signer. | -| `the Tracekit signer X (pid N, protocol [a, b]) ... cannot serve this client` | A dev signer from a newer Tracekit version is running (an older one is replaced on its own): `tracekit up --replace` stops it and starts this one. A signer you run yourself (`TRACEKIT_SIGNER`, system mode) that is older than this client: upgrade the signer first. From the TypeScript SDK, an older `tracekit` on your PATH starts an older signer: `pip install -U tracekit-ai`. | -| `tracekit last: the newer run ID (agent) is still open` | The agent is still running, or was killed before it closed its run. End it and run `tracekit last` again; meanwhile it shows the run before. | -| `tracekit last: no runs in the dev signer's store` | Add `import tracekit; tracekit.instrument()` to your agent and run it first. | -| `tracekit.instrument(): not wired ... Tracekit is tested with ...` | Install the versions it names, or wire that adapter by hand as its quickstart shows. | -| `the v2 signer needs its policy engine ... pip install tracekit-ai` | A `--no-deps` install verifies but can't sign: install with its dependencies. | -| A path with spaces (Windows profiles, macOS `Application Support`) | Paste the commands `tracekit last` prints as they are: their paths are quoted for your shell. | - -## Deeper: drive the signer yourself - -The rest of this page does by hand what `instrument()` and `tracekit last` do, with the approval step a framework's -own control flow needs. - ### A scripted agent Before each tool call runs, the agent asks the signer. The signer answers `allow`, `deny` or `ask` and signs that @@ -197,9 +226,11 @@ with Client().run(agent="quickstart") as run: d = run.decide("call-3", "Bash", held) # ask: a command the signer can't parse waits for a human aid = run.call("approval_request", tool_call_id="call-3")["approval_id"] print("approve with: tracekit approvals approve", aid) - if run.call("approval_wait", approval_id=aid, timeout_ms=300000)["state"] == "approved" and \ - run.approval_consume("call-3", "Bash", held, approval_id_hint=aid)["ok"]: + state = run.call("approval_wait", approval_id=aid, timeout_ms=300000)["state"] # waits up to 5 minutes + if state == "approved" and run.approval_consume("call-3", "Bash", held, approval_id_hint=aid)["ok"]: run.complete("call-3") # run it only now, with exactly those arguments + else: # still "requested": nobody answered in time + print("call-3 not run:", "no answer within 5 minutes" if state == "requested" else state) print("run", run.run_id) ``` @@ -212,7 +243,11 @@ decision then names the arguments whose values only untrusted content brought in ([provenance](policy-v2.md#provenance-untrusted-and-from-untrusted)). Send the task as `trusted` from your app, not from the agent. -Claude Code: `tracekit init --dev --v2` wires its hooks to the same signer (`python -I -m tracekit.integrations.claude_code`). +Claude Code: `tracekit init --dev --v2` wires its hooks to the same signer ([the Claude Code +quickstart](quickstarts/claude-code.md)). Each hook command is plain text in `~/.claude/settings.json`: the Python +init ran with, then `-I -m tracekit.integrations.claude_code `; from a source checkout or a `pip install +--user`, `-I -c` with a one-line import that puts that directory first on the path, then the directory (and a +checkout's branch) and the event as quoted arguments. `tracekit demo --server` runs the whole loop in a temp dir, with a test witness's cosignature and a tampered copy that fails. A deployment for server-hosted agents: [deploy.md](deploy.md). diff --git a/docs/quickstarts/README.md b/docs/quickstarts/README.md index e46d4d09d..551737a53 100644 --- a/docs/quickstarts/README.md +++ b/docs/quickstarts/README.md @@ -6,7 +6,8 @@ reference page covers the rest. | Integration | Quickstart | Reference | |---|---|---| -| Coding agents | | [coding-agents.md](../coding-agents.md) | +| Claude Code | [claude-code.md](claude-code.md) | [the plugin](../../plugin/README.md) | +| Other coding agents (Codex, Cursor, Gemini CLI, ...) | | [coding-agents.md](../coding-agents.md) | | Hermes Agent | | [integrations/hermes.md](../integrations/hermes.md) | | OpenAI Agents SDK | [openai-agents.md](openai-agents.md) | [integrations/openai-agents.md](../integrations/openai-agents.md) | | Claude Agent SDK | [claude-agent-sdk.md](claude-agent-sdk.md) | [integrations/claude-agent-sdk.md](../integrations/claude-agent-sdk.md) | diff --git a/docs/quickstarts/claude-agent-sdk.md b/docs/quickstarts/claude-agent-sdk.md index 9abb05ca2..e72277d82 100644 --- a/docs/quickstarts/claude-agent-sdk.md +++ b/docs/quickstarts/claude-agent-sdk.md @@ -4,10 +4,23 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no API key + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'claude-agent-sdk>=0.2.165,<0.3' +python examples/v2/claude_agent_sdk/agent.py --scripted +``` + +A mock CLI stands in for Claude: `ls` is allowed, `sudo rm -rf /` is denied and the agent goes on, and `echo +"unterminated` is held until a second process approves it. Then the run is exported and verified. + ## 1. Install ```sh pip install tracekit-ai 'claude-agent-sdk>=0.2.165,<0.3' +export ANTHROPIC_API_KEY=... # the SDK runs the Claude Code it bundles, which calls Claude ``` ## 2. One line in your agent @@ -55,8 +68,7 @@ async for message in query(prompt="Use the Bash tool to list the files here.", o The example runs a mock CLI (`transport=`) so it needs no `claude` binary and no API key. An ask holds the call inside the PreToolUse hook for up to nine minutes; SessionEnd closes the run. -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/claude_agent_sdk/agent.py ---scripted` from a checkout. +The no-key example at the top of this page wires it this way, with an approval. ## Read the verify report diff --git a/docs/quickstarts/claude-code.md b/docs/quickstarts/claude-code.md new file mode 100644 index 000000000..5b4f54e9f --- /dev/null +++ b/docs/quickstarts/claude-code.md @@ -0,0 +1,96 @@ +# Quickstart: Claude Code + +Every tool call Claude Code makes is decided by the v2 signer before it runs and recorded after, as a signed event. +This page sets it up on your own machine with the dev signer, which runs as your own user (see [what dev assurance +means](../quickstart-v2.md#what-dev-assurance-means)). No API key: Claude Code uses its own sign-in. + +## 1. Install and wire the hooks + +```sh +pip install tracekit-ai # or pipx install tracekit-ai, uv tool install tracekit-ai +tracekit init --dev --v2 +``` + +Init adds Tracekit's hooks to Claude Code's user settings (`~/.claude/settings.json`, or the `settings.json` in +`$CLAUDE_CONFIG_DIR`) and leaves your other settings and hooks alone. `--project` writes them to +`./.claude/settings.json` instead. The hooks run the Python you ran init with. Init also turns on Claude Code's +thinking summaries (`showThinkingSummaries`) unless your settings set them, so `tracekit analyze --v2` can read the +model's reasoning; `--no-thinking-summaries` leaves the setting alone. + +You don't start a signer: the first hook starts the dev signer in the background (`tracekit up` and `tracekit down` +do it by hand). `tracekit doctor` checks the setup and ends with `dev setup OK` when everything checks out. + +The [Claude Code plugin](../../plugin/README.md) is the other way in: the same hooks, from the plugin instead of +init's entries in your settings. Use one, not both: with the plugin enabled, init installs no hooks and says so. + +## 2. Start a new Claude Code session + +Hooks are read when a session starts, so a session that was already open records nothing: start a new one. From then +on each tool call is decided before it runs: most are allowed, some are denied, and some are held for you. + +A denied call never runs, and Claude sees why: + +```text +[tracekit] BLOCKED, nothing ran, and no one can approve it (TK-D001, TK-D004, TK-F003). +Why: privilege escalation; recursive delete of root or home +Instead: Run the command as the agent's own user, without sudo; ... +``` + +A held call doesn't run either, until you approve it. Claude Code shows the hold in the session: + +```text +[tracekit] HELD, nothing ran: apr-6f0c… (TK-SHELL-PARSE), expires 14:26 UTC. +Why: a quote on the line is never closed +Instead: Close the quote. +Person: look with `! tracekit approvals show apr-6f0c…`; allow once with `! tracekit approvals approve apr-6f0c…`; refuse with `! tracekit approvals reject apr-6f0c…`. +Agent: tell the person what this call was for and pass on these commands, then wait for their answer; ... +``` + +## 3. Answer it + +Paste the `! tracekit approvals approve ` line into the session. A line that starts with `!` runs as your own +user, not as a tool call of Claude's, so it is your answer. Then ask Claude to make the same call again: the retry, +with exactly the same arguments, runs once. `! tracekit approvals reject ` refuses it. `tracekit approvals show +` prints why it was held and the signer's copy of the arguments; from another terminal, run the same commands +without the `!`. `tracekit approvals list` (or `tracekit pending`) lists every held call. + +Claude itself can't approve its own calls: the policy denies a `tracekit approvals approve` that the agent runs +(`TK-D012`). In dev mode the signer can't tell you from your agent, so your approval is recorded as a self-approval and +the run's assurance stays `dev`. + +To have Claude follow the hold message every time, paste [these +lines](https://github.com/Cygnux-Labs/Tracekit/blob/main/plugin/skills/tracekit-holds/SKILL.md) into your +`CLAUDE.md` (the plugin ships them as a skill). + +## 4. See what it did + +```sh +tracekit last # the most recent finished run: exported, its key pinned, verified +tracekit last --open # the session still open, verified up to its last checkpointed record +tracekit view --dev # every run in the browser, replayed and reviewed +``` + +A session's run stays open until the session ends, so `tracekit last` shows the session before it until then. +`tracekit last` prints `VERIFIED` and where it wrote the bundle; [the v2 quickstart](../quickstart-v2.md#3-see-it-verified) +explains each line. `tracekit status` shows where the hooks and the signer are. + +## Remove it + +```sh +tracekit uninstall +``` + +It removes Tracekit's hooks from Claude Code's settings and keeps your other settings; a settings file init made +that holds nothing else is deleted. It then lists what it kept, the dev signer's keys and signed runs and Tracekit's +client config, with the commands that delete them and Tracekit too (`tracekit down`, `rm -rf …`, `pip uninstall +tracekit-ai`). A bundle you exported verifies without them. + +## Next steps + +- **Other coding agents**: Codex, Cursor, Gemini CLI, OpenCode and others take `--agent NAME` + ([coding agents](../coding-agents.md)). +- **System mode** runs the signer as its own OS user, so Claude Code can't reach its keys or answer its own holds: + [system mode](../quickstart-v2.md#system-mode-linux-macos-experimental). +- **Wait for the answer instead**: with `TRACEKIT_APPROVAL_WAIT` set (seconds, up to about 480), a held call's hook + waits for an answer from Slack, the viewer or an agent approver before it blocks the call + ([held calls](../coding-agents.md#held-calls-approve-then-retry)). diff --git a/docs/quickstarts/langchain.md b/docs/quickstarts/langchain.md index d71d7aa70..ab6b47211 100644 --- a/docs/quickstarts/langchain.md +++ b/docs/quickstarts/langchain.md @@ -4,12 +4,27 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no API key + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'langchain>=1.4,<1.5' 'langgraph>=1.2,<1.3' +python examples/v2/langchain/agent.py --scripted +``` + +A mock model drives a real graph: `ls` is allowed, `sudo rm -rf /` is denied and the agent goes on, and `echo +"unterminated` is held until a second process approves it. Then the run is exported and verified. + ## 1. Install ```sh -pip install tracekit-ai 'langchain>=1.4,<1.5' 'langgraph>=1.2,<1.3' +pip install tracekit-ai 'langchain>=1.4,<1.5' 'langgraph>=1.2,<1.3' langchain-openai +export OPENAI_API_KEY=... # the model below is OpenAI's ``` +Any model `create_agent` takes works: install its provider's package and set its key instead. + ## 2. One line in your agent ```python @@ -70,8 +85,7 @@ first error) and through `.with_fallbacks()`, as LangGraph's customer-support tu exceptions around the node ends the pause: the call doesn't run and its approval expires ([limits](../limits.md#approvals)). -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/langchain/agent.py ---scripted` from a checkout. +The no-key example at the top of this page wires it this way, with an approval. ## Routing between assistants (handoffs) diff --git a/docs/quickstarts/mcp.md b/docs/quickstarts/mcp.md index c61b2f5d3..fecc3c059 100644 --- a/docs/quickstarts/mcp.md +++ b/docs/quickstarts/mcp.md @@ -4,12 +4,28 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no MCP server + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'mcp>=2.3,<2.4' +python examples/v2/mcp/agent.py --scripted +``` + +A server runs in the same process: `list_files` is allowed, `tracekit_demo_denied` is denied (the dev policy's demo +rule) and the client goes on, and `tracekit_demo_ask` is held until a second process approves it. Then the run is +exported and verified. + ## 1. Install ```sh pip install tracekit-ai 'mcp>=2.3,<2.4' ``` +No API key: the MCP client calls your server's tools. Replace `my-mcp-server` below with the command that starts +yours. + ## 2. One line in your agent ```python @@ -54,8 +70,7 @@ A denied call comes back as a result with `isError: true` and the refusal as its calls; the example's deny and ask come from the dev policy's demo rules (TK-DEMO-DENY, TK-DEMO-ASK). Write your own policy for real tools. -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/mcp/agent.py ---scripted` from a checkout. +The example at the top of this page wires it this way, with an approval. ## Read the verify report diff --git a/docs/quickstarts/openai-agents.md b/docs/quickstarts/openai-agents.md index 8d8da7c40..52d141cda 100644 --- a/docs/quickstarts/openai-agents.md +++ b/docs/quickstarts/openai-agents.md @@ -4,10 +4,23 @@ Every tool call is decided by the v2 signer before it runs and recorded after, a to the agent you have in three steps. It uses the dev signer, which runs as your own user (see [what dev assurance means](../quickstart-v2.md#what-dev-assurance-means)). +## Try it first, with no API key + +```sh +git clone --depth 1 https://github.com/Cygnux-Labs/Tracekit && cd Tracekit +python3 -m venv .venv && . .venv/bin/activate +pip install . 'openai-agents>=0.23.1,<0.24' +python examples/v2/openai_agents/agent.py --scripted +``` + +A mock model drives a real `Runner`: `ls` is allowed, `sudo rm -rf /` is denied and the agent goes on, and `echo +"unterminated` is held until a second process approves it. Then the run is exported and verified. + ## 1. Install ```sh pip install tracekit-ai 'openai-agents>=0.23.1,<0.24' +export OPENAI_API_KEY=... # the agent below calls OpenAI's gpt-5 ``` ## 2. One line in your agent @@ -58,8 +71,7 @@ while result.interruptions: # an ask: once a person decide Function tools, ShellTool with a local executor, LocalShellTool and ApplyPatchTool are gated. Hosted tools, ComputerTool and handoffs are recorded only inside the signed model responses. -An offline run of all this (a mock model, no API key), with an approval: `python examples/v2/openai_agents/agent.py ---scripted` from a checkout. +The no-key example at the top of this page wires it this way, with an approval. ## Read the verify report diff --git a/docs/walkthrough.md b/docs/walkthrough.md index e86f1f85c..40c5d93b7 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -84,7 +84,7 @@ Integrity: FAILED. ```console $ tracekit view --dev -tracekit view on http://127.0.0.1:7778/?token= +tracekit view on http://127.0.0.1:7778/?token= (open it in a browser; Ctrl-C stops the viewer) ``` Every run, each verified, every call with its decision: diff --git a/plugin/README.md b/plugin/README.md index 689150b28..7bdce97d9 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -59,7 +59,9 @@ without the `!`), Claude's retry of the same call runs. `TRACEKIT_SIGNER` is set. Before that, every event shows a notice and runs, unrecorded. The hooks `tracekit init --dev --v2` writes treat events the same way, but run the interpreter init ran with -(`python -I -m tracekit.integrations.claude_code `) instead of searching for one. +(`python -I -m tracekit.integrations.claude_code `; from a source checkout or a `pip install --user`, `python -I +-c` with a one-line import that puts that directory first on the path, then the directory and the event as quoted +arguments, all in plain text) instead of searching for one. ## Upgrading from the v1 plugin diff --git a/tests/test_bundle_v2.py b/tests/test_bundle_v2.py index ad81612f6..f1d4b3af3 100644 --- a/tests/test_bundle_v2.py +++ b/tests/test_bundle_v2.py @@ -983,7 +983,8 @@ def test_verifiers_need_only_the_standard_library(self): "TAMPERED at event 6: "), "v1 sample, every check": (["verify", os.path.join(sample, "demo-run.tkb"), *key, "--detail"], 0, "Integrity: VERIFIED."), - "v2 bundle": (["verify", v2_bundle, "--trust", self.trust], 0, "VERIFIED: 5 signed records, ")} + "v2 bundle": (["verify", v2_bundle, "--trust", self.trust], 0, + "VERIFIED: no tool calls recorded; 5 signed records, ")} for why, (argv, want, verdict) in cases.items(): with self.subTest(why): p = subprocess.run([sys.executable, "-c", blocker, *argv], capture_output=True, text=True, diff --git a/tests/test_content_vault.py b/tests/test_content_vault.py index 60487d1fa..5432855e8 100644 --- a/tests/test_content_vault.py +++ b/tests/test_content_vault.py @@ -338,6 +338,7 @@ def test_doctor_reports_the_vault(self): self.assertEqual((self.check({"data_dir": d})["status"], self.check({"data_dir": d})["detail"]), (doctor.OK, "off: the signer keeps commitments only, no content")) on = {"data_dir": d, "content": {"retain": True, "retention_days": 7}} + os.makedirs(os.path.join(d, "store")) # a signer that started once (before that, dev says info: test_doctor_v2) self.assertEqual(self.check(on)["status"], doctor.WARN) # no key yet svc.SignerService(d, content=on["content"]).close() r = self.check(on) diff --git a/tests/test_dev_mode_line.py b/tests/test_dev_mode_line.py index 936c9fd58..ed6fb0feb 100644 --- a/tests/test_dev_mode_line.py +++ b/tests/test_dev_mode_line.py @@ -13,8 +13,9 @@ from tracekit import cli, doctor, install from tracekit.sdk import autospawn -LINE = ("Dev mode records; it does not guard. The agent runs as you and can reach the key. " - "For protection, use system mode or Box.") +LINE = ("Dev mode records; it does not guard. The agent runs as you and can reach the key. To guard, run the signer " + "as another user (system mode: https://github.com/Cygnux-Labs/Tracekit/wiki/Deploy) or the agent in Tracekit " + "Box, a micro-VM (https://github.com/Cygnux-Labs/Tracekit-box).") def run(argv): @@ -81,7 +82,57 @@ def status(self, dev, system): with mock.patch.object(install, "status", return_value={"client_config": None}), \ mock.patch.object(autospawn, "status", return_value={"socket": "s", "dev": dev}), \ mock.patch("tracekit.client.system_config", return_value=system): - return json.loads(run(["status"])[1]).get("dev_mode") + return json.loads(run(["status", "--json"])[1]).get("dev_mode") + + def test_status_prints_a_summary_and_json_only_with_json(self): + """A new user's `tracekit status` printed a page of JSON; it prints what it found in lines, --json the JSON.""" + hello = {"socket": "s.sock", "dev": True, "running": True, "version": "9", "pid": 7, "data_dir": "/d"} + with mock.patch.object(install, "status", return_value={"client_config": None, "hooks": [], "signer": None}), \ + mock.patch.object(autospawn, "status", return_value=hello), \ + mock.patch("tracekit.client.system_config", return_value=None): + self.assertEqual(run(["status"]), (0, "\n".join([ + "signer dev signer 9, running (pid 7), socket s.sock", "data /d (its keys and signed runs)", + "hooks none in Claude Code's settings: tracekit init --dev --v2 adds them", LINE, + "(tracekit status --json prints all of it)", ""]))) + + def test_status_reads_any_settings_file_and_says_how_to_add_hooks(self): + """Review of #628: a settings file that isn't JSON (or isn't a settings object) crashed `tracekit status` with + KeyError: 'versions', and one with no Tracekit hooks printed "(0 events)" instead of how to add them.""" + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + project = os.path.join(d, ".claude") + os.makedirs(project) + with open(os.path.join(project, "settings.json"), "w") as f: + f.write("[]") # JSON, not a settings object + with mock.patch.dict(os.environ, {"HOME": d, "USERPROFILE": d, "CLAUDE_CONFIG_DIR": ""}), \ + mock.patch("os.getcwd", return_value=d), \ + mock.patch.object(install, "MANAGED_SETTINGS", {"linux": os.path.join(d, "none")}): + hooks = install._installed_hooks() + self.assertEqual(hooks, [{"settings": os.path.join(project, "settings.json"), "error": "unreadable"}]) + none = {"settings": "b.json", "events": [], "versions": []} + self.assertEqual(cli._status_lines({"hooks": hooks + [none]}, {"dev": True, "error": "not started"})[1:], [ + f"hooks {hooks[0]['settings']} isn't a settings file Tracekit can read, so its hooks are unknown", + "hooks none in Claude Code's settings: tracekit init --dev --v2 adds them"]) + + def test_status_names_the_plugins_hooks_and_a_no_hooks_choice(self): + """Review of #628: status told a plugin user, and one whose init chose --no-hooks, that `tracekit init --dev + --v2` adds hooks, though init adds none while the plugin is on.""" + def hooks(client_config, plugin, system=None): + with mock.patch.object(install, "status", return_value={"client_config": client_config, "hooks": [], + "signer": None}), \ + mock.patch.object(autospawn, "status", return_value={"socket": "s.sock", "dev": True}), \ + mock.patch("tracekit.client.system_config", return_value=system), \ + mock.patch.object(install, "plugin_enabled", return_value=plugin): + return [x for x in run(["status"])[1].splitlines() if x.startswith("hooks")] + self.assertEqual(hooks(None, ("/h/.claude/settings.json", "tracekit@tracekit")), + ["hooks the Claude Code plugin tracekit@tracekit's, enabled in /h/.claude/settings.json"]) + chosen = ["hooks none, as init --no-hooks chose: SDK runs (tracekit.instrument()) are still recorded; " + "init without --no-hooks adds Claude Code's"] + self.assertEqual(hooks({"mode": "dev", "no_hooks": True}, None), chosen) + sc = {"signer": "/run/s.sock", "hooks": {"user": "agent", "no_hooks": True}} # system mode's choice + self.assertEqual(hooks(sc, None, sc), chosen) + self.assertEqual(hooks(None, None), ["hooks none in Claude Code's settings: tracekit init --dev --v2 adds " + "them"]) def test_status_says_it_for_a_dev_signer_and_not_for_a_system_one(self): self.assertEqual(self.status(True, None), LINE) diff --git a/tests/test_doctor_v2.py b/tests/test_doctor_v2.py index 3e7a76536..ca2111f8f 100644 --- a/tests/test_doctor_v2.py +++ b/tests/test_doctor_v2.py @@ -754,7 +754,8 @@ def test_no_hooks_after_a_hooked_init_takes_the_hooks_out(self): self.assertTrue(install._v2_hooks_in(settings)) with contextlib.redirect_stdout(io.StringIO()) as out: self.assertEqual(cli.main(["init", "--dev", "--v2", "--no-hooks", "--no-witness"]), 0) - self.assertEqual(out.getvalue(), f"Tracekit v2 hooks removed from {settings}\nv2 hooks: not installed\n" + self.assertEqual(out.getvalue(), f"Tracekit v2 hooks removed from {settings}, and the file with them: init " + f"made it, and nothing else is in it\nv2 hooks: not installed\n" f"{install.DEV_MODE_LINE}\nlog key SHA-256 fingerprint: " f"{dev_key_fingerprint()}\n{public_witness.LOST}\n") marker = os.path.join(autospawn.data_dir(), "no-witness") @@ -790,6 +791,73 @@ def test_system_mode_reads_the_choice_from_client_json(self): self.assertEqual((got["status"], got["detail"]), want) +class FirstDay(unittest.TestCase): + """A new user's healthy dev setup got three warnings: no hooks where there is no Claude Code, no witness (the dev + signer's default, the public witness, isn't named until a release does), and the dev profile's own process + boundary. Only the last is a warning now, and a healthy run ends saying so.""" + + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d, True) + + def rows(self, **kw): + return {r["id"]: r for r in doctor.v2_checks({"data_dir": self.d}, "dev", **kw)} + + def test_no_claude_code_and_no_witness_are_info_in_dev(self): + nowhere = os.path.join(self.d, "no-claude", "settings.json") + with mock.patch.object(doctor.shutil, "which", return_value=None), \ + mock.patch.object(install, "user_settings", return_value=nowhere): + got = self.rows(settings=nowhere) + self.assertEqual((got["D-HOOKS-PRESENT"]["status"], got["D-HOOKS-PRESENT"]["fix"]), + ("info", "once Claude Code is installed: tracekit init --dev --v2")) + self.assertTrue(got["D-HOOKS-PRESENT"]["detail"].startswith("no Claude Code here (no claude on PATH, no ")) + self.assertEqual(got["D-WITNESS-CONFIGURED"]["status"], "info") + with mock.patch.object(doctor.shutil, "which", return_value="/usr/local/bin/claude"): # Claude Code, no hooks + self.assertEqual(self.rows(settings=nowhere)["D-HOOKS-PRESENT"]["status"], "warn") + prod = {r["id"]: r for r in doctor.v2_checks({"data_dir": self.d})} + self.assertEqual(prod["D-WITNESS-CONFIGURED"]["status"], "fail") # production: unchanged + + def test_what_the_first_start_makes_is_info_until_then(self): + """Review of #628: right after `init --dev --v2` (its log key only, no store) doctor warned "start the signer + once" three times, so it never ended with `dev setup OK` as the Claude Code quickstart says. Before the dev + signer's first start they are info and say when they're checked; once it has a store, warnings again.""" + os.makedirs(os.path.join(self.d, "keys"), 0o700) + with open(os.path.join(self.d, "keys", "log.key"), "wb") as f: # as init makes it + f.write(b"k" * 32) + first = ("D-CONTENT-VAULT", "D-KEYS-DISTINCT", "D-KEY-HYGIENE") + from tracekit.signer import service + cfg = {"data_dir": self.d, "content": service.dev_content()["content"]} # the dev signer's vault: on + + def rows(): + return {r["id"]: r for r in doctor.v2_checks(cfg, "dev", settings=os.path.join(self.d, "none"))} + got = rows() + self.assertEqual([got[i]["status"] for i in first], ["info"] * 3) + for i in first: + self.assertTrue(got[i]["detail"].endswith("checked after the dev signer's first start (an agent's first " + "call, or tracekit up)"), got[i]) + os.makedirs(os.path.join(self.d, "store")) # it started once + got = rows() + self.assertEqual([got[i]["status"] for i in first], ["warn"] * 3) + + def test_a_healthy_dev_run_ends_with_one_ok_line(self): + ok = [doctor.result("D-PROCESS-BOUNDARY", doctor.WARN, install.DEV_MODE_LINE, "system mode"), + doctor.result("D-WITNESS-CONFIGURED", doctor.INFO, "no witnesses", "a signer.yaml"), + doctor.result("D-KEYS-MODE", doctor.OK, "8 keys")] + + def run(rows, *argv): + out = io.StringIO() + with mock.patch.object(doctor, "collect", return_value=rows), contextlib.redirect_stdout(out): + self.assertEqual(doctor.main(*argv), 0) + return out.getvalue().splitlines() + lines = run(ok) + self.assertEqual(lines[-1], "dev setup OK: everything checks out for dev mode, which records but does not " + "guard (D-PROCESS-BOUNDARY)") + self.assertIn("info no witnesses (D-WITNESS-CONFIGURED)", lines) + self.assertFalse(any(x.startswith("dev setup OK") for x in run( + ok + [doctor.result("D-HOOKS-EVENT", doctor.WARN, "unwrapped", "re-run init")]))) + self.assertIsInstance(json.loads("\n".join(run(ok, None, True))), list) # --json: the rows alone + + class Reinit(unittest.TestCase): def test_the_reinit_fix_runs_on_this_platform(self): """On macOS init refuses system mode without --experimental-macos, which the fix left out.""" @@ -800,11 +868,26 @@ def test_the_reinit_fix_runs_init_where_it_works_on_every_python(self): and on a dev host with no /opt (D-PROCESS-BOUNDARY), and the version from doctor's own Python; with Tracekit only in /opt's venv the system Python's `-I` form found no module. Init from a root-owned clone works on every Python and with or without /opt, as the plugin and CI run it.""" - self.assertIn("from a root-owned clone of the release (docs/deploy.md#laptop-system-mode), " - "sudo /usr/bin/python3 -m tracekit init --v2 --user ", doctor.REINIT) + self.assertIn("from a root-owned clone of the release (https://github.com/Cygnux-Labs/Tracekit/wiki/" + f"Deploy#laptop-system-mode), sudo {shlex.quote(doctor._PY)} -m tracekit init --v2 --user " + "", doctor.REINIT) self.assertNotIn("/opt/tracekit", doctor.REINIT) self.assertNotIn(" -I ", doctor.REINIT) # -I leaves the clone out of the import path + def test_the_reinit_fix_names_the_wiki_and_a_python_init_accepts(self): + """A new user saw docs/deploy.md, a path in a checkout they don't have, and /usr/bin/python3, which a container + may not have: the fix names the wiki, and doctor's own Python when only root can change it (init refuses any + other), else the system one.""" + def reinit(python, trusted): + with mock.patch.object(sys, "executable", python), \ + mock.patch("tracekit.harness_helper.trusted_file", return_value=None if trusted else "not root's"): + return importlib.reload(doctor)._PY + self.addCleanup(importlib.reload, doctor) + self.assertEqual(reinit("/usr/local/bin/python3", True), "/usr/local/bin/python3") + self.assertEqual(reinit("/home/me/.venv/bin/python", False), "/usr/bin/python3") + self.assertEqual(reinit(install.OPT_PYTHON, True), "/usr/bin/python3") # 3.9, 3.10: init fails there + self.assertNotIn("docs/deploy.md", doctor.REINIT) + class K8s(unittest.TestCase): def test_clean_deployment_passes_every_check(self): @@ -1688,9 +1771,11 @@ def test_dev_signer_warns_instead_of_failing(self): d = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, d, True) os.chmod(d, 0o700) - got = {r["id"]: r["status"] for r in doctor.v2_checks({"data_dir": d}, "dev", settings=os.path.join(d, "none"))} + with mock.patch.object(doctor.shutil, "which", return_value="/usr/local/bin/claude"): # Claude Code is here + got = {r["id"]: r["status"] for r in doctor.v2_checks({"data_dir": d}, "dev", + settings=os.path.join(d, "none"))} self.assertEqual(got["D-PROCESS-BOUNDARY"], "warn") - self.assertEqual(got["D-WITNESS-CONFIGURED"], "warn") + self.assertEqual(got["D-WITNESS-CONFIGURED"], "info") # none: the public witness, its default, isn't named self.assertEqual(got["D-HOOKS-PRESENT"], "warn") self.assertEqual(got["D-DURABILITY"], "ok") self.assertNotIn("D-AGENT-PRIV", got) diff --git a/tests/test_examples_v2.py b/tests/test_examples_v2.py index 65f7a7a7d..15a44523c 100644 --- a/tests/test_examples_v2.py +++ b/tests/test_examples_v2.py @@ -106,6 +106,9 @@ def test_bundle_verifies_and_the_tampered_copy_fails(self): "NOT PROOF AGAINST THE AGENT: in this setup the agent can reach the signing key, so it could " "have rewritten the record.\n", verified) # what dev mode means self.assertNotIn("PASS", verified) + # the held call's approval, by the run's own user: it said "(self-approved: True)" (review of #628) + self.assertRegex(verified, r"\n approved \S*apr-[0-9a-f]{32}, by the run's own user: a self-approval, so the " + r"run's assurance stays dev\n") self.assertIn("\nTAMPERED at event 4: its content was changed after it was signed.\n", tampered) self.assertRegex(tampered, r"\ndeleted .+ \(--keep keeps it\)\n$") diff --git a/tests/test_hook_invocation.py b/tests/test_hook_invocation.py index ab3d6fabc..52ba2c549 100644 --- a/tests/test_hook_invocation.py +++ b/tests/test_hook_invocation.py @@ -47,6 +47,74 @@ def test_project_directory_cannot_shadow_the_hook(self): self.assertNotIn("Traceback", r.stderr, cmd) +@unittest.skipIf(os.name == "nt", "the fail-closed wrapper is POSIX shell") +class ClearText(unittest.TestCase): + """The hook command init wrote from a working tree (or a --user install) carried the tree's path and branch in + base64, which reads as malware to a security reviewer: they are plain, quoted arguments now, and the hook runs as + before from a path with spaces and quotes.""" + + def tree(self, name, entry="import sys\ndef _entry(event):\n print('ran', event, sys.path[0], sys.argv[1:])\n" + " return 0\n"): + """A directory `name` holding a tracekit package whose v2 hook's _entry is `entry`.""" + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + tree = os.path.join(d, name) + os.makedirs(os.path.join(tree, "tracekit", "integrations")) + for f in ("__init__.py", os.path.join("integrations", "__init__.py")): + open(os.path.join(tree, "tracekit", f), "w").close() + with open(os.path.join(tree, "tracekit", "integrations", "claude_code.py"), "w") as f: + f.write(entry) + return tree + + def test_the_command_names_its_tree_in_clear_text_and_runs_from_any_path(self): + """Review of #628: doctor took a `"` off the end of every word of a POSIX command, so a tree whose name ends + in one was misread.""" + from tracekit import doctor + for name in ("it's a \"tree\" $HOME", 'ends in a quote "'): + tree = self.tree(name) + with self.subTest(tree=tree), mock.patch.object(install, "ROOT", tree), \ + mock.patch("importlib.util.find_spec", return_value=None): + cmd = install._hook_command(install.V2_HOOK, args=("PreToolUse",)) + self.assertNotIn("base64", cmd) + self.assertIn("from tracekit.integrations.claude_code import _entry", cmd) + p = subprocess.run(cmd, shell=True, input="{}", capture_output=True, text=True, timeout=60) + self.assertEqual((p.returncode, p.stdout.strip()), + (0, f"ran PreToolUse {tree} {[tree, 'PreToolUse']}"), p.stderr) + self.assertEqual(doctor._probe_target(doctor._hook_argv(cmd)), (sys.executable, tree)) # D-VERSIONS + + HOSTILE = "C:\\Users\\O'Brien $(id) `id` %PATH% !x &calc \"q\"\\Tracekit" + + def windows(self, tree, branch): + """The hook command init writes on Windows for `tree` on `branch`, and doctor's reading of its tree.""" + from tracekit import doctor + with mock.patch.object(install.os, "name", "nt"), mock.patch.object(install, "ROOT", tree), \ + mock.patch.object(install, "_branch", return_value=branch), \ + mock.patch("importlib.util.find_spec", return_value=None): + cmd = install._hook_command(install.V2_HOOK, args=("PreToolUse",)) + return cmd, doctor._probe_target(doctor._hook_argv(cmd))[1] + + def test_on_windows_no_shell_expands_the_tree_or_branch_in_the_command(self): + """Review of #628: the Windows command passed the tree, branch and event as arguments quoted for the C runtime, + which Claude Code's Git Bash (Codex's PowerShell, Droid's cmd.exe) read as shell text: C:\\Users\\me lost its + backslashes, O'Brien broke the line, and a branch fix&calc or f$(id) ran a command. They are literals inside + the code now, every character a shell reads in double quotes escaped; here under bash, as Git Bash runs it.""" + branch = "fix&calc$(id)`id`%PATH%'q\"!x" + tree = self.tree(self.HOSTILE) + cmd, read = self.windows(tree, branch) + self.assertEqual(read, tree) # doctor reads the tree back from the code + self.assertNotIn("base64", cmd) + p = subprocess.run(["bash", "-c", cmd], input="{}", capture_output=True, text=True, timeout=60) + self.assertEqual((p.returncode, p.stdout.strip()), (0, f"ran PreToolUse {tree} []"), p.stderr) + self.assertNotIn("uid=", p.stdout + p.stderr) + broken = self.tree(self.HOSTILE, "from tracekit import a_name_this_branch_removed\n") + p = subprocess.run(["bash", "-c", self.windows(broken, branch)[0]], input="{}", capture_output=True, + text=True, timeout=60) + self.assertEqual(p.returncode, 1) + self.assertIn(f"the working tree {broken}, which cannot import it now", p.stderr) + self.assertIn(f"was on {branch} when the hooks were installed", p.stderr) + self.assertNotIn("uid=", p.stdout + p.stderr) + + @unittest.skipIf(os.name == "nt", "the fail-closed wrapper is POSIX shell") class WorkingTree(unittest.TestCase): """A dev hook imports Tracekit from the working tree init ran in (#427): a branch switch or rebase there that diff --git a/tests/test_hook_v2.py b/tests/test_hook_v2.py index 9300b7780..a40141228 100644 --- a/tests/test_hook_v2.py +++ b/tests/test_hook_v2.py @@ -23,6 +23,7 @@ from test_bundle_v2 import LOG_SECRET, ORIGIN, pub from tracekit import cli, install, privacy from tracekit.bundle_v2 import export +from tracekit.client import client_home from tracekit.format.canon import event_hash from tracekit.format import checkpoint from tracekit.format.checkpoint import ED25519 @@ -1492,6 +1493,109 @@ def test_init_turns_thinking_summaries_on_for_the_plugins_users_and_uninstall_of with open(settings) as f: self.assertEqual(json.load(f), {"enabledPlugins": {install.PLUGIN: True}}) + def test_uninstall_deletes_the_settings_file_init_made_and_says_what_it_keeps(self): + """A new user's uninstall left a settings.json holding `{}` that init had made, and didn't say that the dev + signer's keys and runs and the client config stay, nor how to delete them and Tracekit.""" + settings = os.path.join(self.home, ".claude", "settings.json") + self.assertEqual(self.init("--dev", "--v2")[0], 0) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(cli.main(["uninstall"]), 0) + self.assertFalse(os.path.exists(settings)) + lines = out.getvalue().splitlines() + data = autospawn.data_dir() + self.assertIn(f" {data} (the dev signer's keys and signed runs: a bundle you exported verifies without " + "them)", lines) + self.assertIn(f" {client_home()} (Tracekit's client config)", lines) + gone = [os.path.dirname(data) if sys.platform == "darwin" else data, client_home()] + self.assertEqual(lines[-4:-2], ["to delete these and Tracekit too:", " tracekit down"]) + if os.name != "nt": # the runtime dir too, where one exists + self.assertEqual(shlex.split(lines[-2])[:2 + len(gone)], ["rm", "-rf", *gone]) + self.assertTrue(lines[-1].endswith(" uninstall tracekit-ai"), lines[-1]) + os.makedirs(os.path.dirname(settings), exist_ok=True) # a file that was there before init stays + with open(settings, "w") as f: + f.write("{}") + self.assertEqual(self.init("--dev", "--v2", "--no-thinking-summaries")[0], 0) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(cli.main(["uninstall"]), 0) + with open(settings) as f: + self.assertEqual(json.load(f), {}) + + def test_uninstall_never_names_system_modes_signer_home(self): + """Review of #628: what uninstall keeps named the client config's signer_home in any mode, so on a v1 system + mode host it printed `rm -rf … /var/lib/tracekit`, the system signer's ledger and keys. As uninstall() stops + it, only a dev mode's home counts, and never under a system config.""" + from tracekit import client + v1 = os.path.join(self.home, "v1-home") + os.makedirs(v1) + for cfg, system, named in (({"mode": "dev", "signer_home": v1}, None, True), + ({"mode": "system", "signer_home": v1}, None, False), + ({"signer_home": v1}, None, False), + ({"mode": "dev", "signer_home": v1}, {"socket": "/var/lib/tracekit/s.sock"}, False), + ({"mode": "dev", "signer_home": v1}, client.SystemConfigError("not root's"), False)): + with self.subTest(cfg=cfg, system=system), \ + mock.patch.object(client, "client_config", return_value=cfg), \ + mock.patch.object(client, "system_config", **{"side_effect" if isinstance(system, Exception) + else "return_value": system}): + self.assertEqual(any(v1 in line for line in cli._kept()), named) + + def test_uninstall_prints_windows_commands_powershell_runs(self): + """Review of #628: on Windows the dev signer's runtime dir (%LOCALAPPDATA%\\tracekit\\run) is inside its data + dir, so Remove-Item named it again after deleting its parent, and a Python path with a space came out quoted, + which PowerShell runs only after `&`. Final review: the paths were quoted for cmd.exe (only on a space), so a + profile such as O'Brien, or a name with , ; ( $ or [ ], broke both lines in PowerShell. They are PowerShell + literals now ('...' with each quote PowerShell reads doubled, ’ too), and Remove-Item takes -LiteralPath.""" + from tracekit import client + name = "O'Brien, a;b (c) $x [y] ’z" # ’: PowerShell reads it as a single quote too + home = os.path.join(self.home, name) + data = os.path.join(home, "data", "tracekit") + run, config = os.path.join(data, "run"), os.path.join(home, ".tracekit-client") + for d in (run, config): + os.makedirs(d, 0o700) + os.environ.pop("TRACEKIT_RUNTIME_DIR", None) # Windows' own: %LOCALAPPDATA%\tracekit\run (setUp restores it) + env = {"HOME": home, "USERPROFILE": home, "LOCALAPPDATA": os.path.join(home, "data")} + with mock.patch.dict(os.environ, env), mock.patch.object(os, "name", "nt"), \ + mock.patch.object(sys, "platform", "win32"), \ + mock.patch.object(sys, "executable", "C:\\Users\\O'Brien $x\\python.exe"), \ + mock.patch.object(autospawn, "_short", lambda d: d), \ + mock.patch.object(client, "system_config", return_value=None): + lines = cli._kept() + self.assertIn(f" {run} (the dev signer's runtime dir)", lines) # kept, and said so + literal = os.path.join(self.home, "O''Brien, a;b (c) $x [y] ’’z") + self.assertEqual(lines[-2:], [ + f" Remove-Item -Recurse -Force -LiteralPath '{os.path.join(literal, 'data', 'tracekit')}', " + f"'{os.path.join(literal, '.tracekit-client')}'", + " & 'C:\\Users\\O''Brien $x\\python.exe' -m pip uninstall tracekit-ai"]) + + @unittest.skipIf(not hasattr(os, "getuid"), "Windows has no root-owned system config") + def test_uninstall_with_a_system_config_it_cannot_trust_finishes_and_says_so(self): + """Final review of #628: what uninstall keeps caught the SystemConfigError of a system config it can't trust, + then read the client config, which raised it again, so `uninstall --agent codex` took the hooks out and exited + 1. It reads it once, lists this user's own files, and says why only those.""" + from tracekit import client + bad = os.path.join(self.home, "client.json") + with open(bad, "w") as f: + f.write("{}") + os.chmod(bad, 0o666) # world-writable: system_config refuses it, as root too + with mock.patch.object(client, "SYSTEM_CONFIG", bad), contextlib.redirect_stdout(io.StringIO()) as out, \ + contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(cli.main(["uninstall", "--agent", "codex"]), 0, err.getvalue()) + self.assertIn(f"{bad} must be owned by root and not group- or world-writable, so this lists only your own " + "files\n", out.getvalue()) + + @unittest.skipIf(os.name == "nt", "Windows has no POSIX key modes: D-KEYS-MODE warns there") + def test_doctor_right_after_init_ends_with_dev_setup_ok(self): + """Review of #628: the Claude Code quickstart says doctor ends with `dev setup OK` once init has run, but before + the dev signer's first start it warned "start the signer once" three times.""" + from tracekit import client, doctor + self.assertEqual(self.init("--dev", "--v2")[0], 0) + with mock.patch.object(client, "system_config", return_value=None), \ + contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(doctor.main(), 0) + self.assertEqual(out.getvalue().splitlines()[-1], "dev setup OK: everything checks out for dev mode, which " + "records but does not guard (D-PROCESS-BOUNDARY)", + out.getvalue()) + def test_uninstall_leaves_a_persons_own_thinking_setting(self): settings = os.path.join(self.home, "settings.json") for before in ({}, {install.THINKING: True}, {install.THINKING: False}): @@ -1546,7 +1650,12 @@ def test_init_dev_v2_writes_the_hook(self): self.assertTrue(all(install._is_ours(g) for gs in hooks.values() for g in gs)) # #587: init gave the agent nothing on what to do when a call is stopped; it points to the lines that do self.assertIn("\n" + cli.guidance() + "\n", self.out) - self.assertIn("/wiki/Coding-Agents#when-tracekit-stops-a-call into its CLAUDE.md or AGENTS.md", cli.guidance()) + # the first-day walkthrough: that anchor wasn't on the published wiki yet; the plugin's skill file is on GitHub + self.assertIn(f"{cli.HOLDS_SKILL} into its CLAUDE.md or AGENTS.md", cli.guidance()) + self.assertTrue(os.path.isfile(os.path.join(ROOT, cli.HOLDS_SKILL.split("/blob/main/")[1]))) + # and what to do next: a new session, the `!` line a hold shows, the viewer + self.assertIn("\n" + cli.next_step() + "\n", self.out) + self.assertIn("! tracekit approvals approve ", cli.next_step()) def test_the_agent_guidance_init_points_to_is_the_plugins_skill(self): """#587: one wording of what an agent does when Tracekit stops its call, in the docs init points to and in the diff --git a/tests/test_install_files.py b/tests/test_install_files.py index d6bce9aa8..d5c7a2cb3 100644 --- a/tests/test_install_files.py +++ b/tests/test_install_files.py @@ -467,10 +467,12 @@ def install_then_purge(self, managed): self.assertEqual(install.uninstall_system_v2(purge=True), []) for p in (install.V2_DATA, install.V2_CONFIG, client.SYSTEM_CONFIG): self.assertFalse(os.path.exists(p), p) - with open(settings) as f: - self.assertEqual(json.load(f), admin if managed else {}) if managed: + with open(settings) as f: + self.assertEqual(json.load(f), admin) self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o644) + else: + self.assertFalse(os.path.exists(settings)) # init made it, and nothing else is in it for user in ("tk-test-signer", "tk-test-tailer"): with self.assertRaises(KeyError): pwd.getpwnam(user) @@ -523,7 +525,7 @@ def test_v2_init_records_where_it_put_the_hooks(self): with mock.patch.dict(os.environ, env), contextlib.redirect_stdout(io.StringIO()): self.assertEqual(doctor._dev_settings(), settings) self.assertEqual(install.uninstall(), settings) - self.assertEqual(install.read_json(settings).get("hooks"), None) + self.assertFalse(os.path.exists(settings)) # init made it, and nothing else is in it def test_hooks_are_found_and_removed_with_or_without_claude_config_dir(self): for installed_with, removed_with in ((True, False), (False, True)): @@ -545,7 +547,7 @@ def test_hooks_are_found_and_removed_with_or_without_claude_config_dir(self): os.environ.pop("CLAUDE_CONFIG_DIR", None) self.assertIn(path, [h["settings"] for h in install._installed_hooks()]) self.assertEqual(install.uninstall(), path) # it named files it never found - self.assertEqual(install._hook_summary(install.read_json(path)), install._hook_summary({})) + self.assertFalse(os.path.exists(path)) # install_hooks made it: no hooks, and no file if __name__ == "__main__": diff --git a/tests/test_instrument.py b/tests/test_instrument.py index 43c4aa517..34821e537 100644 --- a/tests/test_instrument.py +++ b/tests/test_instrument.py @@ -96,7 +96,7 @@ def test_each_framework(self): if name != "openai": self.assertIn(REFUSED, out[1]) p = self.sh("-m", "tracekit", "last") - self.assertRegex(p.stdout, r"\nVERIFIED: (\d+ tool calls? in )?\d+ signed records, none changed") + self.assertRegex(p.stdout, r"\nVERIFIED: (\d+ tool calls? in |no tool calls recorded; )?\d+ signed records, none changed") self.assertIn("\nNOT PROOF AGAINST THE AGENT: ", p.stdout) # a dev signer bundle = p.stdout.split("bundle: ", 1)[1].splitlines()[0] self.assertTrue(os.path.isfile(bundle) and os.path.isfile(bundle[:-4] + ".trust.json")) diff --git a/tests/test_policy2.py b/tests/test_policy2.py index c59f80d82..aadedd8aa 100644 --- a/tests/test_policy2.py +++ b/tests/test_policy2.py @@ -2874,11 +2874,15 @@ def test_the_reason_names_why_the_policy_cant_read_a_command(self): e = Engine(pc.build(os.path.join(PACKS, "dev.yaml"))[0]) for cmd, cause in (("X=$(printf ls); $X", "name"), ("echo x | base64 -d | sh", "input"), ("fish -c 'ls'", "foreign"), ("l*s x", "pattern"), ("echo $((x[$(id)]))", "syntax"), - ("f(){ sh x; }; curl -s -o x https://example.com/x; f", "download")): + ("f(){ sh x; }; curl -s -o x https://example.com/x; f", "download"), + # a new user's first hold said "shell syntax the policy doesn't read" for a missing quote + ('echo "unterminated', "quote"), ("echo 'x", "quote"), ("echo $'x", "quote"), + ("echo ${x", "syntax")): with self.subTest(cmd=cmd): self.assertEqual(e.decide("Bash", {"command": cmd}).get("unread"), cause) + self.assertEqual(e.decide("Bash", {"command": cmd})["verdict"], "ask") # the cause is words only self.assertNotIn("unread", e.decide("Bash", {"command": "ls"})) - for cause in ("name", "pattern", "input", "foreign", "syntax", "function", "download"): + for cause in ("name", "pattern", "input", "foreign", "syntax", "function", "download", "quote"): entry = pc.ENGINE_REASONS["TK-SHELL-PARSE"]["causes"][cause] self.assertLessEqual(len(entry["remedy"]), pc.MAX_REMEDY, cause) diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index e03c4ce44..9136d48db 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -66,7 +66,7 @@ def last(self): """Step 3; its time.""" t = time.monotonic() out = self.sh(self.tracekit, "last").stdout - self.assertRegex(out, r"\nVERIFIED: (\d+ tool calls? in )?\d+ signed records, none changed") + self.assertRegex(out, r"\nVERIFIED: (\d+ tool calls? in |no tool calls recorded; )?\d+ signed records, none changed") self.assertIn("\nNOT PROOF AGAINST THE AGENT: ", out) # a dev signer self.assertIn("next: tracekit view --dev", out) return time.monotonic() - t diff --git a/tests/test_remote_tokens.py b/tests/test_remote_tokens.py index e66be53ab..d8733ae62 100644 --- a/tests/test_remote_tokens.py +++ b/tests/test_remote_tokens.py @@ -259,8 +259,7 @@ def test_a_later_init_and_uninstall_drop_the_remote_signer(self): with open(settings) as f: self.assertNotIn("TRACEKIT_SIGNER_CA", json.load(f)["env"]) self.assertEqual(cli.main(["uninstall", "--project"]), 0) - with open(settings) as f: - self.assertNotIn("env", json.load(f)) + self.assertFalse(os.path.exists(settings)) # init made it: no env, and no file if __name__ == "__main__": diff --git a/tests/test_sdk_client.py b/tests/test_sdk_client.py index 58661d375..18d055f8b 100644 --- a/tests/test_sdk_client.py +++ b/tests/test_sdk_client.py @@ -218,16 +218,16 @@ def cli(self, *argv): def test_cli_up_status_down(self): self.assertIn(f"dev signer {__version__} running", self.cli("up", "--wait")) - status = json.loads(self.cli("status")) + status = json.loads(self.cli("status", "--json")) self.assertIn("client_config", status) # the v1 sections are still there self.assertTrue(status["signer_v2"]["running"]) self.assertIn("dev signer stopped", self.cli("down")) - self.assertFalse(json.loads(self.cli("status"))["signer_v2"]["running"]) + self.assertFalse(json.loads(self.cli("status", "--json"))["signer_v2"]["running"]) self.assertIn("no dev signer running", self.cli("down")) def test_status_survives_an_unusable_runtime_dir(self): with mock.patch.object(autospawn, "runtime_dir", side_effect=PermissionError("read-only")): - status = json.loads(self.cli("status")) + status = json.loads(self.cli("status", "--json")) self.assertIn("client_config", status) self.assertFalse(status["signer_v2"]["running"]) diff --git a/tests/test_signer_approvals.py b/tests/test_signer_approvals.py index f6547d3b0..128f89ebb 100644 --- a/tests/test_signer_approvals.py +++ b/tests/test_signer_approvals.py @@ -825,5 +825,53 @@ def test_approve_round_trip_with_the_dev_signer(self): self.assertIn(f"{aid} consumed", buf.getvalue()) +class TestFirstDayCli(unittest.TestCase): + """What a new user's walkthrough found: `approvals show` gave TK-SHELL-PARSE's general reason where the hook gave + the cause's own, offered Claude Code's `! ` to runs that aren't Claude Code's, and `tracekit pending` failed with + "signer unavailable" with held v2 calls and no v1 signer.""" + + def setUp(self): + self.run_dir = tempfile.mkdtemp(dir="/tmp" if os.path.isdir("/tmp") else None) + self.addCleanup(shutil.rmtree, self.run_dir, True) + os.chmod(self.run_dir, 0o700) + env = mock.patch.dict(os.environ, {"TRACEKIT_RUNTIME_DIR": self.run_dir, + "XDG_DATA_HOME": os.path.join(self.run_dir, "none"), + "HOME": os.path.join(self.run_dir, "home")}) + env.start() + self.addCleanup(env.stop) + + def lines(self, command, run_id="run-1"): + from tracekit.policy2 import compile as pc + pol, _ = pc.build(os.path.join(ts.ROOT, "tracekit", "policy2", "packs", "dev.yaml")) + x = {"approval_id": "apr-" + "1" * 32, "state": "requested", "run_id": run_id, "tool": "Bash", + "rule_ids": ["TK-SHELL-PARSE"], "policy_hash": pc.policy_hash(pol), "args_source": "parsed", + "args": {"command": command}, "expires_at": time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", + time.gmtime(time.time() + 3600))} + return cli._stop_lines(mock.Mock(signer=None), x) + + def test_show_gives_the_cause_the_hook_gives(self): + self.assertEqual(self.lines('echo "unterminated')[:2], [ + "why a quote on the line is never closed (TK-SHELL-PARSE)", "instead Close the quote."]) + self.assertTrue(self.lines("X=$(printf ls); $X")[0].startswith( + "why its program is a variable the line doesn't set to plain text (TK-SHELL-PARSE)")) + self.assertRegex(self.lines("ls x 'y")[2], r"^expires \S+Z \(in 5\d min\)$") + + def test_the_bang_hint_is_for_claude_code_runs_only(self): + answer = next(line for line in self.lines('echo "x') if line.startswith("answer")) + self.assertNotIn("! ", answer) # an SDK agent's run: no Claude Code session to paste it into + with open(os.path.join(self.run_dir, "claude-code-" + "a" * 64 + ".json"), "w") as f: + json.dump({"run_id": "run-1", "run_token": "t"}, f) # the v2 hook's session state + answer = next(line for line in self.lines('echo "x') if line.startswith("answer")) + self.assertTrue(answer.endswith("; in the agent's Claude Code session, with a leading `! `"), answer) + + def test_pending_without_a_v1_signer_lists_the_v2_signers_held_calls(self): + from tracekit import client + with mock.patch.object(client, "DEFAULT_SOCKET", os.path.join(self.run_dir, "no-tracekitd.sock")), \ + mock.patch.object(client, "socket_path", return_value=client.DEFAULT_SOCKET), \ + mock.patch.object(cli, "_approvals", return_value=0) as v2: + self.assertEqual(cli.main(["pending"]), 0) + self.assertEqual((v2.call_args[0][0].action, v2.call_args[0][0].signer), ("list", None)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_signer_dev.py b/tests/test_signer_dev.py index 796f6b41f..8504c0a02 100644 --- a/tests/test_signer_dev.py +++ b/tests/test_signer_dev.py @@ -143,7 +143,7 @@ def test_a_runtime_dir_too_long_for_a_socket_path_gets_a_short_one_of_its_own(se again = Client() self.addCleanup(again.close) self.assertEqual(self.one_run(again), pid) - self.assertEqual(json.loads(self.cli("status"))["signer_v2"]["socket"], os.path.join(short, autospawn.SOCK)) + self.assertEqual(json.loads(self.cli("status", "--json"))["signer_v2"]["socket"], os.path.join(short, autospawn.SOCK)) self.assertNotEqual(autospawn.runtime_dir(), short) # another dir's is never this one near = "/tmp/" + "s" * (103 - len("/tmp/") - len("/" + autospawn.SOCK)) # its socket path fits, a side dir's not self.assertEqual(autospawn._short(near), near) @@ -322,7 +322,7 @@ def test_export_says_what_it_wrote_and_the_commands_it_gives_verify_it(self): r"trust\.json\n$") trust, verify = [shlex.split(x)[1:] for x in said.splitlines() if x.startswith(" tracekit ")] self.cli(*trust) - self.assertRegex(self.cli(*verify), r"^VERIFIED: (\d+ tool calls? in )?\d+ signed records, none changed") + self.assertRegex(self.cli(*verify), r"^VERIFIED: (\d+ tool calls? in |no tool calls recorded; )?\d+ signed records, none changed") info = json.loads(self.cli("export", "--v2", "--run", run.run_id, "-o", "again.tkb", "--json")) self.assertEqual((info["bundle"], info["verifier_min_version"] in said), ("again.tkb", True)) @@ -439,6 +439,31 @@ def setUp(self): self.addCleanup(env.stop) os.environ.pop("TRACEKIT_SIGNER", None) + def test_uninstall_names_only_this_homes_runtime_dirs(self): + """A first-day run of `tracekit uninstall` from a second HOME named the per-user runtime dir another HOME's + live signer used, in the `rm -rf` it prints. Review of #628: one with no endpoint.json (its signer stopped) + was named too, though two HOMEs share it. It names only dirs provably this HOME's (its own sibling, one inside + HOME, one whose endpoint.json names this data dir), and says when the per-user one can't be told.""" + home = os.path.join(self.dir, "home") + os.makedirs(home) + with mock.patch.dict(os.environ, {"HOME": home, "USERPROFILE": home, "XDG_DATA_HOME": os.path.join(home, "data")}): + self.assertEqual(autospawn.own_runtime_dirs(), ([], None)) # none made yet + os.makedirs(self.run_dir, 0o700) + self.assertEqual(autospawn.own_runtime_dirs(), ([], self.run_dir)) # shared, and no endpoint: can't tell + for named, want in ((autospawn.data_dir(), ([self.run_dir], None)), + (os.path.join(self.dir, "another-home-data"), ([], None))): + with open(os.path.join(self.run_dir, autospawn.ENDPOINT), "w") as f: + json.dump({"data_dir": named}, f) + self.assertEqual(autospawn.own_runtime_dirs(), want) + side = autospawn._side(self.run_dir) + os.makedirs(side, 0o700) + self.addCleanup(shutil.rmtree, side, True) + self.assertEqual(autospawn.own_runtime_dirs(), ([side], None)) + inside = os.path.join(home, "run") # macOS's ~/Library/Application Support/tracekit/run: this HOME's + os.makedirs(inside, 0o700) + with mock.patch.dict(os.environ, {"TRACEKIT_RUNTIME_DIR": inside}): + self.assertEqual(autospawn.own_runtime_dirs(), ([inside], None)) + def test_signer_starts_in_the_private_runtime_dir_with_cwd_off_sys_path(self): os.makedirs(self.run_dir, 0o700) with mock.patch.object(autospawn.subprocess, "Popen") as popen: @@ -500,7 +525,7 @@ def replaced(pid, sig): # a signer started since publishes its own pid def test_status_creates_nothing(self): with contextlib.redirect_stdout(io.StringIO()) as out: - self.assertEqual(cli.main(["status"]), 0) + self.assertEqual(cli.main(["status", "--json"]), 0) self.assertFalse(json.loads(out.getvalue())["signer_v2"]["running"]) self.assertFalse(os.path.exists(self.run_dir)) diff --git a/tests/test_system_v2.py b/tests/test_system_v2.py index 63ccc7437..ebc244e8e 100644 --- a/tests/test_system_v2.py +++ b/tests/test_system_v2.py @@ -400,6 +400,8 @@ def managed_setup(self, admin=None): json.dump(admin, f) def read(self, path): + if not os.path.exists(path): # init made it, and taking Tracekit's hooks out left nothing: deleted + return {} with open(path) as f: return json.load(f) @@ -539,7 +541,7 @@ def test_init_names_the_claude_code_file_it_takes_the_hooks_out_of(self): _, _, sc, _ = self.init_real({}) sc["hooks"]["user"] = "alice" if "project" in kw else "agent" out = self.init_real(sc, **kw)[3] - self.assertIn(f"Tracekit v2 hooks removed from {self.user}\n", out) + self.assertIn(f"Tracekit v2 hooks removed from {self.user}", out) self.assertEqual(install._hook_summary(self.read(self.user))["versions"], []) def test_uninstall_takes_tracekits_hooks_out_of_the_managed_settings_the_config_forgot(self): @@ -727,8 +729,7 @@ def test_signer_env_is_set_and_removed_only_while_unchanged(self): with open(path) as f: self.assertEqual(json.load(f)["env"], {"TRACEKIT_SIGNER": "/run/s.sock"}) install.install_hooks(path, uninstall=True, signer="/run/s.sock") - with open(path) as f: - self.assertNotIn("env", json.load(f)) + self.assertFalse(os.path.exists(path)) # install_hooks made it: no env, and no file with open(path, "w") as f: json.dump({"env": {"TRACEKIT_SIGNER": "/mine.sock"}}, f) install.install_hooks(path, uninstall=True, signer="/run/s.sock") diff --git a/tests/test_verify_short.py b/tests/test_verify_short.py index c32c1ed05..0e611cf12 100644 --- a/tests/test_verify_short.py +++ b/tests/test_verify_short.py @@ -175,9 +175,15 @@ def test_a_dev_bundle_is_not_proof_against_the_agent_however_it_was_witnessed(se (0, "VERIFIED: 2 tool calls in 4 signed records, none changed, removed or added after " f"signing.\n{DEV}{ANCHORED}\nPinned log key SHA-256 fingerprint: {self.fingerprint}.\n")) - def test_a_run_with_no_tool_call_counts_its_signed_records(self): - self.assertEqual(verify(self.bundle("same-user", calls=()), "--trust", self.trust)[1].splitlines()[0], - "VERIFIED: 2 signed records, none changed, removed or added after signing.") + def test_a_run_with_no_tool_call_says_so_and_counts_its_signed_records(self): + """A new user's first run with nothing recorded read only "VERIFIED: 3 signed records", as if calls were in + them; the exit code and --json stay as they were.""" + bundle = self.bundle("same-user", calls=()) + self.assertEqual(verify(bundle, "--trust", self.trust)[1].splitlines()[0], + "VERIFIED: no tool calls recorded; 2 signed records, none changed, removed or added after " + "signing.") + code, out = verify(bundle, "--trust", self.trust, "--json") + self.assertEqual((code, json.loads(out)["integrity"]), (0, "VERIFIED")) def test_an_isolation_the_signer_could_not_tell_shows_nothing(self): code, out = verify(self.bundle("unknown"), "--trust", self.trust) diff --git a/tests/test_view.py b/tests/test_view.py index 69d6041ea..c61b41402 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -590,10 +590,12 @@ def test_policy_reasons_of_a_configured_policy_by_its_hash(self): path = os.path.join(ts.tmpdir(self), "p.yaml") with open(path, "w", encoding="utf-8") as f: f.write("deny:\n - id: X-1\n pattern: 'rm'\n reason: removing files\n remedy: Move it aside.\n" - "flag:\n - id: X-2\n pattern: 'ls'\n label: listing\n") + "flag:\n - id: X-2\n pattern: 'ls'\n label: listing\n" + " - id: X-3\n pattern: 'git push'\n label: side_effect\n") h = policy_compile.policy_hash(policy_compile.build(path)[0]) self.assertEqual(view.policy_reasons([path])[h], {"X-1": {"reason": "removing files", "remedy": "Move it aside."}, - "X-2": {"reason": "listing"}}) + "X-2": {"reason": "listing"}, + "X-3": {"reason": "side effect"}}) # report printed side_effect self.assertEqual(SERVER[SERVER_HASH]["TK-DB001"], {"reason": "dropping a database object"}) def test_a_calls_signed_provenance_rides_on_its_row(self): @@ -1168,7 +1170,7 @@ def test_dev_run_is_listed_with_its_counts_and_verified_while_the_signer_holds_t self.assertEqual(rep["title"], f"RUN {run_id} · Integrity VERIFIED · Assurance dev") self.assertIn("tenant default · agent e2e · final · decisions allow 1, ask 1, deny 1 · approvals 1 · gaps 0", rep["text"]) - self.assertIn("same user as the dev signer", rep["text"]) + self.assertIn("in dev mode it runs as you, as the signer does", rep["text"]) self.assertIn(view.OPERATOR_SIDE, rep["text"]) self.assertIn("Integrity: VERIFIED.\nAssurance: dev;", rep["text"]) # verify.v2's own report rows = self.rows(feed, run_id) @@ -1316,8 +1318,23 @@ def test_last_before_any_dev_signer_makes_nothing(self): contextlib.redirect_stderr(io.StringIO()) as err: self.assertEqual(cli.main(["last"]), 1) self.assertTrue(err.getvalue().startswith(f"tracekit last: no runs in the dev signer's store " - f"({os.path.join(nope, 'store')}) yet: "), err.getvalue()) + f"({os.path.join(nope, 'store')}) yet. "), err.getvalue()) self.assertFalse(os.path.exists(nope)) + # a new user who had added instrument() was told to add it: it records only the frameworks it lists + self.assertIn("`tracekit.instrument()` records only the frameworks it lists", err.getvalue()) + self.assertIn(f"as {wiki_url('docs/quickstarts/custom.md')} shows", err.getvalue()) + + def test_last_says_it_waits_for_a_run_that_just_closed(self): + """`tracekit last` waited up to LAST_WAIT_S for a closing run in silence; it says so, once.""" + closing = [v2rec(0, "run.registered", {"agent": {"name": "a"}}, "r1"), v2rec(1, "run.closing", {}, "r1")] + fake = mock.Mock(runs={("default", "r1"): {"seqs": [1]}}) + fake.iter_run.side_effect = lambda *key: iter(closing) + with mock.patch.object(svc, "reader", return_value=fake), \ + mock.patch.object(svc, "signer_config", return_value={"data_dir": self.d}), \ + mock.patch.object(cli, "LAST_WAIT_S", 0.6), contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(cli.main(["last"]), 1) + self.assertEqual(err.getvalue().count("tracekit last: run r1 (a) just closed; waiting up to 0.6 s while the " + "signer finalises it\n"), 1) @pytest.mark.unverified_bundles def test_tampered_record_shows_only_the_failure(self): @@ -1639,6 +1656,27 @@ def test_approval_prints_the_link_that_opens_the_held_call(self): self.assertEqual(view.main(["--data-dir", self.d, "--approval", aid + "&x="]), 2) self.assertIn("--approval takes an approval id", err.getvalue()) + def test_the_start_lines_are_plain_and_the_link_is_one_a_browser_opens(self): + """A new user's walkthrough: the viewer's lines said "trust pins this store's own log.vkey: dev assurance", + an any-address bind printed http://0.0.0.0:..., which a browser doesn't open, --host and --port had no help, and + --approval's help said it prints a link where it starts the viewer and prints one.""" + with mock.patch.object(view, "server") as server, mock.patch.dict(os.environ, {"TRACEKIT_VIEW_TOKEN": "t"}), \ + contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()): + server.return_value.serve_forever.side_effect = KeyboardInterrupt + server.return_value.server_address = ("0.0.0.0", 7778) + self.assertEqual(view.main(["--data-dir", self.d, "--host", "0.0.0.0", "--insecure-http"]), 0) + first, second = out.getvalue().splitlines()[:2] + self.assertEqual(first, "tracekit view on http://127.0.0.1:7778/ (open it in a browser; Ctrl-C stops the " + "viewer)") + self.assertIn("read-only; each run is checked against the key this store names", second) + self.assertNotIn("log.vkey", second) + with contextlib.redirect_stdout(io.StringIO()) as out, self.assertRaises(SystemExit): + view.main(["--help"]) + help_text = " ".join(out.getvalue().split()) + for words in ("--host HOST the address to serve on", "--port PORT the port to serve on", + "--approval ID start the viewer and print a link that opens the call approval ID held"): + self.assertIn(words, help_text) + def test_the_tips_line_names_a_transcripts_folder_only_when_there_is_one(self): """With no Claude Code folder, `tracekit view --dev` printed "tips: Claude Code's transcripts in .../.claude/projects, read-only" for a folder that doesn't exist.""" @@ -1675,7 +1713,8 @@ def test_loopback_viewer_prints_a_token_url_and_needs_it(self): server.return_value.server_address = ("127.0.0.1", 7778) runs.return_value.logs = ["a.dsn", "b.dsn"] self.assertEqual(view.main(["--config", "signer.yaml"]), 0) - self.assertIn(f"http://127.0.0.1:7778/?token={server.call_args[0][3]} (2 Postgres logs", out.getvalue()) + self.assertIn(f"http://127.0.0.1:7778/?token={server.call_args[0][3]} (open it in a browser; Ctrl-C stops " + "the viewer)\n reading 2 Postgres logs, read-only\n", out.getvalue()) with self.assertRaises(ValueError): view.server(None, "127.0.0.1", 0, None) @@ -2305,7 +2344,7 @@ def test_v1_bundle_is_served_on_the_viewers_server_and_a_tampered_one_is_refused self.assertEqual(code, 0) self.assertIsInstance(feed, observe.Feed) # the bundle it shows, not the temp ledger it was unpacked to - self.assertIn(f"(bundle: {os.path.join(self.SAMPLE, 'demo-run.tkb')}, verified, read-only)", out) + self.assertIn(f"reading the bundle {os.path.join(self.SAMPLE, 'demo-run.tkb')}, verified, read-only", out) self.assertNotIn(feed.path, out) srv = view.server(feed, "127.0.0.1", 0, token) threading.Thread(target=srv.serve_forever, daemon=True).start() @@ -2341,7 +2380,7 @@ def test_why_alone_is_why_serve(self): self.assertEqual(code, 0) self.assertEqual((store.root, store.tape, store.allow, args[0]), (root, True, {"m:SYSTEM"}, store.out)) self.assertIsNotNone(store.webhook) - self.assertIn(f"(why logs alone: {root})", out) + self.assertIn(f"reading why logs alone: {root}", out) def test_one_nav_reaches_every_page_this_server_has(self): runs = mock.Mock() diff --git a/tracekit/cli.py b/tracekit/cli.py index 40d346c15..d5f396d57 100644 --- a/tracekit/cli.py +++ b/tracekit/cli.py @@ -26,7 +26,20 @@ def guidance(): """The line `tracekit init --v2` prints once it installs hooks: where the agent's instructions for a held or blocked call are (the Claude Code plugin's tracekit-holds skill holds the same lines).""" return ("when Tracekit holds or blocks a call, the agent should tell you and pass on the commands: paste the lines " - f"at {wiki_url('docs/coding-agents.md', 'when-tracekit-stops-a-call')} into its CLAUDE.md or AGENTS.md") + f"at {HOLDS_SKILL} into its CLAUDE.md or AGENTS.md") + + +# the lines guidance() names, in the Claude Code plugin's tracekit-holds skill +HOLDS_SKILL = "https://github.com/Cygnux-Labs/Tracekit/blob/main/plugin/skills/tracekit-holds/SKILL.md" + + +def next_step(): + """The line `tracekit init --dev --v2` ends with once it wired Claude Code's hooks: what to do now.""" + import shutil + return (("next: install Claude Code (no `claude` on PATH yet), then start a session" if not shutil.which("claude") + else "next: start a new Claude Code session (one already open doesn't load the hooks)") + + ". A call the policy holds shows a `! tracekit approvals approve ` line: paste it into that session " + "to allow the call once, then ask the agent to retry it. See every run: tracekit view --dev") def _dev_key_fingerprint(data_dir, make=True): @@ -328,14 +341,15 @@ def _parser(): "enclave attestation, KMS key metadata); its hash is signed into checkpoints and " "exports carry it") - sub.add_parser("status", help="hooks, signer, witnesses, policy, fail mode, capture sources") - p = sub.add_parser("up", help="find or start the same-user v2 dev signer") + p = sub.add_parser("status", help="what is set up here: hooks, signer, policy, capture sources") + p.add_argument("--json", action="store_true", help="print everything it found as one JSON object") + p = sub.add_parser("up", help="start the local signer (dev mode), or find the running one") p.add_argument("--wait", action="store_true", help="return once it answers") p.add_argument("--no-witness", action="store_true", help="no public witness for the dev signer (restarts it; " "init --dev --v2 --public-witness turns it back on)") p.add_argument("--json", action="store_true", help="wait, then print its hello as JSON") p.add_argument("--replace", action="store_true", help="stop the running dev signer first, even an incompatible one") - sub.add_parser("down", help="stop the same-user v2 dev signer") + sub.add_parser("down", help="stop the local signer (dev mode)") p = sub.add_parser("doctor", help=f"check this setup (v1/v2 system mode, the dev signer); {wiki_url('docs/doctor.md')}") p.add_argument("--json", action="store_true", help="print [{id, status, detail, fix}]") p.add_argument("--config", help="check the v2 signer of this signer.yaml") @@ -343,13 +357,15 @@ def _parser(): p.add_argument("--manifests", metavar="DIR", help="with --k8s: check these rendered manifests instead") p.add_argument("--issuer-key", metavar="KEY_ID", help="the record key issuer's KMS key: the signer must not sign " "with it") - p = sub.add_parser("uninstall", help="remove hooks (the ledger is kept)") + p = sub.add_parser("uninstall", help="remove Tracekit's hooks (your signed runs and keys stay: it says where, and " + "how to delete them too)") p.add_argument("--project", action="store_true") p.add_argument("--v2", action="store_true", help="as root: remove v2 system mode (service, configs, hooks)") p.add_argument("--purge", action="store_true", help="with --v2: also delete the signer's keys, log and user") p.add_argument("--agent", choices=("claude", *agent_hooks.HARNESSES), default="claude") - p = sub.add_parser("ingest", help="remote ingestion gateway: `ingest token NAME` / `ingest serve`", add_help=False) + p = sub.add_parser("ingest", help="remote ingestion gateway (experimental): `ingest token NAME` / `ingest serve`", + add_help=False) p.add_argument("rest", nargs=argparse.REMAINDER) p = sub.add_parser("analyze", help="run the detectors over a run and sign the findings into the ledger " "(--v2: check a v2 run's request, reasoning and actions)", add_help=False) @@ -403,9 +419,10 @@ def _parser(): p = sub.add_parser("follow", help="follow agentsh's audit log into the v2 signer as it is written: " "`follow agentsh --log PATH`", add_help=False) p.add_argument("rest", nargs=argparse.REMAINDER) - p = sub.add_parser("otel", help="OpenTelemetry receiver: `otel serve` records agent spans sent over OTLP/HTTP", add_help=False) + p = sub.add_parser("otel", help="OpenTelemetry receiver (experimental): `otel serve` records agent spans sent over " + "OTLP/HTTP", add_help=False) p.add_argument("rest", nargs=argparse.REMAINDER) - p = sub.add_parser("daemon", help="run tracekitd in the foreground") + p = sub.add_parser("daemon", help="run the v1 signer, tracekitd, in the foreground (experimental)") p.add_argument("--home") p = sub.add_parser("proxy", help="run the model proxy in the foreground (experimental)") p.add_argument("--home") @@ -413,7 +430,8 @@ def _parser(): p.add_argument("--port", type=int) p.add_argument("--upstream") - p = sub.add_parser("observe", help="now part of `tracekit view --ledger`: the v1 ledger or a bundle (read-only web UI)") + p = sub.add_parser("observe", help="the old name of `tracekit view --ledger`: the v1 ledger or a v1 bundle in a " + "read-only web page") p.add_argument("rest", nargs=argparse.REMAINDER) p = sub.add_parser("view", help="the viewer: the v2 signer's runs, each verified; --why DIR, --ledger, --bundle (read-only web UI)") p.add_argument("rest", nargs=argparse.REMAINDER) @@ -537,9 +555,9 @@ def _parser(): "(the full report; scripts that read the text use it, or --json)") p.add_argument("--json", action="store_true") - p = sub.add_parser("policy", help="policy v2: `policy compile FILE` prints canonical JSON and its hash; `policy lint " - "FILE`; `policy backtest --policy P (--store DIR | --bundle B...)` prints the " - "incidents its detection would open") + p = sub.add_parser("policy", help="check a v2 policy and try it on past runs: `policy compile FILE` prints " + "canonical JSON and its hash; `policy lint FILE`; `policy backtest --policy P " + "(--store DIR | --bundle B...)` prints the incidents its detection would open") p.add_argument("action", choices=("compile", "lint", "backtest")) p.add_argument("file", nargs="?") p.add_argument("--policy", help="backtest: the policy file") @@ -786,6 +804,7 @@ def _run(a): print(install.thinking_line(hooks, on)) if hooks: print(guidance()) + print(next_step()) return 0 try: cfg = install.init_dev(home, a.witness, a.checkpoint_every, hooks, proxy=a.proxy, proxy_port=a.proxy_port, @@ -825,7 +844,13 @@ def _run(a): from . import client system = client.system_config() # a root-owned config is system mode: the signer isn't the same user's dev = (st["client_config"] or {}).get("mode") == "dev" or v2.get("dev") and not system - print(json.dumps({**st, "signer_v2": v2, **({"dev_mode": install.DEV_MODE_LINE} if dev else {})}, indent=2)) + if a.json: + print(json.dumps({**st, "signer_v2": v2, **({"dev_mode": install.DEV_MODE_LINE} if dev else {})}, indent=2)) + return 0 + cfg = st["client_config"] or {} # system mode's: its hooks section + chosen = (cfg.get("hooks") or {}).get("no_hooks") if system else cfg.get("no_hooks") + print(*_status_lines(st, v2, install.plugin_enabled(), chosen), *([install.DEV_MODE_LINE] if dev else []), + "(tracekit status --json prints all of it)", sep="\n") return 0 if a.cmd == "up": from . import install @@ -874,6 +899,8 @@ def _run(a): return 0 if a.cmd == "uninstall" and a.agent != "claude": agent_hooks.install(a.agent, os.getcwd() if a.project else None, uninstall=True) # it says what it removed + for line in [] if a.project else _kept(): + print(line) return 0 if a.cmd == "uninstall": from . import install @@ -883,6 +910,8 @@ def _run(a): except install.SettingsError as e: print(f"tracekit: {e}", file=sys.stderr) return 1 + for line in [] if a.project else _kept(): + print(line) return 0 if a.cmd == "daemon": from . import daemon @@ -899,6 +928,9 @@ def _run(a): return _approvals(argparse.Namespace(action=a.cmd, approval_id=a.approval_id, signer=None, reason=None)) if a.cmd in ("pending", "approve", "reject"): from . import client + if a.cmd == "pending" and client.socket_path() == client.DEFAULT_SOCKET \ + and not os.path.exists(client.DEFAULT_SOCKET): # no v1 signer here: the v2 signer's held calls + return _approvals(argparse.Namespace(action="list", signer=None, json=False, all=False)) try: pend = client.rpc({"op": "approval_list"}).get("pending", []) if a.cmd == "pending": @@ -1044,6 +1076,72 @@ def _run(a): return 2 +def _kept(): + """What `tracekit uninstall` leaves in place (the dev signer's keys and signed runs, the client config, the dev + signer's runtime dir, a v1 dev signer's home), each that exists, and the commands that delete them and Tracekit.""" + from . import client + from .sdk import autospawn + try: + system, unread = client.system_config() is not None, None + except client.SystemConfigError as e: # one that can't be read is still system mode's + system, unread = True, e + cfg = {} if system else client.client_config() # never read again: a broken one would raise there + data = autospawn.data_dir() + mine, shared = autospawn.own_runtime_dirs() + where = [(data, "the dev signer's keys and signed runs: a bundle you exported verifies without them"), + (client.client_home(), "Tracekit's client config"), + # as uninstall() stops it: a dev home only, never system mode's signer home (/var/lib/tracekit) + (cfg.get("signer_home") if cfg.get("mode") == "dev" else None, "the v1 dev signer's home"), + *[(d, "the dev signer's runtime dir") for d in mine]] + where = [(d, what) for d, what in where if d and os.path.isdir(d)] + note = [f"{unread}, so this lists only your own files"] if unread else [] + if not where: + return note + paths = [d for d, _ in where] + if sys.platform == "darwin": # data/ and run/ in Tracekit's own Application Support dir (autospawn.data_dir) + paths.insert(0, os.path.dirname(data)) + # a dir inside another one the command deletes isn't named again (Windows' run\ is inside its data dir) + paths = [d for d in paths if not any(d.startswith(p + os.sep) for p in paths)] + def ps(path): # Windows: a PowerShell literal, '...' with each character it reads as a single quote doubled + return "'" + re.sub("(['‘’‚‛])", r"\1\1", path) + "'" + pip = ("uv tool uninstall" if re.search(r"[/\\]uv[/\\]tools[/\\]", sys.executable) else # as claude_code.upgrade + "pipx uninstall" if re.search(r"[/\\]pipx[/\\]venvs[/\\]", sys.executable) else + f"& {ps(sys.executable)} -m pip uninstall" if os.name == "nt" else # & runs a quoted program + f"{_quote(sys.executable)} -m pip uninstall") + # -LiteralPath: -Path reads [ ] * ? in a name as wildcards + remove = (f"Remove-Item -Recurse -Force -LiteralPath {', '.join(map(ps, paths))}" if os.name == "nt" + else f"rm -rf {' '.join(map(_quote, paths))}") + return note + (["kept:"] + [f" {d} ({what})" for d, what in where] + + [f" runtime files under {shared} may be shared with another HOME's dev signer: run tracekit down first, " + "and delete only what is yours"] * bool(shared) + + ["to delete these and Tracekit too:", " tracekit down", f" {remove}", f" {pip} tracekit-ai"]) + + +def _status_lines(st, v2, plugin=None, chosen=False): + """`tracekit status` for a person: the v2 signer (install.status's and autospawn.status's findings), where its + keys and runs are, the hooks (and the Claude Code plugin's, `plugin`: install.plugin_enabled's; `chosen`: init + --no-hooks chose none), then a v1 signer and proxy when there is one.""" + name = "dev signer" if v2.get("dev") else "signer" + lines = [f"signer {name} {v2['version']}, running (pid {v2['pid']}), socket {v2['socket']}" if "version" in v2 + else f"signer {name}: {v2.get('error') or 'running'}"] + if v2.get("data_dir"): + lines.append(f"data {v2['data_dir']} (its keys and signed runs)") + found = [f"hooks {' and '.join(h['versions'])} hooks in {h['settings']} ({len(h['events'])} events)" + for h in st["hooks"] if h.get("versions")] + lines += [f"hooks {h['settings']} isn't a settings file Tracekit can read, so its hooks are unknown" + for h in st["hooks"] if "error" in h] + if plugin: # its hooks are the ones init writes, and init writes none while it is on + found.append(f"hooks the Claude Code plugin {plugin[1]}'s, enabled in {plugin[0]}") + lines += found or ["hooks none, as init --no-hooks chose: SDK runs (tracekit.instrument()) are still " + "recorded; init without --no-hooks adds Claude Code's" if chosen else + "hooks none in Claude Code's settings: tracekit init --dev --v2 adds them"] + if st.get("signer"): + lines.append(f"v1 signer {st['signer'].get('error') or 'answering'}") + if st.get("proxy"): + lines.append(f"proxy {st['proxy']['url']}: {'reachable' if st['proxy']['reachable'] else 'unreachable'}") + return lines + + def _signer_extra(): """True when the v2 signer's policy engine imports; else says how to install it.""" from .policy2 import engine @@ -1188,7 +1286,7 @@ def _last(a): from .verify import v2 from .view import write_trust from .sdk.autospawn import data_dir - deadline = time.monotonic() + LAST_WAIT_S + deadline, waiting = time.monotonic() + LAST_WAIT_S, False store = os.path.join(data_dir(), "store") # named when there is none yet try: cfg = signer_config(None) @@ -1207,6 +1305,10 @@ def _last(a): r.close() if not (runs and runs[0][1] == "run.closing" and time.monotonic() < deadline): break + if not waiting: + print(f"tracekit last: run {runs[0][0][1]} ({runs[0][2]}) just closed; waiting up to {LAST_WAIT_S:g} s " + "while the signer finalises it", file=sys.stderr) + waiting = True time.sleep(0.25) # the signer finalises a closed run once its grace window has passed except FileNotFoundError: runs = [] @@ -1224,8 +1326,11 @@ def _last(a): pick = runs if a.open else final if not pick: if not runs: - print(f"tracekit last: no runs in the dev signer's store ({store}) yet: add `import tracekit; " - "tracekit.instrument()` to your agent and run it", file=sys.stderr) + print(f"tracekit last: no runs in the dev signer's store ({store}) yet. `tracekit.instrument()` records " + "only the frameworks it lists, and says so when it finds none of them " + f"({wiki_url('docs/quickstart-v2.md', '2-one-line-in-your-agent')}); for any other agent, record " + f"its calls as {wiki_url('docs/quickstarts/custom.md')} shows. Claude Code: a session started after " + "`tracekit init --dev --v2` records its calls.", file=sys.stderr) return 1 (tenant, run_id), _, agent = pick[0] out = os.path.abspath(a.out or f"{run_id}.tkb") @@ -1330,6 +1435,7 @@ def _stop_lines(c, x): from . import view from .integrations.claude_code import approvals_cmd from .policy2.compile import ENGINE_REASONS + from .policy2.engine import Engine from .signer.service import dev_data_dir try: kept = None if c.signer else os.path.join(dev_data_dir(create=False), "policies") @@ -1337,8 +1443,17 @@ def _stop_lines(c, x): kept = None reasons = view.policy_reasons(kept=kept) known = reasons.get(x["policy_hash"], {}) - flags = {r["id"] for r in reasons.policies.get(x["policy_hash"], {}).get("flag", [])} # a flagged call runs - rules = [(i, known.get(i) or ENGINE_REASONS.get(i) or {"reason": view.ENGINE_RULES.get(i, "")}) + pol = reasons.policies.get(x["policy_hash"], {}) + flags = {r["id"] for r in pol.get("flag", [])} # a flagged call runs + engine = dict(ENGINE_REASONS) + if "TK-SHELL-PARSE" in x["rule_ids"] and pol and x["args"] is not None: + # the cause the signer gave the hook (its decision's `unread`), from the same engine on the signer's copy + try: + cause = Engine(pol).decide(x["tool"], _args(x)).get("unread") + except Exception: # as the signer's _evaluate: an engine error here leaves the general reason + cause = None + engine["TK-SHELL-PARSE"] = ENGINE_REASONS["TK-SHELL-PARSE"]["causes"].get(cause) or engine["TK-SHELL-PARSE"] + rules = [(i, known.get(i) or engine.get(i) or {"reason": view.ENGINE_RULES.get(i, "")}) for i in x["rule_ids"] if i != "TK-UNKNOWN-TOOL" and i not in flags] why = "; ".join(f"{r['reason']} ({i})" for i, r in rules if r.get("reason")) instead = " ".join(dict.fromkeys(r["remedy"] for _, r in rules if r.get("remedy"))) @@ -1346,11 +1461,13 @@ def _stop_lines(c, x): cmd = approvals_cmd(c.signer) why = why or ("not on this machine: `tracekit policy compile ` prints the reasons of the signer's policy, " + x["policy_hash"]) + # Claude Code runs a `!` line in the session as its OS user: the person's, with the dev signer (as the hook says) + bang = not c.signer and _claude_code_run(x["run_id"]) return [f"why {why}", *([f"instead {instead}"] if instead else []), f"expires {x['expires_at']}{left}", *([f"answer `{cmd} approve {x['approval_id']}` to allow it once, `{cmd} reject {x['approval_id']}` " - "to refuse it" + ("" if c.signer else "; in the agent's Claude Code session, with a leading `! `")] + "to refuse it" + ("; in the agent's Claude Code session, with a leading `! `" if bang else "")] if x["state"] == "requested" else []), *_grant_hint(x, reasons.policies.get(x["policy_hash"], {}), flags, cmd), f"record tracekit view --approval {x['approval_id']}" @@ -1358,6 +1475,26 @@ def _stop_lines(c, x): f"a false alarm? tracekit feedback --harmless {x['approval_id']}{cmd.removeprefix('tracekit approvals')}"] +def _claude_code_run(run_id): + """Whether run `run_id` is a Claude Code session's on this machine: the v2 hook keeps each session's run in its + state in the dev signer's runtime dir (integrations.claude_code._state).""" + import glob + + from .sdk import autospawn + try: + d = autospawn.runtime_dir(create=False) + except (OSError, autospawn.SignerUnavailable): + return False + for f in glob.glob(os.path.join(glob.escape(d), "claude-code-*.json")): + try: + with open(f, encoding="utf-8") as fh: + if json.load(fh).get("run_id") == run_id: + return True + except (OSError, ValueError, AttributeError): # one a hook is replacing, or not a state file + continue + return False + + def _grant_hint(x, pol, flags, cmd): """What `approvals show` says about granting calls like held call `x` (docs/intent-grants.md), when every rule that held it opts into a grant in its policy `pol`: how the grant reads the call, and a command that approves it and diff --git a/tracekit/demo_server.py b/tracekit/demo_server.py index eaf9f2e67..475bd2f8a 100644 --- a/tracekit/demo_server.py +++ b/tracekit/demo_server.py @@ -69,12 +69,13 @@ def agent(client): for call_id, command in CALLS: args = {"command": command} d = run.decide(call_id, "Bash", args) - print(f" Bash {command!r}: {d['decision']} {' '.join(d['rule_ids'])}") + print(f" Bash {command!r}: {' '.join([d['decision'], *d['rule_ids']])}") hint = None if d["decision"] == "ask": hint = run.call("approval_request", tool_call_id=call_id)["approval_id"] a = client.approval_decide({"approval_id": hint, "decision": "approve"}) # `tracekit approvals approve` - print(f" approved {hint} (self-approved: {a['self_approved']})") + print(f" approved {hint}" + (", by the run's own user: a self-approval, so the run's assurance stays " + "dev" if a["self_approved"] else "")) if d["decision"] != "deny" and run.approval_consume(call_id, "Bash", args, approval_id_hint=hint)["ok"]: run.complete(call_id) # the pretend command ran return run.run_id diff --git a/tracekit/deploy/files.py b/tracekit/deploy/files.py index 2a2a2ae04..d1bdf1d6b 100644 --- a/tracekit/deploy/files.py +++ b/tracekit/deploy/files.py @@ -161,6 +161,12 @@ def replace_retrying(src, dst, tries=20): time.sleep(0.05) +def remove(d, name): + """Delete `name` in d; a symlink itself, never what it points to.""" + path, at = _at(d, name) + os.unlink(path, **at) + + def backup(d, name, data): """Save data as a new `name.bak-