diff --git a/.gitignore b/.gitignore index eccd5de..a6e14e8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ node_modules/ dist/ *.db *.db-* +*.db.bak* +*.bak-* .env diff --git a/BUILDLOG.md b/BUILDLOG.md index c1326e2..47d4e9b 100644 --- a/BUILDLOG.md +++ b/BUILDLOG.md @@ -634,3 +634,29 @@ titles now have a single named writer, `renameSession` in emit.ts, which masks. Claude Code's own overwrite moved onto it too. Grepping `UPDATE sessions SET title` across the sources returns emit.ts and nothing else, which is the property worth having: an adapter cannot reach that column directly. + +## 2026-09-07 - Masking the history that predates masking + +Masking guards writes, which leaves everything written before it in the clear: +event bodies from before v0.10.0, titles and turn errors from before v0.12.4. +`agentlens redact-history` walks the store and masks what is left, dry by +default, copying the database before it writes. + +It can run over every row without knowing which are old, because masking is +idempotent, and that is worth stating precisely rather than assuming: a mask +literal contains no pattern, so re-masking returns the text byte for byte. The +one case that looks like it should bite does not - `[REDACTED:bearer-token]` +contains "earer" and so trips the cheap hint gate, but the bearer pattern needs +whitespace after it and finds a hyphen. Verified for all twenty-two. + +`seq` is deliberately left alone. A shipper has already sent these events with +their content stripped, so there is nothing for a shared server to catch up on, +and bumping it would resend tens of thousands of rows to no purpose. + +On the local store: 17 events, no titles, no errors. Integrity check clean, row +counts identical, second pass finds nothing. + +And a thing worth remembering rather than the code change: the local dev server +had been running since the previous morning, holding a checkout from before any +of this existed, quietly writing unmasked rows the whole time. A backfill is +worth nothing until the process that made the mess is restarted. diff --git a/README.md b/README.md index 38563e5..6d44123 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,10 @@ patterns from Gitleaks; see [NOTICE](NOTICE)): cloud and provider API keys, GitHub and Slack tokens, private-key blocks, connection strings with credentials, bearer tokens. Their tier-2 key/value guesses are deliberately left out, since `DB_PASSWORD=hunter2` in a transcript is often the thing you -opened the transcript to find. `AGENTLENS_REDACT=0` turns masking off, and rows -written before this existed are not rewritten. +opened the transcript to find. `AGENTLENS_REDACT=0` turns masking off. Masking guards writes, so anything +stored before it existed stays as it was; `agentlens redact-history` reports +what is left and `--apply` masks it, copying the database first. It is safe to +run repeatedly, since masking an already-masked row changes nothing. Claude Code and dsh transcripts are tailed with per-file cursors (subagent transcripts become threads); OpenCode is polled read-only from its SQLite diff --git a/package-lock.json b/package-lock.json index 9a0711c..4365005 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3533,7 +3533,7 @@ }, "packages/server": { "name": "@gengwg/agentlens", - "version": "0.12.5", + "version": "0.13.0", "license": "MIT", "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/packages/server/package.json b/packages/server/package.json index 2834457..ee6056a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -24,7 +24,7 @@ "tsx": "^4.20.0", "typescript": "^5.9.0" }, - "version": "0.12.5", + "version": "0.13.0", "description": "Local observability for coding-agent harnesses: fleet view and trace viewer for Claude Code, OpenCode, dsh, Cursor Agent, Antigravity, TrueForge and more.", "license": "MIT", "repository": { diff --git a/packages/server/src/backfill.ts b/packages/server/src/backfill.ts new file mode 100644 index 0000000..d48b2da --- /dev/null +++ b/packages/server/src/backfill.ts @@ -0,0 +1,74 @@ +import { copyFileSync, existsSync } from "node:fs"; +import { db } from "./db.js"; +import { maskText, redactSecrets } from "./redact-secrets.js"; + +// Masking guards writes, so anything stored before it existed is still in the +// clear: event bodies from before v0.10.0, session titles and turn errors from +// before v0.12.4. This walks the store once and masks what is left. +// +// Safe to run over everything, repeatedly: masking is idempotent. A mask +// literal contains no pattern, so an already-masked row comes back byte for +// byte, and every mask is plain ASCII, so rewriting a serialized event cannot +// break its JSON. +// +// agentlens redact-history what would change +// agentlens redact-history --apply change it, after copying the database + +type Change = { events: number; titles: number; errors: number }; + +export function scan(): Change { + const counts: Change = { events: 0, titles: 0, errors: 0 }; + for (const { raw } of db.prepare(`SELECT raw FROM events`).iterate() as Iterable<{ raw: string }>) + if (redactSecrets(raw) !== raw) counts.events++; + for (const { title } of db.prepare(`SELECT title FROM sessions WHERE title IS NOT NULL`).iterate() as Iterable<{ title: string }>) + if (maskText(title) !== title) counts.titles++; + for (const { error } of db.prepare(`SELECT error FROM turns WHERE error IS NOT NULL`).iterate() as Iterable<{ error: string }>) + if (maskText(error) !== error) counts.errors++; + return counts; +} + +export function apply(): Change { + const counts: Change = { events: 0, titles: 0, errors: 0 }; + // seq is left alone on purpose: a shipper has already sent these events with + // their content stripped, so there is nothing for the shared server to catch + // up on, and bumping it would resend tens of thousands of rows for nothing. + const setRaw = db.prepare(`UPDATE events SET raw = ? WHERE id = ?`); + const setTitle = db.prepare(`UPDATE sessions SET title = ? WHERE id = ?`); + const setError = db.prepare(`UPDATE turns SET error = ? WHERE id = ?`); + + db.transaction(() => { + for (const { id, raw } of db.prepare(`SELECT id, raw FROM events`).all() as { id: string; raw: string }[]) { + const masked = redactSecrets(raw); + if (masked !== raw) (setRaw.run(masked, id), counts.events++); + } + for (const { id, title } of db.prepare(`SELECT id, title FROM sessions WHERE title IS NOT NULL`).all() as { id: string; title: string }[]) { + const masked = maskText(title); + if (masked !== title) (setTitle.run(masked, id), counts.titles++); + } + for (const { id, error } of db.prepare(`SELECT id, error FROM turns WHERE error IS NOT NULL`).all() as { id: string; error: string }[]) { + const masked = maskText(error); + if (masked !== error) (setError.run(masked, id), counts.errors++); + } + })(); + return counts; +} + +const describe = (c: Change) => `${c.events} events, ${c.titles} titles, ${c.errors} turn errors`; + +export function backfillMain(argv: string[]) { + const path = process.env.AGENTLENS_DB ?? "agentlens.db"; + if (!argv.includes("--apply")) { + const found = scan(); + console.log(`${path}: ${describe(found)} hold something the masker would remove`); + console.log(found.events + found.titles + found.errors ? "run again with --apply to mask them" : "nothing to do"); + return; + } + // The store is the only copy of this history; keep one before rewriting it. + // Named to match the *.db-* ignore rule: this file is the whole store. + const backup = `${path}-bak-before-redact`; + if (path !== ":memory:" && existsSync(path) && !existsSync(backup)) { + copyFileSync(path, backup); + console.log(`copied ${path} to ${backup}`); + } + console.log(`masked ${describe(apply())}`); +} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index f94475d..9686b42 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { serve } from "@hono/node-server"; import { serveStatic } from "@hono/node-server/serve-static"; import { app } from "./api.js"; +import { backfillMain } from "./backfill.js"; import { startCollector } from "./collector.js"; import { refreshPrices } from "./db.js"; import { loadPrices, priceFor } from "./prices.js"; @@ -60,4 +61,6 @@ function serveMain() { // `agentlens ship ...` is a client of another AgentLens, not a server. if (process.argv[2] === "ship") await shipMain(process.argv.slice(3)); +// A one-off pass over history written before masking existed. +else if (process.argv[2] === "redact-history") backfillMain(process.argv.slice(3)); else serveMain(); diff --git a/packages/server/test/backfill.test.ts b/packages/server/test/backfill.test.ts new file mode 100644 index 0000000..2501291 --- /dev/null +++ b/packages/server/test/backfill.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { db, seedSession } from "./fixtures.ts"; + +const { apply, scan } = await import("../src/backfill.ts"); +const { redactSecrets } = await import("../src/redact-secrets.ts"); + +// Rows written before masking existed. Inserted straight through SQL, since the +// point is that they bypassed the guarded writers. +const key = "ghp_" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij"; + +test("history written before masking is found and masked, once", () => { + seedSession("bf-old", { turns: [{ id: "bf-t1" }] }); + db.prepare(`INSERT INTO events (id, session_id, turn_id, type, created_at, raw, seq) + VALUES (?, ?, ?, 'tool.response', '2026-09-01T00:00:00Z', ?, 999999)`) + .run("bf-e1", "bf-old", "bf-t1", JSON.stringify({ content: `token is ${key}` })); + db.prepare(`UPDATE sessions SET title = ? WHERE id = ?`).run(`deploy with ${key}`, "bf-old"); + db.prepare(`UPDATE turns SET error = ? WHERE id = ?`).run(`auth failed for ${key}`, "bf-t1"); + + const found = scan(); + assert.ok(found.events >= 1 && found.titles >= 1 && found.errors >= 1, JSON.stringify(found)); + + const changed = apply(); + assert.equal(changed.events, found.events); + assert.equal(changed.titles, found.titles); + assert.equal(changed.errors, found.errors); + + const row = db.prepare(`SELECT raw FROM events WHERE id = ?`).get("bf-e1") as { raw: string }; + const s = db.prepare(`SELECT title FROM sessions WHERE id = ?`).get("bf-old") as { title: string }; + const t = db.prepare(`SELECT error FROM turns WHERE id = ?`).get("bf-t1") as { error: string }; + for (const text of [row.raw, s.title, t.error]) { + assert.ok(!text.includes(key), text); + assert.ok(text.includes("[REDACTED:github-pat]")); + } + assert.ok(row.raw.includes("token is"), "surrounding text survives"); + assert.deepEqual(JSON.parse(row.raw).content, "token is [REDACTED:github-pat]", "still valid JSON"); + + // Idempotent: a second pass finds nothing, which is what makes it safe to run + // over every row rather than tracking which are old. + assert.deepEqual(scan(), { events: 0, titles: 0, errors: 0 }); + assert.deepEqual(apply(), { events: 0, titles: 0, errors: 0 }); +}); + +test("the pass leaves the shipping cursor alone", () => { + // seq must not move: a shipper already sent these events without content, so + // bumping it would resend everything for no gain. + const before = db.prepare(`SELECT seq FROM events WHERE id = ?`).get("bf-e1") as { seq: number }; + apply(); + const after = db.prepare(`SELECT seq FROM events WHERE id = ?`).get("bf-e1") as { seq: number }; + assert.equal(after.seq, before.seq); +}); + +test("a mask literal is not a secret", () => { + assert.equal(redactSecrets("[REDACTED:bearer-token] [REDACTED:private-key]"), "[REDACTED:bearer-token] [REDACTED:private-key]"); +});