From 3f4395b806f05e33564858370d90df0b8c85f344 Mon Sep 17 00:00:00 2001 From: Weigang Geng <3356786+gengwg@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:16:23 -0700 Subject: [PATCH] Add redact-history for the store written before masking Masking guards writes, so event bodies from before v0.10.0 and titles and turn errors from before v0.12.4 are still in the clear. agentlens redact-history reports what is left; --apply masks it after copying the database. Safe over every row, repeatedly: masking is idempotent, verified for all twenty-two patterns. [REDACTED:bearer-token] trips the hint gate, since it contains "earer", but the pattern needs whitespace and finds a hyphen. seq is left alone. A shipper already sent these events without content, so bumping it would resend tens of thousands of rows for nothing. On the local store: 17 events, no titles, no errors, integrity clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NZDHk4yVYP4jMZkTUMQ1Lw --- .gitignore | 2 + BUILDLOG.md | 26 ++++++++++ README.md | 6 ++- package-lock.json | 2 +- packages/server/package.json | 2 +- packages/server/src/backfill.ts | 74 +++++++++++++++++++++++++++ packages/server/src/index.ts | 3 ++ packages/server/test/backfill.test.ts | 55 ++++++++++++++++++++ 8 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 packages/server/src/backfill.ts create mode 100644 packages/server/test/backfill.test.ts diff --git a/.gitignore b/.gitignore index eccd5de..a6e14e8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ node_modules/ dist/ *.db *.db-* +*.db.bak* +*.bak-* .env diff --git a/BUILDLOG.md b/BUILDLOG.md index c1326e2..47d4e9b 100644 --- a/BUILDLOG.md +++ b/BUILDLOG.md @@ -634,3 +634,29 @@ titles now have a single named writer, `renameSession` in emit.ts, which masks. Claude Code's own overwrite moved onto it too. Grepping `UPDATE sessions SET title` across the sources returns emit.ts and nothing else, which is the property worth having: an adapter cannot reach that column directly. + +## 2026-09-07 - Masking the history that predates masking + +Masking guards writes, which leaves everything written before it in the clear: +event bodies from before v0.10.0, titles and turn errors from before v0.12.4. +`agentlens redact-history` walks the store and masks what is left, dry by +default, copying the database before it writes. + +It can run over every row without knowing which are old, because masking is +idempotent, and that is worth stating precisely rather than assuming: a mask +literal contains no pattern, so re-masking returns the text byte for byte. The +one case that looks like it should bite does not - `[REDACTED:bearer-token]` +contains "earer" and so trips the cheap hint gate, but the bearer pattern needs +whitespace after it and finds a hyphen. Verified for all twenty-two. + +`seq` is deliberately left alone. A shipper has already sent these events with +their content stripped, so there is nothing for a shared server to catch up on, +and bumping it would resend tens of thousands of rows to no purpose. + +On the local store: 17 events, no titles, no errors. Integrity check clean, row +counts identical, second pass finds nothing. + +And a thing worth remembering rather than the code change: the local dev server +had been running since the previous morning, holding a checkout from before any +of this existed, quietly writing unmasked rows the whole time. A backfill is +worth nothing until the process that made the mess is restarted. diff --git a/README.md b/README.md index 38563e5..6d44123 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,10 @@ patterns from Gitleaks; see [NOTICE](NOTICE)): cloud and provider API keys, GitHub and Slack tokens, private-key blocks, connection strings with credentials, bearer tokens. Their tier-2 key/value guesses are deliberately left out, since `DB_PASSWORD=hunter2` in a transcript is often the thing you -opened the transcript to find. `AGENTLENS_REDACT=0` turns masking off, and rows -written before this existed are not rewritten. +opened the transcript to find. `AGENTLENS_REDACT=0` turns masking off. Masking guards writes, so anything +stored before it existed stays as it was; `agentlens redact-history` reports +what is left and `--apply` masks it, copying the database first. It is safe to +run repeatedly, since masking an already-masked row changes nothing. Claude Code and dsh transcripts are tailed with per-file cursors (subagent transcripts become threads); OpenCode is polled read-only from its SQLite diff --git a/package-lock.json b/package-lock.json index 9a0711c..4365005 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3533,7 +3533,7 @@ }, "packages/server": { "name": "@gengwg/agentlens", - "version": "0.12.5", + "version": "0.13.0", "license": "MIT", "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/packages/server/package.json b/packages/server/package.json index 2834457..ee6056a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -24,7 +24,7 @@ "tsx": "^4.20.0", "typescript": "^5.9.0" }, - "version": "0.12.5", + "version": "0.13.0", "description": "Local observability for coding-agent harnesses: fleet view and trace viewer for Claude Code, OpenCode, dsh, Cursor Agent, Antigravity, TrueForge and more.", "license": "MIT", "repository": { diff --git a/packages/server/src/backfill.ts b/packages/server/src/backfill.ts new file mode 100644 index 0000000..d48b2da --- /dev/null +++ b/packages/server/src/backfill.ts @@ -0,0 +1,74 @@ +import { copyFileSync, existsSync } from "node:fs"; +import { db } from "./db.js"; +import { maskText, redactSecrets } from "./redact-secrets.js"; + +// Masking guards writes, so anything stored before it existed is still in the +// clear: event bodies from before v0.10.0, session titles and turn errors from +// before v0.12.4. This walks the store once and masks what is left. +// +// Safe to run over everything, repeatedly: masking is idempotent. A mask +// literal contains no pattern, so an already-masked row comes back byte for +// byte, and every mask is plain ASCII, so rewriting a serialized event cannot +// break its JSON. +// +// agentlens redact-history what would change +// agentlens redact-history --apply change it, after copying the database + +type Change = { events: number; titles: number; errors: number }; + +export function scan(): Change { + const counts: Change = { events: 0, titles: 0, errors: 0 }; + for (const { raw } of db.prepare(`SELECT raw FROM events`).iterate() as Iterable<{ raw: string }>) + if (redactSecrets(raw) !== raw) counts.events++; + for (const { title } of db.prepare(`SELECT title FROM sessions WHERE title IS NOT NULL`).iterate() as Iterable<{ title: string }>) + if (maskText(title) !== title) counts.titles++; + for (const { error } of db.prepare(`SELECT error FROM turns WHERE error IS NOT NULL`).iterate() as Iterable<{ error: string }>) + if (maskText(error) !== error) counts.errors++; + return counts; +} + +export function apply(): Change { + const counts: Change = { events: 0, titles: 0, errors: 0 }; + // seq is left alone on purpose: a shipper has already sent these events with + // their content stripped, so there is nothing for the shared server to catch + // up on, and bumping it would resend tens of thousands of rows for nothing. + const setRaw = db.prepare(`UPDATE events SET raw = ? WHERE id = ?`); + const setTitle = db.prepare(`UPDATE sessions SET title = ? WHERE id = ?`); + const setError = db.prepare(`UPDATE turns SET error = ? WHERE id = ?`); + + db.transaction(() => { + for (const { id, raw } of db.prepare(`SELECT id, raw FROM events`).all() as { id: string; raw: string }[]) { + const masked = redactSecrets(raw); + if (masked !== raw) (setRaw.run(masked, id), counts.events++); + } + for (const { id, title } of db.prepare(`SELECT id, title FROM sessions WHERE title IS NOT NULL`).all() as { id: string; title: string }[]) { + const masked = maskText(title); + if (masked !== title) (setTitle.run(masked, id), counts.titles++); + } + for (const { id, error } of db.prepare(`SELECT id, error FROM turns WHERE error IS NOT NULL`).all() as { id: string; error: string }[]) { + const masked = maskText(error); + if (masked !== error) (setError.run(masked, id), counts.errors++); + } + })(); + return counts; +} + +const describe = (c: Change) => `${c.events} events, ${c.titles} titles, ${c.errors} turn errors`; + +export function backfillMain(argv: string[]) { + const path = process.env.AGENTLENS_DB ?? "agentlens.db"; + if (!argv.includes("--apply")) { + const found = scan(); + console.log(`${path}: ${describe(found)} hold something the masker would remove`); + console.log(found.events + found.titles + found.errors ? "run again with --apply to mask them" : "nothing to do"); + return; + } + // The store is the only copy of this history; keep one before rewriting it. + // Named to match the *.db-* ignore rule: this file is the whole store. + const backup = `${path}-bak-before-redact`; + if (path !== ":memory:" && existsSync(path) && !existsSync(backup)) { + copyFileSync(path, backup); + console.log(`copied ${path} to ${backup}`); + } + console.log(`masked ${describe(apply())}`); +} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index f94475d..9686b42 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { serve } from "@hono/node-server"; import { serveStatic } from "@hono/node-server/serve-static"; import { app } from "./api.js"; +import { backfillMain } from "./backfill.js"; import { startCollector } from "./collector.js"; import { refreshPrices } from "./db.js"; import { loadPrices, priceFor } from "./prices.js"; @@ -60,4 +61,6 @@ function serveMain() { // `agentlens ship ...` is a client of another AgentLens, not a server. if (process.argv[2] === "ship") await shipMain(process.argv.slice(3)); +// A one-off pass over history written before masking existed. +else if (process.argv[2] === "redact-history") backfillMain(process.argv.slice(3)); else serveMain(); diff --git a/packages/server/test/backfill.test.ts b/packages/server/test/backfill.test.ts new file mode 100644 index 0000000..2501291 --- /dev/null +++ b/packages/server/test/backfill.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { db, seedSession } from "./fixtures.ts"; + +const { apply, scan } = await import("../src/backfill.ts"); +const { redactSecrets } = await import("../src/redact-secrets.ts"); + +// Rows written before masking existed. Inserted straight through SQL, since the +// point is that they bypassed the guarded writers. +const key = "ghp_" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij"; + +test("history written before masking is found and masked, once", () => { + seedSession("bf-old", { turns: [{ id: "bf-t1" }] }); + db.prepare(`INSERT INTO events (id, session_id, turn_id, type, created_at, raw, seq) + VALUES (?, ?, ?, 'tool.response', '2026-09-01T00:00:00Z', ?, 999999)`) + .run("bf-e1", "bf-old", "bf-t1", JSON.stringify({ content: `token is ${key}` })); + db.prepare(`UPDATE sessions SET title = ? WHERE id = ?`).run(`deploy with ${key}`, "bf-old"); + db.prepare(`UPDATE turns SET error = ? WHERE id = ?`).run(`auth failed for ${key}`, "bf-t1"); + + const found = scan(); + assert.ok(found.events >= 1 && found.titles >= 1 && found.errors >= 1, JSON.stringify(found)); + + const changed = apply(); + assert.equal(changed.events, found.events); + assert.equal(changed.titles, found.titles); + assert.equal(changed.errors, found.errors); + + const row = db.prepare(`SELECT raw FROM events WHERE id = ?`).get("bf-e1") as { raw: string }; + const s = db.prepare(`SELECT title FROM sessions WHERE id = ?`).get("bf-old") as { title: string }; + const t = db.prepare(`SELECT error FROM turns WHERE id = ?`).get("bf-t1") as { error: string }; + for (const text of [row.raw, s.title, t.error]) { + assert.ok(!text.includes(key), text); + assert.ok(text.includes("[REDACTED:github-pat]")); + } + assert.ok(row.raw.includes("token is"), "surrounding text survives"); + assert.deepEqual(JSON.parse(row.raw).content, "token is [REDACTED:github-pat]", "still valid JSON"); + + // Idempotent: a second pass finds nothing, which is what makes it safe to run + // over every row rather than tracking which are old. + assert.deepEqual(scan(), { events: 0, titles: 0, errors: 0 }); + assert.deepEqual(apply(), { events: 0, titles: 0, errors: 0 }); +}); + +test("the pass leaves the shipping cursor alone", () => { + // seq must not move: a shipper already sent these events without content, so + // bumping it would resend everything for no gain. + const before = db.prepare(`SELECT seq FROM events WHERE id = ?`).get("bf-e1") as { seq: number }; + apply(); + const after = db.prepare(`SELECT seq FROM events WHERE id = ?`).get("bf-e1") as { seq: number }; + assert.equal(after.seq, before.seq); +}); + +test("a mask literal is not a secret", () => { + assert.equal(redactSecrets("[REDACTED:bearer-token] [REDACTED:private-key]"), "[REDACTED:bearer-token] [REDACTED:private-key]"); +});