diff --git a/.vedeiignore b/.vedeiignore index 81a635a..e958aa5 100644 --- a/.vedeiignore +++ b/.vedeiignore @@ -79,6 +79,7 @@ d0dead6eceff137cafd15c0029bf1214 # AKIA...LE example key, cmd/vedei-hook/main_t 2040ad664a71a504a6ff863b0b78a4f0 # generated PIS vector, space-separated, detect/br/extract_test.go 62949e15935851bd24d617292e7230f8 # generated titulo vector, space-separated, detect/br/extract_test.go ec2a7642f9821664294f09e54573ca66 # generated CNS vector, detect/br/extract_test.go +c3fca3e7a0151668ec85c974b0f4f57f # the made-up credential from the corpus, also in transcript/stats_test.go 3d916d891e1aab79bc68592c2d7cd3a5 # the first alphanumeric CNPJ the Receita issued; an earlier version of this file quoted it in a comment, so git history still holds it # --- the one path rule: the labeled corpus --- diff --git a/README.md b/README.md index fd0eb64..0bb3f6e 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,15 @@ vedei reduces what lands there and shows what already did. It does not replace d Documented upstream: [#44868](https://github.com/anthropics/claude-code/issues/44868), [#50014](https://github.com/anthropics/claude-code/issues/50014), [#95680](https://github.com/anthropics/claude-code/issues/95680). +**Which agents' transcripts are read.** `vedei transcript scan` and `scrub` cover Claude Code (`~/.claude/projects/`) and Codex (`~/.codex/sessions/`). Two other agents keep the same kind of record and are **not** read yet: + +| agent | where it keeps sessions | why not yet | +|---|---|---| +| Gemini CLI | `~/.gemini/tmp//chats/`, JSON files | the format is documented but untested against real sessions; reading it blind would report coverage it does not have | +| Cursor | `state.vscdb`, a SQLite database under the Cursor user directory | reading it needs a SQLite driver, and `scrub` cannot safely rewrite a database | + +Both are plaintext on disk too. Until they are supported, treat their directories as holding whatever their agents read. + ## What "validate" means here | | Secret | Personal data | diff --git a/README.pt-BR.md b/README.pt-BR.md index 8030306..f50ccbe 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -99,6 +99,16 @@ O vedei reduz o que entra ali e mostra o que já entrou. Não substitui criptogr Documentado no próprio repositório do Claude Code: [#44868](https://github.com/anthropics/claude-code/issues/44868), [#50014](https://github.com/anthropics/claude-code/issues/50014), [#95680](https://github.com/anthropics/claude-code/issues/95680). + +**De quais agentes o transcript é lido.** `vedei transcript scan` e `scrub` cobrem Claude Code (`~/.claude/projects/`) e Codex (`~/.codex/sessions/`). Dois outros agentes guardam o mesmo tipo de registro e **ainda não** são lidos: + +| agente | onde guarda as sessões | por que ainda não | +|---|---|---| +| Gemini CLI | `~/.gemini/tmp//chats/`, arquivos JSON | o formato está documentado, mas não foi testado contra sessões reais; ler às cegas reportaria uma cobertura que não existe | +| Cursor | `state.vscdb`, um banco SQLite no diretório de usuário do Cursor | ler exige um driver SQLite, e o `scrub` não reescreve um banco com segurança | + +Os dois também ficam em texto puro no disco. Até serem suportados, trate esses diretórios como contendo tudo que os agentes leram. + ## O que "validar" significa aqui | | Segredo | Dado pessoal | diff --git a/SECURITY.md b/SECURITY.md index c6c5ddf..0016ba8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -102,6 +102,14 @@ tokens are detected by the upstream corpus. written to disk. `--idle-timeout` makes it exit when unused. - A request is capped at 16 MB, so one client cannot decide how much memory the process holds. +- **None of the above holds on Windows.** vedei builds there, and Windows 10 + 1803+ supports Unix-domain sockets, but `os.Chmod` on Windows only toggles the + read-only attribute: the `0600` and `0700` modes restrict nobody, because + Windows controls access with ACLs. The socket's privacy then rests on the + default ACL of the user's profile directory — normally the user, + Administrators and SYSTEM — which vedei does not verify. It is also untested + on Windows. Until that changes, run `vedei hook` without the daemon on + Windows (see `hooks/README.md`). **Findings crossing the socket cannot carry the raw value.** `Finding.Raw` is `json:"-"`, so the process boundary enforces ADR-003 rather than trusting it. diff --git a/cmd/vedei-hook/main.go b/cmd/vedei-hook/main.go index c5282dd..b32e6d8 100644 --- a/cmd/vedei-hook/main.go +++ b/cmd/vedei-hook/main.go @@ -38,7 +38,9 @@ func main() { // for as long as it lived. fallbackTimeout := flag.Duration("fallback-timeout", 30*time.Second, "give up on the fallback command after this long") - fallback := flag.String("fallback", "", `command to fall back to, or "" to look for vedei beside this binary; "-" to disable`) + fallback := flag.String("fallback", "", `vedei binary to fall back to, or "" to look for one beside this binary; "-" to disable`) + failClosed := flag.Bool("fail-closed", false, + "when detection cannot run, withhold the tool output instead of passing it through unscanned") showVersion := flag.Bool("version", false, "print the version and exit") flag.Parse() @@ -52,9 +54,9 @@ func main() { return // nothing to write back } - out, ok := viaDaemon(context.Background(), input, *socket, *timeout) + out, ok := viaDaemon(context.Background(), input, *socket, *timeout, *failClosed) if !ok { - out = viaFallback(input, *fallback, *fallbackTimeout) + out = viaFallback(input, *fallback, *fallbackTimeout, *failClosed) } _, _ = os.Stdout.Write(out) } @@ -62,7 +64,7 @@ func main() { // viaDaemon rewrites the event through the daemon, reporting whether it was // able to. A false means the daemon could not be reached or could not answer; // it does not mean the event was clean. -func viaDaemon(ctx context.Context, input []byte, socket string, timeout time.Duration) ([]byte, bool) { +func viaDaemon(ctx context.Context, input []byte, socket string, timeout time.Duration, failClosed bool) ([]byte, bool) { if socket == "" { return nil, false } @@ -82,6 +84,10 @@ func viaDaemon(ctx context.Context, input []byte, socket string, timeout time.Du // being unreachable. Re-running it elsewhere would fail the same way, // so the event is served, unredacted. served = true + if resp.Degraded && failClosed { + fmt.Fprintln(os.Stderr, "vedei: output withheld, the daemon could not scan it") + return hookevent.Withheld, true + } if resp.Degraded || !resp.Redacted { return "", false } @@ -95,11 +101,29 @@ func viaDaemon(ctx context.Context, input []byte, socket string, timeout time.Du // viaFallback hands the event to the full binary. On any failure — no binary, // a non-zero exit, empty output — the original event is returned. -func viaFallback(input []byte, override string, timeout time.Duration) []byte { +func viaFallback(input []byte, override string, timeout time.Duration, failClosed bool) []byte { + // Every way this function gives up returns the same thing: the input as it + // arrived, or — under --fail-closed — the input with its output withheld. + giveUp := func() []byte { + if failClosed { + return hookevent.Withhold(input) + } + return input + } + bin, args := resolveFallback(override) if bin == "" { - fmt.Fprintln(os.Stderr, "vedei: no daemon and no vedei binary found; event passed through unredacted") - return input + if failClosed { + fmt.Fprintln(os.Stderr, "vedei: no daemon and no vedei binary found; output withheld") + } else { + fmt.Fprintln(os.Stderr, "vedei: no daemon and no vedei binary found; event passed through unredacted") + } + return giveUp() + } + // The full binary is told to fail closed too, so a fallback that cannot + // scan withholds rather than passing the output through. + if failClosed { + args = append(args, "--fail-closed") } ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -110,37 +134,43 @@ func viaFallback(input []byte, override string, timeout time.Duration) []byte { cmd.Stderr = os.Stderr out, err := cmd.Output() if err != nil || len(out) == 0 { - return input + return giveUp() } return out } -// resolveFallback picks the command to fall back to: the --fallback flag, then -// VEDEI_BIN, then a vedei sitting beside this binary, then one on PATH. +// resolveFallback picks the vedei binary to fall back to: the --fallback flag, +// then VEDEI_BIN, then a vedei sitting beside this binary, then one on PATH. +// Every source is run the same way, as "vedei hook --no-daemon". +// +// --fallback used to be run with no arguments at all, which made vedei print +// its help — and the help text was written back to the agent in place of the +// tool's output. A path given here is a vedei binary like any other source. // // The sibling is tried before PATH so an installation is self-contained: the // pair is built and shipped together, and a different vedei earlier on PATH // should not quietly take over. func resolveFallback(override string) (bin string, args []string) { + hookArgs := []string{"hook", "--no-daemon"} switch override { case "-": return "", nil case "": default: - return override, nil + return override, hookArgs } if env := os.Getenv("VEDEI_BIN"); env != "" { - return env, []string{"hook", "--no-daemon"} + return env, hookArgs } if self, err := os.Executable(); err == nil { sibling := filepath.Join(filepath.Dir(self), "vedei") if st, err := os.Stat(sibling); err == nil && !st.IsDir() { - return sibling, []string{"hook", "--no-daemon"} + return sibling, hookArgs } } if found, err := exec.LookPath("vedei"); err == nil { - return found, []string{"hook", "--no-daemon"} + return found, hookArgs } return "", nil } diff --git a/cmd/vedei-hook/main_test.go b/cmd/vedei-hook/main_test.go index 41ac12a..c32c6da 100644 --- a/cmd/vedei-hook/main_test.go +++ b/cmd/vedei-hook/main_test.go @@ -141,8 +141,27 @@ func TestHookClient_FallsBackToTheFullBinary(t *testing.T) { out := runHook(t, hook, event("cpf "+testCPF), "--socket", shortSocket(t), "--fallback", vedei) - if strings.Contains(out, testCPF) { - t.Fatalf("the fallback did not redact: %s", out) + assertRedactedEvent(t, out) +} + +// assertRedactedEvent checks that out is the event, with its output redacted. +// +// "The value is gone" is not enough on its own. --fallback once ran vedei with +// no arguments, so the output was vedei's help text: no CPF in it, and a test +// that only looked for the CPF passed while the agent received a usage screen +// instead of its tool's result. +func assertRedactedEvent(t *testing.T, out string) { + t.Helper() + var ev map[string]any + if err := json.Unmarshal([]byte(out), &ev); err != nil { + t.Fatalf("the output is not the event: %v\n%s", err, out) + } + resp, _ := ev["tool_response"].(string) + if strings.Contains(resp, testCPF) { + t.Fatalf("the value got through: %s", resp) + } + if !strings.Contains(resp, "vedei: cpf redacted") { + t.Errorf("tool_response = %q, want the value redacted", resp) } } @@ -158,6 +177,32 @@ func TestHookClient_PassesThroughWithNothingAvailable(t *testing.T) { } } +// No daemon and no fallback is the case --fail-closed exists for: nothing can +// scan the output, so it is withheld instead of passed on. +func TestHookClient_FailClosedWithholdsWithNothingAvailable(t *testing.T) { + _, hook := bins(t) + + out := runHook(t, hook, event("cpf "+testCPF), + "--socket", shortSocket(t), "--fallback", "-", "--fail-closed") + if strings.Contains(out, testCPF) { + t.Fatalf("the unscanned value got through: %s", out) + } + if !strings.Contains(out, "output withheld") { + t.Errorf("no withheld notice: %s", out) + } +} + +// The flag is passed on to the full binary when the client falls back to it, +// so a fallback that cannot scan withholds too. Here the fallback works, and +// the value is redacted rather than withheld. +func TestHookClient_FailClosedFallbackStillRedacts(t *testing.T) { + vedei, hook := bins(t) + + out := runHook(t, hook, event("cpf "+testCPF), + "--socket", shortSocket(t), "--fallback", vedei, "--fail-closed") + assertRedactedEvent(t, out) +} + func TestHookClient_FailsOpenOnMalformedInput(t *testing.T) { vedei, hook := bins(t) socket := shortSocket(t) diff --git a/cmd/vedei/failclosed_test.go b/cmd/vedei/failclosed_test.go new file mode 100644 index 0000000..fc995ed --- /dev/null +++ b/cmd/vedei/failclosed_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mkmuniz/vedei/internal/hookevent" +) + +// degradedOpts makes detection fail on purpose: an input cap smaller than any +// event marks the scan degraded, the same state a panicking engine produces. +func degradedOpts(failClosed bool) redactOpts { + return redactOpts{noDaemon: true, maxInput: 5, failClosed: failClosed} +} + +// The default stays fail-open (ADR-005): a scan that could not run passes the +// event through unchanged, so the session keeps working. +func TestRunHook_FailOpenByDefault(t *testing.T) { + in := []byte(`{"tool_name":"Read","tool_response":"cpf ` + testCPF + `"}`) + if got := runHook(context.Background(), in, degradedOpts(false)); !bytes.Equal(got, in) { + t.Errorf("fail-open altered the event:\n got %s\nwant %s", got, in) + } +} + +// --fail-closed is for the environments where an unscanned CPF reaching a +// model is a reportable incident: the output is withheld instead. +func TestRunHook_FailClosedWithholds(t *testing.T) { + in := []byte(`{"tool_name":"Read","tool_response":"cpf ` + testCPF + `"}`) + got := string(runHook(context.Background(), in, degradedOpts(true))) + + if strings.Contains(got, testCPF) { + t.Fatalf("the unscanned value got through: %s", got) + } + if !strings.Contains(got, "output withheld") { + t.Errorf("no withheld notice: %s", got) + } + if !strings.Contains(got, `"tool_name":"Read"`) { + t.Errorf("the rest of the event was lost: %s", got) + } +} + +// --fail-closed changes only what happens when detection fails. A scan that +// runs and finds nothing still passes the text through. +func TestRunHook_FailClosedLeavesCleanTextAlone(t *testing.T) { + in := []byte(`{"tool_response":"nada aqui"}`) + opts := redactOpts{noDaemon: true, failClosed: true} + if got := runHook(context.Background(), in, opts); !bytes.Equal(got, in) { + t.Errorf("a clean event was altered: %s", got) + } +} + +// stream under --fail-closed writes the notice, none of the input, and exits +// with an error rather than reporting clean. +func TestStream_FailClosed(t *testing.T) { + bin := os.Getenv("VEDEI_BIN") + if bin == "" { + bin = "../../bin/vedei" + } + if _, err := os.Stat(bin); err != nil { + t.Skip("binary not built; run make build") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, bin, "stream", "--no-daemon", "--max-input", "5", "--fail-closed", //nolint:gosec // fixed path + "--socket", filepath.Join(t.TempDir(), "absent.sock")) + cmd.Stdin = strings.NewReader("cpf " + testCPF) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + + var exitErr *exec.ExitError + if err == nil || !errors.As(err, &exitErr) || exitErr.ExitCode() != exitError { + t.Fatalf("exit = %v, want %d", err, exitError) + } + if strings.Contains(out.String(), testCPF) { + t.Fatalf("the unscanned value got through: %q", out.String()) + } + if strings.TrimSpace(out.String()) != hookevent.Withheld { + t.Errorf("stdout = %q, want only the withheld notice", out.String()) + } +} diff --git a/cmd/vedei/hook.go b/cmd/vedei/hook.go index 39b3f2e..f459daa 100644 --- a/cmd/vedei/hook.go +++ b/cmd/vedei/hook.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "os" @@ -40,19 +41,7 @@ unchanged. A hook that blocks is worse than a hook that misses something.`, if err != nil { return nil } - out := hookevent.Rewrite(input, func(text string) (string, bool) { - res := redact(cmd.Context(), text, detect.Metadata{Source: "agent-hook"}, opts) - if res.Degraded || !res.Redacted { - return "", false - } - // Report on stderr, which the agent shows the user but does - // not send to the model, so a person can see what was held back. - for _, f := range res.Findings { - fmt.Fprintf(os.Stderr, "vedei: redacted %s\n", f.Type) - } - return res.Text, true - }) - if _, err := os.Stdout.Write(out); err != nil { + if _, err := os.Stdout.Write(runHook(cmd.Context(), input, opts)); err != nil { return err } return nil @@ -62,3 +51,26 @@ unchanged. A hook that blocks is worse than a hook that misses something.`, bindRedactFlags(cmd, &opts, true) return cmd } + +// runHook rewrites one hook event. Reports go to stderr, which the agent shows +// the user but does not send to the model, so a person can see what was held +// back. +func runHook(ctx context.Context, input []byte, opts redactOpts) []byte { + return hookevent.Rewrite(input, func(text string) (string, bool) { + res := redact(ctx, text, detect.Metadata{Source: "agent-hook"}, opts) + if res.Degraded { + if opts.failClosed { + fmt.Fprintf(os.Stderr, "vedei: output withheld, detection failed: %v\n", res.Err) + return hookevent.Withheld, true + } + return "", false + } + if !res.Redacted { + return "", false + } + for _, f := range res.Findings { + fmt.Fprintf(os.Stderr, "vedei: redacted %s\n", f.Type) + } + return res.Text, true + }) +} diff --git a/cmd/vedei/redact.go b/cmd/vedei/redact.go index 3f2894a..f83d7cd 100644 --- a/cmd/vedei/redact.go +++ b/cmd/vedei/redact.go @@ -14,10 +14,11 @@ import ( // redactOpts are the flags every redaction command shares. type redactOpts struct { - socket string - noDaemon bool - fast bool - maxInput int + socket string + noDaemon bool + fast bool + maxInput int + failClosed bool } // bindRedactFlags registers the shared flags on cmd. @@ -29,6 +30,8 @@ func bindRedactFlags(cmd *cobra.Command, o *redactOpts, withFast bool) { "daemon socket to use when one is listening") cmd.Flags().BoolVar(&o.noDaemon, "no-daemon", false, "always scan in-process, even when a daemon is listening") + cmd.Flags().BoolVar(&o.failClosed, "fail-closed", false, + "when detection cannot run, withhold the text instead of passing it through unscanned") if withFast { cmd.Flags().BoolVar(&o.fast, "fast", false, "skip the secret rule set, cutting startup at the cost of missing credentials (ignored when the daemon answers)") diff --git a/cmd/vedei/stream.go b/cmd/vedei/stream.go index ff837d4..ce1db5a 100644 --- a/cmd/vedei/stream.go +++ b/cmd/vedei/stream.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/mkmuniz/vedei/detect" + "github.com/mkmuniz/vedei/internal/hookevent" ) func newStreamCmd() *cobra.Command { @@ -36,6 +37,13 @@ rule set; without one the work happens in-process.`, } res := redact(cmd.Context(), string(content), detect.Metadata{Source: "stdin"}, opts) + if res.Degraded && opts.failClosed { + // Nothing of the input is written: it could not be checked. + if _, err := fmt.Fprintln(os.Stdout, hookevent.Withheld); err != nil { + return err + } + return fmt.Errorf("detection failed, output withheld: %w", res.Err) + } if _, err := os.Stdout.WriteString(res.Text); err != nil { return err } diff --git a/cmd/vedei/transcript.go b/cmd/vedei/transcript.go index 2233c64..bacdfdd 100644 --- a/cmd/vedei/transcript.go +++ b/cmd/vedei/transcript.go @@ -25,6 +25,7 @@ func newTranscriptScanCmd() *cobra.Command { var ( path string asJSON bool + stats bool ) cmd := &cobra.Command{ @@ -37,7 +38,11 @@ and no security tool watching them. A .env has file permissions and a gitignore; a transcript has neither, while aggregating secrets from every source the agent ever touched. Whoever reads the disk reads all of it. -This command reports. It does not modify anything.`, +This command reports. It does not modify anything. + +Gemini CLI and Cursor keep transcripts too and are not read yet: Gemini's JSON +format is untested against real sessions, and Cursor's is a SQLite database. +See the README for where each keeps them.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { roots := map[transcript.Agent]string{} @@ -68,13 +73,22 @@ This command reports. It does not modify anything.`, } } - if asJSON { + switch { + case stats && asJSON: + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(transcript.Summarize(all, scanned)); err != nil { + return err + } + case stats: + printStats(transcript.Summarize(all, scanned)) + case asJSON: enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") if err := enc.Encode(all); err != nil { return err } - } else { + default: printSessions(all, scanned) } @@ -91,9 +105,30 @@ This command reports. It does not modify anything.`, cmd.Flags().StringVar(&path, "path", "", "audit this directory instead of the default agent locations") cmd.Flags().BoolVar(&asJSON, "json", false, "emit JSON instead of a table") + cmd.Flags().BoolVar(&stats, "stats", false, + "print counts only — no values, paths or fingerprints — so the result can be shared") return cmd } +// printStats renders the shareable summary. Everything it prints is a count. +func printStats(st transcript.Stats) { + fmt.Printf("%d transcript(s) scanned\n", st.Transcripts) + fmt.Printf("%d contain at least one finding\n", st.WithFinding) + fmt.Printf("%d contain a credential\n", st.WithCredential) + fmt.Printf("%d contain Brazilian personal data\n", st.WithPersonalData) + + if types := st.Types(); len(types) > 0 { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "\nTYPE\tTRANSCRIPTS\tDISTINCT VALUES") + for _, t := range types { + ts := st.ByType[t] + _, _ = fmt.Fprintf(w, "%s\t%d\t%d\n", t, ts.Transcripts, ts.Values) + } + _ = w.Flush() + } + fmt.Println("\nCounts only: no values, paths or fingerprints. Safe to share.") +} + func printSessions(sessions []transcript.Session, scanned int) { if len(sessions) == 0 { fmt.Printf("%d transcript(s) scanned, nothing sensitive found\n", scanned) diff --git a/hooks/README.md b/hooks/README.md index 685684b..c3c11f8 100644 --- a/hooks/README.md +++ b/hooks/README.md @@ -109,3 +109,23 @@ vedei daemon status Pass `--idle-timeout 8h` if you would rather it exit when unused. Socket path resolution, in order: `VEDEI_SOCKET`, `$XDG_RUNTIME_DIR/vedei/sock`, `~/.vedei/sock`. + +**Windows: run without the daemon, for now.** The daemon builds on Windows but +its access control does not: the `0600`/`0700` modes that keep the socket +private on macOS and Linux are ignored there, since Windows uses ACLs, and the +daemon is untested on it. Configure the hook as `vedei hook --no-daemon`. It +costs about 29 ms per tool call instead of 8, and nothing crosses a socket. + +The plan to close it, in order: + +1. Create the socket's directory with an explicit security descriptor that + grants access to the current user's SID only, through + `golang.org/x/sys/windows`, already a transitive dependency — rather than + relying on the profile directory's default ACL. +2. Assert it in a test that reads the descriptor back, the Windows counterpart + of `TestListen_SocketAndDirectoryArePrivate`. +3. Run the daemon and hook test suites on a Windows CI runner, so "untested" + stops being true. + +A named pipe with the same descriptor is the alternative if Unix-domain sockets +turn out to misbehave on older Windows builds. diff --git a/hooks/claude-code/README.md b/hooks/claude-code/README.md index 61ad1fb..4b517c5 100644 --- a/hooks/claude-code/README.md +++ b/hooks/claude-code/README.md @@ -87,3 +87,27 @@ rule it out. `vedei daemon status` says whether the daemon is answering. When it is not, `vedei-hook` runs the full binary instead — correct, just slower. + +## Regulated environments: `--fail-closed` + +Fail-open is the right default for most people: a hook that breaks the session +gets uninstalled. It is the wrong trade where an unscanned CPF reaching a model +is a reportable incident. For those environments: + +```json +{ "type": "command", "command": "vedei-hook --fail-closed", "timeout": 5 } +``` + +With the flag, when detection cannot run — the engine fails, the input is over +the size cap, or there is no daemon and no vedei binary to fall back to — the +tool output is replaced by a notice instead of being passed through: + +``` +[vedei: output withheld — detection could not run, and --fail-closed is set] +``` + +The agent loses that one result and can retry; the value never reaches the +model unscanned. Everything else is unchanged: a scan that runs still redacts, +and a clean result still passes. Two things stay fail-open even with the flag, +because there is no tool output in them to withhold: malformed JSON, and an +event shape vedei does not recognize. diff --git a/internal/hookevent/event.go b/internal/hookevent/event.go index 92db65b..77be0d6 100644 --- a/internal/hookevent/event.go +++ b/internal/hookevent/event.go @@ -75,3 +75,19 @@ func Find(event map[string]json.RawMessage) (field, text string, ok bool) { } return "", "", false } + +// Withheld replaces a tool output when detection could not run and the caller +// chose to fail closed. It says what happened and nothing about the content, +// because the content is exactly what could not be checked. +const Withheld = "[vedei: output withheld — detection could not run, and --fail-closed is set]" + +// Withhold returns the event with its tool output replaced by Withheld. +// +// It is the opposite of the default. Fail-open (ADR-005) passes unscanned text +// through, because breaking a session is worse than missing one redaction — +// true for most users, and the wrong trade where an unscanned CPF reaching a +// model is a reportable incident. An event with no recognizable output is +// returned unchanged: there is nothing in it this function knows how to hold. +func Withhold(input []byte) []byte { + return Rewrite(input, func(string) (string, bool) { return Withheld, true }) +} diff --git a/internal/hookevent/event_test.go b/internal/hookevent/event_test.go index 30f1396..0d3eb45 100644 --- a/internal/hookevent/event_test.go +++ b/internal/hookevent/event_test.go @@ -108,3 +108,28 @@ func TestRewrite_EmptyInput(t *testing.T) { t.Errorf("Rewrite(nil) = %q", got) } } + +// Withhold is the fail-closed answer: the output is replaced by a notice that +// says nothing about it, and the rest of the event — which the agent needs to +// route the result — survives untouched. +func TestWithhold(t *testing.T) { + in := []byte(`{"tool_name":"Read","tool_response":"cpf 529.982.247-25"}`) + out := hookevent.Withhold(in) + + var ev map[string]any + if err := json.Unmarshal(out, &ev); err != nil { + t.Fatalf("the withheld event is not JSON: %v", err) + } + if ev["tool_response"] != hookevent.Withheld { + t.Errorf("tool_response = %v, want the withheld notice", ev["tool_response"]) + } + if ev["tool_name"] != "Read" { + t.Errorf("tool_name was lost: %v", ev["tool_name"]) + } + + // An event with no recognizable output has nothing to withhold. + other := []byte(`{"other":"x"}`) + if got := hookevent.Withhold(other); string(got) != string(other) { + t.Errorf("an event without output was rewritten: %s", got) + } +} diff --git a/transcript/stats.go b/transcript/stats.go new file mode 100644 index 0000000..88d8e22 --- /dev/null +++ b/transcript/stats.go @@ -0,0 +1,94 @@ +package transcript + +import "sort" + +// Stats is a transcript audit reduced to counts, safe to send off the machine. +// +// It exists for one study: run the audit on volunteers' machines and learn how +// often agent transcripts hold a credential or a Brazilian document — the +// number that says whether the problem vedei addresses is real. That only works +// if a volunteer can hand over the result without handing over the data. +// +// So the type holds no field that could carry a value, a path or a +// fingerprint. Fingerprints are left out on purpose, not just values: a CPF's +// fingerprint is a hash of eleven digits, and eleven digits are few enough to +// enumerate, so the hash would give the CPF back. +type Stats struct { + Transcripts int `json:"transcripts"` + WithFinding int `json:"transcripts_with_finding"` + WithCredential int `json:"transcripts_with_credential"` + WithPersonalData int `json:"transcripts_with_personal_data"` + ByType map[string]TypeStats `json:"by_type"` +} + +// TypeStats counts one finding type. +type TypeStats struct { + // Transcripts is how many transcripts hold at least one value of the type. + Transcripts int `json:"transcripts"` + // Values is how many distinct values of the type were seen, across all + // transcripts: one CPF in ten files counts once. + Values int `json:"values"` +} + +// engineSecrets names the credential engine; every other engine reports +// Brazilian personal data. +const engineSecrets = "secrets" + +// Summarize reduces sessions to counts. scanned is how many transcripts were +// read, including the ones that held nothing. +func Summarize(sessions []Session, scanned int) Stats { + st := Stats{Transcripts: scanned, ByType: map[string]TypeStats{}} + // Distinct values are counted by fingerprint, which never leaves this + // function. + seen := map[string]map[string]bool{} + + for _, s := range sessions { + if len(s.Findings) == 0 { + continue + } + st.WithFinding++ + + credential, personal := false, false + typesHere := map[string]bool{} + for _, f := range s.Findings { + if f.Engine == engineSecrets { + credential = true + } else { + personal = true + } + typesHere[f.Type] = true + if seen[f.Type] == nil { + seen[f.Type] = map[string]bool{} + } + seen[f.Type][f.Fingerprint] = true + } + if credential { + st.WithCredential++ + } + if personal { + st.WithPersonalData++ + } + for t := range typesHere { + ts := st.ByType[t] + ts.Transcripts++ + st.ByType[t] = ts + } + } + + for t, fps := range seen { + ts := st.ByType[t] + ts.Values = len(fps) + st.ByType[t] = ts + } + return st +} + +// Types returns the finding types in the summary, sorted. +func (s Stats) Types() []string { + out := make([]string, 0, len(s.ByType)) + for t := range s.ByType { + out = append(out, t) + } + sort.Strings(out) + return out +} diff --git a/transcript/stats_test.go b/transcript/stats_test.go new file mode 100644 index 0000000..54c97a6 --- /dev/null +++ b/transcript/stats_test.go @@ -0,0 +1,92 @@ +package transcript_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/mkmuniz/vedei/detect" + "github.com/mkmuniz/vedei/transcript" +) + +func finding(engine, typ, fp, raw string) detect.Finding { + return detect.Finding{ + Engine: engine, Type: typ, Fingerprint: fp, Raw: raw, + Redacted: "***" + raw[len(raw)-2:], + Locations: []detect.Location{{Path: "/home/dev/.claude/projects/acme/session.jsonl", Line: 3}}, + } +} + +func sampleSessions() []transcript.Session { + cpf := finding("br", "cpf", "fp-cpf-1", "529.982.247-25") + key := finding("secrets", "generic-api-key", "fp-key-1", "8f3kq9zLw2Xc7vB1nM4p") + return []transcript.Session{ + {Path: "/home/dev/.claude/projects/acme/a.jsonl", Findings: []detect.Finding{cpf, key}}, + {Path: "/home/dev/.claude/projects/acme/b.jsonl", Findings: []detect.Finding{cpf}}, + {Path: "/home/dev/.claude/projects/acme/c.jsonl"}, + } +} + +func TestSummarize(t *testing.T) { + st := transcript.Summarize(sampleSessions(), 10) + + if st.Transcripts != 10 { + t.Errorf("Transcripts = %d, want 10 (every file read, not only the dirty ones)", st.Transcripts) + } + if st.WithFinding != 2 { + t.Errorf("WithFinding = %d, want 2", st.WithFinding) + } + if st.WithCredential != 1 || st.WithPersonalData != 2 { + t.Errorf("WithCredential = %d, WithPersonalData = %d, want 1 and 2", + st.WithCredential, st.WithPersonalData) + } + + // One CPF in two transcripts: two transcripts, one distinct value. + if got := st.ByType["cpf"]; got.Transcripts != 2 || got.Values != 1 { + t.Errorf("cpf = %+v, want 2 transcripts and 1 value", got) + } + if got := st.ByType["generic-api-key"]; got.Transcripts != 1 || got.Values != 1 { + t.Errorf("generic-api-key = %+v, want 1 and 1", got) + } +} + +// The property the whole type exists for: the summary can leave the machine. +// No value, no redacted form, no path, no file name, and no fingerprint — +// a CPF's fingerprint is a hash of eleven digits, which enumeration reverses. +func TestStats_CarriesNothingIdentifying(t *testing.T) { + sessions := sampleSessions() + out, err := json.Marshal(transcript.Summarize(sessions, 10)) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + text := string(out) + + for _, s := range sessions { + for _, f := range s.Findings { + for name, leak := range map[string]string{ + "value": f.Raw, "redacted value": f.Redacted, + "fingerprint": f.Fingerprint, "location path": f.Locations[0].Path, + } { + if strings.Contains(text, leak) { + t.Errorf("the summary contains the %s %q:\n%s", name, leak, text) + } + } + } + if strings.Contains(text, s.Path) || strings.Contains(text, "acme") || strings.Contains(text, "/home/") { + t.Errorf("the summary contains a path:\n%s", text) + } + } +} + +func TestSummarize_Empty(t *testing.T) { + st := transcript.Summarize(nil, 0) + if st.WithFinding != 0 || len(st.ByType) != 0 { + t.Errorf("empty summary = %+v", st) + } + // An empty map, not null, so a script aggregating many machines does not + // have to special-case a clean one. + out, _ := json.Marshal(st) + if !strings.Contains(string(out), `"by_type":{}`) { + t.Errorf("by_type is not an empty object: %s", out) + } +}