diff --git a/.vedeiignore b/.vedeiignore index 7332e6b..e1bebc6 100644 --- a/.vedeiignore +++ b/.vedeiignore @@ -72,6 +72,22 @@ a83871a7916f6b9c44907082aa289cfc # +***********21 - a second phone in the same d554cea51a6155fd70c74664b448aff2 # +***********94 - a third 9ee2bb2c7f2a591ab50bb3070faa1a6d # a UUID v4 Pix key, in the overlap test +# --- M5 corpus and test fixtures --- +# The labeled corpus and the mask/AWS tests carry example values on purpose. +# Fingerprinted, not path-ignored, so the principle holds: every value is named, +# and a real secret committed into these files would still be reported. +d0dead6eceff137cafd15c0029bf1214 # AKIA...LE example key in corpus +a8cadd7b8a5caff89d90408d8a6c31bc # AKIA...TQ assembled test key +e58b0b5d6b990a005bb44e5d490a851e # ASIA...TQ assembled test key +2ffebaa6e81961e4cef8beeb61c5f8d1 # generated CNH vector +648782aa9ca9e35c33bddea812bb255f # generated CNH vector +2040ad664a71a504a6ff863b0b78a4f0 # generated PIS vector, space-separated +4d4eb30077a44f6a1e088b2c4cff6f1d # generated PIS vector +62949e15935851bd24d617292e7230f8 # generated titulo vector, space-separated +f60e47e7fe914c3e5d10a9b2a99d8b96 # generated titulo vector +ec2a7642f9821664294f09e54573ca66 # generated CNS vector +83f7388d806b21f4c0c4ea6e6a651937 # Pix email key fixture + # Nothing is ignored by path, on purpose. Silencing *_test.go wholesale would # also silence a credential genuinely committed into a test, which is a place # credentials genuinely end up. Every entry above is one value, named. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b718cf3..6c4fb10 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -122,7 +122,7 @@ const ( │ (ex.: node_modules, testdata, binario) │ [2] extract regex por detector, tolerante a mascara - │ 123.456.789-09 / 12345678909 / 123.456.78909 + │ 123.456.789-09 / 12345678909 / 123 456 789 09 │ [3] validate digito verificador | Luhn | ISPB | estrutura │ -> invalido morre aqui, nao vira achado @@ -137,7 +137,7 @@ const ( Etapa 3 é o que separa vedei de um scanner de regex. Um `grep` de CPF acha `000.000.000-00`; vedei não, porque o dígito não fecha. -**Divergência conhecida na etapa 2.** "Tolerante a máscara" não é uniforme entre detectores. CNS, título e PAN aceitam espaço como separador; CPF, CNPJ e PIS aceitam só `.`, `-` e `/`. Então `529 982 247 25` **não** é detectado como CPF hoje. Isso foi descoberto escrevendo um teste que assumia o contrário, e é uma decisão de calibração, não um esquecimento: aceitar espaço no CPF casa com qualquer coluna de tabela de três dígitos, e medir esse custo é justamente o que o corpus do M5 existe para fazer. Até lá o comportamento está documentado aqui em vez de prometido no código. +**Separadores de máscara, padronizados (M5).** Todo detector aceita o mesmo conjunto entre grupos de dígitos: a pontuação própria do documento, um espaço, um tab, ou nada — **nunca** uma quebra de linha. Antes disso havia duas divergências: CNS e título usavam `[\s.]`, e `\s` casa `\n`, então casavam um valor partido em duas linhas (o mesmo bug que o padrão de cartão já evitava); e CPF, CNPJ e PIS recusavam espaço, então `529 982 247 25` não era detectado. O medo de aceitar espaço era casar coluna de tabela de três dígitos — medido contra o corpus, o dígito verificador mantém essas colunas quietas, então a precision ficou em 1.000. A medição está em `corpus/cases/cpf.txt`. --- diff --git a/Makefile b/Makefile index 32299d0..9c6298a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ HOOKBIN := vedei-hook VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X main.version=$(VERSION) -.PHONY: build test lint bench cover clean install-tools all fuzz security vuln sast latency +.PHONY: build test lint bench cover clean install-tools all fuzz security vuln sast latency corpus all: lint test build @@ -50,6 +50,12 @@ FUZZ_TRANSCRIPT := FuzzScrubRecord_PreservesMeaning FuzzScrubRecord_NumbersSurvi FuzzScrubRecord_RedactionKeepsItValid FuzzReadableText FUZZ_SCAN := FuzzParseIgnore FuzzIgnoreStack FuzzIsBinary +# The labeled precision/recall corpus. Separate from the unit tests because it +# is the number that answers "is this getting noisier", and worth running on its +# own when calibrating. +corpus: + go test -v -run TestCorpus ./corpus/ + fuzz: @for t in $(FUZZ_DETECT); do \ echo "== $$t"; go test -run '^$$' -fuzz "^$$t$$" -fuzztime=$(FUZZTIME) ./detect/br/ || exit 1; \ diff --git a/README.md b/README.md index ba893b2..e78446e 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Full records in [`docs/adr/`](docs/adr/). | M2 | Secret engine via betterleaks | ✅ | | M3 | **MVP — AI agent surface** | ✅ | | M4 | CLI, CI/CD, SARIF, transcript scrub | ✅ | -| M5 | Multilingual false-positive corpus + pt-BR calibration | ⬜ | +| M5 | Multilingual false-positive corpus + pt-BR calibration | 🚧 | | M6 | Runtime SDK | ⬜ | | M7 | AI layer | ⬜ | | M8 | Observability collector | ⬜ | @@ -149,16 +149,16 @@ Features, implementation steps, test strategy and exit criteria per stage: [`MIL ``` detect/br/ # the core: pure validators, zero dependencies detect/engine.go # the Engine interface — insulation from betterleaks' v2 API -engine/secrets/ # betterleaks wrapped behind that interface +detect/aws/ # AWS access key ids — the one secret rule vedei owns +engine/secrets/ # betterleaks wrapped behind that interface, plus detect/aws redact/ # format-preserving redaction -report/ # JSON, JSONL, SARIF -scan/ # directory walker, gitignore matcher, git history report/ # table, JSON, JSONL, SARIF +scan/ # directory walker, gitignore matcher, git history stream/ # stdin -> stdout, the MVP path daemon/ # Unix socket server and client — the 8.4 ms path transcript/ # agent session log reader hooks/ # agent hooks, git hooks, launchd and systemd units -corpus/ # multilingual false-positive corpus +corpus/ # labeled precision/recall harness (make corpus) ``` ## The name diff --git a/README.pt-BR.md b/README.pt-BR.md index 046a490..76a2879 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -132,7 +132,7 @@ Registros completos em [`docs/adr/`](docs/adr/). | M2 | Motor de segredos via betterleaks | ✅ | | M3 | **MVP — superfície de IA** | ✅ | | M4 | CLI, CI/CD, SARIF, scrub de transcript | ✅ | -| M5 | Corpus multilíngue + calibração pt-BR | ⬜ | +| M5 | Corpus multilíngue + calibração pt-BR | 🚧 | | M6 | SDK de runtime | ⬜ | | M7 | Camada de IA | ⬜ | | M8 | Coletor de observabilidade | ⬜ | @@ -157,7 +157,7 @@ stream/ # stdin -> stdout, o caminho do MVP daemon/ # servidor e cliente do socket Unix — o caminho de 8,4 ms transcript/ # leitor de log de sessao de agente hooks/ # hooks de agente, hooks de git, units de launchd e systemd -corpus/ # corpus multilingue de falso positivo +corpus/ # harness rotulado de precision/recall (make corpus) ``` ## O nome diff --git a/SECURITY.md b/SECURITY.md index 254055f..48339bf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -71,11 +71,15 @@ Run the same checks locally with `make security` and `make fuzz`. check digits, never identity, and never will (ADR-002). A generated CPF in a test fixture is indistinguishable from a real one by arithmetic alone. -**AWS access key IDs are not detected.** Neither the canonical example nor a -random one: the upstream corpus has no rule for `AKIA…` alone, which is a -defensible position — an access key ID is an identifier, and the secret access -key is the credential. For keeping data out of an agent's context it is still a -gap, and a real one. Stripe, GitHub PAT and Slack tokens are detected. M5. +**AWS access key IDs are detected; the secret access key is not.** vedei owns one +rule the upstream corpus lacks: an access key id (`AKIA…`, `ASIA…`) is matched +structurally — a credential-bearing prefix and a 16-character base32 body. It is +reported High confidence and Unknown validity, because the shape proves it is +well-formed, not that it still works. The 40-character *secret* access key has no +structure to key on and is left to context-based rules, which is a known gap. The +account-id checksum an access key id also carries is not yet verified; doing so +would raise confidence and surface the account. Stripe, GitHub PAT and Slack +tokens are detected by the upstream corpus. **The daemon is a long-lived process holding a Unix socket.** If you run `vedei daemon`, note what it is and is not: diff --git a/corpus/README.md b/corpus/README.md new file mode 100644 index 0000000..7f008ae --- /dev/null +++ b/corpus/README.md @@ -0,0 +1,48 @@ +# Corpus + +Labeled examples that measure what vedei catches and what it cries wolf over. + +A scanner's worth is not that it compiles. It is precision and recall on real +input, and for Brazilian data the hard one is precision: the upstream secret +rules score candidates against an English dictionary, so a Portuguese word like +`senhapadrao` reads as random and gets reported. That cost is invisible until +you measure it against labeled data. This directory is that data. + +## Format + +One directive per line in `cases/*.txt`: + +``` +expect cpf 529.982.247-25 # must be found, as a cpf +reject 529.982.247-00 # must not be reported — wrong check digit +``` + +- **`expect `** — a true positive. The engine must report a + finding of `` over ``. A miss is a recall failure: a leak walked + past. +- **`reject # reason`** — a false positive. The engine must report + nothing. A hit is a precision failure: on the agent hook, redacting something + the model needed. The reason is required — a reject nobody can explain is a + reject nobody can safely remove. + +Everything after `#` is a comment. + +## Running + +```bash +make corpus # or: go test ./corpus/ +``` + +The gate is strict: every `expect` must be found and every `reject` must stay +quiet. That is right for hand-labeled data, where each line is a deliberate +claim — a threshold is for scraped corpora where some error is expected. The +test still prints precision and recall, because the numbers are the point and a +reviewer should see them move. + +## Adding a case + +When a scan produces a false positive, add the value here as a `reject` with the +reason, then fix the detector until this test passes again. When a detector +gains a document type, add `expect` cases for it. The file +`cases/false-positives.txt` is the running record of what scanning the real +world actually produced — every entry in it was once a wrong finding. diff --git a/corpus/cases/cnpj.txt b/corpus/cases/cnpj.txt new file mode 100644 index 0000000..546e000 --- /dev/null +++ b/corpus/cases/cnpj.txt @@ -0,0 +1,9 @@ +# CNPJ — numeric and the alphanumeric format in force from 2026-07-06. + +expect cnpj 11.222.333/0001-81 +expect cnpj 11222333000181 + +expect cnpj 00.000.000/0001-91 # the Uniao's CNPJ: body all zeros, but check digits close + +reject 11.222.333/0001-82 # check digit wrong +reject 12.345.678/0001-00 # check digits do not close diff --git a/corpus/cases/cpf.txt b/corpus/cases/cpf.txt new file mode 100644 index 0000000..3b47726 --- /dev/null +++ b/corpus/cases/cpf.txt @@ -0,0 +1,23 @@ +# CPF — true positives and the near-misses that must stay quiet. +# All valid values have correct check digits by construction; a fixture with a +# wrong one would only prove the generator agrees with the validator. + +expect cpf 529.982.247-25 +expect cpf 52998224725 +expect cpf 111.444.777-35 + +# The check digit is the whole point: these are one digit off and must not be +# reported, or vedei is just a regex and claims otherwise. +reject 529.982.247-00 # last digit wrong, fails mod 11 +reject 111.444.777-30 # check digit does not close +reject 000.000.000-00 # repeated sequence, invalid by rule even though mod 11 passes + +# Shapes that look CPF-ish but are not. +reject 1234567890 # ten digits, too short +reject app version 1.2.3 # dotted, but not eleven digits + +# Space as a separator, standardized across every document type. The feared +# cost is a three-digit table column; the check digit is what keeps it quiet. +expect cpf 529 982 247 25 +reject linha 100 200 300 40 # three-digit columns, check digit does not close +reject valores 123 456 789 10 # same shape, fails mod 11 diff --git a/corpus/cases/false-positives.txt b/corpus/cases/false-positives.txt new file mode 100644 index 0000000..97e17f9 --- /dev/null +++ b/corpus/cases/false-positives.txt @@ -0,0 +1,22 @@ +# The precision corpus proper: things that pass a checksum or match a shape by +# accident. Every one of these was a real false positive, or the exact class of +# one, that an earlier version reported. This file is the record of what running +# vedei against the world actually produced. + +# A Go module pseudo-version. The timestamp YYYYMMDDHHMMSS passes Luhn about one +# time in ten; this one did, and was reported as a credit card. +reject v0.0.0-20240915155400-7ee5256398cf # timestamp read as card-pan +reject migration_20240915155400_add.sql # same timestamp, in a filename + +# A UUID. A v4 is a valid Pix EVP key by shape and essentially never one in +# fact; the sixteen digits straddling a hyphen also passed Luhn once. +reject session 11111111-2222-3333-4444-555555555555 # UUID read as card-pan + +# A Brazilian phone number in E.164 once read as a card-pan: it begins with 55, +# inside the Mastercard issuer range, and its digits passed Luhn. That reading is +# gone. It is still a Pix key by type, which is correct and lives in pix.txt — the +# bare number here must simply never be a card. +reject boleto 4532 1111 1111 111 # 15 digits, grouped as a card but fails Luhn + +# A base64 module hash in go.sum: entropy by design, not a secret. +reject h1:o1uxfymjZ7jZ4MsgCErcwWGtVKSiNAXtS59Lhs6uI/g= # go.sum hash diff --git a/corpus/cases/other-documents.txt b/corpus/cases/other-documents.txt new file mode 100644 index 0000000..ea0c144 --- /dev/null +++ b/corpus/cases/other-documents.txt @@ -0,0 +1,16 @@ +# CNH, PIS, título and CNS. Vectors generated by each published algorithm and +# confirmed against the live extractor, never copied from a third party. + +expect cnh 73022314994 +expect cnh 20909550300 +expect pis 43912547832 +expect pis 22322316163 +expect titulo-eleitor 306636791040 +expect titulo-eleitor 862441572097 +expect cns 100000000000007 + +reject 11111111111 # a repeated digit is not a valid document of any kind + +# Space-separated forms, now caught uniformly. +expect pis 439 12547 83 2 +expect titulo-eleitor 3066 3679 1040 diff --git a/corpus/cases/pix.txt b/corpus/cases/pix.txt new file mode 100644 index 0000000..9977d77 --- /dev/null +++ b/corpus/cases/pix.txt @@ -0,0 +1,9 @@ +# Pix keys. A phone or email used as a Pix key is sensitive data, not noise, so +# these are true positives. The phone is the one that used to be read as a card. + +expect pix-key recebedor +5511980198775 +expect pix-key chave pix fulano@exemplo.com.br + +# A random UUID with no Pix context is not a key: a v4 is a valid EVP key by +# shape and almost never one in fact. +reject trace 11111111-2222-3333-4444-555555555555 # UUID, no Pix context diff --git a/corpus/cases/secrets.txt b/corpus/cases/secrets.txt new file mode 100644 index 0000000..27afc56 --- /dev/null +++ b/corpus/cases/secrets.txt @@ -0,0 +1,12 @@ +# Credentials. The AWS access-key cases are the gap this milestone closes: the +# upstream corpus reports none of them, measured, not even with an assignment +# context. Values are assembled so the file carries no live-shaped secret. + +expect aws-access-key-id AKIA4KJ7WQZR2LMNXBTQ +expect aws-access-key-id ASIA4KJ7WQZR2LMNXBTQ +expect aws-access-key-id export AWS_ACCESS_KEY_ID=AKIA4KJ7WQZR2LMNXBTQ + +# base32 has no 0, 1, 8 or 9, and the prefix must be a credential-bearing one. +reject ABCD4KJ7WQZR2LMNXBTQ # 20 chars, base32 body, but not an AWS prefix +reject AKIA1KJ7WQZR2LMNXBTQ # AKIA prefix but a 1 is not base32 +reject the quick brown fox jumped over the lazy dog # ordinary prose diff --git a/corpus/corpus.go b/corpus/corpus.go new file mode 100644 index 0000000..ddf45cd --- /dev/null +++ b/corpus/corpus.go @@ -0,0 +1,197 @@ +// Package corpus loads the labeled examples that measure detection quality. +// +// It is the measurement tool the milestones call for: a scanner's worth is not +// "does it compile" but "what does it catch and what does it cry wolf over", +// and in Portuguese the second question is the hard one. The upstream secret +// rules score candidates against an English dictionary, so a Portuguese word +// like "senhapadrao" reads as random and is reported. That cost is only visible +// against labeled data, which is what this package is. +// +// A case file is plain text, one directive per line: +// +// expect cpf 529.982.247-25 # a true positive: must be found, as a cpf +// reject 529.982.247-00 # a false positive: must not be reported +// reject commit 20240915... # a timestamp, not a document +// +// Everything after '#' is a comment, and a comment is not decoration: a reject +// case without a reason is a line nobody can re-judge later. +package corpus + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +// Kind is what a case asserts about its text. +type Kind int + +const ( + // Expect means the text must be reported, with the named type. It measures + // recall: a miss here is a leak the scanner walked past. + Expect Kind = iota + // Reject means the text must not be reported at all. It measures precision: + // a hit here is the scanner crying wolf, which on the agent hook means + // redacting something the model needed. + Reject +) + +// Case is one labeled example. +type Case struct { + Kind Kind + // Type is the finding type an Expect case requires, such as "cpf" or + // "aws-access-key-id". Empty for a Reject case. + Type string + // Text is the line fed to the engine. + Text string + // Reason is the comment that followed the directive. Required on a Reject, + // because a false-positive fixture nobody can explain is one nobody can + // safely remove. + Reason string + // Source locates the case for an error message. + Source string + Line int +} + +// Parse reads cases from one file's contents. +// +// A malformed line is an error, not a skip. A corpus that silently drops a case +// it could not parse is a corpus that reports a precision it did not measure. +func Parse(source string, r io.Reader) ([]Case, error) { + var cases []Case + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 4<<10), 1<<20) + + for line := 1; sc.Scan(); line++ { + raw := sc.Text() + + text, reason := splitComment(raw) + text = strings.TrimSpace(text) + if text == "" { + // A blank line, or a line that was only a comment. Either is fine. + continue + } + + c, err := parseDirective(text, reason) + if err != nil { + return nil, fmt.Errorf("%s:%d: %w", source, line, err) + } + c.Source, c.Line = source, line + cases = append(cases, c) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("reading %s: %w", source, err) + } + return cases, nil +} + +// parseDirective reads one non-comment line into a Case. +func parseDirective(text, reason string) (Case, error) { + fields := strings.Fields(text) + switch fields[0] { + case "expect": + // expect + if len(fields) < 3 { + return Case{}, fmt.Errorf("expect needs a type and a value: %q", text) + } + rest := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(text), "expect")) + typ := fields[1] + value := strings.TrimSpace(strings.TrimPrefix(rest, typ)) + if value == "" { + return Case{}, fmt.Errorf("expect %s has no value", typ) + } + return Case{Kind: Expect, Type: typ, Text: value}, nil + + case "reject": + // reject + value := strings.TrimSpace(strings.TrimPrefix(text, "reject")) + if value == "" { + return Case{}, fmt.Errorf("reject has no value") + } + if strings.TrimSpace(reason) == "" { + return Case{}, fmt.Errorf("reject %q has no reason; add one after '#'", value) + } + return Case{Kind: Reject, Text: value, Reason: strings.TrimSpace(reason)}, nil + + default: + return Case{}, fmt.Errorf("unknown directive %q, want expect or reject", fields[0]) + } +} + +// splitComment separates a line from its trailing comment. A '#' inside the +// value would be unusual for these document types, so the first one wins. +func splitComment(line string) (text, comment string) { + if i := strings.IndexByte(line, '#'); i >= 0 { + return line[:i], line[i+1:] + } + return line, "" +} + +// Load reads every .txt file under dir, in a stable order so a failure names +// the same case across runs. +func Load(dir string) ([]Case, error) { + entries, err := filepath.Glob(filepath.Join(dir, "*.txt")) + if err != nil { + return nil, fmt.Errorf("corpus: listing %s: %w", dir, err) + } + sort.Strings(entries) + + var all []Case + for _, path := range entries { + f, err := os.Open(path) //#nosec G304 -- a corpus fixture path from this repo's own tree + if err != nil { + return nil, fmt.Errorf("corpus: %w", err) + } + cases, err := Parse(filepath.Base(path), f) + _ = f.Close() + if err != nil { + return nil, err + } + all = append(all, cases...) + } + return all, nil +} + +// Metrics is the precision and recall a run measured. +type Metrics struct { + // TruePositives is Expect cases reported with the right type. + TruePositives int + // FalseNegatives is Expect cases the scanner missed. + FalseNegatives int + // FalsePositives is Reject cases the scanner reported anyway. + FalsePositives int + // TrueNegatives is Reject cases the scanner correctly left alone. + TrueNegatives int +} + +// Precision is of what was reported, how much should have been. The number this +// corpus exists to defend. +func (m Metrics) Precision() float64 { + reported := m.TruePositives + m.FalsePositives + if reported == 0 { + return 1 + } + return float64(m.TruePositives) / float64(reported) +} + +// Recall is of what should have been reported, how much was. The counterweight: +// precision is trivial to maximize by reporting nothing, and recall is what +// stops that. +func (m Metrics) Recall() float64 { + actual := m.TruePositives + m.FalseNegatives + if actual == 0 { + return 1 + } + return float64(m.TruePositives) / float64(actual) +} + +// String renders the metrics for a test log. +func (m Metrics) String() string { + return fmt.Sprintf("precision %.3f (%d reported, %d wrong), recall %.3f (%d/%d found)", + m.Precision(), m.TruePositives+m.FalsePositives, m.FalsePositives, + m.Recall(), m.TruePositives, m.TruePositives+m.FalseNegatives) +} diff --git a/corpus/corpus_test.go b/corpus/corpus_test.go new file mode 100644 index 0000000..51033fe --- /dev/null +++ b/corpus/corpus_test.go @@ -0,0 +1,145 @@ +package corpus_test + +import ( + "context" + "strings" + "testing" + + "github.com/mkmuniz/vedei/corpus" + "github.com/mkmuniz/vedei/detect" + "github.com/mkmuniz/vedei/engine" +) + +// casesDir is the labeled data, relative to this test. +const casesDir = "cases" + +// TestCorpus runs every labeled case through the full engine and fails on any +// violation, reporting precision and recall as it goes. +// +// The gate is strict — every Expect must be found, every Reject must stay quiet +// — rather than a threshold, because this corpus is hand-labeled. A threshold is +// for a noisy, scraped corpus where some error is expected; here every case is a +// claim someone wrote down on purpose, so any miss is a regression. The metrics +// are still printed, because the numbers are the point of the exercise and a +// reviewer should see them move. +func TestCorpus(t *testing.T) { + cases, err := corpus.Load(casesDir) + if err != nil { + t.Fatalf("loading the corpus: %v", err) + } + if len(cases) == 0 { + t.Fatal("the corpus is empty") + } + + eng := engine.All() + var m corpus.Metrics + + for _, c := range cases { + types := typesFound(t, eng, c.Text) + + switch c.Kind { + case corpus.Expect: + if types[c.Type] { + m.TruePositives++ + } else { + m.FalseNegatives++ + t.Errorf("%s:%d: expected %s in %q, found %v", + c.Source, c.Line, c.Type, c.Text, keys(types)) + } + case corpus.Reject: + if len(types) == 0 { + m.TrueNegatives++ + } else { + m.FalsePositives++ + t.Errorf("%s:%d: %q should be quiet (%s), but was reported as %v", + c.Source, c.Line, c.Text, c.Reason, keys(types)) + } + } + } + + t.Logf("%d cases: %s", len(cases), m) +} + +// typesFound returns the set of finding types the engine reports for text. +func typesFound(t *testing.T, eng detect.Engine, text string) map[string]bool { + t.Helper() + findings, err := eng.Scan(context.Background(), []byte(text), detect.Metadata{Source: "corpus"}) + if err != nil { + t.Fatalf("scanning %q: %v", text, err) + } + types := make(map[string]bool, len(findings)) + for _, f := range findings { + types[f.Type] = true + } + return types +} + +func keys(m map[string]bool) []string { + if len(m) == 0 { + return []string{"nothing"} + } + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +// TestCorpus_MetricsMath guards the two formulas, since a corpus that miscounts +// its own precision is worse than none. +func TestCorpus_MetricsMath(t *testing.T) { + m := corpus.Metrics{TruePositives: 8, FalsePositives: 2, FalseNegatives: 1, TrueNegatives: 5} + if got := m.Precision(); got != 0.8 { + t.Errorf("Precision() = %v, want 0.8", got) + } + if got := m.Recall(); !approx(got, 8.0/9.0) { + t.Errorf("Recall() = %v, want 0.889", got) + } + + // An empty count is defined as perfect, so a run with no cases of a kind + // does not read as a failure. + empty := corpus.Metrics{} + if empty.Precision() != 1 || empty.Recall() != 1 { + t.Errorf("empty metrics = %v", empty) + } +} + +func approx(a, b float64) bool { + d := a - b + return d < 1e-9 && d > -1e-9 +} + +func TestParse_RejectsMalformed(t *testing.T) { + for _, bad := range []string{ + "expect", // no type, no value + "expect cpf", // no value + "reject", // no value + "reject 123", // no reason + "wat 123 # x", // unknown directive + } { + if _, err := corpus.Parse("t", strings.NewReader(bad)); err == nil { + t.Errorf("Parse(%q) accepted a malformed line", bad) + } + } +} + +func TestParse_ReadsBothDirectives(t *testing.T) { + in := ` +# a comment +expect cpf 529.982.247-25 +reject 000.000.000-00 # repeated digits +` + cases, err := corpus.Parse("t", strings.NewReader(in)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if len(cases) != 2 { + t.Fatalf("got %d cases, want 2", len(cases)) + } + if cases[0].Kind != corpus.Expect || cases[0].Type != "cpf" || cases[0].Text != "529.982.247-25" { + t.Errorf("first case = %+v", cases[0]) + } + if cases[1].Kind != corpus.Reject || cases[1].Reason != "repeated digits" { + t.Errorf("second case = %+v", cases[1]) + } +} diff --git a/detect/aws/aws.go b/detect/aws/aws.go new file mode 100644 index 0000000..d369036 --- /dev/null +++ b/detect/aws/aws.go @@ -0,0 +1,92 @@ +// Package aws detects AWS credentials that the upstream secret corpus misses. +// +// It exists because of a measurement, not a hunch: betterleaks, which vedei +// relies on for secrets (ADR-001), reports nothing for an AWS access key id — +// not bare, not with an "AWS_ACCESS_KEY_ID=" context, not for ASIA either. An +// access key id in a log or a transcript is exactly the kind of thing vedei is +// meant to keep out of an agent's context, so this is a gap worth a rule of its +// own. It is the only detection logic vedei owns outside the Brazilian core. +// +// Only the two credential-bearing prefixes are matched. AKIA is a long-term +// access key and ASIA a temporary one from STS; both pair with a secret access +// key and are what leaks. The other documented prefixes — AIDA, AROA, AGPA and +// so on — are unique ids for IAM resources, not secrets, so matching them would +// be noise. +package aws + +import "regexp" + +// Match is one detected value and where it sits in the scanned text. +type Match struct { + Value string + Start int // byte offset of the first character + End int // byte offset just past the last +} + +// AccessKeyID is the finding type vedei reports these under. +const AccessKeyID = "aws-access-key-id" + +// accessKeyRe matches an access key id: a 4-character prefix, then 16 base32 +// characters. +// +// base32 is A–Z and 2–7 — it has no 0, 1, 8 or 9. That is a real structural +// check, not a shape: a 20-character run that contains one of those digits after +// the prefix is not an access key id, which rules out most lookalikes for free. +// The surrounding negative lookahead-substitutes keep a longer alphanumeric +// token (a hash, a longer id) from yielding a 20-character fragment. +var accessKeyRe = regexp.MustCompile(`(?:AKIA|ASIA)[A-Z2-7]{16}`) + +// ValidateAccessKeyID reports whether s is structurally a valid AWS access key +// id: a known credential prefix followed by sixteen base32 characters. +// +// Structural, never live. Like a CPF whose check digits close, this says the +// value is well-formed, not that the key still works — confirming that means +// calling AWS, which is M9. AWS access key ids also encode the account id with a +// checksum, which a future revision could verify to raise confidence and surface +// the account; it is deliberately not invented here. +func ValidateAccessKeyID(s string) bool { + if len(s) != 20 { + return false + } + loc := accessKeyRe.FindStringIndex(s) + return loc != nil && loc[0] == 0 && loc[1] == len(s) +} + +// Extract returns every access key id in text, in order. +// +// A candidate flanked by another base32 character on either side is skipped: it +// is a slice of a longer token, not a standalone key. RE2 has no lookaround, so +// the boundary is checked after the match rather than in the pattern. +func Extract(text string) []Match { + var out []Match + for _, loc := range accessKeyRe.FindAllStringIndex(text, -1) { + start, end := loc[0], loc[1] + if !boundedLeft(text, start) || !boundedRight(text, end) { + continue + } + out = append(out, Match{Value: text[start:end], Start: start, End: end}) + } + return out +} + +// boundedLeft reports whether the byte before start is not part of the token. +func boundedLeft(s string, start int) bool { + return start == 0 || !isTokenByte(s[start-1]) +} + +// boundedRight reports whether the byte at end is not part of the token. +func boundedRight(s string, end int) bool { + return end == len(s) || !isTokenByte(s[end]) +} + +// isTokenByte reports whether b could belong to the same identifier. A base32 +// character or a digit next door means the 20-character run is a fragment of +// something longer. +func isTokenByte(b byte) bool { + switch { + case b >= 'A' && b <= 'Z', b >= 'a' && b <= 'z', b >= '0' && b <= '9': + return true + default: + return false + } +} diff --git a/detect/aws/aws_test.go b/detect/aws/aws_test.go new file mode 100644 index 0000000..d9d6179 --- /dev/null +++ b/detect/aws/aws_test.go @@ -0,0 +1,83 @@ +package aws + +import "testing" + +// Keys are assembled from a prefix plus a 16-char base32 body, so the file +// carries no 20-char string that another scanner reads as a live key. +const ( + body = "4KJ7WQZR2LMNXBTQ" // 16 base32 chars + akiaKey = "AKIA" + body + asiaKey = "ASIA" + body +) + +func TestValidateAccessKeyID(t *testing.T) { + for _, s := range []string{akiaKey, asiaKey} { + if len(s) != 20 { + t.Fatalf("test vector %q is %d chars, not 20", s, len(s)) + } + if !ValidateAccessKeyID(s) { + t.Errorf("valid key rejected: %s", s) + } + } + + invalid := map[string]string{ + "no AWS prefix": "ABCD" + body, + "lowercase prefix": "akia" + body, + "too short": "AKIA" + body[:15], + "too long": "AKIA" + body + "A", + "a 1 is not base32": "AKIA1" + body[1:], + "an 8 is not base32": "AKIA8" + body[1:], + "lowercase body": "AKIA" + "4kj7wqzr2lmnxbtq", + "empty": "", + } + for name, s := range invalid { + if ValidateAccessKeyID(s) { + t.Errorf("%s: accepted %q (len %d)", name, s, len(s)) + } + } +} + +func TestExtract(t *testing.T) { + t.Run("in context", func(t *testing.T) { + text := "AWS_ACCESS_KEY_ID=" + akiaKey + "\n" + ms := Extract(text) + if len(ms) != 1 { + t.Fatalf("got %d matches, want 1", len(ms)) + } + if ms[0].Value != akiaKey { + t.Errorf("value = %q", ms[0].Value) + } + if text[ms[0].Start:ms[0].End] != akiaKey { + t.Errorf("offsets select %q, not the key", text[ms[0].Start:ms[0].End]) + } + }) + + t.Run("two on one line", func(t *testing.T) { + if ms := Extract(akiaKey + " " + asiaKey); len(ms) != 2 { + t.Fatalf("got %d, want 2", len(ms)) + } + }) + + t.Run("a longer token is not a key", func(t *testing.T) { + if ms := Extract("X" + akiaKey + "Y"); len(ms) != 0 { + t.Errorf("a fragment of a longer token was matched: %+v", ms) + } + if ms := Extract(akiaKey + "EXTRA"); len(ms) != 0 { + t.Errorf("a key followed by more base32 was matched: %+v", ms) + } + }) + + t.Run("nothing to find", func(t *testing.T) { + if ms := Extract("the quick brown fox AKIA jumped"); len(ms) != 0 { + t.Errorf("matched something in prose: %+v", ms) + } + }) +} + +func TestExtract_RejectsNonAWSPrefix(t *testing.T) { + for _, s := range []string{"ABCD" + body, "ZZZZ" + body} { + if ms := Extract(s); len(ms) != 0 { + t.Errorf("%q matched despite a non-AWS prefix", s) + } + } +} diff --git a/detect/br/extract.go b/detect/br/extract.go index 66d77f1..5e04530 100644 --- a/detect/br/extract.go +++ b/detect/br/extract.go @@ -39,17 +39,32 @@ type Match struct { // // Second, patterns accept the masks a document actually circulates in, not // only the canonical one. +// A document's digit groups are written apart by one separator: its own +// punctuation, or a space, or a tab, or nothing. The separator classes below +// all include " \t" and none include "\s". +// +// That is a correctness fix, not only a tidy-up. CNS and título used "[\s.]", +// and \s matches a line break, so both matched a value that straddled a +// newline — the same bug the card-number pattern was already written to avoid. +// Space and tab are allowed everywhere now; "\n" is allowed nowhere. Which +// means a CPF written "529 982 247 25" is finally caught, uniformly with the +// documents that already tolerated spaces. +// +// Whether admitting a space widens false positives is a precision question, and +// it is answered by the corpus rather than guessed: a three-column table of +// three-digit numbers is the feared case, and corpus/cases has it, kept quiet by +// the check digit that a table column does not satisfy. var patterns = []struct { kind Kind re *regexp.Regexp }{ {KindE2EID, regexp.MustCompile(`[Ee][0-9]{8}[0-9]{12}[0-9A-Za-z]{11}`)}, - {KindCNPJ, regexp.MustCompile(`[0-9A-Za-z]{2}\.?[0-9A-Za-z]{3}\.?[0-9A-Za-z]{3}/?[0-9A-Za-z]{4}-?[0-9]{2}`)}, - {KindCNS, regexp.MustCompile(`[1-2789][0-9]{2}[\s.]?[0-9]{4}[\s.]?[0-9]{4}[\s.]?[0-9]{4}`)}, - {KindTitulo, regexp.MustCompile(`[0-9]{4}[\s.]?[0-9]{4}[\s.]?[0-9]{4}`)}, + {KindCNPJ, regexp.MustCompile(`[0-9A-Za-z]{2}[ \t.]?[0-9A-Za-z]{3}[ \t.]?[0-9A-Za-z]{3}[ \t/]?[0-9A-Za-z]{4}[ \t-]?[0-9]{2}`)}, + {KindCNS, regexp.MustCompile(`[1-2789][0-9]{2}[ \t.]?[0-9]{4}[ \t.]?[0-9]{4}[ \t.]?[0-9]{4}`)}, + {KindTitulo, regexp.MustCompile(`[0-9]{4}[ \t.]?[0-9]{4}[ \t.]?[0-9]{4}`)}, {KindPAN, panRe}, - {KindCPF, regexp.MustCompile(`[0-9]{3}\.?[0-9]{3}\.?[0-9]{3}[-.]?[0-9]{2}`)}, - {KindPIS, regexp.MustCompile(`[0-9]{3}\.?[0-9]{5}\.?[0-9]{2}-?[0-9]`)}, + {KindCPF, regexp.MustCompile(`[0-9]{3}[ \t.]?[0-9]{3}[ \t.]?[0-9]{3}[ \t.-]?[0-9]{2}`)}, + {KindPIS, regexp.MustCompile(`[0-9]{3}[ \t.]?[0-9]{5}[ \t.]?[0-9]{2}[ \t-]?[0-9]`)}, {KindCNH, regexp.MustCompile(`[0-9]{11}`)}, } diff --git a/detect/br/extract_test.go b/detect/br/extract_test.go index feec768..4936282 100644 --- a/detect/br/extract_test.go +++ b/detect/br/extract_test.go @@ -484,3 +484,56 @@ func TestExtract_PlusAfterAValueIsStillABoundary(t *testing.T) { } } } + +// The separator standardization is also a bug fix: CNS and título used [\s.], +// and \s matches a line break, so a value split across two lines matched as one. +// The card-number pattern was already written to avoid exactly this; now every +// document type does. Space and tab separate groups; a newline never does. +func TestExtract_SeparatorNeverCrossesALine(t *testing.T) { + // A valid CNS and a valid título, each split by a newline where a separator + // would go. Neither may be reported. + for _, text := range []string{ + "100000000000007", // valid CNS, as a control it IS found when whole + } { + if len(Extract(text)) == 0 { + t.Fatalf("control failed: %q should be found when whole", text) + } + } + + for _, text := range []string{ + "100\n000000000007", // CNS split across a newline + "1000000\n00000007", + "3066\n3679\n1040", // título split across newlines + "529\n982\n247\n25", // CPF split across newlines + } { + for _, m := range Extract(text) { + // A match is only a bug if it spans the newline. A fragment on one + // line that happens to be valid on its own is fine. + if strings.Contains(m.Value, "\n") { + t.Errorf("%q matched across a newline: %q", text, m.Value) + } + } + } +} + +// Space and tab are accepted uniformly now, which they were not before: CPF, +// CNPJ and PIS rejected a space outright. +func TestExtract_SpaceAndTabAreUniformSeparators(t *testing.T) { + cases := map[string]Kind{ + "529 982 247 25": KindCPF, + "529\t982\t247\t25": KindCPF, + "439 12547 83 2": KindPIS, + "3066 3679 1040": KindTitulo, + } + for text, want := range cases { + found := false + for _, m := range Extract(text) { + if m.Kind == want { + found = true + } + } + if !found { + t.Errorf("%q was not found as %s", text, want) + } + } +} diff --git a/engine/secrets/engine.go b/engine/secrets/engine.go index f6aec49..41555c5 100644 --- a/engine/secrets/engine.go +++ b/engine/secrets/engine.go @@ -13,6 +13,7 @@ import ( "github.com/rs/zerolog" "github.com/mkmuniz/vedei/detect" + "github.com/mkmuniz/vedei/detect/aws" "github.com/mkmuniz/vedei/fingerprint" "github.com/mkmuniz/vedei/redact" ) @@ -106,17 +107,13 @@ func (e *Engine) Scan(ctx context.Context, content []byte, meta detect.Metadata) return nil, nil } - raw := e.detector.DetectString(string(content)) - if len(raw) == 0 { - return nil, nil - } + text := string(content) + raw := e.detector.DetectString(text) // One credential found in several places is one finding with several // locations, matching how the Brazilian engine reports. byPrint := make(map[string]*detect.Finding, len(raw)) order := make([]string, 0, len(raw)) - - text := string(content) for _, f := range raw { if f.Secret == "" { continue @@ -151,6 +148,12 @@ func (e *Engine) Scan(ctx context.Context, content []byte, meta detect.Metadata) order = append(order, fp) } + // The one rule vedei owns. betterleaks reports no AWS access key id at all — + // measured, not assumed — so this runs alongside it rather than relying on + // it. ADR-001 keeps betterleaks the only *imported* corpus; this is a + // deliberate, documented exception for a gap that corpus/cases proves. + addAWSFindings(text, meta, byPrint, &order) + out := make([]detect.Finding, 0, len(order)) for _, fp := range order { out = append(out, *byPrint[fp]) @@ -158,6 +161,36 @@ func (e *Engine) Scan(ctx context.Context, content []byte, meta detect.Metadata) return out, nil } +// addAWSFindings appends vedei's own AWS detections to the finding set, keyed +// the same way as betterleaks' so a value seen twice stays one finding. +// +// An access key id is structural, like a CPF whose check digits close: the shape +// and the base32 alphabet prove it is well-formed, not that it still works. It +// is reported High confidence because the prefix-plus-base32 combination is +// specific — a random 20-character token almost never lands on it — and Unknown +// validity, because confirming a live key means calling AWS, which is M9. +func addAWSFindings(text string, meta detect.Metadata, byPrint map[string]*detect.Finding, order *[]string) { + for _, m := range aws.Extract(text) { + fp := fingerprint.Of(aws.AccessKeyID, m.Value) + if existing, seen := byPrint[fp]; seen { + existing.Locations = append(existing.Locations, locationOf(text, m.Start, m.End, meta.Path)) + continue + } + byPrint[fp] = &detect.Finding{ + Type: aws.AccessKeyID, + Engine: "secrets", + Redacted: redact.Mask(m.Value), + Raw: m.Value, + Validity: detect.ValidityUnknown, + Confidence: detect.ConfidenceHigh, + Reason: "AWS access key id: known prefix and base32 body", + Fingerprint: fp, + Locations: []detect.Location{locationOf(text, m.Start, m.End, meta.Path)}, + } + *order = append(*order, fp) + } +} + // locationsOf returns every byte range in text holding secret. // // betterleaks reports a line and a column, not a byte offset, and the