From 9d1c734880b7d1043fb27f63fc6bb06d7236ae72 Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Thu, 27 Aug 2026 18:57:32 +0000 Subject: [PATCH 1/7] fix(redaction): strip C0/C1 bytes before shape matching NUL or ESC inside a key body splits the shape so RedactString misses it. Normalize those control bytes out first, then match. Cover NUL and ESC splits. Fixes Gitlawb/zero#969 --- internal/redaction/redaction.go | 63 +++++++++++++++++++++++++++- internal/redaction/redaction_test.go | 35 ++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index 24685eca9..0865e12c1 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -9,6 +9,7 @@ import ( "sort" "strings" "unicode" + "unicode/utf8" ) const ( @@ -170,9 +171,69 @@ func keyLooksSensitive(normalized string) bool { return false } +// stripControlBytes removes C0/C1 controls (Cc other than tab, LF, and CR) so +// shape matching sees a secret that was split by an embedded NUL, ESC, or C1 +// byte. Tab/LF/CR stay so log line structure is preserved. Lone Latin-1 C1 +// bytes (0x80–0x9F, invalid UTF-8) are stripped too; UTF-8 continuation bytes +// are not, because they are not controls. Must run before any pattern match. +func stripControlBytes(s string) string { + for i := 0; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + return stripControlBytesFrom(s, i) + } + i++ + continue + } + if c <= 0x9F { + // 0x80–0x9F at a rune boundary is a lone C1 byte, not UTF-8. + return stripControlBytesFrom(s, i) + } + r, size := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(r) { + return stripControlBytesFrom(s, i) + } + i += size + } + return s +} + +func stripControlBytesFrom(s string, start int) string { + var b strings.Builder + b.Grow(len(s)) + b.WriteString(s[:start]) + for i := start; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + i++ + continue + } + b.WriteByte(c) + i++ + continue + } + if c <= 0x9F { + i++ + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(r) { + i += size + continue + } + b.WriteString(s[i : i+size]) + i += size + } + return b.String() +} + func RedactString(value string, options Options) string { replacement := replacement(options) - redacted := value + // Strip C0/C1 first: a NUL, ESC, or C1 byte inside a key body splits the + // shape so the patterns miss it, and a later strip would rejoin the secret. + redacted := stripControlBytes(value) if len(options.ExtraSecretValues) > 0 { secrets := append([]string{}, options.ExtraSecretValues...) sort.SliceStable(secrets, func(i, j int) bool { diff --git a/internal/redaction/redaction_test.go b/internal/redaction/redaction_test.go index 3ae228c8d..d472650de 100644 --- a/internal/redaction/redaction_test.go +++ b/internal/redaction/redaction_test.go @@ -141,3 +141,38 @@ func containsCircular(v any) bool { } return false } + +func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { + // Unsplit passing is not coverage: a NUL/ESC/C1 in the body splits the + // shape so the patterns miss it unless controls are stripped first. + const prefix = "sk-ant-api03-" + const body = "abcdefghijklmnopqrstuvwxyz" + unsplit := prefix + body + if got := RedactString(unsplit, Options{}); strings.Contains(got, body) { + t.Fatalf("unsplit secret not redacted (test setup): %q", got) + } + + cases := []struct { + name string + split string + }{ + {name: "NUL", split: "\x00"}, + {name: "ESC", split: "\x1b"}, + {name: "C1", split: "\x9b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + input := prefix + tc.split + body + got := RedactString(input, Options{}) + if strings.Contains(got, body) { + t.Fatalf("secret split by %s leaked in %q", tc.name, got) + } + if strings.Contains(got, prefix) { + t.Fatalf("secret prefix split by %s leaked in %q", tc.name, got) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("expected %q after %s split, got %q", RedactedSecret, tc.name, got) + } + }) + } +} From 73911d438bd239c08564eb20ca467d3a41fcc042 Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Fri, 28 Aug 2026 02:16:19 +0000 Subject: [PATCH 2/7] test(redaction): cover UTF-8 C1 split and whitespace preservation Add a valid UTF-8 U+009B control split case alongside the lone invalid 0x9b byte, and assert tab/LF/CR plus non-control UTF-8 stay unchanged. --- internal/redaction/redaction_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/redaction/redaction_test.go b/internal/redaction/redaction_test.go index d472650de..a80c7e1cd 100644 --- a/internal/redaction/redaction_test.go +++ b/internal/redaction/redaction_test.go @@ -159,6 +159,7 @@ func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { {name: "NUL", split: "\x00"}, {name: "ESC", split: "\x1b"}, {name: "C1", split: "\x9b"}, + {name: "UTF-8 C1", split: string(rune(0x9B))}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -176,3 +177,10 @@ func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { }) } } + +func TestRedactStringPreservesAllowedWhitespaceAndUTF8(t *testing.T) { + input := "safe\tline\nnext\rfinal café" + if got := RedactString(input, Options{}); got != input { + t.Fatalf("unexpected normalization: %q", got) + } +} From eeb2e07bf2671ca1e291088fb31db1d81648911b Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Fri, 28 Aug 2026 18:24:38 +0000 Subject: [PATCH 3/7] fix(redaction): match split secrets without joining tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matching on a control-stripped copy made \b fail when a word character preceded the deleted control, so id42\x00sk-ant-… leaked. Allow C0/C1 gaps between shape characters on the original string instead, and do not return a stripped copy when no secret matched. --- internal/redaction/redaction.go | 69 ++++++++++++++++++++-------- internal/redaction/redaction_test.go | 43 ++++++++++++++++- 2 files changed, 91 insertions(+), 21 deletions(-) diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index 0865e12c1..b5a401095 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -69,12 +69,36 @@ var sensitiveKeys = map[string]struct{}{ "zero_api_key": {}, } +// ctrlGap matches C0/C1 bytes (Cc other than tab/LF/CR, plus lone Latin-1 C1) +// between characters of a secret shape. Matching stays on the original string: +// a deleted control is never a join, so \b still treats wordchar+control as a +// boundary and tokens that were never adjacent stay that way. Tab/LF/CR are +// excluded so log line structure is unchanged. \x{FFFD} is how Go's regexp +// engine reports a lone invalid UTF-8 C1 byte such as 0x9B. +const ctrlGap = `[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f\x{FFFD}]*` + +// ctrlLit quotes s as a regexp literal with ctrlGap after every rune, so a +// NUL/ESC/C1 may split the literal without breaking the match. +func ctrlLit(s string) string { + var b strings.Builder + b.Grow(len(s) * (1 + len(ctrlGap))) + for _, r := range s { + b.WriteString(regexp.QuoteMeta(string(r))) + b.WriteString(ctrlGap) + } + return b.String() +} + +func secretBody(class, quant string) string { + return `(?:` + class + ctrlGap + `)` + quant +} + // openaiKeyPattern mirrors secrets.Scan's broad sk- body. Known OpenAI // prefixes (sk-proj-/sk-svcacct-/sk-admin-) are always redacted; other sk- // digit-free matches with an interior hyphen are left alone (kebab-case false // positives), while digit-free legacy sk- credentials are still redacted. // Applied via ReplaceAllStringFunc rather than the plain list below. -var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) +var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlLit("sk-") + secretBody(`[A-Za-z0-9_-]`, `{20,}`)) // textSecretPatterns mirror secrets.Scan for end-boundary behavior and the // shared high-confidence shapes. A leading \b keeps each pattern from firing @@ -85,16 +109,18 @@ var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) // (not in secrets.Scan); ASIA temporary access keys are kept alongside AKIA. // openai keys are handled separately (digit filter). JWT has a strict form // (both segments start with eyJ) and a looser three-segment form. +// ctrlGap between shape characters keeps NUL/ESC/C1 split secrets matching +// without stripping those bytes out of the subject first. var textSecretPatterns = []*regexp.Regexp{ - regexp.MustCompile(`\bsk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`), - regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), - regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36,}`), - regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{12,}`), - regexp.MustCompile(`\bAIza[0-9A-Za-z\-_]{35,}`), - regexp.MustCompile(`\bxox[baprs]-[A-Za-z0-9-]{10,}`), - regexp.MustCompile(`\b(?:AKIA|ASIA)[A-Z0-9]{16}`), - regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), - regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), + regexp.MustCompile(`\b` + ctrlLit("sk-ant-") + `(?:` + ctrlLit("api") + `\d` + ctrlGap + `\d` + ctrlGap + ctrlLit("-") + `)?` + secretBody(`[A-Za-z0-9_-]`, `{20,}`)), + regexp.MustCompile(`\b` + ctrlLit("github_pat_") + secretBody(`[A-Za-z0-9_]`, `{22,}`)), + regexp.MustCompile(`\b` + ctrlLit("gh") + `[pousr]` + ctrlGap + `_` + ctrlGap + secretBody(`[A-Za-z0-9]`, `{36,}`)), + regexp.MustCompile(`\b` + ctrlLit("glpat-") + secretBody(`[A-Za-z0-9_-]`, `{12,}`)), + regexp.MustCompile(`\b` + ctrlLit("AIza") + secretBody(`[0-9A-Za-z\-_]`, `{35,}`)), + regexp.MustCompile(`\b` + ctrlLit("xox") + `[baprs]` + ctrlGap + `-` + ctrlGap + secretBody(`[A-Za-z0-9-]`, `{10,}`)), + regexp.MustCompile(`\b(?:` + ctrlLit("AKIA") + `|` + ctrlLit("ASIA") + `)` + secretBody(`[A-Z0-9]`, `{16}`)), + regexp.MustCompile(`\b` + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`)), + regexp.MustCompile(`\b` + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`)), } var ( @@ -171,11 +197,12 @@ func keyLooksSensitive(normalized string) bool { return false } -// stripControlBytes removes C0/C1 controls (Cc other than tab, LF, and CR) so -// shape matching sees a secret that was split by an embedded NUL, ESC, or C1 -// byte. Tab/LF/CR stay so log line structure is preserved. Lone Latin-1 C1 -// bytes (0x80–0x9F, invalid UTF-8) are stripped too; UTF-8 continuation bytes -// are not, because they are not controls. Must run before any pattern match. +// stripControlBytes removes C0/C1 controls (Cc other than tab, LF, and CR). +// Used to normalize an already-matched secret so prefix/digit checks see the +// rejoined shape. It is matching-time only and must not be applied to +// RedactString's input or return value. Tab/LF/CR stay. Lone Latin-1 C1 bytes +// (0x80–0x9F, invalid UTF-8) are stripped too; UTF-8 continuation bytes are +// not, because they are not controls. func stripControlBytes(s string) string { for i := 0; i < len(s); { c := s[i] @@ -231,9 +258,10 @@ func stripControlBytesFrom(s string, start int) string { func RedactString(value string, options Options) string { replacement := replacement(options) - // Strip C0/C1 first: a NUL, ESC, or C1 byte inside a key body splits the - // shape so the patterns miss it, and a later strip would rejoin the secret. - redacted := stripControlBytes(value) + // Match on the original string. Shape patterns allow C0/C1 gaps between + // characters so a split secret still matches; stripping first would join + // tokens that were never adjacent and make \b miss a leading wordchar. + redacted := value if len(options.ExtraSecretValues) > 0 { secrets := append([]string{}, options.ExtraSecretValues...) sort.SliceStable(secrets, func(i, j int) bool { @@ -288,8 +316,9 @@ func RedactString(value string, options Options) string { // openai keys first so the filter can drop kebab-case false positives // before any other pattern rewrites nearby text. redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { - if !knownOpenAIKeyPrefix(match) && !secretMatchHasDigit(match) && - strings.Contains(strings.TrimPrefix(match, "sk-"), "-") { + normalized := stripControlBytes(match) + if !knownOpenAIKeyPrefix(normalized) && !secretMatchHasDigit(normalized) && + strings.Contains(strings.TrimPrefix(normalized, "sk-"), "-") { return match } return replacement diff --git a/internal/redaction/redaction_test.go b/internal/redaction/redaction_test.go index a80c7e1cd..48c214d53 100644 --- a/internal/redaction/redaction_test.go +++ b/internal/redaction/redaction_test.go @@ -144,7 +144,8 @@ func containsCircular(v any) bool { func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { // Unsplit passing is not coverage: a NUL/ESC/C1 in the body splits the - // shape so the patterns miss it unless controls are stripped first. + // shape so the patterns miss it unless matching allows those controls as + // gaps between body characters (without joining unrelated tokens). const prefix = "sk-ant-api03-" const body = "abcdefghijklmnopqrstuvwxyz" unsplit := prefix + body @@ -184,3 +185,43 @@ func TestRedactStringPreservesAllowedWhitespaceAndUTF8(t *testing.T) { t.Fatalf("unexpected normalization: %q", got) } } + +func TestRedactStringWordcharBeforeNULAnthropicKey(t *testing.T) { + // Matching on a control-stripped copy joins "id42" and the key, so \b in + // textSecretPatterns misses and the secret leaks. Matching on the original + // treats the NUL as a boundary; leaked must be false. + const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + if got := RedactString(secret, Options{}); strings.Contains(got, "sk-ant-api03-") { + t.Fatalf("unsplit secret not redacted (test setup): %q", got) + } + input := "id42\x00" + secret + got := RedactString(input, Options{}) + leaked := strings.Contains(got, secret) || strings.Contains(got, "sk-ant-api03-") + if leaked { + t.Fatalf("wordchar-before-NUL+anthropic-key leaked=true out=%q", got) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("wordchar-before-NUL+anthropic-key leaked=false want %q, got %q", RedactedSecret, got) + } +} + +func TestRedactStringControlBytesWithoutSecretStayIdentical(t *testing.T) { + // scrubResultSecrets sets Result.Redacted when RedactString's result != + // Output. Stripping is matching-time only: no-secret control bytes must + // remain byte-identical so Redacted stays false. + cases := []struct { + name string + input string + }{ + {name: "form feed in source", input: "package main\n\ffunc main() {}\n"}, + {name: "Windows-1252 quotes", input: "Don\x92t \x93quote\x94 me\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := RedactString(tc.input, Options{}) + if got != tc.input { + t.Fatalf("no-secret input not byte-identical:\n in=%q\nout=%q", tc.input, got) + } + }) + } +} From 24d84b73721f530a9227cd36dd4a3c0cd02dad61 Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Tue, 1 Sep 2026 03:07:44 -0400 Subject: [PATCH 4/7] fix(redaction): preserve secret match boundaries --- internal/redaction/redaction.go | 99 +++++++++++++++++++++------- internal/redaction/redaction_test.go | 62 ++++++++++++++--- 2 files changed, 130 insertions(+), 31 deletions(-) diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index b5a401095..6ec8e8ce2 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "unicode" "unicode/utf8" @@ -73,24 +74,46 @@ var sensitiveKeys = map[string]struct{}{ // between characters of a secret shape. Matching stays on the original string: // a deleted control is never a join, so \b still treats wordchar+control as a // boundary and tokens that were never adjacent stay that way. Tab/LF/CR are -// excluded so log line structure is unchanged. \x{FFFD} is how Go's regexp -// engine reports a lone invalid UTF-8 C1 byte such as 0x9B. +// excluded so log line structure is unchanged. \x{FFFD} lets the regexp locate +// a lone invalid UTF-8 byte; validSecretControlGaps subsequently accepts only +// raw C1 bytes and rejects a real, valid UTF-8 U+FFFD rune. const ctrlGap = `[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f\x{FFFD}]*` -// ctrlLit quotes s as a regexp literal with ctrlGap after every rune, so a -// NUL/ESC/C1 may split the literal without breaking the match. +// ctrlLit quotes s as a regexp literal with ctrlGap strictly between runes, so +// a NUL/ESC/C1 may split the literal without letting a match end on a gap. func ctrlLit(s string) string { var b strings.Builder b.Grow(len(s) * (1 + len(ctrlGap))) + first := true for _, r := range s { + if !first { + b.WriteString(ctrlGap) + } b.WriteString(regexp.QuoteMeta(string(r))) - b.WriteString(ctrlGap) + first = false } return b.String() } -func secretBody(class, quant string) string { - return `(?:` + class + ctrlGap + `)` + quant +func ctrlJoin(parts ...string) string { + return strings.Join(parts, ctrlGap) +} + +// secretBody keeps gaps strictly between the minimum required body characters. +// Once an unbounded shape has reached that high-confidence minimum, its +// optional tail stays contiguous: a later control is a suffix delimiter rather +// than permission to absorb the following token into the secret (which could +// feed unrelated suffix text into the OpenAI kebab-case exception). +func secretBody(class string, minimum int, unbounded bool) string { + if minimum <= 0 { + return "" + } + quantifier := strconv.Itoa(minimum - 1) + body := class + `(?:` + ctrlGap + class + `){` + quantifier + `}` + if unbounded { + body += class + `*` + } + return body } // openaiKeyPattern mirrors secrets.Scan's broad sk- body. Known OpenAI @@ -98,7 +121,7 @@ func secretBody(class, quant string) string { // digit-free matches with an interior hyphen are left alone (kebab-case false // positives), while digit-free legacy sk- credentials are still redacted. // Applied via ReplaceAllStringFunc rather than the plain list below. -var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlLit("sk-") + secretBody(`[A-Za-z0-9_-]`, `{20,}`)) +var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("sk-"), secretBody(`[A-Za-z0-9_-]`, 20, true))) // textSecretPatterns mirror secrets.Scan for end-boundary behavior and the // shared high-confidence shapes. A leading \b keeps each pattern from firing @@ -112,15 +135,15 @@ var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlLit("sk-") + secretBody(`[A // ctrlGap between shape characters keeps NUL/ESC/C1 split secrets matching // without stripping those bytes out of the subject first. var textSecretPatterns = []*regexp.Regexp{ - regexp.MustCompile(`\b` + ctrlLit("sk-ant-") + `(?:` + ctrlLit("api") + `\d` + ctrlGap + `\d` + ctrlGap + ctrlLit("-") + `)?` + secretBody(`[A-Za-z0-9_-]`, `{20,}`)), - regexp.MustCompile(`\b` + ctrlLit("github_pat_") + secretBody(`[A-Za-z0-9_]`, `{22,}`)), - regexp.MustCompile(`\b` + ctrlLit("gh") + `[pousr]` + ctrlGap + `_` + ctrlGap + secretBody(`[A-Za-z0-9]`, `{36,}`)), - regexp.MustCompile(`\b` + ctrlLit("glpat-") + secretBody(`[A-Za-z0-9_-]`, `{12,}`)), - regexp.MustCompile(`\b` + ctrlLit("AIza") + secretBody(`[0-9A-Za-z\-_]`, `{35,}`)), - regexp.MustCompile(`\b` + ctrlLit("xox") + `[baprs]` + ctrlGap + `-` + ctrlGap + secretBody(`[A-Za-z0-9-]`, `{10,}`)), - regexp.MustCompile(`\b(?:` + ctrlLit("AKIA") + `|` + ctrlLit("ASIA") + `)` + secretBody(`[A-Z0-9]`, `{16}`)), - regexp.MustCompile(`\b` + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`)), - regexp.MustCompile(`\b` + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`)), + regexp.MustCompile(`\b` + ctrlLit("sk-ant-") + ctrlGap + `(?:` + ctrlJoin(ctrlLit("api"), `\d`, `\d`, `-`) + ctrlGap + `)?` + secretBody(`[A-Za-z0-9_-]`, 20, true)), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("github_pat_"), secretBody(`[A-Za-z0-9_]`, 22, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("gh"), `[pousr]`, `_`, secretBody(`[A-Za-z0-9]`, 36, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("glpat-"), secretBody(`[A-Za-z0-9_-]`, 12, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("AIza"), secretBody(`[0-9A-Za-z\-_]`, 35, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("xox"), `[baprs]`, `-`, secretBody(`[A-Za-z0-9-]`, 10, true))), + regexp.MustCompile(`\b` + ctrlJoin(`(?:`+ctrlLit("AKIA")+`|`+ctrlLit("ASIA")+`)`, secretBody(`[A-Z0-9]`, 16, false))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), } var ( @@ -256,6 +279,26 @@ func stripControlBytesFrom(s string, start int) string { return b.String() } +// validSecretControlGaps disambiguates RuneError matches at the byte boundary. +// Go's regexp engine represents both a malformed single byte and a legitimate +// U+FFFD rune as RuneError. Only raw invalid C1 bytes (0x80-0x9F) are supported +// gaps; a valid UTF-8 replacement rune, or another malformed byte, must keep +// the candidate split and prevent redaction. +func validSecretControlGaps(match string) bool { + for i := 0; i < len(match); { + r, size := utf8.DecodeRuneInString(match[i:]) + if r != utf8.RuneError { + i += size + continue + } + if size != 1 || match[i] < 0x80 || match[i] > 0x9F { + return false + } + i++ + } + return true +} + func RedactString(value string, options Options) string { replacement := replacement(options) // Match on the original string. Shape patterns allow C0/C1 gaps between @@ -313,9 +356,24 @@ func RedactString(value string, options Options) string { } return parts[1] + parts[2] + "=" + replacement }) - // openai keys first so the filter can drop kebab-case false positives - // before any other pattern rewrites nearby text. + // Match high-confidence specialized shapes first. In particular, the broad + // sk- pattern may reach its minimum before a control inside a longer + // Anthropic key; letting the Anthropic shape consume that split first avoids + // leaving a recognizable credential suffix behind. + for _, pattern := range textSecretPatterns { + redacted = pattern.ReplaceAllStringFunc(redacted, func(match string) string { + if !validSecretControlGaps(match) { + return match + } + return replacement + }) + } + // Apply the broad OpenAI shape after specialized keys so its kebab-case + // false-positive filter considers only the matched key, never suffix text. redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { + if !validSecretControlGaps(match) { + return match + } normalized := stripControlBytes(match) if !knownOpenAIKeyPrefix(normalized) && !secretMatchHasDigit(normalized) && strings.Contains(strings.TrimPrefix(normalized, "sk-"), "-") { @@ -323,9 +381,6 @@ func RedactString(value string, options Options) string { } return replacement }) - for _, pattern := range textSecretPatterns { - redacted = pattern.ReplaceAllString(redacted, replacement) - } return redacted } diff --git a/internal/redaction/redaction_test.go b/internal/redaction/redaction_test.go index 48c214d53..43d45b34a 100644 --- a/internal/redaction/redaction_test.go +++ b/internal/redaction/redaction_test.go @@ -164,21 +164,65 @@ func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - input := prefix + tc.split + body - got := RedactString(input, Options{}) - if strings.Contains(got, body) { - t.Fatalf("secret split by %s leaked in %q", tc.name, got) + inputs := []struct { + placement string + input string + }{ + {placement: "prefix-body boundary", input: prefix + tc.split + body}, + {placement: "inside body", input: prefix + body[:13] + tc.split + body[13:]}, } - if strings.Contains(got, prefix) { - t.Fatalf("secret prefix split by %s leaked in %q", tc.name, got) - } - if !strings.Contains(got, RedactedSecret) { - t.Fatalf("expected %q after %s split, got %q", RedactedSecret, tc.name, got) + for _, input := range inputs { + t.Run(input.placement, func(t *testing.T) { + got := RedactString(input.input, Options{}) + if strings.Contains(got, body) { + t.Fatalf("secret split by %s %s leaked in %q", tc.name, input.placement, got) + } + if strings.Contains(got, prefix) { + t.Fatalf("secret prefix split by %s %s leaked in %q", tc.name, input.placement, got) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("expected %q after %s %s split, got %q", RedactedSecret, tc.name, input.placement, got) + } + }) } }) } } +func TestRedactStringPreservesControlAfterCredential(t *testing.T) { + const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + input := "key=" + secret + "\x00path/one.go\x00path/two.go" + want := "key=" + RedactedSecret + "\x00path/one.go\x00path/two.go" + if got := RedactString(input, Options{}); got != want { + t.Fatalf("terminal credential separator changed:\n got=%q\nwant=%q", got, want) + } +} + +func TestRedactStringSuffixCannotDisableOpenAIKeyMatch(t *testing.T) { + const secret = "sk-aaaaaaaaaaaaaaaaaaaabcdefgh" + input := "key " + secret + "\x1bkebab-case tail" + want := "key " + RedactedSecret + "\x1bkebab-case tail" + if got := RedactString(input, Options{}); got != want { + t.Fatalf("suffix changed OpenAI key classification:\n got=%q\nwant=%q", got, want) + } +} + +func TestRedactStringDistinguishesInvalidC1FromValidReplacementRune(t *testing.T) { + const prefix = "sk-ant-api03-" + const body = "abcdefghijklmnopqrstuvwxyz" + + invalidC1 := prefix + body[:13] + "\x9b" + body[13:] + if got := RedactString(invalidC1, Options{}); got != RedactedSecret { + t.Fatalf("raw invalid C1 split was not redacted: %q", got) + } + + validReplacement := prefix + body[:13] + "\uFFFD" + body[13:] + got := RedactString(validReplacement, Options{}) + if !strings.Contains(got, "\uFFFD"+body[13:]) { + t.Fatalf("valid U+FFFD was treated as a control gap: %q", got) + } +} + func TestRedactStringPreservesAllowedWhitespaceAndUTF8(t *testing.T) { input := "safe\tline\nnext\rfinal café" if got := RedactString(input, Options{}); got != input { From 8fbc413d20f4d16382c27a095b6df999621a3e32 Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Tue, 1 Sep 2026 16:28:18 -0400 Subject: [PATCH 5/7] fix(redaction): separate logical candidate matching and trailing control delimiters --- internal/redaction/redaction.go | 73 +++++++++++----- internal/redaction/split_harness_test.go | 102 +++++++++++++++++++++++ 2 files changed, 154 insertions(+), 21 deletions(-) create mode 100644 internal/redaction/split_harness_test.go diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index 6ec8e8ce2..5105e0d5c 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -99,21 +99,18 @@ func ctrlJoin(parts ...string) string { return strings.Join(parts, ctrlGap) } -// secretBody keeps gaps strictly between the minimum required body characters. -// Once an unbounded shape has reached that high-confidence minimum, its -// optional tail stays contiguous: a later control is a suffix delimiter rather -// than permission to absorb the following token into the secret (which could -// feed unrelated suffix text into the OpenAI kebab-case exception). +// secretBody generates a regex matching at least minimum body characters, +// allowing C0/C1 control gaps between any characters. It always starts and ends +// on a class character (never on a gap). func secretBody(class string, minimum int, unbounded bool) string { if minimum <= 0 { return "" } quantifier := strconv.Itoa(minimum - 1) - body := class + `(?:` + ctrlGap + class + `){` + quantifier + `}` if unbounded { - body += class + `*` + return class + `(?:` + ctrlGap + class + `){` + quantifier + `,}` } - return body + return class + `(?:` + ctrlGap + class + `){` + quantifier + `}` } // openaiKeyPattern mirrors secrets.Scan's broad sk- body. Known OpenAI @@ -362,28 +359,62 @@ func RedactString(value string, options Options) string { // leaving a recognizable credential suffix behind. for _, pattern := range textSecretPatterns { redacted = pattern.ReplaceAllStringFunc(redacted, func(match string) string { - if !validSecretControlGaps(match) { - return match - } - return replacement + return redactMatchedPattern(match, pattern, replacement, nil) }) } // Apply the broad OpenAI shape after specialized keys so its kebab-case // false-positive filter considers only the matched key, never suffix text. redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { - if !validSecretControlGaps(match) { - return match - } - normalized := stripControlBytes(match) - if !knownOpenAIKeyPrefix(normalized) && !secretMatchHasDigit(normalized) && - strings.Contains(strings.TrimPrefix(normalized, "sk-"), "-") { - return match - } - return replacement + return redactMatchedPattern(match, openaiKeyPattern, replacement, func(m string) bool { + normalized := stripControlBytes(m) + if !knownOpenAIKeyPrefix(normalized) && !secretMatchHasDigit(normalized) && + strings.Contains(strings.TrimPrefix(normalized, "sk-"), "-") { + return false + } + return true + }) }) return redacted } +func firstControlIndex(s string) int { + for i := 0; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + return i + } + i++ + continue + } + if c <= 0x9F { + return i + } + r, size := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(r) { + return i + } + i += size + } + return -1 +} + +func redactMatchedPattern(match string, pattern *regexp.Regexp, replacement string, isValid func(string) bool) string { + if !validSecretControlGaps(match) { + return match + } + if ctrlIdx := firstControlIndex(match); ctrlIdx >= 0 { + pre := match[:ctrlIdx] + if pattern.MatchString(pre) && (isValid == nil || isValid(pre)) { + return replacement + match[ctrlIdx:] + } + } + if isValid != nil && !isValid(match) { + return match + } + return replacement +} + // knownOpenAIKeyPrefix is the redaction-side twin of secrets.knownOpenAIKeyPrefix: // known OpenAI-issued forms redact even with an alphabet-only body. func knownOpenAIKeyPrefix(match string) bool { diff --git a/internal/redaction/split_harness_test.go b/internal/redaction/split_harness_test.go new file mode 100644 index 000000000..2aa3fa6a7 --- /dev/null +++ b/internal/redaction/split_harness_test.go @@ -0,0 +1,102 @@ +package redaction + +import ( + "strings" + "testing" +) + +func TestSplitRedactionHarness(t *testing.T) { + // Representative secrets for all supported shapes + secrets := []struct { + name string + secret string + }{ + {"Anthropic", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz1234"}, + {"OpenAI standard", "sk-abcdefghijklmnopqrstuvwxyz12345678"}, + {"OpenAI with hyphen and digit", "sk-aaaaaaaaaa-bbbbbbbbb1234567890"}, + {"OpenAI proj", "sk-proj-abcdefghijklmnopqrstuvwxyz12345"}, + {"GitHub PAT", "github_pat_11AAAAAAA0123456789abcdefghijklmnopqrstuvwxyz"}, + {"GitHub Fine-Grained", "ghp_123456789012345678901234567890123456"}, + {"GitLab PAT", "glpat-12345678901234567890"}, + {"Google API", "AIzaSyD-1234567890123456789012345678901"}, + {"Slack bot", "xoxb-123456789012-abcdefghijklmno"}, + {"AWS AKIA", "AKIAIOSFODNN7EXAMPLE"}, + {"JWT", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"}, + } + + controls := []struct { + name string + char string + }{ + {"NUL", "\x00"}, + {"ESC", "\x1b"}, + {"lone C1", "\x9b"}, + {"UTF-8 C1", "\u009b"}, + } + + for _, s := range secrets { + t.Run(s.name, func(t *testing.T) { + // First verify unsplit redacts + gotUnsplit := RedactString(s.secret, Options{}) + if strings.Contains(gotUnsplit, s.secret) || !strings.Contains(gotUnsplit, RedactedSecret) { + t.Fatalf("unsplit secret %q failed to redact: %q", s.secret, gotUnsplit) + } + + // Test split at all interior positions throughout the secret + for _, ctrl := range controls { + for pos := 1; pos < len(s.secret); pos++ { + splitSecret := s.secret[:pos] + ctrl.char + s.secret[pos:] + got := RedactString(splitSecret, Options{}) + + // Strip controls from output and assert original secret cannot be recovered + strippedOutput := stripControlBytes(got) + if strings.Contains(strippedOutput, s.secret) { + t.Fatalf("split at pos %d with %s leaked secret!\n split input=%q\n got=%q\n stripped=%q", pos, ctrl.name, splitSecret, got, strippedOutput) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("split at pos %d with %s did not contain RedactedSecret!\n got=%q", pos, ctrl.name, got) + } + } + } + }) + } +} + +func TestSplitRedactionNegativeCases(t *testing.T) { + controls := []string{"\x00", "\x1b", "\x9b", "\u009b"} + + t.Run("OpenAI kebab false positive with control", func(t *testing.T) { + kebab := "sk-my-awesome-kebab-project" + for _, ctrl := range controls { + input := kebab[:10] + ctrl + kebab[10:] + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("digit-free kebab falsely redacted with control %q: got %q, want %q", ctrl, got, input) + } + } + }) + + t.Run("Control immediately before complete credential", func(t *testing.T) { + secret := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + for _, ctrl := range controls { + input := "prefix" + ctrl + secret + got := RedactString(input, Options{}) + want := "prefix" + ctrl + RedactedSecret + if got != want { + t.Fatalf("control before secret mutated boundary: got %q, want %q", got, want) + } + } + }) + + t.Run("Control immediately after complete credential", func(t *testing.T) { + secret := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + for _, ctrl := range controls { + input := secret + ctrl + "suffix" + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + "suffix" + if got != want { + t.Fatalf("control after secret mutated delimiter: got %q, want %q", got, want) + } + } + }) +} From 64b21008d6f6163262e414a6cbd1a8350178336f Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Tue, 1 Sep 2026 18:18:21 -0400 Subject: [PATCH 6/7] fix(redaction): recognize terminal delimiters after earlier internal gaps --- internal/redaction/redaction.go | 55 +++++++++++++---- internal/redaction/split_harness_test.go | 76 ++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 12 deletions(-) diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index 5105e0d5c..a673425c0 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -377,38 +377,69 @@ func RedactString(value string, options Options) string { return redacted } -func firstControlIndex(s string) int { +type controlSpan struct { + start int + end int +} + +func findControlSpans(s string) []controlSpan { + var spans []controlSpan for i := 0; i < len(s); { c := s[i] if c < 0x80 { if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { - return i + start := i + for i < len(s) && s[i] < 0x80 && s[i] != '\t' && s[i] != '\n' && s[i] != '\r' && (s[i] < 0x20 || s[i] == 0x7F) { + i++ + } + spans = append(spans, controlSpan{start: start, end: i}) + continue } i++ continue } if c <= 0x9F { - return i + start := i + for i < len(s) && s[i] >= 0x80 && s[i] <= 0x9F { + i++ + } + spans = append(spans, controlSpan{start: start, end: i}) + continue } r, size := utf8.DecodeRuneInString(s[i:]) - if unicode.IsControl(r) { - return i + if unicode.IsControl(r) || r == utf8.RuneError { + start := i + i += size + for i < len(s) { + nr, nsize := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(nr) || nr == utf8.RuneError { + i += nsize + } else { + break + } + } + spans = append(spans, controlSpan{start: start, end: i}) + continue } i += size } - return -1 + return spans } func redactMatchedPattern(match string, pattern *regexp.Regexp, replacement string, isValid func(string) bool) string { + spans := findControlSpans(match) + for _, span := range spans { + pre := match[:span.start] + if pre == "" { + continue + } + if validSecretControlGaps(pre) && pattern.MatchString(pre) && (isValid == nil || isValid(pre)) { + return replacement + match[span.start:] + } + } if !validSecretControlGaps(match) { return match } - if ctrlIdx := firstControlIndex(match); ctrlIdx >= 0 { - pre := match[:ctrlIdx] - if pattern.MatchString(pre) && (isValid == nil || isValid(pre)) { - return replacement + match[ctrlIdx:] - } - } if isValid != nil && !isValid(match) { return match } diff --git a/internal/redaction/split_harness_test.go b/internal/redaction/split_harness_test.go index 2aa3fa6a7..0d517f130 100644 --- a/internal/redaction/split_harness_test.go +++ b/internal/redaction/split_harness_test.go @@ -62,6 +62,82 @@ func TestSplitRedactionHarness(t *testing.T) { } } +func TestSplitRedactionMultiControlCases(t *testing.T) { + t.Run("Internal gap before minimum then terminal delimiter", func(t *testing.T) { + input := "sk-ant-api03-\x00abcdefghijklmnopqrstuvwxyz\x00path/file.go" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00path/file.go" + if got != want { + t.Fatalf("multi-control anthropic mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("OpenAI internal gap then terminal delimiter before kebab suffix", func(t *testing.T) { + input := "sk-\x00abcdefghijklmnopqrstuv\x1bkebab-case tail" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x1bkebab-case tail" + if got != want { + t.Fatalf("multi-control openai mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("OpenAI internal gap before digit suffix then terminal delimiter", func(t *testing.T) { + input := "sk-aaaaaaaaaa-bbbbbbbbb\x001234567890\x00path/one.go" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00path/one.go" + if got != want { + t.Fatalf("multi-control openai with digits mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("JWT multiple internal gaps and terminal delimiter", func(t *testing.T) { + input := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\x00.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ\x1b.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\x00trailing/text" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00trailing/text" + if got != want { + t.Fatalf("multi-control jwt mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("Multiple internal controls in credential body", func(t *testing.T) { + input := "sk-ant-\x00api03-\x1babcdefghijklmnopqrstuvwxyz" + got := RedactString(input, Options{}) + if got != RedactedSecret { + t.Fatalf("multiple internal gaps in anthropic key mismatch: got %q, want %q", got, RedactedSecret) + } + }) + + t.Run("Terminal delimiter separating two credentials", func(t *testing.T) { + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz\x00ghp_123456789012345678901234567890123456" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00" + RedactedSecret + if got != want { + t.Fatalf("two credentials separated by delimiter mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Complete credential followed by invalid bytes", func(t *testing.T) { + cases := []struct { + name string + suffix string + }{ + {"valid U+FFFD", "\uFFFDsuffix"}, + {"malformed byte 0xFF", "\xffsuffix"}, + {"malformed byte 0xC0", "\xc0suffix"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + tc.suffix + got := RedactString(input, Options{}) + want := RedactedSecret + tc.suffix + if got != want { + t.Fatalf("suffix %s mismatch: got %q, want %q", tc.name, got, want) + } + }) + } + }) +} + func TestSplitRedactionNegativeCases(t *testing.T) { controls := []string{"\x00", "\x1b", "\x9b", "\u009b"} From 2d9e806567502adbf291cfbef442611e886318af Mon Sep 17 00:00:00 2001 From: cairn-intern Date: Wed, 2 Sep 2026 04:33:35 -0400 Subject: [PATCH 7/7] fix(redaction): address review findings for split credential scanning and boundary resolution --- internal/redaction/redaction.go | 255 +++++++++++++++++++---- internal/redaction/split_harness_test.go | 126 +++++++++++ 2 files changed, 346 insertions(+), 35 deletions(-) diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index a673425c0..6cf9c9a7a 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -120,6 +120,10 @@ func secretBody(class string, minimum int, unbounded bool) string { // Applied via ReplaceAllStringFunc rather than the plain list below. var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("sk-"), secretBody(`[A-Za-z0-9_-]`, 20, true))) +// plainOpenaiKeyPattern is the non-gap-aware counterpart of openaiKeyPattern, +// used for boundary resolution on logical (control-stripped) candidates. +var plainOpenaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) + // textSecretPatterns mirror secrets.Scan for end-boundary behavior and the // shared high-confidence shapes. A leading \b keeps each pattern from firing // mid-word; a trailing \b is omitted so a secret followed by more word @@ -143,6 +147,21 @@ var textSecretPatterns = []*regexp.Regexp{ regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), } +// plainSecretPatterns are non-gap-aware counterparts of textSecretPatterns, +// used for boundary resolution on logical (control-stripped) candidates. +// Each entry corresponds 1:1 with textSecretPatterns by index. +var plainSecretPatterns = []*regexp.Regexp{ + regexp.MustCompile(`\bsk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`), + regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), + regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36,}`), + regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{12,}`), + regexp.MustCompile(`\bAIza[0-9A-Za-z\-_]{35,}`), + regexp.MustCompile(`\bxox[baprs]-[A-Za-z0-9-]{10,}`), + regexp.MustCompile(`\b(?:AKIA|ASIA)[A-Z0-9]{16}`), + regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), + regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), +} + var ( privateKeyPattern = regexp.MustCompile(`(?s)-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----`) jsonStringPattern = regexp.MustCompile(`("([^"\\]*(?:\\.[^"\\]*)*)"\s*:\s*)"([^"\\]*(?:\\.[^"\\]*)*)"`) @@ -357,33 +376,67 @@ func RedactString(value string, options Options) string { // sk- pattern may reach its minimum before a control inside a longer // Anthropic key; letting the Anthropic shape consume that split first avoids // leaving a recognizable credential suffix behind. - for _, pattern := range textSecretPatterns { - redacted = pattern.ReplaceAllStringFunc(redacted, func(match string) string { - return redactMatchedPattern(match, pattern, replacement, nil) - }) + for i, pattern := range textSecretPatterns { + plain := plainSecretPatterns[i] + minLen := minSecretLens[i] + redacted = replaceAllSecretMatches(redacted, i, pattern, plain, replacement, minLen, nil) } // Apply the broad OpenAI shape after specialized keys so its kebab-case // false-positive filter considers only the matched key, never suffix text. - redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { - return redactMatchedPattern(match, openaiKeyPattern, replacement, func(m string) bool { - normalized := stripControlBytes(m) - if !knownOpenAIKeyPrefix(normalized) && !secretMatchHasDigit(normalized) && - strings.Contains(strings.TrimPrefix(normalized, "sk-"), "-") { - return false - } - return true - }) + redacted = replaceAllSecretMatches(redacted, -1, openaiKeyPattern, plainOpenaiKeyPattern, replacement, minOpenAILen, func(m string) bool { + // m is the logical (control-stripped) candidate. + if !knownOpenAIKeyPrefix(m) && !secretMatchHasDigit(m) && + strings.Contains(strings.TrimPrefix(m, "sk-"), "-") { + return false + } + return true }) return redacted } +var minSecretLens = []int{ + 27, // sk-ant- (7) + 20 + 33, // github_pat_ (11) + 22 + 40, // gh[pousr]_ (4) + 36 + 18, // glpat- (6) + 12 + 39, // AIza (4) + 35 + 15, // xox[baprs]- (5) + 10 + 20, // AKIA/ASIA (4) + 16 + 38, // JWT (3 + 10 + 1 + 3 + 10 + 1 + 10) + 34, // JWT (3 + 10 + 1 + 10 + 1 + 10) +} + +const minOpenAILen = 23 // sk- (3) + 20 + +func isCandidateLength(logPre string, patternIndex int, minLen int) bool { + if len(logPre) < minLen { + return false + } + if patternIndex == 7 || patternIndex == 8 { + return strings.Count(logPre, ".") >= 2 + } + return true +} + type controlSpan struct { - start int - end int + start int + end int + validGap bool } -func findControlSpans(s string) []controlSpan { +type logicalCandidate struct { + logical string + origEnds []int + spans []controlSpan +} + +func extractLogicalCandidate(s string) logicalCandidate { + var logical strings.Builder + logical.Grow(len(s)) + var origEnds []int + origEnds = make([]int, 0, len(s)) var spans []controlSpan + for i := 0; i < len(s); { c := s[i] if c < 0x80 { @@ -392,58 +445,190 @@ func findControlSpans(s string) []controlSpan { for i < len(s) && s[i] < 0x80 && s[i] != '\t' && s[i] != '\n' && s[i] != '\r' && (s[i] < 0x20 || s[i] == 0x7F) { i++ } - spans = append(spans, controlSpan{start: start, end: i}) + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) continue } + logical.WriteByte(c) i++ + origEnds = append(origEnds, i) continue } - if c <= 0x9F { + if c >= 0x80 && c <= 0x9F { start := i for i < len(s) && s[i] >= 0x80 && s[i] <= 0x9F { i++ } - spans = append(spans, controlSpan{start: start, end: i}) + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) continue } r, size := utf8.DecodeRuneInString(s[i:]) - if unicode.IsControl(r) || r == utf8.RuneError { + if r == utf8.RuneError { + start := i + i += size + spans = append(spans, controlSpan{start: start, end: i, validGap: false}) + continue + } + if unicode.IsControl(r) && r != '\t' && r != '\n' && r != '\r' { start := i i += size for i < len(s) { nr, nsize := utf8.DecodeRuneInString(s[i:]) - if unicode.IsControl(nr) || nr == utf8.RuneError { + if unicode.IsControl(nr) && nr != '\t' && nr != '\n' && nr != '\r' { i += nsize } else { break } } - spans = append(spans, controlSpan{start: start, end: i}) + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) continue } + logical.WriteRune(r) i += size + runeLen := len(string(r)) + for b := 0; b < runeLen; b++ { + origEnds = append(origEnds, i) + } + } + return logicalCandidate{ + logical: logical.String(), + origEnds: origEnds, + spans: spans, } - return spans } -func redactMatchedPattern(match string, pattern *regexp.Regexp, replacement string, isValid func(string) bool) string { - spans := findControlSpans(match) - for _, span := range spans { - pre := match[:span.start] - if pre == "" { +func findCredentialBoundary(match string, patternIndex int, plainPattern *regexp.Regexp, minLen int, isValid func(string) bool) (int, bool) { + cand := extractLogicalCandidate(match) + if len(cand.spans) == 0 { + if isValid != nil && !isValid(match) { + return len(match), false + } + return len(match), true + } + + logicalStr := cand.logical + + // Fast path for OpenAI kebab false positives: if the logical string has no digits, + // is not a known prefix, and the token before the first control span already contains + // an interior hyphen, no sub-span can ever be valid. + if patternIndex == -1 && isValid != nil && len(cand.spans) > 0 && cand.spans[0].start > 0 { + if !knownOpenAIKeyPrefix(logicalStr) && !secretMatchHasDigit(logicalStr) { + firstLogLen := sort.SearchInts(cand.origEnds, cand.spans[0].start+1) + firstPre := logicalStr[:firstLogLen] + if strings.Contains(strings.TrimPrefix(firstPre, "sk-"), "-") { + return len(match), false + } + } + } + + for _, span := range cand.spans { + if span.start == 0 { continue } - if validSecretControlGaps(pre) && pattern.MatchString(pre) && (isValid == nil || isValid(pre)) { - return replacement + match[span.start:] + // If scanning OpenAI keys and text after span starts with "sk-", + // this span is a delimiter before a new key. + if patternIndex == -1 && strings.HasPrefix(match[span.end:], "sk-") { + logLen := sort.SearchInts(cand.origEnds, span.start+1) + if isCandidateLength(logicalStr[:logLen], patternIndex, minLen) { + logPre := logicalStr[:logLen] + if plainPattern.MatchString(logPre) && (isValid == nil || isValid(logPre)) { + return span.start, true + } + } + return span.start, false + } + + logLen := sort.SearchInts(cand.origEnds, span.start+1) + if !isCandidateLength(logicalStr[:logLen], patternIndex, minLen) { + if !span.validGap { + return span.start, false + } + continue + } + logPre := logicalStr[:logLen] + if plainPattern.MatchString(logPre) && (isValid == nil || isValid(logPre)) { + return span.start, true + } + if !span.validGap { + return span.start, false + } + // If logPre is a kebab false positive, check if the remaining suffix in match + // has fewer than 4 characters. If so, logPre was an independent kebab token + // followed by a delimiter (e.g. "sk-my-kebab\x00v2/file.go"). + if isValid != nil && !isValid(logPre) { + suffixLen := len(logicalStr) - logLen + if suffixLen < 4 { + return span.start, false + } } } - if !validSecretControlGaps(match) { - return match + + for _, span := range cand.spans { + if !span.validGap { + return span.start, false + } + } + + if !isCandidateLength(logicalStr, patternIndex, minLen) { + return len(match), false + } + + if !plainPattern.MatchString(logicalStr) { + return len(match), false + } + + if isValid != nil && !isValid(logicalStr) { + return len(match), false + } + + if len(cand.origEnds) > 0 { + lastEnd := cand.origEnds[len(cand.origEnds)-1] + return lastEnd, true + } + return len(match), true +} + +func replaceAllSecretMatches(src string, patternIndex int, pattern *regexp.Regexp, plainPattern *regexp.Regexp, replacement string, minLen int, isValid func(string) bool) string { + loc := pattern.FindStringIndex(src) + if loc == nil { + return src } - if isValid != nil && !isValid(match) { - return match + + var b strings.Builder + b.Grow(len(src)) + + lastIndex := 0 + for { + loc := pattern.FindStringIndex(src[lastIndex:]) + if loc == nil { + b.WriteString(src[lastIndex:]) + break + } + + matchStart := lastIndex + loc[0] + matchEnd := lastIndex + loc[1] + match := src[matchStart:matchEnd] + + advanceLen, shouldRedact := findCredentialBoundary(match, patternIndex, plainPattern, minLen, isValid) + + b.WriteString(src[lastIndex:matchStart]) + if shouldRedact { + b.WriteString(replacement) + lastIndex = matchStart + advanceLen + } else { + if advanceLen <= 0 { + advanceLen = 1 + } + b.WriteString(src[matchStart : matchStart+advanceLen]) + lastIndex = matchStart + advanceLen + } + if lastIndex <= matchStart { + lastIndex = matchStart + 1 + } + if lastIndex >= len(src) { + break + } } - return replacement + return b.String() } // knownOpenAIKeyPrefix is the redaction-side twin of secrets.knownOpenAIKeyPrefix: diff --git a/internal/redaction/split_harness_test.go b/internal/redaction/split_harness_test.go index 0d517f130..f6382de7f 100644 --- a/internal/redaction/split_harness_test.go +++ b/internal/redaction/split_harness_test.go @@ -116,6 +116,59 @@ func TestSplitRedactionMultiControlCases(t *testing.T) { } }) + t.Run("Two same-shape keys separated by control bytes", func(t *testing.T) { + keyPairs := []struct { + name string + key1 string + key2 string + }{ + {"OpenAI", "sk-aaaaaaaaaaaaaaaaaaaabcdefgh", "sk-bbbbbbbbbbbbbbbbbbbbcdefghi"}, + {"GitHub Fine-Grained", "ghp_123456789012345678901234567890123456", "ghp_abcdefghijklmnopqrstuvwxyz1234567890"}, + {"GitHub PAT", "github_pat_11AAAAAAA0123456789abcdefghijklmnopqrstuvwxyz", "github_pat_22BBBBBBB0123456789abcdefghijklmnopqrstuvwxyz"}, + {"GitLab PAT", "glpat-12345678901234567890", "glpat-abcdefghijklmnopqrst"}, + {"Google API", "AIzaSyD-1234567890123456789012345678901", "AIzaSyD-abcdefghijklmnopqrstuvwxyz12345"}, + {"Slack", "xox" + "b-123456789012-abcdefghijklmno", "xox" + "b-987654321098-zyxwvutsrqponml"}, + } + ctrls := []string{"\x00", "\x1b", "\x9b", "\u009b"} + for _, pair := range keyPairs { + for _, ctrl := range ctrls { + input := pair.key1 + ctrl + pair.key2 + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + RedactedSecret + if got != want { + t.Fatalf("two %s keys separated by %q mismatch: got %q, want %q", pair.name, ctrl, got, want) + } + } + } + }) + + t.Run("Three same-shape keys separated by control bytes", func(t *testing.T) { + input := "sk-aaaaaaaaaaaaaaaaaaaabcdefgh\x00sk-bbbbbbbbbbbbbbbbbbbbcdefghi\x1bsk-ccccccccccccccccccccdefghij" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00" + RedactedSecret + "\x1b" + RedactedSecret + if got != want { + t.Fatalf("three OpenAI keys mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Short sk- token before credential", func(t *testing.T) { + input := "sk-ab\x00sk-aaaaaaaaaaaaaaaaaaaabcdefgh" + got := RedactString(input, Options{}) + want := "sk-ab\x00" + RedactedSecret + if got != want { + t.Fatalf("short sk- token before credential mismatch: got %q, want %q", got, want) + } + }) + + t.Run("OpenAI kebab false positive before path with digit", func(t *testing.T) { + input := "sk-my-awesome-kebab-project\x00v2/file.go" + got := RedactString(input, Options{}) + want := "sk-my-awesome-kebab-project\x00v2/file.go" + if got != want { + t.Fatalf("kebab project before path with digit mismatch: got %q, want %q", got, want) + } + }) + t.Run("Complete credential followed by invalid bytes", func(t *testing.T) { cases := []struct { name string @@ -176,3 +229,76 @@ func TestSplitRedactionNegativeCases(t *testing.T) { } }) } + +func TestSplitRedactionLinearScaling(t *testing.T) { + sizes := []int{8 * 1024, 16 * 1024, 32 * 1024, 64 * 1024, 128 * 1024, 800 * 1024} + + t.Run("OpenAI kebab repeated gaps scaling", func(t *testing.T) { + for _, size := range sizes { + // Construct "sk-kebab-" + repeated "\x00a" up to size + var b strings.Builder + b.WriteString("sk-kebab-") + for b.Len() < size { + b.WriteString("\x00a") + } + input := b.String() + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("kebab false positive was falsely redacted at size %d", size) + } + } + }) + + t.Run("JWT repeated gaps scaling and correct redaction", func(t *testing.T) { + for _, size := range sizes { + // Construct "eyJ" + repeated "\x00a" + ".eyJ" + repeated "\x00b" + ".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" + segLen := size / 2 + var b strings.Builder + b.WriteString("eyJ") + for b.Len() < segLen { + b.WriteString("\x00a") + } + b.WriteString(".eyJ") + for b.Len() < size { + b.WriteString("\x00b") + } + b.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := b.String() + got := RedactString(input, Options{}) + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("JWT at size %d failed to redact", size) + } + } + }) +} + +func BenchmarkRedactOpenAIKebabGaps128KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("sk-kebab-") + for builder.Len() < 128*1024 { + builder.WriteString("\x00a") + } + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +} + +func BenchmarkRedactJWTGaps128KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("eyJ") + for builder.Len() < 64*1024 { + builder.WriteString("\x00a") + } + builder.WriteString(".eyJ") + for builder.Len() < 128*1024 { + builder.WriteString("\x00b") + } + builder.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +}