diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index 24685eca9..6cf9c9a7a 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -7,8 +7,10 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "unicode" + "unicode/utf8" ) const ( @@ -68,12 +70,59 @@ var sensitiveKeys = map[string]struct{}{ "zero_api_key": {}, } +// ctrlGap matches C0/C1 bytes (Cc other than tab/LF/CR, plus lone Latin-1 C1) +// between characters of a secret shape. Matching stays on the original string: +// a deleted control is never a join, so \b still treats wordchar+control as a +// boundary and tokens that were never adjacent stay that way. Tab/LF/CR are +// excluded so log line structure is unchanged. \x{FFFD} lets the regexp locate +// a lone invalid UTF-8 byte; validSecretControlGaps subsequently accepts only +// raw C1 bytes and rejects a real, valid UTF-8 U+FFFD rune. +const ctrlGap = `[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f\x{FFFD}]*` + +// ctrlLit quotes s as a regexp literal with ctrlGap strictly between runes, so +// a NUL/ESC/C1 may split the literal without letting a match end on a gap. +func ctrlLit(s string) string { + var b strings.Builder + b.Grow(len(s) * (1 + len(ctrlGap))) + first := true + for _, r := range s { + if !first { + b.WriteString(ctrlGap) + } + b.WriteString(regexp.QuoteMeta(string(r))) + first = false + } + return b.String() +} + +func ctrlJoin(parts ...string) string { + return strings.Join(parts, ctrlGap) +} + +// secretBody generates a regex matching at least minimum body characters, +// allowing C0/C1 control gaps between any characters. It always starts and ends +// on a class character (never on a gap). +func secretBody(class string, minimum int, unbounded bool) string { + if minimum <= 0 { + return "" + } + quantifier := strconv.Itoa(minimum - 1) + if unbounded { + return class + `(?:` + ctrlGap + class + `){` + quantifier + `,}` + } + return class + `(?:` + ctrlGap + class + `){` + quantifier + `}` +} + // openaiKeyPattern mirrors secrets.Scan's broad sk- body. Known OpenAI // prefixes (sk-proj-/sk-svcacct-/sk-admin-) are always redacted; other sk- // digit-free matches with an interior hyphen are left alone (kebab-case false // positives), while digit-free legacy sk- credentials are still redacted. // Applied via ReplaceAllStringFunc rather than the plain list below. -var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) +var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("sk-"), secretBody(`[A-Za-z0-9_-]`, 20, true))) + +// plainOpenaiKeyPattern is the non-gap-aware counterpart of openaiKeyPattern, +// used for boundary resolution on logical (control-stripped) candidates. +var plainOpenaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) // textSecretPatterns mirror secrets.Scan for end-boundary behavior and the // shared high-confidence shapes. A leading \b keeps each pattern from firing @@ -84,7 +133,24 @@ var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`) // (not in secrets.Scan); ASIA temporary access keys are kept alongside AKIA. // openai keys are handled separately (digit filter). JWT has a strict form // (both segments start with eyJ) and a looser three-segment form. +// ctrlGap between shape characters keeps NUL/ESC/C1 split secrets matching +// without stripping those bytes out of the subject first. var textSecretPatterns = []*regexp.Regexp{ + regexp.MustCompile(`\b` + ctrlLit("sk-ant-") + ctrlGap + `(?:` + ctrlJoin(ctrlLit("api"), `\d`, `\d`, `-`) + ctrlGap + `)?` + secretBody(`[A-Za-z0-9_-]`, 20, true)), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("github_pat_"), secretBody(`[A-Za-z0-9_]`, 22, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("gh"), `[pousr]`, `_`, secretBody(`[A-Za-z0-9]`, 36, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("glpat-"), secretBody(`[A-Za-z0-9_-]`, 12, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("AIza"), secretBody(`[0-9A-Za-z\-_]`, 35, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("xox"), `[baprs]`, `-`, secretBody(`[A-Za-z0-9-]`, 10, true))), + regexp.MustCompile(`\b` + ctrlJoin(`(?:`+ctrlLit("AKIA")+`|`+ctrlLit("ASIA")+`)`, secretBody(`[A-Z0-9]`, 16, false))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), + regexp.MustCompile(`\b` + ctrlJoin(ctrlLit("eyJ"), secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true), `\.`, secretBody(`[A-Za-z0-9_-]`, 10, true))), +} + +// plainSecretPatterns are non-gap-aware counterparts of textSecretPatterns, +// used for boundary resolution on logical (control-stripped) candidates. +// Each entry corresponds 1:1 with textSecretPatterns by index. +var plainSecretPatterns = []*regexp.Regexp{ regexp.MustCompile(`\bsk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`), regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36,}`), @@ -170,8 +236,90 @@ func keyLooksSensitive(normalized string) bool { return false } +// stripControlBytes removes C0/C1 controls (Cc other than tab, LF, and CR). +// Used to normalize an already-matched secret so prefix/digit checks see the +// rejoined shape. It is matching-time only and must not be applied to +// RedactString's input or return value. Tab/LF/CR stay. Lone Latin-1 C1 bytes +// (0x80–0x9F, invalid UTF-8) are stripped too; UTF-8 continuation bytes are +// not, because they are not controls. +func stripControlBytes(s string) string { + for i := 0; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + return stripControlBytesFrom(s, i) + } + i++ + continue + } + if c <= 0x9F { + // 0x80–0x9F at a rune boundary is a lone C1 byte, not UTF-8. + return stripControlBytesFrom(s, i) + } + r, size := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(r) { + return stripControlBytesFrom(s, i) + } + i += size + } + return s +} + +func stripControlBytesFrom(s string, start int) string { + var b strings.Builder + b.Grow(len(s)) + b.WriteString(s[:start]) + for i := start; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + i++ + continue + } + b.WriteByte(c) + i++ + continue + } + if c <= 0x9F { + i++ + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(r) { + i += size + continue + } + b.WriteString(s[i : i+size]) + i += size + } + return b.String() +} + +// validSecretControlGaps disambiguates RuneError matches at the byte boundary. +// Go's regexp engine represents both a malformed single byte and a legitimate +// U+FFFD rune as RuneError. Only raw invalid C1 bytes (0x80-0x9F) are supported +// gaps; a valid UTF-8 replacement rune, or another malformed byte, must keep +// the candidate split and prevent redaction. +func validSecretControlGaps(match string) bool { + for i := 0; i < len(match); { + r, size := utf8.DecodeRuneInString(match[i:]) + if r != utf8.RuneError { + i += size + continue + } + if size != 1 || match[i] < 0x80 || match[i] > 0x9F { + return false + } + i++ + } + return true +} + func RedactString(value string, options Options) string { replacement := replacement(options) + // Match on the original string. Shape patterns allow C0/C1 gaps between + // characters so a split secret still matches; stripping first would join + // tokens that were never adjacent and make \b miss a leading wordchar. redacted := value if len(options.ExtraSecretValues) > 0 { secrets := append([]string{}, options.ExtraSecretValues...) @@ -224,21 +372,265 @@ func RedactString(value string, options Options) string { } return parts[1] + parts[2] + "=" + replacement }) - // openai keys first so the filter can drop kebab-case false positives - // before any other pattern rewrites nearby text. - redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { - if !knownOpenAIKeyPrefix(match) && !secretMatchHasDigit(match) && - strings.Contains(strings.TrimPrefix(match, "sk-"), "-") { - return match + // Match high-confidence specialized shapes first. In particular, the broad + // sk- pattern may reach its minimum before a control inside a longer + // Anthropic key; letting the Anthropic shape consume that split first avoids + // leaving a recognizable credential suffix behind. + for i, pattern := range textSecretPatterns { + plain := plainSecretPatterns[i] + minLen := minSecretLens[i] + redacted = replaceAllSecretMatches(redacted, i, pattern, plain, replacement, minLen, nil) + } + // Apply the broad OpenAI shape after specialized keys so its kebab-case + // false-positive filter considers only the matched key, never suffix text. + redacted = replaceAllSecretMatches(redacted, -1, openaiKeyPattern, plainOpenaiKeyPattern, replacement, minOpenAILen, func(m string) bool { + // m is the logical (control-stripped) candidate. + if !knownOpenAIKeyPrefix(m) && !secretMatchHasDigit(m) && + strings.Contains(strings.TrimPrefix(m, "sk-"), "-") { + return false } - return replacement + return true }) - for _, pattern := range textSecretPatterns { - redacted = pattern.ReplaceAllString(redacted, replacement) - } return redacted } +var minSecretLens = []int{ + 27, // sk-ant- (7) + 20 + 33, // github_pat_ (11) + 22 + 40, // gh[pousr]_ (4) + 36 + 18, // glpat- (6) + 12 + 39, // AIza (4) + 35 + 15, // xox[baprs]- (5) + 10 + 20, // AKIA/ASIA (4) + 16 + 38, // JWT (3 + 10 + 1 + 3 + 10 + 1 + 10) + 34, // JWT (3 + 10 + 1 + 10 + 1 + 10) +} + +const minOpenAILen = 23 // sk- (3) + 20 + +func isCandidateLength(logPre string, patternIndex int, minLen int) bool { + if len(logPre) < minLen { + return false + } + if patternIndex == 7 || patternIndex == 8 { + return strings.Count(logPre, ".") >= 2 + } + return true +} + +type controlSpan struct { + start int + end int + validGap bool +} + +type logicalCandidate struct { + logical string + origEnds []int + spans []controlSpan +} + +func extractLogicalCandidate(s string) logicalCandidate { + var logical strings.Builder + logical.Grow(len(s)) + var origEnds []int + origEnds = make([]int, 0, len(s)) + var spans []controlSpan + + for i := 0; i < len(s); { + c := s[i] + if c < 0x80 { + if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) { + start := i + for i < len(s) && s[i] < 0x80 && s[i] != '\t' && s[i] != '\n' && s[i] != '\r' && (s[i] < 0x20 || s[i] == 0x7F) { + i++ + } + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) + continue + } + logical.WriteByte(c) + i++ + origEnds = append(origEnds, i) + continue + } + if c >= 0x80 && c <= 0x9F { + start := i + for i < len(s) && s[i] >= 0x80 && s[i] <= 0x9F { + i++ + } + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError { + start := i + i += size + spans = append(spans, controlSpan{start: start, end: i, validGap: false}) + continue + } + if unicode.IsControl(r) && r != '\t' && r != '\n' && r != '\r' { + start := i + i += size + for i < len(s) { + nr, nsize := utf8.DecodeRuneInString(s[i:]) + if unicode.IsControl(nr) && nr != '\t' && nr != '\n' && nr != '\r' { + i += nsize + } else { + break + } + } + spans = append(spans, controlSpan{start: start, end: i, validGap: true}) + continue + } + logical.WriteRune(r) + i += size + runeLen := len(string(r)) + for b := 0; b < runeLen; b++ { + origEnds = append(origEnds, i) + } + } + return logicalCandidate{ + logical: logical.String(), + origEnds: origEnds, + spans: spans, + } +} + +func findCredentialBoundary(match string, patternIndex int, plainPattern *regexp.Regexp, minLen int, isValid func(string) bool) (int, bool) { + cand := extractLogicalCandidate(match) + if len(cand.spans) == 0 { + if isValid != nil && !isValid(match) { + return len(match), false + } + return len(match), true + } + + logicalStr := cand.logical + + // Fast path for OpenAI kebab false positives: if the logical string has no digits, + // is not a known prefix, and the token before the first control span already contains + // an interior hyphen, no sub-span can ever be valid. + if patternIndex == -1 && isValid != nil && len(cand.spans) > 0 && cand.spans[0].start > 0 { + if !knownOpenAIKeyPrefix(logicalStr) && !secretMatchHasDigit(logicalStr) { + firstLogLen := sort.SearchInts(cand.origEnds, cand.spans[0].start+1) + firstPre := logicalStr[:firstLogLen] + if strings.Contains(strings.TrimPrefix(firstPre, "sk-"), "-") { + return len(match), false + } + } + } + + for _, span := range cand.spans { + if span.start == 0 { + continue + } + // If scanning OpenAI keys and text after span starts with "sk-", + // this span is a delimiter before a new key. + if patternIndex == -1 && strings.HasPrefix(match[span.end:], "sk-") { + logLen := sort.SearchInts(cand.origEnds, span.start+1) + if isCandidateLength(logicalStr[:logLen], patternIndex, minLen) { + logPre := logicalStr[:logLen] + if plainPattern.MatchString(logPre) && (isValid == nil || isValid(logPre)) { + return span.start, true + } + } + return span.start, false + } + + logLen := sort.SearchInts(cand.origEnds, span.start+1) + if !isCandidateLength(logicalStr[:logLen], patternIndex, minLen) { + if !span.validGap { + return span.start, false + } + continue + } + logPre := logicalStr[:logLen] + if plainPattern.MatchString(logPre) && (isValid == nil || isValid(logPre)) { + return span.start, true + } + if !span.validGap { + return span.start, false + } + // If logPre is a kebab false positive, check if the remaining suffix in match + // has fewer than 4 characters. If so, logPre was an independent kebab token + // followed by a delimiter (e.g. "sk-my-kebab\x00v2/file.go"). + if isValid != nil && !isValid(logPre) { + suffixLen := len(logicalStr) - logLen + if suffixLen < 4 { + return span.start, false + } + } + } + + for _, span := range cand.spans { + if !span.validGap { + return span.start, false + } + } + + if !isCandidateLength(logicalStr, patternIndex, minLen) { + return len(match), false + } + + if !plainPattern.MatchString(logicalStr) { + return len(match), false + } + + if isValid != nil && !isValid(logicalStr) { + return len(match), false + } + + if len(cand.origEnds) > 0 { + lastEnd := cand.origEnds[len(cand.origEnds)-1] + return lastEnd, true + } + return len(match), true +} + +func replaceAllSecretMatches(src string, patternIndex int, pattern *regexp.Regexp, plainPattern *regexp.Regexp, replacement string, minLen int, isValid func(string) bool) string { + loc := pattern.FindStringIndex(src) + if loc == nil { + return src + } + + var b strings.Builder + b.Grow(len(src)) + + lastIndex := 0 + for { + loc := pattern.FindStringIndex(src[lastIndex:]) + if loc == nil { + b.WriteString(src[lastIndex:]) + break + } + + matchStart := lastIndex + loc[0] + matchEnd := lastIndex + loc[1] + match := src[matchStart:matchEnd] + + advanceLen, shouldRedact := findCredentialBoundary(match, patternIndex, plainPattern, minLen, isValid) + + b.WriteString(src[lastIndex:matchStart]) + if shouldRedact { + b.WriteString(replacement) + lastIndex = matchStart + advanceLen + } else { + if advanceLen <= 0 { + advanceLen = 1 + } + b.WriteString(src[matchStart : matchStart+advanceLen]) + lastIndex = matchStart + advanceLen + } + if lastIndex <= matchStart { + lastIndex = matchStart + 1 + } + if lastIndex >= len(src) { + break + } + } + return b.String() +} + // knownOpenAIKeyPrefix is the redaction-side twin of secrets.knownOpenAIKeyPrefix: // known OpenAI-issued forms redact even with an alphabet-only body. func knownOpenAIKeyPrefix(match string) bool { diff --git a/internal/redaction/redaction_test.go b/internal/redaction/redaction_test.go index 3ae228c8d..43d45b34a 100644 --- a/internal/redaction/redaction_test.go +++ b/internal/redaction/redaction_test.go @@ -141,3 +141,131 @@ func containsCircular(v any) bool { } return false } + +func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) { + // Unsplit passing is not coverage: a NUL/ESC/C1 in the body splits the + // shape so the patterns miss it unless matching allows those controls as + // gaps between body characters (without joining unrelated tokens). + const prefix = "sk-ant-api03-" + const body = "abcdefghijklmnopqrstuvwxyz" + unsplit := prefix + body + if got := RedactString(unsplit, Options{}); strings.Contains(got, body) { + t.Fatalf("unsplit secret not redacted (test setup): %q", got) + } + + cases := []struct { + name string + split string + }{ + {name: "NUL", split: "\x00"}, + {name: "ESC", split: "\x1b"}, + {name: "C1", split: "\x9b"}, + {name: "UTF-8 C1", split: string(rune(0x9B))}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + inputs := []struct { + placement string + input string + }{ + {placement: "prefix-body boundary", input: prefix + tc.split + body}, + {placement: "inside body", input: prefix + body[:13] + tc.split + body[13:]}, + } + for _, input := range inputs { + t.Run(input.placement, func(t *testing.T) { + got := RedactString(input.input, Options{}) + if strings.Contains(got, body) { + t.Fatalf("secret split by %s %s leaked in %q", tc.name, input.placement, got) + } + if strings.Contains(got, prefix) { + t.Fatalf("secret prefix split by %s %s leaked in %q", tc.name, input.placement, got) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("expected %q after %s %s split, got %q", RedactedSecret, tc.name, input.placement, got) + } + }) + } + }) + } +} + +func TestRedactStringPreservesControlAfterCredential(t *testing.T) { + const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + input := "key=" + secret + "\x00path/one.go\x00path/two.go" + want := "key=" + RedactedSecret + "\x00path/one.go\x00path/two.go" + if got := RedactString(input, Options{}); got != want { + t.Fatalf("terminal credential separator changed:\n got=%q\nwant=%q", got, want) + } +} + +func TestRedactStringSuffixCannotDisableOpenAIKeyMatch(t *testing.T) { + const secret = "sk-aaaaaaaaaaaaaaaaaaaabcdefgh" + input := "key " + secret + "\x1bkebab-case tail" + want := "key " + RedactedSecret + "\x1bkebab-case tail" + if got := RedactString(input, Options{}); got != want { + t.Fatalf("suffix changed OpenAI key classification:\n got=%q\nwant=%q", got, want) + } +} + +func TestRedactStringDistinguishesInvalidC1FromValidReplacementRune(t *testing.T) { + const prefix = "sk-ant-api03-" + const body = "abcdefghijklmnopqrstuvwxyz" + + invalidC1 := prefix + body[:13] + "\x9b" + body[13:] + if got := RedactString(invalidC1, Options{}); got != RedactedSecret { + t.Fatalf("raw invalid C1 split was not redacted: %q", got) + } + + validReplacement := prefix + body[:13] + "\uFFFD" + body[13:] + got := RedactString(validReplacement, Options{}) + if !strings.Contains(got, "\uFFFD"+body[13:]) { + t.Fatalf("valid U+FFFD was treated as a control gap: %q", got) + } +} + +func TestRedactStringPreservesAllowedWhitespaceAndUTF8(t *testing.T) { + input := "safe\tline\nnext\rfinal café" + if got := RedactString(input, Options{}); got != input { + t.Fatalf("unexpected normalization: %q", got) + } +} + +func TestRedactStringWordcharBeforeNULAnthropicKey(t *testing.T) { + // Matching on a control-stripped copy joins "id42" and the key, so \b in + // textSecretPatterns misses and the secret leaks. Matching on the original + // treats the NUL as a boundary; leaked must be false. + const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + if got := RedactString(secret, Options{}); strings.Contains(got, "sk-ant-api03-") { + t.Fatalf("unsplit secret not redacted (test setup): %q", got) + } + input := "id42\x00" + secret + got := RedactString(input, Options{}) + leaked := strings.Contains(got, secret) || strings.Contains(got, "sk-ant-api03-") + if leaked { + t.Fatalf("wordchar-before-NUL+anthropic-key leaked=true out=%q", got) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("wordchar-before-NUL+anthropic-key leaked=false want %q, got %q", RedactedSecret, got) + } +} + +func TestRedactStringControlBytesWithoutSecretStayIdentical(t *testing.T) { + // scrubResultSecrets sets Result.Redacted when RedactString's result != + // Output. Stripping is matching-time only: no-secret control bytes must + // remain byte-identical so Redacted stays false. + cases := []struct { + name string + input string + }{ + {name: "form feed in source", input: "package main\n\ffunc main() {}\n"}, + {name: "Windows-1252 quotes", input: "Don\x92t \x93quote\x94 me\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := RedactString(tc.input, Options{}) + if got != tc.input { + t.Fatalf("no-secret input not byte-identical:\n in=%q\nout=%q", tc.input, got) + } + }) + } +} diff --git a/internal/redaction/split_harness_test.go b/internal/redaction/split_harness_test.go new file mode 100644 index 000000000..f6382de7f --- /dev/null +++ b/internal/redaction/split_harness_test.go @@ -0,0 +1,304 @@ +package redaction + +import ( + "strings" + "testing" +) + +func TestSplitRedactionHarness(t *testing.T) { + // Representative secrets for all supported shapes + secrets := []struct { + name string + secret string + }{ + {"Anthropic", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz1234"}, + {"OpenAI standard", "sk-abcdefghijklmnopqrstuvwxyz12345678"}, + {"OpenAI with hyphen and digit", "sk-aaaaaaaaaa-bbbbbbbbb1234567890"}, + {"OpenAI proj", "sk-proj-abcdefghijklmnopqrstuvwxyz12345"}, + {"GitHub PAT", "github_pat_11AAAAAAA0123456789abcdefghijklmnopqrstuvwxyz"}, + {"GitHub Fine-Grained", "ghp_123456789012345678901234567890123456"}, + {"GitLab PAT", "glpat-12345678901234567890"}, + {"Google API", "AIzaSyD-1234567890123456789012345678901"}, + {"Slack bot", "xoxb-123456789012-abcdefghijklmno"}, + {"AWS AKIA", "AKIAIOSFODNN7EXAMPLE"}, + {"JWT", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"}, + } + + controls := []struct { + name string + char string + }{ + {"NUL", "\x00"}, + {"ESC", "\x1b"}, + {"lone C1", "\x9b"}, + {"UTF-8 C1", "\u009b"}, + } + + for _, s := range secrets { + t.Run(s.name, func(t *testing.T) { + // First verify unsplit redacts + gotUnsplit := RedactString(s.secret, Options{}) + if strings.Contains(gotUnsplit, s.secret) || !strings.Contains(gotUnsplit, RedactedSecret) { + t.Fatalf("unsplit secret %q failed to redact: %q", s.secret, gotUnsplit) + } + + // Test split at all interior positions throughout the secret + for _, ctrl := range controls { + for pos := 1; pos < len(s.secret); pos++ { + splitSecret := s.secret[:pos] + ctrl.char + s.secret[pos:] + got := RedactString(splitSecret, Options{}) + + // Strip controls from output and assert original secret cannot be recovered + strippedOutput := stripControlBytes(got) + if strings.Contains(strippedOutput, s.secret) { + t.Fatalf("split at pos %d with %s leaked secret!\n split input=%q\n got=%q\n stripped=%q", pos, ctrl.name, splitSecret, got, strippedOutput) + } + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("split at pos %d with %s did not contain RedactedSecret!\n got=%q", pos, ctrl.name, got) + } + } + } + }) + } +} + +func TestSplitRedactionMultiControlCases(t *testing.T) { + t.Run("Internal gap before minimum then terminal delimiter", func(t *testing.T) { + input := "sk-ant-api03-\x00abcdefghijklmnopqrstuvwxyz\x00path/file.go" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00path/file.go" + if got != want { + t.Fatalf("multi-control anthropic mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("OpenAI internal gap then terminal delimiter before kebab suffix", func(t *testing.T) { + input := "sk-\x00abcdefghijklmnopqrstuv\x1bkebab-case tail" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x1bkebab-case tail" + if got != want { + t.Fatalf("multi-control openai mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("OpenAI internal gap before digit suffix then terminal delimiter", func(t *testing.T) { + input := "sk-aaaaaaaaaa-bbbbbbbbb\x001234567890\x00path/one.go" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00path/one.go" + if got != want { + t.Fatalf("multi-control openai with digits mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("JWT multiple internal gaps and terminal delimiter", func(t *testing.T) { + input := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\x00.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ\x1b.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\x00trailing/text" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00trailing/text" + if got != want { + t.Fatalf("multi-control jwt mismatch:\n got=%q\nwant=%q", got, want) + } + }) + + t.Run("Multiple internal controls in credential body", func(t *testing.T) { + input := "sk-ant-\x00api03-\x1babcdefghijklmnopqrstuvwxyz" + got := RedactString(input, Options{}) + if got != RedactedSecret { + t.Fatalf("multiple internal gaps in anthropic key mismatch: got %q, want %q", got, RedactedSecret) + } + }) + + t.Run("Terminal delimiter separating two credentials", func(t *testing.T) { + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz\x00ghp_123456789012345678901234567890123456" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00" + RedactedSecret + if got != want { + t.Fatalf("two credentials separated by delimiter mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Two same-shape keys separated by control bytes", func(t *testing.T) { + keyPairs := []struct { + name string + key1 string + key2 string + }{ + {"OpenAI", "sk-aaaaaaaaaaaaaaaaaaaabcdefgh", "sk-bbbbbbbbbbbbbbbbbbbbcdefghi"}, + {"GitHub Fine-Grained", "ghp_123456789012345678901234567890123456", "ghp_abcdefghijklmnopqrstuvwxyz1234567890"}, + {"GitHub PAT", "github_pat_11AAAAAAA0123456789abcdefghijklmnopqrstuvwxyz", "github_pat_22BBBBBBB0123456789abcdefghijklmnopqrstuvwxyz"}, + {"GitLab PAT", "glpat-12345678901234567890", "glpat-abcdefghijklmnopqrst"}, + {"Google API", "AIzaSyD-1234567890123456789012345678901", "AIzaSyD-abcdefghijklmnopqrstuvwxyz12345"}, + {"Slack", "xox" + "b-123456789012-abcdefghijklmno", "xox" + "b-987654321098-zyxwvutsrqponml"}, + } + ctrls := []string{"\x00", "\x1b", "\x9b", "\u009b"} + for _, pair := range keyPairs { + for _, ctrl := range ctrls { + input := pair.key1 + ctrl + pair.key2 + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + RedactedSecret + if got != want { + t.Fatalf("two %s keys separated by %q mismatch: got %q, want %q", pair.name, ctrl, got, want) + } + } + } + }) + + t.Run("Three same-shape keys separated by control bytes", func(t *testing.T) { + input := "sk-aaaaaaaaaaaaaaaaaaaabcdefgh\x00sk-bbbbbbbbbbbbbbbbbbbbcdefghi\x1bsk-ccccccccccccccccccccdefghij" + got := RedactString(input, Options{}) + want := RedactedSecret + "\x00" + RedactedSecret + "\x1b" + RedactedSecret + if got != want { + t.Fatalf("three OpenAI keys mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Short sk- token before credential", func(t *testing.T) { + input := "sk-ab\x00sk-aaaaaaaaaaaaaaaaaaaabcdefgh" + got := RedactString(input, Options{}) + want := "sk-ab\x00" + RedactedSecret + if got != want { + t.Fatalf("short sk- token before credential mismatch: got %q, want %q", got, want) + } + }) + + t.Run("OpenAI kebab false positive before path with digit", func(t *testing.T) { + input := "sk-my-awesome-kebab-project\x00v2/file.go" + got := RedactString(input, Options{}) + want := "sk-my-awesome-kebab-project\x00v2/file.go" + if got != want { + t.Fatalf("kebab project before path with digit mismatch: got %q, want %q", got, want) + } + }) + + t.Run("Complete credential followed by invalid bytes", func(t *testing.T) { + cases := []struct { + name string + suffix string + }{ + {"valid U+FFFD", "\uFFFDsuffix"}, + {"malformed byte 0xFF", "\xffsuffix"}, + {"malformed byte 0xC0", "\xc0suffix"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + input := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + tc.suffix + got := RedactString(input, Options{}) + want := RedactedSecret + tc.suffix + if got != want { + t.Fatalf("suffix %s mismatch: got %q, want %q", tc.name, got, want) + } + }) + } + }) +} + +func TestSplitRedactionNegativeCases(t *testing.T) { + controls := []string{"\x00", "\x1b", "\x9b", "\u009b"} + + t.Run("OpenAI kebab false positive with control", func(t *testing.T) { + kebab := "sk-my-awesome-kebab-project" + for _, ctrl := range controls { + input := kebab[:10] + ctrl + kebab[10:] + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("digit-free kebab falsely redacted with control %q: got %q, want %q", ctrl, got, input) + } + } + }) + + t.Run("Control immediately before complete credential", func(t *testing.T) { + secret := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + for _, ctrl := range controls { + input := "prefix" + ctrl + secret + got := RedactString(input, Options{}) + want := "prefix" + ctrl + RedactedSecret + if got != want { + t.Fatalf("control before secret mutated boundary: got %q, want %q", got, want) + } + } + }) + + t.Run("Control immediately after complete credential", func(t *testing.T) { + secret := "sk-ant-api03-abcdefghijklmnopqrstuvwxyz" + for _, ctrl := range controls { + input := secret + ctrl + "suffix" + got := RedactString(input, Options{}) + want := RedactedSecret + ctrl + "suffix" + if got != want { + t.Fatalf("control after secret mutated delimiter: got %q, want %q", got, want) + } + } + }) +} + +func TestSplitRedactionLinearScaling(t *testing.T) { + sizes := []int{8 * 1024, 16 * 1024, 32 * 1024, 64 * 1024, 128 * 1024, 800 * 1024} + + t.Run("OpenAI kebab repeated gaps scaling", func(t *testing.T) { + for _, size := range sizes { + // Construct "sk-kebab-" + repeated "\x00a" up to size + var b strings.Builder + b.WriteString("sk-kebab-") + for b.Len() < size { + b.WriteString("\x00a") + } + input := b.String() + got := RedactString(input, Options{}) + if got != input { + t.Fatalf("kebab false positive was falsely redacted at size %d", size) + } + } + }) + + t.Run("JWT repeated gaps scaling and correct redaction", func(t *testing.T) { + for _, size := range sizes { + // Construct "eyJ" + repeated "\x00a" + ".eyJ" + repeated "\x00b" + ".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" + segLen := size / 2 + var b strings.Builder + b.WriteString("eyJ") + for b.Len() < segLen { + b.WriteString("\x00a") + } + b.WriteString(".eyJ") + for b.Len() < size { + b.WriteString("\x00b") + } + b.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := b.String() + got := RedactString(input, Options{}) + if !strings.Contains(got, RedactedSecret) { + t.Fatalf("JWT at size %d failed to redact", size) + } + } + }) +} + +func BenchmarkRedactOpenAIKebabGaps128KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("sk-kebab-") + for builder.Len() < 128*1024 { + builder.WriteString("\x00a") + } + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +} + +func BenchmarkRedactJWTGaps128KB(b *testing.B) { + var builder strings.Builder + builder.WriteString("eyJ") + for builder.Len() < 64*1024 { + builder.WriteString("\x00a") + } + builder.WriteString(".eyJ") + for builder.Len() < 128*1024 { + builder.WriteString("\x00b") + } + builder.WriteString(".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c") + input := builder.String() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = RedactString(input, Options{}) + } +}