From d9ac429ab54ac3f87d77855d003a153e52bef68f Mon Sep 17 00:00:00 2001 From: alyldas Date: Mon, 29 Jun 2026 17:25:38 +0300 Subject: [PATCH 1/4] Align email scanner with shared hints --- README.md | 14 +++- docs/architecture.md | 47 +++++++++----- scripts/benchmark-email.mjs | 25 ++++++- src/index.ts | 4 ++ src/scanner.ts | 126 ++++++++++++++++++++++++++++++++++-- src/types.ts | 14 ++++ tests/scanner.spec.ts | 61 +++++++++++++++++ 7 files changed, 267 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2eebd8f..b88e586 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,15 @@ import { createEmailScanner } from "@textfilters/email"; const scanner = createEmailScanner(); const codePoints = Array.from("contact user@example.com"); const result = scanner.scan({ text: "contact user@example.com", codePoints }); +const hasEmail = scanner.check({ + text: "contact user@example.com", + codePoints, +}); + +scanner.scan({ text: "contact user@example.com", codePoints }, (match) => { + console.log(match.range); + return false; +}); ``` The default shared instance is exported as `filter`. It has stable `name: "email"`. @@ -98,8 +107,9 @@ False-positive guards avoid package scopes, social handles, prose-only `at` and See [docs/architecture.md](docs/architecture.md) for the scanner flow, module map, and change guide. -Run `npm run benchmark:email` from this package to compare clean, direct, -obfuscated, and late-match email censoring cases on the same machine. +Run `npm run benchmark:email` from this package to compare scanner setup, +`check()`, clean, no-dot `@` text, direct, obfuscated, and late-match email +censoring cases on the same machine. ## Related Textfilters Packages diff --git a/docs/architecture.md b/docs/architecture.md index 51b091f..ddec3da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,11 @@ The package provides composable email censoring for direct addresses and common The default `filter` export is a shared instance with default domain rules. `emailFilter(options?)` is an alias for `createEmailFilter(options?)`. -`createEmailScanner(options?)` exposes the same matching behavior as a range scanner. `scanEmailRanges(text, options?)` returns code point ranges directly for callers that want to compose masking through `@textfilters/core`. +`createEmailScanner(options?)` exposes the same matching behavior as a range +scanner. It supports `check(input)` for boolean checks and `scan(input, sink)` +for allocation-aware range streaming with early stop support. `scanEmailRanges` +returns code point ranges directly for callers that want to compose masking +through `@textfilters/core`. `EmailFilterOptions` supports: @@ -65,27 +69,36 @@ graph TD ## File Responsibilities -| File or directory | Responsibility | Out of scope | -| --------------------------------------- | ------------------------------------------------------------------------------ | ----------------------------------- | -| `src/index.ts` | Public entrypoint exports. | Scanner details. | -| `src/types.ts` | Public constants, options, filter types, and scanner contract types. | Internal token and metadata shapes. | -| `src/normalization.ts` | Code point metadata, NFKC lowercase folding, and character predicates. | Email matching policy. | -| `src/scanner.ts` | Scanner factory, cheap prefilter, option normalization, and final range merge. | Tokenization and rule internals. | -| `src/scanner/matching/direct.ts` | Direct literal candidate discovery around `@`. | Exclusions and boundary policy. | -| `src/scanner/matching/obfuscated.ts` | Optional obfuscated candidate discovery through the token stream. | Public API construction or masking. | -| `src/scanner/matching/candidate.ts` | Shared candidate validation, exclusions, and surrounding-boundary checks. | Candidate discovery. | -| `src/scanner/context/bare-at-policy.ts` | Bare `at` phrase policy for prose, labels, and accepted address lists. | Direct `user@example.com` matching. | -| `src/scanner/context` | Introducer, label, phrase, and address-list helpers for bare `at` policy. | Direct `user@example.com` matching. | -| `src/scanner/rules` | Local/domain validation, scanner lexicon, and exclusions. | Metadata construction. | -| `src/scanner/core/types.ts` | Shared internal token, punctuation, candidate, and scanner option types. | Public types. | -| `src/filter.ts` | Factory, shared instance, alias, and code point masking orchestration. | Tokenization and domain validation. | -| `tests/*.spec.ts` | Public behavior grouped by matching, contexts, prose guards, and integration. | Exhaustive RFC email validation. | +| File or directory | Responsibility | Out of scope | +| --------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `src/index.ts` | Public entrypoint exports. | Scanner details. | +| `src/types.ts` | Public constants, options, filter types, and scanner contract types. | Internal token and metadata shapes. | +| `src/normalization.ts` | Code point metadata, NFKC lowercase folding, and character predicates. | Email matching policy. | +| `src/scanner.ts` | Scanner factory, cheap prefilter, boolean checks, sink streaming, option normalization, and final range merge. | Tokenization and rule internals. | +| `src/scanner/matching/direct.ts` | Direct literal candidate discovery around `@`. | Exclusions and boundary policy. | +| `src/scanner/matching/obfuscated.ts` | Optional obfuscated candidate discovery through the token stream. | Public API construction or masking. | +| `src/scanner/matching/candidate.ts` | Shared candidate validation, exclusions, and surrounding-boundary checks. | Candidate discovery. | +| `src/scanner/context/bare-at-policy.ts` | Bare `at` phrase policy for prose, labels, and accepted address lists. | Direct `user@example.com` matching. | +| `src/scanner/context` | Introducer, label, phrase, and address-list helpers for bare `at` policy. | Direct `user@example.com` matching. | +| `src/scanner/rules` | Local/domain validation, scanner lexicon, and exclusions. | Metadata construction. | +| `src/scanner/core/types.ts` | Shared internal token, punctuation, candidate, and scanner option types. | Public types. | +| `src/filter.ts` | Factory, shared instance, alias, and code point masking orchestration. | Tokenization and domain validation. | +| `tests/*.spec.ts` | Public behavior grouped by matching, contexts, prose guards, and integration. | Exhaustive RFC email validation. | ## Scanner Flow Direct email scanning searches for normalized `@` characters, expands a local part to the left, expands a domain to the right, and emits an internal direct candidate. This path does not tokenize text and remains the small path for literal `user@example.com` matching. -Before metadata creation, the scanner checks for cheap direct or obfuscated email candidate signals. Clearly clean text returns no ranges without tokenization; candidate text still runs through the same direct, obfuscated, exclusion, and false-positive guards as before. +Before metadata creation, the scanner checks for cheap direct or obfuscated +email candidate signals. Shared-style hints such as `hasAtSign`, `hasDot`, +`hasNonAscii`, and text length can reject clearly clean input before metadata +allocation. Candidate text still runs through the same direct, obfuscated, +exclusion, and false-positive guards as before. + +Boolean `check()` scans direct `@` windows first and stops after the first valid +candidate. Sink-based `scan(input, sink)` emits ranges as they are found and +stops when the sink returns `false`; the legacy `scan(input)` result shape is +preserved. Obfuscated scanning is isolated under `src/scanner/matching/obfuscated.ts` and only runs when `matchObfuscated` is not `false`. It tokenizes words and separator tokens while ignoring whitespace around bracketed separators. It accepts `at`, `dot`, `@`, and `.` separator forms when they appear between a plausible local part and a valid dotted domain, then emits an internal obfuscated candidate. diff --git a/scripts/benchmark-email.mjs b/scripts/benchmark-email.mjs index c2f4f4e..c06a2d7 100644 --- a/scripts/benchmark-email.mjs +++ b/scripts/benchmark-email.mjs @@ -1,11 +1,12 @@ import { performance } from "node:perf_hooks"; -import { createEmailFilter } from "../dist/index.js"; +import { createEmailFilter, createEmailScanner } from "../dist/index.js"; const ITERATIONS = 1_000; const SETUP_ITERATIONS = 100; const SHORT_CLEAN = "Hello world"; const LONG_CLEAN = "The quick brown fox jumps over the lazy dog. ".repeat(50); +const NO_DOT_AT = "Contact user@example for details"; const DIRECT_EMAIL = "Contact user@example.com for details"; const OBFUSCATED_EMAIL = "Contact user [at] example [dot] com for details"; const LATE_MATCH = @@ -38,11 +39,33 @@ function printResults(results) { } const filter = createEmailFilter(); +const scanner = createEmailScanner(); +const input = (text) => ({ text, codePoints: Array.from(text) }); +const hintedInput = (text) => ({ + text, + codePoints: Array.from(text), + hints: { + textLength: text.length, + hasNonAscii: /[^\x00-\x7f]/u.test(text), + hasAtSign: text.includes("@"), + hasDot: text.includes("."), + }, +}); printResults([ bench("createEmailFilter()", () => createEmailFilter(), SETUP_ITERATIONS), + bench("createEmailScanner()", () => createEmailScanner(), SETUP_ITERATIONS), + bench("check short clean", () => scanner.check(input(SHORT_CLEAN))), + bench("check hinted short clean", () => + scanner.check(hintedInput(SHORT_CLEAN)), + ), + bench("check long clean", () => scanner.check(input(LONG_CLEAN))), + bench("check no-dot at text", () => scanner.check(input(NO_DOT_AT))), + bench("check direct email", () => scanner.check(input(DIRECT_EMAIL))), + bench("check late-match email", () => scanner.check(input(LATE_MATCH))), bench("censor short clean", () => filter.censor(SHORT_CLEAN)), bench("censor long clean", () => filter.censor(LONG_CLEAN)), + bench("censor no-dot at text", () => filter.censor(NO_DOT_AT)), bench("censor direct email", () => filter.censor(DIRECT_EMAIL)), bench("censor obfuscated email", () => filter.censor(OBFUSCATED_EMAIL)), bench("censor late-match email", () => filter.censor(LATE_MATCH)), diff --git a/src/index.ts b/src/index.ts index 065ba9e..4a7412e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,8 @@ export { createEmailFilter, emailFilter, filter } from "./filter.js"; export { EMAIL_FILTER_NAME, + type EmailRangeMatch, + type EmailRangeMatchSink, type EmailRangeScanner, type EmailRangeScanResult, type EmailFilter, @@ -8,8 +10,10 @@ export { type EmailScanInput, } from "./types.js"; export { + checkEmailRanges, collectEmailRanges, createEmailScanner, + scanEmailRangeMatches, scanEmailRanges, type EmailScannerConfig, } from "./scanner.js"; diff --git a/src/scanner.ts b/src/scanner.ts index 29c585c..169b9d7 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -12,7 +12,9 @@ import { TOKEN_VALUE, type ScannerOptions } from "./scanner/core/types.js"; import { EMAIL_FILTER_NAME, type EmailFilterOptions, + type EmailRangeMatchSink, type EmailRangeScanner, + type EmailScanInput, } from "./types.js"; export interface EmailScannerConfig extends EmailFilterOptions {} @@ -22,13 +24,26 @@ export function createEmailScanner( ): EmailRangeScanner { const scannerOptions = createScannerOptions(config); - return { - name: EMAIL_FILTER_NAME, - scan(input) { + function scan(input: EmailScanInput): { + ranges: readonly TextCodePointRange[]; + }; + function scan(input: EmailScanInput, sink: EmailRangeMatchSink): boolean; + function scan(input: EmailScanInput, sink?: EmailRangeMatchSink) { + if (sink === undefined) { return { ranges: scanEmailRangesWithOptions(input.text, scannerOptions), }; + } + + return scanEmailRangeMatchesWithOptions(input, scannerOptions, sink); + } + + return { + name: EMAIL_FILTER_NAME, + check(input) { + return checkEmailRangesWithOptions(input, scannerOptions); }, + scan, }; } @@ -66,7 +81,9 @@ function scanEmailRangesWithOptions( value: string, scannerOptions: ScannerOptions, ): readonly TextCodePointRange[] { - if (!value || !hasEmailCandidate(value, scannerOptions)) { + if ( + !hasEmailCandidateInput({ text: value, codePoints: [] }, scannerOptions) + ) { return []; } @@ -92,6 +109,103 @@ function scanEmailRangesWithOptions( return mergeCodePointRanges(ranges); } +export function checkEmailRanges( + input: EmailScanInput, + options: EmailFilterOptions = {}, +): boolean { + return checkEmailRangesWithOptions(input, createScannerOptions(options)); +} + +export function scanEmailRangeMatches( + input: EmailScanInput, + sink: EmailRangeMatchSink, + options: EmailFilterOptions = {}, +): boolean { + return scanEmailRangeMatchesWithOptions( + input, + createScannerOptions(options), + sink, + ); +} + +function checkEmailRangesWithOptions( + input: EmailScanInput, + scannerOptions: ScannerOptions, +): boolean { + if (!hasEmailCandidateInput(input, scannerOptions)) return false; + + const meta = createEmailTextMeta(input.text); + for (let i = 0; i < meta.normalized.length; i++) { + if (meta.normalized[i] !== TOKEN_VALUE.atSymbol) continue; + if (collectDirectEmailRange(meta, i, scannerOptions)) return true; + } + + return ( + scannerOptions.matchObfuscated && + collectObfuscatedEmailRanges(meta, scannerOptions).length > 0 + ); +} + +function scanEmailRangeMatchesWithOptions( + input: EmailScanInput, + scannerOptions: ScannerOptions, + sink: EmailRangeMatchSink, +): boolean { + if (!hasEmailCandidateInput(input, scannerOptions)) return true; + + const meta = createEmailTextMeta(input.text); + const emitted: TextCodePointRange[] = []; + const emit = (range: TextCodePointRange): boolean => { + if ( + emitted.some( + (existing) => existing[0] === range[0] && existing[1] === range[1], + ) + ) { + return true; + } + + emitted.push(range); + return sink({ range }) !== false; + }; + + for (let i = 0; i < meta.normalized.length; i++) { + if (meta.normalized[i] !== TOKEN_VALUE.atSymbol) continue; + const range = collectDirectEmailRange(meta, i, scannerOptions); + if (range) { + if (!emit(range)) return false; + i = range[1] - 1; + } + } + + if (scannerOptions.matchObfuscated) { + for (const range of collectObfuscatedEmailRanges(meta, scannerOptions)) { + if (!emit(range)) return false; + } + } + + return true; +} + +function hasEmailCandidateInput( + input: EmailScanInput, + scannerOptions: ScannerOptions, +): boolean { + if (!input.text) return false; + + const hints = input.hints; + if ( + hints !== undefined && + hints.hasAtSign === false && + hints.hasDot === false && + hints.hasNonAscii === false && + !hasObfuscatedEmailWordCandidate(input.text) + ) { + return false; + } + + return hasEmailCandidate(input.text, scannerOptions); +} + function hasEmailCandidate( value: string, scannerOptions: ScannerOptions, @@ -109,6 +223,10 @@ function hasEmailCandidate( ); } +function hasObfuscatedEmailWordCandidate(value: string): boolean { + return /[aA][tT]|[dD][oO0][tT]/u.test(value); +} + function hasWordCandidate(value: string, word: string): boolean { for ( let index = value.indexOf(word); diff --git a/src/types.ts b/src/types.ts index 1aadcc3..ccfada7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -20,13 +20,27 @@ export interface EmailFilter { export interface EmailScanInput { readonly text: string; readonly codePoints: readonly string[]; + readonly hints?: { + readonly textLength?: number; + readonly hasNonAscii?: boolean; + readonly hasAtSign?: boolean; + readonly hasDot?: boolean; + }; } export interface EmailRangeScanResult { readonly ranges: readonly TextCodePointRange[]; } +export interface EmailRangeMatch { + readonly range: TextCodePointRange; +} + +export type EmailRangeMatchSink = (match: EmailRangeMatch) => boolean | void; + export interface EmailRangeScanner { readonly name: typeof EMAIL_FILTER_NAME; + check(input: EmailScanInput): boolean; scan(input: EmailScanInput): EmailRangeScanResult; + scan(input: EmailScanInput, sink: EmailRangeMatchSink): boolean | void; } diff --git a/tests/scanner.spec.ts b/tests/scanner.spec.ts index a7abc79..5d4ae6d 100644 --- a/tests/scanner.spec.ts +++ b/tests/scanner.spec.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; import { + checkEmailRanges, createEmailScanner, + scanEmailRangeMatches, scanEmailRanges, EMAIL_FILTER_NAME, } from "../src/index.js"; @@ -43,6 +45,65 @@ describe("@textfilters/email scanner", () => { ).toEqual({ ranges: [] }); }); + it("checks candidates without collecting every range", () => { + const scanner = createEmailScanner(); + const text = "contact first@example.com and second@example.com"; + const input = { text, codePoints: Array.from(text) }; + + expect(scanner.check(input)).toBe(true); + expect(checkEmailRanges(input)).toBe(true); + expect(scanner.check({ text: "plain words only", codePoints: [] })).toBe( + false, + ); + }); + + it("streams ranges into a sink and supports early stop", () => { + const scanner = createEmailScanner(); + const text = "contact first@example.com and second@example.com"; + const seen: Array = []; + + const completed = scanner.scan( + { text, codePoints: Array.from(text) }, + (match) => { + seen.push(match.range); + return false; + }, + ); + + expect(completed).toBe(false); + expect(seen).toEqual([[8, 25]]); + }); + + it("uses shared-style hints to skip clearly clean text", () => { + expect( + checkEmailRanges({ + text: "plain words only", + codePoints: Array.from("plain words only"), + hints: { + textLength: "plain words only".length, + hasNonAscii: false, + hasAtSign: false, + hasDot: false, + }, + }), + ).toBe(false); + }); + + it("streams direct, punctuation-trimmed, and obfuscated ranges", () => { + const text = "mail user@example.com, then admin [at] example [dot] org."; + const seen: Array = []; + + expect( + scanEmailRangeMatches({ text, codePoints: Array.from(text) }, (match) => { + seen.push(match.range); + }), + ).toBe(true); + expect(seen).toEqual([ + [5, 21], + [28, 56], + ]); + }); + it("preserves scanner option behavior", () => { expect( scanEmailRanges("contact user at example dot com", { From ed200930d8eca07a510bb25c1c3fa702942c9845 Mon Sep 17 00:00:00 2001 From: alyldas Date: Mon, 29 Jun 2026 17:55:36 +0300 Subject: [PATCH 2/4] Stream email scanner matches in source order --- src/scanner.ts | 13 +++++++++---- tests/scanner.spec.ts | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/scanner.ts b/src/scanner.ts index 169b9d7..a0ba7ef 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -154,6 +154,7 @@ function scanEmailRangeMatchesWithOptions( if (!hasEmailCandidateInput(input, scannerOptions)) return true; const meta = createEmailTextMeta(input.text); + const ranges: TextCodePointRange[] = []; const emitted: TextCodePointRange[] = []; const emit = (range: TextCodePointRange): boolean => { if ( @@ -172,15 +173,19 @@ function scanEmailRangeMatchesWithOptions( if (meta.normalized[i] !== TOKEN_VALUE.atSymbol) continue; const range = collectDirectEmailRange(meta, i, scannerOptions); if (range) { - if (!emit(range)) return false; + ranges.push(range); i = range[1] - 1; } } if (scannerOptions.matchObfuscated) { - for (const range of collectObfuscatedEmailRanges(meta, scannerOptions)) { - if (!emit(range)) return false; - } + ranges.push(...collectObfuscatedEmailRanges(meta, scannerOptions)); + } + + ranges.sort((left, right) => left[0] - right[0] || left[1] - right[1]); + + for (const range of ranges) { + if (!emit(range)) return false; } return true; diff --git a/tests/scanner.spec.ts b/tests/scanner.spec.ts index 5d4ae6d..2e3f026 100644 --- a/tests/scanner.spec.ts +++ b/tests/scanner.spec.ts @@ -74,6 +74,23 @@ describe("@textfilters/email scanner", () => { expect(seen).toEqual([[8, 25]]); }); + it("streams mixed direct and obfuscated ranges in source order", () => { + const scanner = createEmailScanner(); + const text = "user at example dot com then admin@example.org"; + const seen: Array = []; + + const completed = scanner.scan( + { text, codePoints: Array.from(text) }, + (match) => { + seen.push(match.range); + return false; + }, + ); + + expect(completed).toBe(false); + expect(seen).toEqual([[0, 23]]); + }); + it("uses shared-style hints to skip clearly clean text", () => { expect( checkEmailRanges({ From 8aaf42cf97fc251a465add7b37e9ec95c9f2e09d Mon Sep 17 00:00:00 2001 From: alyldas Date: Mon, 29 Jun 2026 18:21:24 +0300 Subject: [PATCH 3/4] Stream email matches with merged ranges --- src/scanner.ts | 132 ++++++++++++++++++++------ src/scanner/matching/obfuscated.ts | 146 ++++++++++++++++++----------- src/types.ts | 1 + tests/scanner.spec.ts | 17 ++++ 4 files changed, 213 insertions(+), 83 deletions(-) diff --git a/src/scanner.ts b/src/scanner.ts index a0ba7ef..7bdb918 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -6,7 +6,11 @@ import { import { createEmailTextMeta } from "./normalization.js"; import { collectDirectEmailRange } from "./scanner/matching/direct.js"; -import { collectObfuscatedEmailRanges } from "./scanner/matching/obfuscated.js"; +import { + collectObfuscatedEmailRangeMatches, + collectObfuscatedEmailRanges, + createObfuscatedEmailRangeCursor, +} from "./scanner/matching/obfuscated.js"; import { createExclusionSets } from "./scanner/rules/exclusions.js"; import { TOKEN_VALUE, type ScannerOptions } from "./scanner/core/types.js"; import { @@ -40,6 +44,7 @@ export function createEmailScanner( return { name: EMAIL_FILTER_NAME, + allocationAware: true, check(input) { return checkEmailRangesWithOptions(input, scannerOptions); }, @@ -140,10 +145,14 @@ function checkEmailRangesWithOptions( if (collectDirectEmailRange(meta, i, scannerOptions)) return true; } - return ( - scannerOptions.matchObfuscated && - collectObfuscatedEmailRanges(meta, scannerOptions).length > 0 - ); + if (!scannerOptions.matchObfuscated) return false; + + let found = false; + collectObfuscatedEmailRangeMatches(meta, scannerOptions, () => { + found = true; + return false; + }); + return found; } function scanEmailRangeMatchesWithOptions( @@ -154,41 +163,108 @@ function scanEmailRangeMatchesWithOptions( if (!hasEmailCandidateInput(input, scannerOptions)) return true; const meta = createEmailTextMeta(input.text); - const ranges: TextCodePointRange[] = []; - const emitted: TextCodePointRange[] = []; - const emit = (range: TextCodePointRange): boolean => { + return streamMergedEmailRanges(meta, scannerOptions, (range) => + sink({ range }), + ); +} + +interface RangeCursor { + next(): TextCodePointRange | null; +} + +function streamMergedEmailRanges( + meta: ReturnType, + scannerOptions: ScannerOptions, + sink: (range: TextCodePointRange) => boolean | void, +): boolean { + const direct = createDirectRangeCursor(meta, scannerOptions); + const obfuscated = scannerOptions.matchObfuscated + ? createObfuscatedRangeCursor(meta, scannerOptions) + : emptyRangeCursor(); + let nextDirect = direct.next(); + let nextObfuscated = obfuscated.next(); + let pending: TextCodePointRange | null = null; + + const takeNext = (): TextCodePointRange | null => { + if (nextDirect === null && nextObfuscated === null) return null; + if (nextDirect === null) { + const range = nextObfuscated; + nextObfuscated = obfuscated.next(); + return range; + } + if (nextObfuscated === null) { + const range = nextDirect; + nextDirect = direct.next(); + return range; + } if ( - emitted.some( - (existing) => existing[0] === range[0] && existing[1] === range[1], - ) + nextDirect[0] < nextObfuscated[0] || + (nextDirect[0] === nextObfuscated[0] && + nextDirect[1] <= nextObfuscated[1]) ) { - return true; + const range = nextDirect; + nextDirect = direct.next(); + return range; } - emitted.push(range); - return sink({ range }) !== false; + const range = nextObfuscated; + nextObfuscated = obfuscated.next(); + return range; }; - for (let i = 0; i < meta.normalized.length; i++) { - if (meta.normalized[i] !== TOKEN_VALUE.atSymbol) continue; - const range = collectDirectEmailRange(meta, i, scannerOptions); - if (range) { - ranges.push(range); - i = range[1] - 1; + for (let range = takeNext(); range !== null; range = takeNext()) { + if (pending === null) { + pending = range; + continue; } - } - if (scannerOptions.matchObfuscated) { - ranges.push(...collectObfuscatedEmailRanges(meta, scannerOptions)); + if (range[0] <= pending[1]) { + pending = [pending[0], Math.max(pending[1], range[1])]; + continue; + } + + if (sink(pending) === false) return false; + pending = range; } - ranges.sort((left, right) => left[0] - right[0] || left[1] - right[1]); + return pending === null || sink(pending) !== false; +} - for (const range of ranges) { - if (!emit(range)) return false; - } +function createDirectRangeCursor( + meta: ReturnType, + scannerOptions: ScannerOptions, +): RangeCursor { + let index = 0; - return true; + return { + next() { + for (; index < meta.normalized.length; index++) { + if (meta.normalized[index] !== TOKEN_VALUE.atSymbol) continue; + const range = collectDirectEmailRange(meta, index, scannerOptions); + if (range) { + index = range[1]; + return range; + } + } + + return null; + }, + }; +} + +function createObfuscatedRangeCursor( + meta: ReturnType, + scannerOptions: ScannerOptions, +): RangeCursor { + return createObfuscatedEmailRangeCursor(meta, scannerOptions); +} + +function emptyRangeCursor(): RangeCursor { + return { + next() { + return null; + }, + }; } function hasEmailCandidateInput( diff --git a/src/scanner/matching/obfuscated.ts b/src/scanner/matching/obfuscated.ts index d30094a..bd0d531 100644 --- a/src/scanner/matching/obfuscated.ts +++ b/src/scanner/matching/obfuscated.ts @@ -7,67 +7,103 @@ import { isValidLocal } from "../rules/validators.js"; import { tokenize } from "../tokenization/tokenizer.js"; import { isCandidateExcluded, isCandidateMatchable } from "./candidate.js"; +export interface EmailRangeCursor { + next(): TextCodePointRange | null; +} + export const collectObfuscatedEmailRanges = ( meta: EmailTextMeta, options: ScannerOptions, ): readonly TextCodePointRange[] => { - // Obfuscated matching needs surrounding token context because plain words - // around "at" are often prose rather than mailbox addresses. - const tokens = tokenize(meta); const ranges: TextCodePointRange[] = []; - const acceptedListLocalIndexes = new Set(); - for (let i = 0; i < tokens.length - 2; i++) { - const local = tokens[i]; - const at = tokens[i + 1]; - if (local.type !== TOKEN_TYPE.word || at.type !== TOKEN_TYPE.at) continue; - if (local.value.length < 3) continue; - if (!isValidLocal(local.value)) continue; - - const labels: string[] = []; - let cursor = i + 2; - if (tokens[cursor]?.type !== TOKEN_TYPE.word) continue; - labels.push(tokens[cursor].value); - cursor++; - - while ( - tokens[cursor]?.type === TOKEN_TYPE.dot && - tokens[cursor + 1]?.type === TOKEN_TYPE.word - ) { - labels.push(tokens[cursor + 1].value); - cursor += 2; - } - - if ( - isBareAtProsePhrase( - meta, - tokens, - i, - local, - at, - options, - acceptedListLocalIndexes, - ) - ) { - continue; - } - - const endToken = tokens[cursor - 1]; - const candidate = { - kind: "obfuscated", - local: local.value, - labels, - start: local.start, - end: endToken.end, - } as const; - if (!isCandidateMatchable(meta, candidate, options)) continue; - - acceptedListLocalIndexes.add(i); - if (!isCandidateExcluded(candidate, options)) { - ranges.push([candidate.start, candidate.end]); - } - i = cursor - 1; - } + collectObfuscatedEmailRangeMatches(meta, options, (range) => { + ranges.push(range); + }); return ranges; }; + +export const collectObfuscatedEmailRangeMatches = ( + meta: EmailTextMeta, + options: ScannerOptions, + visit: (range: TextCodePointRange) => boolean | void, +): boolean => { + const cursor = createObfuscatedEmailRangeCursor(meta, options); + + for (let range = cursor.next(); range !== null; range = cursor.next()) { + if (visit(range) === false) return false; + } + + return true; +}; + +export const createObfuscatedEmailRangeCursor = ( + meta: EmailTextMeta, + options: ScannerOptions, +): EmailRangeCursor => { + const tokens = tokenize(meta); + const acceptedListLocalIndexes = new Set(); + let index = 0; + + return { + next() { + for (; index < tokens.length - 2; index++) { + const localIndex = index; + const local = tokens[localIndex]; + const at = tokens[localIndex + 1]; + if (local.type !== TOKEN_TYPE.word || at.type !== TOKEN_TYPE.at) { + continue; + } + if (local.value.length < 3) continue; + if (!isValidLocal(local.value)) continue; + + const labels: string[] = []; + let cursor = localIndex + 2; + if (tokens[cursor]?.type !== TOKEN_TYPE.word) continue; + labels.push(tokens[cursor].value); + cursor++; + + while ( + tokens[cursor]?.type === TOKEN_TYPE.dot && + tokens[cursor + 1]?.type === TOKEN_TYPE.word + ) { + labels.push(tokens[cursor + 1].value); + cursor += 2; + } + + if ( + isBareAtProsePhrase( + meta, + tokens, + localIndex, + local, + at, + options, + acceptedListLocalIndexes, + ) + ) { + continue; + } + + const endToken = tokens[cursor - 1]; + const candidate = { + kind: "obfuscated", + local: local.value, + labels, + start: local.start, + end: endToken.end, + } as const; + if (!isCandidateMatchable(meta, candidate, options)) continue; + + acceptedListLocalIndexes.add(localIndex); + index = cursor; + if (!isCandidateExcluded(candidate, options)) { + return [candidate.start, candidate.end]; + } + } + + return null; + }, + }; +}; diff --git a/src/types.ts b/src/types.ts index ccfada7..b995889 100644 --- a/src/types.ts +++ b/src/types.ts @@ -40,6 +40,7 @@ export type EmailRangeMatchSink = (match: EmailRangeMatch) => boolean | void; export interface EmailRangeScanner { readonly name: typeof EMAIL_FILTER_NAME; + readonly allocationAware: true; check(input: EmailScanInput): boolean; scan(input: EmailScanInput): EmailRangeScanResult; scan(input: EmailScanInput, sink: EmailRangeMatchSink): boolean | void; diff --git a/tests/scanner.spec.ts b/tests/scanner.spec.ts index 2e3f026..d586dea 100644 --- a/tests/scanner.spec.ts +++ b/tests/scanner.spec.ts @@ -91,6 +91,23 @@ describe("@textfilters/email scanner", () => { expect(seen).toEqual([[0, 23]]); }); + it("merges overlapping direct and obfuscated ranges before streaming", () => { + const scanner = createEmailScanner(); + const text = "user@example.com dot org"; + const seen: Array = []; + + const completed = scanner.scan( + { text, codePoints: Array.from(text) }, + (match) => { + seen.push(match.range); + return false; + }, + ); + + expect(completed).toBe(false); + expect(seen).toEqual([[0, 24]]); + }); + it("uses shared-style hints to skip clearly clean text", () => { expect( checkEmailRanges({ From 3277c9f753823d00531ce10936c0a3b6e2b5632e Mon Sep 17 00:00:00 2001 From: alyldas Date: Mon, 29 Jun 2026 18:47:24 +0300 Subject: [PATCH 4/4] Resume obfuscated email scans after exclusions --- src/scanner/matching/obfuscated.ts | 9 ++++++--- tests/options-and-integration.spec.ts | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/scanner/matching/obfuscated.ts b/src/scanner/matching/obfuscated.ts index bd0d531..a65df37 100644 --- a/src/scanner/matching/obfuscated.ts +++ b/src/scanner/matching/obfuscated.ts @@ -97,10 +97,13 @@ export const createObfuscatedEmailRangeCursor = ( if (!isCandidateMatchable(meta, candidate, options)) continue; acceptedListLocalIndexes.add(localIndex); - index = cursor; - if (!isCandidateExcluded(candidate, options)) { - return [candidate.start, candidate.end]; + if (isCandidateExcluded(candidate, options)) { + index = cursor - 1; + continue; } + + index = cursor; + return [candidate.start, candidate.end]; } return null; diff --git a/tests/options-and-integration.spec.ts b/tests/options-and-integration.spec.ts index 134b0cc..e8c2955 100644 --- a/tests/options-and-integration.spec.ts +++ b/tests/options-and-integration.spec.ts @@ -100,6 +100,11 @@ describe("@textfilters/email options and integration", () => { "mail user at example dot com and admin at example dot com", ), ).toBe("mail user at example dot com and ************************"); + + const admin = "admin at example dot org"; + expect(configured.censor(`mail user at example dot com, ${admin}`)).toBe( + `mail user at example dot com, ${"*".repeat(admin.length)}`, + ); }); it("applies full-address exclusions consistently to direct and obfuscated candidates", () => {