Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions lib/humanizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { getSystemPrompt, getRehumanizePrompt, getCorpusAwareSystemPrompt } from
import { getCorpusCalibratedThresholds, hasStyleModel, loadStyleModelAsync } from './style-model';
import { generateWithProvider, getProvider, generateAlternatives } from './providers';
import { detectAI } from './detector';
import { postprocess, corpusAwarePostprocess } from './postprocess';
import { postprocess, corpusAwarePostprocess, normalizePunctuation } from './postprocess';
import { chunkText, countWords, addToHistory } from './storage';
import { extractRegions, restoreRegions, containsPlaceholders } from './protect-regions';

Expand Down Expand Up @@ -106,10 +106,16 @@ export async function humanizeText(
const targetScore = options.targetScore || calibratedThresholds?.targetScore || 80;
const maxPasses = options.level === 'ninja' ? 2 : options.level === 'aggressive' ? 2 : 1;

// Normalize typographic punctuation up front so curly quotes / ellipsis from
// the original input (often pasted from Word/Pages/macOS keyboards) don't
// survive inside protected regions. Code spans/fences are normalized too, but
// the substitutions only affect quotes, ellipsis, and NBSP, not code syntax.
const normalizedInput = normalizePunctuation(text);

// Extract structural regions (code, links, URLs, mentions, blockquotes, ...)
// before any processing. They survive the pipeline as opaque placeholders and
// are restored verbatim at the end.
const { masked, regions } = extractRegions(text);
const { masked, regions } = extractRegions(normalizedInput);
const chunks = chunkText(masked, 2500);

let humanizedText = '';
Expand Down
21 changes: 21 additions & 0 deletions lib/postprocess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,23 @@ function stripAIDashes(text: string): string {
.replace(new RegExp(RANGE_PLACEHOLDER, 'g'), '–');
}

// ==================== 2b''. NORMALIZE TYPOGRAPHIC PUNCTUATION ====================

// Curly quotes, smart apostrophes, and horizontal ellipsis are another common AI
// tell (and also a copy-paste hazard from Word/Pages/macOS keyboards). Normalize
// to ASCII so the body of the rewrite uses only `'`, `"`, and `...`. Code spans
// and fences are preserved upstream by the protect-regions pass; this only ever
// runs on prose.
export function normalizePunctuation(text: string): string {
return text
.replace(/[‘’‚‛]/g, "'") // curly singles, low-9, high-reversed-9
.replace(/[“”„‟]/g, '"') // curly doubles, low-9, high-reversed-9
.replace(/[‹›]/g, "'") // single guillemets
.replace(/[«»]/g, '"') // double guillemets
.replace(/…/g, '...') // horizontal ellipsis
.replace(/ /g, ' '); // non-breaking space
}

// ==================== 2c. PUNCTUATION & FORMATTING NOISE ====================

const CONTRACTIONS: [string, string][] = [
Expand Down Expand Up @@ -700,6 +717,10 @@ export function postprocess(text: string, options?: PostProcessOptions): string
const style = options?.style;
let result = text;

// ALWAYS: Normalize typographic punctuation (curly quotes, ellipsis, NBSP).
// Done early so downstream steps see ASCII-only chars.
result = normalizePunctuation(result);

// ALWAYS: Strip em-dashes (strongest AI tell). Numeric ranges preserved.
result = stripAIDashes(result);

Expand Down