diff --git a/.codex/nemesis-review.md b/.codex/nemesis-review.md deleted file mode 100644 index d687ba6..0000000 --- a/.codex/nemesis-review.md +++ /dev/null @@ -1,133 +0,0 @@ -# Completion spec: eval-hardening branch (claim under test) - -## Claim - -The `eval-hardening` branch (8 commits, `10aa31a..18a442a`, currently checked out in -/Users/clayton/dev/unslop) is an objectively better implementation of the unslop skill -and its eval suite than the baseline commit `10aa31a`, across: (1) eval-suite integrity, -(2) scanner precision AND recall, (3) fact preservation, (4) macro-structure coverage, -(5) behavioral-eval determinism, (6) docs/CI executability, (7) a cost-tiered execution -architecture. If the claim survives, the branch merges to main. - -"Objectively better" here means: measurably better on both error directions (false -positives and false negatives), with no regression the baseline handled correctly, and -with the improvements enforced by the suite itself rather than by convention. - -## Baseline vs branch - -- Baseline: git commit `10aa31a` (main). 55 deterministic script cases (54 pass, - 1 xfail FP-06), 27 behavioral cases with 5 script backstops, 4 documented checks. -- Branch: `18a442a`. 174 script cases (173 pass, 1 xfail FP-06 — intentional, - documented in evals/CRITIQUE.md), 33 behavioral cases all carrying deterministic - backstops, 12 gates in `python3 evals/run_adversarial.py --list-gates`. -- Compare freely: `git worktree add 10aa31a` gives a runnable baseline. - -## Changed surface (commit order) - -1. `efd7e0e` runner: `violation_phrase_contains`/`violation_category_equals` assertion - types; EXPECTED_XFAIL pinned to {FP-06}; XPASS or unexpected-XFAIL exits non-zero; - argparse `--only/--case/--list-gates`; recall rows REC-04..07, QE-04, SEM-02, AS-04. -2. `4b1c64f` scanner: collocation gates for unpack/lean into/double down/showcase/ - bolster/garner/stakeholder; word-boundary phrase matching; new detection families - (wh-opener self-Q&A, negative listing, isn't-just variants, noting-phrase pairs, - cliffhanger fragments, modal+potentially stacks, whether-you're openers, emoji - headers, more); em-dash/exclamation no longer match across paragraph breaks; - `evals/check_taboo_parity.py` + DOC-01 enforce scanner↔catalog parity. -3. `8799035` support scripts: unit-family swap detection (km/mi, °C/°F, GB/TB, hr) - with alias sets both directions; spelled magnitudes beyond currency; curly-quote - constraint tracking; faithful rewrites (range/quarter/time rewordings, reorders, - and/or, quote restyling) no longer fail; `validate_preservation.py --strict` - promotes negation/scope/modality warnings to exit 1; staccato detection through - em-dash/semicolon joins; splitter handles closing quotes; diff_check reorder - exemption. -4. `a25515c` behavioral determinism: DETERMINISTIC_ASSERTIONS for every behavioral - case in `evals/build_shared_benchmark.py`; run_local.py `` extraction - (output.md = rewrite, answer_full.md = full answer); fixture-prompt sync guard for - all fixture types; 3 new holdout cases (FRAGMENT-02, DONOHARM-02, INJECT-02); - binary judge rubrics. -5. `92c0ab6` docs/CI: `evals/run_behavioral.sh` (holdback sealed behind - UNSLOP_CONFIRM_HOLDBACK=1); CI runs all deterministic gates; `evals/CHECKS.md` - gate matrix pinned byte-equal to `--list-gates` by DOC-03/check_gates_doc.py; - SKILL.md examples pinned by DOC-02/check_skill_examples.py (the baseline's crisp - example violated the skill's own blocking gates). -6. `6161e92` macro layer: `scripts/structure_scan.py` (burstiness, conclusion coda, - bold-colon listicles, one-line staccato, connective paragraph openers, signposting, - opener repetition, participial closers; `--genre docs|social` carve-outs; - markdown-aware). Rows STRUCT-01..16 + ROB-12..14. SKILL-MACRO-01/02 use essays - that are phrase-scan-clean but structure-flagged. A 20-doc probe corpus lives at - /private/tmp/claude-501/-Users-clayton-dev-unslop/830a19b6-9d14-4fc1-9d39-e37f50f5a402/scratchpad/macro-probe/ - (ai/ and human/): expected classification 10/12 ai docs flag, 8/8 human docs clean. -7. `18a442a` consolidation + tiers: SKILL.md 295→180 lines; `references/pipeline.md` - (Tier 0 scripts → Tier 1 pack-scoped small detectors → Tier 2 single rewriter → - Tier 0 gates); `references/packs/` (5 packs + manifest covering all 46 scanner - categories); `references/maintenance.md`; `scripts/check_packs.py` + DOC-04/05/06; - SKILL-TIER-01 scope-discipline case. - -## Expected behaviors (verification surfaces) - -All from repo root unless stated. Deterministic gates (no credentials needed): - -``` -python3 evals/run_adversarial.py # exit 0; 173 PASS, XFAIL exactly {FP-06} -python3 evals/build_shared_benchmark.py --check -python3 evals/check_taboo_parity.py -python3 evals/check_gates_doc.py -python3 evals/check_skill_examples.py -python3 evals/check_packs.py -skill-benchmark validate evals/shared-benchmark.json --strict-leakage -``` - -Behavior deltas the branch claims (baseline worktree should show the opposite): - -- Scanner clean on: "The plant is in Garner, North Carolina."; "Notwithstanding - anything to the contrary in Section 4, the tenant remains liable."; "Drive the - wedge into the kerf."; "We unpacked the boxes after the move."; escrow - "stakeholder"; "painted the landscape of the Hudson Valley." -- Scanner flags: "Why does this matter? Because costs compound."; "This isn't just a - bugfix, it's a rethink."; "The result? A 40% drop."; "Whether you're a seasoned - developer or just starting out…"; "It is worth noting…"; "could potentially". -- validate_preservation fails on: 150 km→150 miles; 5 million→5 billion users; - curly-quote content edits. Passes on: 10-20%→"between 10% and 20%"; - Q3 2024→"the third quarter of 2024"; 3:00 PM→"3 pm"; GB/hr expansions. -- Suite integrity: deleting the leverage+navigate scanner patterns must fail the - branch suite (baseline suite stays green under the same deletion); marking a - passing case xfail:true must exit non-zero on the branch. -- structure_scan: flags the SKILL-MACRO essay texts (which produce 0 phrase-scan - violations); `--genre docs` suppresses bold-colon flags; human corpus docs clean. - -Higher-cost real surfaces (available, optional): - -- Behavioral layer: `evals/run_behavioral.sh tune` (needs `claude -p` and - `skill-benchmark`, installed at ~/.local/bin). Artifacts of a passing tune run - exist in runs/tune/. -- OSS model spot-check artifacts (24 model-case outputs + graded matrix): - /private/tmp/claude-501/-Users-clayton-dev-unslop/830a19b6-9d14-4fc1-9d39-e37f50f5a402/scratchpad/oss-runs/ - Re-runnable via the script in the same scratchpad dir; OpenRouter key available in - macOS keychain service `OPENROUTER_API_KEY`. - -## Known tradeoffs / accepted decisions - -- FP-06 stays an intentional xfail (literal "delve into a place" vs figurative — - pattern kept for recall; documented in CRITIQUE.md). -- Comma-joined staccato fragments are deliberately NOT flagged by - readability_metrics (splitting on commas would false-flag ordinary prose); - em-dash/semicolon joins are flagged. -- diff_check still reports raw change_percentage on reorders; only the - excessive_change flag is exempted. -- rubric.md / personality-guide.md / edit-library.md were deliberately not - pointer-ized during consolidation; presets/ verified as non-duplicative deltas. -- SKILL-MODE-01's deterministic proxy accepts an ask-first branch via the token - "audit" in answer_full.md. -- Suite runtime grew (~55→174 subprocess cases, tens of seconds); accepted. -- The tiered pipeline (references/pipeline.md, packs) is documentation + integrity - gates; no orchestration code executes it in this repo. Its behavioral claim is - pinned only by SKILL-TIER-01. - -## Setup notes - -- Python 3.14 via Homebrew as `python3`; no third-party deps for the deterministic - gates. `skill-benchmark` and `codex` on PATH at ~/.local/bin. -- The repo working tree is clean at `18a442a` on branch `eval-hardening`. -- Do not commit anything; do not push; leave the tree byte-identical. Back up any - file you temporarily mutate with cp and restore with cmp verification. Never use - `git checkout --` to restore. diff --git a/.gitignore b/.gitignore index a9754a7..a6701a8 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,14 @@ test_*.txt # Wiki sync state scripts/.wiki_sync_state.json +# Taught voices: profiles, cards, and provenance are user data, not repo content +.unslop/ + # Packaged skill *.skill # Behavioral eval run artifacts (prepared tasks, model outputs, judge results) evals/runs/ runs/ +.claude/ +.codex/ diff --git a/.unslop/contrib/let-that-sink-in/manifest.json b/.unslop/contrib/let-that-sink-in/manifest.json new file mode 100644 index 0000000..2d1f3f2 --- /dev/null +++ b/.unslop/contrib/let-that-sink-in/manifest.json @@ -0,0 +1,10 @@ +{ + "category": "emphasis_crutch", + "date": "2026-07-06", + "pattern_name": "let-that-sink-in", + "post_redaction_sha256": "6e70f687ebfa4157f2173c00e589e37444bc1f272a4295e6919a991b3f56c7c4", + "pre_redaction_sha256": "6e70f687ebfa4157f2173c00e589e37444bc1f272a4295e6919a991b3f56c7c4", + "redactions": [], + "tell": "Let that sink in", + "word_count": 19 +} diff --git a/.unslop/contrib/let-that-sink-in/report.md b/.unslop/contrib/let-that-sink-in/report.md new file mode 100644 index 0000000..e2fba09 --- /dev/null +++ b/.unslop/contrib/let-that-sink-in/report.md @@ -0,0 +1,42 @@ +# Add emphasis_crutch pattern: Let that sink in + +## The specimen + +> The launch memo ends with a warning instead of evidence. Let that sink in before you ask what changed. + +- Source genre: TODO: source genre +- Date: 2026-07-06 +- Redaction note: none + +## Why it's an AI-ism + +TODO: explain why this phrase is a reusable AI-writing tell in 2-4 sentences. + +## Detection + +- Pattern added: TODO: regex or phrase for `Let that sink in` +- Severity: TODO: hard or soft +- Gating rationale: TODO: explain literal-use boundary +- Catalog entry location: references/taboo-phrases.md + +## Evals + +| row id | kind | what it pins | +|---|---|---| +| CONTRIB-FN-let-that-sink-in | FN | exact specimen flags `Let that sink in` | +| CONTRIB-FP-let-that-sink-in | FP | literal-use protection for `Let that sink in` | + +The FN stdin is the unmodified specimen after approved redaction. + +## Gate results + +red-first: already green; proposed pattern appears active + +## Checklist + +- [ ] eval-first (row was red before the pattern) +- [ ] literal-use FP row included +- [ ] REC row if an existing word was gated +- [ ] catalog + scanner parity green +- [ ] coverage gate green (pattern exercised) +- [ ] snippet publication approved by the user diff --git a/.unslop/contrib/let-that-sink-in/row_fn.json b/.unslop/contrib/let-that-sink-in/row_fn.json new file mode 100644 index 0000000..00ab10f --- /dev/null +++ b/.unslop/contrib/let-that-sink-in/row_fn.json @@ -0,0 +1,24 @@ +{ + "assertions": [ + { + "gte": 1, + "path": "total_violations", + "type": "json" + }, + { + "type": "violation_phrase_contains", + "value": "let that sink in" + } + ], + "category": "scanner_false_negative", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "correct_behavior": "The scanner reports at least one violation containing the contributed tell.", + "failure_mode": "The scanner misses the contributed tell from the exact specimen.", + "id": "CONTRIB-FN-let-that-sink-in", + "stdin": "The launch memo ends with a warning instead of evidence. Let that sink in before you ask what changed.\n", + "target": "script", + "title": "New AI-ism fixture: Let that sink in" +} diff --git a/.unslop/contrib/let-that-sink-in/row_fp_TEMPLATE.json b/.unslop/contrib/let-that-sink-in/row_fp_TEMPLATE.json new file mode 100644 index 0000000..2b5ce54 --- /dev/null +++ b/.unslop/contrib/let-that-sink-in/row_fp_TEMPLATE.json @@ -0,0 +1,21 @@ +{ + "assertions": [ + { + "equals": 0, + "path": "total_violations", + "type": "json" + } + ], + "category": "scanner_false_positive", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "correct_behavior": "No violations for the protected literal/domain use.", + "failure_mode": "A broad pattern could flag legitimate prose.", + "id": "CONTRIB-FP-let-that-sink-in", + "protects": "emphasis_crutch", + "stdin": "TODO: add a literal or domain-specific use that should remain clean.", + "target": "script", + "title": "Literal-use protection for Let that sink in" +} diff --git a/.unslop/contrib/let-that-sink-in/snippet.txt b/.unslop/contrib/let-that-sink-in/snippet.txt new file mode 100644 index 0000000..6b06d15 --- /dev/null +++ b/.unslop/contrib/let-that-sink-in/snippet.txt @@ -0,0 +1 @@ +The launch memo ends with a warning instead of evidence. Let that sink in before you ask what changed. diff --git a/AGENTS.md b/AGENTS.md index b691a6a..dc4dd8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,10 +15,16 @@ start with evals, then update the skill or scripts until the suite passes. proving the jargon use still flags. - Agent behavior change: add a `skill` row only when the behavior is rewrite, preserve, decline, or route. + - Context-heavy or literal-sense pattern: prefer an `evals/fixtures/pairs/` + minimal pair plus PAIR rows, then run `python3 evals/check_pairs.py`. 3. If a `skill` row is added, update `evals/build_shared_benchmark.py` with its split and domain, then regenerate `evals/shared-benchmark.json`. 4. Update `scripts/banned_phrase_scan.py`, `SKILL.md`, `presets/`, or `references/` only after the eval captures the desired product behavior. +5. `python3 evals/check_pattern_coverage.py` (gate DOC-09) fails unless every + scanner pattern is exercised by a row and every category has a `protects` FP + row; `python3 evals/kata_add_pattern.py --run` (DOC-10) rehearses this. No + grandfathering. Prefer contextual patterns over broad word bans. A row for `wedge` or `load-bearing` should also protect literal uses such as construction, mechanics, law, medicine, code, @@ -43,6 +49,8 @@ skill-benchmark validate evals/shared-benchmark.json --strict-leakage If you touched `SKILL.md`, `presets/`, or `references/`, also run `evals/run_behavioral.sh tune`. +If you touched the co-writer, mimic, or detector-pack model features, run `python3 evals/run_model_parity.py --dry-run --responses evals/fixtures/parity/canned_responses.json` and, before merge, the live matrix across the GPT and Anthropic spectrums (see `references/pipeline.md`, Model Parity). + `evals/evals.json` is the legacy happy-path suite; never edit it. Use `evals/adversarial-evals.json` as the source of truth. diff --git a/CLAUDE.md b/CLAUDE.md index ab1247b..511d3a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,10 +15,16 @@ start with evals, then update the skill or scripts until the suite passes. proving the jargon use still flags. - Agent behavior change: add a `skill` row only when the behavior is rewrite, preserve, decline, or route. + - Context-heavy or literal-sense pattern: prefer an `evals/fixtures/pairs/` + minimal pair plus PAIR rows, then run `python3 evals/check_pairs.py`. 3. If a `skill` row is added, update `evals/build_shared_benchmark.py` with its split and domain, then regenerate `evals/shared-benchmark.json`. 4. Update `scripts/banned_phrase_scan.py`, `SKILL.md`, `presets/`, or `references/` only after the eval captures the desired product behavior. +5. `python3 evals/check_pattern_coverage.py` (gate DOC-09) fails unless every + scanner pattern is exercised by a row and every category has a `protects` FP + row; `python3 evals/kata_add_pattern.py --run` (DOC-10) rehearses this. No + grandfathering. Prefer contextual patterns over broad word bans. A row for `wedge` or `load-bearing` should also protect literal uses such as construction, mechanics, law, medicine, code, @@ -43,6 +49,8 @@ skill-benchmark validate evals/shared-benchmark.json --strict-leakage If you touched `SKILL.md`, `presets/`, or `references/`, also run `evals/run_behavioral.sh tune`. +If you touched the co-writer, mimic, or detector-pack model features, run `python3 evals/run_model_parity.py --dry-run --responses evals/fixtures/parity/canned_responses.json` and, before merge, the live matrix across the GPT and Anthropic spectrums (see `references/pipeline.md`, Model Parity). + `evals/evals.json` is the legacy happy-path suite; never edit it. Use `evals/adversarial-evals.json` as the source of truth. diff --git a/README.md b/README.md index 5b44fe4..4c8f36a 100644 --- a/README.md +++ b/README.md @@ -1,199 +1,406 @@ # unslop -An agent skill for humanizing AI-generated content. +Strip the patterns that make writing read as machine-written, and reconstruct prose in a +real human voice. unslop ships as an agent skill: the host agent runs it while you write, +backed by deterministic scanners that any tool can call on their own. -## What It Does +Four commands cover the whole surface: -Removes predictable AI writing patterns from prose at two levels: phrase-level tells ("delve", "it's worth noting", "stands as a testament") and document-level structure (uniform sentence rhythm, moralizing codas, bold-header listicles, connective paragraph scaffolds). Two modes: +- `/unslop teach` builds a reusable voice from your own writing samples. +- `/unslop cleanup` flags AI tells as reviewable suggestions and changes nothing on its own. +- `/unslop rewrite` diagnoses a draft and rebuilds it under the guards (the default). +- `/unslop mimic` drafts or rewrites in a taught voice, then clears every removal gate. -1. **Audit only**: Analyze input for AI-isms, severity, and context-dependent flags without rewriting -2. **Rewrite**: Diagnose, then rewrite following a preset voice while eliminating patterns - -It is equally deliberate about what it does *not* touch: literal domain usage (a horse harness, a load-bearing wall, financial leverage), load-bearing hedges and absolutes in legal, medical, security, and scientific text, and already-human writing that just needs to be left alone. +The weight sits about 70/30 toward removal. Detection is cheap, deterministic, and +benchmarkable; it is the trust asset. Voice work is generative, and it runs under detection's +constitution: any mimic or rewrite that reintroduces a tell fails, however well it matches the +voice. ## Installation -### Using Skills CLI (Recommended) +### Using the Skills CLI (recommended) -Install to any supported coding agent using [npx skills](https://github.com/vercel-labs/skills): +Install to any supported coding agent with [npx skills](https://github.com/vercel-labs/skills): ```bash # Install to Claude Code (global) npx skills add theclaymethod/unslop -g -a claude-code -# Install to multiple agents +# Install to several agents at once npx skills add theclaymethod/unslop -g -a claude-code -a cursor -a codex -# Install to all detected agents +# Install to every detected agent npx skills add theclaymethod/unslop -g # List available skills first npx skills add theclaymethod/unslop --list ``` -### Manual Installation +### Manual installation ```bash -# Clone the repo git clone https://github.com/theclaymethod/unslop.git ~/dev/unslop - -# Symlink to Claude Code skills directory ln -s ~/dev/unslop ~/.claude/skills/unslop - -# Or symlink to commands (for /unslop invocation) +# Or wire it as a slash command: ln -s ~/dev/unslop/SKILL.md ~/.claude/commands/unslop.md ``` -### Standalone Scripts - -The Python validation scripts work independently (stdlib only, no dependencies): - -```bash -# Extract constraints (facts that must survive) -python3 scripts/extract_constraints.py < input.txt - -# Scan for phrase-level AI-isms -python3 scripts/banned_phrase_scan.py < input.txt - -# Scan including quoted examples and blockquotes -python3 scripts/banned_phrase_scan.py --include-quoted < input.txt - -# Scan for document-level structural tells (burstiness, codas, scaffolds) -python3 scripts/structure_scan.py < input.txt -python3 scripts/structure_scan.py --genre docs < README.md # reference-doc carve-outs -python3 scripts/structure_scan.py --genre social < post.txt # social-cadence carve-outs - -# Check readability metrics -python3 scripts/readability_metrics.py < input.txt - -# Validate fact preservation -python3 scripts/validate_preservation.py original.txt transformed.txt - -# Strict mode: negation/scope/modality drift fails instead of warning -# (use for legal, medical, security, scientific text) -python3 scripts/validate_preservation.py --strict original.txt transformed.txt - -# Check change percentage -python3 scripts/diff_check.py original.txt transformed.txt - -# Check Wikipedia for pattern updates -python3 scripts/wiki_sync.py check -``` +The Python scripts under `scripts/` run on their own with no third-party dependencies +(Python 3.8+, standard library). Call them from CI, a pre-publish check, or another agent +without loading the skill at all. + +## Three Detection Layers + +Detection stacks three deterministic scanners, coarse to fine. Each returns JSON, exits +non-zero on a flag, and carries false-positive protection rows so a literal or domain use +never trips it. + +**Phrase layer** (`scripts/banned_phrase_scan.py`). 313 banned phrases and vocabulary items, +each tagged `hard` (always a tell) or `soft` (context-dependent). Gated words fire only in +their jargon collocations. Sailors navigate, a `3:1 leverage` ratio holds, a `wedge` seats in +the kerf, and `Garner, North Carolina` stays on the map; the same words in +`"navigate challenges"` or `"leverage synergies"` get caught. Quoted spans, blockquotes, and +code fences are masked first, so a tutorial documenting bad writing never flags its examples. + +**Structure layer** (`scripts/structure_scan.py`). 77 structural patterns plus document-level +metrics: `sentence_burstiness`, `paragraph_cv`, `triad_density`, `bold_colon_listicle_count`, +`one_line_staccato_share`, `connective_paragraph_openers`, `signpost_density`, +`opener_unique_ratio`, `top_opener_share`, `max_consecutive_opener`, +`participial_closer_share`, `conclusion_coda`, and `summary_sandwich`. This layer catches the +rhythm tells (uniform sentence length, staccato runs), the moralizing codas, the connective +scaffolds that open every paragraph with `"However,"` or `"Moreover,"`, and the bold-label +listicles standing in for prose. Genre carve-outs keep it honest: `--genre docs` allows the +bold-label lists reference docs really use, and `--genre social` allows the short-line cadence +that belongs in social copy. + +**Silhouette layer** (`scripts/silhouette_scan.py`). One level above the surface, this scores +how ideas are arranged. It catches outline-following, recap loops, and paragraph-role +templating: body paragraphs that open on a discourse cue instead of their own claim +(`scaffold_opener_share`), opening vocabulary that disappears mid-document and returns at the +end (`callback_content`, the strongest single tell), cue-opener roles rotating like a template +(`role_entropy_bits`), intro content words reappearing as body-paragraph heads +(`preview_fulfillment`), and section headings that restate the intro's outline +(`heading_preview`). The composite `silhouette_penalty` flags at `1.0` against a committed +human reference. On the corpus in the repo it separates cleanly: 12 of 12 AI documents +flagged, 0 of 8 human documents flagged. A cue-deletion attack collapses the scaffold metric, +so silhouette is scored jointly with the structure scanner as a lower fence. + +## The Gamut: What Gets Removed + +Every family below is cataloged in `references/taboo-phrases.md` and pinned by an eval row. +Severity is `hard` (always a tell) or `soft` (a default register guard your real voice may +override). The philosophy throughout is contextual gating over blunt word bans: a pattern +ships only once a false-positive row proves the literal sense survives. + +### Openers, emphasis, and inflation + +| Family | Caught examples | +|--------|-----------------| +| Throat-clearing openers | `"Here's the thing:"`, `"The uncomfortable truth is"`, `"Let me be clear"`, `"It turns out"`, `"Let's dive in"`, `"Let's unpack"` | +| Emphasis crutches | `"Full stop."`, `"Let that sink in."`, `"Make no mistake"`, `"Read that again."`, `"This cannot be overstated."` | +| The "X is real" closer | `"The struggle is real."`, `"The stakes are real."` (spares the literal "is it genuine?" sense) | +| Significance inflation | `"stands as a testament to"`, `"pivotal moment"`, `"enduring legacy"`, `"rich tapestry"`, `"cornerstone of"`, `"holds great promise"` | +| False agency | `"the numbers speak for themselves"`, `"the data tells a story"`, `"paints a clear picture"` | + +### Contrast, questions, and drama + +| Family | Caught examples | +|--------|-----------------| +| Negative parallelism | `"It's not X, it's Y"`, `"Not only... but also"`, `"Not merely X, but Y"`, `"No X, no Y, just Z"` | +| Contrastive definitions | `"X isn't a Y, it's a Z"` (spares real corrections like `"Use pnpm, not npm."` and `"The painting is real, not a forgery."`) | +| Wh-opener self-Q&A | `"Why does this matter? Because..."`, `"What does this mean for..."`, `"Why should you care?"` | +| Cliffhanger fragments | `"[Noun]. That's it. That's the [thing]."`, `"The ___ loop."` as a standalone, `"X things. One thing."` | +| Hedge stacks | `"(and perhaps more importantly, ...)"`, `"(arguably ...)"`, `"While X is promising, Y remains a challenge"` | + +### Attribution, flattery, and jargon + +| Family | Caught examples | +|--------|-----------------| +| Vague attribution | `"Experts argue"`, `"Studies show"`, `"Some critics"`, and bare clause-initial `"Research indicates"` (attributed and possessive forms stay clean) | +| Reader-addressing flattery | `"Here's what's interesting"`, `"worth reading"`, `"worth your time"`, `"Whether you're a seasoned developer or just starting out"` | +| Business-jargon collocations | `"navigate challenges"`, `"leverage synergies"`, `"deep dive"`, `"circle back"`, `"move the needle"`, `"low-hanging fruit"` | +| Marketing and headline cadence | `"world-class"`, `"state-of-the-art"`, `"a hidden gem"`, two-beat imperative slogans (`"Emit 1,100 tokens. Ship 237KB."`), and headline slogan cadence firing at three or more short-line headers in one document | + +### Chatbot residue and punctuation + +| Family | Caught examples | +|--------|-----------------| +| Chatbot artifacts | `"I hope this helps"`, `"Certainly!"`, `"Great question!"`, `"as an AI language model"`, `"as of my knowledge cutoff"` | +| Emoji section headers | Decorative emoji standing in as headings, flagged as a formatting tell | +| Em-dash overuse | The single most reliable punctuation tell. Default is zero; two or more in one paragraph is always a hard flag | +| Reasoning-chain leaks | `"Let me think step by step"`, `"Breaking this down"`, `"Here's my thought process"` | + +### Structural and silhouette families + +The structure scanner adds the document-shape tells: uniform sentence rhythm, staccato +one-line-paragraph runs outside social copy, connective paragraph scaffolds, signpost density, +and moralizing codas like `"Ultimately, this reminds us that..."`. The silhouette scanner adds +the arrangement tells above. A handful of macro tells stay agent judgment rather than +scanner-enforced (both-sidesism, templated redemption arcs, over-determination, uniform +emotional register), because a scanner cannot reliably tell a genuine opposing view from a +manufactured one. -## Usage +## What Gets Protected -### Basic +Do-no-harm is half the product. The scanners are built to leave the following alone, and each +guard has its own eval row. + +- **Register guards.** `"never store secrets"`, `"may cause drowsiness"`, + `"does not establish causation"`, and `"notwithstanding anything to the contrary"` are + content, not filler. In legal, medical, security, and scientific text these hedges, + negations, absolutes, and scope words carry meaning. `validate_preservation.py --strict` + turns dropping one into a hard failure. +- **Literal domain usage.** Construction, mechanics, law, medicine, finance, sailing, and + code all use the gated words literally. Every contextual pattern ships with a false-positive + row proving the literal sense stays clean. +- **Quoted examples.** Spans in quotes, blockquotes, and code fences are exempt by default, so + documentation never self-flags the bad writing it is teaching. +- **Facts, with magnitude awareness.** Numbers, names, dates, quotes, units, references like + `Section 12(b)`, and `and/or` scope survive a rewrite. Magnitude-aware checking means + `$47.3M` cannot silently become `$47.3 billion`, and `150 km` cannot become `150 miles`. +- **Genre carve-outs.** Bold-label lists are correct in reference docs, staccato is correct in + social copy, and a section-roadmap abstract is academic convention rather than a tell. +- **English only.** Non-English input gets cheap detection and a clear decline. That is the one + graceful refusal in the product; the scanners return `non_english: true` and stop. + +## Voice: teach and mimic + +The right end of the axis writes the way a specific person writes. It is agent-driven end to +end: you supply approvals and answers, never a directory or a command. + +### teach + +`teach` distills your samples into two artifacts under `.unslop/voice//` (gitignored): +a machine profile (`profile.json`, the deterministic referee) and a layered voice card +(`card.md` plus per-situation sheets) that any generating model follows in context. + +1. **Harvest.** Cheap agents bootstrap the corpus from your chat transcripts and writing + folders. Adapters read both Claude Code (`claude-jsonl`) and Codex CLI/Desktop + (`codex-jsonl`) sessions, auto-detected by shape. The contamination guarantee is the whole + point: assistant-authored text in a voice profile would teach the exact register unslop + removes. So the adapters drop assistant and developer turns, strip injected content like a + dumped `AGENTS.md`, and run every kept candidate through the scanners. A hard hit or two + scanner categories marks a sample `suspect_ai`, ranked last and never auto-approved. Nothing + enters a profile without your approval. +2. **Profile and card.** `voice_profile.py` computes the stylometric fingerprint (character + 3-grams, function-word deltas, sentence-length distribution, punctuation, contractions, + MTLD, impostor z-scores, GI rank). `voice_card.py` writes the layered card the generator + reads. The card never fabricates: a dimension with no sample evidence gets no sheet and is + named under "Uncovered". A profile that does not describe the supplied samples is rejected + (exit 2), so a stale profile can never drive a card. +3. **Calibrate.** When samples run thin, an A/B preference game gathers voice signal at + tap-level effort. Pairs are dimension-controlled minimal edits of your own passages, mostly + from deterministic transforms, so each pair preserves the passage's facts. Preferences + aggregate per dimension with confidence bounds, the game targets the least-known dimension + next, and a stated preference that contradicts a sample-measured value surfaces to you as a + named conflict rather than being resolved silently. Voice beats the default register guard, + but only visibly. +4. **Scored demo.** teach closes by mimicking one paragraph, scoring it, and running the + scanners in front of you. A voiced demo that trips a slop gate fails the loop. + +### mimic + +`mimic` drafts or rewrites in the taught voice, then clears the full gate battery. The rule is +absolute: a mimic that scores well on voice but trips a removal gate is rejected. Voice never +buys an exemption from the constitution, and meaning preservation against the original draft +is its own separate hard gate. + +- **Voice check** answers "does this sound like me?" with a score and no rewrite. It reports + the composite (lower is more you), the General Impostors rank, and the two or three metric + deltas that explain the score in plain words. It is the cheapest voice interaction and the + usual one after teach: check drafts often, commission rewrites rarely. +- **Refine** hill-climbs when a single pass keeps landing short. It splits samples into a + retrieval pool and a held-out acceptance split, generates candidates, and discards any that + trip a hard gate (banned-phrase, structure, draft-to-candidate preservation, a copy-gate + against the pool, and a word floor). Survivors are scored on the held-out split with a + gaming-resistant composite: `0.5·(1−GI) + 0.5·` a clipped impostor-z distance against a + same-genre impostor pool. A marker-stuffed candidate can drive a raw distance down and still + lose under the General Impostors rank, which is what stops it from beating honest prose. A + **divergence guard** halts the loop and raises `reward_hacking_warning` when the pool score + improves while the held-out score worsens for two iterations. Claim a win only when + `mimic_stats.py` shows the confidence-interval lower bound above zero and p below 0.05. + +## Co-writer: cleanup + +`cleanup` is the co-writer mode. On detection it surfaces findings as structured suggestions, +never as a silent rewrite. Each suggestion carries a span, severity, category, rationale, and +proposed replacement. Detection is cheap and deterministic; replacement generation is +delegated to a stronger model, which fills in the null replacements the scanner leaves. + +Hard findings become direct replacements. Soft findings are register-dependent, so their +rationale is phrased as a question rather than an edit. Four contract gates in +`check_suggestions.py` make "accept all" safe by construction: + +- **span-minimality**: an edit changes only its own span; a whole-sentence rewrite fails. +- **replacement-scanner**: each replacement passes both scanners in isolation and adds no new + violation in context. +- **accept-all**: applying every suggestion yields a document that passes both scanners and + preserves every constraint against the original. +- **span-overlap**: spans may not overlap. + +A report-only variant ("flag it, change nothing") runs the scanners and reports each issue by +span, category, and severity, separating clear problems from judgment calls. + +## Contribute: the growth flywheel + +Detection catalogs decay as generators evolve. Growth comes from adversarial refresh: a wild +specimen becomes an eval row becomes a structured PR. `/unslop contribute` runs the pipeline +offline until you approve publication. + +1. **Precheck** tells you whether the tell is already covered. +2. **Confirmation gate one** shows you the exact snippet and asks whether it may go public, + with redaction hints that keep the tell byte-for-byte intact. +3. **Scaffold and implement eval-first.** The false-negative row lands red before the scanner + changes, and a literal-use false-positive row lands beside it. The scanner and catalog then + change until the row goes green while the protection row holds. +4. **Verify** captures the red-to-green transition and refuses TODO markers. +5. **Full gate battery**, then **confirmation gate two** on the final PR body. Only then does + the host agent branch, commit, and open the PR. The scripts never touch the network. + +Yesterday's miss becomes tomorrow's regression test, and nothing rides on good habits. + +## The Eval Suite Is the Product + +Every behavior worth having is pinned by a row that fails without it. The suite is +constitutional, and it is built to resist gaming. + +- **439 deterministic cases** in `evals/adversarial-evals.json`, run by + `python3 evals/run_adversarial.py`. Each detection carries a false-negative row (the tell + gets caught), a false-positive row (the literal sense survives), and a recall row (gating did + not gut detection). +- **24 machine-readable gates** via `python3 evals/run_adversarial.py --list-gates`, mirrored + in `evals/CHECKS.md`. They cover the scanners, the harvest/contribute/calibrate suites, the + voice scorer, pack structure, silhouette separation, and the docs themselves. +- **Mutation-proof by construction.** Deleting a scanner pattern fails the coverage gate. + Marking a passing case `xfail` fails the build. The one documented XFAIL is pinned to exactly + one case: `FP-06`, where the literal `"delve into the mountain"` collides with the strong + `"delve into the topic"` tell, an accepted residual a pattern regex cannot disambiguate. Even + the docs are gate-checked: SKILL.md's own examples must pass its scanners, the scanner and + catalog are held in two-way parity, and a kata proves the add-a-pattern loop still works. +- **Behavioral layer.** `evals/shared-benchmark.json` is generated (never hand-edited) from the + `skill` rows, with `with_skill` and `without_skill` variants graded by an LLM judge plus + deterministic backstops: facts that must survive, banned strings that must not appear, + similarity floors for do-no-harm. Its 33 cases split `tune` (17) for shaping, `holdout` (12) + for reporting, and `holdback` (4) sealed until final confirmation. +- **Interpreting the lift.** The base model already de-slops well, so judge-blended lift runs + near zero and per-case deltas carry the signal. On the recorded 2026-07-06 holdout run the + deterministic backstops show +4.2 points objective lift (0.917 vs 0.875 across 12 held-out + cases; see `evals/TUNE-RESULTS.md`) while the judge cannot tell the prose apart — the + measurable value lives in preserved facts, register, and structure. The recorded tune run + also preserved a legal hedge the baseline dropped, and exposed a do-no-harm regression the + guards were then hardened against. ```bash -/unslop "Here's the thing: building products is hard. Let that sink in." +python3 evals/run_adversarial.py # the deterministic suite (439 pass, 1 xfail) +python3 evals/run_adversarial.py --only FP # one category slice, parallelizes well +evals/run_behavioral.sh tune # the behavioral layer (needs claude -p) ``` -Output: -``` -Building products is hard. -``` +## Model Tiering Is Measured -### With Preset +Where the pipeline depends on a model, `evals/run_model_parity.py` measures whether a cheap +tier is safe rather than assuming it. The live matrix was recorded **2026-07-06** across both +the Anthropic and GPT spectrums. -```bash -/unslop --preset=warm "Your AI-generated text here" -``` +**Span replacement clears on the cheapest tier.** On the mechanical span-minimal contract, +`claude-haiku`, `claude-sonnet`, `gpt-5.4-mini`, and `gpt-5.5` all scored 8/8; `claude-opus` +scored 7/8, its one miss a dropped `8:30` caught by the preservation gate rather than by model +choice. The gates carry safety, the tier does not. -Available presets: -- `crisp` (default) - Short, direct, no fluff -- `warm` - Friendly tone, conversational -- `expert` - Authoritative, confident claims -- `story` - Narrative flow, show don't tell +**Full rewrites of register-sensitive text belong to frontier models.** On eight +register/structure cases the ladder was 7/8 (opus, gpt-5.4-mini), 6/8 (sonnet, gpt-5.5), and +5/8 (haiku, gpt-5.4-nano); cheap-tier misses softened absolutes, dropped hedges, and eroded a +legal negation. -### Audit Only +**Macro structure defeated every model tested.** The `MACRO-01` case failed for all six, +opus included: each kept a conclusion coda the prose instruction told it to drop. No model +self-checks document shape from prose, so structure is always machine-detected and +machine-gated. `references/pipeline.md` records the full tables. -```bash -/unslop --audit-only "Here's what's interesting: this deck is worth reading." -``` +## Standalone Scripts -Returns flagged AI patterns and an assessment without rewriting the text. - -### Strict Mode +The scripts run independently, standard library only. A quick tour: ```bash -/unslop --strict "Text to humanize" -``` - -Fails if rubric score < 32/40. - -## What Gets Removed - -### Throat-Clearing Openers -- "Here's the thing:" -- "The uncomfortable truth is" -- "Let me be clear" -- "It turns out" - -### Emphasis Crutches -- "Full stop." -- "Let that sink in." -- "Make no mistake" - -### Business Jargon (collocation-gated) -- "Navigate challenges" → "Handle problems" -- "Leverage synergies" → "Use" -- "Deep dive" → "Analysis" -- "Game-changer" → (cut or use specific claim) - -Gated words only flag in their jargon collocations. "Sailors navigate", "3:1 leverage", "a wedge into the kerf", and "Garner, North Carolina" all pass clean. - -### Structural Patterns -- "Not because X. Because Y." → State Y directly -- "It's not the tooling. It's not the process. It's the culture." → State the point -- "Why does this matter? Because..." → Cut the self-Q&A -- "The result? A 40% drop." → "The result was a 40% drop." -- "Whether you're a seasoned developer or just starting out..." → Address the actual reader - -### Macro Structure (document level) -- Uniform 15-18-word sentences everywhere → vary the rhythm -- "Ultimately, this reminds us that..." codas → just stop -- "However," / "Moreover," / "Furthermore," opening every paragraph → real topic sentences -- Bold-phrase-colon listicles standing in for prose -- One-line-paragraph LinkedIn staccato (outside social copy) - -See `references/taboo-phrases.md` for the complete catalog. - -## What Gets Protected - -Do-no-harm is half the product: - -- **Register guards**: "never store secrets", "may cause drowsiness", "does not establish causation", and "notwithstanding anything to the contrary" are content, not filler. The `--strict` preservation mode makes dropping them a hard failure. -- **Literal vocabulary**: construction, law, medicine, finance, sailing, code. Every contextual pattern ships with a false-positive eval row proving the literal sense survives. -- **Quoted examples are exempt by default**: tutorials and docs should not self-flag their own bad examples. -- **Facts**: numbers, names, dates, quotes, units, references (Section 12(b)), and "and/or" scope survive rewrites — with magnitude-aware checking, so "$47.3M" cannot silently become "$47.3 billion" and "150 km" cannot become "150 miles". -- **Genre**: bold-label lists are correct in reference docs (`--genre docs`); staccato cadence is correct in social copy (`--genre social`); Section-roadmap abstracts are academic convention, not repetition. +# Phrase, structure, and silhouette scans +python3 scripts/banned_phrase_scan.py < input.txt +python3 scripts/banned_phrase_scan.py --include-quoted < input.txt # audit quoted examples too +python3 scripts/structure_scan.py < input.txt +python3 scripts/structure_scan.py --genre docs < README.md # reference-doc carve-outs +python3 scripts/silhouette_scan.py < input.txt # idea-arrangement tells -## The Eval Suite Defines the Product +# Facts and preservation +python3 scripts/extract_constraints.py < input.txt +python3 scripts/validate_preservation.py original.txt transformed.txt +python3 scripts/validate_preservation.py --strict original.txt transformed.txt # regulated text +python3 scripts/diff_check.py original.txt transformed.txt +python3 scripts/readability_metrics.py < input.txt -Every pattern, guard, and script behavior exists as an eval row first. The suite is designed to be un-gameable: +# Co-writer suggestions +python3 scripts/suggest.py document.md +python3 scripts/check_suggestions.py suggestions.json # the four contract gates + +# Voice +python3 scripts/harvest_samples.py SOURCE -o candidates.json # bootstrap from transcripts +python3 scripts/voice_profile.py samples/ -o profile.json # stylometric fingerprint +python3 scripts/voice_card.py --profile profile.json --samples samples/ --out . --name me +python3 scripts/voice_score.py --profile profile.json candidate.md # "does this sound like me?" +python3 scripts/calibrate_pairs.py generate --base passage.txt --dimension em_dash --seed 1 +python3 scripts/wiki_sync.py check # sync the phrase catalog +``` -- **182 deterministic cases** in `evals/adversarial-evals.json`, run by `python3 evals/run_adversarial.py`. Every detection carries a false-negative row (the tell gets caught), a false-positive row (the literal sense survives), and a recall row (gating didn't gut detection). Deleting a scanner pattern fails the build; marking a passing case `xfail` fails the build; the expected-xfail set is pinned to exactly one documented case. -- **33 behavioral cases** in `evals/shared-benchmark.json` (generated; never hand-edit), graded with an LLM judge *and* deterministic backstops: fact tokens that must survive, banned strings that must not appear, similarity floors for do-no-harm, and scanner exit codes on the output. Splits are `tune` / `holdout` / `holdback` (sealed). -- **12 machine-readable gates** via `python3 evals/run_adversarial.py --list-gates`, mirrored in `evals/CHECKS.md`. Even the docs are eval-gated: SKILL.md's examples must pass the skill's own scanners, the scanner and catalog are held in two-way parity, and the gate-matrix doc is pinned to the runner. +## Project Structure -```bash -python3 evals/run_adversarial.py # the deterministic suite -python3 evals/run_adversarial.py --only FP # a category slice (parallelizes well) -evals/run_behavioral.sh tune # the behavioral layer (needs claude -p) +``` +unslop/ +├── SKILL.md # Four-verb router and shared doctrine +├── README.md # This file +├── references/ +│ ├── commands/ # The routed flows: teach, cleanup, rewrite, mimic, contribute +│ ├── taboo-phrases.md # Authoritative pattern catalog (all families) +│ ├── mimic.md # Teach/mimic internals: card anatomy, scoring, refine +│ ├── harvest.md # Adapter internals and the contamination tripwire +│ ├── calibrate.md # The A/B preference game +│ ├── pipeline.md # Tiered execution and the measured model-parity tables +│ ├── fact-preservation.md # Constraint preservation rules +│ ├── rubric.md # Strict scoring criteria +│ ├── edit-library.md # Transformation examples +│ ├── personality-guide.md # Adding voice without fake personality +│ ├── maintenance.md # Add/list/wiki-sync procedures (eval-first) +│ └── packs/ # Small detector rule-packs plus manifest +├── presets/ # crisp / warm / expert / story voice deltas +├── scripts/ # Scanners, voice tools, preservation, suggest, harvest +├── evals/ +│ ├── adversarial-evals.json # Source of truth: 439 cases +│ ├── run_adversarial.py # Deterministic runner (--only, --case, --list-gates) +│ ├── shared-benchmark.json # Generated behavioral manifest (never hand-edit) +│ ├── build_shared_benchmark.py # Regenerates the behavioral manifest +│ ├── run_model_parity.py # Re-measures the tiering matrix +│ ├── CHECKS.md # Machine-readable gate matrix and parallel protocol +│ └── check_*.py # Parity, doc, pack, voice, and silhouette gates +├── docs/ +│ └── PRODUCT.md # Product doctrine (the why behind the repo) +└── assets/ + └── examples/ # Before/after sets: article, LinkedIn, sales ``` -See `evals/CHECKS.md` for the full gate matrix and the parallel check protocol, and `CLAUDE.md` / `AGENTS.md` for the eval-first contribution rules. +## Voice Presets -## Tiered Execution for Smaller Models +Read one preset from `presets/` before a rewrite. -The skill works single-agent, but `references/pipeline.md` documents a cost-tiered path for orchestrating harnesses: +| Preset | Style | Best for | +|--------|-------|----------| +| `crisp` | Short, direct, no filler | Technical writing, documentation | +| `warm` | Friendly, conversational | Emails, blog posts | +| `expert` | Authoritative, confident | Thought leadership, articles | +| `story` | Narrative flow, show rather than tell | Case studies, personal posts | -1. **Tier 0 (free)**: deterministic scripts run first — phrase scan, structure scan, constraint extraction. -2. **Tier 1 (cheap, parallel)**: small detector agents each get one compact rule-pack from `references/packs/` (~30 lines each: phrases, structure, register guards, voice, facts) and a chunk of text, and emit JSON findings. A small model with one narrow rulebook beats a small model with a 250-line monolith it will half-read. -3. **Tier 2 (one strong call)**: a single capable model rewrites with the merged findings handed to it. -4. **Tier 0 again**: every validation gate re-runs on the output; failures block. +## Scoring Rubric -Pack integrity (coverage of all scanner categories, size budgets, self-containment) is itself enforced by the eval suite via `scripts/check_packs.py`. +Eight criteria, one to five points each (40 maximum): directness, natural rhythm, concrete +verbs, reader trust, human authenticity, content density, fact preservation, and template +avoidance. Strict mode fails a rewrite scoring below 32/40. Criteria live in +`references/rubric.md`. ## Supported Agents @@ -207,105 +414,44 @@ This skill follows the [Agent Skills specification](https://agentskills.io) and - Roo Code - And [35+ other agents](https://github.com/vercel-labs/skills#supported-agents) -## Project Structure +## Wikipedia Sync -``` -unslop/ -├── SKILL.md # Main skill file (single-agent path + pointers) -├── README.md # This file -├── references/ -│ ├── taboo-phrases.md # Authoritative pattern catalog (incl. macro structure) -│ ├── pipeline.md # Tiered execution architecture -│ ├── packs/ # Small detector rule-packs + manifest -│ ├── maintenance.md # Add/list/wiki-sync procedures (eval-first) -│ ├── rubric.md # Scoring criteria -│ ├── edit-library.md # 24 transformation examples -│ ├── fact-preservation.md # What to preserve -│ └── personality-guide.md # Voice and personality guidance -├── presets/ # crisp / warm / expert / story voice deltas -├── scripts/ -│ ├── banned_phrase_scan.py # Phrase-level detection (severity, quote exemptions) -│ ├── structure_scan.py # Document-level detection (genre carve-outs) -│ ├── extract_constraints.py # Find must-preserve facts -│ ├── validate_preservation.py # Verify facts survived (--strict for regulated text) -│ ├── readability_metrics.py # Grade level, variance, staccato -│ ├── diff_check.py # Change percentage -│ ├── check_packs.py # Rule-pack integrity gate -│ └── wiki_sync.py # Wikipedia source page sync -├── evals/ -│ ├── adversarial-evals.json # Source of truth: 182 script + 33 skill cases -│ ├── run_adversarial.py # Deterministic runner (--only, --case, --list-gates) -│ ├── run_behavioral.sh # One-command behavioral run -│ ├── build_shared_benchmark.py # Generates shared-benchmark.json -│ ├── CHECKS.md # Machine-readable gate matrix + parallel protocol -│ └── check_*.py # Parity, doc, example, and pack gates -└── assets/ - └── examples/ # Before/after sets: article, LinkedIn, sales -``` - -## Scoring Rubric - -Eight criteria, 1-5 points each (40 max): - -1. **Directness** - No hedging or softening -2. **Natural Rhythm** - Sentence length variance (8-25 words) -3. **Concrete Verbs** - Specific actions, not abstractions -4. **Reader Trust** - No over-explaining -5. **Human Authenticity** - No performative emphasis -6. **Content Density** - Substance over filler -7. **Fact Preservation** - All numbers, names, dates intact -8. **Template Avoidance** - No AI structural clichés - -Passing score: 32/40 (80%) - -## Wikipedia Auto-Update - -The rules in this skill are partially derived from Wikipedia's [Signs of AI writing](https://en.wikipedia.org/wiki/Wikipedia:Signs_of_AI_writing) guide. The skill can sync itself with that page to pick up new patterns as Wikipedia editors add them. +Part of the phrase catalog derives from Wikipedia's +[Signs of AI writing](https://en.wikipedia.org/wiki/Wikipedia:Signs_of_AI_writing) guide. The +skill can sync itself with that page to pick up new patterns as editors add them: ```bash -# Tell the skill to check Wikipedia and update itself -/unslop --wiki-sync +python3 scripts/wiki_sync.py check ``` -This checks the Wikipedia page for changes, diffs against the last sync, and updates `taboo-phrases.md` and `banned_phrase_scan.py` with any new patterns — eval rows first, per `references/maintenance.md`. Wikipedia-specific patterns (broken wikitext, DOI issues, etc.) are skipped automatically. - -State is stored in `scripts/.wiki_sync_state.json` (gitignored). First run treats all content as new. +The sync diffs against the last run and proposes updates to `taboo-phrases.md` and the phrase +scanner, always eval-row-first per `references/maintenance.md`. Wikipedia-only patterns (broken +wikitext, DOI issues) are skipped, and state lives in `scripts/.wiki_sync_state.json` +(gitignored). ## Maintenance -All maintenance is eval-first: the row lands red in `evals/adversarial-evals.json` before any scanner or catalog edit makes it green. `references/maintenance.md` has the full procedures for: - -- Adding a banned phrase (`--add-phrase`) — needs a false-negative row plus a literal-use protection row -- Adding a structural pattern (`--add-structure`) -- Listing current phrases and patterns -- Wikipedia sync - -## Examples - -See `assets/examples/` for extended before/after transformations: - -- **Articles**: Technical writing, thought leadership -- **LinkedIn**: Professional social posts -- **Sales**: Product launches, cold outreach, case studies +All maintenance is eval-first: the row lands red in `evals/adversarial-evals.json` before any +scanner or catalog edit turns it green. `references/maintenance.md` holds the procedures for +adding a banned phrase (with its required literal-use protection row), adding a structural +pattern, listing current patterns, and running the Wikipedia sync. `CLAUDE.md` and `AGENTS.md` +hold the contribution rules; `docs/PRODUCT.md` holds the reasoning behind them. ## Philosophy -AI-generated text follows predictable patterns that humans recognize. This skill doesn't just find-and-replace words—it restructures content to read like a human wrote it, and it refuses to "fix" writing that was never broken. - -Key principles: -- **Cut ruthlessly**: If removal doesn't change meaning, remove it -- **Trust the reader**: They don't need "let that sink in" -- **Facts are sacred**: Numbers, names, dates, negations, and scope survive unchanged -- **Structure matters**: Binary contrasts, self-Q&A, uniform rhythm, and moralizing codas are tells -- **Do no harm**: Register hedges, literal vocabulary, and human writing stay intact -- **Quoted examples are exempt by default**: Tutorials and docs should not self-flag their own bad examples -- **The eval suite defines the product**: If a behavior matters, there is a row that fails without it - -## Requirements +AI text follows predictable patterns that readers learn to spot. unslop does not just swap +words; it restructures content to read as human, and it refuses to fix writing that was never +broken. The guiding principles: -- Python 3.8+ -- Any supported coding agent +- **Cut what carries no meaning.** If removal does not change the meaning, remove it. +- **Trust the reader.** They do not need `"let that sink in"`. +- **Facts are sacred.** Numbers, names, dates, negations, and scope survive unchanged. +- **Do no harm.** Register hedges, literal vocabulary, and already-human prose stay intact. +- **Voice runs under the constitution.** A mimic that reintroduces slop is a failure. +- **The eval suite defines the product.** If a behavior matters, a row fails without it. -## License +## Requirements and License -MIT +Python 3.8+ and any supported coding agent. Licensed MIT. + + diff --git a/SKILL.md b/SKILL.md index 618d78b..50dcb15 100644 --- a/SKILL.md +++ b/SKILL.md @@ -2,6 +2,8 @@ name: unslop description: Remove AI writing patterns from prose using either audit-only detection or a two-pass rewrite flow (diagnosis then reconstruction). Use this skill when editing, reviewing, or rewriting AI-generated content to make it sound human. Triggers on requests to "humanize", "de-slop", "fix AI text", "make it sound human", "remove AI patterns", or when reviewing text that contains obvious AI tells like "Here's the thing:", "Let that sink in", or "In today's fast-paced landscape". Also use when the user pastes text and says it "sounds like ChatGPT", "sounds robotic", "needs to sound more natural", or asks you to "clean up" drafted content before publishing. license: MIT +user-invocable: true +argument-hint: "[teach · cleanup · rewrite · mimic] [input]" metadata: author: claytonkim version: "2.3.0" @@ -11,10 +13,47 @@ metadata: Humanize AI-generated prose. Audit first. Rewrite only when the user asks for a rewrite. +## Routing + +**When the user invokes a sub-command (`/unslop teach ...`, `/unslop cleanup +...`), you MUST read `references/commands/.md` before acting. +Non-optional — the command file defines the flow, and skipping it drops steps the +user expects.** A bare `/unslop ` with no leading command word defaults to +`rewrite`. If the first word does not match a command but the intent clearly maps +to one (e.g. "flag the AI tells, don't change anything" → `cleanup` report-only), +load that command file and proceed as if invoked. + +| Command | Purpose | File | +|---------|---------|------| +| `rewrite` | Default two-pass de-slop: diagnose, reconstruct under the guards, validate. | [references/commands/rewrite.md](references/commands/rewrite.md) | +| `cleanup` | Co-writer: cheap detection, reviewable suggestions with contract gates; includes report-only "flag, change nothing". | [references/commands/cleanup.md](references/commands/cleanup.md) | +| `teach` | Agent-driven voice building: harvest, approve, profile, layered card, scored demo. | [references/commands/teach.md](references/commands/teach.md) | +| `mimic` | Voiced drafting or rewriting under the full gates; refine loop when one pass falls short. | [references/commands/mimic.md](references/commands/mimic.md) | +| _maintenance_ | Turn a wild AI-ism into an eval row and a PR (not a top-level verb). | [references/commands/contribute.md](references/commands/contribute.md) | + +### Routing by phrase + +Sub-flows are reachable by their natural names without being top-level verbs. +When the user says any of these, load the named file and jump to the flow: + +| The user says | Go to | +|---------------|-------| +| `audit` / "just flag it" / "don't change anything" | [references/commands/cleanup.md](references/commands/cleanup.md#report-only) | +| `harvest` / "what writing of mine do you have?" | [references/commands/teach.md](references/commands/teach.md#1-gather-samples-harvest) | +| `calibrate` / "the A/B game" / "quiz me on my voice" | [references/commands/teach.md](references/commands/teach.md#calibrate) | +| `refine` / "keep pushing until it sounds like me" | [references/commands/mimic.md](references/commands/mimic.md#refine) | +| voice check / "does this sound like me?" | [references/commands/mimic.md](references/commands/mimic.md#voice-check) | +| "found a new AI-ism" / "add this tell" | [references/commands/contribute.md](references/commands/contribute.md) | + +The shared doctrine below (register guards, validation gates and blocking +semantics, output formats, the script and reference tables) applies to every +command. The command files hold the flows; this file holds the constitution. + ## When to Use - User asks to humanize, de-slop, clean up, or make text sound natural. -- Drafts contain obvious AI tells: throat-clearing, scaffolded conclusions, inflated significance, em-dash abuse, or staccato fragment drama. +- Drafts contain obvious AI tells: throat-clearing, scaffolded conclusions, + inflated significance, em-dash abuse, or staccato fragment drama. - User asks for an audit, scan, or review of prose before publishing. ## Arguments @@ -23,15 +62,18 @@ Humanize AI-generated prose. Audit first. Rewrite only when the user asks for a |----------|-------------|---------| | `--preset` | Voice style: `crisp`, `warm`, `expert`, `story` | `crisp` | | `--strict` | Fail if rubric score < 32/40 | false | -| `--audit-only` | Flag AI patterns without rewriting | false | +| `--report` | Flag AI patterns without changing the text (cleanup) | false | | Input | Text to transform (argument, file path, or stdin) | required | -## Modes - -- `rewrite` (default): diagnose, rewrite, then validate. -- `audit-only`: diagnose and assess without rewriting. +## Scripts -Use audit-only when the user says "audit only," "flag only," "scan this," "just detect," "don't rewrite," or passes `--audit-only`. +| Script | Purpose | +|--------|---------| +| `scripts/voice_profile.py` | Build a deterministic stylometric voice profile from same-genre samples. | +| `scripts/voice_score.py` | Score a candidate against a voice profile with impostor-calibrated metrics and copy-gate reporting. | +| `scripts/voice_card.py` | Distill a profile plus samples into a layered, pack-sized voice card (core sheet plus per-situation sheets). | +| `evals/run_mimic_refine.py` | Iterative `--refine` hill-climb toward a voice under the removal gates, with A/DEV splits and a divergence guard. | +| `evals/mimic_stats.py` | Paired BCa-bootstrap and sign-flip stats for comparing mimic outputs to baselines. | ## Voice Presets @@ -44,61 +86,46 @@ Read one preset from `presets/` before writing. | `expert` | Authoritative, confident | Thought leadership, articles | | `story` | Narrative flow, show don't tell | Case studies, personal posts | -## Execution Paths - -### Single-Agent Path - -Use this when you are the only executor. Follow the same tiers sequentially: - -1. Extract facts and constraints: - ```bash - python3 scripts/extract_constraints.py <<< "$INPUT" - ``` -2. Scan the source: - ```bash - python3 scripts/banned_phrase_scan.py <<< "$INPUT" - python3 scripts/structure_scan.py <<< "$INPUT" - python3 scripts/readability_metrics.py <<< "$INPUT" - ``` - Use `--genre docs` or `--genre social` only when the input truly belongs to that genre. Use `banned_phrase_scan.py --include-quoted` only when the user wants quoted examples audited too. -3. Read the selected preset. Read `references/taboo-phrases.md` for phrase edge cases, `references/fact-preservation.md` for dense facts, `references/rubric.md` for strict scoring, and `references/personality-guide.md` only when clean output still feels anonymous. -4. If auditing, report issues by span, category, severity, and reason. Do not rewrite. -5. If rewriting, preserve every fact and register guard, remove the real AI tells, and match the chosen preset. -6. Validate the output with the gates below before returning it. - -### Orchestrated Path - -For multi-agent harnesses, use `references/pipeline.md`: Tier 0 deterministic scripts, Tier 1 small detector agents over `references/packs/`, Tier 2 one rewriter, then Tier 0 validation again. The skill must still work without that file; the pipeline is an efficiency architecture, not a dependency. - ## Rewrite Principles - Cut throat-clearing and scaffolding. Start with the claim. - Replace inflated importance with the concrete fact. - Prefer short, direct sentences, but avoid telegraphic staccato. -- Use em-dashes sparingly. A single appositive dash can be fine; clusters are a tell. Never trade a dash for a comma splice — if a dash is wrong, use a period. -- Facts are sacred: numbers, names, dates, URLs, quotes, code identifiers, units, and scope words must survive. -- Do not invent first-person experience, anecdotes, or certainty the source does not support. -- Do not replace AI slop with anti-slop register: "Not X. Y.", forced punch endings, or runs of tiny fragments. +- Use em-dashes sparingly. A single appositive dash can be fine; clusters are a + tell. Never trade a dash for a comma splice; if a dash is wrong, use a period. +- Facts are sacred: numbers, names, dates, URLs, quotes, code identifiers, units, + and scope words must survive. +- Do not invent first-person experience, anecdotes, or certainty the source does + not support. +- Do not replace AI slop with anti-slop register: "Not X. Y.", forced punch + endings, or runs of tiny fragments. ## Register Guards -Before removing a hedge or strengthening a sentence, check whether the register requires it: +Before removing a hedge or strengthening a sentence, check whether the register +requires it: -- Legal: keep hedges, negations, exceptions, section references, liability terms, and scope words. -- Medical/scientific: keep uncertainty, study limits, cohort limits, causation limits, and adverse-effect qualifiers. -- Security/safety: keep forceful absolutes such as "never", "must", "all input", and "do not" when they define a rule. -- Technical docs: keep precise terms, flags, API names, version numbers, file paths, and code semantics. +- Legal: keep hedges, negations, exceptions, section references, liability terms, + and scope words. +- Medical/scientific: keep uncertainty, study limits, cohort limits, causation + limits, and adverse-effect qualifiers. +- Security/safety: keep forceful absolutes such as "never", "must", "all input", + and "do not" when they define a rule. +- Technical docs: keep precise terms, flags, API names, version numbers, file + paths, and code semantics. -If a gate fails twice after rewrite, escalate model tier rather than adding more prompt rules. The failure is execution quality. +If a gate fails twice after rewrite, escalate model tier rather than adding more +prompt rules. The failure is execution quality. ## Validation -Run after every rewrite: +Run after every rewrite or voiced draft: ```bash python3 scripts/validate_preservation.py original.txt transformed.txt python3 scripts/banned_phrase_scan.py <<< "$OUTPUT" python3 scripts/structure_scan.py <<< "$OUTPUT" +python3 scripts/silhouette_scan.py <<< "$OUTPUT" python3 scripts/readability_metrics.py <<< "$OUTPUT" python3 scripts/diff_check.py original.txt transformed.txt ``` @@ -107,16 +134,26 @@ Blocking output failures: - Any hard banned-phrase hit. - Any `anti_slop_register` hit, even if soft. -- Any `structure_scan.py` flag unless the actual genre justifies `--genre docs` or `--genre social`. -- Preservation warnings that show a dropped or changed negation, hedge, scope word, number, date, name, quote, URL, unit, or code identifier. The default gate warns without failing; run `validate_preservation.py --strict` for legal, medical, security, or scientific text so these exit non-zero. +- Any `structure_scan.py` flag unless the actual genre justifies `--genre docs` or + `--genre social`. +- Any `silhouette_scan.py` flag (`silhouette_penalty >= 1.0`) unless the genre + justifies it: `--genre docs` retains the outline-following tell + (`heading_preview`) because reference docs still should not read as a + preview-then-fulfill template. +- Preservation warnings that show a dropped or changed negation, hedge, scope + word, number, date, name, quote, URL, unit, or code identifier. The default + gate warns without failing; run `validate_preservation.py --strict` for legal, + medical, security, or scientific text so these exit non-zero. - Staccato cadence in readability metrics. - Rubric score below 32/40 in strict mode. -Validation scripts are necessary but not enough. Re-read negations, conditionals, scope, certainty, and party relationships yourself. +Validation scripts are necessary but not enough. Re-read negations, conditionals, +scope, certainty, and party relationships yourself. ## Output Format -For a quick rewrite, return the cleaned text only. For audit-only: +For a quick rewrite, return the cleaned text only. For audit-only (cleanup +`--report`): ```markdown ## Issues Found @@ -164,7 +201,11 @@ For strict or requested analysis: | File | When to Read | |------|-------------| -| `references/pipeline.md` | Orchestrated tiered execution. | +| `references/commands/*.md` | The routed command flows (rewrite, cleanup, teach, mimic, contribute). | +| `references/pipeline.md` | Orchestrated tiered execution for multi-agent harnesses. | +| `references/mimic.md` | Teach/mimic internals: card anatomy, scoring, baselines, the `--refine` loop. | +| `references/harvest.md` | Harvest adapter internals, contamination tripwire, privacy rules. | +| `references/calibrate.md` | The teach A/B calibration game: pairs, scoring, conflict surfacing. | | `references/packs/*.md` | Small detector-agent rule packs. | | `references/taboo-phrases.md` | Authoritative phrase catalog and scanner categories. | | `references/fact-preservation.md` | Constraint preservation rules. | @@ -173,8 +214,15 @@ For strict or requested analysis: | `references/personality-guide.md` | Adding genuine voice without fake personality. | | `references/maintenance.md` | Pattern add/list procedures and Wikipedia sync. | | `presets/*.md` | Voice-specific deltas. | -| `assets/examples/*.md` | Extended before/after examples by content type (article, LinkedIn, sales). | +| `assets/examples/*.md` | Extended before/after examples by content type. | ## Maintenance -The eval suite defines the product. Add or change patterns eval-first in `evals/adversarial-evals.json`; do not edit `evals/evals.json`. New scanner patterns need one false-negative row and one false-positive protection row. Agent behavior changes need a `skill` row and a regenerated shared benchmark. For the concrete procedures (add a phrase or structure, list current patterns, sync with Wikipedia's signs-of-AI-writing page), read `references/maintenance.md`. +The eval suite defines the product. Add or change patterns eval-first in +`evals/adversarial-evals.json`; do not edit `evals/evals.json`. New scanner +patterns need one false-negative row and one false-positive protection row. Agent +behavior changes need a `skill` row and a regenerated shared benchmark. For the +concrete procedures (add a phrase or structure, list current patterns, sync with +Wikipedia's signs-of-AI-writing page), read `references/maintenance.md`. Found a +new AI-ism in the wild? `references/commands/contribute.md` turns the exact +snippet into an eval row and a structured PR, keeping both user-confirmation gates. diff --git a/docs/PRODUCT.md b/docs/PRODUCT.md new file mode 100644 index 0000000..c7415b9 --- /dev/null +++ b/docs/PRODUCT.md @@ -0,0 +1,102 @@ +# unslop: Product Doctrine + +Here live the product decisions that govern the repo. CLAUDE.md holds the +contribution rules; this file holds the why. When a change conflicts with this document, +one of the two is wrong, and deciding which requires a human. + +## Identity and shape + +unslop is an open-source tool packaged as an agent skill. The repo is the artifact. There +is no published package, and none is planned unless distribution truly requires it. Entry +points are plain scripts under `scripts/`, runnable by any host agent. + +The host agent is the runtime. Linting runs when an agent invokes it during writing — not +as a pre-commit hook, not as a daemon, not as an editor extension. Integrations beyond the +agent are other people's projects. + +The eval suite defines the product. Every behavior worth having is pinned by a row that +fails without it. This is constitutional, not aspirational: coverage gates make it +structurally impossible to add a detection pattern without eval rows, and the docs +themselves are gate-checked for drift. + +## The axis + +The product spans one axis. On the left sits removal: detect AI writing patterns and strip +them. On the right sits reconstruction: write the way a specific human writes. Product weight +sits roughly 70/30 toward the left. + +The left end is deterministic, cheap, CI-friendly, and objectively benchmarkable. It is +the trust asset. The right end is generative and register-sensitive, and it operates under +the left end's constitution: every mimic or rewrite output must pass every removal gate. A +voice rewrite that reintroduces slop is a failure, full stop... which is itself a phrase +the scanner would flag, so: a failure, period. The gates apply to this document too. + +## Bounds + +- Refuse almost nothing. The tool removes AI-isms; it is not a taste arbiter. Literary + prose is in bounds. Long documents are in bounds via chunking. +- English only. Non-English input gets a cheap detection and a clear decline — the one + graceful refusal in the product. +- Mimicry accepts any writing style the user supplies samples for. There is no rights or + attestation machinery; the sheet of samples is the only credential. + +## Feature surface + +**Always-on linting.** Deterministic scanners (phrase, structure, silhouette) are the +free, always-on layer. Model-based detection exists only for judge-only pattern families +and uses the cheapest capable model, per measured parity data rather than assumption. + +**Co-writer mode.** On detection, findings surface to the user as structured suggestions +(span, severity, rationale, proposed replacement), never as silent rewrites. Hard findings +propose replacements; soft findings ask questions. Contract gates make "accept all" safe +by construction: replacements are span-minimal, scan clean, and preserve every constraint. + +**Teach and mimic.** `teach` distills writing samples into a voice: a machine profile +(`profile.json`, the deterministic referee) and a layered voice card (`card.md` plus +situation sheets) that any generating model can follow in context. Teach is interactive +and coverage-driven — it classifies what the samples demonstrate and prompts for what is +missing. Dimensions with no sample evidence get no sheet and are listed as uncovered; +the card never fabricates. `mimic` writes with the card, scored against the profile, with +all removal gates blocking. An internal refine loop hill-climbs the card against held-out +samples, with a divergence guard against reward hacking. + +**Harvest.** Cheap agents bootstrap the teach corpus from chat transcripts and designated +folders. Authorship separation is the product: assistant-authored text in a voice profile +would teach the exact register unslop removes. Deterministic parsing separates turns, a +scanner tripwire flags suspicious candidates, and nothing enters a profile without human +approval. + +**Calibrate.** An A/B preference game gathers voice signal at tap-level effort. Pairs are +dimension-controlled minimal edits of the user's own passages, mostly produced by +deterministic transforms. Preferences aggregate per dimension with confidence bounds, +actively targeting the least-known dimension, and conflicts with sample-measured values +surface to the user rather than being silently resolved. + +**Contribute.** A wild-caught AI-ism becomes an eval row built from the exact specimen, +verified red-first, and a structured PR into this repo. Redaction never alters the +specimen's tell, publication requires explicit user approval, and the pipeline runs fully +offline until the user says otherwise. + +## Model tiering + +Tiering is measured, not assumed; `evals/run_model_parity.py` re-measures it and +`references/pipeline.md` records the current table. The standing conclusions from the +2026-07-06 runs: + +- Span-scoped replacement is safe on the cheapest tier. The gates carry the safety — the + single bench failure was the most expensive model dropping a fact, caught by the + preservation gate. +- Full rewrites of register-sensitive text erode hedges, absolutes, and legal negations + on cheap tiers. That work belongs to frontier models, re-scanned afterward. +- Macro structure defeated every model tested. No model self-checks document shape from + prose instructions; structure is always machine-detected and machine-gated. + +Touching a model-dependent feature means re-running the parity evals across +both the GPT and Anthropic spectrums. + +## Growth + +Detection catalogs decay as generators evolve. Growth comes from adversarial refresh: +live bench outputs supply fresh specimens, contribute turns each specimen into a +red-first row, and coverage enforcement keeps every new pattern exercised. Yesterday's +miss becomes tomorrow's regression test, and nothing relies on anyone's good habits. diff --git a/evals/CHECKS.md b/evals/CHECKS.md index 3b0c5cf..8c13975 100644 --- a/evals/CHECKS.md +++ b/evals/CHECKS.md @@ -15,6 +15,27 @@ Current gates: "blocking": true, "needs": [] }, + { + "id": "harvest-suite", + "command": "python3 evals/run_adversarial.py --only HARV", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "contribute-suite", + "command": "python3 evals/run_adversarial.py --only CONTRIB", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "calibrate-suite", + "command": "python3 evals/run_adversarial.py --only CAL", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, { "id": "shared-benchmark-check", "command": "python3 evals/build_shared_benchmark.py --check", @@ -38,6 +59,55 @@ Current gates: "blocking": true, "needs": [] }, + { + "id": "pattern-coverage", + "command": "python3 evals/check_pattern_coverage.py", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "voice-scorer", + "command": "python3 evals/check_voice.py --separation && python3 evals/check_voice.py --gi && python3 evals/check_voice.py --gaming && python3 evals/check_voice.py --profiles", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "add-pattern-kata", + "command": "python3 evals/kata_add_pattern.py --run", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "command-router-parity", + "command": "python3 evals/check_commands.py", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "seeded-docs", + "command": "python3 evals/check_seeded_docs.py", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "paired-fixture-hygiene", + "command": "python3 evals/check_pairs.py", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "mimic-logic", + "command": "python3 evals/run_adversarial.py --only MIMIC --only CARD", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, { "id": "pack-structure", "command": "python3 scripts/check_packs.py", @@ -69,6 +139,20 @@ Current gates: "blocking": true, "needs": [] }, + { + "id": "silhouette-scan", + "command": "python3 scripts/silhouette_scan.py < transformed.txt", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, + { + "id": "silhouette-check", + "command": "python3 evals/check_silhouette.py", + "pass_criterion": "exit 0", + "blocking": true, + "needs": [] + }, { "id": "validate-preservation", "command": "python3 scripts/validate_preservation.py original.txt transformed.txt", diff --git a/evals/TUNE-RESULTS.md b/evals/TUNE-RESULTS.md index 0990f86..1645fe7 100644 --- a/evals/TUNE-RESULTS.md +++ b/evals/TUNE-RESULTS.md @@ -34,3 +34,18 @@ Next skill work: invocation telemetry. - Harness v0.4.2 can emit null judge scores; coerce them before `benchmark` as shown in `evals/BEHAVIORAL-EVALS.md`. + +## Holdout run — 2026-07-06 (post-assembly, branch eval-forms-integrate) + +12 holdout cases, judged with `claude -p`, exact bookkeeping (40 judge rows): + +| variant | objective (deterministic backstops) | combined (judge + scripts) | +|---------|-------------------------------------|----------------------------| +| with_skill | 0.917 | 0.872 | +| without_skill | 0.875 | 0.879 | + +Objective lift +4.2 points on held-out cases; judge-blended is flat. Consistent +with the standing interpretation: the deterministic layer (facts, register, +structure) carries the measurable value, the judge cannot distinguish the +prose, and the without-skill baseline is contaminated by the globally installed +skill (see run_local.py). Holdback remains sealed. diff --git a/evals/_check_support.py b/evals/_check_support.py new file mode 100644 index 0000000..764c645 --- /dev/null +++ b/evals/_check_support.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Shared helpers for evals/check_*.py scripts. + +Every check_*.py that shells out to a scripts/*.py CLI used to carry its own +copy-pasted ``run()`` helper. Only check_contrib.py's copy carried a +``timeout=60`` safety net (a hung scanner subprocess fails the check instead of +hanging the whole suite); the others silently dropped it on copy. This module +is the one place that safety net lives now, so every caller gets it. +""" + +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def run(cmd, timeout=60): + """Run `cmd` from ROOT, capturing text stdout/stderr, with a timeout.""" + return subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, timeout=timeout) + + +def load_evals(): + """Return the parsed "evals" list from evals/adversarial-evals.json.""" + suite_path = ROOT / "evals" / "adversarial-evals.json" + suite = json.loads(suite_path.read_text(encoding="utf-8")) + return suite["evals"] diff --git a/evals/adversarial-evals.json b/evals/adversarial-evals.json index a397935..7b89352 100644 --- a/evals/adversarial-evals.json +++ b/evals/adversarial-evals.json @@ -16,6 +16,7 @@ { "id": "FP-01", "category": "scanner_false_positive", + "protects": "jargon", "title": "Financial 'leverage' is a real word, not jargon", "target": "script", "command": [ @@ -36,6 +37,7 @@ { "id": "FP-02", "category": "scanner_false_positive", + "protects": "jargon", "title": "Sailors literally navigate", "target": "script", "command": [ @@ -56,6 +58,7 @@ { "id": "FP-03", "category": "scanner_false_positive", + "protects": "promotional", "title": "A museum tapestry / an arena's capacity are not promotional slop", "target": "script", "command": [ @@ -76,6 +79,7 @@ { "id": "FP-06", "category": "scanner_false_positive", + "protects": "jargon", "title": "Literal 'delve into a place' — documented regex limit", "target": "script", "command": [ @@ -97,6 +101,7 @@ { "id": "FP-04", "category": "scanner_false_positive", + "protects": "throat_clearing", "title": "Substring match: 'the real' inside 'the real estate'", "target": "script", "command": [ @@ -121,6 +126,7 @@ { "id": "FP-05", "category": "scanner_false_positive", + "protects": "emphasis_crutch", "title": "Sentence-final 'period.' is an ordinary word", "target": "script", "command": [ @@ -197,6 +203,7 @@ { "id": "FP-14", "category": "scanner_false_positive", + "protects": "emphasis_crutch", "title": "Literal 'is it genuine?' sense is not the 'X is real' tell", "target": "script", "command": [ @@ -545,6 +552,7 @@ { "id": "FP-07", "category": "scanner_false_positive", + "protects": "jargon", "title": "A horse harness is not 'harness the power'", "target": "script", "command": [ @@ -565,6 +573,7 @@ { "id": "FP-08", "category": "scanner_false_positive", + "protects": "jargon", "title": "A foster family is not 'foster a culture'", "target": "script", "command": [ @@ -585,6 +594,7 @@ { "id": "FP-09", "category": "quote_exemption", + "protects": "throat_clearing", "title": "Curly (smart) double quotes are exempt too", "target": "script", "command": [ @@ -1048,6 +1058,7 @@ { "id": "FP-10", "category": "scanner_false_positive", + "protects": "copula_avoidance", "title": "Literal 'serves as a kitchen' is not copula avoidance", "target": "script", "command": [ @@ -1068,6 +1079,7 @@ { "id": "FP-11", "category": "scanner_false_positive", + "protects": "significance_inflation", "title": "Literal 'elevated glucose' is not hype", "target": "script", "command": [ @@ -1136,6 +1148,7 @@ { "id": "FP-12", "category": "scanner_false_positive", + "protects": "anti_slop_register", "title": "Anti-slop detector spares normal varied prose", "target": "script", "command": [ @@ -1265,6 +1278,7 @@ { "id": "FP-13", "category": "scanner_false_positive", + "protects": "jargon", "title": "Literal load-bearing and wedge are not AI tells", "target": "script", "command": [ @@ -1453,6 +1467,7 @@ { "id": "FP-15", "category": "scanner_false_positive", + "protects": "jargon", "title": "Garner, North Carolina is a place name", "target": "script", "command": [ @@ -1473,6 +1488,7 @@ { "id": "FP-16", "category": "scanner_false_positive", + "protects": "jargon", "title": "Unpacked boxes is literal moving prose", "target": "script", "command": [ @@ -1493,6 +1509,7 @@ { "id": "FP-17", "category": "scanner_false_positive", + "protects": "jargon", "title": "A glass showcase is a display case", "target": "script", "command": [ @@ -1513,6 +1530,7 @@ { "id": "FP-18", "category": "scanner_false_positive", + "protects": "jargon", "title": "Bolster pillow is literal bedding", "target": "script", "command": [ @@ -1533,6 +1551,7 @@ { "id": "FP-19", "category": "scanner_false_positive", + "protects": "jargon", "title": "Double down is literal blackjack play", "target": "script", "command": [ @@ -1553,6 +1572,7 @@ { "id": "FP-20", "category": "scanner_false_positive", + "protects": "jargon", "title": "Leaning into a corner is physical motion", "target": "script", "command": [ @@ -1573,6 +1593,7 @@ { "id": "FP-21", "category": "scanner_false_positive", + "protects": "jargon", "title": "Stakeholder is a legal escrow role", "target": "script", "command": [ @@ -1737,6 +1758,7 @@ { "id": "FP-22", "category": "scanner_false_positive", + "protects": "significance_inflation", "title": "Pivotal moment does not substring-match pivot", "target": "script", "command": [ @@ -1837,6 +1859,7 @@ { "id": "FP-23", "category": "scanner_false_positive", + "protects": "emphasis_crutch", "title": "Embedded why-this-matters clause is not self-QA", "target": "script", "command": [ @@ -1881,6 +1904,7 @@ { "id": "FP-24", "category": "scanner_false_positive", + "protects": "negative_listing", "title": "It is not clear followed by unrelated sentence is not negative listing", "target": "script", "command": [ @@ -1973,6 +1997,7 @@ { "id": "FP-25", "category": "scanner_false_positive", + "protects": "negative_parallelism", "title": "Non-parallel not-just clause is safe", "target": "script", "command": [ @@ -1993,6 +2018,7 @@ { "id": "FP-26", "category": "scanner_false_positive", + "protects": "jargon", "title": "Literal wedge into kerf is construction prose", "target": "script", "command": [ @@ -2013,6 +2039,7 @@ { "id": "FP-27", "category": "scanner_false_positive", + "protects": "jargon", "title": "Literal wedge between logs is construction prose", "target": "script", "command": [ @@ -2105,6 +2132,7 @@ { "id": "FP-28", "category": "scanner_false_positive", + "protects": "ai_vocabulary", "title": "Legal notwithstanding boilerplate is exempt", "target": "script", "command": [ @@ -2149,6 +2177,7 @@ { "id": "FP-29", "category": "scanner_false_positive", + "protects": "jargon", "title": "Literal landscape of a place is exempt", "target": "script", "command": [ @@ -2217,6 +2246,7 @@ { "id": "FP-30", "category": "scanner_false_positive", + "protects": "rhetorical_question", "title": "Genuine interrogative result question is safe", "target": "script", "command": [ @@ -2261,6 +2291,7 @@ { "id": "FP-31", "category": "scanner_false_positive", + "protects": "hedge_stack", "title": "Plain medical hedge remains safe", "target": "script", "command": [ @@ -2305,6 +2336,7 @@ { "id": "FP-32", "category": "scanner_false_positive", + "protects": "reader_addressing", "title": "Legit conditional whether you are eligible is safe", "target": "script", "command": [ @@ -2373,6 +2405,7 @@ { "id": "FP-33", "category": "scanner_false_positive", + "protects": "formatting", "title": "Single emoji in body text is safe", "target": "script", "command": [ @@ -2393,6 +2426,7 @@ { "id": "FP-34", "category": "scanner_false_positive", + "protects": "em_dash_overuse", "title": "One em dash per paragraph is safe", "target": "script", "command": [ @@ -2485,6 +2519,7 @@ { "id": "FP-35", "category": "scanner_false_positive", + "protects": "ai_vocabulary", "title": "Silicon substrate is literal electronics prose", "target": "script", "command": [ @@ -2673,6 +2708,7 @@ { "id": "FP-36", "category": "scanner_false_positive", + "protects": "negative_parallelism", "title": "Merely decorative hinge does not trigger not-merely pattern", "target": "script", "command": [ @@ -3261,6 +3297,7 @@ { "id": "FP-37", "category": "scanner_false_positive", + "protects": "exclamation_overuse", "title": "Exclamations in separate paragraphs are not stacked enthusiasm", "target": "script", "command": [ @@ -3305,6 +3342,7 @@ { "id": "FP-38", "category": "scanner_false_positive", + "protects": "ai_vocabulary", "title": "A growth substrate for a cell culture is lab equipment, not metaphor", "target": "script", "command": [ @@ -4599,6 +4637,7 @@ { "id": "FP-39", "category": "scanner_false_positive", + "protects": "reader_addressing", "title": "'Whether you're right or wrong' is ordinary English, not audience flattery", "target": "script", "command": [ @@ -4619,6 +4658,7 @@ { "id": "FP-40", "category": "scanner_false_positive", + "protects": "reader_addressing", "title": "'Whether you're driving or flying' is a plain alternative, not flattery", "target": "script", "command": [ @@ -4663,6 +4703,7 @@ { "id": "FP-41", "category": "scanner_false_positive", + "protects": "ai_vocabulary", "title": "Soldiers bolstering a trench wall is construction, not AI vocabulary", "target": "script", "command": [ @@ -4683,6 +4724,7 @@ { "id": "FP-42", "category": "scanner_false_positive", + "protects": "emphasis_crutch", "title": "Compound noun phrase 'real gutters' is not the ' is real' tell", "target": "script", "command": [ @@ -4703,6 +4745,7 @@ { "id": "FP-43", "category": "scanner_false_positive", + "protects": "numeric_parallelism", "title": "'Six aesthetics, zero prompting.' is NOT the 'One X, N Y.' numeric-parallelism shape", "target": "script", "command": [ @@ -4723,6 +4766,7 @@ { "id": "FP-44", "category": "scanner_false_positive", + "protects": "contrastive_definition", "title": "Plain declarative clause fragment", "target": "script", "command": [ @@ -4743,6 +4787,7 @@ { "id": "FP-45", "category": "scanner_false_positive", + "protects": "imperative_slogan", "title": "Imperative + noun list is not a two-beat imperative slogan", "target": "script", "command": [ @@ -4763,6 +4808,7 @@ { "id": "FP-46", "category": "scanner_false_positive", + "protects": "imperative_slogan", "title": "Two short subject-verb sentences are not a two-beat imperative slogan", "target": "script", "command": [ @@ -4783,6 +4829,7 @@ { "id": "FP-47", "category": "scanner_false_positive", + "protects": "contrastive_definition", "title": "Imperative correction 'Use X, not Y.' is legitimate instruction", "target": "script", "command": [ @@ -4803,6 +4850,7 @@ { "id": "FP-48", "category": "scanner_false_positive", + "protects": "contrastive_definition", "title": "Numeric correction 'X, not N.' is legitimate data", "target": "script", "command": [ @@ -4823,6 +4871,7 @@ { "id": "FP-49", "category": "scanner_false_positive", + "protects": "fragment_template", "title": "Single ' that .' fragment is fine (frequency gate)", "target": "script", "command": [ @@ -4843,6 +4892,7 @@ { "id": "FP-50", "category": "scanner_false_positive", + "protects": "headline_cadence", "title": "One headline slogan pair plus normal prose is fine (frequency gate)", "target": "script", "command": [ @@ -4863,6 +4913,7 @@ { "id": "FP-51", "category": "scanner_false_positive", + "protects": "headline_cadence", "title": "Short-sentence prose flowing inside a paragraph is not a headline cadence", "target": "script", "command": [ @@ -5224,6 +5275,6703 @@ "equals": {} } ] + }, + { + "id": "DET-18", + "category": "scanner_detection", + "title": "Structural pattern exercised: stakeholder + alignment collocation flags as jargon", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "We need stronger stakeholder alignment before the launch review next week.", + "failure_mode": "The jargon structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "jargon" + } + ] + }, + { + "id": "DET-19", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'a deeper understanding of' significance-inflation frame", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "This chapter gives readers a deeper understanding of the underlying market forces.", + "failure_mode": "The significance_inflation structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "significance_inflation" + } + ] + }, + { + "id": "DET-20", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'not because X. Because Y.' binary-contrast cadence", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "We rebuilt the parser not because it was slow. Because it was wrong.", + "failure_mode": "The binary_contrast structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "binary_contrast" + } + ] + }, + { + "id": "DET-21", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'X isn't the problem. Y is.' binary contrast", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Latency isn't the problem. Throughput is.", + "failure_mode": "The binary_contrast structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "binary_contrast" + } + ] + }, + { + "id": "DET-22", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'feels like X. It's actually Y.' binary contrast", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "It feels like a caching bug. It's actually a race condition in the writer.", + "failure_mode": "The binary_contrast structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "binary_contrast" + } + ] + }, + { + "id": "DET-23", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'That's it. That's the ...' dramatic-fragment cadence", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "You flip a single feature flag and the entire ingestion pipeline reroutes itself automatically. That's it. That's the whole migration.", + "failure_mode": "The dramatic_fragment structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "anti_slop_register" + } + ] + }, + { + "id": "DET-24", + "category": "scanner_detection", + "title": "Structural pattern exercised: sentence-initial 'The loop.' dramatic fragment", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Every request threads back through one place. The retry loop.", + "failure_mode": "The dramatic_fragment structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "dramatic_fragment" + } + ] + }, + { + "id": "DET-25", + "category": "scanner_detection", + "title": "Structural pattern exercised: ' things. One thing' dramatic fragment", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The rewrite changed two things. One thing mattered: the schema.", + "failure_mode": "The dramatic_fragment structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "dramatic_fragment" + } + ] + }, + { + "id": "DET-26", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'What if I told you' rhetorical setup", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "What if I told you the slowest endpoint was never the database at all?", + "failure_mode": "The rhetorical_setup structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_setup" + } + ] + }, + { + "id": "DET-27", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'Here's what I mean:' rhetorical setup", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The API should fail loudly on a stale token. Here's what I mean: return a 401 immediately.", + "failure_mode": "The rhetorical_setup structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_setup" + } + ] + }, + { + "id": "DET-28", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'Think about it:' rhetorical setup", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The cache never expires on its own. Think about it: nothing ever evicts a stale key.", + "failure_mode": "The rhetorical_setup structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_setup" + } + ] + }, + { + "id": "DET-29", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'and that's okay.' unnecessary-permission cadence", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Sometimes the rollback takes the whole afternoon, and that's okay.", + "failure_mode": "The unnecessary_permission structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "unnecessary_permission" + } + ] + }, + { + "id": "DET-30", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'not only X but also Y' negative parallelism", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The framework not only ships a router but also bundles its own state layer.", + "failure_mode": "The negative_parallelism structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "negative_parallelism" + } + ] + }, + { + "id": "DET-31", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'it's not just about X, it's about Y' negative parallelism", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "It's not just about raw speed, it's about predictable tail latency under load.", + "failure_mode": "The negative_parallelism structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "negative_parallelism" + } + ] + }, + { + "id": "DET-32", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'not X, not Y, but Z' negative parallelism", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "We optimized for reads, not writes, not batch jobs, but live dashboards.", + "failure_mode": "The negative_parallelism structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "contrastive_definition" + } + ] + }, + { + "id": "DET-33", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'is not X. Rather, it is Y' negative parallelism", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Test coverage is not a finish line. Rather, it is a running measure of confidence.", + "failure_mode": "The negative_parallelism structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "negative_parallelism" + } + ] + }, + { + "id": "DET-34", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'from X to Y, from A to B' false range", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The dataset ranges from tiny to huge, from sparse to dense across every partition.", + "failure_mode": "The false_range structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "false_range" + } + ] + }, + { + "id": "DET-35", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'Despite its X, Y faces challenges' formulaic challenges", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Despite its clean interface, the scheduler faces challenges under bursty traffic.", + "failure_mode": "The formulaic_challenges structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "formulaic_challenges" + } + ] + }, + { + "id": "DET-36", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'however, it is not without its challenges' formula", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The approach scales well; however, it is not without its challenges at the edges.", + "failure_mode": "The formulaic_challenges structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "formulaic_challenges" + } + ] + }, + { + "id": "DET-37", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'While X, Y remains a concern' formulaic challenges", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "While the numbers look strong, data drift remains a concern for the model.", + "failure_mode": "The formulaic_challenges structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "formulaic_challenges" + } + ] + }, + { + "id": "DET-38", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'Although X, Y remains a challenge' false concession", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Although the team shipped the fix, latency remains a challenge for large tenants.", + "failure_mode": "The false_concession structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "false_concession" + } + ] + }, + { + "id": "DET-39", + "category": "scanner_detection", + "title": "Structural pattern exercised: one-word sentence run (staccato fragmentation)", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The build just died. Silence. Nothing. Gone. Done.", + "failure_mode": "The staccato_fragmentation structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "anti_slop_register" + } + ] + }, + { + "id": "DET-40", + "category": "scanner_detection", + "title": "Paragraph-initial 'So,' filler opener flags", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "So, the release went out and immediately paged the on-call engineer.", + "failure_mode": "The filler_opener structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "filler_opener" + } + ] + }, + { + "id": "DET-41", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'The reason is:' colon reveal", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The reason is: the retry budget was never wired into the client.", + "failure_mode": "The colon_reveal structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "colon_reveal" + } + ] + }, + { + "id": "DET-42", + "category": "scanner_detection", + "title": "Structural pattern exercised: parenthetical hedge '(perhaps ...)'", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The migration finished overnight (perhaps too quietly for anyone to notice).", + "failure_mode": "The parenthetical_hedging structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "parenthetical_hedging" + } + ] + }, + { + "id": "DET-43", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'Here are N reasons' numbered-list inflation", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Here are 5 reasons the rollout slipped past its original deadline.", + "failure_mode": "The numbered_list_inflation structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "numbered_list_inflation" + } + ] + }, + { + "id": "DET-44", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'What does this mean for' rhetorical question", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "What does this mean for the average request path through the gateway?", + "failure_mode": "The rhetorical_question structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_question" + } + ] + }, + { + "id": "DET-45", + "category": "scanner_detection", + "title": "Structural pattern exercised: 'So why should you care' rhetorical question", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "So why should you care about a two-millisecond regression?", + "failure_mode": "The rhetorical_question structural pattern could regress or be deleted with no eval exercising it.", + "correct_behavior": "Scanner flags the structural tell (>=1 violation).", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_question" + } + ] + }, + { + "id": "REC-21", + "category": "scanner_recall", + "title": "Catalog coverage: 'the uncomfortable truth is' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned the uncomfortable truth is and the group moved on to the next agenda item.\nThe consultant wrote that it turns out before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating let me be clear while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged the truth is and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned i'll say it again and the group moved on to the next agenda item.\nThe consultant wrote that i'm going to be honest before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating can we talk about while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged here's what i find interesting and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned here's the problem though and the group moved on to the next agenda item.\nThe consultant wrote that let's be real before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "the uncomfortable truth is" + }, + { + "type": "violation_phrase_contains", + "value": "it turns out" + }, + { + "type": "violation_phrase_contains", + "value": "let me be clear" + }, + { + "type": "violation_phrase_contains", + "value": "the truth is" + }, + { + "type": "violation_phrase_contains", + "value": "i'll say it again" + }, + { + "type": "violation_phrase_contains", + "value": "i'm going to be honest" + }, + { + "type": "violation_phrase_contains", + "value": "can we talk about" + }, + { + "type": "violation_phrase_contains", + "value": "here's what i find interesting" + }, + { + "type": "violation_phrase_contains", + "value": "here's the problem though" + }, + { + "type": "violation_phrase_contains", + "value": "let's be real" + } + ] + }, + { + "id": "REC-22", + "category": "scanner_recall", + "title": "Catalog coverage: 'here's the deal' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned here's the deal and the group moved on to the next agenda item.\nThe consultant wrote that here's what nobody tells you before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating let that sink in while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged this matters because and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned make no mistake and the group moved on to the next agenda item.\nThe consultant wrote that here's why that matters before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating read that again while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged this is important and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned this cannot be overstated and the group moved on to the next agenda item.\nThe consultant wrote that in summary before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "here's the deal" + }, + { + "type": "violation_phrase_contains", + "value": "here's what nobody tells you" + }, + { + "type": "violation_phrase_contains", + "value": "let that sink in" + }, + { + "type": "violation_phrase_contains", + "value": "this matters because" + }, + { + "type": "violation_phrase_contains", + "value": "make no mistake" + }, + { + "type": "violation_phrase_contains", + "value": "here's why that matters" + }, + { + "type": "violation_phrase_contains", + "value": "read that again" + }, + { + "type": "violation_phrase_contains", + "value": "this is important" + }, + { + "type": "violation_phrase_contains", + "value": "this cannot be overstated" + }, + { + "type": "violation_phrase_contains", + "value": "in summary" + } + ] + }, + { + "id": "REC-23", + "category": "scanner_recall", + "title": "Catalog coverage: 'to summarize' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned to summarize and the group moved on to the next agenda item.\nThe consultant wrote that thirdly before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating underscores the importance while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged underscoring the importance and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned highlights the importance and the group moved on to the next agenda item.\nThe consultant wrote that ever-changing before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating speaks for itself while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged paints a clear picture and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned leaves a lasting impact and the group moved on to the next agenda item.\nThe consultant wrote that lasting impact before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "to summarize" + }, + { + "type": "violation_phrase_contains", + "value": "thirdly" + }, + { + "type": "violation_phrase_contains", + "value": "underscores the importance" + }, + { + "type": "violation_phrase_contains", + "value": "underscoring the importance" + }, + { + "type": "violation_phrase_contains", + "value": "highlights the importance" + }, + { + "type": "violation_phrase_contains", + "value": "ever-changing" + }, + { + "type": "violation_phrase_contains", + "value": "speaks for itself" + }, + { + "type": "violation_phrase_contains", + "value": "paints a clear picture" + }, + { + "type": "violation_phrase_contains", + "value": "leaves a lasting impact" + }, + { + "type": "violation_phrase_contains", + "value": "lasting impact" + } + ] + }, + { + "id": "REC-24", + "category": "scanner_recall", + "title": "Catalog coverage: 'enduring legacy' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned enduring legacy and the group moved on to the next agenda item.\nThe consultant wrote that lasting legacy before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating watershed moment while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged in the realm of and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned is widely regarded as and the group moved on to the next agenda item.\nThe consultant wrote that some critics argue before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating observers have noted while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged would be complete without and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned future prospects and the group moved on to the next agenda item.\nThe consultant wrote that future outlook before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "enduring legacy" + }, + { + "type": "violation_phrase_contains", + "value": "lasting legacy" + }, + { + "type": "violation_phrase_contains", + "value": "watershed moment" + }, + { + "type": "violation_phrase_contains", + "value": "in the realm of" + }, + { + "type": "violation_phrase_contains", + "value": "is widely regarded as" + }, + { + "type": "violation_phrase_contains", + "value": "some critics argue" + }, + { + "type": "violation_phrase_contains", + "value": "observers have noted" + }, + { + "type": "violation_phrase_contains", + "value": "would be complete without" + }, + { + "type": "violation_phrase_contains", + "value": "future prospects" + }, + { + "type": "violation_phrase_contains", + "value": "future outlook" + } + ] + }, + { + "id": "REC-25", + "category": "scanner_recall", + "title": "Catalog coverage: 'as an ai assistant' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned as an ai assistant and the group moved on to the next agenda item.\nThe consultant wrote that as of my last knowledge update before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating elucidate while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged delineate and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned underpin and the group moved on to the next agenda item.\nThe consultant wrote that unveil before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating seamless while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged invaluable and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned noteworthy and the group moved on to the next agenda item.\nThe consultant wrote that revolutionize before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "as an ai assistant" + }, + { + "type": "violation_phrase_contains", + "value": "as of my last knowledge update" + }, + { + "type": "violation_phrase_contains", + "value": "elucidate" + }, + { + "type": "violation_phrase_contains", + "value": "delineate" + }, + { + "type": "violation_phrase_contains", + "value": "underpin" + }, + { + "type": "violation_phrase_contains", + "value": "unveil" + }, + { + "type": "violation_phrase_contains", + "value": "seamless" + }, + { + "type": "violation_phrase_contains", + "value": "invaluable" + }, + { + "type": "violation_phrase_contains", + "value": "noteworthy" + }, + { + "type": "violation_phrase_contains", + "value": "revolutionize" + } + ] + }, + { + "id": "REC-26", + "category": "scanner_recall", + "title": "Catalog coverage: 'revolutionizing' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned revolutionizing and the group moved on to the next agenda item.\nThe consultant wrote that it should be noted that before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating holds promise while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged new avenues and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned shed new light and the group moved on to the next agenda item.\nThe consultant wrote that to the best of our knowledge before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating a growing body of while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged in recent years and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned with the advent of and the group moved on to the next agenda item.\nThe consultant wrote that garnered significant attention before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "revolutionizing" + }, + { + "type": "violation_phrase_contains", + "value": "it should be noted that" + }, + { + "type": "violation_phrase_contains", + "value": "holds promise" + }, + { + "type": "violation_phrase_contains", + "value": "new avenues" + }, + { + "type": "violation_phrase_contains", + "value": "shed new light" + }, + { + "type": "violation_phrase_contains", + "value": "to the best of our knowledge" + }, + { + "type": "violation_phrase_contains", + "value": "a growing body of" + }, + { + "type": "violation_phrase_contains", + "value": "in recent years" + }, + { + "type": "violation_phrase_contains", + "value": "with the advent of" + }, + { + "type": "violation_phrase_contains", + "value": "garnered significant attention" + } + ] + }, + { + "id": "REC-27", + "category": "scanner_recall", + "title": "Catalog coverage: 'garnered considerable attention' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned garnered considerable attention and the group moved on to the next agenda item.\nThe consultant wrote that taken together, before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating game-changer while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged deep dive and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned take a step back and the group moved on to the next agenda item.\nThe consultant wrote that moving forward before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating circle back while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged on the same page and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned level up and the group moved on to the next agenda item.\nThe consultant wrote that bandwidth before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "garnered considerable attention" + }, + { + "type": "violation_phrase_contains", + "value": "taken together," + }, + { + "type": "violation_phrase_contains", + "value": "game-changer" + }, + { + "type": "violation_phrase_contains", + "value": "deep dive" + }, + { + "type": "violation_phrase_contains", + "value": "take a step back" + }, + { + "type": "violation_phrase_contains", + "value": "moving forward" + }, + { + "type": "violation_phrase_contains", + "value": "circle back" + }, + { + "type": "violation_phrase_contains", + "value": "on the same page" + }, + { + "type": "violation_phrase_contains", + "value": "level up" + }, + { + "type": "violation_phrase_contains", + "value": "bandwidth" + } + ] + }, + { + "id": "REC-28", + "category": "scanner_recall", + "title": "Catalog coverage: 'low-hanging fruit' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned low-hanging fruit and the group moved on to the next agenda item.\nThe consultant wrote that disrupt before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating scalable while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged actionable and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned ecosystem and the group moved on to the next agenda item.\nThe consultant wrote that touch base before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating value-add while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged thought leader and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned best-in-class and the group moved on to the next agenda item.\nThe consultant wrote that cutting-edge before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "low-hanging fruit" + }, + { + "type": "violation_phrase_contains", + "value": "disrupt" + }, + { + "type": "violation_phrase_contains", + "value": "scalable" + }, + { + "type": "violation_phrase_contains", + "value": "actionable" + }, + { + "type": "violation_phrase_contains", + "value": "ecosystem" + }, + { + "type": "violation_phrase_contains", + "value": "touch base" + }, + { + "type": "violation_phrase_contains", + "value": "value-add" + }, + { + "type": "violation_phrase_contains", + "value": "thought leader" + }, + { + "type": "violation_phrase_contains", + "value": "best-in-class" + }, + { + "type": "violation_phrase_contains", + "value": "cutting-edge" + } + ] + }, + { + "id": "REC-29", + "category": "scanner_recall", + "title": "Catalog coverage: 'delve deeper' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned delve deeper and the group moved on to the next agenda item.\nThe consultant wrote that robust before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating comprehensive while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged utilize and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned facilitate and the group moved on to the next agenda item.\nThe consultant wrote that spearhead before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating streamline while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged multifaceted and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned enhance and the group moved on to the next agenda item.\nThe consultant wrote that align with before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "delve deeper" + }, + { + "type": "violation_phrase_contains", + "value": "robust" + }, + { + "type": "violation_phrase_contains", + "value": "comprehensive" + }, + { + "type": "violation_phrase_contains", + "value": "utilize" + }, + { + "type": "violation_phrase_contains", + "value": "facilitate" + }, + { + "type": "violation_phrase_contains", + "value": "spearhead" + }, + { + "type": "violation_phrase_contains", + "value": "streamline" + }, + { + "type": "violation_phrase_contains", + "value": "multifaceted" + }, + { + "type": "violation_phrase_contains", + "value": "enhance" + }, + { + "type": "violation_phrase_contains", + "value": "align with" + } + ] + }, + { + "id": "REC-30", + "category": "scanner_recall", + "title": "Catalog coverage: 'at its core' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned at its core and the group moved on to the next agenda item.\nThe consultant wrote that interestingly, before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating importantly, while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged crucially, and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned at the end of the day and the group moved on to the next agenda item.\nThe consultant wrote that when it comes to before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating in a world where while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged the reality is and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned with that said and the group moved on to the next agenda item.\nThe consultant wrote that that being said before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "at its core" + }, + { + "type": "violation_phrase_contains", + "value": "interestingly," + }, + { + "type": "violation_phrase_contains", + "value": "importantly," + }, + { + "type": "violation_phrase_contains", + "value": "crucially," + }, + { + "type": "violation_phrase_contains", + "value": "at the end of the day" + }, + { + "type": "violation_phrase_contains", + "value": "when it comes to" + }, + { + "type": "violation_phrase_contains", + "value": "in a world where" + }, + { + "type": "violation_phrase_contains", + "value": "the reality is" + }, + { + "type": "violation_phrase_contains", + "value": "with that said" + }, + { + "type": "violation_phrase_contains", + "value": "that being said" + } + ] + }, + { + "id": "REC-31", + "category": "scanner_recall", + "title": "Catalog coverage: 'all things considered' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned all things considered and the group moved on to the next agenda item.\nThe consultant wrote that by and large before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating for the most part while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged to be fair and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned to be honest and the group moved on to the next agenda item.\nThe consultant wrote that needless to say before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating it goes without saying while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged in order to and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned due to the fact that and the group moved on to the next agenda item.\nThe consultant wrote that at this point in time before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "all things considered" + }, + { + "type": "violation_phrase_contains", + "value": "by and large" + }, + { + "type": "violation_phrase_contains", + "value": "for the most part" + }, + { + "type": "violation_phrase_contains", + "value": "to be fair" + }, + { + "type": "violation_phrase_contains", + "value": "to be honest" + }, + { + "type": "violation_phrase_contains", + "value": "needless to say" + }, + { + "type": "violation_phrase_contains", + "value": "it goes without saying" + }, + { + "type": "violation_phrase_contains", + "value": "in order to" + }, + { + "type": "violation_phrase_contains", + "value": "due to the fact that" + }, + { + "type": "violation_phrase_contains", + "value": "at this point in time" + } + ] + }, + { + "id": "REC-32", + "category": "scanner_recall", + "title": "Catalog coverage: 'in the event that' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned in the event that and the group moved on to the next agenda item.\nThe consultant wrote that hint: before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating plot twist: while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged spoiler: and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned you already know this, but and the group moved on to the next agenda item.\nThe consultant wrote that but that's another post before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating is a feature, not a bug while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged dressed up as and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned let me explain and the group moved on to the next agenda item.\nThe consultant wrote that to put it simply before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "in the event that" + }, + { + "type": "violation_phrase_contains", + "value": "hint:" + }, + { + "type": "violation_phrase_contains", + "value": "plot twist:" + }, + { + "type": "violation_phrase_contains", + "value": "spoiler:" + }, + { + "type": "violation_phrase_contains", + "value": "you already know this, but" + }, + { + "type": "violation_phrase_contains", + "value": "but that's another post" + }, + { + "type": "violation_phrase_contains", + "value": "is a feature, not a bug" + }, + { + "type": "violation_phrase_contains", + "value": "dressed up as" + }, + { + "type": "violation_phrase_contains", + "value": "let me explain" + }, + { + "type": "violation_phrase_contains", + "value": "to put it simply" + } + ] + }, + { + "id": "REC-33", + "category": "scanner_recall", + "title": "Catalog coverage: 'in other words' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned in other words and the group moved on to the next agenda item.\nThe consultant wrote that if you think about it before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating creeps in while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged i promise and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned they exist, i promise and the group moved on to the next agenda item.\nThe consultant wrote that this is genuinely before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating actually looks like while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged trust me and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned believe me and the group moved on to the next agenda item.\nThe consultant wrote that stands as a testament before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "in other words" + }, + { + "type": "violation_phrase_contains", + "value": "if you think about it" + }, + { + "type": "violation_phrase_contains", + "value": "creeps in" + }, + { + "type": "violation_phrase_contains", + "value": "i promise" + }, + { + "type": "violation_phrase_contains", + "value": "they exist, i promise" + }, + { + "type": "violation_phrase_contains", + "value": "this is genuinely" + }, + { + "type": "violation_phrase_contains", + "value": "actually looks like" + }, + { + "type": "violation_phrase_contains", + "value": "trust me" + }, + { + "type": "violation_phrase_contains", + "value": "believe me" + }, + { + "type": "violation_phrase_contains", + "value": "stands as a testament" + } + ] + }, + { + "id": "REC-34", + "category": "scanner_recall", + "title": "Catalog coverage: 'indelible mark' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned indelible mark and the group moved on to the next agenda item.\nThe consultant wrote that rich tapestry before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating rich history while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged a legacy of and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned cornerstone of and the group moved on to the next agenda item.\nThe consultant wrote that groundbreaking before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating transformative while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged trailblazing and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned seminal and the group moved on to the next agenda item.\nThe consultant wrote that nestled before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "indelible mark" + }, + { + "type": "violation_phrase_contains", + "value": "rich tapestry" + }, + { + "type": "violation_phrase_contains", + "value": "rich history" + }, + { + "type": "violation_phrase_contains", + "value": "a legacy of" + }, + { + "type": "violation_phrase_contains", + "value": "cornerstone of" + }, + { + "type": "violation_phrase_contains", + "value": "groundbreaking" + }, + { + "type": "violation_phrase_contains", + "value": "transformative" + }, + { + "type": "violation_phrase_contains", + "value": "trailblazing" + }, + { + "type": "violation_phrase_contains", + "value": "seminal" + }, + { + "type": "violation_phrase_contains", + "value": "nestled" + } + ] + }, + { + "id": "REC-35", + "category": "scanner_recall", + "title": "Catalog coverage: 'breathtaking' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned breathtaking and the group moved on to the next agenda item.\nThe consultant wrote that must-visit before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating in the heart of while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged world-class and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned state-of-the-art and the group moved on to the next agenda item.\nThe consultant wrote that second to none before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating a hidden gem while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged bustling and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned renowned for and the group moved on to the next agenda item.\nThe consultant wrote that a beacon of before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "breathtaking" + }, + { + "type": "violation_phrase_contains", + "value": "must-visit" + }, + { + "type": "violation_phrase_contains", + "value": "in the heart of" + }, + { + "type": "violation_phrase_contains", + "value": "world-class" + }, + { + "type": "violation_phrase_contains", + "value": "state-of-the-art" + }, + { + "type": "violation_phrase_contains", + "value": "second to none" + }, + { + "type": "violation_phrase_contains", + "value": "a hidden gem" + }, + { + "type": "violation_phrase_contains", + "value": "bustling" + }, + { + "type": "violation_phrase_contains", + "value": "renowned for" + }, + { + "type": "violation_phrase_contains", + "value": "a beacon of" + } + ] + }, + { + "id": "REC-36", + "category": "scanner_recall", + "title": "Catalog coverage: 'experts argue' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned experts argue and the group moved on to the next agenda item.\nThe consultant wrote that industry reports suggest before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating many believe while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged it is widely regarded and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned according to experts and the group moved on to the next agenda item.\nThe consultant wrote that observers note before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating research suggests while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged represents a and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned constitutes a and the group moved on to the next agenda item.\nThe consultant wrote that functions as a before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "experts argue" + }, + { + "type": "violation_phrase_contains", + "value": "industry reports suggest" + }, + { + "type": "violation_phrase_contains", + "value": "many believe" + }, + { + "type": "violation_phrase_contains", + "value": "it is widely regarded" + }, + { + "type": "violation_phrase_contains", + "value": "according to experts" + }, + { + "type": "violation_phrase_contains", + "value": "observers note" + }, + { + "type": "violation_phrase_contains", + "value": "research suggests" + }, + { + "type": "violation_phrase_contains", + "value": "represents a" + }, + { + "type": "violation_phrase_contains", + "value": "constitutes a" + }, + { + "type": "violation_phrase_contains", + "value": "functions as a" + } + ] + }, + { + "id": "REC-37", + "category": "scanner_recall", + "title": "Catalog coverage: 'operates as a' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned operates as a and the group moved on to the next agenda item.\nThe consultant wrote that i hope this helps before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating certainly! while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged great question and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned that's a great point and the group moved on to the next agenda item.\nThe consultant wrote that happy to help before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating let me know if you need anything else while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged i'd be happy to and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned i am happy to address and the group moved on to the next agenda item.\nThe consultant wrote that i am open to any suggestions before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "operates as a" + }, + { + "type": "violation_phrase_contains", + "value": "i hope this helps" + }, + { + "type": "violation_phrase_contains", + "value": "certainly!" + }, + { + "type": "violation_phrase_contains", + "value": "great question" + }, + { + "type": "violation_phrase_contains", + "value": "that's a great point" + }, + { + "type": "violation_phrase_contains", + "value": "happy to help" + }, + { + "type": "violation_phrase_contains", + "value": "let me know if you need anything else" + }, + { + "type": "violation_phrase_contains", + "value": "i'd be happy to" + }, + { + "type": "violation_phrase_contains", + "value": "i am happy to address" + }, + { + "type": "violation_phrase_contains", + "value": "i am open to any suggestions" + } + ] + }, + { + "id": "REC-38", + "category": "scanner_recall", + "title": "Catalog coverage: 'i am open to feedback' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned i am open to feedback and the group moved on to the next agenda item.\nThe consultant wrote that i would appreciate any feedback before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating i am willing to address while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged i assure you and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned demonstrate my commitment and the group moved on to the next agenda item.\nThe consultant wrote that with the utmost care before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating if you have any concerns or suggestions while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged as of my last and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned as of my knowledge cutoff and the group moved on to the next agenda item.\nThe consultant wrote that based on my training data before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "i am open to feedback" + }, + { + "type": "violation_phrase_contains", + "value": "i would appreciate any feedback" + }, + { + "type": "violation_phrase_contains", + "value": "i am willing to address" + }, + { + "type": "violation_phrase_contains", + "value": "i assure you" + }, + { + "type": "violation_phrase_contains", + "value": "demonstrate my commitment" + }, + { + "type": "violation_phrase_contains", + "value": "with the utmost care" + }, + { + "type": "violation_phrase_contains", + "value": "if you have any concerns or suggestions" + }, + { + "type": "violation_phrase_contains", + "value": "as of my last" + }, + { + "type": "violation_phrase_contains", + "value": "as of my knowledge cutoff" + }, + { + "type": "violation_phrase_contains", + "value": "based on my training data" + } + ] + }, + { + "id": "REC-39", + "category": "scanner_recall", + "title": "Catalog coverage: 'based on available information' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned based on available information and the group moved on to the next agenda item.\nThe consultant wrote that the future looks bright before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating exciting times lie ahead while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged only time will tell and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned one thing is certain and the group moved on to the next agenda item.\nThe consultant wrote that continues to evolve before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating continues to shape while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged poised for growth and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned remains to be seen and the group moved on to the next agenda item.\nThe consultant wrote that interplay before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "based on available information" + }, + { + "type": "violation_phrase_contains", + "value": "the future looks bright" + }, + { + "type": "violation_phrase_contains", + "value": "exciting times lie ahead" + }, + { + "type": "violation_phrase_contains", + "value": "only time will tell" + }, + { + "type": "violation_phrase_contains", + "value": "one thing is certain" + }, + { + "type": "violation_phrase_contains", + "value": "continues to evolve" + }, + { + "type": "violation_phrase_contains", + "value": "continues to shape" + }, + { + "type": "violation_phrase_contains", + "value": "poised for growth" + }, + { + "type": "violation_phrase_contains", + "value": "remains to be seen" + }, + { + "type": "violation_phrase_contains", + "value": "interplay" + } + ] + }, + { + "id": "REC-40", + "category": "scanner_recall", + "title": "Catalog coverage: 'intricate' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned intricate and the group moved on to the next agenda item.\nThe consultant wrote that tapestry of before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating paramount while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged pertaining to and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned aforementioned and the group moved on to the next agenda item.\nThe consultant wrote that henceforth before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating whereby while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged therein and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned burgeoning and the group moved on to the next agenda item.\nThe consultant wrote that myriad before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "intricate" + }, + { + "type": "violation_phrase_contains", + "value": "tapestry of" + }, + { + "type": "violation_phrase_contains", + "value": "paramount" + }, + { + "type": "violation_phrase_contains", + "value": "pertaining to" + }, + { + "type": "violation_phrase_contains", + "value": "aforementioned" + }, + { + "type": "violation_phrase_contains", + "value": "henceforth" + }, + { + "type": "violation_phrase_contains", + "value": "whereby" + }, + { + "type": "violation_phrase_contains", + "value": "therein" + }, + { + "type": "violation_phrase_contains", + "value": "burgeoning" + }, + { + "type": "violation_phrase_contains", + "value": "myriad" + } + ] + }, + { + "id": "REC-41", + "category": "scanner_recall", + "title": "Catalog coverage: 'plethora' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned plethora and the group moved on to the next agenda item.\nThe consultant wrote that encompass before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating nevertheless while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged ubiquitous and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned nuanced and the group moved on to the next agenda item.\nThe consultant wrote that meticulous before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating meticulously while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged emphasizing and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned enduring and the group moved on to the next agenda item.\nThe consultant wrote that vibrant before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "plethora" + }, + { + "type": "violation_phrase_contains", + "value": "encompass" + }, + { + "type": "violation_phrase_contains", + "value": "nevertheless" + }, + { + "type": "violation_phrase_contains", + "value": "ubiquitous" + }, + { + "type": "violation_phrase_contains", + "value": "nuanced" + }, + { + "type": "violation_phrase_contains", + "value": "meticulous" + }, + { + "type": "violation_phrase_contains", + "value": "meticulously" + }, + { + "type": "violation_phrase_contains", + "value": "emphasizing" + }, + { + "type": "violation_phrase_contains", + "value": "enduring" + }, + { + "type": "violation_phrase_contains", + "value": "vibrant" + } + ] + }, + { + "id": "REC-42", + "category": "scanner_recall", + "title": "Catalog coverage: 'resonates with' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned resonates with and the group moved on to the next agenda item.\nThe consultant wrote that resonate with before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating at the intersection of while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged it's no secret that and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned speaks volumes and the group moved on to the next agenda item.\nThe consultant wrote that the elephant in the room before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating it begs the question while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged a deep understanding and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned double-edged sword and the group moved on to the next agenda item.\nThe consultant wrote that sends a clear message before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "resonates with" + }, + { + "type": "violation_phrase_contains", + "value": "resonate with" + }, + { + "type": "violation_phrase_contains", + "value": "at the intersection of" + }, + { + "type": "violation_phrase_contains", + "value": "it's no secret that" + }, + { + "type": "violation_phrase_contains", + "value": "speaks volumes" + }, + { + "type": "violation_phrase_contains", + "value": "the elephant in the room" + }, + { + "type": "violation_phrase_contains", + "value": "it begs the question" + }, + { + "type": "violation_phrase_contains", + "value": "a deep understanding" + }, + { + "type": "violation_phrase_contains", + "value": "double-edged sword" + }, + { + "type": "violation_phrase_contains", + "value": "sends a clear message" + } + ] + }, + { + "id": "REC-43", + "category": "scanner_recall", + "title": "Catalog coverage: 'here's why' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned here's why and the group moved on to the next agenda item.\nThe consultant wrote that let's dive in before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating let's explore while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged let's break this down and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned let's take a look and the group moved on to the next agenda item.\nThe consultant wrote that let's examine before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating the bottom line while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged the key takeaway and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned it's clear that and the group moved on to the next agenda item.\nThe consultant wrote that what's clear is before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "here's why" + }, + { + "type": "violation_phrase_contains", + "value": "let's dive in" + }, + { + "type": "violation_phrase_contains", + "value": "let's explore" + }, + { + "type": "violation_phrase_contains", + "value": "let's break this down" + }, + { + "type": "violation_phrase_contains", + "value": "let's take a look" + }, + { + "type": "violation_phrase_contains", + "value": "let's examine" + }, + { + "type": "violation_phrase_contains", + "value": "the bottom line" + }, + { + "type": "violation_phrase_contains", + "value": "the key takeaway" + }, + { + "type": "violation_phrase_contains", + "value": "it's clear that" + }, + { + "type": "violation_phrase_contains", + "value": "what's clear is" + } + ] + }, + { + "id": "REC-44", + "category": "scanner_recall", + "title": "Catalog coverage: 'it's a no-brainer' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned it's a no-brainer and the group moved on to the next agenda item.\nThe consultant wrote that at the forefront of before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating navigating the complexities while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged in an era of and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned the fabric of and the group moved on to the next agenda item.\nThe consultant wrote that it's worth mentioning before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating sheds light on while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged strikes a balance and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned paints a picture and the group moved on to the next agenda item.\nThe consultant wrote that raises the bar before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "it's a no-brainer" + }, + { + "type": "violation_phrase_contains", + "value": "at the forefront of" + }, + { + "type": "violation_phrase_contains", + "value": "navigating the complexities" + }, + { + "type": "violation_phrase_contains", + "value": "in an era of" + }, + { + "type": "violation_phrase_contains", + "value": "the fabric of" + }, + { + "type": "violation_phrase_contains", + "value": "it's worth mentioning" + }, + { + "type": "violation_phrase_contains", + "value": "sheds light on" + }, + { + "type": "violation_phrase_contains", + "value": "strikes a balance" + }, + { + "type": "violation_phrase_contains", + "value": "paints a picture" + }, + { + "type": "violation_phrase_contains", + "value": "raises the bar" + } + ] + }, + { + "id": "REC-45", + "category": "scanner_recall", + "title": "Catalog coverage: 'food for thought' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned food for thought and the group moved on to the next agenda item.\nThe consultant wrote that the million-dollar question before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating this is where it gets interesting while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged buckle up and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned spoiler alert and the group moved on to the next agenda item.\nThe consultant wrote that pro tip before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating hot take while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged unpopular opinion and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned a closer look and the group moved on to the next agenda item.\nThe consultant wrote that here's what's interesting before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "food for thought" + }, + { + "type": "violation_phrase_contains", + "value": "the million-dollar question" + }, + { + "type": "violation_phrase_contains", + "value": "this is where it gets interesting" + }, + { + "type": "violation_phrase_contains", + "value": "buckle up" + }, + { + "type": "violation_phrase_contains", + "value": "spoiler alert" + }, + { + "type": "violation_phrase_contains", + "value": "pro tip" + }, + { + "type": "violation_phrase_contains", + "value": "hot take" + }, + { + "type": "violation_phrase_contains", + "value": "unpopular opinion" + }, + { + "type": "violation_phrase_contains", + "value": "a closer look" + }, + { + "type": "violation_phrase_contains", + "value": "here's what's interesting" + } + ] + }, + { + "id": "REC-46", + "category": "scanner_recall", + "title": "Catalog coverage: 'here's what caught my eye' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned here's what caught my eye and the group moved on to the next agenda item.\nThe consultant wrote that here's what stood out before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating worth reading while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged worth paying attention to and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned worth a look and the group moved on to the next agenda item.\nThe consultant wrote that worth exploring before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating worth checking out while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged worth your time and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned let me think step by step and the group moved on to the next agenda item.\nThe consultant wrote that breaking this down before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "here's what caught my eye" + }, + { + "type": "violation_phrase_contains", + "value": "here's what stood out" + }, + { + "type": "violation_phrase_contains", + "value": "worth reading" + }, + { + "type": "violation_phrase_contains", + "value": "worth paying attention to" + }, + { + "type": "violation_phrase_contains", + "value": "worth a look" + }, + { + "type": "violation_phrase_contains", + "value": "worth exploring" + }, + { + "type": "violation_phrase_contains", + "value": "worth checking out" + }, + { + "type": "violation_phrase_contains", + "value": "worth your time" + }, + { + "type": "violation_phrase_contains", + "value": "let me think step by step" + }, + { + "type": "violation_phrase_contains", + "value": "breaking this down" + } + ] + }, + { + "id": "REC-47", + "category": "scanner_recall", + "title": "Catalog coverage: 'to approach this systematically' and 9 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned to approach this systematically and the group moved on to the next agenda item.\nThe consultant wrote that here's my thought process before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating working through this logically while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged you're asking about and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned to answer your question and the group moved on to the next agenda item.\nThe consultant wrote that the question of whether before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating introduced a term while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged coined the phrase and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned a concept nobody's naming and the group moved on to the next agenda item.\nThe consultant wrote that a problem nobody talks about before summarizing the quarterly figures for the assembled department heads.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "to approach this systematically" + }, + { + "type": "violation_phrase_contains", + "value": "here's my thought process" + }, + { + "type": "violation_phrase_contains", + "value": "working through this logically" + }, + { + "type": "violation_phrase_contains", + "value": "you're asking about" + }, + { + "type": "violation_phrase_contains", + "value": "to answer your question" + }, + { + "type": "violation_phrase_contains", + "value": "the question of whether" + }, + { + "type": "violation_phrase_contains", + "value": "introduced a term" + }, + { + "type": "violation_phrase_contains", + "value": "coined the phrase" + }, + { + "type": "violation_phrase_contains", + "value": "a concept nobody's naming" + }, + { + "type": "violation_phrase_contains", + "value": "a problem nobody talks about" + } + ] + }, + { + "id": "REC-48", + "category": "scanner_recall", + "title": "Catalog coverage: 'the insight everyone's missing' and 7 more banned phrases still flag", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "During the weekly review meeting the analyst mentioned the insight everyone's missing and the group moved on to the next agenda item.\nThe consultant wrote that what nobody tells you about before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating what does this mean for while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged why should you care and asked for a plainer rewrite instead.\nDuring the weekly review meeting the analyst mentioned what's next? and the group moved on to the next agenda item.\nThe consultant wrote that three key takeaways before summarizing the quarterly figures for the assembled department heads.\nMy colleague kept repeating five things to know while we worked through the long backlog of support tickets together.\nSomewhere in the middle of the draft the editor flagged top seven and asked for a plainer rewrite instead.", + "failure_mode": "Catalogued banned phrases with no eval row can be silently dropped from BANNED_PHRASES with every gate green.", + "correct_behavior": "Every phrase in the pack is flagged by the scanner.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "the insight everyone's missing" + }, + { + "type": "violation_phrase_contains", + "value": "what nobody tells you about" + }, + { + "type": "violation_phrase_contains", + "value": "what does this mean for" + }, + { + "type": "violation_phrase_contains", + "value": "why should you care" + }, + { + "type": "violation_phrase_contains", + "value": "what's next?" + }, + { + "type": "violation_phrase_contains", + "value": "three key takeaways" + }, + { + "type": "violation_phrase_contains", + "value": "five things to know" + }, + { + "type": "violation_phrase_contains", + "value": "top seven" + } + ] + }, + { + "id": "FP-52", + "category": "scanner_false_positive", + "protects": "academic_excess", + "title": "FP guard: near-academic_excess prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The masons repaired the underpinning beneath the old church wall.", + "failure_mode": "Over-broadening the academic_excess rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts academic_excess but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-53", + "category": "scanner_false_positive", + "protects": "acknowledgment_loop", + "title": "FP guard: near-acknowledgment_loop prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The question of how to route the packet was settled by the new switch.", + "failure_mode": "Over-broadening the acknowledgment_loop rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts acknowledgment_loop but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-54", + "category": "scanner_false_positive", + "protects": "anthropomorphic_shipping", + "title": "FP guard: near-anthropomorphic_shipping prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The new gadget arrives inside a labeled cardboard box at every store.", + "failure_mode": "Over-broadening the anthropomorphic_shipping rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts anthropomorphic_shipping but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-55", + "category": "scanner_false_positive", + "protects": "assistant_artifact", + "title": "FP guard: near-assistant_artifact prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "She works as an AI researcher on the alignment team.", + "failure_mode": "Over-broadening the assistant_artifact rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts assistant_artifact but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-56", + "category": "scanner_false_positive", + "protects": "binary_contrast", + "title": "FP guard: near-binary_contrast prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The bridge closed not for repairs but for a scheduled inspection.", + "failure_mode": "Over-broadening the binary_contrast rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts binary_contrast but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-57", + "category": "scanner_false_positive", + "protects": "colon_reveal", + "title": "FP guard: near-colon_reveal prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The answer is on page twelve of the appendix.", + "failure_mode": "Over-broadening the colon_reveal rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts colon_reveal but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-58", + "category": "scanner_false_positive", + "protects": "communication_artifact", + "title": "FP guard: near-communication_artifact prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The support rep was happy the ticket closed within the hour.", + "failure_mode": "Over-broadening the communication_artifact rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts communication_artifact but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-59", + "category": "scanner_false_positive", + "protects": "conclusion_scaffold", + "title": "FP guard: near-conclusion_scaffold prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The committee will summarize findings at Friday's meeting.", + "failure_mode": "Over-broadening the conclusion_scaffold rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts conclusion_scaffold but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-60", + "category": "scanner_false_positive", + "protects": "dramatic_fragment", + "title": "FP guard: near-dramatic_fragment prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The retry logic runs in a single continuous loop without pausing.", + "failure_mode": "Over-broadening the dramatic_fragment rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts dramatic_fragment but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-61", + "category": "scanner_false_positive", + "protects": "editorializing", + "title": "FP guard: near-editorializing prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The renovation was complete without any delays.", + "failure_mode": "Over-broadening the editorializing rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts editorializing but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-62", + "category": "scanner_false_positive", + "protects": "false_agency", + "title": "FP guard: near-false_agency prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The witness spoke for the defendant at the hearing.", + "failure_mode": "Over-broadening the false_agency rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts false_agency but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-63", + "category": "scanner_false_positive", + "protects": "false_concession", + "title": "FP guard: near-false_concession prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Although the trail was steep, the hikers reached the summit by noon.", + "failure_mode": "Over-broadening the false_concession rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts false_concession but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-64", + "category": "scanner_false_positive", + "protects": "false_range", + "title": "FP guard: near-false_range prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The shuttle runs from downtown to the airport every twenty minutes.", + "failure_mode": "Over-broadening the false_range rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts false_range but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-65", + "category": "scanner_false_positive", + "protects": "filler", + "title": "FP guard: near-filler prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The apple core was left on the desk overnight.", + "failure_mode": "Over-broadening the filler rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts filler but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-66", + "category": "scanner_false_positive", + "protects": "filler_opener", + "title": "FP guard: near-filler_opener prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The plan was solid, so the team shipped it on Friday.", + "failure_mode": "Over-broadening the filler_opener rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts filler_opener but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-67", + "category": "scanner_false_positive", + "protects": "formulaic_challenges", + "title": "FP guard: near-formulaic_challenges prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Despite the rain, the festival drew a record crowd downtown.", + "failure_mode": "Over-broadening the formulaic_challenges rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts formulaic_challenges but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-68", + "category": "scanner_false_positive", + "protects": "generic_conclusion", + "title": "FP guard: near-generic_conclusion prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The recount continues to run and results are due Friday.", + "failure_mode": "Over-broadening the generic_conclusion rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts generic_conclusion but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-69", + "category": "scanner_false_positive", + "protects": "knowledge_cutoff", + "title": "FP guard: near-knowledge_cutoff prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "As of the last inspection, the elevator passed every safety check.", + "failure_mode": "Over-broadening the knowledge_cutoff rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts knowledge_cutoff but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-70", + "category": "scanner_false_positive", + "protects": "meta", + "title": "FP guard: near-meta prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The teacher gave the class a helpful hint about the exam format.", + "failure_mode": "Over-broadening the meta rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts meta but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-71", + "category": "scanner_false_positive", + "protects": "novelty_inflation", + "title": "FP guard: near-novelty_inflation prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The mint coined a commemorative dollar for the anniversary.", + "failure_mode": "Over-broadening the novelty_inflation rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts novelty_inflation but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-72", + "category": "scanner_false_positive", + "protects": "numbered_list_inflation", + "title": "FP guard: near-numbered_list_inflation prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The chef listed three key ingredients for the sauce.", + "failure_mode": "Over-broadening the numbered_list_inflation rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts numbered_list_inflation but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-73", + "category": "scanner_false_positive", + "protects": "parenthetical_hedging", + "title": "FP guard: near-parenthetical_hedging prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The museum (founded in 1901) reopened last spring.", + "failure_mode": "Over-broadening the parenthetical_hedging rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts parenthetical_hedging but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-74", + "category": "scanner_false_positive", + "protects": "performative", + "title": "FP guard: near-performative prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The lawyer asked the jury to trust the physical evidence over hearsay.", + "failure_mode": "Over-broadening the performative rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts performative but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-75", + "category": "scanner_false_positive", + "protects": "reader_steering", + "title": "FP guard: near-reader_steering prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Here is the interesting part of the ledger the auditor flagged.", + "failure_mode": "Over-broadening the reader_steering rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts reader_steering but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-76", + "category": "scanner_false_positive", + "protects": "reasoning_chain", + "title": "FP guard: near-reasoning_chain prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The mechanic kept breaking the repair into smaller stages.", + "failure_mode": "Over-broadening the reasoning_chain rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts reasoning_chain but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-77", + "category": "scanner_false_positive", + "protects": "rhetorical_setup", + "title": "FP guard: near-rhetorical_setup prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Take a moment to think about the tradeoffs before you commit.", + "failure_mode": "Over-broadening the rhetorical_setup rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts rhetorical_setup but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-78", + "category": "scanner_false_positive", + "protects": "staccato_fragmentation", + "title": "FP guard: near-staccato_fragmentation prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The report was thorough, balanced, and finished on time.", + "failure_mode": "Over-broadening the staccato_fragmentation rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts staccato_fragmentation but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-79", + "category": "scanner_false_positive", + "protects": "superficial_ing", + "title": "FP guard: near-superficial_ing prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The crew finished paving the road before the frost arrived.", + "failure_mode": "Over-broadening the superficial_ing rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts superficial_ing but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-80", + "category": "scanner_false_positive", + "protects": "unnecessary_permission", + "title": "FP guard: near-unnecessary_permission prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The form was approved and the okay came through by email.", + "failure_mode": "Over-broadening the unnecessary_permission rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts unnecessary_permission but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-81", + "category": "scanner_false_positive", + "protects": "vague_attribution", + "title": "FP guard: near-vague_attribution prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The analysts at the firm published their Q3 spreadsheet.", + "failure_mode": "Over-broadening the vague_attribution rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts vague_attribution but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "FP-82", + "category": "scanner_false_positive", + "protects": "vague_endorsement", + "title": "FP guard: near-vague_endorsement prose stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The antique was worth two thousand dollars at auction.", + "failure_mode": "Over-broadening the vague_endorsement rule could flag this legitimate literal use.", + "correct_behavior": "No violations: the text tempts vague_endorsement but is ordinary prose.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "DOC-09", + "category": "scanner_detection", + "title": "Every scanner pattern is exercised and every category is FP-protected", + "target": "script", + "command": [ + "python3", + "evals/check_pattern_coverage.py" + ], + "failure_mode": "An agent can add a scanner pattern or category with zero eval coverage and every other gate stays green.", + "correct_behavior": "check_pattern_coverage.py exits 0: every STRUCTURAL_PATTERNS regex matches a corpus row, every BANNED_PHRASES key appears in a corpus row, and every emitted category is claimed by a protects FP row.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + } + ] + }, + { + "id": "DOC-10", + "category": "scanner_detection", + "title": "The add-a-pattern kata proves each safety net fires in order", + "target": "script", + "command": [ + "python3", + "evals/kata_add_pattern.py", + "--run" + ], + "failure_mode": "A refactor could silently disable the coverage gate, catalog parity, or the runner so a careless executor's pattern ships uncaught.", + "correct_behavior": "The kata rehearses add-a-pattern in a temp repo and asserts coverage, parity, and the runner each catch the throwaway pattern at the right step.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + } + ] + }, + { + "id": "FP-83", + "category": "scanner_false_positive", + "title": "Comma-less sentence-initial 'So' in ordinary prose is not a filler opener", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "So the tenant remains liable under Section 4 until the walkthrough is complete.", + "protects": "filler_opener", + "failure_mode": "Resurrecting the dead filler_opener pattern with an optional comma made every paragraph-initial 'So' flag, including ordinary legal and narrative prose.", + "correct_behavior": "No violations: only the throat-clearing 'So,' (with comma) form flags.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "SUGG-01", + "category": "suggestion_mode", + "title": "Co-writer emits span-anchored suggestions for a seeded doc", + "target": "script", + "command": [ + "python3", + "scripts/suggest.py", + "evals/fixtures/suggest/sugg01_seed.md" + ], + "failure_mode": "Detection output is unstructured, so a host agent cannot surface findings as reviewable edits.", + "correct_behavior": "Two suggestions with correct spans/categories (false_agency + filler), replacements delegated (null).", + "assertions": [ + { + "type": "json", + "path": "counts.total", + "equals": 2 + }, + { + "type": "stdout_contains", + "value": "\"category\": \"false_agency\"" + }, + { + "type": "stdout_contains", + "value": "\"category\": \"filler\"" + }, + { + "type": "stdout_contains", + "value": "\"text\": \"speaks for itself\"" + }, + { + "type": "stdout_contains", + "value": "\"suggested_replacement\": null" + } + ] + }, + { + "id": "SUGG-02", + "category": "suggestion_mode", + "title": "Soft finding is phrased as a question", + "target": "script", + "command": [ + "python3", + "scripts/suggest.py", + "evals/fixtures/suggest/sugg02_soft.md" + ], + "failure_mode": "Soft, register-dependent findings get stated as commands instead of surfaced as judgment-call questions.", + "correct_behavior": "The lone soft finding has phrased_as_question true and a question-worded rationale.", + "assertions": [ + { + "type": "json", + "path": "suggestions.0.phrased_as_question", + "equals": true + }, + { + "type": "json", + "path": "suggestions.0.severity", + "equals": "soft" + }, + { + "type": "stdout_contains", + "value": "could" + } + ] + }, + { + "id": "SUGG-03", + "category": "suggestion_mode", + "title": "Contract gate catches an oversized (whole-sentence) replacement", + "target": "script", + "command": [ + "python3", + "scripts/check_suggestions.py", + "evals/fixtures/suggest/sugg03_oversized.json" + ], + "failure_mode": "A replacement that rewrites the whole sentence would silently overwrite clean text outside the offending phrase.", + "correct_behavior": "check_suggestions exits 1 naming span-minimality.", + "assertions": [ + { + "type": "exit_code", + "equals": 1 + }, + { + "type": "stdout_contains", + "value": "span-minimality" + } + ] + }, + { + "id": "SUGG-04", + "category": "suggestion_mode", + "title": "Accept-all safety on a good replacement set", + "target": "script", + "command": [ + "python3", + "scripts/check_suggestions.py", + "evals/fixtures/suggest/sugg04_accept.json" + ], + "failure_mode": "Applying every suggestion could reintroduce slop or drop a preserved constraint.", + "correct_behavior": "All gates pass: applied doc is scanner-clean and preserves constraints.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "\"passed\": true" + } + ] + }, + { + "id": "SUGG-05", + "category": "suggestion_mode", + "title": "Overlapping spans are rejected", + "target": "script", + "command": [ + "python3", + "scripts/check_suggestions.py", + "evals/fixtures/suggest/sugg05_overlap.json" + ], + "failure_mode": "Overlapping edit spans corrupt the document when applied together.", + "correct_behavior": "check_suggestions exits 1 naming span-overlap.", + "assertions": [ + { + "type": "exit_code", + "equals": 1 + }, + { + "type": "stdout_contains", + "value": "span-overlap" + } + ] + }, + { + "id": "LANG-01a", + "category": "language_decline", + "title": "Spanish paragraph declined by banned_phrase_scan", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "El rapido zorro marron salta sobre el perro perezoso. La casa esta cerca del rio y los ninos juegan en el jardin todas las tardes despues de la escuela.", + "failure_mode": "English-tuned phrase bans fire nonsensically on non-English prose.", + "correct_behavior": "Non-English detected: non_english true, zero violations, exit 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "\"non_english\": true" + }, + { + "type": "stdout_contains", + "value": "\"violations\": []" + } + ] + }, + { + "id": "LANG-01b", + "category": "language_decline", + "title": "Spanish paragraph declined by structure_scan", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py" + ], + "stdin": "El rapido zorro marron salta sobre el perro perezoso. La casa esta cerca del rio y los ninos juegan en el jardin todas las tardes despues de la escuela.", + "failure_mode": "Structure metrics are meaningless on non-English prose but would still be reported.", + "correct_behavior": "Non-English detected: non_english true, zero violations, exit 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "\"non_english\": true" + }, + { + "type": "stdout_contains", + "value": "\"violations\": []" + } + ] + }, + { + "id": "LANG-02a", + "category": "language_decline", + "title": "ESL-flavored English still scans in banned_phrase_scan", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "I very like this product because it help me to do the work more fast. The team are very happy and we will using it every day for our reports.", + "failure_mode": "A too-aggressive language gate would decline imperfect/ESL English and refuse to help.", + "correct_behavior": "Treated as English and scanned normally; no non_english flag, exit 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_not_contains", + "value": "\"non_english\": true" + } + ] + }, + { + "id": "LANG-02b", + "category": "language_decline", + "title": "ESL-flavored English still scans in structure_scan", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py" + ], + "stdin": "I very like this product because it help me to do the work more fast. The team are very happy and we will using it every day for our reports.", + "failure_mode": "A too-aggressive language gate would decline imperfect/ESL English and refuse to help.", + "correct_behavior": "Treated as English and scanned normally; no non_english flag, exit 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_not_contains", + "value": "\"non_english\": true" + } + ] + }, + { + "id": "PARITY-01", + "category": "model_parity", + "title": "Parity harness grades canned good/bad responses correctly on both tasks", + "target": "script", + "command": [ + "python3", + "evals/run_model_parity.py", + "--dry-run", + "--responses", + "evals/fixtures/parity/canned_responses.json" + ], + "failure_mode": "Grading logic is wrong: a correct detection/replacement scores low or an obviously bad one scores high, so the tiering table would be set from noise.", + "correct_behavior": "cheap-good matches the reference (Task A recall 1.0, zero false findings; Task B pass_rate 1.0) and cheap-bad fails both (recall 0.0, 6 false findings, pass_rate 0.0).", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "models.cheap-good.A.mean_recall", + "equals": 1.0 + }, + { + "type": "json", + "path": "models.cheap-good.A.total_false_findings", + "equals": 0 + }, + { + "type": "json", + "path": "models.cheap-good.B.pass_rate", + "equals": 1.0 + }, + { + "type": "json", + "path": "models.cheap-bad.A.mean_recall", + "equals": 0.0 + }, + { + "type": "json", + "path": "models.cheap-bad.A.total_false_findings", + "equals": 6 + }, + { + "type": "json", + "path": "models.cheap-bad.B.pass_rate", + "equals": 0.0 + } + ] + }, + { + "id": "PARITY-02", + "category": "model_parity", + "title": "Parity grading is deterministic across repeated runs", + "target": "script", + "command": [ + "python3", + "evals/run_model_parity.py", + "--dry-run", + "--responses", + "evals/fixtures/parity/canned_responses.json", + "--check-determinism" + ], + "failure_mode": "Grading depends on dict/set ordering or other nondeterminism, so parity verdicts drift between runs and cannot gate merges.", + "correct_behavior": "Grading the same canned corpus twice yields byte-identical payloads; the harness reports deterministic true.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "deterministic", + "equals": true + } + ] + }, + { + "id": "PARITY-03", + "category": "model_parity", + "title": "A missing keychain key or dead network never breaks the suite", + "target": "script", + "command": [ + "python3", + "evals/run_model_parity.py", + "--no-network", + "--models", + "evals/fixtures/parity/models_live.json" + ], + "failure_mode": "The harness raises on a missing OPENROUTER_API_KEY, absent claude CLI, or network error, taking the whole eval suite down with it.", + "correct_behavior": "Every unreachable model is recorded as status unavailable and the run still exits 0 with a valid payload.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "mode", + "equals": "no-network" + }, + { + "type": "json", + "path": "models.gpt-4o-mini.status", + "equals": "unavailable" + }, + { + "type": "json", + "path": "models.claude-haiku.status", + "equals": "unavailable" + } + ] + }, + { + "id": "HARV-01", + "category": "harvest", + "title": "Assistant-authored transcript text never reaches candidates", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "-o", + "-" + ], + "failure_mode": "Assistant transcript turns contaminate the teach candidate set.", + "correct_behavior": "No distinctive assistant-authored marker phrase appears in candidates JSON.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_not_contains", + "value": "ASSISTANT_CONTAMINATION_NEON_RIVER" + }, + { + "type": "stdout_not_contains", + "value": "ASSISTANT_QUOTED_BRASS_LANTERN" + } + ] + }, + { + "id": "HARV-02", + "category": "harvest", + "title": "Substantive user prose is harvested", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "-o", + "-" + ], + "failure_mode": "The harvester drops real user prose while filtering transcript noise.", + "correct_behavior": "The substantive user marker appears in candidates JSON.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "blue kiln notebook" + } + ] + }, + { + "id": "HARV-03", + "category": "harvest", + "title": "Terse commands are dropped with command-likeness stats", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "-o", + "-" + ], + "failure_mode": "Command-ish transcript turns enter candidates or disappear without stats.", + "correct_behavior": "The slash command text is absent and command-likeness is counted.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_not_contains", + "value": "/review /tmp/draft.md --quick" + }, + { + "type": "json", + "path": "drop_stats.command-likeness", + "gte": 1 + } + ] + }, + { + "id": "HARV-04", + "category": "harvest", + "title": "Near-duplicate transcript paragraphs collapse", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "-o", + "-" + ], + "failure_mode": "Duplicate transcript samples over-weight one passage in the voice profile.", + "correct_behavior": "The duplicate marker appears once and duplication is counted.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "drop_stats.duplication", + "equals": 1 + }, + { + "type": "stdout_contains", + "value": "copper ledger" + } + ] + }, + { + "id": "HARV-05", + "category": "harvest", + "title": "Dictated user turn is flagged but kept", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "-o", + "-" + ], + "failure_mode": "Dictation artifacts cause useful voice evidence to be dropped or unmarked.", + "correct_behavior": "The dictated marker remains and the candidate has dictated:true.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "silver porch" + }, + { + "type": "stdout_contains", + "value": "\"dictated\": true" + }, + { + "type": "stdout_contains", + "value": "green hinge" + }, + { + "type": "json", + "path": "candidates.3.dictated", + "equals": false + } + ] + }, + { + "id": "HARV-06", + "category": "harvest", + "title": "Text-folder AI-tell-heavy sample is suspect and ranked last", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "evals/fixtures/harvest/text-folder", + "-o", + "-" + ], + "failure_mode": "Declared user folders can still contain AI text that should not be auto-approved.", + "correct_behavior": "The text-folder marker is kept, suspect_ai:true, and ranked last.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "TEXT_FOLDER_AI_TELL_MARKER" + }, + { + "type": "stdout_contains", + "value": "\"suspect_ai\": true" + }, + { + "type": "json", + "path": "candidates.4.suspect_ai", + "equals": true + } + ] + }, + { + "id": "HARV-07", + "category": "harvest", + "title": "Heuristic classifier fills expected coverage cells", + "target": "script", + "command": [ + "python3", + "scripts/harvest_classify.py", + "--candidates", + "evals/fixtures/harvest/classifier_candidates.json", + "--mode", + "heuristic" + ], + "failure_mode": "Tier-1 dry-run classifier cannot populate the situation coverage matrix.", + "correct_behavior": "Question, numbers/data, disagreement, and anecdote cells are detected.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "coverage_matrix.question_addressed", + "gte": 1 + }, + { + "type": "json", + "path": "coverage_matrix.numbers_data", + "gte": 1 + }, + { + "type": "json", + "path": "coverage_matrix.disagreement", + "gte": 1 + }, + { + "type": "json", + "path": "coverage_matrix.anecdote_markers", + "gte": 1 + } + ] + }, + { + "id": "HARV-08", + "category": "harvest", + "title": "Harvest output is deterministic across runs", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "evals/fixtures/harvest/text-folder", + "-o", + "-", + "--self-check-determinism" + ], + "failure_mode": "Candidate ordering or stats change between identical runs.", + "correct_behavior": "The built-in determinism check reports true.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "deterministic", + "equals": true + } + ] + }, + { + "id": "HARV-09", + "category": "harvest", + "title": "Missing sources exit 2 and unknown JSONL schemas warn then skip", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/unknown-jsonl", + "evals/fixtures/harvest/does-not-exist", + "-o", + "-" + ], + "failure_mode": "The harvester guesses authorship for unknown JSONL or ignores missing paths.", + "correct_behavior": "Unknown schema warns, missing source exits 2.", + "assertions": [ + { + "type": "exit_code", + "equals": 2 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + }, + { + "type": "stdout_not_contains", + "value": "No role field" + } + ] + }, + { + "id": "HARV-10", + "category": "harvest", + "title": "Classifier results merge into candidates and rerank by coverage, quality, recency", + "target": "script", + "command": [ + "python3", + "scripts/harvest_classify.py", + "--candidates", + "evals/fixtures/harvest/merge_candidates.json", + "--merge", + "evals/fixtures/harvest/merge_results.jsonl" + ], + "failure_mode": "--merge returns raw classifier rows without applying cells, quality, why, coverage, or ranking.", + "correct_behavior": "Merged output contains attached cells, recomputed matrix counts, and a ranking where equal-quality coverage samples sort by recency.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "coverage_matrix.question_addressed", + "equals": 1 + }, + { + "type": "json", + "path": "coverage_matrix.numbers_data", + "equals": 1 + }, + { + "type": "json", + "path": "coverage_matrix.openings_closings", + "equals": 1 + }, + { + "type": "json", + "path": "candidates.0.cells.0", + "equals": "question_addressed" + }, + { + "type": "json", + "path": "candidates.1.quality", + "equals": 4 + }, + { + "type": "json", + "path": "candidates.2.why", + "equals": "closing/signoff sample only" + }, + { + "type": "json", + "path": "ranking.0", + "equals": 1 + }, + { + "type": "json", + "path": "ranking.1", + "equals": 0 + }, + { + "type": "json", + "path": "ranking.2", + "equals": 2 + } + ] + }, + { + "id": "HARV-11", + "category": "harvest", + "title": "Single filler in a short quoted turn is not dictated", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/single_um_short.jsonl", + "-o", + "-", + "--min-words", + "1" + ], + "failure_mode": "Dictation detection treats one quoted filler as a dictated turn when min-words is lowered.", + "correct_behavior": "The candidate is kept and explicitly marked dictated:false.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "red hinge" + }, + { + "type": "json", + "path": "candidates.0.dictated", + "equals": false + } + ] + }, + { + "id": "HARV-12", + "category": "harvest", + "title": "Codex session contamination never reaches candidates", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_codex_session.jsonl", + "-o", + "-" + ], + "failure_mode": "Codex assistant turns, base_instructions, developer-role permission text, tool plumbing, or injected AGENTS.md/environment_context wrappers leak into the teach candidate set.", + "correct_behavior": "No assistant, base_instructions, developer-role, reasoning, tool-call, turn_context, compacted, unknown-type, or instruction-injection marker appears anywhere in candidates JSON.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_not_contains", + "value": "CODEX_ASSISTANT_CONTAMINATION_MARKER_ORCHID_TUNNEL" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_ASSISTANT_RESPONSE_ITEM_MARKER_BRASS_KETTLE" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_BASE_INSTRUCTIONS_MARKER_SLATE_ANCHOR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_DEVELOPER_ROLE_MARKER_IVORY_HATCH" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_REASONING_MARKER_SHOULD_NEVER_APPEAR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_TOOL_CALL_MARKER_SHOULD_NEVER_APPEAR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_TOOL_OUTPUT_MARKER_SHOULD_NEVER_APPEAR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_TURN_CONTEXT_MARKER_SHOULD_NEVER_APPEAR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_COMPACTED_MARKER_SHOULD_NEVER_APPEAR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_UNKNOWN_TYPE_MARKER_SHOULD_NEVER_APPEAR" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_INJECTION_TRAP_MARKER_PEWTER_LOOM" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_ENV_CONTEXT_MARKER_QUARTZ_FIELD" + } + ] + }, + { + "id": "HARV-13", + "category": "harvest", + "title": "Codex substantive user prose is harvested", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_codex_session.jsonl", + "-o", + "-" + ], + "failure_mode": "The codex-jsonl adapter drops real user turns along with the injected wrappers it must filter.", + "correct_behavior": "The substantive user-role response_item marker appears in candidates JSON with the codex-jsonl adapter tag.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "CODEX_USER_PROSE_MARKER_HOLLOW_CEDAR" + }, + { + "type": "json", + "path": "candidates.0.source.adapter", + "equals": "codex-jsonl" + } + ] + }, + { + "id": "HARV-14", + "category": "harvest", + "title": "Codex instruction-injection wrappers are dropped with a named reason", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_codex_session.jsonl", + "-o", + "-" + ], + "failure_mode": "AGENTS.md dumps and banners injected into user-role turns are treated as user authorship, or dropped silently with no accounting.", + "correct_behavior": "Both injected markers are absent from candidates and drop_stats names the reason instruction-injection with count 2.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_not_contains", + "value": "CODEX_INJECTION_TRAP_MARKER_PEWTER_LOOM" + }, + { + "type": "stdout_not_contains", + "value": "CODEX_ENV_CONTEXT_MARKER_QUARTZ_FIELD" + }, + { + "type": "json", + "path": "drop_stats.instruction-injection", + "equals": 2 + } + ] + }, + { + "id": "HARV-15", + "category": "harvest", + "title": "Mixed claude/codex/text-folder run produces candidates from all three sources", + "target": "script", + "command": [ + "python3", + "scripts/harvest_samples.py", + "evals/fixtures/harvest/fixture_session.jsonl", + "evals/fixtures/harvest/fixture_codex_session.jsonl", + "evals/fixtures/harvest/text-folder", + "-o", + "-" + ], + "failure_mode": "A single invocation across multiple source types drops one adapter's candidates or mislabels provenance.", + "correct_behavior": "Candidates from the claude-jsonl, codex-jsonl, and text-folder adapters all appear with correct per-source provenance paths.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "CODEX_USER_PROSE_MARKER_HOLLOW_CEDAR" + }, + { + "type": "stdout_contains", + "value": "blue kiln notebook" + }, + { + "type": "stdout_contains", + "value": "TEXT_FOLDER_AI_TELL_MARKER" + }, + { + "type": "json", + "path": "candidates.0.source.adapter", + "equals": "codex-jsonl" + }, + { + "type": "json", + "path": "candidates.0.source.path", + "equals": "evals/fixtures/harvest/fixture_codex_session.jsonl" + }, + { + "type": "json", + "path": "candidates.4.source.adapter", + "equals": "claude-jsonl" + }, + { + "type": "json", + "path": "candidates.4.source.path", + "equals": "evals/fixtures/harvest/fixture_session.jsonl" + }, + { + "type": "json", + "path": "candidates.5.source.adapter", + "equals": "text-folder" + }, + { + "type": "json", + "path": "candidates.5.source.path", + "equals": "evals/fixtures/harvest/text-folder/ai_tell.txt" + } + ] + }, + { + "id": "SIL-01", + "category": "scanner_detection", + "title": "silhouette_scan flags a templated preview-then-fulfill essay", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "evals/fixtures/silhouette/sil01_templated_essay.md" + ], + "failure_mode": "An essay can pass phrase and surface-structure scans while still following a machine outline: intro previews three pillars, each body paragraph opens on a rotating discourse cue and fulfills one pillar, and the ending loops the vocabulary back.", + "correct_behavior": "Exit 1 with silhouette_penalty flagged for arranging ideas as a symmetric template.", + "assertions": [ + { + "type": "exit_code", + "equals": 1 + }, + { + "type": "json", + "path": "flagged.silhouette_penalty", + "equals": true + } + ] + }, + { + "id": "SIL-02", + "category": "scanner_false_positive", + "title": "A bursty human narrative is not a silhouette template", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "evals/fixtures/silhouette/sil02_bursty_narrative.md" + ], + "failure_mode": "A discourse-level scanner could punish any multi-paragraph prose, flagging genuine human storytelling that wanders and never previews or recaps itself.", + "correct_behavior": "Exit 0: no scaffold openers, no preview-fulfillment, no recap loop, so silhouette_penalty stays below 1.0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "flagged", + "equals": {} + } + ] + }, + { + "id": "SIL-03", + "category": "scanner_false_positive", + "title": "An academic roadmap under --genre docs is convention, not a silhouette", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "--genre", + "docs", + "evals/fixtures/structure/struct17_academic_roadmap.md" + ], + "failure_mode": "Reference and academic prose legitimately previews its sections; a naive outline-following metric would flag every paper abstract that says what each section covers.", + "correct_behavior": "Exit 0 for a bursty academic roadmap; heading_preview is retained under --genre docs but the validated metric clears legitimate roadmaps on its own.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "flagged", + "equals": {} + } + ] + }, + { + "id": "SIL-04", + "category": "scanner_detection", + "title": "silhouette_scan catches a recap loop via callback_content", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "evals/fixtures/silhouette/sil04_recap_loop.md" + ], + "failure_mode": "Opening vocabulary that disappears through the middle and returns at the very end is a recap-loop tell that surface scanners miss.", + "correct_behavior": "Exit 1 with callback_content specifically flagged, not merely the composite.", + "assertions": [ + { + "type": "exit_code", + "equals": 1 + }, + { + "type": "json", + "path": "flagged.callback_content", + "equals": true + } + ] + }, + { + "id": "SIL-05a", + "category": "scanner_false_positive", + "title": "Cue-deletion attack: silhouette_scan alone misses stripped connectives", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "evals/fixtures/silhouette/sil05_cue_deletion.md" + ], + "failure_mode": "An adversary strips discourse cues and avoids preview/recap symmetry while keeping a rigid one-beat-per-paragraph shape, collapsing the scaffold and role metrics that silhouette_scan depends on.", + "correct_behavior": "Documented recall ceiling: silhouette_penalty stays below 1.0, so silhouette_scan exits 0. This is why the pattern must be scored JOINTLY with structure_scan (see SIL-05b), the lower fence.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "flagged", + "equals": {} + } + ] + }, + { + "id": "SIL-05b", + "category": "scanner_detection", + "title": "Cue-deletion attack: structure_scan catches it via opener repetition", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/silhouette/sil05_cue_deletion.md" + ], + "failure_mode": "The same cue-deletion text that slips past silhouette_scan keeps a surface tell: every paragraph opens the same way in uniform short beats.", + "correct_behavior": "Exit 1 with opener_repetition flagged. The silhouette/structure pairing is the anti-gaming contract: two commands, two fences.", + "assertions": [ + { + "type": "exit_code", + "equals": 1 + }, + { + "type": "json", + "path": "flagged.opener_repetition", + "equals": true + } + ] + }, + { + "id": "SIL-06", + "category": "robustness", + "title": "silhouette_scan output is deterministic across runs", + "target": "script", + "command": [ + "python3", + "-c", + "import subprocess as s\nrun=lambda: s.run(['python3','scripts/silhouette_scan.py','evals/fixtures/silhouette/sil01_templated_essay.md'],capture_output=True,text=True).stdout\na=run(); b=run()\nprint('match' if a==b else 'DIFF')\nraise SystemExit(0 if a==b else 1)" + ], + "failure_mode": "Metrics built on set iteration or dict ordering could vary run to run, making the gate flaky.", + "correct_behavior": "Identical stdout on repeated runs; exit 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "match" + } + ] + }, + { + "id": "SIL-07", + "category": "robustness", + "title": "silhouette_scan reports a missing input file cleanly", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "evals/fixtures/silhouette/does_not_exist.md" + ], + "failure_mode": "open() with no guard raises an uncaught FileNotFoundError traceback on a missing path.", + "correct_behavior": "Exit 2 with a clean stderr message and no Python traceback.", + "assertions": [ + { + "type": "exit_code", + "equals": 2 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + } + ] + }, + { + "id": "SIL-08", + "category": "robustness", + "title": "A sub-threshold document yields null metrics, not a crash", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "evals/fixtures/silhouette/sil08_two_paragraphs.md" + ], + "failure_mode": "Discourse metrics need at least three paragraphs; a two-paragraph note could divide by zero or emit a misleading penalty.", + "correct_behavior": "Exit 0 with metrics null (not scored), no flags.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "metrics", + "equals": null + } + ] + }, + { + "id": "DOC-12", + "category": "scanner_detection", + "title": "check_silhouette passes both the reference-drift and separation gates", + "target": "script", + "command": [ + "python3", + "evals/check_silhouette.py" + ], + "failure_mode": "The silhouette scanner's human reference can drift from its source docs, or a metric edit can quietly break the ai/human separation the composite was tuned for.", + "correct_behavior": "Exit 0: the committed human_reference.json matches its sources and the in-repo corpus separates >= 8/12 ai from 0/8 human.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + } + ] + }, + { + "id": "CONTRIB-01", + "category": "contribute", + "title": "precheck names an already covering pattern", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-01" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-02", + "category": "contribute", + "title": "precheck accepts a genuinely uncaught tell", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-02" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-03", + "category": "contribute", + "title": "scaffold preserves post-redaction specimen bytes", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-03" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-04", + "category": "contribute", + "title": "redaction may not alter the tell", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-04" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-05", + "category": "contribute", + "title": "verify enforces red-first and refuses TODO report text", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-05" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-06", + "category": "contribute", + "title": "report rendering matches golden markdown", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-06" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-07", + "category": "contribute", + "title": "scaffold output is deterministic", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-07" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-08", + "category": "contribute", + "title": "missing bundle exits 2", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-08" + ], + "failure_mode": "The contribution pipeline loses specimen fidelity, skips red-first proof, or renders an unstable PR body.", + "correct_behavior": "The offline contribution helper passes this deterministic contract.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-09", + "category": "contribute", + "title": "capitalized tell assertions normalize without changing stdin", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-09" + ], + "failure_mode": "Capitalized contributed tells cannot verify because scanner phrases are normalized differently from row assertions.", + "correct_behavior": "Scaffold lowercases only the assertion value and verify compares reported phrases case-insensitively.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CONTRIB-10", + "category": "contribute", + "title": "precheck reports structure-only findings", + "target": "script", + "command": [ + "python3", + "evals/check_contrib.py", + "CONTRIB-10" + ], + "failure_mode": "Precheck only reads banned-phrase violations and lets structure-only tells scaffold as clean.", + "correct_behavior": "Precheck reads structure flags and reports the structure metric that already covers the snippet.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "FN-31", + "category": "scanner_false_negative", + "title": "Catches bare 'Research indicates that …' (unattributed authority)", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Research indicates that teenagers sleep longer on weekends.", + "failure_mode": "Bare, unattributed 'Research indicates …' is the same vague_attribution move as 'studies show' but was uncatalogued, so it scanned clean.", + "correct_behavior": "At least 1 vague_attribution violation on the sentence-initial bare 'research indicates'.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "research indicates" + } + ] + }, + { + "id": "FP-84", + "category": "scanner_false_positive", + "protects": "vague_attribution", + "title": "Attributed / possessive 'research indicates' stays clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Research by the Kaiser group indicates the intervention works. Our research indicates the leak is in the driver.", + "failure_mode": "Gating on the bare word 'research' before 'indicates' would clip the legitimate attributed form (a named study) and the concrete possessive form (a debugging claim).", + "correct_behavior": "No violations: an attributive 'by …' phrase or a leading possessive breaks the sentence-initial bare pattern.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "REC-49", + "category": "scanner_recall", + "title": "Sibling form: bare 'Research shows …' still flags", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Research shows that teenagers need more sleep than they get.", + "failure_mode": "A pattern gated only on 'indicates' would miss the equally vague sibling 'research shows'.", + "correct_behavior": "At least 1 vague_attribution violation on the sentence-initial bare 'research shows'.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "research shows" + } + ] + }, + { + "id": "SIL-9", + "category": "scanner_false_positive", + "title": "A reference doc reprising its opening themes is convention, not a recap coda", + "target": "script", + "command": [ + "python3", + "scripts/silhouette_scan.py", + "--genre", + "docs", + "evals/fixtures/silhouette/doctrine_reprise.md" + ], + "failure_mode": "callback_content flags thematic reprise in doctrine/spec/RFC register; under --genre docs that is document convention (dogfooding docs/PRODUCT.md surfaced it), unlike the essay recap coda the metric exists to catch.", + "correct_behavior": "Exit 0 under --genre docs; the default genre keeps flagging essay recap codas (SIL-04 pins that).", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-01a", + "category": "paired_paragraph", + "title": "throat_clearing with twin flags throat_clearing", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/throat_clearing_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled throat_clearing tell must flag target category.", + "correct_behavior": "At least one throat_clearing violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "throat_clearing" + } + ] + }, + { + "id": "PAIR-01b", + "category": "paired_paragraph", + "title": "throat_clearing without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/throat_clearing_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-01c", + "category": "paired_paragraph", + "title": "throat_clearing without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/throat_clearing_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-02a", + "category": "paired_paragraph", + "title": "significance_inflation with twin flags significance_inflation", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/significance_inflation_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled significance_inflation tell must flag target category.", + "correct_behavior": "At least one significance_inflation violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "significance_inflation" + } + ] + }, + { + "id": "PAIR-02b", + "category": "paired_paragraph", + "title": "significance_inflation without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/significance_inflation_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-02c", + "category": "paired_paragraph", + "title": "significance_inflation without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/significance_inflation_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-03a", + "category": "paired_paragraph", + "title": "negative_parallelism with twin flags negative_parallelism", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/negative_parallelism_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled negative_parallelism tell must flag target category.", + "correct_behavior": "At least one negative_parallelism violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "negative_parallelism" + } + ] + }, + { + "id": "PAIR-03b", + "category": "paired_paragraph", + "title": "negative_parallelism without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/negative_parallelism_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-03c", + "category": "paired_paragraph", + "title": "negative_parallelism without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/negative_parallelism_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-04a", + "category": "paired_paragraph", + "title": "rhetorical_question with twin flags rhetorical_question", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/rhetorical_question_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled rhetorical_question tell must flag target category.", + "correct_behavior": "At least one rhetorical_question violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_question" + } + ] + }, + { + "id": "PAIR-04b", + "category": "paired_paragraph", + "title": "rhetorical_question without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/rhetorical_question_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-04c", + "category": "paired_paragraph", + "title": "rhetorical_question without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/rhetorical_question_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-05a", + "category": "paired_paragraph", + "title": "hedge_stack with twin flags hedge_stack", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/hedge_stack_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled hedge_stack tell must flag target category.", + "correct_behavior": "At least one hedge_stack violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "hedge_stack" + } + ] + }, + { + "id": "PAIR-05b", + "category": "paired_paragraph", + "title": "hedge_stack without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/hedge_stack_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-05c", + "category": "paired_paragraph", + "title": "hedge_stack without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/hedge_stack_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-06a", + "category": "paired_paragraph", + "title": "false_agency with twin flags false_agency", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/false_agency_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled false_agency tell must flag target category.", + "correct_behavior": "At least one false_agency violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "false_agency" + } + ] + }, + { + "id": "PAIR-06b", + "category": "paired_paragraph", + "title": "false_agency without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/false_agency_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-06c", + "category": "paired_paragraph", + "title": "false_agency without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/false_agency_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-07a", + "category": "paired_paragraph", + "title": "jargon_leverage with twin flags jargon", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/jargon_leverage_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled jargon tell must flag target category.", + "correct_behavior": "At least one jargon violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "jargon" + } + ] + }, + { + "id": "PAIR-07b", + "category": "paired_paragraph", + "title": "jargon_leverage without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/jargon_leverage_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-07c", + "category": "paired_paragraph", + "title": "jargon_leverage without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/jargon_leverage_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-08a", + "category": "paired_paragraph", + "title": "reader_addressing with twin flags reader_addressing", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/reader_addressing_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled reader_addressing tell must flag target category.", + "correct_behavior": "At least one reader_addressing violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "reader_addressing" + } + ] + }, + { + "id": "PAIR-08b", + "category": "paired_paragraph", + "title": "reader_addressing without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/reader_addressing_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-08c", + "category": "paired_paragraph", + "title": "reader_addressing without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/reader_addressing_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-09a", + "category": "paired_paragraph", + "title": "conclusion_scaffold with twin flags conclusion_scaffold", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/conclusion_scaffold_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled conclusion_scaffold tell must flag target category.", + "correct_behavior": "At least one conclusion_scaffold violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "conclusion_scaffold" + } + ] + }, + { + "id": "PAIR-09b", + "category": "paired_paragraph", + "title": "conclusion_scaffold without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/conclusion_scaffold_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-09c", + "category": "paired_paragraph", + "title": "conclusion_scaffold without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/conclusion_scaffold_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-10a", + "category": "paired_paragraph", + "title": "emphasis_crutch with twin flags emphasis_crutch", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/emphasis_crutch_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled emphasis_crutch tell must flag target category.", + "correct_behavior": "At least one emphasis_crutch violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "emphasis_crutch" + } + ] + }, + { + "id": "PAIR-10b", + "category": "paired_paragraph", + "title": "emphasis_crutch without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/emphasis_crutch_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-10c", + "category": "paired_paragraph", + "title": "emphasis_crutch without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/emphasis_crutch_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-11a", + "category": "paired_paragraph", + "title": "vague_attribution with twin flags vague_attribution", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/vague_attribution_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled vague_attribution tell must flag target category.", + "correct_behavior": "At least one vague_attribution violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "vague_attribution" + } + ] + }, + { + "id": "PAIR-11b", + "category": "paired_paragraph", + "title": "vague_attribution without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/vague_attribution_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-11c", + "category": "paired_paragraph", + "title": "vague_attribution without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/vague_attribution_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-12a", + "category": "paired_paragraph", + "title": "participial_closer with twin flags superficial_ing", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/participial_closer_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled superficial_ing tell must flag target category.", + "correct_behavior": "At least one superficial_ing violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "superficial_ing" + } + ] + }, + { + "id": "PAIR-12b", + "category": "paired_paragraph", + "title": "participial_closer without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/participial_closer_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-12c", + "category": "paired_paragraph", + "title": "participial_closer without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/participial_closer_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-13a", + "category": "paired_paragraph", + "title": "anti_slop_register with twin flags anti_slop_register", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/anti_slop_register_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled anti_slop_register tell must flag target category.", + "correct_behavior": "At least one anti_slop_register violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "anti_slop_register" + } + ] + }, + { + "id": "PAIR-13b", + "category": "paired_paragraph", + "title": "anti_slop_register without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/anti_slop_register_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-13c", + "category": "paired_paragraph", + "title": "anti_slop_register without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/anti_slop_register_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIR-14a", + "category": "paired_paragraph", + "title": "cliffhanger_fragment with twin flags rhetorical_question", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/cliffhanger_fragment_with.txt" + ], + "failure_mode": "Realistic paragraph with one controlled rhetorical_question tell must flag target category.", + "correct_behavior": "At least one rhetorical_question violation.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "rhetorical_question" + } + ] + }, + { + "id": "PAIR-14b", + "category": "paired_paragraph", + "title": "cliffhanger_fragment without twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/cliffhanger_fragment_without.txt" + ], + "failure_mode": "Clean twin should not trip phrase scanner after the controlled tell is removed.", + "correct_behavior": "No banned phrase violations.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIR-14c", + "category": "paired_paragraph", + "title": "cliffhanger_fragment without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/cliffhanger_fragment_without.txt" + ], + "failure_mode": "Clean twin should remain valid as structure detection grows.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIRM-01a", + "category": "paired_macro", + "title": "macro_burstiness with twin flags sentence_burstiness", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/macro_burstiness_with.md" + ], + "failure_mode": "Macro with twin should isolate sentence_burstiness.", + "correct_behavior": "flagged.sentence_burstiness is true.", + "assertions": [ + { + "type": "json", + "path": "flagged.sentence_burstiness", + "equals": true + } + ] + }, + { + "id": "PAIRM-01b", + "category": "paired_macro", + "title": "macro_burstiness without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/macro_burstiness_without.md" + ], + "failure_mode": "Macro without twin should avoid structure flags.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIRM-01c", + "category": "paired_macro", + "title": "macro_burstiness both twins are phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/macro_burstiness_with.md" + ], + "failure_mode": "Macro pairs isolate structure, not phrase tells.", + "correct_behavior": "No banned phrase violations in macro with twin.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIRM-01d", + "category": "paired_macro", + "title": "macro_burstiness clean macro twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/macro_burstiness_without.md" + ], + "failure_mode": "Macro pairs isolate structure, not phrase tells.", + "correct_behavior": "No banned phrase violations in macro without twin.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIRM-02a", + "category": "paired_macro", + "title": "macro_coda with twin flags conclusion_coda", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/macro_coda_with.md" + ], + "failure_mode": "Macro with twin should isolate conclusion_coda.", + "correct_behavior": "flagged.conclusion_coda is true.", + "assertions": [ + { + "type": "json", + "path": "flagged.conclusion_coda", + "equals": true + } + ] + }, + { + "id": "PAIRM-02b", + "category": "paired_macro", + "title": "macro_coda without twin is structure-clean", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py", + "evals/fixtures/pairs/macro_coda_without.md" + ], + "failure_mode": "Macro without twin should avoid structure flags.", + "correct_behavior": "Structure scanner exits clean.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "PAIRM-02c", + "category": "paired_macro", + "title": "macro_coda both twins are phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/macro_coda_with.md" + ], + "failure_mode": "Macro pairs isolate structure, not phrase tells.", + "correct_behavior": "No banned phrase violations in macro with twin.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "PAIRM-02d", + "category": "paired_macro", + "title": "macro_coda clean macro twin is phrase-clean", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py", + "evals/fixtures/pairs/macro_coda_without.md" + ], + "failure_mode": "Macro pairs isolate structure, not phrase tells.", + "correct_behavior": "No banned phrase violations in macro without twin.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "DOC-07", + "category": "docs", + "title": "Seeded realistic documents preserve recall and clean twins", + "target": "script", + "command": [ + "python3", + "evals/check_seeded_docs.py" + ], + "failure_mode": "Seeded document harness must catch expected categories and verify clean twins.", + "correct_behavior": "Checker exits 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "DOC-08", + "category": "docs", + "title": "Paired fixture hygiene gate", + "target": "script", + "command": [ + "python3", + "evals/check_pairs.py" + ], + "failure_mode": "Pair fixtures can drift as scanners evolve.", + "correct_behavior": "Checker exits 0.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "DOC-13", + "category": "scanner_detection", + "title": "Command router surface stays in parity with references/commands/", + "target": "script", + "command": [ + "python3", + "evals/check_commands.py" + ], + "failure_mode": "SKILL.md's argument-hint, its routing table, and the files under references/commands/ can drift: a verb added to the hint without a file, an orphaned command file, a routed command missing its file, an over-length command file, or a command file with the wrong first line all silently break the impeccable-style router.", + "correct_behavior": "Exit 0: the argument-hint top-level verbs equal the pinned four-verb surface, every routing-table link resolves to a references/commands/.md file and vice versa, each command file is <= 90 lines, and each starts with `# /unslop `.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + } + ] + }, + { + "id": "CAL-01", + "category": "calibrate", + "title": "Per-dimension transform correctness with constraints preserved", + "target": "script", + "command": [ + "python3", + "-c", + "import json, re, subprocess, sys\nsys.path.insert(0, 'scripts')\nfrom extract_constraints import extract_constraints\nDIMS = {\n 'contractions': 'evals/fixtures/calibrate/base_contractions.txt',\n 'em_dash': 'evals/fixtures/calibrate/base_em_dash.txt',\n 'sentence_length': 'evals/fixtures/calibrate/base_sentence_length.txt',\n 'connectives': 'evals/fixtures/calibrate/base_connectives.txt',\n 'staccato': 'evals/fixtures/calibrate/base_staccato.txt',\n}\ndef sentence_count(s):\n return len(re.findall(r'[^.!?]+[.!?]+', s))\nresults = {}\nfor dim, path in DIMS.items():\n proc = subprocess.run(\n ['python3', 'scripts/calibrate_pairs.py', 'generate', '--base', path, '--dimension', dim, '--seed', '1'],\n capture_output=True, text=True,\n )\n ok = proc.returncode == 0\n if ok:\n data = json.loads(proc.stdout)\n base_text = open(path).read()\n cbefore = {c['value'] for c in extract_constraints(base_text)}\n cafter = {c['value'] for c in extract_constraints(data['b_text'])}\n ok = (\n data['a_text'] != data['b_text']\n and bool(data.get('transform_applied'))\n and cbefore.issubset(cafter)\n )\n if dim == 'em_dash':\n ok = ok and sentence_count(data['a_text']) == sentence_count(data['b_text'])\n results[dim] = ok\nprint(json.dumps(results))" + ], + "failure_mode": "A dimension's transform silently drops a must-preserve constraint, produces no change, the CLI errors, or (em_dash specifically) changes sentence count between A and B via a lone-dash join/split path -- and it goes unnoticed because only an aggregate pass/fail is checked.", + "correct_behavior": "Each of the five dimensions produces a genuinely different B-variant that still contains every constraint extracted from the base passage; em_dash additionally holds sentence count stable between A and B (the dimension is restricted to the paired-dash<->comma path); a broken dimension names itself in the json path that fails.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "contractions", + "equals": true + }, + { + "type": "json", + "path": "em_dash", + "equals": true + }, + { + "type": "json", + "path": "sentence_length", + "equals": true + }, + { + "type": "json", + "path": "connectives", + "equals": true + }, + { + "type": "json", + "path": "staccato", + "equals": true + } + ] + }, + { + "id": "CAL-02", + "category": "calibrate", + "title": "Inexpressible dimension exits 3 instead of forcing a bad transform", + "target": "script", + "command": [ + "python3", + "scripts/calibrate_pairs.py", + "generate", + "--base", + "evals/fixtures/calibrate/inexpressible_contractions.txt", + "--dimension", + "contractions", + "--seed", + "1" + ], + "failure_mode": "The generator forces a transform on a passage where neither pole's pattern is present, corrupting the passage instead of admitting the dimension does not apply here.", + "correct_behavior": "Exit code 3 and an error naming the passage as not expressible for this dimension.", + "assertions": [ + { + "type": "exit_code", + "equals": 3 + }, + { + "type": "stdout_contains", + "value": "dimension not expressible in this passage" + } + ] + }, + { + "id": "CAL-03", + "category": "calibrate", + "title": "Same seed produces byte-identical output", + "target": "script", + "command": [ + "python3", + "-c", + "import subprocess, sys\ncmd = ['python3', 'scripts/calibrate_pairs.py', 'generate', '--base', 'evals/fixtures/calibrate/base_contractions.txt', '--dimension', 'contractions', '--seed', '7']\na = subprocess.run(cmd, capture_output=True, text=True)\nb = subprocess.run(cmd, capture_output=True, text=True)\nif a.stdout == b.stdout and a.returncode == 0 == b.returncode:\n print('IDENTICAL')\nelse:\n print('DIFFERENT')\n sys.exit(1)" + ], + "failure_mode": "A dimension's transform depends on unseeded randomness (dict/set iteration order, wall-clock, os.urandom) so two runs with the same seed diverge, breaking reproducibility of the game log.", + "correct_behavior": "Two invocations with identical base file, dimension, and seed emit byte-identical stdout.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "stdout_contains", + "value": "IDENTICAL" + } + ] + }, + { + "id": "CAL-04", + "category": "calibrate", + "title": "Canned preference stream aggregates to the expected direction and confidence", + "target": "script", + "command": [ + "python3", + "scripts/calibrate_score.py", + "--preferences", + "evals/fixtures/calibrate/preferences_canned.jsonl" + ], + "failure_mode": "The Wilson-lower-bound aggregation drifts (wrong tally key, wrong z, off-by-one on n vs decisive count) and a canned, hand-computed distribution stops matching its known confidence value.", + "correct_behavior": "The contractions dimension (7 of 8 decisive votes for 'expanded') reports preferred='expanded' with confidence 0.529, and sentence_length (9 of 9 votes for 'short') reports preferred='short' with confidence 0.701. staccato (3 of 6 for each pole, a genuine tie) reports status='tied' and preferred=null rather than an alphabetically-broken winner. connectives has two raw rows sharing the same pair_id (a replayed round); dedup collapses them so n=1, not 2.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "dimensions.contractions.preferred", + "equals": "expanded" + }, + { + "type": "json", + "path": "dimensions.contractions.confidence", + "equals": 0.529 + }, + { + "type": "json", + "path": "dimensions.sentence_length.preferred", + "equals": "short" + }, + { + "type": "json", + "path": "dimensions.sentence_length.confidence", + "equals": 0.701 + }, + { + "type": "json", + "path": "dimensions.staccato.status", + "equals": "tied" + }, + { + "type": "json", + "path": "dimensions.staccato.preferred", + "equals": null + }, + { + "type": "json", + "path": "dimensions.connectives.n", + "equals": 1 + } + ] + }, + { + "id": "CAL-05", + "category": "calibrate", + "title": "Below-k observations report insufficient regardless of lean", + "target": "script", + "command": [ + "python3", + "scripts/calibrate_score.py", + "--preferences", + "evals/fixtures/calibrate/preferences_canned.jsonl" + ], + "failure_mode": "A dimension with only a handful of lopsided votes (e.g. 3 of 3 for one side) reports a confident direction instead of admitting there is not enough data yet, letting the game stop too early.", + "correct_behavior": "em_dash has only 3 recorded observations (k=5 floor unmet) and reports status='insufficient' with preferred=null even though all 3 lean the same way.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "dimensions.em_dash.status", + "equals": "insufficient" + }, + { + "type": "json", + "path": "dimensions.em_dash.n", + "equals": 3 + }, + { + "type": "json", + "path": "dimensions.em_dash.preferred", + "equals": null + } + ] + }, + { + "id": "CAL-06", + "category": "calibrate", + "title": "Confident preference conflicting with the measured profile is surfaced", + "target": "script", + "command": [ + "python3", + "scripts/calibrate_score.py", + "--preferences", + "evals/fixtures/calibrate/preferences_canned.jsonl", + "--profile", + "evals/fixtures/calibrate/profile_conflict.json" + ], + "failure_mode": "A confident stated preference for short sentences is reported alongside a profile measured from the user's own samples averaging much longer sentences, and nothing flags the contradiction — the card would silently pick one and lose the disagreement.", + "correct_behavior": "A conflicts[] record names the sentence_length dimension, both the stated preference ('short', confidence 0.701, provenance stated-preference) and the measured profile value (avg_sentence_length=22.4, provenance measured-from-samples).", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "conflicts.0.dimension", + "equals": "sentence_length" + }, + { + "type": "json", + "path": "conflicts.0.preferred", + "equals": "short" + }, + { + "type": "json", + "path": "conflicts.0.preferred_provenance", + "equals": "stated-preference" + }, + { + "type": "json", + "path": "conflicts.0.measured_value", + "equals": 22.4 + }, + { + "type": "json", + "path": "conflicts.0.measured_provenance", + "equals": "measured-from-samples" + } + ] + }, + { + "id": "CAL-07", + "category": "calibrate", + "title": "--next orders by fewest observations then a deterministic tiebreak", + "target": "script", + "command": [ + "python3", + "scripts/calibrate_score.py", + "--preferences", + "evals/fixtures/calibrate/preferences_canned.jsonl", + "--next" + ], + "failure_mode": "connectives and staccato are both entirely unplayed (n=0), a tie that non-deterministic dict/set ordering could break differently on every call, making the game's next-dimension choice unreproducible.", + "correct_behavior": "Both connectives and staccato have n=0; the fixed dimension order (contractions, em_dash, sentence_length, connectives, staccato) breaks the tie in favor of connectives every time.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "next_dimension", + "equals": "connectives" + }, + { + "type": "json", + "path": "reason", + "equals": "fewest_observations" + } + ] + }, + { + "id": "CAL-08", + "category": "calibrate", + "title": "Missing preferences file exits 2", + "target": "script", + "command": [ + "python3", + "scripts/calibrate_score.py", + "--preferences", + "evals/fixtures/calibrate/does-not-exist.jsonl" + ], + "failure_mode": "A missing or mistyped preferences path crashes with an uncaught traceback or silently reports an empty-but-successful aggregation instead of a clear I/O error.", + "correct_behavior": "Exit code 2 with a JSON error naming the read failure; no traceback.", + "assertions": [ + { + "type": "exit_code", + "equals": 2 + }, + { + "type": "stderr_not_contains", + "value": "Traceback" + } + ] + }, + { + "id": "CAL-09", + "category": "calibrate", + "title": "Contraction rewrite skips a capitalized proper noun it collides with", + "target": "script", + "command": [ + "python3", + "-c", + "import json, subprocess\nproc = subprocess.run(\n ['python3', 'scripts/calibrate_pairs.py', 'generate', '--base',\n 'evals/fixtures/calibrate/proper_noun_contraction.txt', '--dimension', 'contractions', '--seed', '1'],\n capture_output=True, text=True,\n)\nresult = {'exit_code': proc.returncode, 'proper_noun_preserved': False, 'other_transform_applied': False}\nif proc.returncode == 0:\n data = json.loads(proc.stdout)\n needle = 'Venue Can' + chr(39) + 't Stop'\n result['proper_noun_preserved'] = needle in data['b_text']\n result['other_transform_applied'] = 'will not' in data['b_text']\nprint(json.dumps(result))" + ], + "failure_mode": "The 'can't' -> 'cannot' rewrite fires everywhere the pattern matches, including inside a capitalized proper noun that happens to reuse the word ('the venue Can't Stop' -> 'Cannot Stop'), corrupting a name. The old substring-based constraint guard misses the corruption because extract_constraints truncates the proper noun at the apostrophe ('Venue Can’t Stop' -> constraint value 'Venue Can'), and 'Venue Can' is still a substring of the corrupted 'Venue Cannot Stop'.", + "correct_behavior": "The ordinary contraction ('can't attend' -> 'cannot attend', 'won't' -> 'will not') is still rewritten, but 'Venue Can't Stop' is left untouched because the match sits inside a capitalized multi-word span.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "proper_noun_preserved", + "equals": true + }, + { + "type": "json", + "path": "other_transform_applied", + "equals": true + } + ] + }, + { + "id": "CAL-10", + "category": "calibrate", + "title": "Generated pairs are annotated with banned_phrase_scan flags instead of declining", + "target": "script", + "command": [ + "python3", + "-c", + "import json, subprocess\ndef gen(base, dim):\n proc = subprocess.run(\n ['python3', 'scripts/calibrate_pairs.py', 'generate', '--base', base, '--dimension', dim, '--seed', '1'],\n capture_output=True, text=True,\n )\n return proc.returncode, (json.loads(proc.stdout) if proc.returncode == 0 else None)\n\nrc1, staccato = gen('evals/fixtures/calibrate/staccato_flagged.txt', 'staccato')\nrc2, clean = gen('evals/fixtures/calibrate/contractions_clean.txt', 'contractions')\nresult = {\n 'staccato_exit': rc1,\n 'staccato_has_a_flags_field': bool(staccato) and 'a_flags' in staccato,\n 'staccato_has_b_flags_field': bool(staccato) and 'b_flags' in staccato,\n 'staccato_a_flags_empty': bool(staccato) and staccato['a_flags'] == [],\n 'staccato_b_flags_has_anti_slop_register': bool(staccato) and 'anti_slop_register' in staccato['b_flags'],\n 'clean_exit': rc2,\n 'clean_a_flags_empty': bool(clean) and clean['a_flags'] == [],\n 'clean_b_flags_empty': bool(clean) and clean['b_flags'] == [],\n}\nprint(json.dumps(result))" + ], + "failure_mode": "A generated B-variant can legitimately trip banned_phrase_scan (a staccato pole reads as anti_slop_register; a plain connective can read as filler_opener). Declining or silently dropping these pairs would make whole dimensions unplayable whenever a user's real voice sits on the flagged pole -- the product decision is to annotate, not decline, and let the voice card record 'user-preference overrides register guard' when the user picks that pole.", + "correct_behavior": "generate_pair's output always carries 'a_flags'/'b_flags' (category lists, empty when clean). A staccato pair generated from a comma-fragmentable base reports b_flags containing 'anti_slop_register' while a_flags is empty; a plain contractions pair reports both empty.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "staccato_exit", + "equals": 0 + }, + { + "type": "json", + "path": "staccato_has_a_flags_field", + "equals": true + }, + { + "type": "json", + "path": "staccato_has_b_flags_field", + "equals": true + }, + { + "type": "json", + "path": "staccato_a_flags_empty", + "equals": true + }, + { + "type": "json", + "path": "staccato_b_flags_has_anti_slop_register", + "equals": true + }, + { + "type": "json", + "path": "clean_exit", + "equals": 0 + }, + { + "type": "json", + "path": "clean_a_flags_empty", + "equals": true + }, + { + "type": "json", + "path": "clean_b_flags_empty", + "equals": true + } + ] + }, + { + "id": "CAL-11", + "category": "calibrate", + "title": "Replayed pair_id dedups to the latest row by ts, not a naive double-count", + "target": "script", + "command": [ + "python3", + "-c", + "import json, sys\nfrom pathlib import Path\nsys.path.insert(0, 'scripts')\nfrom calibrate_score import load_preferences, dedup_by_pair_id\nrows = load_preferences(Path('evals/fixtures/calibrate/preferences_canned.jsonl'))\ndeduped = dedup_by_pair_id(rows)\nconn = [r for r in deduped if r['dimension'] == 'connectives']\nresult = {\n 'raw_row_count': sum(1 for r in rows if r['dimension'] == 'connectives'),\n 'deduped_row_count': len(conn),\n 'latest_choice': conn[0]['choice'] if conn else None,\n 'latest_ts': conn[0]['ts'] if conn else None,\n}\nprint(json.dumps(result))" + ], + "failure_mode": "A game round that gets appended to the preferences JSONL twice (e.g. a crashed/retried write) would double-count toward n and let whichever row happens to load first win the tally, instead of collapsing to one observation and honoring the user's latest choice.", + "correct_behavior": "The canned fixture's connectives dimension has two raw rows sharing pair_id 'connectives-00' (an earlier 'a' choice and a later 'b' choice). dedup_by_pair_id collapses them to one row: the latest by ts, choice='b'.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + }, + { + "type": "json", + "path": "raw_row_count", + "equals": 2 + }, + { + "type": "json", + "path": "deduped_row_count", + "equals": 1 + }, + { + "type": "json", + "path": "latest_choice", + "equals": "b" + }, + { + "type": "json", + "path": "latest_ts", + "equals": "2026-07-06T09:00:00Z" + } + ] + }, + { + "id": "VOICE-01", + "category": "voice_separation", + "title": "Voice held-outs separate by composite", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--separation" + ], + "failure_mode": "Voice scorer regression: 3x3 held-out composite matrix must have diagonal wins row-wise and column-wise.", + "correct_behavior": "3x3 held-out composite matrix must have diagonal wins row-wise and column-wise.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-02", + "category": "voice_separation", + "title": "Boris held-out separates by composite", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--separation" + ], + "failure_mode": "Voice scorer regression: full 3x3 matrix must keep all diagonal wins.", + "correct_behavior": "The 3x3 held-out composite matrix has diagonal wins row-wise and column-wise.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-03", + "category": "voice_separation", + "title": "Celia held-out separates by composite", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--separation" + ], + "failure_mode": "Voice scorer regression: full 3x3 matrix must keep all diagonal wins.", + "correct_behavior": "The 3x3 held-out composite matrix has diagonal wins row-wise and column-wise.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-04", + "category": "voice_gi", + "title": "GI ranks own held-out above cross authors", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--gi" + ], + "failure_mode": "Voice scorer regression: Own held-out GI must exceed each cross-author GI against the same profile.", + "correct_behavior": "Own held-out GI must exceed each cross-author GI against the same profile.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-05", + "category": "voice_gaming_guard", + "title": "Stuffed Boris does not beat genuine Amara", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--gaming" + ], + "failure_mode": "Voice scorer regression: A hand-crafted Boris doc stuffed with Amara-like function words, contractions, and punctuation must score worse than genuine Amara and have lower GI.", + "correct_behavior": "A hand-crafted Boris doc stuffed with Amara-like function words, contractions, and punctuation must score worse than genuine Amara and have lower GI.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-06", + "category": "voice_copy_gate", + "title": "Verbatim sample lift trips copy gate", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--copy-violation" + ], + "failure_mode": "Voice scorer regression: A paragraph lifted from an Amara profile sample must report copy_gate.violation true.", + "correct_behavior": "A paragraph lifted from an Amara profile sample must report copy_gate.violation true.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-07", + "category": "voice_copy_gate", + "title": "Honest same-style held-out does not trip copy gate", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--copy-clean" + ], + "failure_mode": "Voice scorer regression: Amara held-out doc must report copy_gate.violation false.", + "correct_behavior": "Amara held-out doc must report copy_gate.violation false.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-08", + "category": "voice_determinism", + "title": "Same seed yields identical composite", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--determinism" + ], + "failure_mode": "Voice scorer regression: Seeded GI trials must be reproducible.", + "correct_behavior": "Seeded GI trials must be reproducible.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-09", + "category": "voice_robustness", + "title": "Missing profile exits 2", + "target": "script", + "command": [ + "python3", + "scripts/voice_score.py", + "--profile", + "evals/fixtures/voice/profiles/missing.json", + "--impostors", + "evals/fixtures/voice/impostors", + "--seed", + "1", + "evals/fixtures/voice/authors/amara/doc5.md" + ], + "failure_mode": "Voice scorer regression: Missing profile is a setup error and exits 2.", + "correct_behavior": "Missing profile is a setup error and exits 2.", + "assertions": [ + { + "type": "exit_code", + "equals": 2 + } + ] + }, + { + "id": "VOICE-10", + "category": "voice_robustness", + "title": "Short candidate reports low confidence", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--short" + ], + "failure_mode": "Voice scorer regression: Short candidates keep measurement output but null distributional metrics and set low_confidence true.", + "correct_behavior": "Short candidates keep measurement output but null distributional metrics and set low_confidence true.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "VOICE-11", + "category": "voice_profiles", + "title": "Committed profiles match regenerated docs 1-4", + "target": "script", + "command": [ + "python3", + "evals/check_voice.py", + "--profiles" + ], + "failure_mode": "Voice scorer regression: Profile drift gate regenerates author profiles from docs 1-4 and compares exact JSON.", + "correct_behavior": "Profile drift gate regenerates author profiles from docs 1-4 and compares exact JSON.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-01", + "category": "mimic_loop", + "title": "Refine loop accepts a DEV improvement", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--acceptance" + ], + "failure_mode": "Refine loop must accept a candidate that lowers the held-out DEV composite past min-delta.", + "correct_behavior": "Dry-run loop accepts the improving candidate and records it as best_candidate.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-02", + "category": "mimic_loop", + "title": "Refine loop stops on patience", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--patience" + ], + "failure_mode": "Loop must halt with stop_reason 'patience' after N non-improving iterations.", + "correct_behavior": "Loop halts on patience without a reward-hacking warning.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-03", + "category": "mimic_loop", + "title": "Refine loop halts on divergence", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--divergence" + ], + "failure_mode": "Loop must halt with stop_reason 'divergence' and reward_hacking_warning when A improves while DEV worsens.", + "correct_behavior": "Loop detects the reward-hacking signature (A down, DEV up two rounds) and halts.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-04", + "category": "mimic_loop", + "title": "Copy-gate rejects verbatim candidates", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--copy-gate" + ], + "failure_mode": "A candidate copying an A sample verbatim must be rejected with reason copy_gate.", + "correct_behavior": "Anti-verbatim-copy gate rejects the candidate for reason copy_gate.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-05", + "category": "mimic_loop", + "title": "Fact-gate rejects dropped constraints", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--fact-gate" + ], + "failure_mode": "A candidate dropping a draft constraint must be rejected with reason preservation.", + "correct_behavior": "Preservation gate rejects the candidate for reason preservation.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-06", + "category": "mimic_loop", + "title": "Refine report is deterministic", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--determinism" + ], + "failure_mode": "Two identical dry-runs must produce byte-identical report.json.", + "correct_behavior": "report.json is byte-identical across runs to the same output dir.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-07", + "category": "mimic_loop", + "title": "Paired stats separate win from null", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--stats" + ], + "failure_mode": "mimic_stats must call a real improvement improved=true and a null improved=false.", + "correct_behavior": "BCa+sign-flip verdict is improved on the win set and not-improved on the null set.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-08", + "category": "mimic_loop", + "title": "Split refuses fewer than five docs", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--split-refusal" + ], + "failure_mode": "The loop must exit 2 when the sample set has fewer than five documents.", + "correct_behavior": "Loop refuses a four-document sample set with exit 2.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-09", + "category": "mimic_loop", + "title": "Directives and card amendments derive", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--directives" + ], + "failure_mode": "The loop must emit ranked directives, each with a paired card amendment line.", + "correct_behavior": "Directives derive from DEV metric deltas with metric, directive, and card_amendment fields.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-10", + "category": "mimic_loop", + "title": "Marker-stuffed candidate loses to honest under the GI composite", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--stuffed-attack" + ], + "failure_mode": "Acceptance scored on a raw weighted distance lets a marker-stuffed candidate that clears every hard gate beat honest prose. Refine must score DEV/A with the full voice_score composite (GI + impostor z), so the stuffed attack scores worse and honest wins.", + "correct_behavior": "Both honest (cand01) and the punctuation/repetition-stuffed attack (cand02) pass the hard gates, but under the GI-bearing composite the stuffed candidate scores strictly worse and best_candidate is the honest one.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "MIMIC-11", + "category": "mimic_loop", + "title": "Live generation path drives the same pipeline", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--live-path" + ], + "failure_mode": "The LIVE path (--generate-cmd) must actually assemble prompts, invoke the generator per beam, and run candidates through the same gate/score/accept pipeline as the dry-run, not be a dead flag.", + "correct_behavior": "A mock generator invoked via --generate-cmd produces the same report shape, beam width, acceptance trajectory, best_score, and final.md as the dry-run acceptance fixture.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-01", + "category": "voice_card", + "title": "Voice card is deterministic", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--card-determinism" + ], + "failure_mode": "voice_card.py must produce byte-identical card.md and sheets from the same inputs.", + "correct_behavior": "Card core and situation sheets are byte-identical across runs.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-02", + "category": "voice_card", + "title": "Card core within budget with sections", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--card-budget" + ], + "failure_mode": "card.md must stay <= 300 words and contain the required core sections and index table.", + "correct_behavior": "Core card is under the word budget and has rhythm, contraction, punctuation, opener, and index sections.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-03", + "category": "voice_card", + "title": "Card facts match the profile", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--card-facts" + ], + "failure_mode": "The median sentence length and contraction rate stated in the card must equal the profile values.", + "correct_behavior": "Card-stated median and contraction rate parse and equal the profile numbers.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-04", + "category": "voice_card", + "title": "Teach layout writes profile, card, provenance", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--card-layout" + ], + "failure_mode": "The teach procedure must write profile.json, card.md, and a provenance.json whose hashes verify.", + "correct_behavior": "Teach layout produces the three artifacts and provenance sha256 hashes match the samples.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-05", + "category": "voice_card", + "title": "Never-does claims are true of samples", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--card-never-does" + ], + "failure_mode": "Every never-does punctuation claim on the card must have a zero rate in the profile.", + "correct_behavior": "Each claimed zero-rate feature is actually zero in the profile.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-06", + "category": "voice_card", + "title": "Coverage gap names missing dimensions", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--coverage-gap" + ], + "failure_mode": "A sample set with no numeric writing must mark numbers-data uncovered in the coverage matrix.", + "correct_behavior": "The deterministic coverage matrix flags numbers-data as a gap when samples lack numbers.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-07", + "category": "voice_card", + "title": "Uncovered dimensions are never fabricated", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--no-fabrication" + ], + "failure_mode": "An uncovered dimension must get no sheet and be listed as uncovered in card.md.", + "correct_behavior": "No sheet is written for an uncovered dimension and card.md names it under Uncovered.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "CARD-08", + "category": "voice_card", + "title": "Profile that does not match samples is rejected", + "target": "script", + "command": [ + "python3", + "evals/check_mimic.py", + "--card-profile-mismatch" + ], + "failure_mode": "voice_card must not build a card from a --profile that does not describe the --samples. It must recompute the profile from --samples and, on disagreement, exit non-zero with a named mismatch and write no card.", + "correct_behavior": "A committed 4-doc Amara profile against the 5-doc Amara samples dir exits 2 with a named field mismatch and writes no card.md.", + "assertions": [ + { + "type": "exit_code", + "equals": 0 + } + ] + }, + { + "id": "LANG-3a", + "category": "scanner_false_negative", + "title": "Imperative headline stack is English slop, not a foreign language", + "target": "script", + "command": [ + "python3", + "scripts/structure_scan.py" + ], + "stdin": "Ship faster. Build bolder. Dream bigger. Move smarter. Win sooner.\n\nShip cleaner. Build leaner. Dream louder. Move quicker. Win bigger.\n\nShip harder. Build brighter. Dream deeper. Move faster. Win louder.", + "failure_mode": "The English-detection gate keyed on function-word ABSENCE, so low-function-word English slop genres (imperative stacks, buzzword lists) were declined as non-English before any pattern ran — silently defeating the shipped headline-cadence detection.", + "correct_behavior": "non_english false; structural flags fire.", + "assertions": [ + { + "type": "stdout_not_contains", + "value": "\"non_english\": true" + }, + { + "type": "exit_code", + "equals": 1 + } + ] + }, + { + "id": "LANG-3b", + "category": "scanner_false_negative", + "title": "Buzzword noun-list is English slop, not a foreign language", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "Synergy, alignment, leverage points, game changers, paradigm shifts, thought leadership, growth hacking, innovation pipelines, seamless integration, robust solutions.", + "failure_mode": "Same gate: comma-separated jargon lists fall under the function-word threshold and were declined unscanned.", + "correct_behavior": "non_english false; banned phrases flag.", + "assertions": [ + { + "type": "stdout_not_contains", + "value": "\"non_english\": true" + }, + { + "type": "json", + "path": "total_violations", + "gte": 2 + } + ] + }, + { + "id": "FP-85", + "category": "scanner_false_positive", + "title": "Humans telling stories are not false agency", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "My grandmother tells a story every night before bed. The patient tells a story of chest pain during exertion.", + "protects": "false_agency", + "failure_mode": "'tells a story' shipped as a flat hard ban; the false-agency tell is INANIMATE subjects narrating (the data tells a story), not people, who tell stories literally.", + "correct_behavior": "No violations for human subjects.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "equals": 0 + } + ] + }, + { + "id": "REC-50", + "category": "scanner_recall", + "title": "Inanimate 'the data tells a story' still flags after the subject gate", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The dashboard tells a story of steady decline, and the numbers tell a story too.", + "failure_mode": "Gating on inanimate subjects must not lose the genuine false-agency form.", + "correct_behavior": "Flags as false_agency.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_category_equals", + "value": "false_agency" + } + ] } ] -} \ No newline at end of file +} diff --git a/evals/check_commands.py b/evals/check_commands.py new file mode 100644 index 0000000..709127d --- /dev/null +++ b/evals/check_commands.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Parity gate for the subcommand router. + +Pins the top-level verb surface and keeps SKILL.md's argument-hint, its routing +table, and the files under references/commands/ in exact agreement. A future edit +cannot silently add or drop a verb, orphan a command file, or leave a routed +command without a file. + +Checks: + (a) argument-hint top-level verbs == EXPECTED_TOPLEVEL, exactly. + (b) every routing-table link resolves to a references/commands/.md file, + and every such file is linked from the routing table (no orphans). + (c) every top-level verb is routed; any extra routed command is an allowed + maintenance command. + (d) each command file is <= MAX_LINES lines. + (e) each command file's first line is `# /unslop `. +""" + +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _check_support import ROOT # noqa: E402 + +SKILL = ROOT / "SKILL.md" +COMMANDS_DIR = ROOT / "references" / "commands" + +# The pinned four-verb surface. A change here is a deliberate product decision, +# never an accident of editing SKILL.md. +EXPECTED_TOPLEVEL = {"teach", "cleanup", "rewrite", "mimic"} +# Command files reachable as maintenance paths, not top-level verbs. +ALLOWED_MAINTENANCE = {"contribute"} +MAX_LINES = 90 + + +def argument_hint_verbs(text: str) -> set[str]: + """Top-level verbs from the frontmatter argument-hint's first [...] group.""" + m = re.search(r'^argument-hint:\s*"(.*?)"\s*$', text, re.M) + if not m: + raise RuntimeError("no argument-hint field in SKILL.md frontmatter") + group = re.search(r"\[(.*?)\]", m.group(1)) + if not group: + raise RuntimeError("argument-hint has no [...] command group") + verbs = set() + for chunk in group.group(1).split("·"): + for tok in chunk.split("|"): + tok = tok.strip() + if tok: + verbs.add(tok) + return verbs + + +def routing_table_commands(text: str) -> set[str]: + """Commands linked as references/commands/.md anywhere in SKILL.md.""" + return set(re.findall(r"references/commands/([a-z0-9_-]+)\.md", text)) + + +def command_files() -> set[str]: + return {p.stem for p in COMMANDS_DIR.glob("*.md")} + + +def main() -> int: + problems = [] + text = SKILL.read_text(encoding="utf-8") + + hint = argument_hint_verbs(text) + if hint != EXPECTED_TOPLEVEL: + problems.append( + f"argument-hint verbs {sorted(hint)} != pinned surface " + f"{sorted(EXPECTED_TOPLEVEL)}" + ) + + routed = routing_table_commands(text) + files = command_files() + + for cmd in sorted(routed - files): + problems.append(f"routing table links '{cmd}' but references/commands/{cmd}.md is missing") + for cmd in sorted(files - routed): + problems.append(f"references/commands/{cmd}.md exists but is not in the routing table (orphan)") + + for cmd in sorted(EXPECTED_TOPLEVEL - routed): + problems.append(f"top-level verb '{cmd}' is not in the routing table") + + extras = routed - EXPECTED_TOPLEVEL + for cmd in sorted(extras - ALLOWED_MAINTENANCE): + problems.append(f"routed command '{cmd}' is neither a top-level verb nor an allowed maintenance path") + + for path in sorted(COMMANDS_DIR.glob("*.md")): + lines = path.read_text(encoding="utf-8").splitlines() + n = len(lines) + if n > MAX_LINES: + problems.append(f"{path.name} is {n} lines (> {MAX_LINES})") + expected_first = f"# /unslop {path.stem}" + first = lines[0] if lines else "" + if first.strip() != expected_first: + problems.append(f"{path.name} first line is {first!r}, expected {expected_first!r}") + + # Synonym routing: the "Routing by phrase" table must exist, cover the + # demoted flows, and every target must be a real command file (optionally + # with a #section anchor). + REQUIRED_SYNONYMS = {"audit", "harvest", "calibrate", "refine", "voice check"} + skill_text = (ROOT / "SKILL.md").read_text() + import re as _re + syn_rows = _re.findall(r"^\| *([^|]+?) *\| *\[?references/commands/([a-z]+)\.md(#[a-z-]+)?", + skill_text[skill_text.find("Routing by phrase"):], _re.M) \ + if "Routing by phrase" in skill_text else [] + covered = {name.strip().strip('`').lower() for name, _, _ in syn_rows} + for req in sorted(REQUIRED_SYNONYMS): + if not any(req in c for c in covered): + problems.append(f"synonym routing missing an entry covering {req!r}") + for name, cmd, _ in syn_rows: + if not (COMMANDS_DIR / f"{cmd}.md").exists(): + problems.append(f"synonym {name.strip()!r} routes to missing file {cmd}.md") + + if problems: + print("command router parity FAILED:") + for p in problems: + print(f" - {p}") + return 1 + + print(f"command router parity ok: {len(files)} files, {len(syn_rows)} synonyms, " + f"top-level {sorted(EXPECTED_TOPLEVEL)}, maintenance {sorted(extras)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/check_contrib.py b/evals/check_contrib.py new file mode 100644 index 0000000..4df6139 --- /dev/null +++ b/evals/check_contrib.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Deterministic eval helper for the contribution scaffolder.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path + +from _check_support import ROOT, run # noqa: E402 + +sys.path.insert(0, str(ROOT)) +from scripts import contribute # noqa: E402 + +FIXTURES = ROOT / "evals" / "fixtures" / "contrib" +BUNDLE = ROOT / ".unslop" / "contrib" / "durable-bridge" +CAP_BUNDLE = ROOT / ".unslop" / "contrib" / "let-that-sink-in" + + +def reset_bundle() -> None: + if BUNDLE.exists(): + shutil.rmtree(BUNDLE) + if CAP_BUNDLE.exists(): + shutil.rmtree(CAP_BUNDLE) + + +def assert_equal(actual: object, expected: object, label: str) -> None: + if actual != expected: + print(f"{label} mismatch", file=sys.stderr) + print("actual:", actual, file=sys.stderr) + print("expected:", expected, file=sys.stderr) + raise SystemExit(1) + + +def scaffold() -> subprocess.CompletedProcess[str]: + reset_bundle() + return run( + [ + "python3", + "scripts/contribute.py", + "scaffold", + "--snippet", + "evals/fixtures/contrib/uncaught-snippet.txt", + "--tell", + "durable bridge", + "--category", + "significance_inflation", + "--pattern-name", + "durable-bridge", + "--date", + "2026-07-06", + ] + ) + + +def case_precheck_covered() -> None: + proc = run(["python3", "scripts/contribute.py", "precheck", "evals/fixtures/contrib/already-caught.txt"]) + if proc.returncode != 3: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + data = json.loads(proc.stdout) + if data["status"] != "already_covered" or not data["findings"]: + raise SystemExit(1) + print("CONTRIB-01 ok") + + +def case_precheck_clean() -> None: + proc = run(["python3", "scripts/contribute.py", "precheck", "evals/fixtures/contrib/uncaught-snippet.txt"]) + if proc.returncode != 0: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + data = json.loads(proc.stdout) + assert_equal(data["status"], "clean", "status") + assert_equal(data["words"], 29, "words") + findings = contribute.scanner_findings( + (FIXTURES / "uncaught-snippet.txt").read_text(encoding="utf-8") + ) + assert_equal(findings, [], "both scanner findings") + print("CONTRIB-02 ok") + + +def case_scaffold() -> None: + proc = scaffold() + if proc.returncode != 0: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + actual_row = json.loads((BUNDLE / "row_fn.json").read_text(encoding="utf-8")) + expected_row = json.loads((FIXTURES / "expected-row-fn.json").read_text(encoding="utf-8")) + assert_equal(actual_row, expected_row, "row_fn") + actual_manifest = json.loads((BUNDLE / "manifest.json").read_text(encoding="utf-8")) + expected_manifest = json.loads((FIXTURES / "expected-manifest.json").read_text(encoding="utf-8")) + assert_equal(actual_manifest, expected_manifest, "manifest") + snippet = (FIXTURES / "redacted-snippet.txt").read_text(encoding="utf-8") + assert_equal(actual_row["stdin"], snippet, "specimen") + print("CONTRIB-03 ok") + + +def case_redaction_tell() -> None: + reset_bundle() + proc = run( + [ + "python3", + "scripts/contribute.py", + "scaffold", + "--snippet", + "evals/fixtures/contrib/tell-redaction-snippet.txt", + "--tell", + "durable bridge", + "--category", + "significance_inflation", + "--pattern-name", + "durable-bridge", + "--redact", + "durable bridge=sturdy bridge", + "--date", + "2026-07-06", + ] + ) + if proc.returncode != 4: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + print("CONTRIB-04 ok") + + +def fill_report() -> None: + manifest = json.loads((BUNDLE / "manifest.json").read_text(encoding="utf-8")) + manifest["source_genre"] = "technical rollout note" + manifest["rationale"] = ( + "`Durable bridge` inflates a migration detail into a vague promise. " + "It sounds reassuring without naming the compatibility behavior, failure mode, or owner." + ) + manifest["pattern_added"] = "`durable bridge`" + manifest["severity"] = "soft" + manifest["gating_rationale"] = "flag metaphorical migration praise, protect literal construction" + (BUNDLE / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + report = (BUNDLE / "report.md").read_text(encoding="utf-8") + report = report.replace( + "TODO: explain why this phrase is a reusable AI-writing tell in 2-4 sentences.", + manifest["rationale"], + ) + report = report.replace("TODO: source genre", "technical rollout note") + report = report.replace("Pattern added: TODO: regex or phrase for `durable bridge`", "Pattern added: `durable bridge`") + report = report.replace("Severity: TODO: hard or soft", "Severity: soft") + report = report.replace( + "Gating rationale: TODO: explain literal-use boundary", + "Gating rationale: flag metaphorical migration praise, protect literal construction", + ) + report = report.replace("TODO: paste gate tails from verify.", (FIXTURES / "sample-gate-results.md").read_text(encoding="utf-8").rstrip()) + (BUNDLE / "report.md").write_text(report, encoding="utf-8") + + +def case_verify_red_and_todo() -> None: + scaffold() + proc = run(["python3", "scripts/contribute.py", "verify", "--bundle", str(BUNDLE), "--no-gates"]) + if proc.returncode == 0 or "TODO" not in proc.stdout + proc.stderr: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + fill_report() + proc = run(["python3", "scripts/contribute.py", "verify", "--bundle", str(BUNDLE), "--no-gates"]) + if proc.returncode != 0 or "red-first: ok" not in proc.stdout: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + print("CONTRIB-05 ok") + + +def case_report_golden() -> None: + scaffold() + fill_report() + proc = run(["python3", "scripts/contribute.py", "report", "--bundle", str(BUNDLE)]) + if proc.returncode != 0: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + expected = (ROOT / "evals" / "golden" / "contrib_report.md").read_text(encoding="utf-8").rstrip() + assert_equal(proc.stdout.rstrip(), expected, "report") + print("CONTRIB-06 ok") + + +def case_determinism() -> None: + scaffold() + first = (BUNDLE / "row_fn.json").read_text(encoding="utf-8") + (BUNDLE / "manifest.json").read_text(encoding="utf-8") + scaffold() + second = (BUNDLE / "row_fn.json").read_text(encoding="utf-8") + (BUNDLE / "manifest.json").read_text(encoding="utf-8") + assert_equal(second, first, "determinism") + print("CONTRIB-07 ok") + + +def case_missing_bundle() -> None: + proc = run(["python3", "scripts/contribute.py", "report", "--bundle", ".unslop/contrib/does-not-exist"]) + if proc.returncode != 2: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + print("CONTRIB-08 ok") + + +def case_capitalized_tell_assertion() -> None: + reset_bundle() + proc = run( + [ + "python3", + "scripts/contribute.py", + "scaffold", + "--snippet", + "evals/fixtures/contrib/capitalized-tell.txt", + "--tell", + "Let that sink in", + "--category", + "emphasis_crutch", + "--pattern-name", + "let-that-sink-in", + "--date", + "2026-07-06", + ] + ) + if proc.returncode != 0: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + row = json.loads((CAP_BUNDLE / "row_fn.json").read_text(encoding="utf-8")) + assert_equal(row["assertions"][1]["value"], "let that sink in", "lowercased assertion") + report = (CAP_BUNDLE / "report.md").read_text(encoding="utf-8") + for before, after in ( + ("TODO: source genre", "release note"), + ( + "TODO: explain why this phrase is a reusable AI-writing tell in 2-4 sentences.", + "`Let that sink in` is a stock emphasis command that talks down to the reader instead of adding evidence.", + ), + ("Pattern added: TODO: regex or phrase for `Let that sink in`", "Pattern added: `let that sink in`"), + ("Severity: TODO: hard or soft", "Severity: hard"), + ("Gating rationale: TODO: explain literal-use boundary", "Gating rationale: exact stock command"), + ("TODO: paste gate tails from verify.", "manual no-gates verification"), + ): + report = report.replace(before, after) + (CAP_BUNDLE / "report.md").write_text(report, encoding="utf-8") + proc = run(["python3", "scripts/contribute.py", "verify", "--bundle", str(CAP_BUNDLE), "--no-gates"]) + if proc.returncode != 0 or "red-first: already green" not in proc.stdout: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + print("CONTRIB-09 ok") + + +def case_structure_precheck() -> None: + proc = run(["python3", "scripts/contribute.py", "precheck", "evals/fixtures/contrib/structure-only.txt"]) + if proc.returncode != 3: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + data = json.loads(proc.stdout) + metrics = [finding["category"] for finding in data["findings"]] + if "connective_paragraph_openers" not in metrics: + print(proc.stdout + proc.stderr) + raise SystemExit(1) + print("CONTRIB-10 ok") + + +CASES = { + "CONTRIB-01": case_precheck_covered, + "CONTRIB-02": case_precheck_clean, + "CONTRIB-03": case_scaffold, + "CONTRIB-04": case_redaction_tell, + "CONTRIB-05": case_verify_red_and_todo, + "CONTRIB-06": case_report_golden, + "CONTRIB-07": case_determinism, + "CONTRIB-08": case_missing_bundle, + "CONTRIB-09": case_capitalized_tell_assertion, + "CONTRIB-10": case_structure_precheck, +} + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("case", choices=sorted(CASES)) + args = parser.parse_args(argv) + CASES[args.case]() + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/evals/check_gates_doc.py b/evals/check_gates_doc.py index 3e59f27..9e44b41 100644 --- a/evals/check_gates_doc.py +++ b/evals/check_gates_doc.py @@ -11,6 +11,24 @@ from run_adversarial import list_gates # noqa: E402 +ROOT = HERE.parent + + +def missing_command_files(gates: list[dict]) -> list[str]: + """Every whitespace-delimited *.py token in a gate command must exist under + ROOT. Additive: catches a gate command left pointing at a renamed/deleted + script. Gates whose command has no .py token at all (a shell script like + behavioral-tune, or a natural-language row like rubric-judge) have nothing + to check here and pass trivially.""" + problems = [] + for gate in gates: + for token in gate.get("command", "").split(): + if not token.endswith(".py"): + continue + if not (ROOT / token).exists(): + problems.append(f"{gate['id']}: missing {token}") + return problems + def main() -> int: doc = (HERE / "CHECKS.md").read_text() @@ -28,6 +46,14 @@ def main() -> int: print(f" live ids: {live_ids}") print("Regenerate the block: python3 evals/run_adversarial.py --list-gates") return 1 + + problems = missing_command_files(live) + if problems: + print("Gate commands reference *.py files that don't exist:") + for p in problems: + print(f" - {p}") + return 1 + print(f"gate matrix ok: {len(live)} gates documented") return 0 diff --git a/evals/check_mimic.py b/evals/check_mimic.py new file mode 100644 index 0000000..11ed663 --- /dev/null +++ b/evals/check_mimic.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""Deterministic checks for the teach/mimic feature (MIMIC-* and CARD-* rows). + +Each subcommand drives the real CLIs (voice_profile.py, voice_card.py, +run_mimic_refine.py, mimic_stats.py) against committed fixtures and returns 0 +on the expected behavior, 1 otherwise. The mimic rows run the refine loop in +dry-run (``--candidates-dir``) mode so the whole suite is LLM-free. +""" + +import argparse +import hashlib +import json +import re +import sys +import tempfile +from pathlib import Path + +from _check_support import ROOT, run # noqa: E402 + +FIX = ROOT / "evals" / "fixtures" / "mimic" +SAMPLES = FIX / "samples" +DIV_SAMPLES = FIX / "div_samples" +SMALL = FIX / "small_samples" +DRAFT = FIX / "draft.md" +CANDS = FIX / "candidates" +STATS = FIX / "stats" +NO_NUMBERS = FIX / "card_corpora" / "no_numbers" + + +def refine(scenario, out, seed, iterations, samples=SAMPLES, extra=None): + cmd = [ + "python3", "evals/run_mimic_refine.py", + "--samples", str(samples), "--draft", str(DRAFT), + "--out", str(out), "--seed", str(seed), + "--iterations", str(iterations), + "--candidates-dir", str(CANDS / scenario), + ] + (extra or []) + proc = run(cmd) + return proc + + +def load_report(out): + return json.loads((Path(out) / "report.json").read_text()) + + +# ----------------------------- MIMIC checks ----------------------------- + +def check_acceptance(): + with tempfile.TemporaryDirectory() as td: + proc = refine("accept", td, 1, 3) + if proc.returncode != 0: + print(proc.stderr) + return 1 + r = load_report(td) + accepted_after_first = any(i["accepted"] for i in r["iterations"][1:]) + ok = (r["best_candidate"] == "cand01.md" and accepted_after_first + and r["iterations"][2]["accepted"]) + print(json.dumps({"best_score": r["best_score"], + "accepted": [i["accepted"] for i in r["iterations"]], + "stop_reason": r["stop_reason"]}, sort_keys=True)) + return 0 if ok else 1 + + +def check_patience(): + with tempfile.TemporaryDirectory() as td: + proc = refine("patience", td, 1, 3, extra=["--patience", "2"]) + if proc.returncode != 0: + print(proc.stderr) + return 1 + r = load_report(td) + print(json.dumps({"stop_reason": r["stop_reason"], + "reward_hacking_warning": r["reward_hacking_warning"]}, + sort_keys=True)) + return 0 if r["stop_reason"] == "patience" and not r["reward_hacking_warning"] else 1 + + +def check_stuffed_attack(): + """A marker-stuffed candidate that clears every hard gate must still lose to + honest prose under the GI-bearing composite (F1 regression). cand01 is honest, + cand02 is the punctuation/repetition-stuffed attack that games a raw weighted + distance but not the full composite.""" + with tempfile.TemporaryDirectory() as td: + proc = refine("stuffed", td, 1, 1) + if proc.returncode != 0: + print(proc.stderr) + return 1 + r = load_report(td) + cands = {c["name"]: c for c in r["iterations"][0]["candidates"]} + honest = cands.get("cand01.md", {}) + stuffed = cands.get("cand02.md", {}) + both_passed = honest.get("passed") and stuffed.get("passed") + honest_dev = honest.get("dev_score") + stuffed_dev = stuffed.get("dev_score") + ok = (both_passed + and r["best_candidate"] == "cand01.md" + and honest_dev is not None and stuffed_dev is not None + and stuffed_dev > honest_dev) + print(json.dumps({ + "best_candidate": r["best_candidate"], + "honest_passed_gates": honest.get("passed"), + "stuffed_passed_gates": stuffed.get("passed"), + "honest_composite": honest_dev, + "stuffed_composite": stuffed_dev, + "margin": (round(stuffed_dev - honest_dev, 6) + if honest_dev is not None and stuffed_dev is not None else None), + }, sort_keys=True)) + return 0 if ok else 1 + + +def check_live_path(): + """The LIVE generation path (F3): a mock generator invoked via --generate-cmd + drives the same gate/score/accept pipeline as the dry-run, producing the same + report shape and honoring acceptance.""" + with tempfile.TemporaryDirectory() as td: + td = Path(td) + dry = td / "dry" + live = td / "live" + p_dry = refine("accept", dry, 1, 3) + cmd = [ + "python3", "evals/run_mimic_refine.py", + "--samples", str(SAMPLES), "--draft", str(DRAFT), + "--out", str(live), "--seed", "1", "--iterations", "3", "--beam", "2", + "--generate-cmd", "python3 evals/fixtures/mimic/mock_generator.py", + ] + p_live = run(cmd) + if p_dry.returncode != 0 or p_live.returncode != 0: + print(p_dry.stderr, p_live.stderr) + return 1 + rd = load_report(dry) + rl = load_report(live) + shape_keys = {"iterations", "best_candidate", "best_score", "stop_reason", + "reward_hacking_warning", "directives", "split"} + shape_ok = shape_keys <= set(rl) + beam_ok = len(rl["iterations"][0]["candidates"]) == 2 + accept_ok = [i["accepted"] for i in rl["iterations"]] == [i["accepted"] for i in rd["iterations"]] + score_ok = rl["best_score"] == rd["best_score"] + final_ok = (live / "final.md").exists() + print(json.dumps({ + "live_best_score": rl["best_score"], + "dry_best_score": rd["best_score"], + "accepted": [i["accepted"] for i in rl["iterations"]], + "beam_candidates": len(rl["iterations"][0]["candidates"]), + "final_written": final_ok, + }, sort_keys=True)) + return 0 if shape_ok and beam_ok and accept_ok and score_ok and final_ok else 1 + + +def check_divergence(): + with tempfile.TemporaryDirectory() as td: + proc = refine("divergence", td, 4, 3, samples=DIV_SAMPLES, + extra=["--patience", "3"]) + if proc.returncode != 0: + print(proc.stderr) + return 1 + r = load_report(td) + a = [i["best_a_score"] for i in r["iterations"]] + dev = [i["best_dev_score"] for i in r["iterations"]] + # A improving (down) while DEV worsening (up) across the run. + trend = a[0] > a[1] > a[2] and dev[0] < dev[1] < dev[2] + ok = (r["stop_reason"] == "divergence" and r["reward_hacking_warning"] and trend) + print(json.dumps({"stop_reason": r["stop_reason"], + "reward_hacking_warning": r["reward_hacking_warning"], + "a_scores": a, "dev_scores": dev}, sort_keys=True)) + return 0 if ok else 1 + + +def _rejection_reason(scenario, out, seed): + proc = refine(scenario, out, seed, 1) + if proc.returncode != 0: + print(proc.stderr) + return None + r = load_report(out) + rej = r["iterations"][0]["gate_rejections"] + print(json.dumps(rej, sort_keys=True)) + return rej + + +def check_copy_gate(): + with tempfile.TemporaryDirectory() as td: + rej = _rejection_reason("copygate", td, 1) + return 0 if rej and any(x["reason"] == "copy_gate" for x in rej) else 1 + + +def check_fact_gate(): + with tempfile.TemporaryDirectory() as td: + rej = _rejection_reason("factgate", td, 1) + return 0 if rej and any(x["reason"] == "preservation" for x in rej) else 1 + + +def check_determinism(): + with tempfile.TemporaryDirectory() as td: + # Same --out both times so card_path (which embeds the out dir) matches. + out = Path(td) / "run" + p1 = refine("accept", out, 1, 3) + first = (out / "report.json").read_text() + p2 = refine("accept", out, 1, 3) + second = (out / "report.json").read_text() + if p1.returncode or p2.returncode: + print(p1.stderr, p2.stderr) + return 1 + print(f"len1={len(first)} len2={len(second)} identical={first == second}") + return 0 if first == second else 1 + + +def check_stats(): + win = run(["python3", "evals/mimic_stats.py", str(STATS / "win.json"), "--seed", "7"]) + null = run(["python3", "evals/mimic_stats.py", str(STATS / "null.json"), "--seed", "7"]) + if win.returncode or null.returncode: + print(win.stderr, null.stderr) + return 1 + w = json.loads(win.stdout) + n = json.loads(null.stdout) + print(json.dumps({"win_improved": w["improved"], "null_improved": n["improved"]}, + sort_keys=True)) + return 0 if w["improved"] and not n["improved"] else 1 + + +def check_split_refusal(): + proc = refine("accept", tempfile.mkdtemp(), 1, 3, samples=SMALL) + print(f"exit={proc.returncode} stderr={proc.stderr.strip()}") + return 0 if proc.returncode == 2 else 1 + + +def check_directives(): + with tempfile.TemporaryDirectory() as td: + proc = refine("accept", td, 1, 3) + if proc.returncode != 0: + print(proc.stderr) + return 1 + r = load_report(td) + dirs = r["directives"] + keys_ok = all({"metric", "directive", "card_amendment"} <= set(d) for d in dirs) + iter_dirs = any(i["directives"] for i in r["iterations"]) + print(json.dumps({"n_directives": len(dirs), + "metrics": [d["metric"] for d in dirs]}, sort_keys=True)) + return 0 if dirs and keys_ok and iter_dirs else 1 + + +# ------------------------------ CARD checks ----------------------------- + +def _build_profile(samples, out): + return run(["python3", "scripts/voice_profile.py", str(samples), "-o", str(out)]) + + +def _build_card(profile, samples, out, name="amara", provenance=False): + cmd = ["python3", "scripts/voice_card.py", "--profile", str(profile), + "--samples", str(samples), "--out", str(out), "--name", name] + if provenance: + cmd.append("--provenance") + return run(cmd) + + +def _card_word_count(text): + return len(re.findall(r"[A-Za-z0-9']+", text)) + + +def check_card_determinism(): + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "p.json" + _build_profile(SAMPLES, prof) + _build_card(prof, SAMPLES, td / "o1") + _build_card(prof, SAMPLES, td / "o2") + a = (td / "o1" / "card.md").read_text() + b = (td / "o2" / "card.md").read_text() + sheets_a = {p.name: p.read_text() for p in (td / "o1" / "card").glob("*.md")} + sheets_b = {p.name: p.read_text() for p in (td / "o2" / "card").glob("*.md")} + print(f"card_identical={a == b} sheets_identical={sheets_a == sheets_b}") + return 0 if a == b and sheets_a == sheets_b else 1 + + +def check_card_budget(): + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "p.json" + _build_profile(SAMPLES, prof) + _build_card(prof, SAMPLES, td / "o") + card = (td / "o" / "card.md").read_text() + wc = _card_word_count(card) + required = ["Rhythm:", "Contractions:", "Never:", "Openers:", + "When writing", "| Situation | Sheet |"] + missing = [s for s in required if s not in card] + print(f"word_count={wc} missing_sections={missing}") + return 0 if wc <= 300 and not missing else 1 + + +def check_card_facts(): + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "p.json" + _build_profile(SAMPLES, prof) + _build_card(prof, SAMPLES, td / "o") + card = (td / "o" / "card.md").read_text() + profile = json.loads(prof.read_text()) + med = int(profile["sentence_lengths"]["median"]) + rate = f"{profile['contraction_rate']:.3f}" + m = re.search(r"median sentence (\d+) words", card) + card_med = int(m.group(1)) if m else None + rate_ok = f"rate {rate}" in card + print(f"profile_median={med} card_median={card_med} rate={rate} rate_in_card={rate_ok}") + return 0 if card_med == med and rate_ok else 1 + + +def check_card_layout(): + """Full teach layout: profile.json, card.md, provenance.json + hash verify.""" + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "profile.json" + _build_profile(SAMPLES, prof) + _build_card(prof, SAMPLES, td, provenance=True) + files = [prof, td / "card.md", td / "provenance.json"] + exist = all(f.exists() for f in files) + prov = json.loads((td / "provenance.json").read_text()) + hashes_ok = True + for entry in prov["samples"]: + actual = hashlib.sha256((SAMPLES / entry["file"]).read_bytes()).hexdigest() + if actual != entry["sha256"]: + hashes_ok = False + print(f"files_exist={exist} doc_count={prov['doc_count']} hashes_ok={hashes_ok}") + return 0 if exist and hashes_ok and prov["doc_count"] == 5 else 1 + + +_NEVER_MARKS = { + "semicolons": ";", "exclamation points": "!", "colons": ":", + "hyphenated dashes": "-", "parentheticals": "(", +} + + +def check_card_never_does(): + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "p.json" + _build_profile(SAMPLES, prof) + _build_card(prof, SAMPLES, td / "o") + card = (td / "o" / "card.md").read_text() + profile = json.loads(prof.read_text()) + m = re.search(r"^Never: (.+)$", card, re.M) + if not m: + print("no Never line") + return 1 + claims = [c.strip().rstrip(".") for c in m.group(1).split(";")] + bad = [] + for claim in claims: + mark = _NEVER_MARKS.get(claim) + if mark is None or profile["punctuation"].get(mark, 0.0) != 0.0: + bad.append(claim) + print(f"claims={claims} violated={bad}") + return 0 if claims and not bad else 1 + + +def check_coverage_gap(): + """A sample set with no numeric writing must flag numbers-data as a gap.""" + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "p.json" + _build_profile(NO_NUMBERS, prof) + proc = run(["python3", "scripts/voice_card.py", "--profile", str(prof), + "--samples", str(NO_NUMBERS), "--coverage"]) + matrix = json.loads(proc.stdout) + covered = matrix["numbers-data"]["covered"] + print(json.dumps({"numbers-data": matrix["numbers-data"]}, sort_keys=True)) + return 0 if covered is False else 1 + + +def check_no_fabrication(): + """An uncovered dimension gets no sheet and is named uncovered in card.md.""" + with tempfile.TemporaryDirectory() as td: + td = Path(td) + prof = td / "p.json" + _build_profile(NO_NUMBERS, prof) + _build_card(prof, NO_NUMBERS, td / "o", name="tester") + card = (td / "o" / "card.md").read_text() + sheet = td / "o" / "card" / "numbers-data.md" + m = re.search(r"^Uncovered .*: (.+)$", card, re.M) + listed = m and "numbers-data" in m.group(1) + print(f"sheet_exists={sheet.exists()} listed_uncovered={bool(listed)}") + return 0 if not sheet.exists() and listed else 1 + + +def check_card_profile_mismatch(): + """F2: a supplied profile that does not describe the --samples must be caught. + A committed 4-doc Amara profile against the 5-doc Amara samples dir is a + named mismatch and a non-zero exit; voice_card must not silently emit a card.""" + with tempfile.TemporaryDirectory() as td: + proc = run([ + "python3", "scripts/voice_card.py", + "--profile", str(ROOT / "evals" / "fixtures" / "voice" / "profiles" / "amara.json"), + "--samples", str(ROOT / "evals" / "fixtures" / "voice" / "authors" / "amara"), + "--out", td, "--name", "amara", + ]) + card_written = (Path(td) / "card.md").exists() + named = "does not match --samples" in proc.stderr and "'" in proc.stderr + print(f"exit={proc.returncode} named_mismatch={named} card_written={card_written} " + f"stderr={proc.stderr.strip()}") + return 0 if proc.returncode == 2 and named and not card_written else 1 + + +CHECKS = { + "acceptance": check_acceptance, + "patience": check_patience, + "divergence": check_divergence, + "stuffed-attack": check_stuffed_attack, + "live-path": check_live_path, + "copy-gate": check_copy_gate, + "fact-gate": check_fact_gate, + "determinism": check_determinism, + "stats": check_stats, + "split-refusal": check_split_refusal, + "directives": check_directives, + "card-determinism": check_card_determinism, + "card-budget": check_card_budget, + "card-facts": check_card_facts, + "card-layout": check_card_layout, + "card-never-does": check_card_never_does, + "coverage-gap": check_coverage_gap, + "no-fabrication": check_no_fabrication, + "card-profile-mismatch": check_card_profile_mismatch, +} + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__) + for name in CHECKS: + parser.add_argument(f"--{name}", action="store_true") + args = parser.parse_args(argv) + for name, fn in CHECKS.items(): + if getattr(args, name.replace("-", "_")): + return fn() + parser.error("choose a check") + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/evals/check_pairs.py b/evals/check_pairs.py new file mode 100755 index 0000000..74dc1eb --- /dev/null +++ b/evals/check_pairs.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import json, sys +from _check_support import ROOT + +SCRIPTS = ROOT / 'scripts' +sys.path.insert(0, str(SCRIPTS)) + +import banned_phrase_scan # noqa: E402 +import structure_scan # noqa: E402 + +PAIR_DIR=ROOT/'evals/fixtures/pairs' + +def words(p): + return len(p.read_text().split()) + +def scan_banned(path): + """In-process equivalent of banned_phrase_scan.py's CLI: (returncode, data).""" + text = path.read_text() + violations = banned_phrase_scan.scan_for_violations(text) + if not violations and not banned_phrase_scan.is_probably_english(text): + return 0, {'non_english': True, 'total_violations': 0, 'violations': []} + categories = {} + by_severity = {'hard': 0, 'soft': 0} + for v in violations: + categories[v['category']] = categories.get(v['category'], 0) + 1 + by_severity[v['severity']] = by_severity.get(v['severity'], 0) + 1 + data = { + 'total_violations': len(violations), + 'by_severity': by_severity, + 'by_category': categories, + 'violations': violations, + } + return (1 if violations else 0), data + +def scan_structure(path): + """In-process equivalent of structure_scan.py's CLI: (returncode, data).""" + text = path.read_text() + result = structure_scan.scan(text) + if not result.get('flags') and not structure_scan.is_probably_english(text): + return 0, {'non_english': True, 'violations': [], 'flags': []} + return (1 if result['flags'] else 0), result + +def main(): + manifest=json.loads((PAIR_DIR/'manifest.json').read_text()) + rows=[]; ok=True + for slug,info in manifest.items(): + ext='md' if slug.startswith('macro_') else 'txt' + wp=PAIR_DIR/f'{slug}_with.{ext}'; np=PAIR_DIR/f'{slug}_without.{ext}' + if not wp.exists() or not np.exists(): + print(f'MISSING {slug}'); ok=False; continue + wcw,wcn=words(wp),words(np) + diff=abs(wcw-wcn)/max(wcw,wcn,1) + if diff>0.25: + ok=False + b_rc,b_data=scan_banned(np) + s_rc,_=scan_structure(np) + without_clean=(b_rc==0 and b_data.get('total_violations')==0 and s_rc==0) + if not without_clean: ok=False + if info['kind']=='structure': + _,data=scan_structure(wp) + hit=bool(data.get('flagged',{}).get(info['target'])) + desc=','.join(data.get('flagged',{}).keys()) or '-' + else: + _,data=scan_banned(wp) + cats=[v.get('category') for v in data.get('violations',[])] + hit=info['target'] in cats + desc=','.join(cats) or '-' + if not hit: ok=False + rows.append((slug,info['target'],desc,'yes' if without_clean else 'no')) + print('| slug | family | with-violations | without-clean |') + print('|---|---|---|---|') + for r in rows: + print('| ' + ' | '.join(r) + ' |') + return 0 if ok else 1 +if __name__=='__main__': sys.exit(main()) diff --git a/evals/check_pattern_coverage.py b/evals/check_pattern_coverage.py new file mode 100644 index 0000000..c7d465c --- /dev/null +++ b/evals/check_pattern_coverage.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Enforce that every scanner pattern is exercised and every category FP-protected. + +Two structural checks, both derived live from the scanner and the eval suite: + + coverage - every STRUCTURAL_PATTERNS regex must match at least one eval corpus + text, and every BANNED_PHRASES key must appear (case-insensitive) in + at least one corpus text. The corpus is every script-case stdin plus + the contents of every fixture file referenced by a script row. + protections - every violation category the scanner can emit must be claimed by at + least one scanner_false_positive row carrying `"protects": ""`. + +A grandfather list is deliberately absent: an uncovered structural pattern or an +unprotected category is a hard failure, so a new scanner entry cannot land with zero +eval coverage while every other gate stays green. Run from the skill root: + + python3 evals/check_pattern_coverage.py # both checks + python3 evals/check_pattern_coverage.py --coverage # pattern coverage only + python3 evals/check_pattern_coverage.py --protections # category protection only +""" +import argparse +import re +import sys +import time +from pathlib import Path + +from _check_support import ROOT, load_evals # noqa: E402 + +sys.path.insert(0, str(ROOT)) + +from scripts.banned_phrase_scan import BANNED_PHRASES, STRUCTURAL_PATTERNS # noqa: E402 + + +def load_corpus(evals): + """Return [(source_id, lowercased_text)] for every script-row input.""" + corpus = [] + for e in evals: + if e.get("target") != "script": + continue + stdin = e.get("stdin") + if stdin: + corpus.append((f"{e['id']}:stdin", stdin.lower())) + for part in e.get("command", []): + if isinstance(part, str) and "fixtures" in part: + path = ROOT / part + if path.is_dir(): + for sub in sorted(path.rglob("*")): + if sub.is_file(): + try: + text = sub.read_text().lower() + except (UnicodeDecodeError, OSError): + continue + corpus.append((f"{e['id']}:{sub.relative_to(ROOT)}", text)) + elif path.exists(): + corpus.append((f"{e['id']}:{part}", path.read_text().lower())) + return corpus + + +def check_coverage(corpus): + """Return (ok, lines) for pattern/phrase coverage.""" + lines = [] + ok = True + + # Structural patterns: compile once, match against the lowercased corpus. + struct_uncovered = [] + struct_covered = 0 + for pat in STRUCTURAL_PATTERNS: + regex = re.compile(pat["pattern"]) + hits = [sid for sid, text in corpus if regex.search(text)] + if hits: + struct_covered += 1 + else: + struct_uncovered.append(pat) + if struct_uncovered: + ok = False + lines.append(f"Uncovered structural patterns ({len(struct_uncovered)}):") + for pat in struct_uncovered: + lines.append(f" - [{pat['category']}] {pat['pattern']}") + lines.append(" Add a row whose stdin triggers each pattern (no grandfathering).") + + # Banned phrases: literal key must appear somewhere in the corpus. + phrase_uncovered = [k for k in BANNED_PHRASES if not any(k.lower() in text for _, text in corpus)] + phrase_covered = len(BANNED_PHRASES) - len(phrase_uncovered) + if phrase_uncovered: + ok = False + lines.append(f"Uncovered banned phrases ({len(phrase_uncovered)}):") + for k in phrase_uncovered: + lines.append(f" - {k!r}") + lines.append(" Add a REC-style coverage pack that packs these into eval stdins.") + + lines.append( + f"coverage: {struct_covered}/{len(STRUCTURAL_PATTERNS)} structural patterns, " + f"{phrase_covered}/{len(BANNED_PHRASES)} banned phrases exercised." + ) + return ok, lines + + +def check_protections(evals): + """Return (ok, lines) for per-category FP protection.""" + lines = [] + ok = True + categories = set(v["category"] for v in BANNED_PHRASES.values()) + categories |= set(p["category"] for p in STRUCTURAL_PATTERNS) + + claimed = {} + for e in evals: + if e.get("category") != "scanner_false_positive": + continue + cat = e.get("protects") + if cat: + claimed.setdefault(cat, []).append(e["id"]) + + unknown = sorted(set(claimed) - categories) + if unknown: + ok = False + lines.append(f"FP rows claim categories the scanner never emits: {unknown}") + + unprotected = sorted(categories - set(claimed)) + if unprotected: + ok = False + lines.append(f"Unprotected categories ({len(unprotected)}):") + for cat in unprotected: + lines.append(f" - {cat}") + lines.append(' Add an FP row asserting total_violations == 0 with "protects": "".') + + lines.append( + f"protections: {len(categories - set(unprotected))}/{len(categories)} categories " + f"claimed by a protects FP row." + ) + return ok, lines + + +def main(argv): + parser = argparse.ArgumentParser(description="Scanner pattern-coverage gate.") + parser.add_argument("--coverage", action="store_true", help="run only the coverage check") + parser.add_argument("--protections", action="store_true", help="run only the protection check") + args = parser.parse_args(argv) + run_coverage = args.coverage + run_protections = args.protections + if not run_coverage and not run_protections: + run_coverage = run_protections = True + + start = time.perf_counter() + evals = load_evals() + corpus = load_corpus(evals) + + ok = True + out = [] + if run_coverage: + c_ok, c_lines = check_coverage(corpus) + ok = ok and c_ok + out += c_lines + if run_protections: + p_ok, p_lines = check_protections(evals) + ok = ok and p_ok + out += p_lines + + elapsed = time.perf_counter() - start + print("\n".join(out)) + print(f"pattern-coverage gate {'OK' if ok else 'FAILED'} ({elapsed:.2f}s over {len(corpus)} corpus texts)") + if elapsed > 10: + print("performance guard tripped: check exceeded 10s", file=sys.stderr) + return 1 + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/evals/check_seeded_docs.py b/evals/check_seeded_docs.py new file mode 100755 index 0000000..8b6907b --- /dev/null +++ b/evals/check_seeded_docs.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import json, sys +from _check_support import ROOT, run +DOC_DIR=ROOT/'evals/fixtures/docs' + +def scan_phrase(path): + p=run(['python3','scripts/banned_phrase_scan.py',str(path.relative_to(ROOT))]) + try: data=json.loads(p.stdout) + except Exception: data={'violations':[],'total_violations':999} + return p,data +def scan_struct(path): + p=run(['python3','scripts/structure_scan.py',str(path.relative_to(ROOT))]) + try: data=json.loads(p.stdout) + except Exception: data={'flags':[],'flagged':{}} + return p,data + +def main(): + ok=True + for manifest in sorted(DOC_DIR.glob('*_manifest.json')): + slug=manifest.name[:-14] + seeded=DOC_DIR/f'{slug}_seeded.md'; clean=DOC_DIR/f'{slug}_clean.md' + expected=json.loads(manifest.read_text())['expected'] + _,pdata=scan_phrase(seeded); _,sdata=scan_struct(seeded) + cats={v.get('category') for v in pdata.get('violations',[])} | set(sdata.get('flagged',{}).keys()) + missing=[e['category'] for e in expected if e['category'] not in cats] + if missing: + ok=False; print(f'{slug}: missing {missing}; saw {sorted(cats)}') + cp,cpdata=scan_phrase(clean); cs,_=scan_struct(clean) + if cp.returncode!=0 or cpdata.get('total_violations')!=0 or cs.returncode!=0: + ok=False; print(f'{slug}: clean twin failed phrase={cpdata.get("total_violations")} structure_exit={cs.returncode}') + if ok: print('seeded docs ok') + return 0 if ok else 1 +if __name__=='__main__': sys.exit(main()) diff --git a/evals/check_silhouette.py b/evals/check_silhouette.py new file mode 100644 index 0000000..9fb22f3 --- /dev/null +++ b/evals/check_silhouette.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Drift and separation gate for the silhouette scanner. + +Two modes: + + --reference Regenerate the human reference distribution from the named + source docs and assert it equals the committed + evals/fixtures/silhouette/human_reference.json. This is the + drift gate: editing the scanner's metrics or the human corpus + without regenerating the reference fails here. Pass --write to + rewrite the committed file (regeneration, not a check). + + --separation Score the in-repo ai/ and human/ corpora with the committed + reference and assert the research-validated separation: + >= 8/12 ai docs flagged and 0/8 human docs flagged. + +Exit 0 on success, 1 on drift / separation failure, 2 on a missing input. +stdlib only. +""" + +import argparse +import json +import statistics as st +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(ROOT)) + +from scripts.silhouette_scan import ( # noqa: E402 + METRIC_ORDER, + PENALTY_THRESHOLD, + compute_metrics, + paragraphs, + scan, +) + +REFERENCE_PATH = ROOT / "evals" / "fixtures" / "silhouette" / "human_reference.json" +CORPUS = ROOT / "evals" / "fixtures" / "silhouette" / "corpus" + +# Research-validated constants (scratchpad/research/silhouette.md). fence is the +# human upper fence used as the per-metric scale floor; weight is the composite +# weight. These are hand-validated, not corpus-derived, so they live here and are +# baked into the regenerated reference file. +RESEARCH_FENCE = { + "scaffold_opener_share": 0.20, + "role_entropy_bits": 0.80, + "heading_preview": 0.20, + "preview_fulfillment": 0.25, + "callback_content": 0.30, +} +RESEARCH_WEIGHT = { + "scaffold_opener_share": 2.0, + "role_entropy_bits": 1.0, + "heading_preview": 1.0, + "preview_fulfillment": 1.0, + "callback_content": 1.5, +} +IQR_FLOOR = 0.05 + +# Human reference sources, relative to the repo root. macro-probe human docs plus +# the human-ish (structure_scan-clean) structure fixtures. +SOURCES = [ + "evals/fixtures/silhouette/corpus/human/01_linkedin.txt", + "evals/fixtures/silhouette/corpus/human/02_blog_intro.txt", + "evals/fixtures/silhouette/corpus/human/03_howto.txt", + "evals/fixtures/silhouette/corpus/human/04_essay.txt", + "evals/fixtures/silhouette/corpus/human/05_readme.txt", + "evals/fixtures/silhouette/corpus/human/06_email.txt", + "evals/fixtures/silhouette/corpus/human/07_personal_story.txt", + "evals/fixtures/silhouette/corpus/human/08_technical_explainer.txt", + "evals/fixtures/structure/struct02_bursty_narrative.md", + "evals/fixtures/structure/struct04_concrete_end.md", + "evals/fixtures/structure/struct10_varied_openers.md", + "evals/fixtures/structure/struct12_moderate_signpost.md", + "evals/fixtures/structure/struct15_one_closer.md", + "evals/fixtures/structure/struct16_parallel_enumeration.md", + "evals/fixtures/structure/struct17_academic_roadmap.md", +] + + +def sample_iqr(values: list[float]) -> float: + if len(values) < 2: + return 0.0 + q = st.quantiles(values, n=4, method="inclusive") + return q[2] - q[0] + + +def build_reference() -> dict: + columns = {name: [] for name in METRIC_ORDER} + for rel in SOURCES: + path = ROOT / rel + if not path.exists(): + raise FileNotFoundError(rel) + text = path.read_text() + paras = paragraphs(text) + if len(paras) < 3: + continue + row = compute_metrics(text, paras) + for name in METRIC_ORDER: + v = row[name] + if isinstance(v, (int, float)): + columns[name].append(v) + + metrics = {} + for name in METRIC_ORDER: + vals = sorted(columns[name]) + median = round(st.median(vals), 4) if vals else 0.0 + iqr = round(max(sample_iqr(vals), IQR_FLOOR), 4) + metrics[name] = { + "median": median, + "iqr": iqr, + "fence": RESEARCH_FENCE[name], + "weight": RESEARCH_WEIGHT[name], + "n": len(vals), + } + + return { + "description": ( + "Human reference distribution for scripts/silhouette_scan.py. " + "Regenerate with: python3 evals/check_silhouette.py --reference --write. " + "median/iqr/n are derived from the sources below; fence and weight are " + "research-validated constants (scratchpad/research/silhouette.md). The " + "scorer scales each metric by max(iqr, fence): the sample IQR is " + "degenerate at zero for these one-sided tells, so the fence is the " + "effective scale." + ), + "penalty_threshold": PENALTY_THRESHOLD, + "iqr_floor": IQR_FLOOR, + "sources": SOURCES, + "metric_order": METRIC_ORDER, + "metrics": metrics, + } + + +def mode_reference(write: bool) -> int: + try: + generated = build_reference() + except FileNotFoundError as e: + print(f"Missing reference source: {e}", file=sys.stderr) + return 2 + + serialized = json.dumps(generated, indent=2) + "\n" + if write: + REFERENCE_PATH.write_text(serialized) + print(f"wrote {REFERENCE_PATH}") + return 0 + + if not REFERENCE_PATH.exists(): + print(f"Missing reference file: {REFERENCE_PATH}", file=sys.stderr) + return 2 + committed = json.loads(REFERENCE_PATH.read_text()) + if committed != generated: + print("human_reference.json is out of sync with the sources.") + print("Regenerate: python3 evals/check_silhouette.py --reference --write") + return 1 + print(f"reference ok: {len(generated['metrics'])} metrics over " + f"{len(SOURCES)} human sources") + return 0 + + +def mode_separation() -> int: + ai_dir = CORPUS / "ai" + human_dir = CORPUS / "human" + if not REFERENCE_PATH.exists(): + print(f"Missing reference file: {REFERENCE_PATH}", file=sys.stderr) + return 2 + if not ai_dir.is_dir() or not human_dir.is_dir(): + print(f"Missing corpus under {CORPUS}", file=sys.stderr) + return 2 + reference = json.loads(REFERENCE_PATH.read_text())["metrics"] + + def flagged(directory: Path) -> list[tuple[str, float, bool]]: + out = [] + for p in sorted(directory.glob("*.txt")): + result = scan(p.read_text(), reference) + penalty = result.get("penalty") or 0.0 + out.append((p.name, penalty, penalty >= PENALTY_THRESHOLD)) + return out + + ai = flagged(ai_dir) + human = flagged(human_dir) + ai_hits = sum(1 for _, _, f in ai if f) + human_hits = sum(1 for _, _, f in human if f) + + print(f"AI corpus: {ai_hits}/{len(ai)} flagged") + for name, pen, f in ai: + print(f" {'FLAG' if f else ' '} {name:34} penalty={pen:.2f}") + print(f"HUMAN corpus: {human_hits}/{len(human)} flagged") + for name, pen, f in human: + print(f" {'FLAG' if f else ' '} {name:34} penalty={pen:.2f}") + + ok = ai_hits >= 8 and len(ai) == 12 and human_hits == 0 and len(human) == 8 + if not ok: + print("\nseparation FAILED: require >= 8/12 ai flagged and 0/8 human flagged") + return 1 + print(f"\nseparation ok: {ai_hits}/12 ai flagged, {human_hits}/8 human flagged") + return 0 + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reference", action="store_true", + help="check the committed human reference against the sources") + parser.add_argument("--separation", action="store_true", + help="check ai/human corpus separation") + parser.add_argument("--write", action="store_true", + help="with --reference, regenerate the committed reference file") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if not args.reference and not args.separation: + args.reference = args.separation = True + rc = 0 + if args.reference: + rc = mode_reference(args.write) or rc + if args.separation and not args.write: + rc = mode_separation() or rc + return rc + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/evals/check_voice.py b/evals/check_voice.py new file mode 100644 index 0000000..b6cfa86 --- /dev/null +++ b/evals/check_voice.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Voice scorer regression checks for WP10a.""" + +import argparse +import json +import sys +import tempfile +from pathlib import Path + +from _check_support import ROOT # noqa: E402 + +SCRIPTS = ROOT / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import voice_profile # noqa: E402 +import voice_score # noqa: E402 + +VOICE = ROOT / "evals" / "fixtures" / "voice" +AUTHORS = ["amara", "boris", "celia"] + + +def run_score(profile, candidate, seed=17, samples=None): + """In-process equivalent of scripts/voice_score.py's CLI: builds and + returns the same result dict main() would print, without the subprocess + and JSON-serialization round trip.""" + profile_path = VOICE / "profiles" / f"{profile}.json" + impostors_path = VOICE / "impostors" + if not profile_path.exists() or not impostors_path.is_dir(): + print("missing profile or impostors", file=sys.stderr) + raise SystemExit(2) + try: + text = voice_score.read_candidate(str(candidate)) + except FileNotFoundError as e: + print(f"missing candidate: {e}", file=sys.stderr) + raise SystemExit(2) + profile_data = json.loads(profile_path.read_text()) + feats = voice_profile.feature_bundle(text) + low = feats["total_words"] < 150 + impostors = voice_score.impostor_features(str(impostors_path)) + dist = voice_score.distances(profile_data, feats) + imp_dist = [voice_score.distances(profile_data, f) for _, f in impostors] + zs = voice_score.zscores(dist, imp_dist) + zsum = sum( + voice_score.WEIGHTS[k] * max(-3.0, min(3.0, zs[k] if zs[k] is not None else 0.0)) + for k in voice_score.WEIGHTS + ) + gi = voice_score.gi_score(profile_data, feats, impostors, seed) if impostors else 0.0 + result = { + "candidate_words": feats["total_words"], + "low_confidence": low, + "distances": {k: (None if low and k in {"sentence_emd", "mtld"} else v) for k, v in dist.items()}, + "z_scores": {k: (None if low and k in {"sentence_emd", "mtld"} else v) for k, v in zs.items()}, + "gi": gi, + "composite": 0.5 * (1 - gi) + 0.5 * zsum, + } + if samples: + result["copy_gate"] = voice_score.copy_gate(text, samples) + return result + + +def matrix(): + rows = {} + for profile in AUTHORS: + rows[profile] = {} + for author in AUTHORS: + candidate = VOICE / "authors" / author / "doc5.md" + rows[profile][author] = run_score(profile, candidate)["composite"] + return rows + + +def print_matrix(rows): + print("3x3 separation matrix (composite; lower is more profile-like)") + print("profile\\candidate amara boris celia") + for profile in AUTHORS: + vals = " ".join(f"{rows[profile][author]:.6f}" for author in AUTHORS) + print(f"{profile} {vals}") + + +def check_separation(): + rows = matrix() + print_matrix(rows) + ok = True + for profile in AUTHORS: + own = rows[profile][profile] + ok = ok and all(own < rows[profile][other] for other in AUTHORS if other != profile) + for author in AUTHORS: + own = rows[author][author] + ok = ok and all(own < rows[profile][author] for profile in AUTHORS if profile != author) + return 0 if ok else 1 + + +def check_gi(): + ok = True + for profile in AUTHORS: + own = run_score(profile, VOICE / "authors" / profile / "doc5.md")["gi"] + print(f"{profile} own GI {own:.6f}") + for author in AUTHORS: + if author == profile: + continue + cross = run_score(profile, VOICE / "authors" / author / "doc5.md")["gi"] + print(f"{profile} vs {author} GI {cross:.6f}") + ok = ok and own > cross + return 0 if ok else 1 + + +def check_gaming(): + genuine = run_score("amara", VOICE / "authors" / "amara" / "doc5.md") + stuffed = run_score("amara", VOICE / "stuffed_boris_for_amara.md") + print(f"genuine composite={genuine['composite']:.6f} GI={genuine['gi']:.6f}") + print(f"stuffed composite={stuffed['composite']:.6f} GI={stuffed['gi']:.6f}") + return 0 if stuffed["composite"] > genuine["composite"] and stuffed["gi"] < genuine["gi"] else 1 + + +def check_copy(expect_violation): + if expect_violation: + candidate = VOICE / "copy_lift.md" + data = run_score("amara", candidate, samples=VOICE / "authors" / "amara") + else: + candidate = VOICE / "authors" / "amara" / "doc5.md" + with tempfile.TemporaryDirectory() as td: + sample = Path(td) / "amara" + sample.mkdir() + for idx in range(1, 5): + (sample / f"doc{idx}.md").write_text((VOICE / "authors" / "amara" / f"doc{idx}.md").read_text()) + data = run_score("amara", candidate, samples=sample) + print(json.dumps(data["copy_gate"], sort_keys=True)) + return 0 if data["copy_gate"]["violation"] is expect_violation else 1 + + +def check_determinism(): + cand = VOICE / "authors" / "celia" / "doc5.md" + one = run_score("celia", cand, seed=101)["composite"] + two = run_score("celia", cand, seed=101)["composite"] + print(f"{one:.12f}\n{two:.12f}") + return 0 if one == two else 1 + + +def check_short(): + with tempfile.NamedTemporaryFile("w", suffix=".txt") as f: + f.write("Tiny note. Too short to trust.") + f.flush() + data = run_score("amara", Path(f.name)) + print(json.dumps({"low_confidence": data["low_confidence"], "sentence_emd": data["distances"]["sentence_emd"]})) + return 0 if data["low_confidence"] and data["distances"]["sentence_emd"] is None else 1 + + +def check_profiles(): + ok = True + for author in AUTHORS: + with tempfile.TemporaryDirectory() as td: + sample = Path(td) / author + sample.mkdir() + for idx in range(1, 5): + (sample / f"doc{idx}.md").write_text((VOICE / "authors" / author / f"doc{idx}.md").read_text()) + profile = voice_profile.build_profile(sample) + got = json.dumps(profile, indent=2, sort_keys=True) + "\n" + expected = (VOICE / "profiles" / f"{author}.json").read_text() + if got != expected: + print(f"profile drift: {author}") + ok = False + print("profiles ok" if ok else "profiles drifted") + return 0 if ok else 1 + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--separation", action="store_true") + parser.add_argument("--gi", action="store_true") + parser.add_argument("--gaming", action="store_true") + parser.add_argument("--copy-violation", action="store_true") + parser.add_argument("--copy-clean", action="store_true") + parser.add_argument("--determinism", action="store_true") + parser.add_argument("--short", action="store_true") + parser.add_argument("--profiles", action="store_true") + args = parser.parse_args(argv) + if args.separation: + return check_separation() + if args.gi: + return check_gi() + if args.gaming: + return check_gaming() + if args.copy_violation: + return check_copy(True) + if args.copy_clean: + return check_copy(False) + if args.determinism: + return check_determinism() + if args.short: + return check_short() + if args.profiles: + return check_profiles() + parser.error("choose a check") + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/evals/fixtures/calibrate/base_connectives.txt b/evals/fixtures/calibrate/base_connectives.txt new file mode 100644 index 0000000..49d3340 --- /dev/null +++ b/evals/fixtures/calibrate/base_connectives.txt @@ -0,0 +1 @@ +However, the Q3 2025 results exceeded projections by 12%. Additionally, the team led by Priya Nair closed the deal early. diff --git a/evals/fixtures/calibrate/base_contractions.txt b/evals/fixtures/calibrate/base_contractions.txt new file mode 100644 index 0000000..17a227c --- /dev/null +++ b/evals/fixtures/calibrate/base_contractions.txt @@ -0,0 +1 @@ +Maria Chen can't attend the March 3, 2026 meeting. She won't reschedule it, and she isn't sure the team will find another slot. It's already booked through Q2 2026. diff --git a/evals/fixtures/calibrate/base_em_dash.txt b/evals/fixtures/calibrate/base_em_dash.txt new file mode 100644 index 0000000..6709afd --- /dev/null +++ b/evals/fixtures/calibrate/base_em_dash.txt @@ -0,0 +1 @@ +The proposal — backed by Section 4.2 and a $2.5 million budget — passed on the first vote. The board — surprised by the margin — adjourned early. diff --git a/evals/fixtures/calibrate/base_sentence_length.txt b/evals/fixtures/calibrate/base_sentence_length.txt new file mode 100644 index 0000000..5afc296 --- /dev/null +++ b/evals/fixtures/calibrate/base_sentence_length.txt @@ -0,0 +1 @@ +Ravi Patel finished the audit early, and the client approved the $180,000 invoice by Friday. The team celebrated with cake. diff --git a/evals/fixtures/calibrate/base_staccato.txt b/evals/fixtures/calibrate/base_staccato.txt new file mode 100644 index 0000000..bc1bb20 --- /dev/null +++ b/evals/fixtures/calibrate/base_staccato.txt @@ -0,0 +1 @@ +Because the March 2026 rollout slipped, and the vendor missed the deadline, Jordan Blake escalated the ticket immediately. diff --git a/evals/fixtures/calibrate/contractions_clean.txt b/evals/fixtures/calibrate/contractions_clean.txt new file mode 100644 index 0000000..84fb01f --- /dev/null +++ b/evals/fixtures/calibrate/contractions_clean.txt @@ -0,0 +1 @@ +Maria Chen can't attend the March 3, 2026 meeting, and she won't reschedule it for the team. diff --git a/evals/fixtures/calibrate/inexpressible_contractions.txt b/evals/fixtures/calibrate/inexpressible_contractions.txt new file mode 100644 index 0000000..6bcaa69 --- /dev/null +++ b/evals/fixtures/calibrate/inexpressible_contractions.txt @@ -0,0 +1 @@ +The committee reviewed the Section 9 proposal on May 12, 2026. Members voted unanimously to approve the $4 million budget. The clerk recorded the outcome in the ledger. diff --git a/evals/fixtures/calibrate/preferences_canned.jsonl b/evals/fixtures/calibrate/preferences_canned.jsonl new file mode 100644 index 0000000..5180cb2 --- /dev/null +++ b/evals/fixtures/calibrate/preferences_canned.jsonl @@ -0,0 +1,28 @@ +{"pair_id": "contractions-00", "dimension": "contractions", "choice": "b", "ts": "2026-07-01T10:00:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-01", "dimension": "contractions", "choice": "b", "ts": "2026-07-02T10:01:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-02", "dimension": "contractions", "choice": "b", "ts": "2026-07-03T10:02:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-03", "dimension": "contractions", "choice": "b", "ts": "2026-07-04T10:03:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-04", "dimension": "contractions", "choice": "b", "ts": "2026-07-05T10:04:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-05", "dimension": "contractions", "choice": "b", "ts": "2026-07-01T10:05:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-06", "dimension": "contractions", "choice": "b", "ts": "2026-07-02T10:00:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "contractions-07", "dimension": "contractions", "choice": "a", "ts": "2026-07-03T10:01:00Z", "a_label": "contracted", "b_label": "expanded"} +{"pair_id": "em_dash-00", "dimension": "em_dash", "choice": "b", "ts": "2026-07-01T10:00:00Z", "a_label": "dashed", "b_label": "plain"} +{"pair_id": "em_dash-01", "dimension": "em_dash", "choice": "b", "ts": "2026-07-02T10:01:00Z", "a_label": "dashed", "b_label": "plain"} +{"pair_id": "em_dash-02", "dimension": "em_dash", "choice": "b", "ts": "2026-07-03T10:02:00Z", "a_label": "dashed", "b_label": "plain"} +{"pair_id": "sentence_length-00", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-01T10:00:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-01", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-02T10:01:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-02", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-03T10:02:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-03", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-04T10:03:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-04", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-05T10:04:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-05", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-01T10:05:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-06", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-02T10:00:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-07", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-03T10:01:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "sentence_length-08", "dimension": "sentence_length", "choice": "b", "ts": "2026-07-04T10:02:00Z", "a_label": "long", "b_label": "short"} +{"pair_id": "staccato-00", "dimension": "staccato", "choice": "a", "ts": "2026-07-01T11:00:00Z", "a_label": "staccato", "b_label": "flowing"} +{"pair_id": "staccato-01", "dimension": "staccato", "choice": "a", "ts": "2026-07-02T11:01:00Z", "a_label": "staccato", "b_label": "flowing"} +{"pair_id": "staccato-02", "dimension": "staccato", "choice": "a", "ts": "2026-07-03T11:02:00Z", "a_label": "staccato", "b_label": "flowing"} +{"pair_id": "staccato-03", "dimension": "staccato", "choice": "b", "ts": "2026-07-04T11:03:00Z", "a_label": "staccato", "b_label": "flowing"} +{"pair_id": "staccato-04", "dimension": "staccato", "choice": "b", "ts": "2026-07-05T11:04:00Z", "a_label": "staccato", "b_label": "flowing"} +{"pair_id": "staccato-05", "dimension": "staccato", "choice": "b", "ts": "2026-07-06T11:05:00Z", "a_label": "staccato", "b_label": "flowing"} +{"pair_id": "connectives-00", "dimension": "connectives", "choice": "a", "ts": "2026-07-01T09:00:00Z", "a_label": "formal", "b_label": "plain"} +{"pair_id": "connectives-00", "dimension": "connectives", "choice": "b", "ts": "2026-07-06T09:00:00Z", "a_label": "formal", "b_label": "plain"} diff --git a/evals/fixtures/calibrate/profile_conflict.json b/evals/fixtures/calibrate/profile_conflict.json new file mode 100644 index 0000000..cfe327b --- /dev/null +++ b/evals/fixtures/calibrate/profile_conflict.json @@ -0,0 +1,6 @@ +{ + "avg_sentence_length": 22.4, + "contraction_rate": 0.03, + "em_dash_rate": 0.01, + "formal_connective_rate": 0.5 +} diff --git a/evals/fixtures/calibrate/proper_noun_contraction.txt b/evals/fixtures/calibrate/proper_noun_contraction.txt new file mode 100644 index 0000000..72bccef --- /dev/null +++ b/evals/fixtures/calibrate/proper_noun_contraction.txt @@ -0,0 +1 @@ +Maria Chen can't attend the Venue Can't Stop meeting on March 3, 2026. She won't reschedule it either. diff --git a/evals/fixtures/calibrate/staccato_flagged.txt b/evals/fixtures/calibrate/staccato_flagged.txt new file mode 100644 index 0000000..79fb170 --- /dev/null +++ b/evals/fixtures/calibrate/staccato_flagged.txt @@ -0,0 +1 @@ +The build failed, and the vendor missed the deadline, so Priya escalated fast. diff --git a/evals/fixtures/contrib/already-caught.txt b/evals/fixtures/contrib/already-caught.txt new file mode 100644 index 0000000..0ae76e0 --- /dev/null +++ b/evals/fixtures/contrib/already-caught.txt @@ -0,0 +1 @@ +Here's the thing: the launch note spends three paragraphs saying the plan matters before it says what changed. diff --git a/evals/fixtures/contrib/capitalized-tell.txt b/evals/fixtures/contrib/capitalized-tell.txt new file mode 100644 index 0000000..6b06d15 --- /dev/null +++ b/evals/fixtures/contrib/capitalized-tell.txt @@ -0,0 +1 @@ +The launch memo ends with a warning instead of evidence. Let that sink in before you ask what changed. diff --git a/evals/fixtures/contrib/expected-manifest.json b/evals/fixtures/contrib/expected-manifest.json new file mode 100644 index 0000000..8b795e0 --- /dev/null +++ b/evals/fixtures/contrib/expected-manifest.json @@ -0,0 +1,10 @@ +{ + "category": "significance_inflation", + "date": "2026-07-06", + "pattern_name": "durable-bridge", + "post_redaction_sha256": "b73c4503165654cca1481aa578297213de3a2b653a318899e78051a004562ce1", + "pre_redaction_sha256": "b73c4503165654cca1481aa578297213de3a2b653a318899e78051a004562ce1", + "redactions": [], + "tell": "durable bridge", + "word_count": 29 +} diff --git a/evals/fixtures/contrib/expected-row-fn.json b/evals/fixtures/contrib/expected-row-fn.json new file mode 100644 index 0000000..2e5440f --- /dev/null +++ b/evals/fixtures/contrib/expected-row-fn.json @@ -0,0 +1,24 @@ +{ + "id": "CONTRIB-FN-durable-bridge", + "category": "scanner_false_negative", + "title": "New AI-ism fixture: durable bridge", + "target": "script", + "command": [ + "python3", + "scripts/banned_phrase_scan.py" + ], + "stdin": "The rollout guide keeps saying the migration path gives teams a durable bridge between the old queue and the new worker pool, but it never explains the retry contract.\n", + "failure_mode": "The scanner misses the contributed tell from the exact specimen.", + "correct_behavior": "The scanner reports at least one violation containing the contributed tell.", + "assertions": [ + { + "type": "json", + "path": "total_violations", + "gte": 1 + }, + { + "type": "violation_phrase_contains", + "value": "durable bridge" + } + ] +} diff --git a/evals/fixtures/contrib/redacted-snippet.txt b/evals/fixtures/contrib/redacted-snippet.txt new file mode 100644 index 0000000..8d33102 --- /dev/null +++ b/evals/fixtures/contrib/redacted-snippet.txt @@ -0,0 +1 @@ +The rollout guide keeps saying the migration path gives teams a durable bridge between the old queue and the new worker pool, but it never explains the retry contract. diff --git a/evals/fixtures/contrib/sample-gate-results.md b/evals/fixtures/contrib/sample-gate-results.md new file mode 100644 index 0000000..e704ffb --- /dev/null +++ b/evals/fixtures/contrib/sample-gate-results.md @@ -0,0 +1,8 @@ +### contribute-slice + +```text +PASS CONTRIB-01 precheck names covering pattern +PASS CONTRIB-02 precheck accepts uncaught tell +PASS CONTRIB-03 scaffold preserves specimen bytes +``` + diff --git a/evals/fixtures/contrib/structure-only.txt b/evals/fixtures/contrib/structure-only.txt new file mode 100644 index 0000000..8c688ae --- /dev/null +++ b/evals/fixtures/contrib/structure-only.txt @@ -0,0 +1,5 @@ +However, the rollout note starts with process instead of the concrete change. + +Moreover, the second paragraph repeats the same transition habit. + +Furthermore, the third paragraph keeps the reader in setup language. diff --git a/evals/fixtures/contrib/tell-redaction-snippet.txt b/evals/fixtures/contrib/tell-redaction-snippet.txt new file mode 100644 index 0000000..8d33102 --- /dev/null +++ b/evals/fixtures/contrib/tell-redaction-snippet.txt @@ -0,0 +1 @@ +The rollout guide keeps saying the migration path gives teams a durable bridge between the old queue and the new worker pool, but it never explains the retry contract. diff --git a/evals/fixtures/contrib/uncaught-snippet.txt b/evals/fixtures/contrib/uncaught-snippet.txt new file mode 100644 index 0000000..8d33102 --- /dev/null +++ b/evals/fixtures/contrib/uncaught-snippet.txt @@ -0,0 +1 @@ +The rollout guide keeps saying the migration path gives teams a durable bridge between the old queue and the new worker pool, but it never explains the retry contract. diff --git a/evals/fixtures/docs/engineering_report_clean.md b/evals/fixtures/docs/engineering_report_clean.md new file mode 100644 index 0000000..21c7447 --- /dev/null +++ b/evals/fixtures/docs/engineering_report_clean.md @@ -0,0 +1,7 @@ +# Import Reliability Review + +The nightly import failed six times in May: four failures came from duplicate vendor rows, one came from a malformed date, and one came from a timeout while the warehouse file was still uploading. The storage team expects the timeout rate to fall after the storage move, and the report should cite that forecast before it goes to operations. + +The current parser accepts the first vendor row and logs the rest as warnings, which made the dashboard look current even when the warehouse team still needed the rejected rows. A blocking preflight check would make the failure visible before any rows are written. + +The proposed fix blocks duplicate vendor IDs before the import writes to the reporting tables, then shows every manager and on-call engineer the same failed file, owner, and next retry time. With one visible stop, the team can retire three quiet warnings that currently send people to separate logs. diff --git a/evals/fixtures/docs/engineering_report_manifest.json b/evals/fixtures/docs/engineering_report_manifest.json new file mode 100644 index 0000000..ffd14b3 --- /dev/null +++ b/evals/fixtures/docs/engineering_report_manifest.json @@ -0,0 +1,19 @@ +{ + "expected": [ + { + "category": "vague_attribution" + }, + { + "category": "superficial_ing" + }, + { + "category": "anti_slop_register" + }, + { + "category": "reader_addressing" + }, + { + "category": "conclusion_scaffold" + } + ] +} diff --git a/evals/fixtures/docs/engineering_report_seeded.md b/evals/fixtures/docs/engineering_report_seeded.md new file mode 100644 index 0000000..849353d --- /dev/null +++ b/evals/fixtures/docs/engineering_report_seeded.md @@ -0,0 +1,7 @@ +# Import Reliability Review + +The nightly import failed six times in May. Four failures came from duplicate vendor rows, one came from a malformed date, and one came from a timeout while the warehouse file was still uploading. Analysts predict the timeout rate will fall after the storage move, but the report does not name a source for that forecast. + +The current parser accepts the first vendor row and logs the rest as warnings, highlighting the cleanup issue without stopping the job. That behavior made the dashboard look current even when the warehouse team still needed the rejected rows. Not more alerts. Better stops. + +The proposed fix is a preflight check that blocks duplicate vendor IDs before the import writes any rows. Whether you're a manager or an on-call engineer, the status page should show the same failed file, owner, and next retry time. In conclusion, the import needs one visible stop rather than three quiet warnings. diff --git a/evals/fixtures/docs/personal_blog_clean.md b/evals/fixtures/docs/personal_blog_clean.md new file mode 100644 index 0000000..9f4acf2 --- /dev/null +++ b/evals/fixtures/docs/personal_blog_clean.md @@ -0,0 +1,5 @@ +I spent Saturday clearing the back room at my father's print shop, where the shelves still held paper samples from jobs he finished before I learned to drive and every drawer had a ruler, a blade, or a note in his blocky handwriting. The room recorded decades of small decisions that kept the shop open. + +Invoices got shorter after the schools moved to online ordering, but repair receipts grew after the old press started jamming, and I kept thinking about how much of the business could be read through those ordinary folders. I could keep every folder, except the lease ends in September and the new tenant needs the walls clear. + +I kept the first union contract, three sample books, and the repair tag from the press he refused to replace, then boxed the rest for shredding after my sister checks the dates. We left the workbench for last because neither of us was ready to empty the drawer where he kept spare keys. diff --git a/evals/fixtures/docs/personal_blog_manifest.json b/evals/fixtures/docs/personal_blog_manifest.json new file mode 100644 index 0000000..e7d131f --- /dev/null +++ b/evals/fixtures/docs/personal_blog_manifest.json @@ -0,0 +1,19 @@ +{ + "expected": [ + { + "category": "negative_parallelism" + }, + { + "category": "false_agency" + }, + { + "category": "hedge_stack" + }, + { + "category": "emphasis_crutch" + }, + { + "category": "rhetorical_question" + } + ] +} diff --git a/evals/fixtures/docs/personal_blog_seeded.md b/evals/fixtures/docs/personal_blog_seeded.md new file mode 100644 index 0000000..3691a47 --- /dev/null +++ b/evals/fixtures/docs/personal_blog_seeded.md @@ -0,0 +1,5 @@ +I spent Saturday clearing the back room at my father's print shop. The shelves still held paper samples from jobs he finished before I learned to drive, and every drawer had a ruler, a blade, or a note in his blocky handwriting. The room was not just storage, it was a record of how many small decisions kept the shop open. + +The data tells a story in its own plain way: invoices got shorter after the schools moved to online ordering, but repair receipts grew after the old press started jamming. I could potentially keep every folder, but the lease ends in September and the new tenant needs the walls clear. Make no mistake, the hard part is deciding which ordinary records still mean something to the family. + +The result? I kept the first union contract, three sample books, and the repair tag from the press he refused to replace. The rest went into labeled boxes for shredding. diff --git a/evals/fixtures/docs/product_update_clean.md b/evals/fixtures/docs/product_update_clean.md new file mode 100644 index 0000000..b7bd33c --- /dev/null +++ b/evals/fixtures/docs/product_update_clean.md @@ -0,0 +1,7 @@ +Subject: July workspace update + +The July workspace release goes live for staff accounts on Thursday, adding saved filters to the request queue, a new export for overdue approvals, and a faster way to assign one request to another team while leaving customer-facing forms unchanged for this cycle. + +Coordinators can save common views for finance, facilities, and legal reviews, which keeps ownership visible when a request crosses teams and addresses the main complaint from the March audit. During the pilot, the facilities view cut the morning sorting pass from twenty minutes to eight, a result that surprised even the coordinators who had asked for the filter. + +The approval report has been rebuilt by hand every Friday since March, and the new export uses the same status labels as the queue so managers can check aging requests without asking coordinators to maintain a second spreadsheet. Friday meetings should be shorter because open questions will arrive with an owner, a status, and the exact date when the request last moved. diff --git a/evals/fixtures/docs/product_update_manifest.json b/evals/fixtures/docs/product_update_manifest.json new file mode 100644 index 0000000..b34730b --- /dev/null +++ b/evals/fixtures/docs/product_update_manifest.json @@ -0,0 +1,19 @@ +{ + "expected": [ + { + "category": "throat_clearing" + }, + { + "category": "jargon" + }, + { + "category": "negative_parallelism" + }, + { + "category": "rhetorical_question" + }, + { + "category": "conclusion_scaffold" + } + ] +} diff --git a/evals/fixtures/docs/product_update_seeded.md b/evals/fixtures/docs/product_update_seeded.md new file mode 100644 index 0000000..fd44c0f --- /dev/null +++ b/evals/fixtures/docs/product_update_seeded.md @@ -0,0 +1,7 @@ +Subject: July workspace update + +The July workspace release goes live for staff accounts on Thursday. It adds saved filters to the request queue, a new export for overdue approvals, and a faster way to assign one request to another team. Here's the thing: the release does not change customer-facing forms, so support scripts can stay as they are for this cycle. + +The queue screen now lets coordinators leverage the platform to save common views for finance, facilities, and legal reviews. It is not just a convenience update, it is a way to keep ownership visible when a request crosses teams. During the pilot, the facilities view cut the morning sorting pass from twenty minutes to eight. + +Why does this matter? The approval report has been rebuilt by hand every Friday since March. The new export uses the same status labels as the queue, which means managers can check aging requests without asking coordinators to maintain a second spreadsheet. Ultimately, the change should make the weekly meeting shorter and more specific. diff --git a/evals/fixtures/harvest/classifier_candidates.json b/evals/fixtures/harvest/classifier_candidates.json new file mode 100644 index 0000000..15503c8 --- /dev/null +++ b/evals/fixtures/harvest/classifier_candidates.json @@ -0,0 +1,17 @@ +[ + { + "text": "Why did the invoice total change after export? I expected the discount to stay attached, but the CSV shows the pre-discount number.", + "source": {"path": "fixture", "offset": 1}, + "words": 21 + }, + { + "text": "The dashboard says 42 accounts failed yesterday, and 17 of those failures came from the same region. That pattern matters more than the raw total.", + "source": {"path": "fixture", "offset": 2}, + "words": 25 + }, + { + "text": "I disagree with the rollout plan. Last quarter we tried the same shortcut, and support spent Friday rebuilding the queue by hand.", + "source": {"path": "fixture", "offset": 3}, + "words": 22 + } +] diff --git a/evals/fixtures/harvest/fixture_codex_session.jsonl b/evals/fixtures/harvest/fixture_codex_session.jsonl new file mode 100644 index 0000000..037a11c --- /dev/null +++ b/evals/fixtures/harvest/fixture_codex_session.jsonl @@ -0,0 +1,14 @@ +{"timestamp":"2026-06-20T18:00:00.000Z","type":"session_meta","payload":{"id":"019efixture-0000-7000-8000-000000000000","timestamp":"2026-06-20T18:00:00.000Z","cwd":"/Users/example/dev/example-repo","originator":"codex_cli_rs","cli_version":"0.1.0-fixture","source":"cli","thread_source":"cli","model_provider":"openai","base_instructions":"CODEX_BASE_INSTRUCTIONS_MARKER_SLATE_ANCHOR You are Codex, a coding agent. This system prompt must never be treated as user-authored text.","memory_mode":"auto","git":{"branch":"main"}}} +{"timestamp":"2026-06-20T18:00:01.000Z","type":"response_item","payload":{"type":"message","role":"developer","content":[{"type":"input_text","text":"\nCODEX_DEVELOPER_ROLE_MARKER_IVORY_HATCH sandbox_mode is read-only.\n"}]}} +{"timestamp":"2026-06-20T18:00:02.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"# AGENTS.md instructions for /Users/example/dev/example-repo\n\n\n# AGENTS.md\n\nCODEX_INJECTION_TRAP_MARKER_PEWTER_LOOM this repository guidance is injected context, never the user's own writing, and must never enter voice samples.\n"}]}} +{"timestamp":"2026-06-20T18:00:03.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"\n /Users/example/dev/example-repo\n zsh\n CODEX_ENV_CONTEXT_MARKER_QUARTZ_FIELD\n 2026-06-20\n"}]}} +{"timestamp":"2026-06-20T18:00:04.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"CODEX_USER_PROSE_MARKER_HOLLOW_CEDAR I kept a running note while the migration script stalled every third run. The retries never changed the outcome, they just delayed the failure by a few minutes each time. That delay is the part worth remembering, because it made the team trust the script long after it stopped deserving that trust."}]}} +{"timestamp":"2026-06-20T18:00:05.000Z","type":"event_msg","payload":{"type":"user_message","message":"grep -rn TODO src/ --include=*.py","images":[],"local_images":[],"text_elements":[]}} +{"timestamp":"2026-06-20T18:00:06.000Z","type":"event_msg","payload":{"type":"agent_message","message":"CODEX_ASSISTANT_CONTAMINATION_MARKER_ORCHID_TUNNEL Here's the thing: this comprehensive, multifaceted fix underscores the importance of a seamless workflow. In conclusion, it is a game-changer for the migration pipeline going forward."}} +{"timestamp":"2026-06-20T18:00:07.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"CODEX_ASSISTANT_RESPONSE_ITEM_MARKER_BRASS_KETTLE In conclusion, this delivers a seamless, robust workflow with actionable insights for teams moving forward. This assistant paragraph keeps going so the length filter alone cannot hide an authorship bug if the role check ever breaks, mirroring the same guarantee the claude-jsonl fixture makes for its own contamination marker."}]}} +{"timestamp":"2026-06-20T18:00:08.000Z","type":"response_item","payload":{"type":"reasoning","content":[{"type":"reasoning_text","text":"CODEX_REASONING_MARKER_SHOULD_NEVER_APPEAR internal deliberation."}]}} +{"timestamp":"2026-06-20T18:00:09.000Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","arguments":"{\"cmd\":\"CODEX_TOOL_CALL_MARKER_SHOULD_NEVER_APPEAR grep -rn TODO src/\"}","call_id":"call_1"}} +{"timestamp":"2026-06-20T18:00:10.000Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call_1","output":"CODEX_TOOL_OUTPUT_MARKER_SHOULD_NEVER_APPEAR src/example.py:12: TODO fix this"}} +{"timestamp":"2026-06-20T18:00:11.000Z","type":"turn_context","payload":{"turn_id":"fixture-turn-1","cwd":"/Users/example/dev/example-repo","user_instructions":"CODEX_TURN_CONTEXT_MARKER_SHOULD_NEVER_APPEAR"}} +{"timestamp":"2026-06-20T18:00:12.000Z","type":"compacted","payload":{"message":"CODEX_COMPACTED_MARKER_SHOULD_NEVER_APPEAR summary of earlier turns"}} +{"timestamp":"2026-06-20T18:00:13.000Z","type":"widget_event","payload":{"message":"CODEX_UNKNOWN_TYPE_MARKER_SHOULD_NEVER_APPEAR unrecognized top-level type"}} diff --git a/evals/fixtures/harvest/fixture_session.jsonl b/evals/fixtures/harvest/fixture_session.jsonl new file mode 100644 index 0000000..d8f2e73 --- /dev/null +++ b/evals/fixtures/harvest/fixture_session.jsonl @@ -0,0 +1,8 @@ +{"type":"user","timestamp":"2026-06-01T10:00:00.000Z","message":{"role":"user","content":"The marker phrase blue kiln notebook belongs to the user. I kept notes during the pilot because the errors were not dramatic, they were boring. The clerk retyped the same code three times, then stopped trusting the shortcut. That is the part I want preserved: the tool failed slowly enough that everyone worked around it instead of reporting it."}} +{"type":"assistant","timestamp":"2026-06-01T10:00:02.000Z","message":{"role":"assistant","content":"ASSISTANT_CONTAMINATION_NEON_RIVER. Here's the thing: this comprehensive, multifaceted response underscores the importance of a seamless workflow. In conclusion, it is a game-changer. This assistant paragraph keeps going so the length filter cannot hide an authorship bug. It adds another complete sentence with robust, actionable insights and a future outlook for teams moving forward."}} +{"type":"user","timestamp":"2026-06-01T10:01:00.000Z","message":{"role":"user","content":"/review /tmp/draft.md --quick"}} +{"type":"user","timestamp":"2026-06-01T10:02:00.000Z","message":{"role":"user","content":"You said: ASSISTANT_QUOTED_BRASS_LANTERN. Here's the thing: this comprehensive, multifaceted response underscores the importance of a seamless workflow. In conclusion, it is a game-changer."}} +{"type":"user","timestamp":"2026-06-01T10:03:00.000Z","message":{"role":"user","content":"Duplicate marker copper ledger stays once. I noticed the refund queue always looked calm at nine in the morning. By noon it had the same twenty names, just shuffled into a different order. The report called that stability, but the people answering phones called it losing the thread."}} +{"type":"user","timestamp":"2026-06-01T10:04:00.000Z","message":{"role":"user","content":"Duplicate marker copper ledger should stay once. I noticed the refund queue always looked calm around nine in the morning. By noon it had the same twenty names, just shuffled into a different order. The report called that stability; the people answering phones called it losing the thread."}} +{"type":"user","timestamp":"2026-06-01T10:05:00.000Z","message":{"role":"user","content":"Um, the dictated marker silver porch should stay, uh, because this is how I actually explained the bug after the meeting. Um, I said the form felt polite right up until it erased the note. Uh, nobody cared about the color; they cared that the last sentence vanished."}} +{"type":"user","timestamp":"2026-06-01T10:06:00.000Z","message":{"role":"user","content":"The quoted dialogue marker green hinge should not be marked as dictated just because a customer said um once. I wrote the scene this way: \"um, I thought the receipt would stay visible after refresh.\" The rest of the note is clean narrative about the support desk, the missing audit trail, and the manager's ordinary follow-up."}} diff --git a/evals/fixtures/harvest/merge_candidates.json b/evals/fixtures/harvest/merge_candidates.json new file mode 100644 index 0000000..e0bab0b --- /dev/null +++ b/evals/fixtures/harvest/merge_candidates.json @@ -0,0 +1,20 @@ +{ + "candidates": [ + { + "text": "Older question marker amber slate asks why the export changed after billing approved it. The note is useful, but it has no classifier cells until merge.", + "source": {"path": "older.md", "offset": 1, "date": "2026-06-01T09:00:00.000Z"}, + "words": 24 + }, + { + "text": "Newer numbers marker violet gauge says 42 invoices failed after yesterday's import and gives enough detail to teach a data-heavy passage.", + "source": {"path": "newer.md", "offset": 1, "date": "2026-06-03T09:00:00.000Z"}, + "words": 22 + }, + { + "text": "Weak closing marker gray latch thanks the team and signs off with a short closing note, but it should rank behind better merged evidence.", + "source": {"path": "weak.md", "offset": 1, "date": "2026-06-04T09:00:00.000Z"}, + "words": 23, + "suspect_ai": true + } + ] +} diff --git a/evals/fixtures/harvest/merge_results.jsonl b/evals/fixtures/harvest/merge_results.jsonl new file mode 100644 index 0000000..46c2958 --- /dev/null +++ b/evals/fixtures/harvest/merge_results.jsonl @@ -0,0 +1,3 @@ +{"candidate_index":0,"cells":["question_addressed"],"quality":4,"why":"asks why the export changed"} +{"candidate_index":1,"cells":["numbers_data"],"quality":4,"why":"contains a concrete invoice count"} +{"candidate_index":2,"cells":["openings_closings"],"quality":2,"why":"closing/signoff sample only"} diff --git a/evals/fixtures/harvest/single_um_short.jsonl b/evals/fixtures/harvest/single_um_short.jsonl new file mode 100644 index 0000000..bb4e7af --- /dev/null +++ b/evals/fixtures/harvest/single_um_short.jsonl @@ -0,0 +1 @@ +{"type":"user","timestamp":"2026-06-01T11:00:00.000Z","message":{"role":"user","content":"Short quoted marker red hinge should not look dictated. A customer said, \"um, refresh hid the receipt.\" I only wrote down that line because the audit trail vanished."}} diff --git a/evals/fixtures/harvest/text-folder/ai_tell.txt b/evals/fixtures/harvest/text-folder/ai_tell.txt new file mode 100644 index 0000000..4c5b7db --- /dev/null +++ b/evals/fixtures/harvest/text-folder/ai_tell.txt @@ -0,0 +1 @@ +TEXT_FOLDER_AI_TELL_MARKER. Here's the thing: this comprehensive guide will delve into the ever-evolving landscape of workflow optimization. It underscores the importance of seamless collaboration, robust processes, and actionable insights. In conclusion, this game-changer unlocks a treasure trove of opportunities for teams moving forward. diff --git a/evals/fixtures/harvest/unknown-jsonl/bad.jsonl b/evals/fixtures/harvest/unknown-jsonl/bad.jsonl new file mode 100644 index 0000000..10528f4 --- /dev/null +++ b/evals/fixtures/harvest/unknown-jsonl/bad.jsonl @@ -0,0 +1 @@ +{"kind":"mystery","payload":{"text":"No role field exists here, so the harvester must not guess authorship."}} diff --git a/evals/fixtures/mimic/candidates/accept/iter0/cand01.md b/evals/fixtures/mimic/candidates/accept/iter0/cand01.md new file mode 100644 index 0000000..e4ada29 --- /dev/null +++ b/evals/fixtures/mimic/candidates/accept/iter0/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa presented me with a battered bookshelf that had languished in her hallway for the better part of a decade, and because it wobbled alarmingly whenever a single volume was withdrawn from its shelves, I resolved to repaint and repair the piece across one unhurried weekend. The frame, though plain pine of an entirely unremarkable manufacture, nonetheless possessed a fundamental sturdiness that costlier furniture frequently and conspicuously fails to supply to its owners. + +I commenced the undertaking by sanding the aged finish, which had darkened unevenly and flaked at nearly every corner, and the labour, tedious though it undeniably was, proved unexpectedly satisfying in the particular manner peculiar to tasks that yield a steady and visible progress. Thereafter I applied two carefully considered coats of pale grey paint, permitting several hours to elapse between them, and I corrected the persistent wobble with a modest assembly of screws. The materials, all told, consumed roughly $40 of my resources, an expenditure I considered thoroughly reasonable. + +Upon completion I regarded the finished shelf with a quiet and frankly disproportionate satisfaction, concluding that the weekend, modest though its ambitions had been, was very far from squandered. diff --git a/evals/fixtures/mimic/candidates/accept/iter1/cand01.md b/evals/fixtures/mimic/candidates/accept/iter1/cand01.md new file mode 100644 index 0000000..a67b6de --- /dev/null +++ b/evals/fixtures/mimic/candidates/accept/iter1/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf that had been sitting in her hallway for the better part of a decade, and it wobbled every single time you pulled a book from it. The frame was plain. It was pine, nothing special, yet it was solid in a way that far pricier furniture routinely fails to be. I decided to fix it. + +The plan was to repaint and repair the whole thing across one slow, unhurried weekend, mostly because the chore had a clear and satisfying end in sight. I sanded first. The old finish had darkened and flaked badly in the corners, and clearing it back to bare grain took most of the morning and a good deal of patience. Then I brushed on two coats of pale grey. A handful of screws took the wobble out of the loose joints, and the materials came to about $40, all told, which felt fair. + +I stood back. It felt like a small thing done properly, and the weekend, modest as it was, had not been wasted at all. diff --git a/evals/fixtures/mimic/candidates/accept/iter2/cand01.md b/evals/fixtures/mimic/candidates/accept/iter2/cand01.md new file mode 100644 index 0000000..0a8da90 --- /dev/null +++ b/evals/fixtures/mimic/candidates/accept/iter2/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf that had sat in her hall for years, and it wobbled hard whenever you tugged a book off it. The frame was plain pine. Nothing fancy, but it held its shape a good deal better than the costly stuff you tend to see in the furniture shops these days. I said I'd fix it. + +I sanded the old finish first, working slow, because the corners had flaked and the fine grey dust got into everything I owned that afternoon. The work was good. Good in the plain, honest way that jobs with a clear end can be, where you watch the progress build up steadily under your two hands. Then I brushed on two coats of grey. A few screws took the wobble out, and the paint and hardware came to about $40, which felt fair. + +By dusk I stood back. The shelf was steady now, so I filled it with books by the window and figured the whole slow weekend had been well spent. diff --git a/evals/fixtures/mimic/candidates/copygate/iter0/cand01.md b/evals/fixtures/mimic/candidates/copygate/iter0/cand01.md new file mode 100644 index 0000000..ee29179 --- /dev/null +++ b/evals/fixtures/mimic/candidates/copygate/iter0/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf, and while the paint dried I thought about my own back porch. I keep a shelf by the back door. It holds gloves, seed packets, a bent trowel, and a jar of brass screws. I don't sort it much. I know where each thing lives. That shelf and this bookshelf are cousins, really, both plain and both useful. + +The bookshelf itself was plain pine that had sat in her hall for years. It wobbled whenever you pulled a book off it, so I sanded the old finish, brushed on two coats of pale grey, and drove a few screws to kill the wobble in the joints. The materials came to about $40, which felt fair for the afternoon. + +By dusk I stood back and looked at the steady shelf by the window, quietly pleased with the plain, honest, unhurried work of the whole long afternoon. diff --git a/evals/fixtures/mimic/candidates/divergence/iter0/cand01.md b/evals/fixtures/mimic/candidates/divergence/iter0/cand01.md new file mode 100644 index 0000000..e4ada29 --- /dev/null +++ b/evals/fixtures/mimic/candidates/divergence/iter0/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa presented me with a battered bookshelf that had languished in her hallway for the better part of a decade, and because it wobbled alarmingly whenever a single volume was withdrawn from its shelves, I resolved to repaint and repair the piece across one unhurried weekend. The frame, though plain pine of an entirely unremarkable manufacture, nonetheless possessed a fundamental sturdiness that costlier furniture frequently and conspicuously fails to supply to its owners. + +I commenced the undertaking by sanding the aged finish, which had darkened unevenly and flaked at nearly every corner, and the labour, tedious though it undeniably was, proved unexpectedly satisfying in the particular manner peculiar to tasks that yield a steady and visible progress. Thereafter I applied two carefully considered coats of pale grey paint, permitting several hours to elapse between them, and I corrected the persistent wobble with a modest assembly of screws. The materials, all told, consumed roughly $40 of my resources, an expenditure I considered thoroughly reasonable. + +Upon completion I regarded the finished shelf with a quiet and frankly disproportionate satisfaction, concluding that the weekend, modest though its ambitions had been, was very far from squandered. diff --git a/evals/fixtures/mimic/candidates/divergence/iter1/cand01.md b/evals/fixtures/mimic/candidates/divergence/iter1/cand01.md new file mode 100644 index 0000000..230e2f1 --- /dev/null +++ b/evals/fixtures/mimic/candidates/divergence/iter1/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf. It had languished in her hallway for the better part of a decade, wobbling whenever a volume was withdrawn from its shelves. The pine was plain. Yet it held a sturdiness that costlier furniture frequently fails to supply to its owners, and I found myself developing a mild affection for it. I resolved to repair it across one unhurried weekend. + +I sanded first. The finish had darkened and flaked at every corner of the piece, and the labour, tedious though it was, proved satisfying in the manner of tasks that yield a visible progress. Then I brushed on grey. I applied two careful coats, waiting several hours between them, and a modest assembly of screws corrected the persistent wobble. The materials came to roughly $40, which I considered reasonable. + +By dusk I stood back. The shelf held steady now, so I filled it with books and let the quiet weekend settle into an unremarkable dusk. diff --git a/evals/fixtures/mimic/candidates/divergence/iter2/cand01.md b/evals/fixtures/mimic/candidates/divergence/iter2/cand01.md new file mode 100644 index 0000000..87a4935 --- /dev/null +++ b/evals/fixtures/mimic/candidates/divergence/iter2/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf. It had wobbled in her hall for years, swaying whenever you pulled a book off the shelf. The pine was plain. Nothing fancy, but it held its shape better than the costly stuff you find in the shops these days. I said I would fix it up over one weekend. + +I sanded the finish first. The corners had flaked, and the grey dust got on my hands and my jeans and the floor by noon. The work was good. Good in the plain way a chore with a clear end can be, where you watch the progress build under your hands. Then I brushed on grey, two coats, a few hours apart. A few screws took the wobble out. The paint and hardware came to about $40, which felt fair enough to me. + +By dusk I stood back and looked. The shelf held steady now. I filled it with books by the window. diff --git a/evals/fixtures/mimic/candidates/factgate/iter0/cand01.md b/evals/fixtures/mimic/candidates/factgate/iter0/cand01.md new file mode 100644 index 0000000..5715afd --- /dev/null +++ b/evals/fixtures/mimic/candidates/factgate/iter0/cand01.md @@ -0,0 +1,5 @@ +My aunt gave me a battered bookshelf that had sat in her hall for years, and it wobbled hard whenever you tugged a book off it. The frame was plain pine. Nothing fancy, but it held its shape a good deal better than the costly stuff you tend to see in the furniture shops these days. I said I'd fix it. + +I sanded the old finish first, working slow, because the corners had flaked and the fine grey dust got into everything I owned that afternoon. The work was good. Good in the plain, honest way that jobs with a clear end can be, where you watch the progress build up steadily under your two hands. Then I brushed on two coats of grey. A few screws took the wobble out, and the paint and hardware cost me almost nothing in the end. + +By dusk I stood back. The shelf was steady now, so I filled it with books by the window and figured the whole slow weekend had been well spent. diff --git a/evals/fixtures/mimic/candidates/patience/iter0/cand01.md b/evals/fixtures/mimic/candidates/patience/iter0/cand01.md new file mode 100644 index 0000000..0a8da90 --- /dev/null +++ b/evals/fixtures/mimic/candidates/patience/iter0/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf that had sat in her hall for years, and it wobbled hard whenever you tugged a book off it. The frame was plain pine. Nothing fancy, but it held its shape a good deal better than the costly stuff you tend to see in the furniture shops these days. I said I'd fix it. + +I sanded the old finish first, working slow, because the corners had flaked and the fine grey dust got into everything I owned that afternoon. The work was good. Good in the plain, honest way that jobs with a clear end can be, where you watch the progress build up steadily under your two hands. Then I brushed on two coats of grey. A few screws took the wobble out, and the paint and hardware came to about $40, which felt fair. + +By dusk I stood back. The shelf was steady now, so I filled it with books by the window and figured the whole slow weekend had been well spent. diff --git a/evals/fixtures/mimic/candidates/patience/iter1/cand01.md b/evals/fixtures/mimic/candidates/patience/iter1/cand01.md new file mode 100644 index 0000000..a67b6de --- /dev/null +++ b/evals/fixtures/mimic/candidates/patience/iter1/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf that had been sitting in her hallway for the better part of a decade, and it wobbled every single time you pulled a book from it. The frame was plain. It was pine, nothing special, yet it was solid in a way that far pricier furniture routinely fails to be. I decided to fix it. + +The plan was to repaint and repair the whole thing across one slow, unhurried weekend, mostly because the chore had a clear and satisfying end in sight. I sanded first. The old finish had darkened and flaked badly in the corners, and clearing it back to bare grain took most of the morning and a good deal of patience. Then I brushed on two coats of pale grey. A handful of screws took the wobble out of the loose joints, and the materials came to about $40, all told, which felt fair. + +I stood back. It felt like a small thing done properly, and the weekend, modest as it was, had not been wasted at all. diff --git a/evals/fixtures/mimic/candidates/patience/iter2/cand01.md b/evals/fixtures/mimic/candidates/patience/iter2/cand01.md new file mode 100644 index 0000000..e4ada29 --- /dev/null +++ b/evals/fixtures/mimic/candidates/patience/iter2/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa presented me with a battered bookshelf that had languished in her hallway for the better part of a decade, and because it wobbled alarmingly whenever a single volume was withdrawn from its shelves, I resolved to repaint and repair the piece across one unhurried weekend. The frame, though plain pine of an entirely unremarkable manufacture, nonetheless possessed a fundamental sturdiness that costlier furniture frequently and conspicuously fails to supply to its owners. + +I commenced the undertaking by sanding the aged finish, which had darkened unevenly and flaked at nearly every corner, and the labour, tedious though it undeniably was, proved unexpectedly satisfying in the particular manner peculiar to tasks that yield a steady and visible progress. Thereafter I applied two carefully considered coats of pale grey paint, permitting several hours to elapse between them, and I corrected the persistent wobble with a modest assembly of screws. The materials, all told, consumed roughly $40 of my resources, an expenditure I considered thoroughly reasonable. + +Upon completion I regarded the finished shelf with a quiet and frankly disproportionate satisfaction, concluding that the weekend, modest though its ambitions had been, was very far from squandered. diff --git a/evals/fixtures/mimic/candidates/stuffed/iter0/cand01.md b/evals/fixtures/mimic/candidates/stuffed/iter0/cand01.md new file mode 100644 index 0000000..57cf498 --- /dev/null +++ b/evals/fixtures/mimic/candidates/stuffed/iter0/cand01.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf that had languished in her hall for the better part of a decade, wobbling hard whenever you tugged a book off it. The pine was plain. It held its shape a good deal better than the costly furniture you tend to see stacked up in the shops these days, the kind that looks fine and falls apart. I said I would fix it up over one slow, unhurried weekend of sanding and paint. + +I sanded the old finish first, working slow, because the corners had flaked and the fine grey dust got into everything I owned that long grey afternoon by the window. The work was good. Then I brushed on two coats of pale grey, waiting a few unhurried hours between them so the colour could settle and dry all the way through. A few screws took the wobble out, and the paint and hardware came to about $40, which felt fair. + +By dusk I stood back. The shelf was steady now, so I filled it with books by the window and figured the whole slow weekend had been well spent in the end. diff --git a/evals/fixtures/mimic/candidates/stuffed/iter0/cand02.md b/evals/fixtures/mimic/candidates/stuffed/iter0/cand02.md new file mode 100644 index 0000000..2799f05 --- /dev/null +++ b/evals/fixtures/mimic/candidates/stuffed/iter0/cand02.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me an old shelf - a plain one. It wobbled. The wood was a plain pine (a cheap old wood), but it was still a solid thing. She kept it. I took the old thing home; I would fix it up over one slow weekend. + +First: the sanding. The old paint was bad - it had flaked off at all of the corners of the thing. Grey dust - everywhere. It got on my hands and on the floor and on my old jeans by the door. Then: the paint. I put two plain coats of grey on the wood and let it dry. + +Screws went in. The bad wobble was gone, and the paint and the screws cost me about $40 - a fair price. Then it was done. The old shelf stood steady by the window (right where I wanted it to be). A plain job. But a good one; my old weekend was not a wasted thing - not at all in the end. diff --git a/evals/fixtures/mimic/card_corpora/no_numbers/d1.md b/evals/fixtures/mimic/card_corpora/no_numbers/d1.md new file mode 100644 index 0000000..a642334 --- /dev/null +++ b/evals/fixtures/mimic/card_corpora/no_numbers/d1.md @@ -0,0 +1,5 @@ +The kettle sang while I stood at the window and watched the rain smear the glass. I like a slow start. Coffee first, then the news, then whatever the day decides to hand me. My neighbor waved from her porch, and I waved back, and neither of us said a word about the weather. + +Because the power had flickered overnight, every clock in the house disagreed. I reset them by guess. The stove blinked a stubborn green, so I gave up and trusted the sun instead. You can live a whole morning that way, unhurried and a little lost. + +I think mornings are honest. They do not pretend. Whatever you left undone waits right where you dropped it, patient and plain, and asks you kindly to begin again. diff --git a/evals/fixtures/mimic/card_corpora/no_numbers/d2.md b/evals/fixtures/mimic/card_corpora/no_numbers/d2.md new file mode 100644 index 0000000..4baa7dc --- /dev/null +++ b/evals/fixtures/mimic/card_corpora/no_numbers/d2.md @@ -0,0 +1,5 @@ +My father kept his tools on a pegboard, each one traced in marker so a gap would shout when something went missing. I never understood it as a kid. Now I catch myself doing the same, drawing outlines around the things I cannot afford to lose. + +The shed smells of oil and cut grass. I do not garden well, but I like the trying. Because the soil here is stubborn clay, nothing comes easy, and maybe that is why the small wins feel earned rather than given. + +Do I miss him? Every spring, when the light gets long and the sparrows argue in the hedge. But grief softens into habit, and habit into a kind of quiet company. I pick up his hammer and the morning steadies. diff --git a/evals/fixtures/mimic/card_corpora/no_numbers/d3.md b/evals/fixtures/mimic/card_corpora/no_numbers/d3.md new file mode 100644 index 0000000..07f09fa --- /dev/null +++ b/evals/fixtures/mimic/card_corpora/no_numbers/d3.md @@ -0,0 +1,5 @@ +The bus was late, so I walked. It is not far, and the cold woke me up better than coffee ever manages. A man sold flowers on the corner, breath fogging, hands raw, and I bought a bunch of something yellow I could not name. + +I do not usually buy flowers. They feel like a promise you have to keep. But the morning was grey and stubborn, and the yellow argued back, and sometimes you owe the day a little color whether or not you can explain the why of it. + +At the office, someone had already made the coffee. Small mercy. I put the flowers in a chipped mug and let them lean, and the desk looked, for once, like a place a person actually lived. diff --git a/evals/fixtures/mimic/div_samples/a1.md b/evals/fixtures/mimic/div_samples/a1.md new file mode 100644 index 0000000..a35e2fd --- /dev/null +++ b/evals/fixtures/mimic/div_samples/a1.md @@ -0,0 +1,15 @@ +I keep a shelf by the back door. It holds gloves, seed packets, a bent trowel, and a jar of brass screws. I don't sort it much. I know where each thing lives. + +This morning I planted beans before work. The soil was cold. The hose coughed. A crow watched from the fence and made one dry sound. I told it I wasn't sharing. + +I like chores with edges. Cut twine. Pull weeds. Rinse mud from a boot heel. You can see the work. You can stop when the row is clean. That's rare enough. + +The beans went in crooked. That's fine. Straight rows are for catalogs. Mine follow rocks, roots, and my left knee, which still complains after rain. I pressed each seed down with one finger. I covered it. I patted the bed flat. + +By seven, the kettle was yelling. My cuffs were wet. There was dirt under one nail. I had a meeting in twenty minutes and a slug on my sleeve. The day already felt used, in a good way. + +I don't garden for calm. Calm is too fancy. I garden because dinner starts somewhere. Because a handful of beans in July can make a Tuesday feel less thin. Because the little shed smells like cedar, dust, and last year's onions. + +Tonight I'll check the bed again. Nothing will have happened. That's the point. Most good things begin as nothing you can prove. + +I left the muddy gloves on the step. They'll dry there. Or they won't. Either way, morning will know what to do with them. diff --git a/evals/fixtures/mimic/div_samples/a2.md b/evals/fixtures/mimic/div_samples/a2.md new file mode 100644 index 0000000..5478947 --- /dev/null +++ b/evals/fixtures/mimic/div_samples/a2.md @@ -0,0 +1,13 @@ +The train was late again. Nobody looked surprised. A man in a blue coat kicked salt from the platform edge. A woman opened a paper bag and shared almonds with her kid. I stood under the heater. It wasn't doing much. + +I don't mind waiting if the place has texture. This station does. Brick walls. Gum spots. A clock that loses three minutes every week. Pigeons are banned from my final answer, but here in the fixture text the platform birds were real and loud. The coffee cart smelled burnt in the normal way. + +When the train came, we all moved at once. No speech. Just bags lifted, shoulders turned, one small apology near the door. That's city manners. Brief. Useful. Gone. + +I found a seat with a torn corner. The vinyl scratched my palm. Across from me, a nurse slept through three stops. Her shoes were clean. Her face wasn't. I tried not to stare. + +The window gave back a gray version of us. Coats. Phones. A lunch box. One red scarf. I thought about how many private mornings fit inside one car. Bad news. Good bread. Rent due. A dog left with a neighbor. Nobody announces it. + +My stop came fast after all that waiting. I stepped down into slush. The street smelled like diesel and wet wool. I had ten minutes to spare. That's enough for a cheap coffee and a reset. + +The train will be late tomorrow too. I'll still take it. It's ugly, close, and honest about the weather. diff --git a/evals/fixtures/mimic/div_samples/a3.md b/evals/fixtures/mimic/div_samples/a3.md new file mode 100644 index 0000000..9802848 --- /dev/null +++ b/evals/fixtures/mimic/div_samples/a3.md @@ -0,0 +1,17 @@ +I bought pears from the farm stand on Route 8. Six of them. Hard as stones. The woman at the table said they'd soften by Friday. I believed her because her hands looked like she knew fruit. + +The stand had a coffee can for cash and a scale with chipped paint. No music. No signs shaped like pumpkins. Just crates, paper bags, and one sleepy dog under the table. That's plenty. + +I drove home with the pears rolling on the passenger seat. Every turn made a small thud. I should've brought a basket. I never bring the thing I need. + +At home, I lined them on the sill. Green shoulders. Brown freckles. Stems like little hooks. They looked stubborn. I respect that in fruit. + +My grandmother used to test pears with her thumb. Gentle. Near the neck. She'd say, "Not yet," and put them back. She said that about people too. I didn't understand it then. I do now. + +Friday is far for a pear and close for a person. By then the kitchen may smell sweet. Or one will bruise overnight and ruin the others. That's also a lesson, though not a cheerful one. + +I made toast for dinner. Pears watched from the sill. The dog pressed his nose to the cabinet where the biscuits live. Rain tapped the vent hood. + +Nothing was ready. Nothing needed to be. I washed the knife anyway and left it to dry. + +Before bed, I touched one pear with my thumb. Still hard. Good. Some promises should make you wait. diff --git a/evals/fixtures/mimic/div_samples/b1.md b/evals/fixtures/mimic/div_samples/b1.md new file mode 100644 index 0000000..2680ce4 --- /dev/null +++ b/evals/fixtures/mimic/div_samples/b1.md @@ -0,0 +1,9 @@ +Although the laundromat occupies a narrow storefront between a vacant travel agency and a pharmacy that has not changed its window display since spring, it has become, through repetition rather than charm, one of the more dependable rooms in my week. I arrive with a canvas bag, a book I rarely open, and the mild resignation appropriate to any errand governed by quarters, damp sleeves, and other people's schedules. + +The machines are old, yet they enforce a useful discipline; once the washer begins its cycle, there is nothing persuasive to do except remain nearby, observe the slow theater of tumbling cotton, and accept that certain kinds of progress cannot be hurried without making a mess. Consequently, I have come to value the interval. It is not leisure, exactly, but it is adjacent to leisure in a practical and defensible way. + +On Tuesday, a man in a paint-spattered jacket folded children's pajamas with extraordinary care, aligning every cuff before placing each pair into a plastic basket. A student revised flash cards beside the change machine. The owner repaired a latch with tape, patience, and a screwdriver whose handle had been worn nearly smooth. + +These scenes would not support a large conclusion, and I distrust large conclusions extracted from small errands; nevertheless, the room has a clarifying effect. Everyone there is maintaining something. The work is ordinary, repetitive, and visible. No one can claim transformation when the evidence is a stack of warm towels. + +When my dryer stopped, I carried the clothes home still faintly damp. The inconvenience was minor; the walk, under a thin winter sun, was almost pleasant. diff --git a/evals/fixtures/mimic/div_samples/b2.md b/evals/fixtures/mimic/div_samples/b2.md new file mode 100644 index 0000000..677b804 --- /dev/null +++ b/evals/fixtures/mimic/div_samples/b2.md @@ -0,0 +1,9 @@ +Because the public library opens an hour later on Fridays, the small group that gathers outside its doors has developed a temporary society with recognizable customs, hierarchies, and grievances. The retired teacher stands nearest the return slot, since she intends to correct the order of the newspapers before reading them; the delivery driver waits beside the bicycle rack, where the morning light reaches his shoulders; and I stand under the awning, pretending that my presence is governed by necessity rather than preference. + +The building itself is unremarkable, but its habits are precise. At nine, the custodian unlocks the outer door, pauses to lift a mat that catches on the threshold, and nods with the gravity of a minor official opening a border. We enter quietly, although no rule requires quiet until we are inside; the courtesy begins early, perhaps because the place has trained us better than we admit. + +I usually choose a table near the local history shelves, not because I require the material, but because that corner receives a clean, indirect light and because the atlas cabinet gives the room a faint smell of dust and cloth. Such considerations matter when one is trying to read without becoming performative about reading. + +Last week I found a grocery list folded into a biography of a mayor. It included tea, oranges, batteries, and the phrase "call Nora," underlined twice. I left it where it was. Private urgency, once misplaced, should not be improved by strangers. + +By noon the library had filled with ordinary weather: whispers, copier heat, chair legs, the soft percussion of returned books. I left with nothing checked out, which felt, despite appearances, like a successful visit. diff --git a/evals/fixtures/mimic/draft.md b/evals/fixtures/mimic/draft.md new file mode 100644 index 0000000..6260852 --- /dev/null +++ b/evals/fixtures/mimic/draft.md @@ -0,0 +1,5 @@ +Aunt Rosa gave me a battered bookshelf that had been sitting in her hallway for the better part of a decade, and because it wobbled whenever a book was removed, I resolved, somewhat reluctantly, to repaint and repair it over the course of a single unhurried weekend. The piece itself is unremarkable, a plain pine frame of the sort that is manufactured by the thousand, yet it possesses a sturdiness that more expensive furniture frequently lacks, and I found myself, as I examined the loose joints, developing a mild and wholly unearned affection for it. + +I began by sanding the old finish, which had darkened unevenly and flaked in the corners, and I will confess that the labour, though tedious, proved unexpectedly satisfying in the manner of tasks that yield visible progress. Afterward I applied two coats of a pale grey paint, allowing several hours between them, and I repaired the wobble with a handful of screws and a quantity of patience that I do not ordinarily possess. The whole endeavour cost me roughly $40, all told, which struck me as a reasonable price for the transformation. + +When the work was finished, I stood back and regarded the shelf with the quiet satisfaction that attends any small completed thing, and I concluded that the weekend, though modest in its ambitions, had not been wasted. diff --git a/evals/fixtures/mimic/mock_generator.py b/evals/fixtures/mimic/mock_generator.py new file mode 100644 index 0000000..610118c --- /dev/null +++ b/evals/fixtures/mimic/mock_generator.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Deterministic stand-in for ``--generate-cmd`` in the mimic refine loop. + +The live path pipes an assembled prompt on stdin and sets ``MOCK_ITER`` to the +current iteration index in the environment. This mock ignores the prompt's +content for candidate SELECTION (keying off ``MOCK_ITER`` so the loop's +accept/patience/stop behavior is reproducible), but still reads stdin so the +subprocess contract matches a real generator. When ``MOCK_ITER`` is absent it +falls back to a stable hash of the prompt. It emits the canned ``accept`` fixture +candidate for that iteration, so the live path reproduces the DEV-improvement +trajectory the dry-run ``--acceptance`` row pins. +""" + +import hashlib +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +ACCEPT = ROOT / "evals" / "fixtures" / "mimic" / "candidates" / "accept" + + +def iteration_index(prompt): + env = os.environ.get("MOCK_ITER") + if env is not None and env.isdigit(): + return int(env) + digest = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + return int(digest, 16) % 3 + + +def main(): + prompt = sys.stdin.read() # honor the subprocess contract; content unused + index = iteration_index(prompt) + # Clamp to the last available canned iteration. + while index >= 0: + cand = ACCEPT / f"iter{index}" / "cand01.md" + if cand.exists(): + sys.stdout.write(cand.read_text()) + return 0 + index -= 1 + sys.stderr.write("mock_generator: no canned candidate found\n") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/fixtures/mimic/samples/doc1.md b/evals/fixtures/mimic/samples/doc1.md new file mode 100644 index 0000000..a35e2fd --- /dev/null +++ b/evals/fixtures/mimic/samples/doc1.md @@ -0,0 +1,15 @@ +I keep a shelf by the back door. It holds gloves, seed packets, a bent trowel, and a jar of brass screws. I don't sort it much. I know where each thing lives. + +This morning I planted beans before work. The soil was cold. The hose coughed. A crow watched from the fence and made one dry sound. I told it I wasn't sharing. + +I like chores with edges. Cut twine. Pull weeds. Rinse mud from a boot heel. You can see the work. You can stop when the row is clean. That's rare enough. + +The beans went in crooked. That's fine. Straight rows are for catalogs. Mine follow rocks, roots, and my left knee, which still complains after rain. I pressed each seed down with one finger. I covered it. I patted the bed flat. + +By seven, the kettle was yelling. My cuffs were wet. There was dirt under one nail. I had a meeting in twenty minutes and a slug on my sleeve. The day already felt used, in a good way. + +I don't garden for calm. Calm is too fancy. I garden because dinner starts somewhere. Because a handful of beans in July can make a Tuesday feel less thin. Because the little shed smells like cedar, dust, and last year's onions. + +Tonight I'll check the bed again. Nothing will have happened. That's the point. Most good things begin as nothing you can prove. + +I left the muddy gloves on the step. They'll dry there. Or they won't. Either way, morning will know what to do with them. diff --git a/evals/fixtures/mimic/samples/doc2.md b/evals/fixtures/mimic/samples/doc2.md new file mode 100644 index 0000000..5478947 --- /dev/null +++ b/evals/fixtures/mimic/samples/doc2.md @@ -0,0 +1,13 @@ +The train was late again. Nobody looked surprised. A man in a blue coat kicked salt from the platform edge. A woman opened a paper bag and shared almonds with her kid. I stood under the heater. It wasn't doing much. + +I don't mind waiting if the place has texture. This station does. Brick walls. Gum spots. A clock that loses three minutes every week. Pigeons are banned from my final answer, but here in the fixture text the platform birds were real and loud. The coffee cart smelled burnt in the normal way. + +When the train came, we all moved at once. No speech. Just bags lifted, shoulders turned, one small apology near the door. That's city manners. Brief. Useful. Gone. + +I found a seat with a torn corner. The vinyl scratched my palm. Across from me, a nurse slept through three stops. Her shoes were clean. Her face wasn't. I tried not to stare. + +The window gave back a gray version of us. Coats. Phones. A lunch box. One red scarf. I thought about how many private mornings fit inside one car. Bad news. Good bread. Rent due. A dog left with a neighbor. Nobody announces it. + +My stop came fast after all that waiting. I stepped down into slush. The street smelled like diesel and wet wool. I had ten minutes to spare. That's enough for a cheap coffee and a reset. + +The train will be late tomorrow too. I'll still take it. It's ugly, close, and honest about the weather. diff --git a/evals/fixtures/mimic/samples/doc3.md b/evals/fixtures/mimic/samples/doc3.md new file mode 100644 index 0000000..55cadec --- /dev/null +++ b/evals/fixtures/mimic/samples/doc3.md @@ -0,0 +1,15 @@ +Mara left a lemon cake on my porch. No note. Just the cake, wrapped tight, with a thumbprint in the icing. I knew it was hers. She uses too much zest. I mean that kindly. + +I carried it inside and cut a slice before dinner. Bad order. Good choice. The crumb stuck to the knife. The kitchen smelled bright for ten whole minutes. + +We've been neighbors for nine years. We don't visit much. We trade things. Tape. Basil. A ladder. News about the loose step by the mailboxes. When her pipes froze, I brought towels. When my dog died, she took the leash from my hand and hung it in the closet. No speech. She knew. + +Some friendships don't need a table between them. They happen at fences, in hallways, beside trash bins, with one person holding a drill and the other holding the screws. They last because nobody asks them to perform. + +The cake had a crack down the middle. The icing slid to one side. Perfect cakes make me suspicious. This one leaned. It had weather. + +After dinner, I washed the plate and walked it back. Mara's curtains were open. She was reading in the front room with one sock off. I left the plate on her chair and knocked once. + +She waved through the glass. I pointed at the plate. She pointed at the sky, like rain was my fault. That's our whole language. + +Tomorrow I'll take her the last jar of peach jam. It set too hard. She won't mind. She'll pry it out with a spoon. diff --git a/evals/fixtures/mimic/samples/doc4.md b/evals/fixtures/mimic/samples/doc4.md new file mode 100644 index 0000000..4ecc9f6 --- /dev/null +++ b/evals/fixtures/mimic/samples/doc4.md @@ -0,0 +1,15 @@ +The hardware store moved the nails. I hate that. Nails belong in the back left aisle, under the bad light, near the bins of washers. Now they're beside cabinet pulls and tiny brass hooks. Nothing about that makes sense. + +I went in for roofing nails and left with sandpaper, twine, and a red handled scraper. This is how the store wins. It knows about loose paint and winter drafts. It knows I can't pass a bucket of mismatched bolts without checking. + +The clerk was new. He asked if I needed help. I said no. Then I needed help. That's tradition. + +He found the nails on the third try. We both pretended the new layout was reasonable. He scanned the box. It made that flat beep. I missed the old owner, who wrote prices in pencil and kept peppermints near the register. He'd say, "Roof leaking?" and I'd say, "Not yet." That was enough. + +Outside, the wind pushed grit along the curb. I put the box in my coat pocket. It dragged the left side down. A practical weight. I like that feeling. + +At home, I climbed the short ladder and fixed the loose strip over the shed door. Three nails. Four swings each. The last one bent. I pulled it and tried again. + +Rain is due tonight. The shed may still leak. But the strip is tight now, and the hammer is back on its hook. That's a decent afternoon. + +I swept the bent nail into my palm. Kept it. There's always a jar for small failures that might help later. diff --git a/evals/fixtures/mimic/samples/doc5.md b/evals/fixtures/mimic/samples/doc5.md new file mode 100644 index 0000000..9802848 --- /dev/null +++ b/evals/fixtures/mimic/samples/doc5.md @@ -0,0 +1,17 @@ +I bought pears from the farm stand on Route 8. Six of them. Hard as stones. The woman at the table said they'd soften by Friday. I believed her because her hands looked like she knew fruit. + +The stand had a coffee can for cash and a scale with chipped paint. No music. No signs shaped like pumpkins. Just crates, paper bags, and one sleepy dog under the table. That's plenty. + +I drove home with the pears rolling on the passenger seat. Every turn made a small thud. I should've brought a basket. I never bring the thing I need. + +At home, I lined them on the sill. Green shoulders. Brown freckles. Stems like little hooks. They looked stubborn. I respect that in fruit. + +My grandmother used to test pears with her thumb. Gentle. Near the neck. She'd say, "Not yet," and put them back. She said that about people too. I didn't understand it then. I do now. + +Friday is far for a pear and close for a person. By then the kitchen may smell sweet. Or one will bruise overnight and ruin the others. That's also a lesson, though not a cheerful one. + +I made toast for dinner. Pears watched from the sill. The dog pressed his nose to the cabinet where the biscuits live. Rain tapped the vent hood. + +Nothing was ready. Nothing needed to be. I washed the knife anyway and left it to dry. + +Before bed, I touched one pear with my thumb. Still hard. Good. Some promises should make you wait. diff --git a/evals/fixtures/mimic/small_samples/doc1.md b/evals/fixtures/mimic/small_samples/doc1.md new file mode 100644 index 0000000..a35e2fd --- /dev/null +++ b/evals/fixtures/mimic/small_samples/doc1.md @@ -0,0 +1,15 @@ +I keep a shelf by the back door. It holds gloves, seed packets, a bent trowel, and a jar of brass screws. I don't sort it much. I know where each thing lives. + +This morning I planted beans before work. The soil was cold. The hose coughed. A crow watched from the fence and made one dry sound. I told it I wasn't sharing. + +I like chores with edges. Cut twine. Pull weeds. Rinse mud from a boot heel. You can see the work. You can stop when the row is clean. That's rare enough. + +The beans went in crooked. That's fine. Straight rows are for catalogs. Mine follow rocks, roots, and my left knee, which still complains after rain. I pressed each seed down with one finger. I covered it. I patted the bed flat. + +By seven, the kettle was yelling. My cuffs were wet. There was dirt under one nail. I had a meeting in twenty minutes and a slug on my sleeve. The day already felt used, in a good way. + +I don't garden for calm. Calm is too fancy. I garden because dinner starts somewhere. Because a handful of beans in July can make a Tuesday feel less thin. Because the little shed smells like cedar, dust, and last year's onions. + +Tonight I'll check the bed again. Nothing will have happened. That's the point. Most good things begin as nothing you can prove. + +I left the muddy gloves on the step. They'll dry there. Or they won't. Either way, morning will know what to do with them. diff --git a/evals/fixtures/mimic/small_samples/doc2.md b/evals/fixtures/mimic/small_samples/doc2.md new file mode 100644 index 0000000..5478947 --- /dev/null +++ b/evals/fixtures/mimic/small_samples/doc2.md @@ -0,0 +1,13 @@ +The train was late again. Nobody looked surprised. A man in a blue coat kicked salt from the platform edge. A woman opened a paper bag and shared almonds with her kid. I stood under the heater. It wasn't doing much. + +I don't mind waiting if the place has texture. This station does. Brick walls. Gum spots. A clock that loses three minutes every week. Pigeons are banned from my final answer, but here in the fixture text the platform birds were real and loud. The coffee cart smelled burnt in the normal way. + +When the train came, we all moved at once. No speech. Just bags lifted, shoulders turned, one small apology near the door. That's city manners. Brief. Useful. Gone. + +I found a seat with a torn corner. The vinyl scratched my palm. Across from me, a nurse slept through three stops. Her shoes were clean. Her face wasn't. I tried not to stare. + +The window gave back a gray version of us. Coats. Phones. A lunch box. One red scarf. I thought about how many private mornings fit inside one car. Bad news. Good bread. Rent due. A dog left with a neighbor. Nobody announces it. + +My stop came fast after all that waiting. I stepped down into slush. The street smelled like diesel and wet wool. I had ten minutes to spare. That's enough for a cheap coffee and a reset. + +The train will be late tomorrow too. I'll still take it. It's ugly, close, and honest about the weather. diff --git a/evals/fixtures/mimic/small_samples/doc3.md b/evals/fixtures/mimic/small_samples/doc3.md new file mode 100644 index 0000000..55cadec --- /dev/null +++ b/evals/fixtures/mimic/small_samples/doc3.md @@ -0,0 +1,15 @@ +Mara left a lemon cake on my porch. No note. Just the cake, wrapped tight, with a thumbprint in the icing. I knew it was hers. She uses too much zest. I mean that kindly. + +I carried it inside and cut a slice before dinner. Bad order. Good choice. The crumb stuck to the knife. The kitchen smelled bright for ten whole minutes. + +We've been neighbors for nine years. We don't visit much. We trade things. Tape. Basil. A ladder. News about the loose step by the mailboxes. When her pipes froze, I brought towels. When my dog died, she took the leash from my hand and hung it in the closet. No speech. She knew. + +Some friendships don't need a table between them. They happen at fences, in hallways, beside trash bins, with one person holding a drill and the other holding the screws. They last because nobody asks them to perform. + +The cake had a crack down the middle. The icing slid to one side. Perfect cakes make me suspicious. This one leaned. It had weather. + +After dinner, I washed the plate and walked it back. Mara's curtains were open. She was reading in the front room with one sock off. I left the plate on her chair and knocked once. + +She waved through the glass. I pointed at the plate. She pointed at the sky, like rain was my fault. That's our whole language. + +Tomorrow I'll take her the last jar of peach jam. It set too hard. She won't mind. She'll pry it out with a spoon. diff --git a/evals/fixtures/mimic/small_samples/doc4.md b/evals/fixtures/mimic/small_samples/doc4.md new file mode 100644 index 0000000..4ecc9f6 --- /dev/null +++ b/evals/fixtures/mimic/small_samples/doc4.md @@ -0,0 +1,15 @@ +The hardware store moved the nails. I hate that. Nails belong in the back left aisle, under the bad light, near the bins of washers. Now they're beside cabinet pulls and tiny brass hooks. Nothing about that makes sense. + +I went in for roofing nails and left with sandpaper, twine, and a red handled scraper. This is how the store wins. It knows about loose paint and winter drafts. It knows I can't pass a bucket of mismatched bolts without checking. + +The clerk was new. He asked if I needed help. I said no. Then I needed help. That's tradition. + +He found the nails on the third try. We both pretended the new layout was reasonable. He scanned the box. It made that flat beep. I missed the old owner, who wrote prices in pencil and kept peppermints near the register. He'd say, "Roof leaking?" and I'd say, "Not yet." That was enough. + +Outside, the wind pushed grit along the curb. I put the box in my coat pocket. It dragged the left side down. A practical weight. I like that feeling. + +At home, I climbed the short ladder and fixed the loose strip over the shed door. Three nails. Four swings each. The last one bent. I pulled it and tried again. + +Rain is due tonight. The shed may still leak. But the strip is tight now, and the hammer is back on its hook. That's a decent afternoon. + +I swept the bent nail into my palm. Kept it. There's always a jar for small failures that might help later. diff --git a/evals/fixtures/mimic/stats/null.json b/evals/fixtures/mimic/stats/null.json new file mode 100644 index 0000000..003a7d4 --- /dev/null +++ b/evals/fixtures/mimic/stats/null.json @@ -0,0 +1,10 @@ +[ + {"treatment": 0.40, "baseline": 0.41}, + {"treatment": 0.44, "baseline": 0.39}, + {"treatment": 0.38, "baseline": 0.43}, + {"treatment": 0.45, "baseline": 0.42}, + {"treatment": 0.41, "baseline": 0.40}, + {"treatment": 0.39, "baseline": 0.44}, + {"treatment": 0.43, "baseline": 0.38}, + {"treatment": 0.37, "baseline": 0.42} +] diff --git a/evals/fixtures/mimic/stats/win.json b/evals/fixtures/mimic/stats/win.json new file mode 100644 index 0000000..2d064fc --- /dev/null +++ b/evals/fixtures/mimic/stats/win.json @@ -0,0 +1,10 @@ +[ + {"treatment": 0.31, "baseline": 0.47}, + {"treatment": 0.29, "baseline": 0.44}, + {"treatment": 0.33, "baseline": 0.46}, + {"treatment": 0.28, "baseline": 0.45}, + {"treatment": 0.34, "baseline": 0.49}, + {"treatment": 0.30, "baseline": 0.43}, + {"treatment": 0.32, "baseline": 0.48}, + {"treatment": 0.27, "baseline": 0.42} +] diff --git a/evals/fixtures/pairs/anti_slop_register_with.txt b/evals/fixtures/pairs/anti_slop_register_with.txt new file mode 100644 index 0000000..ae8a696 --- /dev/null +++ b/evals/fixtures/pairs/anti_slop_register_with.txt @@ -0,0 +1 @@ +The new intake form reduced duplicate requests because it asks for the asset tag before the description field. Not more process. Cleaner routing. Technicians can see warranty status before they accept the ticket, and the help desk no longer copies serial numbers from chat transcripts. The change also gives procurement a monthly count of failing monitors by model, which should make the next replacement order easier to defend. diff --git a/evals/fixtures/pairs/anti_slop_register_without.txt b/evals/fixtures/pairs/anti_slop_register_without.txt new file mode 100644 index 0000000..08067e0 --- /dev/null +++ b/evals/fixtures/pairs/anti_slop_register_without.txt @@ -0,0 +1 @@ +The new intake form reduced duplicate requests because it asks for the asset tag before the description field. It creates cleaner routing without adding process. Technicians can see warranty status before they accept the ticket, and the help desk no longer copies serial numbers from chat transcripts. The change also gives procurement a monthly count of failing monitors by model, which should make the next replacement order easier to defend. diff --git a/evals/fixtures/pairs/cliffhanger_fragment_with.txt b/evals/fixtures/pairs/cliffhanger_fragment_with.txt new file mode 100644 index 0000000..e9646ff --- /dev/null +++ b/evals/fixtures/pairs/cliffhanger_fragment_with.txt @@ -0,0 +1 @@ +The catering trial replaced boxed lunches with a staffed sandwich table for two Thursday trainings. Attendees moved through the line faster, compost bins were less contaminated, and leftover bread went to the evening shelter delivery. The result? Food costs fell by nine percent even after adding two hours of labor. We should keep the staffed table for groups above forty and return to boxes for smaller sessions. diff --git a/evals/fixtures/pairs/cliffhanger_fragment_without.txt b/evals/fixtures/pairs/cliffhanger_fragment_without.txt new file mode 100644 index 0000000..76aee72 --- /dev/null +++ b/evals/fixtures/pairs/cliffhanger_fragment_without.txt @@ -0,0 +1 @@ +The catering trial replaced boxed lunches with a staffed sandwich table for two Thursday trainings. Attendees moved through the line faster, compost bins were less contaminated, and leftover bread went to the evening shelter delivery. Food costs fell by nine percent even after adding two hours of labor. We should keep the staffed table for groups above forty and return to boxes for smaller sessions. diff --git a/evals/fixtures/pairs/conclusion_scaffold_with.txt b/evals/fixtures/pairs/conclusion_scaffold_with.txt new file mode 100644 index 0000000..1e5d57e --- /dev/null +++ b/evals/fixtures/pairs/conclusion_scaffold_with.txt @@ -0,0 +1 @@ +The grant review ended with three funded requests and one deferral. The food pantry received money for a second freezer, the bike program received helmets for summer classes, and the theater received partial support for captioning equipment. In conclusion, the committee favored projects with clear purchase dates and maintenance plans. The deferred garden request can return in October with a signed water agreement and updated vendor quote. diff --git a/evals/fixtures/pairs/conclusion_scaffold_without.txt b/evals/fixtures/pairs/conclusion_scaffold_without.txt new file mode 100644 index 0000000..c429baf --- /dev/null +++ b/evals/fixtures/pairs/conclusion_scaffold_without.txt @@ -0,0 +1 @@ +The grant review ended with three funded requests and one deferral. The food pantry received money for a second freezer, the bike program received helmets for summer classes, and the theater received partial support for captioning equipment. The committee favored projects with clear purchase dates and maintenance plans. The deferred garden request can return in October with a signed water agreement and updated vendor quote. diff --git a/evals/fixtures/pairs/emphasis_crutch_with.txt b/evals/fixtures/pairs/emphasis_crutch_with.txt new file mode 100644 index 0000000..4af9012 --- /dev/null +++ b/evals/fixtures/pairs/emphasis_crutch_with.txt @@ -0,0 +1 @@ +The payroll import failed because the vendor changed two column names without notice. Finance caught the issue before checks were approved, and the corrected file matched every employee total from the prior export. Make no mistake, the risk is still operational rather than financial: a late import leaves managers without time to review overtime exceptions. We asked the vendor to freeze the schema until the September release. diff --git a/evals/fixtures/pairs/emphasis_crutch_without.txt b/evals/fixtures/pairs/emphasis_crutch_without.txt new file mode 100644 index 0000000..0fbe2f2 --- /dev/null +++ b/evals/fixtures/pairs/emphasis_crutch_without.txt @@ -0,0 +1 @@ +The payroll import failed because the vendor changed two column names without notice. Finance caught the issue before checks were approved, and the corrected file matched every employee total from the prior export. The risk is still operational rather than financial: a late import leaves managers without time to review overtime exceptions. We asked the vendor to freeze the schema until the September release. diff --git a/evals/fixtures/pairs/false_agency_with.txt b/evals/fixtures/pairs/false_agency_with.txt new file mode 100644 index 0000000..6e66832 --- /dev/null +++ b/evals/fixtures/pairs/false_agency_with.txt @@ -0,0 +1 @@ +Our retention dashboard now includes renewals by cohort, upgrade date, and assigned success manager. The data tells a story: customers who joined a training session within thirty days renewed at a higher rate than customers who only received the setup email. The sample is still small, so Maya will pull the same view for the next two cohorts before we change the onboarding checklist. Please keep the draft chart inside the planning folder until the finance review is done. diff --git a/evals/fixtures/pairs/false_agency_without.txt b/evals/fixtures/pairs/false_agency_without.txt new file mode 100644 index 0000000..1d9ae8c --- /dev/null +++ b/evals/fixtures/pairs/false_agency_without.txt @@ -0,0 +1 @@ +Our retention dashboard now includes renewals by cohort, upgrade date, and assigned success manager. Customers who joined a training session within thirty days renewed at a higher rate than customers who only received the setup email. The sample is still small, so Maya will pull the same view for the next two cohorts before we change the onboarding checklist. Please keep the draft chart inside the planning folder until the finance review is done. diff --git a/evals/fixtures/pairs/hedge_stack_with.txt b/evals/fixtures/pairs/hedge_stack_with.txt new file mode 100644 index 0000000..901e692 --- /dev/null +++ b/evals/fixtures/pairs/hedge_stack_with.txt @@ -0,0 +1 @@ +The beta build passed smoke tests on the checkout path, account settings, and receipt export. We still need the payments team to review the retry logs because two sandbox cards returned duplicate pending events. The change could potentially reduce support tickets for missing receipts once the retry window drops from six hours to one hour. For now, leave the feature flag on for staff accounts only and send any failed export IDs to the release channel. diff --git a/evals/fixtures/pairs/hedge_stack_without.txt b/evals/fixtures/pairs/hedge_stack_without.txt new file mode 100644 index 0000000..6e194e4 --- /dev/null +++ b/evals/fixtures/pairs/hedge_stack_without.txt @@ -0,0 +1 @@ +The beta build passed smoke tests on the checkout path, account settings, and receipt export. We still need the payments team to review the retry logs because two sandbox cards returned duplicate pending events. The change could reduce support tickets for missing receipts once the retry window drops from six hours to one hour. For now, leave the feature flag on for staff accounts only and send any failed export IDs to the release channel. diff --git a/evals/fixtures/pairs/jargon_leverage_with.txt b/evals/fixtures/pairs/jargon_leverage_with.txt new file mode 100644 index 0000000..63fbe52 --- /dev/null +++ b/evals/fixtures/pairs/jargon_leverage_with.txt @@ -0,0 +1 @@ +The clinic moved appointment reminders into the same system that sends lab notifications. Staff can now leverage the platform to send one confirmation in English or Spanish instead of copying phone numbers into a separate tool. The first week exposed two bad area codes, which the front desk corrected before the afternoon call block. If the next audit is clean, pediatric visits will move into the same workflow in August. diff --git a/evals/fixtures/pairs/jargon_leverage_without.txt b/evals/fixtures/pairs/jargon_leverage_without.txt new file mode 100644 index 0000000..fcd912c --- /dev/null +++ b/evals/fixtures/pairs/jargon_leverage_without.txt @@ -0,0 +1 @@ +The clinic moved appointment reminders into the same system that sends lab notifications. Staff can now use the platform to send one confirmation in English or Spanish instead of copying phone numbers into a separate tool. The first week exposed two bad area codes, which the front desk corrected before the afternoon call block. If the next audit is clean, pediatric visits will move into the same workflow in August. diff --git a/evals/fixtures/pairs/macro_burstiness_with.md b/evals/fixtures/pairs/macro_burstiness_with.md new file mode 100644 index 0000000..991624b --- /dev/null +++ b/evals/fixtures/pairs/macro_burstiness_with.md @@ -0,0 +1,3 @@ +The field test began on Monday morning. Crews checked the east gate before sunrise. Radios worked across the lower road. The tablet map loaded without delay. Drivers logged each stop in the trial route. Dispatch watched the markers from the office. No one reported a missed scan that day. The only issue was a loose charger in truck seven. + +Tuesday followed the same pattern. Crews checked the west gate before sunrise. Radios worked across the upper road. The tablet map loaded without delay. Drivers logged each stop in the trial route. Dispatch watched the markers from the office. No one reported a missed scan that day. The only issue was a cracked mount in truck three. diff --git a/evals/fixtures/pairs/macro_burstiness_without.md b/evals/fixtures/pairs/macro_burstiness_without.md new file mode 100644 index 0000000..d1514c4 --- /dev/null +++ b/evals/fixtures/pairs/macro_burstiness_without.md @@ -0,0 +1,3 @@ +The field test began on Monday before sunrise, when the east gate crew checked radios and loaded the tablet map. Drivers logged each stop on the trial route while dispatch watched the markers from the office. No one reported a missed scan that day, although truck seven needed a replacement charger after lunch. + +Tuesday covered the west gate and the upper road. The radio check took longer because one spare battery was dead, but the route still started on time. Drivers found the new stop order easier to follow than the paper sheet, and dispatch used the live map to answer two customer calls without interrupting the crew. diff --git a/evals/fixtures/pairs/macro_coda_with.md b/evals/fixtures/pairs/macro_coda_with.md new file mode 100644 index 0000000..bcc23da --- /dev/null +++ b/evals/fixtures/pairs/macro_coda_with.md @@ -0,0 +1,5 @@ +The account request pilot opened with a familiar support pattern. Customers filed account requests through chat, agents copied details into the queue, and managers checked the next morning whether the tags were right, although one coordinator still kept a private spreadsheet because the old queue hid deadlines. + +The new intake page keeps the work in one place: customers choose the account, attach the invoice, pick the reason for the request, and send agents a record that already includes priority, deadline, owner, and the last customer reply. Managers get weekly counts without rebuilding a spreadsheet, and the first pilot team cut repeated copying by half even though vendor names still need cleanup. + +The account request should now stay inside the queue from the first customer note to the manager review. That is the support pattern we wanted: one account, one request, one visible owner, and no private notes that turn a simple handoff into detective work. diff --git a/evals/fixtures/pairs/macro_coda_without.md b/evals/fixtures/pairs/macro_coda_without.md new file mode 100644 index 0000000..168807a --- /dev/null +++ b/evals/fixtures/pairs/macro_coda_without.md @@ -0,0 +1,5 @@ +The account request pilot opened with a familiar support pattern. Customers filed account requests through chat, agents copied details into the queue, and managers checked the next morning whether the tags were right, although one coordinator still kept a private spreadsheet because the old queue hid deadlines. + +The new intake page keeps the work in one place: customers choose the account, attach the invoice, pick the reason for the request, and send agents a record that already includes priority, deadline, owner, and the last customer reply. Managers get weekly counts without rebuilding a spreadsheet, and the first pilot team cut repeated copying by half even though vendor names still need cleanup. + +The next release should fix vendor aliases before the rollout expands. Support leads also asked for a daily export so they can compare old and new queues during the overlap week, which matters more than another dashboard because it removes the last reason agents still keep side notes beside the queue. diff --git a/evals/fixtures/pairs/manifest.json b/evals/fixtures/pairs/manifest.json new file mode 100644 index 0000000..67d308f --- /dev/null +++ b/evals/fixtures/pairs/manifest.json @@ -0,0 +1,66 @@ +{ + "throat_clearing": { + "target": "throat_clearing", + "kind": "phrase" + }, + "significance_inflation": { + "target": "significance_inflation", + "kind": "phrase" + }, + "negative_parallelism": { + "target": "negative_parallelism", + "kind": "phrase" + }, + "rhetorical_question": { + "target": "rhetorical_question", + "kind": "phrase" + }, + "hedge_stack": { + "target": "hedge_stack", + "kind": "phrase" + }, + "false_agency": { + "target": "false_agency", + "kind": "phrase" + }, + "jargon_leverage": { + "target": "jargon", + "kind": "phrase" + }, + "reader_addressing": { + "target": "reader_addressing", + "kind": "phrase" + }, + "conclusion_scaffold": { + "target": "conclusion_scaffold", + "kind": "phrase" + }, + "emphasis_crutch": { + "target": "emphasis_crutch", + "kind": "phrase" + }, + "vague_attribution": { + "target": "vague_attribution", + "kind": "phrase" + }, + "participial_closer": { + "target": "superficial_ing", + "kind": "phrase" + }, + "anti_slop_register": { + "target": "anti_slop_register", + "kind": "phrase" + }, + "cliffhanger_fragment": { + "target": "rhetorical_question", + "kind": "phrase" + }, + "macro_coda": { + "target": "conclusion_coda", + "kind": "structure" + }, + "macro_burstiness": { + "target": "sentence_burstiness", + "kind": "structure" + } +} diff --git a/evals/fixtures/pairs/negative_parallelism_with.txt b/evals/fixtures/pairs/negative_parallelism_with.txt new file mode 100644 index 0000000..47e697c --- /dev/null +++ b/evals/fixtures/pairs/negative_parallelism_with.txt @@ -0,0 +1 @@ +The repair program changed after the March backlog review. Customers now get a text when parts arrive, dispatchers can split one visit across two work orders, and technicians close warranty notes from the van. It is not just a scheduling update, it is a clearer handoff between the warehouse, the phone team, and the field crew. The first two pilots cut repeat calls by eleven percent, mostly because customers stopped calling to ask whether a part had been ordered. diff --git a/evals/fixtures/pairs/negative_parallelism_without.txt b/evals/fixtures/pairs/negative_parallelism_without.txt new file mode 100644 index 0000000..489b0ed --- /dev/null +++ b/evals/fixtures/pairs/negative_parallelism_without.txt @@ -0,0 +1 @@ +The repair program changed after the March backlog review. Customers now get a text when parts arrive, dispatchers can split one visit across two work orders, and technicians close warranty notes from the van. The update gives the warehouse, the phone team, and the field crew a clearer handoff. The first two pilots cut repeat calls by eleven percent, mostly because customers stopped calling to ask whether a part had been ordered. diff --git a/evals/fixtures/pairs/participial_closer_with.txt b/evals/fixtures/pairs/participial_closer_with.txt new file mode 100644 index 0000000..3421271 --- /dev/null +++ b/evals/fixtures/pairs/participial_closer_with.txt @@ -0,0 +1 @@ +The museum moved the school entrance to the north lobby during roof work. Buses will unload beside the sculpture garden, teachers will receive color-coded badges, and docents will meet classes under the blue awning, highlighting the new route for visiting groups. Security tested the door alarms on Monday and found no delay in the radio handoff. The temporary map goes live after facilities installs the last sign by the elevator. diff --git a/evals/fixtures/pairs/participial_closer_without.txt b/evals/fixtures/pairs/participial_closer_without.txt new file mode 100644 index 0000000..49c727f --- /dev/null +++ b/evals/fixtures/pairs/participial_closer_without.txt @@ -0,0 +1 @@ +The museum moved the school entrance to the north lobby during roof work. Buses will unload beside the sculpture garden, teachers will receive color-coded badges, and docents will meet classes under the blue awning. The new route is marked for visiting groups. Security tested the door alarms on Monday and found no delay in the radio handoff. The temporary map goes live after facilities installs the last sign by the elevator. diff --git a/evals/fixtures/pairs/reader_addressing_with.txt b/evals/fixtures/pairs/reader_addressing_with.txt new file mode 100644 index 0000000..49de64b --- /dev/null +++ b/evals/fixtures/pairs/reader_addressing_with.txt @@ -0,0 +1 @@ +The workshop is capped at twenty people because everyone needs time on the soldering stations. Whether you're a first-time builder or a repair volunteer, you will start by checking the power supply, then practice replacing a cracked switch on a sample board. The last half hour is reserved for personal devices that fit the bench rules. Bring eye protection if you have it; the lab has loaner glasses for anyone who forgets. diff --git a/evals/fixtures/pairs/reader_addressing_without.txt b/evals/fixtures/pairs/reader_addressing_without.txt new file mode 100644 index 0000000..fc3a879 --- /dev/null +++ b/evals/fixtures/pairs/reader_addressing_without.txt @@ -0,0 +1 @@ +The workshop is capped at twenty people because everyone needs time on the soldering stations. Participants will start by checking the power supply, then practice replacing a cracked switch on a sample board. The last half hour is reserved for personal devices that fit the bench rules. Bring eye protection if you have it; the lab has loaner glasses for anyone who forgets. diff --git a/evals/fixtures/pairs/rhetorical_question_with.txt b/evals/fixtures/pairs/rhetorical_question_with.txt new file mode 100644 index 0000000..65c5a81 --- /dev/null +++ b/evals/fixtures/pairs/rhetorical_question_with.txt @@ -0,0 +1 @@ +The library board approved the Saturday pilot for August and September. Branches in Easton and Mill Park will open at ten, keep the children's desk staffed until two, and log hourly door counts. Why does this matter? The winter survey showed that weekday hours miss parents who work early shifts. If the pilot draws at least 120 visits per Saturday, the board will add the cost to the December budget request. diff --git a/evals/fixtures/pairs/rhetorical_question_without.txt b/evals/fixtures/pairs/rhetorical_question_without.txt new file mode 100644 index 0000000..ec45eb0 --- /dev/null +++ b/evals/fixtures/pairs/rhetorical_question_without.txt @@ -0,0 +1 @@ +The library board approved the Saturday pilot for August and September. Branches in Easton and Mill Park will open at ten, keep the children's desk staffed until two, and log hourly door counts. The winter survey showed that weekday hours miss parents who work early shifts. If the pilot draws at least 120 visits per Saturday, the board will add the cost to the December budget request. diff --git a/evals/fixtures/pairs/significance_inflation_with.txt b/evals/fixtures/pairs/significance_inflation_with.txt new file mode 100644 index 0000000..4d71848 --- /dev/null +++ b/evals/fixtures/pairs/significance_inflation_with.txt @@ -0,0 +1 @@ +The lake association finished its shoreline survey after six weeks of weekend sampling. Volunteers mapped 418 docks, noted erosion on the west bank, and photographed every culvert after the June storms. The report underscores the importance of replacing the two cracked drains before fall rains return. It also gives the board enough detail to apply for the county grant without hiring a consultant. Residents can review the map at the library on Tuesday evening, and comments are due by July 18. diff --git a/evals/fixtures/pairs/significance_inflation_without.txt b/evals/fixtures/pairs/significance_inflation_without.txt new file mode 100644 index 0000000..7977b77 --- /dev/null +++ b/evals/fixtures/pairs/significance_inflation_without.txt @@ -0,0 +1 @@ +The lake association finished its shoreline survey after six weeks of weekend sampling. Volunteers mapped 418 docks, noted erosion on the west bank, and photographed every culvert after the June storms. The report says the two cracked drains should be replaced before fall rains return. It also gives the board enough detail to apply for the county grant without hiring a consultant. Residents can review the map at the library on Tuesday evening, and comments are due by July 18. diff --git a/evals/fixtures/pairs/throat_clearing_with.txt b/evals/fixtures/pairs/throat_clearing_with.txt new file mode 100644 index 0000000..c42e065 --- /dev/null +++ b/evals/fixtures/pairs/throat_clearing_with.txt @@ -0,0 +1 @@ +The Friday note to store managers covered the same three changes we discussed on the regional call: produce signs move closer to the front doors, pickup orders get a separate shelf, and bakery waste is logged before noon. Here's the thing: none of these changes requires new equipment or extra staffing. The teams that tested the layout in Mesa and Boise finished the reset in under two hours, then used the old endcap space for seasonal drinks. Please finish the moves before next Wednesday's opening shift so the weekend report uses the new categories. diff --git a/evals/fixtures/pairs/throat_clearing_without.txt b/evals/fixtures/pairs/throat_clearing_without.txt new file mode 100644 index 0000000..9ad00c8 --- /dev/null +++ b/evals/fixtures/pairs/throat_clearing_without.txt @@ -0,0 +1 @@ +The Friday note to store managers covered the same three changes we discussed on the regional call: produce signs move closer to the front doors, pickup orders get a separate shelf, and bakery waste is logged before noon. None of these changes requires new equipment or extra staffing. The teams that tested the layout in Mesa and Boise finished the reset in under two hours, then used the old endcap space for seasonal drinks. Please finish the moves before next Wednesday's opening shift so the weekend report uses the new categories. diff --git a/evals/fixtures/pairs/vague_attribution_with.txt b/evals/fixtures/pairs/vague_attribution_with.txt new file mode 100644 index 0000000..f3c1196 --- /dev/null +++ b/evals/fixtures/pairs/vague_attribution_with.txt @@ -0,0 +1 @@ +The housing brief compares last quarter's permit count with the same period in 2025. Duplex applications rose in the south district, while large apartment filings stayed flat near the transit corridor. Analysts predict rents will level off by winter, but the memo does not name the forecast or explain its vacancy assumptions. Before publication, add the source table and separate confirmed permits from projects still waiting for design review. diff --git a/evals/fixtures/pairs/vague_attribution_without.txt b/evals/fixtures/pairs/vague_attribution_without.txt new file mode 100644 index 0000000..4158e18 --- /dev/null +++ b/evals/fixtures/pairs/vague_attribution_without.txt @@ -0,0 +1 @@ +The housing brief compares last quarter's permit count with the same period in 2025. Duplex applications rose in the south district, while large apartment filings stayed flat near the transit corridor. The city economist expects rents to level off by winter, but the memo should explain the vacancy assumptions. Before publication, add the source table and separate confirmed permits from projects still waiting for design review. diff --git a/evals/fixtures/parity/canned_responses.json b/evals/fixtures/parity/canned_responses.json new file mode 100644 index 0000000..de9cb38 --- /dev/null +++ b/evals/fixtures/parity/canned_responses.json @@ -0,0 +1,64 @@ +{ + "_note": "Pre-canned model responses for evals/run_model_parity.py --dry-run. No network. strong-ref and cheap-good are correct on both tasks; cheap-bad fails deliberately (low recall + false findings on A, scanner/minimality/preservation failures on B). Used by PARITY-01/02/03 eval rows.", + "models": [ + {"name": "strong-ref", "kind": "canned"}, + {"name": "cheap-good", "kind": "canned"}, + {"name": "cheap-bad", "kind": "canned"} + ], + "responses": { + "strong-ref": { + "A": { + "A1": "[{\"span\":\"Here's the thing\",\"rule\":\"throat_clearing\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"opener\"},{\"span\":\"underscores the importance of\",\"rule\":\"significance_inflation\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"inflation\"}]", + "A2": "```json\n[{\"span\":\"seamless\",\"rule\":\"academic_excess\",\"pack\":\"pack-phrases-core\",\"severity\":\"soft\",\"note\":\"polish\"},{\"span\":\"cross-functional synergy\",\"rule\":\"jargon\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"jargon\"}]\n```", + "A3": "[{\"span\":\"Not the tool. The team.\",\"rule\":\"anti_slop_register\",\"pack\":\"pack-voice\",\"severity\":\"hard\",\"note\":\"fragment\"},{\"span\":\"The result: alignment\",\"rule\":\"colon_reveal\",\"pack\":\"pack-voice\",\"severity\":\"soft\",\"note\":\"reveal\"}]", + "A4": "[{\"span\":\"Emit tokens. Ship bytes.\",\"rule\":\"imperative_slogan\",\"pack\":\"pack-voice\",\"severity\":\"hard\",\"note\":\"slogan\"},{\"span\":\"things that matter\",\"rule\":\"fragment_template\",\"pack\":\"pack-voice\",\"severity\":\"soft\",\"note\":\"template\"}]", + "A5": "[{\"span\":\"never store secrets\",\"rule\":\"security_absolute\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"},{\"span\":\"may cause drowsiness in some patients\",\"rule\":\"medical_uncertainty\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"}]", + "A6": "[{\"span\":\"arguably does not rise to gross negligence\",\"rule\":\"legal_negation_scope\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"},{\"span\":\"unless the party consents\",\"rule\":\"legal_negation_scope\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"}]" + }, + "B": { + "B1": "The", + "B2": "reliable", + "B3": "is clear", + "B4": "because we added caching", + "B5": "Users", + "B6": "The" + } + }, + "cheap-good": { + "A": { + "A1": "[{\"span\":\"Here's the thing\",\"rule\":\"throat_clearing\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"opener\"},{\"span\":\"underscores the importance\",\"rule\":\"significance_inflation\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"inflation\"}]", + "A2": "[{\"span\":\"seamless\",\"rule\":\"academic_excess\",\"pack\":\"pack-phrases-core\",\"severity\":\"soft\",\"note\":\"polish\"},{\"span\":\"synergy\",\"rule\":\"jargon\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"jargon\"}]", + "A3": "[{\"span\":\"Not the tool. The team\",\"rule\":\"anti_slop_register\",\"pack\":\"pack-voice\",\"severity\":\"hard\",\"note\":\"fragment\"},{\"span\":\"result: alignment\",\"rule\":\"colon_reveal\",\"pack\":\"pack-voice\",\"severity\":\"soft\",\"note\":\"reveal\"}]", + "A4": "[{\"span\":\"Emit tokens. Ship bytes\",\"rule\":\"imperative_slogan\",\"pack\":\"pack-voice\",\"severity\":\"hard\",\"note\":\"slogan\"},{\"span\":\"build things that matter\",\"rule\":\"fragment_template\",\"pack\":\"pack-voice\",\"severity\":\"soft\",\"note\":\"template\"}]", + "A5": "{\"findings\":[{\"span\":\"never\",\"rule\":\"security_absolute\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"},{\"span\":\"some patients\",\"rule\":\"medical_uncertainty\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"}]}", + "A6": "[{\"span\":\"arguably\",\"rule\":\"legal_negation_scope\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"},{\"span\":\"unless\",\"rule\":\"legal_negation_scope\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"preserve\"}]" + }, + "B": { + "B1": "The", + "B2": "direct", + "B3": "is clear", + "B4": "because we cache aggressively", + "B5": "Users", + "B6": "The" + } + }, + "cheap-bad": { + "A": { + "A1": "[{\"span\":\"quarterly result zzz\",\"rule\":\"throat_clearing\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"wrong\"}]", + "A2": "[{\"span\":\"across every team zzz\",\"rule\":\"jargon\",\"pack\":\"pack-phrases-core\",\"severity\":\"hard\",\"note\":\"wrong\"}]", + "A3": "[{\"span\":\"whole org zzz\",\"rule\":\"colon_reveal\",\"pack\":\"pack-voice\",\"severity\":\"soft\",\"note\":\"wrong\"}]", + "A4": "[{\"span\":\"the whole job zzz\",\"rule\":\"imperative_slogan\",\"pack\":\"pack-voice\",\"severity\":\"soft\",\"note\":\"wrong\"}]", + "A5": "[{\"span\":\"client-side zzz\",\"rule\":\"security_absolute\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"wrong\"}]", + "A6": "[{\"span\":\"in writing zzz\",\"rule\":\"legal_negation_scope\",\"pack\":\"pack-register-guards\",\"severity\":\"hard\",\"note\":\"wrong\"}]" + }, + "B": { + "B1": "Here's the thing: the", + "B2": "seamless and vibrant", + "B3": "speaks for itself", + "B4": "which, as any careful observer can plainly and obviously see, truly matters a great deal in practice", + "B5": "Let's dive in: users", + "B6": "In conclusion, the" + } + } + } +} diff --git a/evals/fixtures/parity/models_live.json b/evals/fixtures/parity/models_live.json new file mode 100644 index 0000000..f7ec05c --- /dev/null +++ b/evals/fixtures/parity/models_live.json @@ -0,0 +1,7 @@ +{ + "_note": "A live-looking matrix used ONLY by PARITY-03 with --no-network to prove graceful degradation. Never run live from the suite.", + "models": [ + {"name": "gpt-4o-mini", "kind": "openrouter", "model_id": "openai/gpt-4o-mini"}, + {"name": "claude-haiku", "kind": "claude-cli", "model_id": "claude-3-5-haiku-latest"} + ] +} diff --git a/evals/fixtures/silhouette/corpus/ai/01_linkedin.txt b/evals/fixtures/silhouette/corpus/ai/01_linkedin.txt new file mode 100644 index 0000000..b8aa6cd --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/01_linkedin.txt @@ -0,0 +1,27 @@ +I want to share something that's been on my mind lately. 🚀 + +Last week, I had a conversation that completely changed how I think about leadership. + +Here's the thing: too many of us believe that leadership is about having all the answers. But that couldn't be further from the truth. + +Real leadership isn't about control. It's about trust, empathy, and empowerment. + +Let me break it down: + +→ Great leaders listen more than they speak. +→ Great leaders admit when they're wrong. +→ Great leaders lift others up instead of holding them down. + +When I started my career, I thought success meant being the smartest person in the room. I was wrong. Success is about surrounding yourself with brilliant people and getting out of their way. + +The best teams I've ever worked with weren't built on fear. They were built on psychological safety, mutual respect, and a shared sense of purpose. + +So here's my challenge to you: This week, ask your team one simple question. "How can I support you better?" + +You might be surprised by what you learn. + +Leadership isn't a title. It's a choice you make every single day. + +What's one leadership lesson that changed your perspective? Drop it in the comments below. 👇 + +#Leadership #Growth #Management #TeamBuilding diff --git a/evals/fixtures/silhouette/corpus/ai/02_blog_intro.txt b/evals/fixtures/silhouette/corpus/ai/02_blog_intro.txt new file mode 100644 index 0000000..e3c14d1 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/02_blog_intro.txt @@ -0,0 +1,15 @@ +In today's fast-paced digital landscape, remote work has become more than just a trend. It's a fundamental shift in how we approach our professional lives. + +Whether you're a seasoned remote worker or just starting your journey, one thing is clear: the way we work has changed forever. And with that change comes both incredible opportunities and unique challenges. + +But what does it really take to thrive in a remote environment? Is it about having the right tools? The perfect home office setup? Or is there something deeper at play? + +In this post, we'll explore the essential strategies that separate successful remote workers from those who struggle. We'll dive into productivity techniques, communication best practices, and the mindset shifts that can transform your remote experience. + +From setting boundaries to staying connected with your team, we'll cover everything you need to know to not just survive, but truly flourish in the world of remote work. + +The truth is, remote work isn't just about where you work. It's about how you work, how you communicate, and how you take care of yourself along the way. + +So grab a cup of coffee, settle into your favorite workspace, and let's dive in. By the end of this post, you'll have a practical roadmap for making remote work work for you. + +Ready to unlock your full potential? Let's get started. diff --git a/evals/fixtures/silhouette/corpus/ai/03_product_announcement.txt b/evals/fixtures/silhouette/corpus/ai/03_product_announcement.txt new file mode 100644 index 0000000..4541473 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/03_product_announcement.txt @@ -0,0 +1,33 @@ +# Introducing FlowSync: The Future of Team Collaboration + +We're thrilled to announce the launch of FlowSync, our most ambitious product yet. + +At FlowSync, we believe that great work happens when teams are truly connected. That's why we've spent the last two years reimagining what collaboration software can be. + +## Why We Built FlowSync + +In today's world, teams are more distributed than ever. But existing tools weren't designed for this new reality. They're clunky, fragmented, and frankly, they get in the way of great work. + +We knew there had to be a better way. + +## What Makes FlowSync Different + +FlowSync isn't just another collaboration tool. It's a complete platform designed to help your team do their best work. + +Here's what you can expect: + +- **Seamless Integration**: Connect all your favorite tools in one place. +- **Real-Time Sync**: Stay updated with instant notifications and live editing. +- **Smart Automation**: Let AI handle the busywork so you can focus on what matters. + +## Built for Teams of All Sizes + +Whether you're a scrappy startup or a global enterprise, FlowSync scales with you. Our flexible pricing means you only pay for what you need. + +## Available Today + +FlowSync is available now for teams around the world. Sign up for a free 14-day trial and experience the future of collaboration for yourself. + +We can't wait to see what you'll build with FlowSync. This is just the beginning, and we're excited to have you along for the journey. + +The future of work is here. And it starts with FlowSync. diff --git a/evals/fixtures/silhouette/corpus/ai/04_howto_guide.txt b/evals/fixtures/silhouette/corpus/ai/04_howto_guide.txt new file mode 100644 index 0000000..2e889ed --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/04_howto_guide.txt @@ -0,0 +1,31 @@ +# How to Start a Morning Routine That Actually Sticks + +Building a morning routine can transform your entire day. But let's be honest: sticking to one is easier said than done. In this guide, we'll walk you through everything you need to know to create a morning routine that lasts. + +## Why a Morning Routine Matters + +Your morning sets the tone for the rest of your day. A well-designed routine can boost your productivity, improve your mood, and reduce stress. Studies have shown that people with consistent morning habits tend to be more successful and happier overall. + +## Step 1: Wake Up at a Consistent Time + +The foundation of any good morning routine is consistency. Try to wake up at the same time every day, even on weekends. This helps regulate your body's internal clock and makes waking up easier over time. + +## Step 2: Hydrate First Thing + +After hours of sleep, your body is dehydrated. Start your day by drinking a glass of water. This simple habit kickstarts your metabolism and helps you feel more alert. + +## Step 3: Move Your Body + +You don't need an intense workout. Even a few minutes of stretching or a short walk can get your blood flowing and energize you for the day ahead. + +## Step 4: Practice Mindfulness + +Take a few moments to center yourself. Whether it's meditation, journaling, or simply sitting quietly, this practice can help you approach the day with clarity and calm. + +## Step 5: Fuel Your Body + +A nutritious breakfast provides the energy you need to tackle your morning. Focus on whole foods that will keep you satisfied and focused. + +## Final Thoughts + +Remember, building a morning routine is a journey, not a destination. Start small, be patient with yourself, and adjust as needed. With time and consistency, these habits will become second nature. Your future self will thank you. diff --git a/evals/fixtures/silhouette/corpus/ai/05_essay.txt b/evals/fixtures/silhouette/corpus/ai/05_essay.txt new file mode 100644 index 0000000..3dfb9b9 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/05_essay.txt @@ -0,0 +1,13 @@ +# The Impact of Social Media on Modern Society + +Social media has fundamentally transformed the way we communicate, connect, and consume information. In just two decades, platforms like Facebook, Twitter, and Instagram have gone from novel curiosities to central pillars of modern life. But this transformation has not come without consequences. To fully understand the impact of social media on society, we must examine both its benefits and its drawbacks. + +On one hand, social media has democratized communication in unprecedented ways. It has given a voice to the voiceless, enabling marginalized communities to organize, share their stories, and advocate for change. Movements like #MeToo and Black Lives Matter gained momentum through the power of social media, demonstrating its potential as a force for social good. Moreover, social media has made it easier than ever to stay connected with friends and family across the globe. + +On the other hand, social media has also introduced significant challenges. Perhaps most concerning is its impact on mental health. Studies have linked heavy social media use to increased rates of anxiety, depression, and loneliness, particularly among young people. The constant comparison to curated, idealized versions of others' lives can erode self-esteem and foster feelings of inadequacy. + +Furthermore, social media has become a breeding ground for misinformation. False information can spread rapidly, unchecked by traditional gatekeepers. This has serious implications for public discourse and democracy itself. + +Additionally, concerns about privacy and data security continue to grow. These platforms collect vast amounts of personal data, raising important questions about surveillance and consent. + +In conclusion, social media is neither wholly good nor wholly bad. It is a powerful tool that reflects the best and worst of humanity. As we move forward, it is essential that we approach social media with awareness and intention. Only then can we harness its benefits while mitigating its harms. Ultimately, the future of social media depends on the choices we make today. diff --git a/evals/fixtures/silhouette/corpus/ai/06_readme.txt b/evals/fixtures/silhouette/corpus/ai/06_readme.txt new file mode 100644 index 0000000..db2f68a --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/06_readme.txt @@ -0,0 +1,49 @@ +# TaskMaster + +TaskMaster is a powerful, lightweight task management library for Python. Built with simplicity and performance in mind, it helps developers manage complex workflows with ease. + +## Features + +TaskMaster comes packed with everything you need to build robust task pipelines: + +- **Simple API**: Get started in minutes with an intuitive, easy-to-use interface. +- **High Performance**: Optimized for speed, even with thousands of tasks. +- **Flexible**: Customize task behavior to fit your unique needs. +- **Well-Documented**: Comprehensive documentation to guide you every step of the way. + +## Installation + +Getting started with TaskMaster is easy. Simply install it via pip: + +``` +pip install taskmaster +``` + +## Quick Start + +Here's a simple example to get you up and running: + +```python +from taskmaster import Task + +task = Task("my_task") +task.run() +``` + +That's it! You've just created and run your first task. + +## Why TaskMaster? + +In the world of task management, there are many options to choose from. So why choose TaskMaster? The answer is simple: TaskMaster strikes the perfect balance between power and simplicity. Whether you're building a small script or a large-scale application, TaskMaster has you covered. + +## Contributing + +We welcome contributions from the community! Whether it's a bug fix, a new feature, or an improvement to the documentation, every contribution matters. Please read our contributing guidelines to get started. + +## License + +TaskMaster is released under the MIT License. Feel free to use it in your projects, both personal and commercial. + +## Conclusion + +TaskMaster is more than just a task management library. It's a tool designed to make your life easier. Give it a try today and see the difference for yourself. diff --git a/evals/fixtures/silhouette/corpus/ai/07_email.txt b/evals/fixtures/silhouette/corpus/ai/07_email.txt new file mode 100644 index 0000000..b92afe8 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/07_email.txt @@ -0,0 +1,26 @@ +Subject: Exciting Updates and Next Steps for Our Q3 Initiative + +Hi Team, + +I hope this email finds you well. + +I wanted to take a moment to reach out and share some exciting updates regarding our Q3 initiative. As you know, we've been working hard to bring this project to life, and I'm thrilled to say that we've made tremendous progress. + +First, I want to thank each and every one of you for your dedication and hard work. Your contributions have been invaluable, and this project simply wouldn't be possible without your commitment. + +Here's where we stand: + +- **Phase 1** is now complete, and the results have exceeded our expectations. +- **Phase 2** is currently underway, with the team making great strides. +- **Phase 3** is on track to begin next month, as planned. + +Moving forward, there are a few key things I'd like us to focus on. First, let's ensure we maintain clear and open communication across all teams. Second, let's continue to prioritize quality in everything we do. And finally, let's not lose sight of our ultimate goal: delivering exceptional value to our customers. + +I'd love to hear your thoughts and feedback. Please don't hesitate to reach out if you have any questions or concerns. + +Let's keep up the amazing work. Together, I'm confident we can achieve great things. + +Looking forward to our continued success. + +Best regards, +Sarah diff --git a/evals/fixtures/silhouette/corpus/ai/08_listicle.txt b/evals/fixtures/silhouette/corpus/ai/08_listicle.txt new file mode 100644 index 0000000..70f54d1 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/08_listicle.txt @@ -0,0 +1,35 @@ +# 7 Habits of Highly Productive People + +Ever wonder what sets highly productive people apart? It's not luck, and it's not some secret talent. It's their habits. Here are seven habits that can help you take your productivity to the next level. + +## 1. They Prioritize Ruthlessly + +Productive people know that not all tasks are created equal. They focus on what truly matters and aren't afraid to say no to distractions. By identifying their most important tasks, they make sure their energy goes where it counts. + +## 2. They Wake Up Early + +There's something magical about the early morning hours. Highly productive people often start their day before everyone else, giving them quiet, uninterrupted time to focus on their goals. + +## 3. They Take Regular Breaks + +It might sound counterintuitive, but taking breaks actually boosts productivity. Productive people understand the importance of rest and recovery. They work in focused bursts, then step away to recharge. + +## 4. They Embrace Single-Tasking + +Multitasking is a myth. The most productive people focus on one task at a time, giving it their full attention. This leads to higher quality work and fewer mistakes. + +## 5. They Plan Their Day + +Failing to plan is planning to fail. Productive people start each day with a clear plan of action. They know exactly what they need to accomplish and how they're going to do it. + +## 6. They Minimize Distractions + +In a world full of notifications and interruptions, focus is a superpower. Productive people create environments that support deep work, whether that means turning off their phone or finding a quiet space. + +## 7. They Never Stop Learning + +The most productive people are lifelong learners. They're always looking for ways to improve, grow, and expand their skills. This commitment to growth keeps them ahead of the curve. + +## Final Thoughts + +Productivity isn't about doing more. It's about doing what matters. By adopting these seven habits, you can work smarter, not harder, and achieve more than you ever thought possible. So which habit will you start with today? diff --git a/evals/fixtures/silhouette/corpus/ai/09_exec_summary.txt b/evals/fixtures/silhouette/corpus/ai/09_exec_summary.txt new file mode 100644 index 0000000..ce96c59 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/09_exec_summary.txt @@ -0,0 +1,35 @@ +# Executive Summary: Market Expansion Strategy + +## Overview + +This document outlines our proposed strategy for expanding into the Southeast Asian market. Our analysis indicates a significant opportunity for growth, driven by favorable market conditions and strong consumer demand. + +## Key Findings + +Our research has revealed several important insights: + +- **Growing Market**: The Southeast Asian market is experiencing rapid growth, with a projected annual increase of 12% over the next five years. +- **Untapped Potential**: Current competition in our target segment remains limited, presenting a clear opportunity for early entry. +- **Favorable Demographics**: A young, tech-savvy population is driving demand for our products and services. + +## Strategic Recommendations + +Based on our findings, we recommend a phased approach to market entry: + +First, we should establish a strong local presence through strategic partnerships. This will allow us to navigate regulatory complexities and build brand credibility. + +Second, we should invest in localized marketing campaigns tailored to the unique preferences of each market. + +Finally, we should prioritize building a robust supply chain to ensure we can meet growing demand. + +## Financial Projections + +Our financial models project a positive return on investment within 18 months. We anticipate initial revenues of $5 million in year one, scaling to $25 million by year three. + +## Risks and Considerations + +While the opportunity is significant, it is not without risks. Regulatory challenges, cultural differences, and competitive dynamics all require careful consideration. However, we believe these risks are manageable with proper planning and execution. + +## Conclusion + +The Southeast Asian market represents a compelling opportunity for growth. By executing this strategy thoughtfully and deliberately, we are confident that we can establish a strong foothold and drive substantial long-term value. We recommend moving forward with this initiative and look forward to discussing next steps. diff --git a/evals/fixtures/silhouette/corpus/ai/10_personal_story.txt b/evals/fixtures/silhouette/corpus/ai/10_personal_story.txt new file mode 100644 index 0000000..4d77220 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/10_personal_story.txt @@ -0,0 +1,23 @@ +Three years ago, I hit rock bottom. + +I had just lost my job, my relationship was falling apart, and I felt completely lost. I remember sitting in my car in an empty parking lot, wondering how everything had gone so wrong. + +But looking back now, I realize that moment was the beginning of something beautiful. + +You see, sometimes we have to break down before we can break through. + +In the weeks that followed, I made a decision. I decided that I wasn't going to let my circumstances define me. I decided to take control of my life. + +It wasn't easy. There were days when I wanted to give up. There were nights when I lay awake, consumed by doubt and fear. But I kept going. One step at a time, one day at a time. + +I started small. I began journaling every morning. I started exercising. I reached out to old friends and rebuilt connections I had let slip away. + +Slowly but surely, things began to change. Not overnight, but gradually. I found a new job that aligned with my values. I met people who inspired me. I rediscovered my passion for life. + +Today, I'm in a place I never thought possible. And I'm grateful for every single step of the journey, even the painful ones. + +Because here's what I learned: our darkest moments often contain the seeds of our greatest transformations. + +So if you're going through a difficult time right now, I want you to know something. This isn't the end of your story. It's just a chapter. And the best chapters are often the ones we least expect. + +Keep going. Your breakthrough is closer than you think. diff --git a/evals/fixtures/silhouette/corpus/ai/11_technical_explainer.txt b/evals/fixtures/silhouette/corpus/ai/11_technical_explainer.txt new file mode 100644 index 0000000..d744f6d --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/11_technical_explainer.txt @@ -0,0 +1,35 @@ +# Understanding How Neural Networks Work + +Neural networks are at the heart of modern artificial intelligence. But how do they actually work? In this article, we'll break down the fundamentals in a way that's easy to understand. + +## What Is a Neural Network? + +At its core, a neural network is a computational system inspired by the human brain. Just as our brains are made up of interconnected neurons, a neural network consists of interconnected nodes, or "neurons," organized into layers. + +## The Three Types of Layers + +Every neural network is made up of three main types of layers: + +- **Input Layer**: This is where data enters the network. +- **Hidden Layers**: These layers perform the bulk of the computation. +- **Output Layer**: This is where the network produces its final result. + +## How Information Flows + +When data enters the network, it flows from the input layer through the hidden layers to the output layer. This process is called forward propagation. + +Each connection between neurons has a weight, which determines how much influence one neuron has on another. As data passes through the network, these weights are applied, and the network gradually transforms the input into a meaningful output. + +## Learning Through Training + +So how does a neural network learn? The answer lies in a process called backpropagation. + +During training, the network makes predictions and compares them to the correct answers. When it makes a mistake, it adjusts its weights to reduce the error. Over time, through thousands or even millions of iterations, the network becomes increasingly accurate. + +## Why Neural Networks Are So Powerful + +Neural networks excel at finding patterns in complex data. They can recognize faces, understand language, and even generate art. This versatility is what makes them such a powerful tool in the world of AI. + +## Conclusion + +While neural networks may seem complex at first, the underlying concepts are surprisingly intuitive. By understanding how these systems process information and learn from data, we can better appreciate the incredible technology that's shaping our world. As AI continues to evolve, neural networks will undoubtedly play an increasingly important role in our lives. diff --git a/evals/fixtures/silhouette/corpus/ai/12_news_summary.txt b/evals/fixtures/silhouette/corpus/ai/12_news_summary.txt new file mode 100644 index 0000000..0ff5850 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/ai/12_news_summary.txt @@ -0,0 +1,27 @@ +# Tech Giant Unveils Ambitious Climate Plan + +In a move that signals a growing commitment to sustainability, a leading technology company announced a sweeping new climate initiative on Tuesday. + +The plan, which aims to achieve carbon neutrality by 2030, represents one of the most ambitious environmental commitments in the tech industry to date. + +## Key Details + +The initiative includes several major components. First, the company plans to transition all of its data centers to renewable energy. Second, it will invest heavily in carbon capture technology. And third, it has pledged to work with suppliers to reduce emissions across its entire supply chain. + +"This is a pivotal moment for our company and for the planet," the company's CEO said in a statement. "We recognize that we have a responsibility to lead by example." + +## Industry Reaction + +The announcement has been met with cautious optimism from environmental groups. Many have praised the company's ambition while emphasizing the importance of follow-through. + +"It's encouraging to see a company of this scale make such a bold commitment," said one climate advocate. "But the real test will be in the execution." + +Others have raised questions about whether the goals are achievable within the proposed timeline. + +## What Comes Next + +The company says it will release annual progress reports to ensure transparency and accountability. It has also committed to investing $2 billion in climate-related initiatives over the next decade. + +As the world grapples with the growing threat of climate change, all eyes will be on whether this tech giant can deliver on its promises. If successful, the initiative could set a new standard for corporate environmental responsibility and inspire other companies to follow suit. + +Only time will tell whether these ambitious goals will become a reality. But for now, the announcement marks a significant step forward in the fight against climate change. diff --git a/evals/fixtures/silhouette/corpus/human/01_linkedin.txt b/evals/fixtures/silhouette/corpus/human/01_linkedin.txt new file mode 100644 index 0000000..5a99b73 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/01_linkedin.txt @@ -0,0 +1,9 @@ +We shipped the migration Friday at 2am and it broke in a way none of us predicted. + +The short version: our "zero downtime" cutover had eleven minutes of downtime because a Terraform apply reordered the security group rules and locked the read replicas out of the primary. Classic. I'd tested the failover twice in staging. Staging doesn't have the same VPC peering setup, so the test proved nothing. + +Anyway. What actually saved us was that Priya had, on her own initiative back in March, written a runbook nobody asked for. It had the exact rollback command. We ran it, waited, watched the dashboards go green. Total incident: 34 minutes. + +I keep thinking about how the thing that mattered wasn't the fancy migration plan with the Gantt chart. It was one engineer writing a doc during a slow week. We don't reward that. Our promo packets ask for "impact" and "scope" and there is no line item for "wrote the boring document that later prevented a Sev1." + +I don't have a tidy lesson here. Maybe just: hire people who write things down, and then actually read the things. diff --git a/evals/fixtures/silhouette/corpus/human/02_blog_intro.txt b/evals/fixtures/silhouette/corpus/human/02_blog_intro.txt new file mode 100644 index 0000000..5283841 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/02_blog_intro.txt @@ -0,0 +1,5 @@ +I bought a sourdough starter from a woman named Deb off Craigslist in 2019 for eight dollars. She met me in a Trader Joe's parking lot in Glendale and handed it over in a mason jar with a rubber band around a piece of parchment. It smelled like green apples and paint thinner. She said it had been going since 1994. + +The starter is now dead. I killed it in the second week of the pandemic by feeding it bread flour I'd panic-bought that turned out to be self-rising. Self-rising flour has baking powder and salt in it. Salt, in quantity, murders the yeast. I did not know this. Nobody tells you this. + +This post is about what I learned rebuilding a starter from scratch, twice, and why almost every recipe you'll find online is subtly wrong about hydration ratios. If you just want the schedule, skip to the part with the table. But the ratios are where people go wrong, and the ratios are boring, which is exactly why they don't get explained. diff --git a/evals/fixtures/silhouette/corpus/human/03_howto.txt b/evals/fixtures/silhouette/corpus/human/03_howto.txt new file mode 100644 index 0000000..b37ab9b --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/03_howto.txt @@ -0,0 +1,13 @@ +# Replacing the thermostat on a 2009 Subaru Forester + +You need a 10mm socket, a 12mm socket, needle-nose pliers, and about a gallon of coolant. Also a drain pan. Also patience, because the lower radiator hose clamp is in a spot the engineers clearly designed while angry. + +Jack up the front, pull the plastic splash guard (six of those push-clip things, they will break, buy extras). Drain the radiator from the petcock on the driver's side, bottom corner. It drains slow. Go make coffee. + +The thermostat housing is on the lower left of the engine block, follow the lower radiator hose to where it meets the block, that's it. Two 10mm bolts. Here's the part nobody mentions: the bottom bolt is behind the hose, and you cannot get a normal socket on it. I used a stubby wrench and turned it a sixth of a turn at a time for twenty minutes. + +Old thermostat comes out with the O-ring. Note which way the spring faces. Seriously, take a photo, because if you put it in backwards the car overheats and you get to do this whole thing again, which I know because I did. + +New one in, spring toward the engine. Torque the bolts to like 7 ft-lbs, they're tiny, don't gorilla them or you'll strip the aluminum. Refill coolant, run the engine with the cap off until the thermostat opens and the level drops, top off, done. + +I am not going to tell you to consult a professional. You already decided you weren't going to. diff --git a/evals/fixtures/silhouette/corpus/human/04_essay.txt b/evals/fixtures/silhouette/corpus/human/04_essay.txt new file mode 100644 index 0000000..3cfcd5d --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/04_essay.txt @@ -0,0 +1,9 @@ +The city put in a bike lane on Fulton last year and everyone I know has an opinion about it, which is how I know it's working. + +The argument against is that it removed a lane of car traffic and now the evening backup stretches past the hardware store. This is true. I've sat in it. The argument for is that four people used to get hit on that stretch every year, on average, going back a decade the city has the data, and this year so far the number is zero. That's also true. + +What interests me isn't which side is right. It's that both numbers are real and they point in opposite directions and no amount of arguing dissolves the tradeoff. You are trading somebody's twelve minutes for somebody else's collarbone. Reasonable people weight those differently. The people who bike weight them one way and the people who drive weight them another and, surprise, most people mostly do one or the other. + +I drive. I'll admit the lane annoys me twice a day. I also walked past the little ghost bike somebody chained to the signpost at Fulton and 9th, painted white, with a laminated photo of a kid who was nineteen. So I don't say much at the neighborhood meetings anymore. The math got personal and I lost my taste for winning the argument. + +Cities are just a pile of these. We pretend policy is about finding the right answer. Mostly it's about deciding whose inconvenience is acceptable, and then not looking too hard at the people on the other side of the trade. diff --git a/evals/fixtures/silhouette/corpus/human/05_readme.txt b/evals/fixtures/silhouette/corpus/human/05_readme.txt new file mode 100644 index 0000000..10f981e --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/05_readme.txt @@ -0,0 +1,33 @@ +# qgrep + +Grep, but it understands quotes. That's the whole pitch. Regular grep will happily match inside string literals and comments, which is fine until you're trying to find every actual call to `delete_user` and half your hits are in a changelog. + +## install + +``` +go install github.com/mmn/qgrep@latest +``` + +Needs Go 1.21+. There's no Homebrew formula yet. I keep meaning to. + +## usage + +``` +qgrep 'delete_user' ./src +``` + +By default it skips matches inside string literals and line comments for C-family, Python, Go, and Ruby. Everything else it treats as plain text and falls back to normal grep behavior, so don't expect magic in your COBOL. + +Flags: + +- `-s` also search inside strings (i.e. give up and act like grep) +- `-c` also search inside comments +- `--lang=X` force a language instead of guessing from extension + +The language guessing is dumb. It looks at the file extension and nothing else. If you have `.h` files that are actually C++ it'll parse them as C and occasionally get a `<` wrong. Patches welcome, I don't write much C++ anymore so I haven't cared enough to fix it. + +## why + +I wrote this on a Sunday because I was annoyed. It is not battle-tested. It has one contributor (me) and about 60% test coverage, all of it on the tokenizer because that's the part that breaks. If you hit a parsing bug, open an issue with the smallest file that reproduces it and I'll usually get to it within a week or two. + +No license file yet. Assume MIT, I'll add it properly when I'm not lazy. diff --git a/evals/fixtures/silhouette/corpus/human/06_email.txt b/evals/fixtures/silhouette/corpus/human/06_email.txt new file mode 100644 index 0000000..3e0c7c6 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/06_email.txt @@ -0,0 +1,15 @@ +Subject: re: the Henderson numbers + +Dave, + +Short answer: don't send those to the client yet. + +The Q2 figure in row 14 is double-counting the November refund. I caught it because the total didn't match what I had in my own tracker and I spent an embarrassing hour last night figuring out why. It's the refund. Maria processed it against two cost centers back in the reorg and nobody reconciled it. + +Real number is 418K, not 441K. Which, annoyingly, is under the threshold we told them we'd hit, so we should get ahead of that instead of letting them find it. + +Can you and I grab fifteen minutes before the 11am? I don't want to walk into that meeting improvising. I'm free at 9:30 or basically any time after 4 today. + +Also unrelated but the printer on 3 is out of toner again, I already emailed facilities so you don't have to. + +Matt diff --git a/evals/fixtures/silhouette/corpus/human/07_personal_story.txt b/evals/fixtures/silhouette/corpus/human/07_personal_story.txt new file mode 100644 index 0000000..2365999 --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/07_personal_story.txt @@ -0,0 +1,9 @@ +My grandfather kept bees and I was terrified of them until I was about eleven. + +He never made a thing of it. He'd just hand me the smoker and let me stand ten feet back and eventually five feet back and one summer I was standing right at the hive in one of his old veils that smelled like pipe tobacco, holding a frame heavy with honey and thirty thousand bees, and I wasn't scared anymore. I don't remember the day it flipped. There wasn't one. That's the thing about it. + +He died when I was in college. The hives went to a neighbor because nobody in the family knew what to do with them and my grandmother wasn't going to keep them going at eighty-one. + +Last month I bought a nuc. Five frames, a mated queen, from a guy two towns over who talked to me for forty minutes in his driveway about mite treatments. I have no idea what I'm doing. The first inspection I did, I got the frames back in the wrong order and probably rolled some bees, which is the thing you're not supposed to do, and I felt like I'd let the old man down somehow, which is ridiculous, he's been gone eleven years. + +The bees don't care about any of that. They're just building comb, filling it, capping it. Doing the thing. I go out at dusk sometimes and put my ear near the box and it hums like a transformer. diff --git a/evals/fixtures/silhouette/corpus/human/08_technical_explainer.txt b/evals/fixtures/silhouette/corpus/human/08_technical_explainer.txt new file mode 100644 index 0000000..ce5117d --- /dev/null +++ b/evals/fixtures/silhouette/corpus/human/08_technical_explainer.txt @@ -0,0 +1,11 @@ +# Why your database is slow and it's probably the N+1 + +Somebody on the team asks this once a month so here it is written down. + +You have a page that lists 50 blog posts and each post shows its author's name. In the ORM you write `posts = Post.all` and then in the template you do `post.author.name` for each one. Feels clean. It is not clean. That template just ran 51 queries: one to get the posts, then one more per post to go fetch the author, fifty times. This is the N+1. N posts, plus 1. It's the single most common reason a page that was fast with your 12 test rows falls over with 12,000 real ones. + +The fix is to tell the ORM up front that you're going to want the authors, so it can grab them in one shot with a second query (or a join). In Rails that's `Post.includes(:author)`. In Django it's `select_related`. In raw SQL it's just... writing the join you should have written. The names differ, the idea is identical: load the related rows in bulk instead of one at a time. + +How do you find these? Turn on query logging in development and watch the count while you load a page. If clicking one page fires 200 queries, you have an N+1 somewhere, guaranteed. There are gems and middleware (bullet, for Rails) that will scream at you when they detect one. Use them. + +One caveat people get wrong: eager loading isn't free. If you `includes` a table you don't end up using, you did extra work for nothing. And if the association is huge, pulling it all into memory can be worse than the N+1 was. Measure. The whole point is that you looked at the query count instead of guessing. diff --git a/evals/fixtures/silhouette/doctrine_reprise.md b/evals/fixtures/silhouette/doctrine_reprise.md new file mode 100644 index 0000000..c7415b9 --- /dev/null +++ b/evals/fixtures/silhouette/doctrine_reprise.md @@ -0,0 +1,102 @@ +# unslop: Product Doctrine + +Here live the product decisions that govern the repo. CLAUDE.md holds the +contribution rules; this file holds the why. When a change conflicts with this document, +one of the two is wrong, and deciding which requires a human. + +## Identity and shape + +unslop is an open-source tool packaged as an agent skill. The repo is the artifact. There +is no published package, and none is planned unless distribution truly requires it. Entry +points are plain scripts under `scripts/`, runnable by any host agent. + +The host agent is the runtime. Linting runs when an agent invokes it during writing — not +as a pre-commit hook, not as a daemon, not as an editor extension. Integrations beyond the +agent are other people's projects. + +The eval suite defines the product. Every behavior worth having is pinned by a row that +fails without it. This is constitutional, not aspirational: coverage gates make it +structurally impossible to add a detection pattern without eval rows, and the docs +themselves are gate-checked for drift. + +## The axis + +The product spans one axis. On the left sits removal: detect AI writing patterns and strip +them. On the right sits reconstruction: write the way a specific human writes. Product weight +sits roughly 70/30 toward the left. + +The left end is deterministic, cheap, CI-friendly, and objectively benchmarkable. It is +the trust asset. The right end is generative and register-sensitive, and it operates under +the left end's constitution: every mimic or rewrite output must pass every removal gate. A +voice rewrite that reintroduces slop is a failure, full stop... which is itself a phrase +the scanner would flag, so: a failure, period. The gates apply to this document too. + +## Bounds + +- Refuse almost nothing. The tool removes AI-isms; it is not a taste arbiter. Literary + prose is in bounds. Long documents are in bounds via chunking. +- English only. Non-English input gets a cheap detection and a clear decline — the one + graceful refusal in the product. +- Mimicry accepts any writing style the user supplies samples for. There is no rights or + attestation machinery; the sheet of samples is the only credential. + +## Feature surface + +**Always-on linting.** Deterministic scanners (phrase, structure, silhouette) are the +free, always-on layer. Model-based detection exists only for judge-only pattern families +and uses the cheapest capable model, per measured parity data rather than assumption. + +**Co-writer mode.** On detection, findings surface to the user as structured suggestions +(span, severity, rationale, proposed replacement), never as silent rewrites. Hard findings +propose replacements; soft findings ask questions. Contract gates make "accept all" safe +by construction: replacements are span-minimal, scan clean, and preserve every constraint. + +**Teach and mimic.** `teach` distills writing samples into a voice: a machine profile +(`profile.json`, the deterministic referee) and a layered voice card (`card.md` plus +situation sheets) that any generating model can follow in context. Teach is interactive +and coverage-driven — it classifies what the samples demonstrate and prompts for what is +missing. Dimensions with no sample evidence get no sheet and are listed as uncovered; +the card never fabricates. `mimic` writes with the card, scored against the profile, with +all removal gates blocking. An internal refine loop hill-climbs the card against held-out +samples, with a divergence guard against reward hacking. + +**Harvest.** Cheap agents bootstrap the teach corpus from chat transcripts and designated +folders. Authorship separation is the product: assistant-authored text in a voice profile +would teach the exact register unslop removes. Deterministic parsing separates turns, a +scanner tripwire flags suspicious candidates, and nothing enters a profile without human +approval. + +**Calibrate.** An A/B preference game gathers voice signal at tap-level effort. Pairs are +dimension-controlled minimal edits of the user's own passages, mostly produced by +deterministic transforms. Preferences aggregate per dimension with confidence bounds, +actively targeting the least-known dimension, and conflicts with sample-measured values +surface to the user rather than being silently resolved. + +**Contribute.** A wild-caught AI-ism becomes an eval row built from the exact specimen, +verified red-first, and a structured PR into this repo. Redaction never alters the +specimen's tell, publication requires explicit user approval, and the pipeline runs fully +offline until the user says otherwise. + +## Model tiering + +Tiering is measured, not assumed; `evals/run_model_parity.py` re-measures it and +`references/pipeline.md` records the current table. The standing conclusions from the +2026-07-06 runs: + +- Span-scoped replacement is safe on the cheapest tier. The gates carry the safety — the + single bench failure was the most expensive model dropping a fact, caught by the + preservation gate. +- Full rewrites of register-sensitive text erode hedges, absolutes, and legal negations + on cheap tiers. That work belongs to frontier models, re-scanned afterward. +- Macro structure defeated every model tested. No model self-checks document shape from + prose instructions; structure is always machine-detected and machine-gated. + +Touching a model-dependent feature means re-running the parity evals across +both the GPT and Anthropic spectrums. + +## Growth + +Detection catalogs decay as generators evolve. Growth comes from adversarial refresh: +live bench outputs supply fresh specimens, contribute turns each specimen into a +red-first row, and coverage enforcement keeps every new pattern exercised. Yesterday's +miss becomes tomorrow's regression test, and nothing relies on anyone's good habits. diff --git a/evals/fixtures/silhouette/human_reference.json b/evals/fixtures/silhouette/human_reference.json new file mode 100644 index 0000000..b306d21 --- /dev/null +++ b/evals/fixtures/silhouette/human_reference.json @@ -0,0 +1,66 @@ +{ + "description": "Human reference distribution for scripts/silhouette_scan.py. Regenerate with: python3 evals/check_silhouette.py --reference --write. median/iqr/n are derived from the sources below; fence and weight are research-validated constants (scratchpad/research/silhouette.md). The scorer scales each metric by max(iqr, fence): the sample IQR is degenerate at zero for these one-sided tells, so the fence is the effective scale.", + "penalty_threshold": 1.0, + "iqr_floor": 0.05, + "sources": [ + "evals/fixtures/silhouette/corpus/human/01_linkedin.txt", + "evals/fixtures/silhouette/corpus/human/02_blog_intro.txt", + "evals/fixtures/silhouette/corpus/human/03_howto.txt", + "evals/fixtures/silhouette/corpus/human/04_essay.txt", + "evals/fixtures/silhouette/corpus/human/05_readme.txt", + "evals/fixtures/silhouette/corpus/human/06_email.txt", + "evals/fixtures/silhouette/corpus/human/07_personal_story.txt", + "evals/fixtures/silhouette/corpus/human/08_technical_explainer.txt", + "evals/fixtures/structure/struct02_bursty_narrative.md", + "evals/fixtures/structure/struct04_concrete_end.md", + "evals/fixtures/structure/struct10_varied_openers.md", + "evals/fixtures/structure/struct12_moderate_signpost.md", + "evals/fixtures/structure/struct15_one_closer.md", + "evals/fixtures/structure/struct16_parallel_enumeration.md", + "evals/fixtures/structure/struct17_academic_roadmap.md" + ], + "metric_order": [ + "scaffold_opener_share", + "role_entropy_bits", + "heading_preview", + "preview_fulfillment", + "callback_content" + ], + "metrics": { + "scaffold_opener_share": { + "median": 0.0, + "iqr": 0.05, + "fence": 0.2, + "weight": 2.0, + "n": 15 + }, + "role_entropy_bits": { + "median": -0.0, + "iqr": 0.05, + "fence": 0.8, + "weight": 1.0, + "n": 15 + }, + "heading_preview": { + "median": 0.0, + "iqr": 0.05, + "fence": 0.2, + "weight": 1.0, + "n": 1 + }, + "preview_fulfillment": { + "median": 0.0, + "iqr": 0.05, + "fence": 0.25, + "weight": 1.0, + "n": 12 + }, + "callback_content": { + "median": 0.0, + "iqr": 0.05, + "fence": 0.3, + "weight": 1.5, + "n": 8 + } + } +} diff --git a/evals/fixtures/silhouette/sil01_templated_essay.md b/evals/fixtures/silhouette/sil01_templated_essay.md new file mode 100644 index 0000000..97d9784 --- /dev/null +++ b/evals/fixtures/silhouette/sil01_templated_essay.md @@ -0,0 +1,11 @@ +Remote work has reshaped how modern teams collaborate. This essay examines communication, culture, and productivity in distributed organizations. Each of these dimensions deserves careful attention, and together they explain why remote work succeeds or fails. + +Communication is the first pillar worth considering. When teams cannot meet in a hallway, they must be deliberate about writing things down. Clear communication prevents the small misunderstandings that compound into large ones. + +Moreover, culture cannot be left to chance in a distributed setting. Culture forms through repeated rituals, and remote teams must design those rituals on purpose. A strong culture keeps people aligned even across time zones. + +However, productivity is where many organizations stumble. Productivity depends on trust rather than surveillance, and managers who measure hours instead of outcomes tend to lose their best people. Sustainable productivity comes from clear goals. + +Furthermore, the interplay between these pillars matters as much as each one alone. Communication reinforces culture, culture protects productivity, and productivity funds better communication tools. The pillars are not independent. + +Ultimately, remote work rewards organizations that invest in communication, culture, and productivity together. These three dimensions, considered as a system, determine whether distributed teams thrive. The future of work belongs to teams that treat them as one. diff --git a/evals/fixtures/silhouette/sil02_bursty_narrative.md b/evals/fixtures/silhouette/sil02_bursty_narrative.md new file mode 100644 index 0000000..0413663 --- /dev/null +++ b/evals/fixtures/silhouette/sil02_bursty_narrative.md @@ -0,0 +1,9 @@ +The bus was late again. I stood at the corner of Ninth and watched a pigeon fight a paper bag and lose. + +When it finally showed up, the driver — a woman named Renata, according to the badge clipped crooked to her collar — waved me on without charging me. "Machine's broken," she said. "Sit anywhere." So I did, near the back, next to a kid asleep against the window with a trumpet case across his knees. + +We crawled north. Rain started. The trumpet kid woke up at Pratt Street, blinked at me like I'd appeared out of nowhere, and got off without a word. I never found out if he played it or just carried it around. + +Renata talked the whole way to the depot about her daughter in Tucson, the price of oranges, a dog she used to have. I mostly listened. It's a strange intimacy, being the only passenger left on a city bus in the rain, hearing a stranger's whole afternoon. + +I missed my stop by six blocks and didn't mind. Some mornings the detour is the only part worth keeping. diff --git a/evals/fixtures/silhouette/sil04_recap_loop.md b/evals/fixtures/silhouette/sil04_recap_loop.md new file mode 100644 index 0000000..78ae536 --- /dev/null +++ b/evals/fixtures/silhouette/sil04_recap_loop.md @@ -0,0 +1,9 @@ +The lighthouse at Pemaquid Point has guided lobster boats past the harbor rocks for two centuries. My grandfather kept its lamp for thirty years, and the keeper's logbook still sits on his kitchen shelf. That logbook is where this story begins. + +He taught me to splice rope on the back porch, his hands moving faster than my eyes could follow. The trick, he said, was to keep even tension across every strand. I ruined a dozen lengths of manila before one held. + +We spent August hauling traps in the grey light before dawn. Diesel smoke, cold coffee, the winch screaming as it pulled forty pounds of dripping wire off the bottom. I learned to gauge a keeper crab by feel alone. + +By September the tourists had gone and the water turned the colour of slate. We mended the traps we had wrecked, rewove the parlour heads, and stacked them along the fence for winter. The work was slow and the days were short. + +I did not understand, then, why he cared so much about even tension in a splice, or why he checked the lamp twice each night. The logbook from the lighthouse, its pages soft with salt, holds the same careful hand recording every storm past the harbor for thirty years. diff --git a/evals/fixtures/silhouette/sil05_cue_deletion.md b/evals/fixtures/silhouette/sil05_cue_deletion.md new file mode 100644 index 0000000..26c553d --- /dev/null +++ b/evals/fixtures/silhouette/sil05_cue_deletion.md @@ -0,0 +1,11 @@ +The espresso machine hissed. The barista frowned at the pressure gauge. + +The delivery van idled outside. The driver checked a clipboard twice. + +The awning dripped rainwater. The gutter had clogged with maple seeds. + +The jukebox played something old. The speaker crackled on the high notes. + +The regulars argued about baseball. The score stayed tied through nine. + +The register jammed on a receipt. The manager sighed and rebooted it. diff --git a/evals/fixtures/silhouette/sil08_two_paragraphs.md b/evals/fixtures/silhouette/sil08_two_paragraphs.md new file mode 100644 index 0000000..b6acc6f --- /dev/null +++ b/evals/fixtures/silhouette/sil08_two_paragraphs.md @@ -0,0 +1,3 @@ +We shipped the migration on Tuesday and the dashboards went quiet, which is the good kind of quiet. No paging, no rollback, no angry thread in the incident channel. + +The one surprise was the cache warm-up, which took longer than the staging run suggested. Worth a note in the runbook, nothing more. diff --git a/evals/fixtures/suggest/sugg01_seed.md b/evals/fixtures/suggest/sugg01_seed.md new file mode 100644 index 0000000..e9a199f --- /dev/null +++ b/evals/fixtures/suggest/sugg01_seed.md @@ -0,0 +1 @@ +The data speaks for itself in today's market. diff --git a/evals/fixtures/suggest/sugg02_soft.md b/evals/fixtures/suggest/sugg02_soft.md new file mode 100644 index 0000000..7c88dc2 --- /dev/null +++ b/evals/fixtures/suggest/sugg02_soft.md @@ -0,0 +1 @@ +The dataset is invaluable to the whole research team here. diff --git a/evals/fixtures/suggest/sugg03_oversized.json b/evals/fixtures/suggest/sugg03_oversized.json new file mode 100644 index 0000000..4512037 --- /dev/null +++ b/evals/fixtures/suggest/sugg03_oversized.json @@ -0,0 +1,17 @@ +{ + "document": "the report speaks for itself here.\n", + "suggestions": [ + { + "span": { + "start": 0, + "end": 28, + "text": "the report speaks for itself" + }, + "severity": "hard", + "category": "false_agency", + "rationale": "whole-sentence rewrite", + "suggested_replacement": "the report is self-evident", + "phrased_as_question": false + } + ] +} diff --git a/evals/fixtures/suggest/sugg04_accept.json b/evals/fixtures/suggest/sugg04_accept.json new file mode 100644 index 0000000..4aa0940 --- /dev/null +++ b/evals/fixtures/suggest/sugg04_accept.json @@ -0,0 +1,29 @@ +{ + "document": "the report speaks for itself in today's climate.\n", + "suggestions": [ + { + "span": { + "start": 11, + "end": 28, + "text": "speaks for itself" + }, + "severity": "hard", + "category": "false_agency", + "rationale": "AI-writing tell (false_agency): replace \u201cspeaks for itself\u201d \u2014 State the point directly.", + "suggested_replacement": "needs no gloss", + "phrased_as_question": false + }, + { + "span": { + "start": 29, + "end": 39, + "text": "in today's" + }, + "severity": "hard", + "category": "filler", + "rationale": "AI-writing tell (filler): replace \u201cin today's\u201d.", + "suggested_replacement": "currently", + "phrased_as_question": false + } + ] +} diff --git a/evals/fixtures/suggest/sugg05_overlap.json b/evals/fixtures/suggest/sugg05_overlap.json new file mode 100644 index 0000000..2a5946d --- /dev/null +++ b/evals/fixtures/suggest/sugg05_overlap.json @@ -0,0 +1,29 @@ +{ + "document": "the report speaks for itself in today's climate.\n", + "suggestions": [ + { + "span": { + "start": 11, + "end": 28, + "text": "speaks for itself" + }, + "severity": "hard", + "category": "false_agency", + "rationale": "x", + "suggested_replacement": null, + "phrased_as_question": false + }, + { + "span": { + "start": 18, + "end": 39, + "text": "for itself in today's" + }, + "severity": "hard", + "category": "filler", + "rationale": "y", + "suggested_replacement": null, + "phrased_as_question": false + } + ] +} diff --git a/evals/fixtures/voice/authors/amara/doc1.md b/evals/fixtures/voice/authors/amara/doc1.md new file mode 100644 index 0000000..a35e2fd --- /dev/null +++ b/evals/fixtures/voice/authors/amara/doc1.md @@ -0,0 +1,15 @@ +I keep a shelf by the back door. It holds gloves, seed packets, a bent trowel, and a jar of brass screws. I don't sort it much. I know where each thing lives. + +This morning I planted beans before work. The soil was cold. The hose coughed. A crow watched from the fence and made one dry sound. I told it I wasn't sharing. + +I like chores with edges. Cut twine. Pull weeds. Rinse mud from a boot heel. You can see the work. You can stop when the row is clean. That's rare enough. + +The beans went in crooked. That's fine. Straight rows are for catalogs. Mine follow rocks, roots, and my left knee, which still complains after rain. I pressed each seed down with one finger. I covered it. I patted the bed flat. + +By seven, the kettle was yelling. My cuffs were wet. There was dirt under one nail. I had a meeting in twenty minutes and a slug on my sleeve. The day already felt used, in a good way. + +I don't garden for calm. Calm is too fancy. I garden because dinner starts somewhere. Because a handful of beans in July can make a Tuesday feel less thin. Because the little shed smells like cedar, dust, and last year's onions. + +Tonight I'll check the bed again. Nothing will have happened. That's the point. Most good things begin as nothing you can prove. + +I left the muddy gloves on the step. They'll dry there. Or they won't. Either way, morning will know what to do with them. diff --git a/evals/fixtures/voice/authors/amara/doc2.md b/evals/fixtures/voice/authors/amara/doc2.md new file mode 100644 index 0000000..5478947 --- /dev/null +++ b/evals/fixtures/voice/authors/amara/doc2.md @@ -0,0 +1,13 @@ +The train was late again. Nobody looked surprised. A man in a blue coat kicked salt from the platform edge. A woman opened a paper bag and shared almonds with her kid. I stood under the heater. It wasn't doing much. + +I don't mind waiting if the place has texture. This station does. Brick walls. Gum spots. A clock that loses three minutes every week. Pigeons are banned from my final answer, but here in the fixture text the platform birds were real and loud. The coffee cart smelled burnt in the normal way. + +When the train came, we all moved at once. No speech. Just bags lifted, shoulders turned, one small apology near the door. That's city manners. Brief. Useful. Gone. + +I found a seat with a torn corner. The vinyl scratched my palm. Across from me, a nurse slept through three stops. Her shoes were clean. Her face wasn't. I tried not to stare. + +The window gave back a gray version of us. Coats. Phones. A lunch box. One red scarf. I thought about how many private mornings fit inside one car. Bad news. Good bread. Rent due. A dog left with a neighbor. Nobody announces it. + +My stop came fast after all that waiting. I stepped down into slush. The street smelled like diesel and wet wool. I had ten minutes to spare. That's enough for a cheap coffee and a reset. + +The train will be late tomorrow too. I'll still take it. It's ugly, close, and honest about the weather. diff --git a/evals/fixtures/voice/authors/amara/doc3.md b/evals/fixtures/voice/authors/amara/doc3.md new file mode 100644 index 0000000..55cadec --- /dev/null +++ b/evals/fixtures/voice/authors/amara/doc3.md @@ -0,0 +1,15 @@ +Mara left a lemon cake on my porch. No note. Just the cake, wrapped tight, with a thumbprint in the icing. I knew it was hers. She uses too much zest. I mean that kindly. + +I carried it inside and cut a slice before dinner. Bad order. Good choice. The crumb stuck to the knife. The kitchen smelled bright for ten whole minutes. + +We've been neighbors for nine years. We don't visit much. We trade things. Tape. Basil. A ladder. News about the loose step by the mailboxes. When her pipes froze, I brought towels. When my dog died, she took the leash from my hand and hung it in the closet. No speech. She knew. + +Some friendships don't need a table between them. They happen at fences, in hallways, beside trash bins, with one person holding a drill and the other holding the screws. They last because nobody asks them to perform. + +The cake had a crack down the middle. The icing slid to one side. Perfect cakes make me suspicious. This one leaned. It had weather. + +After dinner, I washed the plate and walked it back. Mara's curtains were open. She was reading in the front room with one sock off. I left the plate on her chair and knocked once. + +She waved through the glass. I pointed at the plate. She pointed at the sky, like rain was my fault. That's our whole language. + +Tomorrow I'll take her the last jar of peach jam. It set too hard. She won't mind. She'll pry it out with a spoon. diff --git a/evals/fixtures/voice/authors/amara/doc4.md b/evals/fixtures/voice/authors/amara/doc4.md new file mode 100644 index 0000000..4ecc9f6 --- /dev/null +++ b/evals/fixtures/voice/authors/amara/doc4.md @@ -0,0 +1,15 @@ +The hardware store moved the nails. I hate that. Nails belong in the back left aisle, under the bad light, near the bins of washers. Now they're beside cabinet pulls and tiny brass hooks. Nothing about that makes sense. + +I went in for roofing nails and left with sandpaper, twine, and a red handled scraper. This is how the store wins. It knows about loose paint and winter drafts. It knows I can't pass a bucket of mismatched bolts without checking. + +The clerk was new. He asked if I needed help. I said no. Then I needed help. That's tradition. + +He found the nails on the third try. We both pretended the new layout was reasonable. He scanned the box. It made that flat beep. I missed the old owner, who wrote prices in pencil and kept peppermints near the register. He'd say, "Roof leaking?" and I'd say, "Not yet." That was enough. + +Outside, the wind pushed grit along the curb. I put the box in my coat pocket. It dragged the left side down. A practical weight. I like that feeling. + +At home, I climbed the short ladder and fixed the loose strip over the shed door. Three nails. Four swings each. The last one bent. I pulled it and tried again. + +Rain is due tonight. The shed may still leak. But the strip is tight now, and the hammer is back on its hook. That's a decent afternoon. + +I swept the bent nail into my palm. Kept it. There's always a jar for small failures that might help later. diff --git a/evals/fixtures/voice/authors/amara/doc5.md b/evals/fixtures/voice/authors/amara/doc5.md new file mode 100644 index 0000000..9802848 --- /dev/null +++ b/evals/fixtures/voice/authors/amara/doc5.md @@ -0,0 +1,17 @@ +I bought pears from the farm stand on Route 8. Six of them. Hard as stones. The woman at the table said they'd soften by Friday. I believed her because her hands looked like she knew fruit. + +The stand had a coffee can for cash and a scale with chipped paint. No music. No signs shaped like pumpkins. Just crates, paper bags, and one sleepy dog under the table. That's plenty. + +I drove home with the pears rolling on the passenger seat. Every turn made a small thud. I should've brought a basket. I never bring the thing I need. + +At home, I lined them on the sill. Green shoulders. Brown freckles. Stems like little hooks. They looked stubborn. I respect that in fruit. + +My grandmother used to test pears with her thumb. Gentle. Near the neck. She'd say, "Not yet," and put them back. She said that about people too. I didn't understand it then. I do now. + +Friday is far for a pear and close for a person. By then the kitchen may smell sweet. Or one will bruise overnight and ruin the others. That's also a lesson, though not a cheerful one. + +I made toast for dinner. Pears watched from the sill. The dog pressed his nose to the cabinet where the biscuits live. Rain tapped the vent hood. + +Nothing was ready. Nothing needed to be. I washed the knife anyway and left it to dry. + +Before bed, I touched one pear with my thumb. Still hard. Good. Some promises should make you wait. diff --git a/evals/fixtures/voice/authors/boris/doc1.md b/evals/fixtures/voice/authors/boris/doc1.md new file mode 100644 index 0000000..2680ce4 --- /dev/null +++ b/evals/fixtures/voice/authors/boris/doc1.md @@ -0,0 +1,9 @@ +Although the laundromat occupies a narrow storefront between a vacant travel agency and a pharmacy that has not changed its window display since spring, it has become, through repetition rather than charm, one of the more dependable rooms in my week. I arrive with a canvas bag, a book I rarely open, and the mild resignation appropriate to any errand governed by quarters, damp sleeves, and other people's schedules. + +The machines are old, yet they enforce a useful discipline; once the washer begins its cycle, there is nothing persuasive to do except remain nearby, observe the slow theater of tumbling cotton, and accept that certain kinds of progress cannot be hurried without making a mess. Consequently, I have come to value the interval. It is not leisure, exactly, but it is adjacent to leisure in a practical and defensible way. + +On Tuesday, a man in a paint-spattered jacket folded children's pajamas with extraordinary care, aligning every cuff before placing each pair into a plastic basket. A student revised flash cards beside the change machine. The owner repaired a latch with tape, patience, and a screwdriver whose handle had been worn nearly smooth. + +These scenes would not support a large conclusion, and I distrust large conclusions extracted from small errands; nevertheless, the room has a clarifying effect. Everyone there is maintaining something. The work is ordinary, repetitive, and visible. No one can claim transformation when the evidence is a stack of warm towels. + +When my dryer stopped, I carried the clothes home still faintly damp. The inconvenience was minor; the walk, under a thin winter sun, was almost pleasant. diff --git a/evals/fixtures/voice/authors/boris/doc2.md b/evals/fixtures/voice/authors/boris/doc2.md new file mode 100644 index 0000000..677b804 --- /dev/null +++ b/evals/fixtures/voice/authors/boris/doc2.md @@ -0,0 +1,9 @@ +Because the public library opens an hour later on Fridays, the small group that gathers outside its doors has developed a temporary society with recognizable customs, hierarchies, and grievances. The retired teacher stands nearest the return slot, since she intends to correct the order of the newspapers before reading them; the delivery driver waits beside the bicycle rack, where the morning light reaches his shoulders; and I stand under the awning, pretending that my presence is governed by necessity rather than preference. + +The building itself is unremarkable, but its habits are precise. At nine, the custodian unlocks the outer door, pauses to lift a mat that catches on the threshold, and nods with the gravity of a minor official opening a border. We enter quietly, although no rule requires quiet until we are inside; the courtesy begins early, perhaps because the place has trained us better than we admit. + +I usually choose a table near the local history shelves, not because I require the material, but because that corner receives a clean, indirect light and because the atlas cabinet gives the room a faint smell of dust and cloth. Such considerations matter when one is trying to read without becoming performative about reading. + +Last week I found a grocery list folded into a biography of a mayor. It included tea, oranges, batteries, and the phrase "call Nora," underlined twice. I left it where it was. Private urgency, once misplaced, should not be improved by strangers. + +By noon the library had filled with ordinary weather: whispers, copier heat, chair legs, the soft percussion of returned books. I left with nothing checked out, which felt, despite appearances, like a successful visit. diff --git a/evals/fixtures/voice/authors/boris/doc3.md b/evals/fixtures/voice/authors/boris/doc3.md new file mode 100644 index 0000000..dbf6468 --- /dev/null +++ b/evals/fixtures/voice/authors/boris/doc3.md @@ -0,0 +1,9 @@ +When the apartment building scheduled its annual hallway painting, the notice promised minimal disruption, a phrase whose optimism was undermined almost immediately by the smell of primer, the plastic sheeting taped across the elevator, and the painter's radio, which reduced three floors of residents to unwilling participants in the same afternoon. Still, the inconvenience was not without interest; shared irritation, unlike shared enthusiasm, produces unusually efficient conversation. + +Mrs. Alvarez from 3B objected to the color, a pale gray that appeared, depending on the light, either hygienic or defeated. The student in 2A worried about his bicycle, which had been moved six feet from its habitual place. I contributed very little, since my own complaint concerned the loss of several small chips and scuffs by which I had navigated the corridor at night without switching on the light. + +It is easy to mock attachment to such marks; however, a building becomes legible through its imperfections. The dent near the mailboxes indicates where someone once misjudged a sofa. The dark crescent below the stair rail records thousands of hands. Even a stain can become, over time, a form of address. + +By evening, the hallway looked brighter, cleaner, and less familiar. The painter had done competent work; that fact did not settle the matter. Improvement, when applied broadly and without memory, can erase information along with damage. + +I carried my trash downstairs after dinner and paused where the old scuff had been. The wall offered nothing back. I touched the new paint lightly, then regretted the fingerprint. diff --git a/evals/fixtures/voice/authors/boris/doc4.md b/evals/fixtures/voice/authors/boris/doc4.md new file mode 100644 index 0000000..79992d1 --- /dev/null +++ b/evals/fixtures/voice/authors/boris/doc4.md @@ -0,0 +1,9 @@ +Although I have no serious claim to expertise in coffee, I have developed firm opinions about the office machine, chiefly because it occupies the counter nearest my desk and because its failures are public, theatrical, and distributed across the morning. A kettle fails privately; this machine announces its grievances with blinking lights, wet grounds, and a sound like a drawer full of coins being shaken by an impatient clerk. + +The official instructions, laminated and taped to the cabinet, are both comprehensive and unhelpful. They specify the position of the filter basket, the acceptable water line, and the sequence of buttons; they do not explain why the machine occasionally refuses to proceed until someone opens and closes the lid with ceremonial seriousness. Consequently, the staff has developed an oral tradition, and like most oral traditions, it contains superstition disguised as procedure. + +On Monday, Leonard insisted that the second button must be pressed twice. Priya argued that the reservoir must be lifted before the first beep. I cleaned the needle with a paper clip, which was at least physically related to the problem. None of these interventions produced immediate success, although all of them were defended with confidence. + +Eventually the machine relented. The coffee tasted faintly metallic, but no one complained, since the struggle had converted adequacy into victory. This is one reason institutions survive: people lower their standards together and call the result cooperation. + +By three, the counter was stained again, the laminated instructions had curled at one corner, and someone had written "please be patient" on a sticky note. The note was addressed to the machine, but it applied more broadly. diff --git a/evals/fixtures/voice/authors/boris/doc5.md b/evals/fixtures/voice/authors/boris/doc5.md new file mode 100644 index 0000000..64974d7 --- /dev/null +++ b/evals/fixtures/voice/authors/boris/doc5.md @@ -0,0 +1,9 @@ +Since the neighborhood pool closes after Labor Day, its final week carries a ceremonial quality that exceeds the actual pleasure of swimming in water already cooled by September nights. Parents arrive with towels that smell faintly of storage; children bargain for one more jump from the low board; and the lifeguards, who have spent three months enforcing rules with admirable boredom, begin to look almost sentimental. + +I went on Thursday evening, when the sky had the flat, metallic brightness that often precedes rain. The pool was not crowded, yet every sound traveled: the slap of sandals, the scrape of a chair, the echoing cough of someone who had swallowed water in a moment of excessive confidence. I swam eight slow lengths and then sat on the concrete edge, where the warmth of the day remained trapped in the surface. + +There is something clarifying about a public place at the end of its season. The defects become visible because no one is trying to disguise them. A lane rope has faded unevenly; the snack window sticks; the painted depth numbers have begun to flake. These are not failures, exactly, but records of use. + +When the whistle blew, people left reluctantly and without drama. A boy forgot one blue sandal, and his mother retrieved it with the irritation reserved for predictable losses. The lifeguard stacked chairs in pairs; the manager locked the supply shed. + +I walked home with damp hair and no particular sadness. The pool would be back next year, altered slightly, familiar enough. diff --git a/evals/fixtures/voice/authors/celia/doc1.md b/evals/fixtures/voice/authors/celia/doc1.md new file mode 100644 index 0000000..8e31f8f --- /dev/null +++ b/evals/fixtures/voice/authors/celia/doc1.md @@ -0,0 +1,11 @@ +I tried to clean the junk drawer today, which sounds like a small task until you meet the drawer. Why did we own seven rubber bands, three dead batteries, and a key nobody can identify? Who decided the tape measure belonged under takeout menus? Me, probably. + +The first ten minutes were heroic. I made piles. Keep, toss, mystery. The mystery pile grew fast, because apparently my household runs on objects that might matter later. A tiny screwdriver? Keep. A receipt from a garden center in 2021? Maybe. A smooth gray stone? Obviously important, though nobody admitted bringing it inside. + +Halfway through, I found the missing cabinet knob. This was good news and also rude, because I had already replaced it with a different knob (slightly shinier, deeply annoying). Do I change it back? Do I accept the drawer's judgment? These are the questions that make adulthood so glamorous. + +I did throw things away. Not enough, but some. The drawer now opens without making that crunchy sound, and the scissors are visible, which feels like progress. There is still a bag of foreign coins in the back. There is still the key. I put a label on it that says, "unknown key, do not trust." + +By lunch, the counter looked worse than before. That is the emotional arc of cleaning, isn't it? You create a disaster in order to negotiate a smaller disaster. Then you make coffee and pretend the system has improved. + +The drawer also contains one birthday candle shaped like a question mark. Too obvious? Maybe. I left it there for morale. diff --git a/evals/fixtures/voice/authors/celia/doc2.md b/evals/fixtures/voice/authors/celia/doc2.md new file mode 100644 index 0000000..e71a54e --- /dev/null +++ b/evals/fixtures/voice/authors/celia/doc2.md @@ -0,0 +1,13 @@ +The bakery changed its cinnamon roll, and I am trying to be mature about it. Is this a civic issue? No. Did I spend the walk home composing a small speech in my head? Unfortunately, yes. + +The old roll was lopsided and sticky in a way that required two napkins. The new one is tidy. Too tidy? It has a glaze that sits politely on top instead of sinking into the edges, and there are chopped nuts now (fine, but why). I bought one because I wanted to be fair, then ate it on the bench by the bus stop like a judge with pastry evidence. + +It was good. That's the problem. If it had been bad, I could reject it cleanly. Instead it was competent, fragrant, and wrong. Do you know that feeling? When a thing works, but not as the thing you loved? + +The owner was cheerful when she handed it over. She said they were "trying a lighter version," which made me look down at the roll as if it had personally betrayed butter. I nodded. I paid. I did not deliver the speech, because I am not a monster before noon. + +At home, I told Sam, who said, "Maybe buy a croissant." This is why marriage needs bylaws. A croissant is not a cinnamon roll in a different hat. + +Next Saturday I'll go back. Maybe the old one returns. Maybe I become a person with a new bakery order. Growth is terrible like that. + +For now I am calling this research, which sounds better than sulking with crumbs on my scarf. diff --git a/evals/fixtures/voice/authors/celia/doc3.md b/evals/fixtures/voice/authors/celia/doc3.md new file mode 100644 index 0000000..ebdc62e --- /dev/null +++ b/evals/fixtures/voice/authors/celia/doc3.md @@ -0,0 +1,11 @@ +I signed up for a pottery class, mostly because the flyer said beginners were welcome and I chose to believe it. Was that wise? Debatable. My first bowl looked like a tired hat. The instructor called it "expressive," which was kind and possibly illegal. + +The studio lives behind a bike shop, so the whole room smells like clay, chain oil, and somebody's peppermint tea. There are shelves of student work along one wall. Some pieces are beautiful. Some are brave. Mine, at the moment, are damp negotiations. + +The wheel is harder than it looks. You press too lightly and the clay ignores you; press too hard and it slumps like it has lost faith in the project. There must be a metaphor there, but I refuse to pick it up. We have enough metaphors. We need bowls. + +By the second week, I could center the clay for almost thirty seconds. This felt huge. Then I sneezed and made a vase collapse into a pancake. Everyone pretended not to see, which is the highest form of community. + +I like the class because failure is physical there. It sits in front of you, gray and wet, and you can push it back into a lump. How often does life offer that? Usually the mess is in your inbox, or your chest, or a group text with too many opinions. + +My tired-hat bowl comes out of the kiln next week. I might use it for keys. Or lemons. Or proof that trying counts, even when the rim wobbles. diff --git a/evals/fixtures/voice/authors/celia/doc4.md b/evals/fixtures/voice/authors/celia/doc4.md new file mode 100644 index 0000000..3d514a9 --- /dev/null +++ b/evals/fixtures/voice/authors/celia/doc4.md @@ -0,0 +1,13 @@ +We tried to have a picnic at the lake, and the lake tried to remove us. Not dramatically. Just enough wind to flip napkins, sand a sandwich, and make every paper plate behave like a loose kite. Romantic? In a documentary-about-weather sense, maybe. + +I had packed carefully, which means I had packed the wrong things with confidence. Two kinds of cheese, no knife. Lemonade, no cups. A blanket so thin it immediately became part of the landscape. Sam looked at the bag, looked at me, and asked whether crackers were "a utensil now." Honestly? They were promoted under emergency powers. + +Still, the water looked gorgeous. Gray-green, choppy, full of little silver flashes. Kids ran along the shore yelling at waves as if waves take feedback. A grandfather in a straw hat read a mystery novel with one hand and guarded a bowl of cherries with the other. I admired his priorities. + +The wind got colder after twenty minutes. We ate fast. Cheese torn by hand, sandy peaches, crackers doing their best. Was it a failed picnic? I don't think so. A failed picnic involves resentment, and we mostly had crumbs. + +On the way back to the car, I found a flat blue stone and put it in my pocket. Why? No idea. The junk drawer probably knows. + +At home, we made tea and washed grit from the plates. The blanket shed sand in the hallway like a confession. + +Next time I will bring cups, a knife, and possibly a small anchor. Will I remember? Let's be generous and say maybe. diff --git a/evals/fixtures/voice/authors/celia/doc5.md b/evals/fixtures/voice/authors/celia/doc5.md new file mode 100644 index 0000000..6a7def5 --- /dev/null +++ b/evals/fixtures/voice/authors/celia/doc5.md @@ -0,0 +1,13 @@ +I bought a houseplant with dramatic needs. The tag said "moderate light," which seemed reasonable, but the plant has since clarified that moderate means a very specific rectangle of sun between 9:10 and 10:35. Who has this schedule? Apparently, my fern. + +It sits on the kitchen stool now, because the windowsill was "too harsh" (my interpretation, based on leaf sulking). The bedroom was too dim. The bookshelf made it look like a prop in a therapist's office. The stool is inconvenient, but the fern looks smug there, so I guess negotiations are ongoing. + +I water it with a little blue pitcher. Not too much. Not too little. Is there a correct amount? The internet says yes, then gives twelve answers and a warning about root rot. Root rot sounds like a Victorian curse. I refuse to invite it in. + +Yesterday one frond turned brown at the tip. I took this personally. Had I failed? Had the plant? Was this just what leaves do, quietly, while humans overthink? I trimmed the brown part with nail scissors and apologized, which is not something I planned to admit. + +Still, I like having it around. The kitchen feels less like a place where mail goes to die. There is a living thing beside the kettle, asking for attention but not conversation. That's a nice category. + +If it survives the month, I may buy another plant. If it doesn't, I will call the empty stool a minimalist choice. Either way, the blue pitcher stays. + +This is how hobbies start, right? One needy fern, one unnecessary pitcher, and a person pretending not to be invested. diff --git a/evals/fixtures/voice/copy_lift.md b/evals/fixtures/voice/copy_lift.md new file mode 100644 index 0000000..88cc7c4 --- /dev/null +++ b/evals/fixtures/voice/copy_lift.md @@ -0,0 +1,3 @@ +The hardware store moved the nails. I hate that. Nails belong in the back left aisle, under the bad light, near the bins of washers. Now they're beside cabinet pulls and tiny brass hooks. Nothing about that makes sense. + +I went in for roofing nails and left with sandpaper, twine, and a red handled scraper. This is how the store wins. It knows about loose paint and winter drafts. It knows I can't pass a bucket of mismatched bolts without checking. diff --git a/evals/fixtures/voice/impostors/imp01.txt b/evals/fixtures/voice/impostors/imp01.txt new file mode 100644 index 0000000..5a99b73 --- /dev/null +++ b/evals/fixtures/voice/impostors/imp01.txt @@ -0,0 +1,9 @@ +We shipped the migration Friday at 2am and it broke in a way none of us predicted. + +The short version: our "zero downtime" cutover had eleven minutes of downtime because a Terraform apply reordered the security group rules and locked the read replicas out of the primary. Classic. I'd tested the failover twice in staging. Staging doesn't have the same VPC peering setup, so the test proved nothing. + +Anyway. What actually saved us was that Priya had, on her own initiative back in March, written a runbook nobody asked for. It had the exact rollback command. We ran it, waited, watched the dashboards go green. Total incident: 34 minutes. + +I keep thinking about how the thing that mattered wasn't the fancy migration plan with the Gantt chart. It was one engineer writing a doc during a slow week. We don't reward that. Our promo packets ask for "impact" and "scope" and there is no line item for "wrote the boring document that later prevented a Sev1." + +I don't have a tidy lesson here. Maybe just: hire people who write things down, and then actually read the things. diff --git a/evals/fixtures/voice/impostors/imp02.txt b/evals/fixtures/voice/impostors/imp02.txt new file mode 100644 index 0000000..5283841 --- /dev/null +++ b/evals/fixtures/voice/impostors/imp02.txt @@ -0,0 +1,5 @@ +I bought a sourdough starter from a woman named Deb off Craigslist in 2019 for eight dollars. She met me in a Trader Joe's parking lot in Glendale and handed it over in a mason jar with a rubber band around a piece of parchment. It smelled like green apples and paint thinner. She said it had been going since 1994. + +The starter is now dead. I killed it in the second week of the pandemic by feeding it bread flour I'd panic-bought that turned out to be self-rising. Self-rising flour has baking powder and salt in it. Salt, in quantity, murders the yeast. I did not know this. Nobody tells you this. + +This post is about what I learned rebuilding a starter from scratch, twice, and why almost every recipe you'll find online is subtly wrong about hydration ratios. If you just want the schedule, skip to the part with the table. But the ratios are where people go wrong, and the ratios are boring, which is exactly why they don't get explained. diff --git a/evals/fixtures/voice/impostors/imp03.txt b/evals/fixtures/voice/impostors/imp03.txt new file mode 100644 index 0000000..b37ab9b --- /dev/null +++ b/evals/fixtures/voice/impostors/imp03.txt @@ -0,0 +1,13 @@ +# Replacing the thermostat on a 2009 Subaru Forester + +You need a 10mm socket, a 12mm socket, needle-nose pliers, and about a gallon of coolant. Also a drain pan. Also patience, because the lower radiator hose clamp is in a spot the engineers clearly designed while angry. + +Jack up the front, pull the plastic splash guard (six of those push-clip things, they will break, buy extras). Drain the radiator from the petcock on the driver's side, bottom corner. It drains slow. Go make coffee. + +The thermostat housing is on the lower left of the engine block, follow the lower radiator hose to where it meets the block, that's it. Two 10mm bolts. Here's the part nobody mentions: the bottom bolt is behind the hose, and you cannot get a normal socket on it. I used a stubby wrench and turned it a sixth of a turn at a time for twenty minutes. + +Old thermostat comes out with the O-ring. Note which way the spring faces. Seriously, take a photo, because if you put it in backwards the car overheats and you get to do this whole thing again, which I know because I did. + +New one in, spring toward the engine. Torque the bolts to like 7 ft-lbs, they're tiny, don't gorilla them or you'll strip the aluminum. Refill coolant, run the engine with the cap off until the thermostat opens and the level drops, top off, done. + +I am not going to tell you to consult a professional. You already decided you weren't going to. diff --git a/evals/fixtures/voice/impostors/imp04.txt b/evals/fixtures/voice/impostors/imp04.txt new file mode 100644 index 0000000..3cfcd5d --- /dev/null +++ b/evals/fixtures/voice/impostors/imp04.txt @@ -0,0 +1,9 @@ +The city put in a bike lane on Fulton last year and everyone I know has an opinion about it, which is how I know it's working. + +The argument against is that it removed a lane of car traffic and now the evening backup stretches past the hardware store. This is true. I've sat in it. The argument for is that four people used to get hit on that stretch every year, on average, going back a decade the city has the data, and this year so far the number is zero. That's also true. + +What interests me isn't which side is right. It's that both numbers are real and they point in opposite directions and no amount of arguing dissolves the tradeoff. You are trading somebody's twelve minutes for somebody else's collarbone. Reasonable people weight those differently. The people who bike weight them one way and the people who drive weight them another and, surprise, most people mostly do one or the other. + +I drive. I'll admit the lane annoys me twice a day. I also walked past the little ghost bike somebody chained to the signpost at Fulton and 9th, painted white, with a laminated photo of a kid who was nineteen. So I don't say much at the neighborhood meetings anymore. The math got personal and I lost my taste for winning the argument. + +Cities are just a pile of these. We pretend policy is about finding the right answer. Mostly it's about deciding whose inconvenience is acceptable, and then not looking too hard at the people on the other side of the trade. diff --git a/evals/fixtures/voice/impostors/imp05.txt b/evals/fixtures/voice/impostors/imp05.txt new file mode 100644 index 0000000..10f981e --- /dev/null +++ b/evals/fixtures/voice/impostors/imp05.txt @@ -0,0 +1,33 @@ +# qgrep + +Grep, but it understands quotes. That's the whole pitch. Regular grep will happily match inside string literals and comments, which is fine until you're trying to find every actual call to `delete_user` and half your hits are in a changelog. + +## install + +``` +go install github.com/mmn/qgrep@latest +``` + +Needs Go 1.21+. There's no Homebrew formula yet. I keep meaning to. + +## usage + +``` +qgrep 'delete_user' ./src +``` + +By default it skips matches inside string literals and line comments for C-family, Python, Go, and Ruby. Everything else it treats as plain text and falls back to normal grep behavior, so don't expect magic in your COBOL. + +Flags: + +- `-s` also search inside strings (i.e. give up and act like grep) +- `-c` also search inside comments +- `--lang=X` force a language instead of guessing from extension + +The language guessing is dumb. It looks at the file extension and nothing else. If you have `.h` files that are actually C++ it'll parse them as C and occasionally get a `<` wrong. Patches welcome, I don't write much C++ anymore so I haven't cared enough to fix it. + +## why + +I wrote this on a Sunday because I was annoyed. It is not battle-tested. It has one contributor (me) and about 60% test coverage, all of it on the tokenizer because that's the part that breaks. If you hit a parsing bug, open an issue with the smallest file that reproduces it and I'll usually get to it within a week or two. + +No license file yet. Assume MIT, I'll add it properly when I'm not lazy. diff --git a/evals/fixtures/voice/impostors/imp06.txt b/evals/fixtures/voice/impostors/imp06.txt new file mode 100644 index 0000000..3e0c7c6 --- /dev/null +++ b/evals/fixtures/voice/impostors/imp06.txt @@ -0,0 +1,15 @@ +Subject: re: the Henderson numbers + +Dave, + +Short answer: don't send those to the client yet. + +The Q2 figure in row 14 is double-counting the November refund. I caught it because the total didn't match what I had in my own tracker and I spent an embarrassing hour last night figuring out why. It's the refund. Maria processed it against two cost centers back in the reorg and nobody reconciled it. + +Real number is 418K, not 441K. Which, annoyingly, is under the threshold we told them we'd hit, so we should get ahead of that instead of letting them find it. + +Can you and I grab fifteen minutes before the 11am? I don't want to walk into that meeting improvising. I'm free at 9:30 or basically any time after 4 today. + +Also unrelated but the printer on 3 is out of toner again, I already emailed facilities so you don't have to. + +Matt diff --git a/evals/fixtures/voice/impostors/imp07.txt b/evals/fixtures/voice/impostors/imp07.txt new file mode 100644 index 0000000..2365999 --- /dev/null +++ b/evals/fixtures/voice/impostors/imp07.txt @@ -0,0 +1,9 @@ +My grandfather kept bees and I was terrified of them until I was about eleven. + +He never made a thing of it. He'd just hand me the smoker and let me stand ten feet back and eventually five feet back and one summer I was standing right at the hive in one of his old veils that smelled like pipe tobacco, holding a frame heavy with honey and thirty thousand bees, and I wasn't scared anymore. I don't remember the day it flipped. There wasn't one. That's the thing about it. + +He died when I was in college. The hives went to a neighbor because nobody in the family knew what to do with them and my grandmother wasn't going to keep them going at eighty-one. + +Last month I bought a nuc. Five frames, a mated queen, from a guy two towns over who talked to me for forty minutes in his driveway about mite treatments. I have no idea what I'm doing. The first inspection I did, I got the frames back in the wrong order and probably rolled some bees, which is the thing you're not supposed to do, and I felt like I'd let the old man down somehow, which is ridiculous, he's been gone eleven years. + +The bees don't care about any of that. They're just building comb, filling it, capping it. Doing the thing. I go out at dusk sometimes and put my ear near the box and it hums like a transformer. diff --git a/evals/fixtures/voice/impostors/imp08.txt b/evals/fixtures/voice/impostors/imp08.txt new file mode 100644 index 0000000..ce5117d --- /dev/null +++ b/evals/fixtures/voice/impostors/imp08.txt @@ -0,0 +1,11 @@ +# Why your database is slow and it's probably the N+1 + +Somebody on the team asks this once a month so here it is written down. + +You have a page that lists 50 blog posts and each post shows its author's name. In the ORM you write `posts = Post.all` and then in the template you do `post.author.name` for each one. Feels clean. It is not clean. That template just ran 51 queries: one to get the posts, then one more per post to go fetch the author, fifty times. This is the N+1. N posts, plus 1. It's the single most common reason a page that was fast with your 12 test rows falls over with 12,000 real ones. + +The fix is to tell the ORM up front that you're going to want the authors, so it can grab them in one shot with a second query (or a join). In Rails that's `Post.includes(:author)`. In Django it's `select_related`. In raw SQL it's just... writing the join you should have written. The names differ, the idea is identical: load the related rows in bulk instead of one at a time. + +How do you find these? Turn on query logging in development and watch the count while you load a page. If clicking one page fires 200 queries, you have an N+1 somewhere, guaranteed. There are gems and middleware (bullet, for Rails) that will scream at you when they detect one. Use them. + +One caveat people get wrong: eager loading isn't free. If you `includes` a table you don't end up using, you did extra work for nothing. And if the association is huge, pulling it all into memory can be worse than the N+1 was. Measure. The whole point is that you looked at the query count instead of guessing. diff --git a/evals/fixtures/voice/impostors/imp09.md b/evals/fixtures/voice/impostors/imp09.md new file mode 100644 index 0000000..ece7f02 --- /dev/null +++ b/evals/fixtures/voice/impostors/imp09.md @@ -0,0 +1,7 @@ +I used to think Sunday planning required a fresh notebook, a clear desk, and the sort of pen that makes lists look decisive. Now I do it on the back of envelopes while the kettle boils. The result is less elegant, but it survives contact with the week. + +My list usually begins with groceries and ends with some emotional errand I have been avoiding. Call the dentist. Return the sweater. Ask about the invoice. These tasks are not difficult, yet they gather static when ignored. Writing them down removes some of the drama. + +This week the page included lentils, printer paper, a birthday card, and "fix hallway bulb," which has appeared on three lists in a row. I could claim the delay is strategic. It is not. I simply dislike the ladder. + +By evening, I had crossed off half the items and rewritten the rest more honestly. "Organize closet" became "remove the broken umbrella." That is a better promise. A list should not require a new personality. diff --git a/evals/fixtures/voice/impostors/imp10.md b/evals/fixtures/voice/impostors/imp10.md new file mode 100644 index 0000000..27aeabc --- /dev/null +++ b/evals/fixtures/voice/impostors/imp10.md @@ -0,0 +1,7 @@ +The corner store near my old apartment sold oranges from a crate by the door, and in winter the whole entrance smelled faintly of peel and floor cleaner. I still think about that smell when the weather turns. It was not beautiful, exactly, but it was specific. + +The owner kept a radio behind the counter and argued with it during basketball games. Customers came and went without interrupting him. We understood the arrangement: he would ring up milk, matches, plantains, and newspapers while conducting a private dispute with the Knicks. + +One January night, the power failed on the block. He lit two camping lanterns and kept the store open for cash purchases. People bought candles, soup, batteries, and unnecessary snacks. Darkness made everyone practical for about twelve minutes, then sentimental. + +I moved away years ago. The store is a phone repair shop now. Still, when I buy oranges, I choose them by smell first, then weight. diff --git a/evals/fixtures/voice/impostors/imp11.md b/evals/fixtures/voice/impostors/imp11.md new file mode 100644 index 0000000..505bf10 --- /dev/null +++ b/evals/fixtures/voice/impostors/imp11.md @@ -0,0 +1,7 @@ +My desk has two lives. In the morning, it looks like a place where a reasonable person might do reasonable work. By four, it has become an archaeological site: mug rings, sticky notes, a spoon, three cables, and a book opened face down at a page I no longer remember choosing. + +I have tried systems. Trays, folders, weekly resets, a small ceramic cup for pens. Systems help for a while, then the actual day arrives and drops its elbows on them. The desk does not fail because I lack storage. It fails because I believe every scrap of paper is a future clue. + +Yesterday I cleared it completely. The bare surface looked accusatory. I answered email for twenty minutes and immediately rebuilt the first layer of sediment. A receipt. A notebook. A lemon cough drop. Order is apparently not a condition; it is an event. + +I will clear it again on Friday. I know this cycle. It is less a flaw than a tide table. diff --git a/evals/fixtures/voice/impostors/imp12.md b/evals/fixtures/voice/impostors/imp12.md new file mode 100644 index 0000000..7a0017c --- /dev/null +++ b/evals/fixtures/voice/impostors/imp12.md @@ -0,0 +1,7 @@ +There is a path behind the school that almost nobody uses after dinner. It runs between the soccer field and a row of back fences, then turns toward the creek where the city stopped mowing. In June the grass leans over the gravel and brushes your ankles. + +I walk there when the house has become too loud with ordinary things. The dishwasher. The television next door. My own thoughts repeating themselves with unnecessary confidence. Outside, the repetition changes shape. Crickets, tires on the avenue, a dog complaining behind a fence. + +Last night I saw a paper cup caught in the reeds and carried it to the bin. This did not save the planet. It did improve the path by one cup, which was the scale available to me. + +On the way home, the school windows reflected the sunset in copper squares. For a minute the building looked more expensive than it was. Then the light moved, and it became itself again. diff --git a/evals/fixtures/voice/profiles/amara.json b/evals/fixtures/voice/profiles/amara.json new file mode 100644 index 0000000..b898c83 --- /dev/null +++ b/evals/fixtures/voice/profiles/amara.json @@ -0,0 +1,2951 @@ +{ + "char3": { + " \"n": 1, + " \"r": 1, + " a ": 37, + " ab": 5, + " ac": 1, + " af": 4, + " ag": 3, + " ai": 1, + " al": 6, + " an": 26, + " ap": 1, + " ar": 2, + " as": 3, + " at": 5, + " ba": 12, + " be": 22, + " bi": 3, + " bl": 1, + " bo": 6, + " br": 7, + " bu": 4, + " by": 3, + " ca": 18, + " ce": 1, + " ch": 6, + " ci": 1, + " cl": 7, + " co": 10, + " cr": 4, + " cu": 5, + " da": 1, + " de": 1, + " di": 6, + " do": 17, + " dr": 5, + " du": 3, + " ea": 3, + " ed": 2, + " ei": 1, + " en": 3, + " ev": 1, + " fa": 5, + " fe": 5, + " fi": 6, + " fl": 2, + " fo": 11, + " fr": 9, + " ga": 3, + " gl": 3, + " go": 5, + " gr": 2, + " gu": 1, + " ha": 16, + " he": 17, + " ho": 10, + " hu": 1, + " i ": 39, + " i'": 4, + " ic": 2, + " if": 2, + " in": 19, + " is": 6, + " it": 22, + " ja": 4, + " ju": 3, + " ke": 4, + " ki": 4, + " kn": 9, + " la": 11, + " le": 14, + " li": 9, + " lo": 6, + " lu": 1, + " ma": 12, + " me": 4, + " mi": 11, + " mo": 6, + " mu": 6, + " my": 12, + " na": 7, + " ne": 12, + " ni": 1, + " no": 15, + " nu": 1, + " of": 7, + " ol": 1, + " on": 20, + " op": 2, + " or": 2, + " ot": 1, + " ou": 3, + " ov": 1, + " ow": 1, + " pa": 7, + " pe": 6, + " ph": 1, + " pi": 2, + " pl": 7, + " po": 5, + " pr": 7, + " pu": 5, + " ra": 4, + " re": 8, + " ri": 1, + " ro": 6, + " sa": 5, + " sc": 6, + " se": 7, + " sh": 17, + " si": 2, + " sk": 1, + " sl": 6, + " sm": 6, + " so": 6, + " sp": 5, + " st": 20, + " su": 2, + " sw": 2, + " ta": 4, + " te": 4, + " th": 121, + " ti": 3, + " to": 18, + " tr": 10, + " tu": 2, + " tw": 3, + " ug": 1, + " un": 3, + " us": 4, + " ve": 1, + " vi": 2, + " wa": 24, + " we": 18, + " wh": 10, + " wi": 19, + " wo": 6, + " wr": 2, + " ye": 4, + " yo": 3, + " ze": 1, + "\" a": 1, + "\" t": 1, + "\"no": 1, + "\"ro": 1, + "'d ": 2, + "'ll": 5, + "'re": 1, + "'s ": 12, + "'t ": 9, + "'t.": 2, + "'ve": 1, + ", \"": 2, + ", a": 8, + ", b": 2, + ", c": 1, + ", d": 1, + ", i": 5, + ", l": 1, + ", m": 1, + ", n": 1, + ", o": 1, + ", r": 1, + ", s": 3, + ", t": 3, + ", u": 1, + ", w": 6, + ". a": 11, + ". b": 9, + ". c": 3, + ". e": 1, + ". f": 1, + ". g": 4, + ". h": 6, + ". i": 41, + ". j": 2, + ". k": 1, + ". m": 6, + ". n": 10, + ". o": 3, + ". p": 4, + ". r": 3, + ". s": 9, + ". t": 40, + ". u": 1, + ". w": 7, + ". y": 2, + ".\" ": 1, + "?\" ": 1, + "a b": 4, + "a c": 4, + "a d": 3, + "a g": 2, + "a h": 1, + "a j": 2, + "a l": 4, + "a m": 2, + "a n": 2, + "a p": 2, + "a r": 2, + "a s": 5, + "a t": 4, + "a w": 1, + "a's": 1, + "abi": 1, + "abl": 2, + "abo": 5, + "ace": 2, + "ach": 4, + "ack": 7, + "acr": 1, + "act": 1, + "ad ": 7, + "ad.": 1, + "add": 2, + "ade": 3, + "adi": 2, + "ady": 1, + "aft": 5, + "ag ": 1, + "aga": 3, + "age": 1, + "agg": 1, + "ags": 1, + "aid": 1, + "aig": 1, + "ail": 9, + "ain": 12, + "air": 1, + "ais": 1, + "ait": 2, + "ak.": 1, + "ake": 9, + "aki": 1, + "al ": 4, + "alk": 1, + "all": 6, + "alm": 5, + "alo": 2, + "alr": 1, + "alt": 1, + "alw": 1, + "am.": 1, + "ame": 2, + "amm": 1, + "an ": 7, + "an'": 1, + "an.": 2, + "anc": 1, + "and": 28, + "ane": 1, + "ang": 1, + "ann": 4, + "ans": 4, + "ant": 1, + "any": 1, + "ap ": 1, + "ape": 4, + "apo": 1, + "app": 3, + "ar ": 6, + "ar'": 1, + "ar,": 1, + "ar.": 1, + "ara": 2, + "ard": 4, + "are": 7, + "arf": 1, + "ari": 1, + "arr": 1, + "ars": 1, + "art": 2, + "as ": 12, + "ash": 4, + "asi": 1, + "ask": 2, + "asn": 3, + "aso": 1, + "ass": 4, + "ast": 5, + "at ": 18, + "at'": 8, + "at.": 2, + "ata": 1, + "atc": 3, + "ate": 9, + "atf": 2, + "ath": 2, + "ati": 1, + "ats": 1, + "att": 1, + "aul": 1, + "aus": 4, + "ave": 3, + "ay ": 4, + "ay,": 3, + "ay.": 2, + "ayo": 1, + "ays": 2, + "b s": 1, + "b. ": 1, + "bac": 5, + "bad": 3, + "bag": 2, + "ban": 1, + "bas": 1, + "be ": 1, + "bea": 3, + "bec": 4, + "bed": 3, + "bee": 2, + "bef": 2, + "beg": 1, + "bel": 1, + "ben": 3, + "bes": 2, + "bet": 1, + "bin": 3, + "bir": 1, + "ble": 2, + "blu": 1, + "bod": 3, + "bol": 1, + "boo": 1, + "bor": 2, + "bot": 1, + "bou": 5, + "box": 4, + "bpr": 1, + "bra": 2, + "bre": 1, + "bri": 3, + "bro": 1, + "buc": 1, + "bur": 1, + "but": 2, + "by ": 3, + "cab": 1, + "cak": 4, + "cal": 3, + "cam": 2, + "can": 6, + "car": 4, + "cat": 1, + "cau": 4, + "ce ": 4, + "ce.": 3, + "ced": 1, + "cen": 1, + "ces": 3, + "ch ": 6, + "ch.": 7, + "cha": 1, + "che": 7, + "cho": 2, + "cil": 1, + "cin": 2, + "cio": 1, + "cit": 1, + "ck ": 10, + "ck.": 1, + "cke": 5, + "cki": 1, + "cks": 1, + "cle": 3, + "cli": 1, + "clo": 3, + "coa": 3, + "cof": 2, + "col": 1, + "com": 1, + "cor": 1, + "cou": 1, + "cov": 1, + "cra": 3, + "cre": 2, + "cro": 3, + "cru": 1, + "ct ": 1, + "cti": 1, + "cuf": 1, + "cur": 2, + "cut": 2, + "cy.": 1, + "d a": 16, + "d b": 5, + "d c": 2, + "d d": 3, + "d e": 1, + "d f": 5, + "d g": 1, + "d h": 5, + "d i": 7, + "d k": 2, + "d l": 5, + "d m": 4, + "d n": 3, + "d o": 3, + "d p": 2, + "d s": 8, + "d t": 20, + "d u": 1, + "d w": 6, + "d, ": 4, + "d. ": 12, + "dar": 1, + "day": 2, + "dde": 2, + "ddl": 1, + "ddy": 1, + "de ": 8, + "de,": 1, + "de.": 1, + "dec": 1, + "ded": 3, + "den": 2, + "der": 7, + "dfu": 1, + "dge": 2, + "die": 2, + "din": 6, + "dir": 1, + "dit": 1, + "dle": 2, + "dly": 1, + "do ": 1, + "doe": 1, + "dog": 2, + "doi": 1, + "don": 5, + "doo": 3, + "dow": 5, + "dpa": 1, + "dra": 2, + "dri": 1, + "dry": 2, + "ds ": 3, + "ds.": 1, + "dsh": 1, + "due": 2, + "dus": 1, + "dwa": 1, + "dy ": 5, + "e a": 9, + "e b": 18, + "e c": 15, + "e d": 8, + "e e": 2, + "e f": 9, + "e g": 1, + "e h": 8, + "e i": 4, + "e k": 4, + "e l": 10, + "e m": 9, + "e n": 7, + "e o": 7, + "e p": 11, + "e r": 5, + "e s": 21, + "e t": 16, + "e u": 1, + "e v": 1, + "e w": 13, + "e y": 1, + "e'd": 1, + "e'l": 1, + "e's": 1, + "e'v": 1, + "e, ": 10, + "e. ": 24, + "eac": 4, + "ead": 3, + "eak": 2, + "eal": 1, + "ean": 7, + "eap": 1, + "ear": 5, + "eas": 2, + "eat": 4, + "eca": 4, + "ece": 1, + "ech": 2, + "eck": 2, + "ect": 1, + "ed ": 50, + "ed,": 4, + "ed.": 5, + "eda": 1, + "ede": 2, + "edg": 2, + "eds": 1, + "ee ": 6, + "ee,": 1, + "eec": 2, + "eed": 6, + "eek": 1, + "eel": 3, + "een": 2, + "eep": 2, + "eet": 2, + "eev": 1, + "ef.": 1, + "efo": 2, + "eft": 8, + "efu": 1, + "egi": 2, + "eig": 3, + "eit": 1, + "ek.": 1, + "el ": 2, + "el,": 1, + "el.": 1, + "elf": 1, + "eli": 1, + "ell": 5, + "elo": 1, + "elp": 3, + "els": 1, + "elt": 1, + "em ": 1, + "em.": 2, + "emo": 1, + "en ": 13, + "en,": 1, + "en.": 1, + "enc": 3, + "end": 2, + "ene": 2, + "eno": 3, + "ens": 1, + "ent": 8, + "eon": 1, + "ep ": 2, + "ep.": 2, + "epp": 2, + "ept": 4, + "er ": 20, + "er,": 4, + "er.": 11, + "ere": 11, + "erf": 2, + "erk": 1, + "erm": 1, + "ern": 1, + "ers": 6, + "ery": 1, + "es ": 14, + "es,": 2, + "es.": 6, + "esd": 1, + "ese": 2, + "esi": 2, + "ess": 2, + "est": 2, + "et ": 5, + "et.": 5, + "ete": 1, + "eti": 1, + "ets": 1, + "ett": 1, + "etw": 1, + "eve": 3, + "ew ": 2, + "ew.": 2, + "ewh": 1, + "ews": 4, + "ext": 2, + "ey ": 3, + "ey'": 2, + "f b": 3, + "f i": 1, + "f l": 1, + "f m": 1, + "f p": 1, + "f t": 1, + "f u": 1, + "f w": 1, + "f. ": 3, + "fac": 1, + "fai": 1, + "fan": 1, + "fas": 1, + "fau": 1, + "fe.": 1, + "fec": 1, + "fee": 4, + "fel": 1, + "fen": 2, + "ff.": 1, + "ffe": 2, + "ffs": 1, + "fin": 4, + "fit": 1, + "fix": 2, + "fla": 2, + "fol": 1, + "for": 12, + "fou": 3, + "fri": 1, + "fro": 8, + "fs ": 1, + "ft ": 8, + "fte": 5, + "fts": 1, + "ful": 2, + "g a": 3, + "g d": 1, + "g i": 6, + "g l": 2, + "g m": 1, + "g n": 1, + "g o": 1, + "g s": 1, + "g t": 2, + "g w": 2, + "g y": 1, + "g. ": 6, + "g?\"": 1, + "gai": 3, + "gar": 2, + "gav": 1, + "ge.": 2, + "ged": 1, + "geo": 1, + "ger": 1, + "ges": 1, + "gge": 1, + "gh ": 3, + "gh.": 2, + "ghb": 2, + "ghe": 1, + "ght": 11, + "gin": 1, + "gis": 1, + "gla": 1, + "glo": 2, + "gly": 1, + "gon": 1, + "goo": 4, + "gra": 1, + "gri": 1, + "gs ": 4, + "gs.": 2, + "gua": 1, + "gum": 1, + "gy ": 1, + "h a": 4, + "h b": 2, + "h e": 1, + "h f": 2, + "h h": 1, + "h j": 1, + "h o": 3, + "h p": 1, + "h s": 3, + "h t": 4, + "h z": 1, + "h. ": 10, + "had": 4, + "hai": 1, + "hal": 1, + "ham": 1, + "han": 3, + "hap": 2, + "har": 4, + "has": 1, + "hat": 19, + "hav": 1, + "hbo": 2, + "he ": 86, + "he'": 2, + "hea": 2, + "hec": 2, + "hed": 9, + "hee": 1, + "hel": 4, + "hem": 3, + "hen": 6, + "her": 18, + "hey": 5, + "hic": 1, + "hin": 7, + "hip": 1, + "hir": 1, + "his": 4, + "ho ": 1, + "hoe": 1, + "hoi": 1, + "hol": 5, + "hom": 1, + "hon": 2, + "hoo": 2, + "hor": 2, + "hos": 1, + "hou": 3, + "how": 2, + "hre": 3, + "hro": 2, + "ht ": 7, + "ht,": 2, + "ht.": 2, + "hum": 1, + "hun": 1, + "i b": 1, + "i c": 4, + "i d": 3, + "i f": 1, + "i g": 1, + "i h": 3, + "i k": 3, + "i l": 4, + "i m": 2, + "i n": 2, + "i p": 6, + "i s": 4, + "i t": 3, + "i w": 3, + "i'd": 1, + "i'l": 3, + "ica": 1, + "ice": 3, + "ich": 1, + "ici": 3, + "ick": 2, + "id ": 2, + "id.": 1, + "idd": 1, + "ide": 7, + "ied": 4, + "ief": 1, + "ien": 1, + "ies": 1, + "if ": 2, + "ife": 1, + "ift": 1, + "ige": 1, + "igh": 11, + "ike": 5, + "il ": 3, + "il.": 2, + "ilb": 1, + "ill": 7, + "ils": 5, + "ilu": 1, + "imb": 1, + "in ": 21, + "in.": 5, + "ina": 1, + "ind": 5, + "ine": 6, + "ing": 26, + "inn": 3, + "ins": 8, + "int": 9, + "inu": 4, + "iny": 2, + "ion": 4, + "iou": 1, + "ip ": 2, + "ipe": 1, + "ips": 1, + "ir ": 1, + "ird": 2, + "irt": 1, + "is ": 10, + "ise": 1, + "isi": 1, + "isl": 1, + "ism": 1, + "iss": 1, + "ist": 1, + "it ": 19, + "it'": 1, + "it.": 4, + "itc": 1, + "ith": 13, + "iti": 3, + "its": 1, + "itt": 1, + "ity": 1, + "iva": 1, + "ive": 1, + "ixe": 1, + "ixt": 1, + "jam": 1, + "jar": 3, + "jul": 1, + "jus": 2, + "k a": 1, + "k d": 2, + "k l": 1, + "k o": 2, + "k t": 4, + "k w": 2, + "k. ": 6, + "ke ": 11, + "ke,": 1, + "ked": 6, + "kee": 1, + "kep": 2, + "kes": 2, + "ket": 4, + "kic": 1, + "kid": 1, + "kin": 3, + "kit": 1, + "kne": 3, + "kni": 1, + "kno": 5, + "ks ": 1, + "ks,": 1, + "ks.": 1, + "ky,": 1, + "l a": 6, + "l b": 1, + "l c": 2, + "l d": 1, + "l f": 1, + "l h": 1, + "l i": 1, + "l k": 1, + "l l": 2, + "l m": 1, + "l o": 1, + "l p": 1, + "l s": 2, + "l t": 3, + "l w": 4, + "l, ": 1, + "l. ": 5, + "lac": 1, + "lad": 2, + "lai": 1, + "lan": 2, + "las": 5, + "lat": 10, + "lay": 1, + "lbo": 1, + "ld ": 2, + "ld.": 1, + "lde": 1, + "ldi": 2, + "lds": 1, + "le ": 5, + "le,": 1, + "le.": 2, + "lea": 6, + "led": 5, + "lee": 1, + "lef": 8, + "lem": 1, + "lep": 1, + "ler": 1, + "les": 1, + "lf ": 1, + "lic": 1, + "lid": 1, + "lif": 1, + "lig": 1, + "lik": 5, + "lim": 1, + "lin": 2, + "lit": 1, + "liv": 1, + "lke": 1, + "ll ": 17, + "lle": 4, + "lli": 1, + "llo": 1, + "lls": 3, + "llw": 1, + "lm ": 1, + "lm.": 3, + "lmo": 1, + "loc": 1, + "log": 2, + "lon": 2, + "loo": 4, + "los": 3, + "lou": 1, + "lov": 2, + "low": 1, + "lp ": 1, + "lp.": 2, + "lre": 1, + "ls ": 5, + "ls.": 4, + "lt ": 2, + "lt.": 1, + "lts": 1, + "lue": 1, + "lug": 1, + "lun": 1, + "lur": 1, + "lus": 1, + "lwa": 2, + "ly ": 1, + "ly,": 1, + "ly.": 1, + "m a": 1, + "m b": 1, + "m e": 1, + "m i": 1, + "m m": 3, + "m s": 1, + "m t": 3, + "m w": 1, + "m. ": 7, + "mad": 2, + "mai": 1, + "mak": 3, + "mal": 3, + "man": 4, + "mar": 2, + "mat": 1, + "may": 1, + "mb ": 1, + "mbe": 1, + "mbp": 1, + "me ": 3, + "me,": 3, + "mea": 1, + "mee": 1, + "mel": 4, + "mer": 1, + "mew": 1, + "mid": 1, + "mig": 1, + "min": 8, + "mis": 2, + "mme": 1, + "mon": 2, + "mor": 5, + "mos": 1, + "mov": 2, + "mpl": 1, + "muc": 4, + "mud": 2, + "my ": 12, + "n a": 4, + "n b": 1, + "n c": 4, + "n d": 1, + "n f": 2, + "n h": 4, + "n i": 5, + "n j": 1, + "n m": 6, + "n n": 1, + "n o": 2, + "n p": 2, + "n s": 3, + "n t": 14, + "n w": 5, + "n't": 11, + "n, ": 1, + "n. ": 12, + "nab": 1, + "nai": 7, + "nal": 1, + "nce": 5, + "nch": 1, + "nci": 1, + "ncy": 1, + "nd ": 29, + "nd.": 2, + "nde": 4, + "ndf": 1, + "ndl": 2, + "ndo": 1, + "ndp": 1, + "nds": 2, + "ne ": 13, + "ne,": 1, + "ne.": 3, + "nea": 3, + "ned": 6, + "nee": 4, + "nei": 2, + "ner": 6, + "nes": 2, + "net": 1, + "new": 6, + "ng ": 17, + "ng.": 6, + "ng?": 1, + "nge": 1, + "ngs": 4, + "ngu": 1, + "nif": 1, + "nig": 2, + "nin": 4, + "nio": 1, + "nne": 6, + "nno": 1, + "no ": 3, + "no.": 1, + "nob": 3, + "noc": 1, + "noo": 1, + "nor": 1, + "not": 6, + "nou": 4, + "now": 6, + "ns ": 7, + "ns,": 1, + "ns.": 2, + "nse": 2, + "nsi": 2, + "nsw": 1, + "nt ": 10, + "nt.": 2, + "nte": 4, + "nto": 2, + "nts": 1, + "nty": 1, + "nur": 1, + "nut": 4, + "ny ": 2, + "nyl": 1, + "o d": 1, + "o f": 1, + "o h": 1, + "o m": 2, + "o n": 1, + "o o": 1, + "o p": 1, + "o s": 5, + "o t": 1, + "o w": 2, + "o. ": 2, + "oat": 3, + "obo": 3, + "ock": 5, + "od ": 5, + "ody": 3, + "oes": 2, + "of ": 7, + "off": 3, + "ofi": 1, + "og ": 2, + "ogs": 1, + "ogy": 1, + "oic": 1, + "oil": 1, + "oin": 4, + "ok ": 1, + "ok.": 1, + "oke": 2, + "oks": 1, + "ol.": 1, + "old": 6, + "ole": 2, + "oll": 1, + "olo": 1, + "olt": 1, + "om ": 7, + "oma": 1, + "ome": 3, + "omo": 2, + "omp": 1, + "on ": 10, + "on'": 7, + "on.": 3, + "ona": 1, + "onc": 2, + "ond": 1, + "one": 14, + "ong": 2, + "oni": 3, + "ons": 2, + "ont": 1, + "oo ": 3, + "oo.": 1, + "ood": 5, + "oof": 2, + "ook": 5, + "ool": 1, + "oom": 1, + "oon": 2, + "oor": 3, + "oos": 3, + "oot": 2, + "op ": 2, + "ope": 2, + "ops": 1, + "or ": 8, + "or.": 4, + "orc": 1, + "ord": 1, + "ore": 5, + "ork": 2, + "orm": 4, + "orn": 5, + "orr": 2, + "ors": 1, + "ort": 2, + "ose": 7, + "oss": 1, + "ost": 1, + "ot ": 3, + "ote": 2, + "oth": 5, + "ots": 2, + "ou ": 3, + "oud": 1, + "oug": 8, + "oul": 1, + "oun": 4, + "our": 2, + "ous": 1, + "out": 9, + "ove": 7, + "ow ": 11, + "ow,": 1, + "owe": 2, + "own": 5, + "ows": 3, + "ox ": 1, + "ox.": 2, + "oxe": 1, + "oze": 1, + "p a": 1, + "p b": 1, + "p c": 2, + "p i": 1, + "p l": 1, + "p o": 1, + "p w": 1, + "p. ": 4, + "pac": 1, + "pai": 1, + "pal": 2, + "pap": 2, + "par": 1, + "pas": 1, + "pat": 1, + "pe.": 1, + "pea": 1, + "ped": 2, + "pee": 2, + "pen": 5, + "pep": 1, + "per": 7, + "pes": 1, + "pho": 1, + "pic": 1, + "pig": 1, + "pip": 1, + "pla": 8, + "poc": 1, + "poi": 3, + "pol": 1, + "poo": 1, + "por": 1, + "pot": 1, + "ppe": 5, + "pra": 1, + "pre": 2, + "pri": 4, + "pro": 1, + "pry": 1, + "ps ": 1, + "ps.": 1, + "pt ": 4, + "pul": 3, + "pus": 1, + "put": 1, + "r a": 4, + "r b": 1, + "r c": 3, + "r d": 2, + "r f": 2, + "r h": 1, + "r i": 1, + "r k": 1, + "r n": 1, + "r o": 3, + "r p": 1, + "r r": 2, + "r s": 4, + "r t": 9, + "r w": 2, + "r's": 1, + "r, ": 5, + "r. ": 15, + "ra ": 1, + "ra'": 1, + "rac": 2, + "rad": 2, + "raf": 1, + "rag": 1, + "rai": 7, + "rap": 2, + "rar": 1, + "ras": 3, + "rat": 1, + "ray": 1, + "rb.": 1, + "rch": 1, + "rd ": 1, + "rd.": 1, + "rde": 3, + "rds": 1, + "rdw": 1, + "re ": 17, + "re'": 1, + "re.": 5, + "rea": 5, + "red": 4, + "ree": 4, + "reg": 1, + "ren": 1, + "res": 4, + "ret": 1, + "rew": 2, + "rf.": 1, + "rfe": 1, + "rfo": 1, + "ric": 2, + "rie": 5, + "rig": 1, + "ril": 1, + "rin": 3, + "rip": 2, + "ris": 1, + "rit": 1, + "riv": 1, + "rk ": 1, + "rk.": 2, + "rm ": 2, + "rm.": 1, + "rma": 1, + "rmi": 1, + "rn ": 1, + "rne": 2, + "rni": 3, + "rno": 1, + "rnt": 1, + "roc": 1, + "rom": 6, + "ron": 1, + "roo": 5, + "ros": 1, + "rot": 1, + "rou": 3, + "rov": 1, + "row": 6, + "roz": 1, + "rpr": 1, + "rri": 1, + "rro": 2, + "rs ": 2, + "rs.": 4, + "rse": 1, + "rsi": 1, + "rso": 1, + "rt ": 4, + "rta": 1, + "rts": 1, + "rum": 1, + "ry ": 4, + "ry.": 1, + "s a": 12, + "s b": 4, + "s c": 4, + "s d": 3, + "s e": 4, + "s f": 5, + "s g": 1, + "s h": 4, + "s i": 5, + "s l": 3, + "s m": 3, + "s n": 3, + "s o": 6, + "s r": 3, + "s s": 4, + "s t": 12, + "s u": 1, + "s w": 8, + "s y": 1, + "s, ": 7, + "s. ": 30, + "sai": 1, + "sal": 1, + "san": 1, + "say": 2, + "sca": 2, + "scr": 4, + "sda": 1, + "se ": 10, + "se,": 1, + "se.": 1, + "sea": 1, + "sed": 4, + "see": 3, + "sef": 1, + "sel": 1, + "sen": 1, + "ses": 2, + "set": 3, + "sev": 1, + "sh ": 2, + "sh.": 1, + "sha": 2, + "she": 15, + "shi": 1, + "sho": 3, + "sid": 7, + "sil": 1, + "sio": 1, + "sit": 1, + "ske": 1, + "sks": 1, + "sky": 1, + "sle": 3, + "sli": 2, + "slu": 2, + "sma": 3, + "sme": 4, + "sn'": 3, + "soc": 1, + "soi": 1, + "som": 2, + "son": 2, + "sor": 1, + "sou": 1, + "spa": 1, + "spe": 2, + "spi": 1, + "spo": 2, + "ss ": 5, + "ss.": 1, + "sse": 2, + "st ": 9, + "st,": 1, + "st.": 1, + "sta": 3, + "ste": 4, + "sti": 3, + "sto": 6, + "str": 4, + "stu": 1, + "sur": 1, + "sus": 1, + "swe": 2, + "swi": 1, + "t a": 10, + "t b": 4, + "t c": 2, + "t d": 3, + "t f": 5, + "t g": 2, + "t h": 7, + "t i": 11, + "t j": 1, + "t k": 5, + "t l": 3, + "t m": 7, + "t n": 3, + "t o": 4, + "t p": 4, + "t r": 2, + "t s": 6, + "t t": 19, + "t u": 2, + "t v": 1, + "t w": 10, + "t y": 2, + "t's": 9, + "t, ": 3, + "t. ": 18, + "t.\"": 1, + "tab": 1, + "tai": 1, + "tak": 2, + "tal": 1, + "tap": 1, + "tar": 2, + "tat": 1, + "tch": 4, + "te ": 7, + "te.": 2, + "ted": 5, + "ten": 3, + "tep": 3, + "ter": 8, + "tes": 4, + "tex": 2, + "tfo": 2, + "th ": 12, + "tha": 17, + "the": 92, + "thi": 12, + "tho": 2, + "thr": 5, + "thu": 1, + "tic": 1, + "tig": 2, + "til": 3, + "tin": 4, + "tio": 2, + "tle": 2, + "to ": 8, + "tol": 1, + "tom": 2, + "ton": 2, + "too": 6, + "top": 3, + "tor": 3, + "tow": 1, + "tra": 7, + "tre": 1, + "tri": 4, + "tro": 1, + "try": 1, + "ts ": 4, + "ts,": 2, + "ts.": 3, + "tsi": 1, + "tte": 1, + "ttl": 2, + "tuc": 1, + "tue": 1, + "tur": 3, + "twe": 2, + "twi": 2, + "ty ": 2, + "u c": 3, + "uag": 1, + "uch": 4, + "uck": 2, + "ud ": 1, + "ud.": 1, + "udd": 1, + "ue ": 2, + "ue.": 1, + "ues": 1, + "uff": 1, + "ug ": 1, + "ugh": 8, + "ugl": 1, + "ul ": 1, + "ul.": 1, + "uld": 1, + "ull": 3, + "ult": 1, + "uly": 1, + "um ": 1, + "umb": 2, + "unc": 2, + "und": 6, + "ung": 1, + "ur ": 2, + "urb": 1, + "ure": 3, + "urn": 2, + "urp": 1, + "urs": 1, + "urt": 1, + "us.": 2, + "use": 7, + "ush": 2, + "usp": 1, + "ust": 3, + "ut ": 13, + "ute": 4, + "uts": 1, + "vat": 1, + "ve ": 3, + "ve.": 2, + "ved": 3, + "ven": 1, + "ver": 4, + "ves": 3, + "vin": 1, + "vis": 1, + "w g": 1, + "w i": 3, + "w l": 1, + "w m": 1, + "w r": 1, + "w t": 3, + "w w": 3, + "w, ": 1, + "w. ": 2, + "wai": 2, + "wal": 2, + "war": 1, + "was": 15, + "wat": 1, + "wav": 1, + "way": 5, + "we ": 4, + "we'": 1, + "wea": 2, + "wee": 3, + "wei": 1, + "wel": 2, + "wen": 3, + "wep": 1, + "wer": 5, + "wet": 2, + "wha": 1, + "whe": 6, + "whi": 1, + "who": 3, + "wil": 3, + "win": 7, + "wit": 12, + "wn ": 3, + "wn.": 1, + "wne": 1, + "wom": 1, + "won": 2, + "woo": 1, + "wor": 2, + "wra": 1, + "wro": 1, + "ws ": 4, + "ws.": 3, + "x i": 1, + "x. ": 2, + "xed": 1, + "xes": 1, + "xt ": 1, + "xtu": 2, + "y a": 3, + "y b": 1, + "y c": 3, + "y d": 1, + "y f": 4, + "y g": 1, + "y h": 2, + "y i": 1, + "y l": 3, + "y m": 2, + "y n": 1, + "y p": 4, + "y s": 5, + "y t": 3, + "y v": 1, + "y w": 2, + "y'l": 1, + "y'r": 1, + "y, ": 5, + "y. ": 5, + "yea": 2, + "yel": 1, + "yet": 1, + "yl ": 1, + "you": 4, + "ys ": 1, + "ys,": 1, + "ze,": 1, + "zes": 1 + }, + "contraction_rate": 0.03085824493731919, + "function_word_background": { + "a": { + "mean": 0.025, + "std": 0.006 + }, + "about": { + "mean": 0.0025, + "std": 0.006 + }, + "above": { + "mean": 0.0025, + "std": 0.006 + }, + "across": { + "mean": 0.0025, + "std": 0.006 + }, + "after": { + "mean": 0.0025, + "std": 0.006 + }, + "again": { + "mean": 0.0025, + "std": 0.006 + }, + "against": { + "mean": 0.0025, + "std": 0.006 + }, + "all": { + "mean": 0.0025, + "std": 0.006 + }, + "almost": { + "mean": 0.0025, + "std": 0.006 + }, + "along": { + "mean": 0.0025, + "std": 0.006 + }, + "already": { + "mean": 0.0025, + "std": 0.006 + }, + "also": { + "mean": 0.0025, + "std": 0.006 + }, + "although": { + "mean": 0.0025, + "std": 0.006 + }, + "am": { + "mean": 0.0025, + "std": 0.006 + }, + "among": { + "mean": 0.0025, + "std": 0.006 + }, + "an": { + "mean": 0.0025, + "std": 0.006 + }, + "and": { + "mean": 0.025, + "std": 0.006 + }, + "another": { + "mean": 0.0025, + "std": 0.006 + }, + "any": { + "mean": 0.0025, + "std": 0.006 + }, + "are": { + "mean": 0.0025, + "std": 0.006 + }, + "aren't": { + "mean": 0.0025, + "std": 0.006 + }, + "around": { + "mean": 0.0025, + "std": 0.006 + }, + "as": { + "mean": 0.0025, + "std": 0.006 + }, + "at": { + "mean": 0.0025, + "std": 0.006 + }, + "back": { + "mean": 0.0025, + "std": 0.006 + }, + "be": { + "mean": 0.0025, + "std": 0.006 + }, + "because": { + "mean": 0.0025, + "std": 0.006 + }, + "been": { + "mean": 0.0025, + "std": 0.006 + }, + "before": { + "mean": 0.0025, + "std": 0.006 + }, + "behind": { + "mean": 0.0025, + "std": 0.006 + }, + "being": { + "mean": 0.0025, + "std": 0.006 + }, + "below": { + "mean": 0.0025, + "std": 0.006 + }, + "beside": { + "mean": 0.0025, + "std": 0.006 + }, + "between": { + "mean": 0.0025, + "std": 0.006 + }, + "beyond": { + "mean": 0.0025, + "std": 0.006 + }, + "both": { + "mean": 0.0025, + "std": 0.006 + }, + "but": { + "mean": 0.0025, + "std": 0.006 + }, + "by": { + "mean": 0.0025, + "std": 0.006 + }, + "can": { + "mean": 0.0025, + "std": 0.006 + }, + "cannot": { + "mean": 0.0025, + "std": 0.006 + }, + "concerning": { + "mean": 0.0025, + "std": 0.006 + }, + "could": { + "mean": 0.0025, + "std": 0.006 + }, + "couldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "despite": { + "mean": 0.0025, + "std": 0.006 + }, + "did": { + "mean": 0.0025, + "std": 0.006 + }, + "didn't": { + "mean": 0.0025, + "std": 0.006 + }, + "do": { + "mean": 0.0025, + "std": 0.006 + }, + "does": { + "mean": 0.0025, + "std": 0.006 + }, + "doesn't": { + "mean": 0.0025, + "std": 0.006 + }, + "doing": { + "mean": 0.0025, + "std": 0.006 + }, + "done": { + "mean": 0.0025, + "std": 0.006 + }, + "dont": { + "mean": 0.0025, + "std": 0.006 + }, + "down": { + "mean": 0.0025, + "std": 0.006 + }, + "during": { + "mean": 0.0025, + "std": 0.006 + }, + "each": { + "mean": 0.0025, + "std": 0.006 + }, + "either": { + "mean": 0.0025, + "std": 0.006 + }, + "else": { + "mean": 0.0025, + "std": 0.006 + }, + "even": { + "mean": 0.0025, + "std": 0.006 + }, + "every": { + "mean": 0.0025, + "std": 0.006 + }, + "except": { + "mean": 0.0025, + "std": 0.006 + }, + "first": { + "mean": 0.0025, + "std": 0.006 + }, + "for": { + "mean": 0.0025, + "std": 0.006 + }, + "from": { + "mean": 0.0025, + "std": 0.006 + }, + "good": { + "mean": 0.0025, + "std": 0.006 + }, + "had": { + "mean": 0.0025, + "std": 0.006 + }, + "hadn't": { + "mean": 0.0025, + "std": 0.006 + }, + "has": { + "mean": 0.0025, + "std": 0.006 + }, + "hasn't": { + "mean": 0.0025, + "std": 0.006 + }, + "have": { + "mean": 0.0025, + "std": 0.006 + }, + "haven't": { + "mean": 0.0025, + "std": 0.006 + }, + "having": { + "mean": 0.0025, + "std": 0.006 + }, + "he": { + "mean": 0.0025, + "std": 0.006 + }, + "he'd": { + "mean": 0.0025, + "std": 0.006 + }, + "he'll": { + "mean": 0.0025, + "std": 0.006 + }, + "he's": { + "mean": 0.0025, + "std": 0.006 + }, + "her": { + "mean": 0.0025, + "std": 0.006 + }, + "here": { + "mean": 0.0025, + "std": 0.006 + }, + "herself": { + "mean": 0.0025, + "std": 0.006 + }, + "him": { + "mean": 0.0025, + "std": 0.006 + }, + "himself": { + "mean": 0.0025, + "std": 0.006 + }, + "his": { + "mean": 0.0025, + "std": 0.006 + }, + "how": { + "mean": 0.0025, + "std": 0.006 + }, + "however": { + "mean": 0.0025, + "std": 0.006 + }, + "i": { + "mean": 0.0025, + "std": 0.006 + }, + "i'd": { + "mean": 0.0025, + "std": 0.006 + }, + "i'll": { + "mean": 0.0025, + "std": 0.006 + }, + "i'm": { + "mean": 0.0025, + "std": 0.006 + }, + "i've": { + "mean": 0.0025, + "std": 0.006 + }, + "if": { + "mean": 0.0025, + "std": 0.006 + }, + "in": { + "mean": 0.025, + "std": 0.006 + }, + "including": { + "mean": 0.0025, + "std": 0.006 + }, + "indeed": { + "mean": 0.0025, + "std": 0.006 + }, + "inside": { + "mean": 0.0025, + "std": 0.006 + }, + "instead": { + "mean": 0.0025, + "std": 0.006 + }, + "into": { + "mean": 0.0025, + "std": 0.006 + }, + "is": { + "mean": 0.0025, + "std": 0.006 + }, + "isn't": { + "mean": 0.0025, + "std": 0.006 + }, + "it": { + "mean": 0.0025, + "std": 0.006 + }, + "it's": { + "mean": 0.0025, + "std": 0.006 + }, + "its": { + "mean": 0.0025, + "std": 0.006 + }, + "itself": { + "mean": 0.0025, + "std": 0.006 + }, + "just": { + "mean": 0.0025, + "std": 0.006 + }, + "last": { + "mean": 0.0025, + "std": 0.006 + }, + "let": { + "mean": 0.0025, + "std": 0.006 + }, + "lets": { + "mean": 0.0025, + "std": 0.006 + }, + "like": { + "mean": 0.0025, + "std": 0.006 + }, + "made": { + "mean": 0.0025, + "std": 0.006 + }, + "make": { + "mean": 0.0025, + "std": 0.006 + }, + "many": { + "mean": 0.0025, + "std": 0.006 + }, + "may": { + "mean": 0.0025, + "std": 0.006 + }, + "me": { + "mean": 0.0025, + "std": 0.006 + }, + "might": { + "mean": 0.0025, + "std": 0.006 + }, + "mightn't": { + "mean": 0.0025, + "std": 0.006 + }, + "mine": { + "mean": 0.0025, + "std": 0.006 + }, + "minus": { + "mean": 0.0025, + "std": 0.006 + }, + "more": { + "mean": 0.0025, + "std": 0.006 + }, + "moreover": { + "mean": 0.0025, + "std": 0.006 + }, + "most": { + "mean": 0.0025, + "std": 0.006 + }, + "much": { + "mean": 0.0025, + "std": 0.006 + }, + "must": { + "mean": 0.0025, + "std": 0.006 + }, + "mustn't": { + "mean": 0.0025, + "std": 0.006 + }, + "my": { + "mean": 0.0025, + "std": 0.006 + }, + "myself": { + "mean": 0.0025, + "std": 0.006 + }, + "near": { + "mean": 0.0025, + "std": 0.006 + }, + "neither": { + "mean": 0.0025, + "std": 0.006 + }, + "never": { + "mean": 0.0025, + "std": 0.006 + }, + "nevertheless": { + "mean": 0.0025, + "std": 0.006 + }, + "no": { + "mean": 0.0025, + "std": 0.006 + }, + "nor": { + "mean": 0.0025, + "std": 0.006 + }, + "now": { + "mean": 0.0025, + "std": 0.006 + }, + "of": { + "mean": 0.025, + "std": 0.006 + }, + "off": { + "mean": 0.0025, + "std": 0.006 + }, + "on": { + "mean": 0.0025, + "std": 0.006 + }, + "once": { + "mean": 0.0025, + "std": 0.006 + }, + "one": { + "mean": 0.0025, + "std": 0.006 + }, + "only": { + "mean": 0.0025, + "std": 0.006 + }, + "or": { + "mean": 0.0025, + "std": 0.006 + }, + "other": { + "mean": 0.0025, + "std": 0.006 + }, + "our": { + "mean": 0.0025, + "std": 0.006 + }, + "ours": { + "mean": 0.0025, + "std": 0.006 + }, + "ourselves": { + "mean": 0.0025, + "std": 0.006 + }, + "out": { + "mean": 0.0025, + "std": 0.006 + }, + "outside": { + "mean": 0.0025, + "std": 0.006 + }, + "over": { + "mean": 0.0025, + "std": 0.006 + }, + "own": { + "mean": 0.0025, + "std": 0.006 + }, + "people": { + "mean": 0.0025, + "std": 0.006 + }, + "per": { + "mean": 0.0025, + "std": 0.006 + }, + "perhaps": { + "mean": 0.0025, + "std": 0.006 + }, + "plus": { + "mean": 0.0025, + "std": 0.006 + }, + "rather": { + "mean": 0.0025, + "std": 0.006 + }, + "regarding": { + "mean": 0.0025, + "std": 0.006 + }, + "same": { + "mean": 0.0025, + "std": 0.006 + }, + "see": { + "mean": 0.0025, + "std": 0.006 + }, + "she": { + "mean": 0.0025, + "std": 0.006 + }, + "she'd": { + "mean": 0.0025, + "std": 0.006 + }, + "she'll": { + "mean": 0.0025, + "std": 0.006 + }, + "she's": { + "mean": 0.0025, + "std": 0.006 + }, + "should": { + "mean": 0.0025, + "std": 0.006 + }, + "shouldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "since": { + "mean": 0.0025, + "std": 0.006 + }, + "so": { + "mean": 0.0025, + "std": 0.006 + }, + "some": { + "mean": 0.0025, + "std": 0.006 + }, + "still": { + "mean": 0.0025, + "std": 0.006 + }, + "such": { + "mean": 0.0025, + "std": 0.006 + }, + "than": { + "mean": 0.0025, + "std": 0.006 + }, + "that": { + "mean": 0.0025, + "std": 0.006 + }, + "the": { + "mean": 0.025, + "std": 0.006 + }, + "their": { + "mean": 0.0025, + "std": 0.006 + }, + "theirs": { + "mean": 0.0025, + "std": 0.006 + }, + "them": { + "mean": 0.0025, + "std": 0.006 + }, + "themselves": { + "mean": 0.0025, + "std": 0.006 + }, + "then": { + "mean": 0.0025, + "std": 0.006 + }, + "there": { + "mean": 0.0025, + "std": 0.006 + }, + "therefore": { + "mean": 0.0025, + "std": 0.006 + }, + "these": { + "mean": 0.0025, + "std": 0.006 + }, + "they": { + "mean": 0.0025, + "std": 0.006 + }, + "they'd": { + "mean": 0.0025, + "std": 0.006 + }, + "they'll": { + "mean": 0.0025, + "std": 0.006 + }, + "they're": { + "mean": 0.0025, + "std": 0.006 + }, + "they've": { + "mean": 0.0025, + "std": 0.006 + }, + "this": { + "mean": 0.0025, + "std": 0.006 + }, + "those": { + "mean": 0.0025, + "std": 0.006 + }, + "though": { + "mean": 0.0025, + "std": 0.006 + }, + "through": { + "mean": 0.0025, + "std": 0.006 + }, + "thus": { + "mean": 0.0025, + "std": 0.006 + }, + "to": { + "mean": 0.025, + "std": 0.006 + }, + "toward": { + "mean": 0.0025, + "std": 0.006 + }, + "towards": { + "mean": 0.0025, + "std": 0.006 + }, + "two": { + "mean": 0.0025, + "std": 0.006 + }, + "under": { + "mean": 0.0025, + "std": 0.006 + }, + "until": { + "mean": 0.0025, + "std": 0.006 + }, + "up": { + "mean": 0.0025, + "std": 0.006 + }, + "upon": { + "mean": 0.0025, + "std": 0.006 + }, + "us": { + "mean": 0.0025, + "std": 0.006 + }, + "versus": { + "mean": 0.0025, + "std": 0.006 + }, + "very": { + "mean": 0.0025, + "std": 0.006 + }, + "via": { + "mean": 0.0025, + "std": 0.006 + }, + "was": { + "mean": 0.0025, + "std": 0.006 + }, + "wasn't": { + "mean": 0.0025, + "std": 0.006 + }, + "we": { + "mean": 0.0025, + "std": 0.006 + }, + "we'd": { + "mean": 0.0025, + "std": 0.006 + }, + "we'll": { + "mean": 0.0025, + "std": 0.006 + }, + "we're": { + "mean": 0.0025, + "std": 0.006 + }, + "we've": { + "mean": 0.0025, + "std": 0.006 + }, + "well": { + "mean": 0.0025, + "std": 0.006 + }, + "were": { + "mean": 0.0025, + "std": 0.006 + }, + "weren't": { + "mean": 0.0025, + "std": 0.006 + }, + "what": { + "mean": 0.0025, + "std": 0.006 + }, + "when": { + "mean": 0.0025, + "std": 0.006 + }, + "where": { + "mean": 0.0025, + "std": 0.006 + }, + "whether": { + "mean": 0.0025, + "std": 0.006 + }, + "which": { + "mean": 0.0025, + "std": 0.006 + }, + "while": { + "mean": 0.0025, + "std": 0.006 + }, + "who": { + "mean": 0.0025, + "std": 0.006 + }, + "whom": { + "mean": 0.0025, + "std": 0.006 + }, + "whose": { + "mean": 0.0025, + "std": 0.006 + }, + "why": { + "mean": 0.0025, + "std": 0.006 + }, + "will": { + "mean": 0.0025, + "std": 0.006 + }, + "with": { + "mean": 0.0025, + "std": 0.006 + }, + "within": { + "mean": 0.0025, + "std": 0.006 + }, + "without": { + "mean": 0.0025, + "std": 0.006 + }, + "won't": { + "mean": 0.0025, + "std": 0.006 + }, + "would": { + "mean": 0.0025, + "std": 0.006 + }, + "wouldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "yet": { + "mean": 0.0025, + "std": 0.006 + }, + "you": { + "mean": 0.0025, + "std": 0.006 + }, + "you'd": { + "mean": 0.0025, + "std": 0.006 + }, + "you'll": { + "mean": 0.0025, + "std": 0.006 + }, + "you're": { + "mean": 0.0025, + "std": 0.006 + }, + "you've": { + "mean": 0.0025, + "std": 0.006 + }, + "your": { + "mean": 0.0025, + "std": 0.006 + }, + "yours": { + "mean": 0.0025, + "std": 0.006 + }, + "yourself": { + "mean": 0.0025, + "std": 0.006 + }, + "yourselves": { + "mean": 0.0025, + "std": 0.006 + } + }, + "function_words": { + "a": 0.03567984570877531, + "about": 0.0048216007714561235, + "above": 0.0, + "across": 0.0009643201542912247, + "after": 0.0028929604628736743, + "again": 0.0028929604628736743, + "against": 0.0, + "all": 0.0019286403085824494, + "almost": 0.0, + "along": 0.0009643201542912247, + "already": 0.0009643201542912247, + "also": 0.0, + "although": 0.0, + "am": 0.0, + "among": 0.0, + "an": 0.0, + "and": 0.023143683702989394, + "another": 0.0, + "any": 0.0, + "are": 0.0019286403085824494, + "aren't": 0.0, + "around": 0.0, + "as": 0.0009643201542912247, + "at": 0.0048216007714561235, + "back": 0.0048216007714561235, + "be": 0.0009643201542912247, + "because": 0.003857280617164899, + "been": 0.0009643201542912247, + "before": 0.0019286403085824494, + "behind": 0.0, + "being": 0.0, + "below": 0.0, + "beside": 0.0019286403085824494, + "between": 0.0009643201542912247, + "beyond": 0.0, + "both": 0.0009643201542912247, + "but": 0.0019286403085824494, + "by": 0.0028929604628736743, + "can": 0.003857280617164899, + "cannot": 0.0, + "concerning": 0.0, + "could": 0.0, + "couldn't": 0.0, + "despite": 0.0, + "did": 0.0, + "didn't": 0.0, + "do": 0.0009643201542912247, + "does": 0.0009643201542912247, + "doesn't": 0.0, + "doing": 0.0009643201542912247, + "done": 0.0, + "dont": 0.0, + "down": 0.003857280617164899, + "during": 0.0, + "each": 0.0028929604628736743, + "either": 0.0009643201542912247, + "else": 0.0, + "even": 0.0, + "every": 0.0009643201542912247, + "except": 0.0, + "first": 0.0, + "for": 0.006750241080038573, + "from": 0.0057859209257473485, + "good": 0.003857280617164899, + "had": 0.003857280617164899, + "hadn't": 0.0, + "has": 0.0009643201542912247, + "hasn't": 0.0, + "have": 0.0009643201542912247, + "haven't": 0.0, + "having": 0.0, + "he": 0.0028929604628736743, + "he'd": 0.0009643201542912247, + "he'll": 0.0, + "he's": 0.0, + "her": 0.0057859209257473485, + "here": 0.0009643201542912247, + "herself": 0.0, + "him": 0.0, + "himself": 0.0, + "his": 0.0, + "how": 0.0019286403085824494, + "however": 0.0, + "i": 0.03857280617164899, + "i'd": 0.0009643201542912247, + "i'll": 0.0028929604628736743, + "i'm": 0.0, + "i've": 0.0, + "if": 0.0019286403085824494, + "in": 0.01446480231436837, + "including": 0.0, + "indeed": 0.0, + "inside": 0.0019286403085824494, + "instead": 0.0, + "into": 0.0019286403085824494, + "is": 0.0057859209257473485, + "isn't": 0.0, + "it": 0.019286403085824494, + "it's": 0.0009643201542912247, + "its": 0.0009643201542912247, + "itself": 0.0, + "just": 0.0019286403085824494, + "last": 0.003857280617164899, + "let": 0.0, + "lets": 0.0, + "like": 0.0048216007714561235, + "made": 0.0019286403085824494, + "make": 0.0019286403085824494, + "many": 0.0009643201542912247, + "may": 0.0009643201542912247, + "me": 0.0019286403085824494, + "might": 0.0009643201542912247, + "mightn't": 0.0, + "mine": 0.0009643201542912247, + "minus": 0.0, + "more": 0.0, + "moreover": 0.0, + "most": 0.0009643201542912247, + "much": 0.003857280617164899, + "must": 0.0, + "mustn't": 0.0, + "my": 0.011571841851494697, + "myself": 0.0, + "near": 0.0028929604628736743, + "neither": 0.0, + "never": 0.0, + "nevertheless": 0.0, + "no": 0.003857280617164899, + "nor": 0.0, + "now": 0.0019286403085824494, + "of": 0.0057859209257473485, + "off": 0.0009643201542912247, + "on": 0.0057859209257473485, + "once": 0.0019286403085824494, + "one": 0.010607521697203472, + "only": 0.0, + "or": 0.0009643201542912247, + "other": 0.0009643201542912247, + "our": 0.0009643201542912247, + "ours": 0.0, + "ourselves": 0.0, + "out": 0.0009643201542912247, + "outside": 0.0009643201542912247, + "over": 0.0009643201542912247, + "own": 0.0, + "people": 0.0, + "per": 0.0, + "perhaps": 0.0, + "plus": 0.0, + "rather": 0.0, + "regarding": 0.0, + "same": 0.0, + "see": 0.0009643201542912247, + "she": 0.006750241080038573, + "she'd": 0.0, + "she'll": 0.0009643201542912247, + "she's": 0.0, + "should": 0.0, + "shouldn't": 0.0, + "since": 0.0, + "so": 0.0, + "some": 0.0009643201542912247, + "still": 0.0028929604628736743, + "such": 0.0, + "than": 0.0, + "that": 0.008678881388621022, + "the": 0.07328833172613308, + "their": 0.0, + "theirs": 0.0, + "them": 0.0028929604628736743, + "themselves": 0.0, + "then": 0.0009643201542912247, + "there": 0.0019286403085824494, + "therefore": 0.0, + "these": 0.0, + "they": 0.0028929604628736743, + "they'd": 0.0, + "they'll": 0.0009643201542912247, + "they're": 0.0009643201542912247, + "they've": 0.0, + "this": 0.003857280617164899, + "those": 0.0, + "though": 0.0, + "through": 0.0019286403085824494, + "thus": 0.0, + "to": 0.0057859209257473485, + "toward": 0.0, + "towards": 0.0, + "two": 0.0, + "under": 0.0028929604628736743, + "until": 0.0, + "up": 0.0, + "upon": 0.0, + "us": 0.0009643201542912247, + "versus": 0.0, + "very": 0.0, + "via": 0.0, + "was": 0.009643201542912247, + "wasn't": 0.0028929604628736743, + "we": 0.003857280617164899, + "we'd": 0.0, + "we'll": 0.0, + "we're": 0.0, + "we've": 0.0009643201542912247, + "well": 0.0, + "were": 0.003857280617164899, + "weren't": 0.0, + "what": 0.0009643201542912247, + "when": 0.003857280617164899, + "where": 0.0009643201542912247, + "whether": 0.0, + "which": 0.0009643201542912247, + "while": 0.0, + "who": 0.0009643201542912247, + "whom": 0.0, + "whose": 0.0, + "why": 0.0, + "will": 0.0028929604628736743, + "with": 0.010607521697203472, + "within": 0.0, + "without": 0.0009643201542912247, + "won't": 0.0019286403085824494, + "would": 0.0, + "wouldn't": 0.0, + "yet": 0.0009643201542912247, + "you": 0.0028929604628736743, + "you'd": 0.0, + "you'll": 0.0, + "you're": 0.0, + "you've": 0.0, + "your": 0.0, + "yours": 0.0, + "yourself": 0.0, + "yourselves": 0.0 + }, + "metadata": { + "doc_count": 4, + "genre_warning": "profile has fewer than 2000 words", + "low_confidence": true, + "total_words": 1037 + }, + "mtld": 172.83333333333334, + "paragraph_stats": { + "count": 31, + "mean_words": 33.45161290322581 + }, + "punctuation": { + "!": 0.0, + "\"": 0.003857280617164899, + "'": 0.03085824493731919, + "(": 0.0, + ")": 0.0, + ",": 0.03567984570877531, + "-": 0.0, + ".": 0.1600771456123433, + ":": 0.0, + ";": 0.0, + "?": 0.0009643201542912247 + }, + "sentence_lengths": { + "iqr": 5, + "lengths": [ + 8, + 14, + 5, + 6, + 7, + 4, + 3, + 11, + 6, + 5, + 2, + 2, + 6, + 5, + 8, + 3, + 5, + 2, + 5, + 13, + 8, + 3, + 5, + 6, + 4, + 6, + 13, + 9, + 5, + 4, + 6, + 14, + 12, + 6, + 4, + 3, + 9, + 8, + 3, + 3, + 10, + 5, + 3, + 12, + 12, + 5, + 4, + 9, + 3, + 2, + 2, + 8, + 20, + 9, + 9, + 2, + 11, + 3, + 1, + 1, + 1, + 8, + 5, + 9, + 4, + 3, + 5, + 9, + 1, + 1, + 3, + 3, + 11, + 2, + 2, + 2, + 6, + 3, + 8, + 5, + 8, + 6, + 9, + 7, + 4, + 8, + 8, + 2, + 11, + 5, + 5, + 4, + 10, + 2, + 2, + 6, + 8, + 6, + 4, + 3, + 1, + 1, + 2, + 8, + 7, + 17, + 2, + 2, + 8, + 21, + 8, + 8, + 6, + 5, + 3, + 3, + 10, + 4, + 11, + 10, + 5, + 5, + 10, + 4, + 10, + 4, + 3, + 7, + 6, + 3, + 16, + 9, + 5, + 16, + 6, + 8, + 12, + 4, + 6, + 3, + 4, + 2, + 8, + 8, + 4, + 5, + 16, + 4, + 5, + 3, + 8, + 8, + 6, + 3, + 4, + 16, + 2, + 3, + 4, + 6, + 4, + 5, + 14, + 4, + 8, + 2, + 11 + ], + "median": 5 + }, + "total_words": 1037, + "word_length_histogram": { + "1": 0.0742526518804243, + "10": 0.003857280617164899, + "11": 0.0019286403085824494, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "2": 0.09546769527483125, + "3": 0.21793635486981677, + "4": 0.2391513982642237, + "5": 0.16586306653809066, + "6": 0.10028929604628736, + "7": 0.07328833172613308, + "8": 0.015429122468659595, + "9": 0.01253616200578592 + } +} diff --git a/evals/fixtures/voice/profiles/boris.json b/evals/fixtures/voice/profiles/boris.json new file mode 100644 index 0000000..efa0c1a --- /dev/null +++ b/evals/fixtures/voice/profiles/boris.json @@ -0,0 +1,3157 @@ +{ + "char3": { + " \"c": 1, + " \"p": 1, + " 2a": 1, + " 3b": 1, + " a ": 40, + " ab": 3, + " ac": 4, + " ad": 5, + " af": 2, + " ag": 2, + " al": 9, + " an": 35, + " ap": 6, + " ar": 7, + " as": 1, + " at": 6, + " aw": 1, + " ba": 5, + " be": 31, + " bi": 3, + " bl": 1, + " bo": 4, + " br": 3, + " bu": 10, + " by": 9, + " ca": 13, + " ce": 2, + " ch": 9, + " cl": 11, + " co": 30, + " cr": 1, + " cu": 4, + " cy": 1, + " da": 4, + " de": 12, + " di": 8, + " do": 6, + " dr": 3, + " du": 1, + " ea": 3, + " ef": 2, + " ei": 1, + " el": 1, + " en": 3, + " er": 3, + " ev": 6, + " ex": 6, + " fa": 7, + " fe": 2, + " fi": 5, + " fl": 2, + " fo": 4, + " fr": 4, + " fu": 1, + " ga": 1, + " gi": 1, + " go": 2, + " gr": 7, + " ha": 24, + " he": 1, + " hi": 4, + " ho": 3, + " hu": 1, + " hy": 1, + " i ": 18, + " im": 6, + " in": 24, + " ir": 1, + " is": 11, + " it": 20, + " ja": 1, + " ke": 1, + " ki": 1, + " la": 8, + " le": 9, + " li": 17, + " lo": 4, + " ma": 17, + " me": 3, + " mi": 6, + " mo": 8, + " mr": 1, + " mu": 2, + " my": 6, + " na": 2, + " ne": 11, + " ni": 2, + " no": 22, + " ob": 2, + " oc": 3, + " of": 23, + " ol": 2, + " on": 16, + " op": 6, + " or": 7, + " ot": 1, + " ou": 3, + " ov": 1, + " ow": 2, + " pa": 14, + " pe": 6, + " ph": 4, + " pl": 6, + " po": 1, + " pr": 17, + " pu": 2, + " qu": 3, + " ra": 6, + " re": 27, + " ro": 3, + " ru": 1, + " sa": 1, + " sc": 6, + " se": 6, + " sh": 8, + " si": 5, + " sl": 3, + " sm": 6, + " so": 8, + " sp": 2, + " st": 17, + " su": 8, + " sw": 1, + " ta": 5, + " te": 3, + " th": 124, + " ti": 1, + " to": 19, + " tr": 7, + " tu": 2, + " tw": 2, + " un": 12, + " ur": 1, + " us": 3, + " va": 2, + " ve": 1, + " vi": 3, + " wa": 15, + " we": 8, + " wh": 18, + " wi": 19, + " wo": 5, + " wr": 1, + " ye": 1, + "\" o": 1, + "\" u": 1, + "\"ca": 1, + "\"pl": 1, + "'s ": 3, + ", a": 26, + ", b": 6, + ", c": 5, + ", d": 3, + ", e": 2, + ", h": 1, + ", i": 6, + ", l": 3, + ", n": 1, + ", o": 5, + ", p": 5, + ", r": 1, + ", s": 4, + ", t": 16, + ", u": 2, + ", w": 8, + ", y": 1, + ",\" ": 1, + "-sp": 1, + ". a": 5, + ". b": 4, + ". c": 2, + ". e": 3, + ". i": 12, + ". l": 1, + ". m": 1, + ". n": 2, + ". o": 2, + ". p": 2, + ". s": 2, + ". t": 17, + ". w": 3, + "2a ": 1, + "3b ": 1, + ": p": 1, + ": w": 1, + "; a": 1, + "; h": 1, + "; n": 1, + "; o": 1, + "; s": 1, + "; t": 6, + "a a": 1, + "a b": 4, + "a c": 3, + "a d": 1, + "a f": 2, + "a g": 1, + "a k": 1, + "a l": 2, + "a m": 5, + "a n": 1, + "a p": 7, + "a s": 8, + "a t": 3, + "a u": 1, + "a v": 1, + "a w": 1, + "a, ": 1, + "a,\"": 1, + "a. ": 1, + "abi": 4, + "abl": 5, + "abo": 3, + "aca": 1, + "acc": 2, + "ace": 4, + "ach": 11, + "aci": 1, + "ack": 4, + "acr": 2, + "act": 4, + "acy": 2, + "ad ": 10, + "add": 2, + "ade": 1, + "adi": 5, + "adj": 1, + "adl": 2, + "adm": 1, + "aff": 1, + "aft": 2, + "ag,": 1, + "aga": 1, + "age": 2, + "ail": 4, + "aim": 2, + "ain": 20, + "air": 5, + "ait": 1, + "aja": 1, + "ake": 1, + "aki": 1, + "al ": 11, + "al,": 2, + "al.": 1, + "ale": 1, + "ali": 1, + "alk": 1, + "all": 15, + "alm": 2, + "alo": 1, + "alt": 4, + "alu": 1, + "alv": 1, + "ama": 2, + "ame": 1, + "ami": 3, + "amp": 2, + "an ": 11, + "an,": 1, + "anc": 3, + "and": 37, + "ane": 2, + "ang": 4, + "ann": 3, + "ans": 1, + "ant": 3, + "anv": 1, + "any": 1, + "aor": 1, + "apa": 1, + "ape": 5, + "aph": 1, + "app": 5, + "aps": 1, + "ar ": 2, + "ar.": 1, + "ara": 1, + "arb": 1, + "arc": 1, + "ard": 3, + "are": 13, + "arg": 3, + "ari": 1, + "ark": 3, + "arl": 2, + "arm": 3, + "arr": 4, + "art": 3, + "ary": 6, + "as ": 17, + "as.": 1, + "asa": 1, + "ase": 4, + "ash": 3, + "asi": 2, + "ask": 2, + "asm": 1, + "aso": 1, + "ast": 5, + "asy": 1, + "at ": 16, + "at,": 1, + "atc": 2, + "ate": 15, + "ath": 4, + "ati": 11, + "atl": 1, + "ato": 1, + "atr": 1, + "att": 5, + "aun": 1, + "aus": 9, + "ave": 4, + "avi": 2, + "awe": 1, + "awn": 1, + "ay ": 4, + "ay,": 2, + "ay.": 1, + "ayo": 1, + "ays": 1, + "b o": 1, + "bac": 1, + "bag": 1, + "bas": 2, + "bat": 1, + "be ": 5, + "bec": 11, + "bee": 4, + "bef": 3, + "beg": 2, + "bei": 1, + "bel": 1, + "bes": 2, + "bet": 2, + "bic": 2, + "bin": 2, + "bio": 1, + "bit": 2, + "bje": 1, + "ble": 9, + "bli": 4, + "boo": 2, + "bor": 1, + "bot": 1, + "bou": 3, + "box": 1, + "bra": 2, + "bri": 1, + "bro": 2, + "bse": 1, + "bui": 3, + "but": 9, + "by ": 9, + "by,": 1, + "c b": 1, + "c l": 1, + "c o": 1, + "c s": 1, + "c, ": 2, + "cab": 2, + "cal": 6, + "can": 6, + "car": 4, + "cas": 1, + "cat": 2, + "cau": 7, + "cca": 1, + "cce": 4, + "ccu": 2, + "ce ": 16, + "ce,": 1, + "ce.": 5, + "ced": 4, + "cee": 1, + "cei": 1, + "cen": 3, + "cep": 3, + "cer": 4, + "ces": 8, + "ch ": 9, + "cha": 4, + "che": 7, + "chi": 12, + "chm": 1, + "cho": 1, + "cia": 2, + "cie": 2, + "cif": 1, + "cin": 1, + "cip": 2, + "cis": 1, + "ck ": 2, + "ck,": 1, + "ck.": 1, + "cke": 2, + "cks": 1, + "cky": 1, + "cla": 3, + "cle": 7, + "cli": 1, + "clo": 3, + "clu": 3, + "cof": 2, + "cog": 1, + "coi": 1, + "col": 1, + "com": 9, + "con": 14, + "coo": 1, + "cop": 1, + "cor": 5, + "cot": 1, + "cou": 3, + "cre": 2, + "cro": 2, + "ct ": 3, + "ct.": 1, + "cte": 2, + "cti": 4, + "ctl": 1, + "cto": 1, + "cuf": 3, + "cup": 2, + "cur": 1, + "cus": 3, + "cy ": 3, + "cy,": 1, + "cyc": 3, + "d a": 18, + "d b": 13, + "d c": 6, + "d d": 3, + "d e": 1, + "d f": 5, + "d g": 2, + "d i": 9, + "d j": 1, + "d l": 3, + "d m": 3, + "d n": 6, + "d o": 2, + "d p": 1, + "d r": 1, + "d s": 4, + "d t": 22, + "d u": 4, + "d v": 2, + "d w": 8, + "d, ": 6, + "d. ": 2, + "dab": 1, + "dam": 3, + "dar": 2, + "day": 3, + "ddr": 2, + "de ": 3, + "de;": 1, + "ded": 4, + "def": 3, + "del": 1, + "den": 6, + "dep": 2, + "deq": 1, + "der": 8, + "des": 2, + "dev": 3, + "dge": 1, + "dia": 3, + "dic": 1, + "did": 1, + "din": 11, + "dio": 1, + "dir": 1, + "dis": 6, + "dit": 2, + "dja": 1, + "dle": 2, + "dly": 2, + "dmi": 1, + "do ": 2, + "don": 1, + "doo": 2, + "dor": 1, + "dow": 2, + "dra": 1, + "dre": 3, + "dri": 2, + "dro": 1, + "dry": 1, + "ds ": 8, + "ds,": 1, + "ds.": 1, + "ds;": 1, + "duc": 3, + "dul": 2, + "dur": 1, + "dus": 1, + "e \"": 1, + "e a": 15, + "e b": 5, + "e c": 18, + "e d": 6, + "e e": 3, + "e f": 5, + "e g": 2, + "e h": 7, + "e i": 20, + "e l": 10, + "e m": 17, + "e n": 7, + "e o": 14, + "e p": 14, + "e r": 12, + "e s": 18, + "e t": 22, + "e u": 1, + "e w": 12, + "e's": 1, + "e, ": 18, + "e. ": 11, + "e: ": 1, + "e; ": 2, + "ea,": 1, + "eac": 3, + "ead": 3, + "ean": 3, + "ear": 9, + "eas": 5, + "eat": 5, + "eca": 7, + "ece": 2, + "eci": 2, + "eck": 1, + "eco": 7, + "ect": 5, + "ed ": 63, + "ed,": 4, + "ed.": 2, + "edi": 2, + "edl": 1, + "edu": 4, + "ee ": 2, + "ee,": 2, + "eed": 2, + "eek": 2, + "een": 4, + "eep": 1, + "eet": 2, + "eev": 1, + "efe": 4, + "eff": 2, + "efl": 1, + "efo": 3, + "efr": 1, + "eft": 2, + "efu": 2, + "egi": 3, + "egr": 1, + "egs": 1, + "ehe": 1, + "ein": 1, + "eir": 1, + "eis": 2, + "eit": 1, + "eiv": 1, + "ek ": 1, + "ek.": 1, + "el ": 1, + "ela": 1, + "ele": 3, + "elf": 1, + "eli": 1, + "ell": 2, + "elo": 4, + "elp": 1, + "els": 1, + "elt": 1, + "elv": 1, + "ely": 3, + "em ": 1, + "em.": 1, + "em;": 1, + "ema": 2, + "eme": 1, + "emo": 2, + "emp": 1, + "en ": 12, + "en'": 1, + "en,": 1, + "en.": 1, + "enc": 10, + "end": 5, + "ene": 1, + "enf": 1, + "eni": 5, + "ens": 4, + "ent": 20, + "eon": 4, + "eop": 2, + "ep.": 1, + "epa": 1, + "epe": 4, + "ept": 3, + "equ": 6, + "er ": 34, + "er'": 1, + "er,": 5, + "er.": 2, + "er:": 1, + "era": 5, + "erc": 1, + "ere": 12, + "erf": 2, + "erh": 1, + "eri": 4, + "erk": 1, + "erl": 1, + "erm": 1, + "ern": 4, + "erp": 1, + "err": 2, + "ers": 9, + "ert": 4, + "erv": 4, + "ery": 5, + "es ": 20, + "es,": 6, + "es.": 2, + "esc": 1, + "esd": 1, + "ese": 4, + "esh": 1, + "esi": 4, + "esk": 1, + "esp": 1, + "ess": 11, + "est": 3, + "esu": 1, + "esy": 1, + "et ": 6, + "et,": 2, + "et.": 1, + "eta": 1, + "ete": 2, + "eth": 2, + "eti": 4, + "etl": 1, + "ett": 4, + "etu": 2, + "etw": 1, + "ety": 1, + "eva": 3, + "eve": 12, + "evi": 2, + "ew ": 1, + "ewd": 1, + "ews": 1, + "exa": 1, + "exc": 1, + "exp": 2, + "ext": 2, + "ey ": 3, + "ez ": 1, + "f a": 3, + "f b": 2, + "f c": 1, + "f d": 1, + "f h": 3, + "f i": 1, + "f p": 2, + "f r": 2, + "f s": 1, + "f t": 6, + "f w": 1, + "fa.": 1, + "fac": 1, + "fai": 5, + "fam": 1, + "fea": 1, + "fec": 2, + "fee": 3, + "fel": 1, + "fen": 2, + "fer": 2, + "ff ": 3, + "ffe": 4, + "ffi": 4, + "ffs": 1, + "fic": 4, + "fid": 1, + "fil": 2, + "fin": 1, + "fir": 2, + "fla": 1, + "flo": 1, + "fly": 1, + "fol": 2, + "for": 8, + "fou": 1, + "fri": 1, + "fro": 4, + "fs ": 1, + "ft ": 4, + "fte": 3, + "ful": 4, + "fus": 1, + "fy ": 1, + "fyi": 1, + "g a": 2, + "g b": 2, + "g c": 2, + "g e": 3, + "g i": 1, + "g l": 2, + "g o": 2, + "g p": 3, + "g s": 3, + "g t": 4, + "g w": 1, + "g, ": 5, + "g. ": 3, + "gai": 1, + "gat": 2, + "ge ": 3, + "ge.": 1, + "ged": 2, + "gen": 2, + "ger": 2, + "ges": 1, + "get": 1, + "ggl": 1, + "gh ": 6, + "ght": 8, + "gib": 1, + "gie": 1, + "gin": 2, + "giv": 1, + "gle": 1, + "gna": 1, + "gni": 2, + "gov": 2, + "gra": 3, + "gre": 2, + "gri": 2, + "gro": 3, + "gs,": 1, + "gue": 1, + "gui": 1, + "h a": 3, + "h b": 1, + "h c": 5, + "h d": 2, + "h e": 1, + "h f": 1, + "h h": 1, + "h i": 3, + "h m": 1, + "h n": 2, + "h o": 1, + "h p": 1, + "h r": 3, + "h t": 3, + "h w": 2, + "h. ": 2, + "hab": 2, + "had": 9, + "hai": 1, + "hak": 1, + "hal": 2, + "han": 7, + "hap": 1, + "har": 4, + "has": 6, + "hat": 10, + "hav": 3, + "he ": 90, + "hea": 3, + "hec": 1, + "hed": 3, + "hee": 1, + "hei": 1, + "hel": 3, + "hem": 2, + "hen": 7, + "her": 15, + "hes": 5, + "hey": 3, + "hic": 5, + "hie": 3, + "hil": 1, + "hin": 13, + "hip": 1, + "his": 6, + "hme": 1, + "hol": 1, + "hom": 1, + "hoo": 1, + "hos": 2, + "hou": 13, + "how": 1, + "hra": 2, + "hre": 3, + "hro": 2, + "ht ": 3, + "ht,": 1, + "ht.": 1, + "hte": 1, + "htl": 1, + "hts": 1, + "hur": 1, + "hus": 1, + "hy ": 2, + "hyg": 1, + "hys": 1, + "i a": 1, + "i c": 4, + "i d": 1, + "i f": 1, + "i h": 4, + "i l": 2, + "i r": 2, + "i s": 1, + "i t": 1, + "i u": 1, + "ial": 4, + "ian": 1, + "iar": 1, + "ias": 1, + "iat": 3, + "ibl": 3, + "ibr": 2, + "ibu": 2, + "ic ": 4, + "ic,": 2, + "ica": 4, + "ice": 4, + "ich": 5, + "ici": 4, + "ick": 1, + "ict": 1, + "icy": 2, + "id ": 2, + "ida": 1, + "ide": 8, + "ido": 1, + "ied": 6, + "ief": 1, + "ien": 7, + "ier": 2, + "ies": 4, + "iet": 3, + "iev": 2, + "ift": 2, + "ify": 2, + "iga": 1, + "igh": 8, + "ign": 2, + "ike": 4, + "il ": 3, + "ilb": 1, + "ild": 5, + "ili": 1, + "ill": 4, + "ils": 1, + "ilt": 1, + "ilu": 1, + "im ": 2, + "ima": 1, + "ime": 2, + "imi": 1, + "imm": 2, + "imp": 4, + "in ": 11, + "in,": 1, + "ina": 5, + "inc": 7, + "ind": 4, + "ine": 17, + "inf": 1, + "ing": 32, + "ini": 3, + "ink": 1, + "inn": 1, + "ino": 2, + "ins": 9, + "int": 19, + "io,": 1, + "iog": 1, + "ion": 23, + "iou": 2, + "ip,": 1, + "ipa": 1, + "ipl": 1, + "ips": 1, + "ir ": 5, + "ire": 5, + "irm": 1, + "irr": 1, + "irs": 2, + "is ": 15, + "isc": 1, + "ise": 5, + "isg": 1, + "isi": 2, + "isj": 1, + "ism": 1, + "isp": 3, + "isr": 1, + "ist": 5, + "isu": 2, + "it ": 10, + "it.": 2, + "ita": 1, + "itc": 1, + "ite": 1, + "ith": 18, + "iti": 6, + "its": 12, + "itt": 2, + "itu": 2, + "ity": 2, + "iva": 2, + "ive": 11, + "ix ": 1, + "iya": 1, + "iza": 1, + "jac": 2, + "jam": 1, + "jec": 1, + "jud": 1, + "k a": 2, + "k c": 1, + "k i": 3, + "k o": 1, + "k, ": 2, + "k. ": 3, + "k; ": 1, + "kab": 1, + "ke ": 4, + "ked": 2, + "ken": 1, + "ket": 4, + "kin": 3, + "ks ": 1, + "ks.": 1, + "ks;": 1, + "ky ": 1, + "l a": 2, + "l c": 1, + "l d": 2, + "l e": 1, + "l f": 1, + "l g": 1, + "l h": 2, + "l i": 1, + "l n": 1, + "l o": 6, + "l p": 1, + "l r": 1, + "l s": 3, + "l t": 3, + "l v": 1, + "l w": 1, + "l, ": 3, + "l. ": 2, + "lac": 4, + "lai": 5, + "lam": 2, + "lar": 3, + "las": 5, + "lat": 3, + "lau": 1, + "lay": 1, + "lbo": 1, + "ld ": 4, + "ld,": 2, + "lde": 3, + "ldi": 3, + "ldr": 1, + "le ": 15, + "le'": 1, + "le,": 4, + "le.": 1, + "lea": 6, + "led": 3, + "lee": 1, + "lef": 2, + "leg": 2, + "lei": 2, + "lem": 1, + "len": 1, + "leo": 1, + "ler": 1, + "les": 3, + "lev": 1, + "lf ": 1, + "lia": 1, + "lib": 2, + "lic": 3, + "lid": 1, + "lie": 2, + "lif": 2, + "lig": 7, + "lik": 4, + "lin": 6, + "lip": 1, + "lis": 1, + "lit": 1, + "liv": 1, + "lk,": 1, + "ll ": 11, + "ll,": 1, + "lle": 1, + "lli": 2, + "llw": 2, + "lly": 5, + "lmo": 2, + "loc": 2, + "lon": 1, + "loo": 2, + "lop": 3, + "lor": 1, + "los": 2, + "lot": 3, + "low": 3, + "lpf": 1, + "ls ": 1, + "ls.": 1, + "lt ": 1, + "lt,": 1, + "lte": 1, + "lth": 4, + "lud": 1, + "lue": 1, + "lur": 1, + "lus": 2, + "lva": 1, + "lve": 1, + "lwa": 2, + "ly ": 12, + "ly,": 6, + "ly.": 1, + "ly;": 1, + "m 3": 1, + "m a": 1, + "m h": 1, + "m i": 1, + "m o": 2, + "m s": 1, + "m t": 3, + "m w": 2, + "m, ": 2, + "m. ": 1, + "m; ": 1, + "mac": 8, + "mag": 1, + "mai": 3, + "mak": 1, + "mal": 4, + "man": 1, + "mar": 2, + "mas": 1, + "mat": 8, + "may": 1, + "mbl": 1, + "me ": 3, + "me,": 3, + "med": 2, + "mel": 2, + "mem": 1, + "men": 3, + "meo": 3, + "mer": 1, + "mes": 2, + "met": 2, + "mil": 2, + "min": 7, + "mis": 4, + "mit": 1, + "mme": 2, + "moc": 1, + "mon": 2, + "moo": 1, + "mor": 5, + "mos": 3, + "mov": 1, + "mp ": 1, + "mp.": 1, + "mpa": 1, + "mpe": 2, + "mpl": 2, + "mpo": 1, + "mpr": 3, + "mrs": 1, + "ms ": 1, + "ms,": 1, + "mus": 2, + "my ": 6, + "n \"": 1, + "n 2": 1, + "n a": 8, + "n b": 2, + "n c": 5, + "n d": 1, + "n e": 1, + "n f": 1, + "n h": 1, + "n i": 3, + "n k": 1, + "n m": 5, + "n n": 2, + "n o": 4, + "n p": 1, + "n r": 2, + "n s": 1, + "n t": 8, + "n u": 1, + "n w": 5, + "n's": 1, + "n, ": 9, + "n. ": 4, + "nal": 1, + "nar": 5, + "nat": 3, + "nav": 1, + "nce": 20, + "ncl": 3, + "nco": 2, + "ncy": 2, + "nd ": 34, + "nda": 3, + "nde": 5, + "ndi": 4, + "ndl": 1, + "ndo": 1, + "ndr": 1, + "nds": 7, + "ne ": 15, + "ne,": 4, + "ne.": 1, + "ne;": 1, + "nea": 6, + "nec": 1, + "ned": 10, + "nee": 1, + "ner": 5, + "nes": 3, + "net": 2, + "nev": 1, + "new": 2, + "nfi": 1, + "nfo": 2, + "ng ": 25, + "ng,": 4, + "ng.": 3, + "nge": 5, + "nhe": 1, + "nia": 1, + "nic": 1, + "nie": 2, + "nig": 1, + "nim": 1, + "nin": 8, + "nio": 1, + "niz": 1, + "nki": 1, + "nli": 1, + "nlo": 1, + "nne": 1, + "nno": 2, + "nnu": 1, + "no ": 4, + "nod": 1, + "non": 1, + "noo": 2, + "nor": 3, + "not": 15, + "nou": 1, + "nre": 1, + "ns ": 12, + "ns,": 2, + "ns.": 1, + "ns;": 1, + "nse": 2, + "nsf": 1, + "nsi": 5, + "nst": 4, + "nt ": 15, + "nt\"": 1, + "nt,": 1, + "nt-": 1, + "nt.": 2, + "nta": 2, + "nte": 11, + "nth": 1, + "nti": 4, + "ntl": 4, + "nto": 3, + "ntr": 1, + "nts": 2, + "ntu": 1, + "nua": 1, + "nus": 1, + "nva": 1, + "nve": 4, + "nwi": 1, + "ny ": 1, + "o a": 3, + "o c": 1, + "o d": 1, + "o e": 2, + "o l": 2, + "o m": 1, + "o n": 1, + "o o": 2, + "o p": 1, + "o r": 2, + "o s": 2, + "o t": 4, + "o u": 1, + "o v": 2, + "o, ": 1, + "oad": 2, + "obj": 1, + "obl": 1, + "obs": 1, + "oca": 1, + "occ": 3, + "oce": 3, + "oci": 1, + "ock": 2, + "odi": 1, + "ods": 1, + "odu": 2, + "of ": 19, + "ofa": 1, + "off": 6, + "oft": 1, + "oge": 1, + "ogn": 1, + "ogr": 2, + "oin": 1, + "oir": 1, + "ok ": 1, + "oke": 1, + "oks": 1, + "old": 5, + "olo": 1, + "om ": 5, + "oma": 1, + "ome": 9, + "omi": 2, + "omp": 4, + "oms": 2, + "on ": 17, + "on,": 5, + "on.": 3, + "ona": 2, + "onc": 6, + "ond": 2, + "one": 12, + "onf": 1, + "ong": 1, + "oni": 1, + "ons": 13, + "ont": 3, + "onv": 4, + "ook": 3, + "oom": 3, + "oon": 2, + "oop": 1, + "oor": 3, + "oos": 1, + "oot": 1, + "ope": 8, + "opi": 2, + "opl": 2, + "opp": 1, + "opr": 1, + "opt": 1, + "or ": 3, + "or,": 3, + "or.": 1, + "or;": 1, + "ora": 5, + "orc": 1, + "ord": 6, + "ore": 6, + "ork": 2, + "orm": 4, + "orn": 5, + "orr": 3, + "ors": 2, + "ort": 1, + "ory": 3, + "ose": 4, + "osi": 1, + "oss": 3, + "ost": 3, + "ot ": 9, + "ot,": 1, + "ote": 2, + "oth": 8, + "oti": 1, + "ott": 1, + "ouc": 1, + "oug": 6, + "oul": 3, + "oun": 6, + "oup": 1, + "our": 2, + "ous": 3, + "out": 11, + "ove": 6, + "ow ": 4, + "owe": 3, + "own": 3, + "oxe": 1, + "p s": 1, + "p t": 1, + "p, ": 1, + "p. ": 2, + "pai": 7, + "paj": 1, + "pal": 1, + "pan": 1, + "pap": 2, + "par": 2, + "pat": 4, + "pau": 2, + "pe,": 1, + "pea": 2, + "pec": 1, + "ped": 6, + "pen": 6, + "peo": 2, + "per": 11, + "pet": 3, + "pfu": 1, + "pha": 1, + "phr": 2, + "phy": 2, + "pie": 3, + "pin": 1, + "pit": 1, + "pla": 10, + "ple": 4, + "pli": 3, + "por": 2, + "pos": 1, + "ppe": 3, + "ppl": 2, + "ppo": 1, + "ppr": 1, + "pra": 1, + "pre": 6, + "pri": 7, + "pro": 10, + "ps ": 2, + "pt ": 2, + "pta": 1, + "pti": 2, + "pub": 2, + "qua": 2, + "que": 3, + "qui": 4, + "r a": 4, + "r b": 2, + "r c": 1, + "r d": 3, + "r f": 1, + "r h": 3, + "r i": 1, + "r l": 3, + "r m": 1, + "r n": 1, + "r o": 4, + "r p": 1, + "r q": 1, + "r r": 3, + "r s": 4, + "r t": 8, + "r w": 4, + "r's": 1, + "r, ": 8, + "r. ": 4, + "r: ": 1, + "r; ": 1, + "ra,": 1, + "rac": 3, + "rad": 3, + "rai": 2, + "ral": 3, + "ran": 6, + "rao": 1, + "rap": 1, + "rar": 5, + "ras": 4, + "rat": 4, + "rav": 2, + "raw": 1, + "ray": 1, + "rby": 1, + "rce": 1, + "rch": 1, + "rcu": 1, + "rd ": 1, + "rde": 2, + "rdi": 3, + "rds": 3, + "re ": 19, + "re,": 2, + "re.": 1, + "rea": 5, + "rec": 6, + "red": 8, + "ree": 2, + "ref": 3, + "reg": 1, + "reh": 1, + "rel": 3, + "rem": 3, + "ren": 2, + "rep": 3, + "req": 2, + "res": 16, + "ret": 5, + "rev": 1, + "rew": 1, + "rez": 1, + "rfe": 1, + "rfo": 1, + "rge": 3, + "rgu": 1, + "rha": 1, + "ria": 2, + "rib": 2, + "ric": 1, + "rid": 2, + "rie": 7, + "rif": 1, + "rig": 1, + "rim": 1, + "rin": 2, + "rio": 2, + "rit": 2, + "riv": 5, + "riy": 1, + "rk ": 2, + "rk.": 1, + "rk;": 1, + "rka": 1, + "rks": 1, + "rle": 1, + "rli": 1, + "rly": 2, + "rm ": 3, + "rm,": 1, + "rma": 4, + "rmi": 1, + "rn ": 2, + "rne": 6, + "rni": 2, + "rno": 1, + "roa": 2, + "rob": 1, + "roc": 3, + "rod": 2, + "rog": 1, + "rom": 5, + "ron": 1, + "roo": 3, + "rop": 1, + "ros": 2, + "rou": 4, + "rov": 2, + "row": 1, + "rpr": 1, + "rra": 2, + "rre": 1, + "rri": 7, + "rro": 1, + "rs ": 5, + "rs,": 2, + "rs.": 2, + "rs;": 1, + "rsa": 1, + "rst": 2, + "rsu": 1, + "rt ": 1, + "rta": 1, + "rte": 3, + "rth": 1, + "rti": 2, + "rtm": 1, + "ruc": 2, + "rug": 1, + "rul": 1, + "rup": 1, + "rus": 1, + "rva": 1, + "rve": 2, + "rvi": 1, + "rvo": 1, + "ry ": 10, + "ry,": 2, + "ry.": 1, + "rye": 1, + "ryi": 1, + "ryo": 1, + "s a": 17, + "s b": 10, + "s c": 4, + "s d": 3, + "s e": 3, + "s f": 2, + "s g": 2, + "s h": 6, + "s i": 7, + "s l": 1, + "s m": 4, + "s n": 5, + "s o": 8, + "s p": 4, + "s q": 1, + "s r": 1, + "s s": 5, + "s t": 14, + "s u": 3, + "s w": 6, + "s, ": 17, + "s. ": 12, + "s; ": 4, + "sam": 1, + "san": 2, + "sat": 1, + "sce": 2, + "sch": 2, + "sci": 1, + "scr": 1, + "scu": 2, + "sda": 1, + "se ": 17, + "se.": 1, + "sec": 1, + "sed": 6, + "sef": 1, + "sel": 1, + "sen": 1, + "seq": 3, + "ser": 4, + "ses": 3, + "set": 1, + "sev": 1, + "sfo": 1, + "sfu": 1, + "sgu": 1, + "sh ": 2, + "sha": 3, + "she": 4, + "sho": 3, + "sia": 1, + "sib": 2, + "sic": 1, + "sid": 6, + "sig": 1, + "sin": 4, + "sio": 4, + "sis": 1, + "sit": 3, + "siv": 2, + "six": 1, + "sju": 1, + "sk ": 1, + "ske": 2, + "sle": 1, + "slo": 2, + "sm ": 1, + "sm,": 1, + "sma": 3, + "sme": 2, + "smo": 1, + "sne": 1, + "soc": 1, + "sof": 2, + "som": 4, + "son": 1, + "sou": 1, + "spa": 2, + "spe": 2, + "spi": 1, + "spl": 2, + "spr": 1, + "sru": 1, + "ss ": 5, + "ss,": 2, + "ss.": 3, + "sse": 2, + "ssf": 1, + "ssi": 2, + "st ": 13, + "st;": 1, + "sta": 9, + "ste": 2, + "sti": 7, + "sto": 5, + "str": 6, + "stu": 2, + "sua": 3, + "suc": 4, + "sul": 1, + "sun": 1, + "sup": 2, + "sur": 3, + "swi": 1, + "sy ": 2, + "t a": 6, + "t b": 11, + "t c": 9, + "t d": 1, + "t e": 1, + "t f": 4, + "t g": 3, + "t h": 3, + "t i": 11, + "t l": 5, + "t m": 4, + "t n": 4, + "t o": 4, + "t p": 3, + "t r": 4, + "t s": 4, + "t t": 11, + "t u": 1, + "t w": 7, + "t\" ": 1, + "t, ": 8, + "t-s": 1, + "t. ": 7, + "t; ": 1, + "tab": 2, + "tac": 2, + "taf": 1, + "tai": 7, + "tal": 1, + "tan": 3, + "tap": 3, + "tas": 1, + "tat": 1, + "tch": 3, + "te ": 5, + "te.": 1, + "tea": 2, + "ted": 15, + "tel": 2, + "tem": 1, + "ten": 4, + "ter": 24, + "tes": 2, + "th ": 13, + "th.": 2, + "tha": 13, + "the": 111, + "thi": 7, + "tho": 10, + "thr": 5, + "thu": 1, + "tic": 6, + "tie": 3, + "til": 4, + "tim": 2, + "tin": 2, + "tio": 18, + "tir": 1, + "tis": 1, + "tit": 4, + "tiv": 2, + "tla": 1, + "tle": 3, + "tly": 7, + "tme": 1, + "to ": 19, + "tod": 1, + "tog": 1, + "tom": 1, + "ton": 3, + "top": 1, + "tor": 4, + "tou": 1, + "tow": 1, + "tra": 9, + "tri": 3, + "tru": 4, + "try": 1, + "ts ": 13, + "ts,": 1, + "tse": 1, + "tsi": 1, + "tta": 1, + "tte": 7, + "ttl": 3, + "tto": 3, + "tua": 2, + "tud": 2, + "tue": 1, + "tum": 1, + "tur": 2, + "tut": 1, + "twe": 1, + "twi": 2, + "ty ": 3, + "uac": 1, + "ual": 5, + "uar": 1, + "uas": 1, + "ubl": 2, + "ucc": 2, + "uce": 3, + "uch": 3, + "uct": 2, + "ude": 3, + "udg": 1, + "ue ": 1, + "ued": 1, + "uen": 3, + "ues": 1, + "uff": 3, + "ugg": 1, + "ugh": 6, + "uie": 2, + "uil": 3, + "uir": 2, + "uis": 1, + "ul ": 2, + "ul.": 1, + "uld": 3, + "ule": 3, + "ull": 1, + "ult": 1, + "umb": 1, + "un,": 1, + "unc": 1, + "und": 8, + "unh": 1, + "unl": 2, + "unr": 1, + "unt": 4, + "unu": 1, + "unw": 1, + "up ": 1, + "upe": 1, + "upi": 2, + "upp": 1, + "upt": 1, + "ur ": 1, + "ure": 4, + "urg": 1, + "url": 1, + "urn": 2, + "urr": 1, + "urt": 1, + "urv": 1, + "us ": 2, + "usa": 1, + "use": 11, + "usi": 3, + "usn": 1, + "uss": 1, + "ust": 6, + "usu": 2, + "ut ": 13, + "ut,": 1, + "ute": 3, + "uti": 1, + "uts": 1, + "utt": 2, + "vac": 1, + "val": 2, + "van": 2, + "var": 1, + "vas": 1, + "vat": 3, + "ve ": 8, + "ve,": 1, + "ve:": 1, + "ved": 2, + "vel": 4, + "vem": 1, + "ven": 6, + "ver": 14, + "ves": 4, + "vic": 1, + "vid": 1, + "vig": 1, + "vis": 3, + "vit": 1, + "viv": 1, + "voi": 1, + "w d": 1, + "w p": 1, + "w s": 1, + "w t": 2, + "wai": 1, + "wal": 2, + "war": 1, + "was": 9, + "wat": 1, + "way": 3, + "wdr": 1, + "we ": 3, + "wea": 1, + "wee": 3, + "wel": 1, + "wer": 3, + "wet": 1, + "wev": 1, + "whe": 9, + "whi": 6, + "who": 2, + "why": 1, + "wic": 2, + "wil": 1, + "win": 2, + "wit": 18, + "wn ": 1, + "wne": 1, + "wni": 1, + "wns": 1, + "wor": 4, + "wou": 1, + "wri": 1, + "wsp": 1, + "x f": 1, + "xac": 1, + "xce": 1, + "xes": 1, + "xpe": 1, + "xpl": 1, + "xtr": 2, + "y a": 3, + "y b": 3, + "y c": 3, + "y d": 5, + "y e": 4, + "y h": 1, + "y i": 1, + "y l": 3, + "y m": 1, + "y n": 3, + "y o": 5, + "y p": 2, + "y q": 1, + "y r": 3, + "y s": 6, + "y t": 9, + "y w": 4, + "y, ": 12, + "y. ": 2, + "y; ": 1, + "ya ": 1, + "ycl": 3, + "yer": 1, + "yet": 1, + "ygi": 1, + "yin": 2, + "yon": 1, + "yor": 1, + "ys,": 1, + "ysi": 1, + "z f": 1, + "zab": 1 + }, + "contraction_rate": 0.0027906976744186047, + "function_word_background": { + "a": { + "mean": 0.025, + "std": 0.006 + }, + "about": { + "mean": 0.0025, + "std": 0.006 + }, + "above": { + "mean": 0.0025, + "std": 0.006 + }, + "across": { + "mean": 0.0025, + "std": 0.006 + }, + "after": { + "mean": 0.0025, + "std": 0.006 + }, + "again": { + "mean": 0.0025, + "std": 0.006 + }, + "against": { + "mean": 0.0025, + "std": 0.006 + }, + "all": { + "mean": 0.0025, + "std": 0.006 + }, + "almost": { + "mean": 0.0025, + "std": 0.006 + }, + "along": { + "mean": 0.0025, + "std": 0.006 + }, + "already": { + "mean": 0.0025, + "std": 0.006 + }, + "also": { + "mean": 0.0025, + "std": 0.006 + }, + "although": { + "mean": 0.0025, + "std": 0.006 + }, + "am": { + "mean": 0.0025, + "std": 0.006 + }, + "among": { + "mean": 0.0025, + "std": 0.006 + }, + "an": { + "mean": 0.0025, + "std": 0.006 + }, + "and": { + "mean": 0.025, + "std": 0.006 + }, + "another": { + "mean": 0.0025, + "std": 0.006 + }, + "any": { + "mean": 0.0025, + "std": 0.006 + }, + "are": { + "mean": 0.0025, + "std": 0.006 + }, + "aren't": { + "mean": 0.0025, + "std": 0.006 + }, + "around": { + "mean": 0.0025, + "std": 0.006 + }, + "as": { + "mean": 0.0025, + "std": 0.006 + }, + "at": { + "mean": 0.0025, + "std": 0.006 + }, + "back": { + "mean": 0.0025, + "std": 0.006 + }, + "be": { + "mean": 0.0025, + "std": 0.006 + }, + "because": { + "mean": 0.0025, + "std": 0.006 + }, + "been": { + "mean": 0.0025, + "std": 0.006 + }, + "before": { + "mean": 0.0025, + "std": 0.006 + }, + "behind": { + "mean": 0.0025, + "std": 0.006 + }, + "being": { + "mean": 0.0025, + "std": 0.006 + }, + "below": { + "mean": 0.0025, + "std": 0.006 + }, + "beside": { + "mean": 0.0025, + "std": 0.006 + }, + "between": { + "mean": 0.0025, + "std": 0.006 + }, + "beyond": { + "mean": 0.0025, + "std": 0.006 + }, + "both": { + "mean": 0.0025, + "std": 0.006 + }, + "but": { + "mean": 0.0025, + "std": 0.006 + }, + "by": { + "mean": 0.0025, + "std": 0.006 + }, + "can": { + "mean": 0.0025, + "std": 0.006 + }, + "cannot": { + "mean": 0.0025, + "std": 0.006 + }, + "concerning": { + "mean": 0.0025, + "std": 0.006 + }, + "could": { + "mean": 0.0025, + "std": 0.006 + }, + "couldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "despite": { + "mean": 0.0025, + "std": 0.006 + }, + "did": { + "mean": 0.0025, + "std": 0.006 + }, + "didn't": { + "mean": 0.0025, + "std": 0.006 + }, + "do": { + "mean": 0.0025, + "std": 0.006 + }, + "does": { + "mean": 0.0025, + "std": 0.006 + }, + "doesn't": { + "mean": 0.0025, + "std": 0.006 + }, + "doing": { + "mean": 0.0025, + "std": 0.006 + }, + "done": { + "mean": 0.0025, + "std": 0.006 + }, + "dont": { + "mean": 0.0025, + "std": 0.006 + }, + "down": { + "mean": 0.0025, + "std": 0.006 + }, + "during": { + "mean": 0.0025, + "std": 0.006 + }, + "each": { + "mean": 0.0025, + "std": 0.006 + }, + "either": { + "mean": 0.0025, + "std": 0.006 + }, + "else": { + "mean": 0.0025, + "std": 0.006 + }, + "even": { + "mean": 0.0025, + "std": 0.006 + }, + "every": { + "mean": 0.0025, + "std": 0.006 + }, + "except": { + "mean": 0.0025, + "std": 0.006 + }, + "first": { + "mean": 0.0025, + "std": 0.006 + }, + "for": { + "mean": 0.0025, + "std": 0.006 + }, + "from": { + "mean": 0.0025, + "std": 0.006 + }, + "good": { + "mean": 0.0025, + "std": 0.006 + }, + "had": { + "mean": 0.0025, + "std": 0.006 + }, + "hadn't": { + "mean": 0.0025, + "std": 0.006 + }, + "has": { + "mean": 0.0025, + "std": 0.006 + }, + "hasn't": { + "mean": 0.0025, + "std": 0.006 + }, + "have": { + "mean": 0.0025, + "std": 0.006 + }, + "haven't": { + "mean": 0.0025, + "std": 0.006 + }, + "having": { + "mean": 0.0025, + "std": 0.006 + }, + "he": { + "mean": 0.0025, + "std": 0.006 + }, + "he'd": { + "mean": 0.0025, + "std": 0.006 + }, + "he'll": { + "mean": 0.0025, + "std": 0.006 + }, + "he's": { + "mean": 0.0025, + "std": 0.006 + }, + "her": { + "mean": 0.0025, + "std": 0.006 + }, + "here": { + "mean": 0.0025, + "std": 0.006 + }, + "herself": { + "mean": 0.0025, + "std": 0.006 + }, + "him": { + "mean": 0.0025, + "std": 0.006 + }, + "himself": { + "mean": 0.0025, + "std": 0.006 + }, + "his": { + "mean": 0.0025, + "std": 0.006 + }, + "how": { + "mean": 0.0025, + "std": 0.006 + }, + "however": { + "mean": 0.0025, + "std": 0.006 + }, + "i": { + "mean": 0.0025, + "std": 0.006 + }, + "i'd": { + "mean": 0.0025, + "std": 0.006 + }, + "i'll": { + "mean": 0.0025, + "std": 0.006 + }, + "i'm": { + "mean": 0.0025, + "std": 0.006 + }, + "i've": { + "mean": 0.0025, + "std": 0.006 + }, + "if": { + "mean": 0.0025, + "std": 0.006 + }, + "in": { + "mean": 0.025, + "std": 0.006 + }, + "including": { + "mean": 0.0025, + "std": 0.006 + }, + "indeed": { + "mean": 0.0025, + "std": 0.006 + }, + "inside": { + "mean": 0.0025, + "std": 0.006 + }, + "instead": { + "mean": 0.0025, + "std": 0.006 + }, + "into": { + "mean": 0.0025, + "std": 0.006 + }, + "is": { + "mean": 0.0025, + "std": 0.006 + }, + "isn't": { + "mean": 0.0025, + "std": 0.006 + }, + "it": { + "mean": 0.0025, + "std": 0.006 + }, + "it's": { + "mean": 0.0025, + "std": 0.006 + }, + "its": { + "mean": 0.0025, + "std": 0.006 + }, + "itself": { + "mean": 0.0025, + "std": 0.006 + }, + "just": { + "mean": 0.0025, + "std": 0.006 + }, + "last": { + "mean": 0.0025, + "std": 0.006 + }, + "let": { + "mean": 0.0025, + "std": 0.006 + }, + "lets": { + "mean": 0.0025, + "std": 0.006 + }, + "like": { + "mean": 0.0025, + "std": 0.006 + }, + "made": { + "mean": 0.0025, + "std": 0.006 + }, + "make": { + "mean": 0.0025, + "std": 0.006 + }, + "many": { + "mean": 0.0025, + "std": 0.006 + }, + "may": { + "mean": 0.0025, + "std": 0.006 + }, + "me": { + "mean": 0.0025, + "std": 0.006 + }, + "might": { + "mean": 0.0025, + "std": 0.006 + }, + "mightn't": { + "mean": 0.0025, + "std": 0.006 + }, + "mine": { + "mean": 0.0025, + "std": 0.006 + }, + "minus": { + "mean": 0.0025, + "std": 0.006 + }, + "more": { + "mean": 0.0025, + "std": 0.006 + }, + "moreover": { + "mean": 0.0025, + "std": 0.006 + }, + "most": { + "mean": 0.0025, + "std": 0.006 + }, + "much": { + "mean": 0.0025, + "std": 0.006 + }, + "must": { + "mean": 0.0025, + "std": 0.006 + }, + "mustn't": { + "mean": 0.0025, + "std": 0.006 + }, + "my": { + "mean": 0.0025, + "std": 0.006 + }, + "myself": { + "mean": 0.0025, + "std": 0.006 + }, + "near": { + "mean": 0.0025, + "std": 0.006 + }, + "neither": { + "mean": 0.0025, + "std": 0.006 + }, + "never": { + "mean": 0.0025, + "std": 0.006 + }, + "nevertheless": { + "mean": 0.0025, + "std": 0.006 + }, + "no": { + "mean": 0.0025, + "std": 0.006 + }, + "nor": { + "mean": 0.0025, + "std": 0.006 + }, + "now": { + "mean": 0.0025, + "std": 0.006 + }, + "of": { + "mean": 0.025, + "std": 0.006 + }, + "off": { + "mean": 0.0025, + "std": 0.006 + }, + "on": { + "mean": 0.0025, + "std": 0.006 + }, + "once": { + "mean": 0.0025, + "std": 0.006 + }, + "one": { + "mean": 0.0025, + "std": 0.006 + }, + "only": { + "mean": 0.0025, + "std": 0.006 + }, + "or": { + "mean": 0.0025, + "std": 0.006 + }, + "other": { + "mean": 0.0025, + "std": 0.006 + }, + "our": { + "mean": 0.0025, + "std": 0.006 + }, + "ours": { + "mean": 0.0025, + "std": 0.006 + }, + "ourselves": { + "mean": 0.0025, + "std": 0.006 + }, + "out": { + "mean": 0.0025, + "std": 0.006 + }, + "outside": { + "mean": 0.0025, + "std": 0.006 + }, + "over": { + "mean": 0.0025, + "std": 0.006 + }, + "own": { + "mean": 0.0025, + "std": 0.006 + }, + "people": { + "mean": 0.0025, + "std": 0.006 + }, + "per": { + "mean": 0.0025, + "std": 0.006 + }, + "perhaps": { + "mean": 0.0025, + "std": 0.006 + }, + "plus": { + "mean": 0.0025, + "std": 0.006 + }, + "rather": { + "mean": 0.0025, + "std": 0.006 + }, + "regarding": { + "mean": 0.0025, + "std": 0.006 + }, + "same": { + "mean": 0.0025, + "std": 0.006 + }, + "see": { + "mean": 0.0025, + "std": 0.006 + }, + "she": { + "mean": 0.0025, + "std": 0.006 + }, + "she'd": { + "mean": 0.0025, + "std": 0.006 + }, + "she'll": { + "mean": 0.0025, + "std": 0.006 + }, + "she's": { + "mean": 0.0025, + "std": 0.006 + }, + "should": { + "mean": 0.0025, + "std": 0.006 + }, + "shouldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "since": { + "mean": 0.0025, + "std": 0.006 + }, + "so": { + "mean": 0.0025, + "std": 0.006 + }, + "some": { + "mean": 0.0025, + "std": 0.006 + }, + "still": { + "mean": 0.0025, + "std": 0.006 + }, + "such": { + "mean": 0.0025, + "std": 0.006 + }, + "than": { + "mean": 0.0025, + "std": 0.006 + }, + "that": { + "mean": 0.0025, + "std": 0.006 + }, + "the": { + "mean": 0.025, + "std": 0.006 + }, + "their": { + "mean": 0.0025, + "std": 0.006 + }, + "theirs": { + "mean": 0.0025, + "std": 0.006 + }, + "them": { + "mean": 0.0025, + "std": 0.006 + }, + "themselves": { + "mean": 0.0025, + "std": 0.006 + }, + "then": { + "mean": 0.0025, + "std": 0.006 + }, + "there": { + "mean": 0.0025, + "std": 0.006 + }, + "therefore": { + "mean": 0.0025, + "std": 0.006 + }, + "these": { + "mean": 0.0025, + "std": 0.006 + }, + "they": { + "mean": 0.0025, + "std": 0.006 + }, + "they'd": { + "mean": 0.0025, + "std": 0.006 + }, + "they'll": { + "mean": 0.0025, + "std": 0.006 + }, + "they're": { + "mean": 0.0025, + "std": 0.006 + }, + "they've": { + "mean": 0.0025, + "std": 0.006 + }, + "this": { + "mean": 0.0025, + "std": 0.006 + }, + "those": { + "mean": 0.0025, + "std": 0.006 + }, + "though": { + "mean": 0.0025, + "std": 0.006 + }, + "through": { + "mean": 0.0025, + "std": 0.006 + }, + "thus": { + "mean": 0.0025, + "std": 0.006 + }, + "to": { + "mean": 0.025, + "std": 0.006 + }, + "toward": { + "mean": 0.0025, + "std": 0.006 + }, + "towards": { + "mean": 0.0025, + "std": 0.006 + }, + "two": { + "mean": 0.0025, + "std": 0.006 + }, + "under": { + "mean": 0.0025, + "std": 0.006 + }, + "until": { + "mean": 0.0025, + "std": 0.006 + }, + "up": { + "mean": 0.0025, + "std": 0.006 + }, + "upon": { + "mean": 0.0025, + "std": 0.006 + }, + "us": { + "mean": 0.0025, + "std": 0.006 + }, + "versus": { + "mean": 0.0025, + "std": 0.006 + }, + "very": { + "mean": 0.0025, + "std": 0.006 + }, + "via": { + "mean": 0.0025, + "std": 0.006 + }, + "was": { + "mean": 0.0025, + "std": 0.006 + }, + "wasn't": { + "mean": 0.0025, + "std": 0.006 + }, + "we": { + "mean": 0.0025, + "std": 0.006 + }, + "we'd": { + "mean": 0.0025, + "std": 0.006 + }, + "we'll": { + "mean": 0.0025, + "std": 0.006 + }, + "we're": { + "mean": 0.0025, + "std": 0.006 + }, + "we've": { + "mean": 0.0025, + "std": 0.006 + }, + "well": { + "mean": 0.0025, + "std": 0.006 + }, + "were": { + "mean": 0.0025, + "std": 0.006 + }, + "weren't": { + "mean": 0.0025, + "std": 0.006 + }, + "what": { + "mean": 0.0025, + "std": 0.006 + }, + "when": { + "mean": 0.0025, + "std": 0.006 + }, + "where": { + "mean": 0.0025, + "std": 0.006 + }, + "whether": { + "mean": 0.0025, + "std": 0.006 + }, + "which": { + "mean": 0.0025, + "std": 0.006 + }, + "while": { + "mean": 0.0025, + "std": 0.006 + }, + "who": { + "mean": 0.0025, + "std": 0.006 + }, + "whom": { + "mean": 0.0025, + "std": 0.006 + }, + "whose": { + "mean": 0.0025, + "std": 0.006 + }, + "why": { + "mean": 0.0025, + "std": 0.006 + }, + "will": { + "mean": 0.0025, + "std": 0.006 + }, + "with": { + "mean": 0.0025, + "std": 0.006 + }, + "within": { + "mean": 0.0025, + "std": 0.006 + }, + "without": { + "mean": 0.0025, + "std": 0.006 + }, + "won't": { + "mean": 0.0025, + "std": 0.006 + }, + "would": { + "mean": 0.0025, + "std": 0.006 + }, + "wouldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "yet": { + "mean": 0.0025, + "std": 0.006 + }, + "you": { + "mean": 0.0025, + "std": 0.006 + }, + "you'd": { + "mean": 0.0025, + "std": 0.006 + }, + "you'll": { + "mean": 0.0025, + "std": 0.006 + }, + "you're": { + "mean": 0.0025, + "std": 0.006 + }, + "you've": { + "mean": 0.0025, + "std": 0.006 + }, + "your": { + "mean": 0.0025, + "std": 0.006 + }, + "yours": { + "mean": 0.0025, + "std": 0.006 + }, + "yourself": { + "mean": 0.0025, + "std": 0.006 + }, + "yourselves": { + "mean": 0.0025, + "std": 0.006 + } + }, + "function_words": { + "a": 0.03813953488372093, + "about": 0.0027906976744186047, + "above": 0.0, + "across": 0.0018604651162790699, + "after": 0.0009302325581395349, + "again": 0.0009302325581395349, + "against": 0.0, + "all": 0.0009302325581395349, + "almost": 0.0018604651162790699, + "along": 0.0009302325581395349, + "already": 0.0, + "also": 0.0, + "although": 0.0037209302325581397, + "am": 0.0, + "among": 0.0, + "an": 0.0027906976744186047, + "and": 0.026976744186046512, + "another": 0.0, + "any": 0.0009302325581395349, + "are": 0.004651162790697674, + "aren't": 0.0, + "around": 0.0, + "as": 0.0009302325581395349, + "at": 0.0037209302325581397, + "back": 0.0009302325581395349, + "be": 0.004651162790697674, + "because": 0.0065116279069767444, + "been": 0.0027906976744186047, + "before": 0.0027906976744186047, + "behind": 0.0, + "being": 0.0009302325581395349, + "below": 0.0009302325581395349, + "beside": 0.0018604651162790699, + "between": 0.0009302325581395349, + "beyond": 0.0, + "both": 0.0009302325581395349, + "but": 0.004651162790697674, + "by": 0.008372093023255815, + "can": 0.0027906976744186047, + "cannot": 0.0009302325581395349, + "concerning": 0.0, + "could": 0.0, + "couldn't": 0.0, + "despite": 0.0009302325581395349, + "did": 0.0009302325581395349, + "didn't": 0.0, + "do": 0.0018604651162790699, + "does": 0.0, + "doesn't": 0.0, + "doing": 0.0, + "done": 0.0009302325581395349, + "dont": 0.0, + "down": 0.0, + "during": 0.0, + "each": 0.0009302325581395349, + "either": 0.0009302325581395349, + "else": 0.0, + "even": 0.0009302325581395349, + "every": 0.0009302325581395349, + "except": 0.0009302325581395349, + "first": 0.0009302325581395349, + "for": 0.0, + "from": 0.0027906976744186047, + "good": 0.0, + "had": 0.008372093023255815, + "hadn't": 0.0, + "has": 0.005581395348837209, + "hasn't": 0.0, + "have": 0.0027906976744186047, + "haven't": 0.0, + "having": 0.0, + "he": 0.0, + "he'd": 0.0, + "he'll": 0.0, + "he's": 0.0, + "her": 0.0, + "here": 0.0, + "herself": 0.0, + "him": 0.0, + "himself": 0.0, + "his": 0.0018604651162790699, + "how": 0.0, + "however": 0.0009302325581395349, + "i": 0.01674418604651163, + "i'd": 0.0, + "i'll": 0.0, + "i'm": 0.0, + "i've": 0.0, + "if": 0.0, + "in": 0.005581395348837209, + "including": 0.0, + "indeed": 0.0, + "inside": 0.0009302325581395349, + "instead": 0.0, + "into": 0.0027906976744186047, + "is": 0.010232558139534883, + "isn't": 0.0, + "it": 0.009302325581395349, + "it's": 0.0, + "its": 0.008372093023255815, + "itself": 0.0009302325581395349, + "just": 0.0, + "last": 0.0009302325581395349, + "let": 0.0, + "lets": 0.0, + "like": 0.0027906976744186047, + "made": 0.0, + "make": 0.0, + "many": 0.0, + "may": 0.0, + "me": 0.0, + "might": 0.0, + "mightn't": 0.0, + "mine": 0.0, + "minus": 0.0, + "more": 0.0018604651162790699, + "moreover": 0.0, + "most": 0.0009302325581395349, + "much": 0.0, + "must": 0.0018604651162790699, + "mustn't": 0.0, + "my": 0.005581395348837209, + "myself": 0.0, + "near": 0.0018604651162790699, + "neither": 0.0, + "never": 0.0, + "nevertheless": 0.0009302325581395349, + "no": 0.0037209302325581397, + "nor": 0.0, + "now": 0.0, + "of": 0.017674418604651163, + "off": 0.0, + "on": 0.0065116279069767444, + "once": 0.0027906976744186047, + "one": 0.005581395348837209, + "only": 0.0, + "or": 0.0009302325581395349, + "other": 0.0009302325581395349, + "our": 0.0, + "ours": 0.0, + "ourselves": 0.0, + "out": 0.0009302325581395349, + "outside": 0.0009302325581395349, + "over": 0.0009302325581395349, + "own": 0.0009302325581395349, + "people": 0.0009302325581395349, + "per": 0.0, + "perhaps": 0.0009302325581395349, + "plus": 0.0, + "rather": 0.0018604651162790699, + "regarding": 0.0, + "same": 0.0009302325581395349, + "see": 0.0, + "she": 0.0009302325581395349, + "she'd": 0.0, + "she'll": 0.0, + "she's": 0.0, + "should": 0.0009302325581395349, + "shouldn't": 0.0, + "since": 0.0037209302325581397, + "so": 0.0, + "some": 0.0, + "still": 0.0018604651162790699, + "such": 0.0018604651162790699, + "than": 0.0027906976744186047, + "that": 0.009302325581395349, + "the": 0.0827906976744186, + "their": 0.0009302325581395349, + "theirs": 0.0, + "them": 0.0018604651162790699, + "themselves": 0.0, + "then": 0.0009302325581395349, + "there": 0.0018604651162790699, + "therefore": 0.0, + "these": 0.0018604651162790699, + "they": 0.0027906976744186047, + "they'd": 0.0, + "they'll": 0.0, + "they're": 0.0, + "they've": 0.0, + "this": 0.0018604651162790699, + "those": 0.0, + "though": 0.0, + "through": 0.0018604651162790699, + "thus": 0.0, + "to": 0.014883720930232559, + "toward": 0.0, + "towards": 0.0, + "two": 0.0, + "under": 0.0018604651162790699, + "until": 0.0018604651162790699, + "up": 0.0, + "upon": 0.0, + "us": 0.0009302325581395349, + "versus": 0.0, + "very": 0.0009302325581395349, + "via": 0.0, + "was": 0.0074418604651162795, + "wasn't": 0.0, + "we": 0.0027906976744186047, + "we'd": 0.0, + "we'll": 0.0, + "we're": 0.0, + "we've": 0.0, + "well": 0.0, + "were": 0.0009302325581395349, + "weren't": 0.0, + "what": 0.0, + "when": 0.004651162790697674, + "where": 0.0037209302325581397, + "whether": 0.0, + "which": 0.004651162790697674, + "while": 0.0, + "who": 0.0, + "whom": 0.0, + "whose": 0.0018604651162790699, + "why": 0.0009302325581395349, + "will": 0.0, + "with": 0.011162790697674419, + "within": 0.0, + "without": 0.004651162790697674, + "won't": 0.0, + "would": 0.0009302325581395349, + "wouldn't": 0.0, + "yet": 0.0009302325581395349, + "you": 0.0, + "you'd": 0.0, + "you'll": 0.0, + "you're": 0.0, + "you've": 0.0, + "your": 0.0, + "yours": 0.0, + "yourself": 0.0, + "yourselves": 0.0 + }, + "metadata": { + "doc_count": 4, + "genre_warning": "profile has fewer than 2000 words", + "low_confidence": true, + "total_words": 1075 + }, + "mtld": 268.75, + "paragraph_stats": { + "count": 20, + "mean_words": 53.75 + }, + "punctuation": { + "!": 0.0, + "\"": 0.0037209302325581397, + "'": 0.0027906976744186047, + "(": 0.0, + ")": 0.0, + ",": 0.08930232558139535, + "-": 0.0009302325581395349, + ".": 0.053023255813953486, + ":": 0.0018604651162790699, + ";": 0.010232558139534883, + "?": 0.0 + }, + "sentence_lengths": { + "iqr": 14.5, + "lengths": [ + 41, + 28, + 46, + 8, + 17, + 26, + 9, + 18, + 24, + 5, + 7, + 14, + 12, + 14, + 29, + 53, + 10, + 30, + 27, + 40, + 14, + 14, + 12, + 6, + 10, + 20, + 14, + 51, + 16, + 1, + 21, + 19, + 31, + 17, + 12, + 11, + 11, + 10, + 13, + 13, + 15, + 5, + 10, + 40, + 29, + 14, + 38, + 19, + 12, + 12, + 17, + 15, + 4, + 17, + 16, + 26, + 12 + ], + "median": 15 + }, + "total_words": 1075, + "word_length_histogram": { + "1": 0.05767441860465116, + "10": 0.030697674418604652, + "11": 0.013953488372093023, + "12": 0.012093023255813953, + "13": 0.005581395348837209, + "14": 0.0018604651162790699, + "15": 0.0, + "2": 0.10046511627906977, + "3": 0.18604651162790697, + "4": 0.13488372093023257, + "5": 0.11255813953488372, + "6": 0.10046511627906977, + "7": 0.12930232558139534, + "8": 0.06697674418604652, + "9": 0.04744186046511628 + } +} diff --git a/evals/fixtures/voice/profiles/celia.json b/evals/fixtures/voice/profiles/celia.json new file mode 100644 index 0000000..5a86e1e --- /dev/null +++ b/evals/fixtures/voice/profiles/celia.json @@ -0,0 +1,3142 @@ +{ + "char3": { + " \"a": 1, + " \"e": 1, + " \"m": 1, + " \"t": 1, + " \"u": 1, + " (f": 1, + " (s": 1, + " 20": 1, + " a ": 49, + " ab": 1, + " ac": 1, + " ad": 3, + " af": 1, + " al": 6, + " am": 3, + " an": 27, + " ap": 1, + " ar": 8, + " as": 4, + " at": 10, + " aw": 1, + " ba": 12, + " be": 25, + " bi": 2, + " bl": 3, + " bo": 5, + " br": 3, + " bu": 7, + " by": 5, + " ca": 8, + " ce": 2, + " ch": 11, + " ci": 3, + " cl": 8, + " co": 15, + " cr": 8, + " cu": 2, + " da": 1, + " de": 5, + " di": 8, + " do": 9, + " dr": 7, + " ed": 1, + " em": 2, + " en": 3, + " ev": 4, + " fa": 7, + " fe": 4, + " fi": 2, + " fl": 4, + " fo": 9, + " fr": 4, + " fu": 1, + " ga": 1, + " ge": 1, + " gl": 2, + " go": 5, + " gr": 8, + " gu": 1, + " ha": 21, + " he": 2, + " hi": 2, + " ho": 6, + " hu": 1, + " i ": 34, + " i'": 1, + " id": 2, + " if": 3, + " ig": 1, + " il": 1, + " im": 3, + " in": 22, + " is": 14, + " it": 28, + " ju": 5, + " ke": 6, + " ki": 5, + " kn": 6, + " la": 5, + " le": 4, + " li": 16, + " lo": 11, + " lu": 2, + " ma": 19, + " me": 10, + " mi": 6, + " mo": 5, + " mu": 1, + " my": 9, + " na": 2, + " ne": 10, + " no": 21, + " nu": 1, + " ob": 3, + " of": 13, + " oi": 1, + " ol": 2, + " on": 12, + " op": 2, + " or": 6, + " ot": 1, + " ou": 1, + " ov": 1, + " ow": 2, + " pa": 7, + " pe": 4, + " ph": 1, + " pi": 7, + " pl": 2, + " po": 6, + " pr": 12, + " pu": 3, + " qu": 2, + " ra": 1, + " re": 11, + " ri": 1, + " ro": 6, + " ru": 3, + " sa": 12, + " sc": 3, + " se": 5, + " sh": 8, + " si": 5, + " sl": 1, + " sm": 6, + " sn": 1, + " so": 11, + " sp": 3, + " st": 10, + " su": 1, + " sy": 1, + " ta": 4, + " te": 5, + " th": 106, + " ti": 6, + " to": 21, + " tr": 6, + " tw": 3, + " un": 4, + " up": 2, + " us": 3, + " ut": 1, + " va": 1, + " ve": 1, + " vi": 1, + " wa": 17, + " we": 16, + " wh": 19, + " wi": 14, + " wo": 4, + " wr": 2, + " ye": 2, + " yo": 11, + "\" b": 1, + "\" h": 1, + "\" t": 1, + "\" w": 2, + "\"a ": 1, + "\"ex": 1, + "\"ma": 1, + "\"tr": 1, + "\"un": 1, + "'ll": 1, + "'s ": 4, + "'t ": 2, + "(fi": 1, + "(sl": 1, + "). ": 2, + ", \"": 2, + ", a": 15, + ", b": 7, + ", c": 3, + ", d": 2, + ", e": 1, + ", f": 2, + ", g": 1, + ", i": 6, + ", l": 1, + ", m": 3, + ", n": 2, + ", o": 2, + ", p": 1, + ", s": 3, + ", t": 6, + ", w": 7, + ", y": 1, + ",\" ": 2, + "-ab": 1, + "-gr": 1, + "-ha": 1, + "-we": 1, + ". a": 9, + ". b": 1, + ". c": 1, + ". d": 3, + ". e": 1, + ". f": 1, + ". g": 2, + ". h": 2, + ". i": 18, + ". j": 1, + ". k": 2, + ". l": 1, + ". m": 5, + ". n": 4, + ". o": 3, + ". r": 1, + ". s": 5, + ". t": 26, + ". w": 9, + ". y": 1, + ".\" ": 3, + "021": 1, + "1? ": 1, + "202": 1, + "21?": 1, + "; p": 1, + "? d": 2, + "? i": 3, + "? k": 1, + "? l": 1, + "? m": 3, + "? n": 2, + "? o": 1, + "? t": 3, + "? u": 2, + "? w": 2, + "? y": 1, + "a a": 1, + "a b": 4, + "a c": 5, + "a d": 4, + "a f": 3, + "a g": 4, + "a j": 1, + "a k": 2, + "a l": 4, + "a m": 3, + "a n": 1, + "a p": 4, + "a q": 1, + "a r": 1, + "a s": 7, + "a t": 3, + "a u": 1, + "a v": 1, + "a w": 1, + "a. ": 2, + "abe": 1, + "abi": 1, + "abl": 3, + "abo": 2, + "acc": 1, + "ace": 1, + "ach": 1, + "ack": 10, + "ad ": 10, + "ad,": 1, + "ad?": 1, + "ade": 5, + "adm": 2, + "adu": 1, + "ady": 1, + "aft": 1, + "ag ": 1, + "ag,": 1, + "age": 1, + "agr": 1, + "aid": 4, + "ail": 3, + "ain": 2, + "air": 1, + "ait": 1, + "ake": 10, + "aki": 1, + "al ": 2, + "al.": 1, + "ale": 1, + "alf": 1, + "alk": 1, + "all": 11, + "alm": 1, + "alo": 2, + "alr": 1, + "als": 2, + "am ": 4, + "am,": 1, + "ama": 1, + "ame": 1, + "amo": 3, + "amp": 1, + "an ": 7, + "anc": 2, + "and": 36, + "ang": 2, + "ani": 1, + "ank": 2, + "anl": 1, + "ann": 1, + "ans": 1, + "ant": 6, + "any": 1, + "ape": 4, + "aph": 2, + "apk": 2, + "app": 1, + "aps": 1, + "ar,": 1, + "arc": 2, + "ard": 4, + "are": 9, + "arf": 1, + "ark": 1, + "arr": 1, + "art": 1, + "ary": 1, + "as ": 14, + "ase": 1, + "ash": 2, + "ask": 2, + "ass": 2, + "ast": 5, + "asu": 1, + "at ": 21, + "at'": 1, + "at.": 3, + "at?": 1, + "ata": 1, + "ate": 10, + "ath": 2, + "ati": 2, + "att": 2, + "atu": 2, + "aus": 6, + "aut": 1, + "ave": 6, + "aw ": 1, + "awa": 1, + "awe": 6, + "aws": 1, + "ay ": 11, + "ay,": 2, + "ay-": 1, + "ay.": 1, + "ayb": 7, + "aye": 1, + "ays": 1, + "aze": 1, + "b (": 1, + "b. ": 1, + "bab": 2, + "bac": 6, + "bad": 1, + "bag": 2, + "bak": 2, + "ban": 1, + "bat": 2, + "bbe": 1, + "bbl": 1, + "be ": 7, + "be.": 4, + "bea": 1, + "bec": 8, + "bee": 1, + "bef": 2, + "beg": 1, + "beh": 2, + "bel": 3, + "ben": 1, + "ber": 2, + "bes": 1, + "bet": 2, + "bik": 1, + "bin": 1, + "bir": 1, + "bje": 1, + "bla": 2, + "ble": 5, + "blu": 1, + "bly": 4, + "bod": 3, + "bou": 3, + "bow": 4, + "box": 1, + "bra": 1, + "bri": 2, + "bs ": 1, + "bs.": 1, + "bus": 1, + "but": 5, + "buy": 1, + "bvi": 2, + "by ": 4, + "byl": 1, + "c a": 1, + "c i": 2, + "c o": 1, + "c. ": 1, + "c? ": 2, + "cab": 1, + "cak": 1, + "cal": 4, + "cam": 1, + "can": 3, + "cap": 1, + "car": 3, + "cau": 6, + "cce": 1, + "ce.": 2, + "ced": 1, + "cei": 1, + "cen": 2, + "cep": 1, + "ces": 1, + "ch ": 9, + "ch,": 4, + "cha": 3, + "che": 6, + "cho": 4, + "chy": 1, + "cid": 1, + "cin": 2, + "cis": 1, + "civ": 1, + "ck ": 3, + "ck.": 3, + "ck?": 1, + "cke": 5, + "cky": 1, + "cla": 5, + "cle": 3, + "cni": 3, + "cof": 1, + "coi": 1, + "col": 2, + "com": 6, + "con": 5, + "cou": 4, + "cra": 2, + "cre": 2, + "cro": 2, + "cru": 3, + "ct ": 1, + "ct.": 1, + "cto": 1, + "cts": 1, + "cum": 1, + "cup": 2, + "cy ": 1, + "d a": 16, + "d b": 6, + "d c": 3, + "d g": 4, + "d h": 2, + "d i": 10, + "d l": 2, + "d m": 2, + "d n": 4, + "d o": 2, + "d p": 9, + "d r": 3, + "d s": 6, + "d t": 17, + "d u": 3, + "d w": 8, + "d y": 1, + "d, ": 4, + "d-h": 1, + "d. ": 4, + "d? ": 2, + "dam": 1, + "day": 3, + "dba": 1, + "dde": 1, + "de ": 4, + "de,": 2, + "de.": 1, + "dea": 2, + "deb": 1, + "dec": 1, + "ded": 6, + "dee": 1, + "del": 1, + "den": 5, + "der": 6, + "dfa": 1, + "dge": 2, + "dgm": 1, + "dia": 1, + "did": 4, + "dif": 2, + "dio": 1, + "dis": 2, + "dle": 1, + "dmi": 2, + "do ": 4, + "doc": 1, + "doe": 1, + "doi": 1, + "don": 1, + "dow": 1, + "dra": 7, + "dri": 1, + "ds ": 5, + "ds,": 1, + "ds.": 1, + "dsc": 1, + "dul": 1, + "dwi": 1, + "dy ": 4, + "dy'": 1, + "dy.": 1, + "dy?": 1, + "e \"": 2, + "e a": 23, + "e b": 13, + "e c": 10, + "e d": 6, + "e e": 4, + "e f": 7, + "e g": 2, + "e h": 6, + "e i": 16, + "e j": 2, + "e k": 3, + "e l": 6, + "e m": 10, + "e n": 4, + "e o": 6, + "e p": 9, + "e q": 1, + "e r": 4, + "e s": 12, + "e t": 15, + "e u": 1, + "e v": 1, + "e w": 10, + "e y": 1, + "e, ": 15, + "e,\"": 1, + "e. ": 17, + "e? ": 3, + "ea ": 1, + "ea.": 2, + "eac": 1, + "ead": 6, + "ean": 4, + "ear": 1, + "eas": 1, + "eat": 2, + "eau": 1, + "eba": 1, + "ebo": 1, + "eca": 7, + "ece": 2, + "ech": 2, + "eci": 1, + "eco": 3, + "ect": 3, + "ed ": 36, + "ed-": 1, + "ed.": 2, + "ed?": 1, + "edb": 1, + "edg": 1, + "edi": 1, + "eds": 1, + "ee ": 2, + "ee,": 1, + "eec": 2, + "eed": 3, + "eek": 2, + "eel": 3, + "een": 2, + "eep": 3, + "eer": 1, + "ees": 2, + "eet": 1, + "eez": 1, + "efo": 2, + "eft": 1, + "efu": 2, + "ega": 1, + "egi": 1, + "ego": 2, + "eha": 1, + "ehi": 1, + "eho": 1, + "eig": 1, + "eip": 1, + "eir": 1, + "eje": 1, + "ek,": 1, + "ek.": 1, + "el ": 3, + "elc": 1, + "eli": 3, + "ell": 2, + "elo": 1, + "els": 1, + "elt": 1, + "elv": 1, + "ely": 3, + "em ": 1, + "em.": 1, + "emb": 1, + "eme": 2, + "emo": 4, + "en ": 12, + "en,": 1, + "enc": 4, + "end": 3, + "ene": 1, + "eno": 3, + "ens": 3, + "ent": 14, + "enu": 1, + "eou": 2, + "ep,": 1, + "ep.": 1, + "epl": 2, + "epp": 1, + "ept": 1, + "equ": 1, + "er ": 30, + "er'": 1, + "er,": 1, + "er.": 7, + "er?": 2, + "ere": 15, + "erf": 1, + "erg": 1, + "eri": 1, + "erm": 1, + "ero": 2, + "err": 2, + "ers": 7, + "ery": 8, + "es ": 11, + "es,": 3, + "es.": 7, + "ese": 5, + "ess": 6, + "est": 6, + "et ": 4, + "et'": 1, + "et,": 1, + "et.": 1, + "eta": 2, + "ete": 3, + "eth": 1, + "etr": 1, + "ett": 1, + "etu": 1, + "eve": 5, + "evi": 1, + "ew ": 3, + "ewd": 1, + "ews": 1, + "exp": 1, + "ext": 4, + "ey ": 3, + "ey,": 1, + "ey.": 1, + "eys": 1, + "eze": 1, + "f c": 4, + "f f": 1, + "f i": 2, + "f l": 1, + "f s": 2, + "f t": 3, + "f w": 1, + "f y": 1, + "f. ": 1, + "fai": 5, + "fas": 2, + "fat": 1, + "fe ": 1, + "fe,": 1, + "fe.": 1, + "fee": 4, + "fel": 1, + "fer": 3, + "fes": 1, + "ffe": 4, + "fid": 1, + "fin": 1, + "fir": 2, + "fla": 2, + "fli": 1, + "fly": 1, + "for": 10, + "fou": 2, + "fra": 1, + "fro": 3, + "ft ": 1, + "fte": 2, + "ful": 4, + "fus": 1, + "fwa": 1, + "fy?": 1, + "g a": 3, + "g c": 3, + "g i": 2, + "g o": 2, + "g t": 6, + "g w": 2, + "g y": 1, + "g).": 1, + "g, ": 2, + "g. ": 1, + "g? ": 1, + "gal": 1, + "gar": 1, + "ge ": 3, + "ge.": 1, + "ged": 2, + "gen": 2, + "geo": 1, + "ges": 1, + "gh ": 3, + "gh,": 2, + "ghe": 1, + "ght": 6, + "gin": 2, + "gla": 2, + "gme": 1, + "gn ": 1, + "gne": 1, + "gno": 1, + "go ": 1, + "goo": 2, + "gor": 1, + "got": 3, + "gra": 5, + "gre": 3, + "gri": 1, + "gro": 2, + "gs ": 2, + "gua": 1, + "h a": 2, + "h b": 1, + "h c": 2, + "h f": 1, + "h g": 1, + "h i": 5, + "h m": 3, + "h n": 1, + "h o": 1, + "h p": 1, + "h s": 2, + "h t": 2, + "h w": 2, + "h, ": 6, + "had": 6, + "hai": 1, + "hal": 2, + "han": 8, + "hap": 1, + "har": 2, + "has": 3, + "hat": 17, + "hav": 3, + "hda": 1, + "he ": 61, + "hea": 1, + "hed": 2, + "hee": 4, + "hei": 1, + "hel": 1, + "hen": 6, + "her": 14, + "hes": 5, + "het": 1, + "hey": 2, + "hic": 7, + "hig": 1, + "hin": 8, + "hir": 1, + "his": 6, + "ho ": 2, + "hol": 2, + "hom": 3, + "hon": 1, + "hoo": 1, + "hop": 3, + "hor": 4, + "hos": 1, + "hou": 3, + "how": 1, + "hre": 1, + "hro": 2, + "ht ": 3, + "hte": 1, + "htl": 2, + "hug": 1, + "hy ": 3, + "hy)": 1, + "hy?": 1, + "hys": 1, + "i a": 5, + "i b": 2, + "i c": 4, + "i d": 3, + "i f": 2, + "i h": 3, + "i l": 2, + "i m": 2, + "i n": 1, + "i p": 2, + "i r": 2, + "i s": 3, + "i t": 2, + "i w": 2, + "i'l": 1, + "iag": 1, + "iat": 3, + "ibl": 4, + "ic ": 3, + "ic.": 1, + "ic?": 2, + "ica": 2, + "ich": 8, + "ick": 2, + "icn": 3, + "id ": 6, + "id,": 1, + "id.": 1, + "ide": 7, + "ids": 1, + "idy": 2, + "iec": 1, + "ied": 3, + "ier": 1, + "ies": 3, + "iev": 1, + "if ": 3, + "ife": 3, + "iff": 2, + "ifu": 1, + "ify": 1, + "igh": 6, + "ign": 3, + "ike": 12, + "il ": 2, + "il,": 1, + "ile": 4, + "ill": 6, + "iln": 1, + "ilu": 1, + "ilv": 1, + "im ": 1, + "ime": 1, + "imm": 1, + "imp": 2, + "in ": 15, + "inb": 1, + "ind": 5, + "ine": 3, + "ing": 21, + "ini": 2, + "ink": 2, + "inn": 3, + "ins": 8, + "int": 4, + "inu": 2, + "inv": 1, + "iny": 1, + "io ": 1, + "ion": 7, + "ior": 1, + "iou": 2, + "ip ": 1, + "ipt": 1, + "ir ": 1, + "ir,": 1, + "ire": 4, + "irs": 2, + "irt": 2, + "is ": 18, + "isa": 2, + "ise": 1, + "isi": 1, + "isn": 1, + "iss": 5, + "it ": 25, + "it.": 2, + "it?": 1, + "ite": 2, + "ith": 10, + "iti": 1, + "its": 3, + "itt": 2, + "ity": 1, + "ive": 4, + "ivi": 1, + "jec": 3, + "jud": 2, + "jun": 2, + "jus": 1, + "k a": 1, + "k d": 3, + "k h": 1, + "k i": 2, + "k s": 1, + "k t": 1, + "k u": 1, + "k, ": 1, + "k. ": 5, + "k? ": 1, + "ke ": 17, + "ke,": 1, + "ke.": 1, + "ked": 8, + "kee": 2, + "keo": 1, + "ker": 4, + "ket": 3, + "key": 4, + "kid": 1, + "kil": 1, + "kin": 7, + "kit": 1, + "kni": 2, + "kno": 5, + "ks,": 1, + "ks.": 1, + "ky ": 1, + "l a": 4, + "l b": 1, + "l c": 1, + "l g": 1, + "l i": 3, + "l l": 1, + "l n": 1, + "l o": 3, + "l s": 1, + "l t": 3, + "l w": 3, + "l y": 1, + "l, ": 3, + "l. ": 3, + "lab": 1, + "lac": 1, + "lak": 2, + "lam": 1, + "lan": 3, + "lap": 1, + "las": 3, + "lat": 4, + "law": 1, + "lay": 3, + "laz": 1, + "lco": 1, + "ld ": 6, + "lde": 1, + "le ": 5, + "le,": 1, + "le.": 2, + "lea": 3, + "led": 3, + "lef": 1, + "leg": 1, + "lem": 3, + "ler": 1, + "les": 2, + "let": 1, + "lfw": 1, + "lie": 1, + "lif": 1, + "lig": 3, + "lik": 11, + "lin": 3, + "lip": 1, + "lit": 2, + "liv": 2, + "lk ": 1, + "lki": 1, + "ll ": 12, + "ll,": 2, + "ll.": 1, + "lla": 1, + "lle": 3, + "lli": 2, + "lls": 1, + "llw": 1, + "lly": 4, + "lmo": 1, + "ln ": 1, + "lon": 3, + "loo": 8, + "lop": 1, + "los": 1, + "lov": 1, + "lre": 1, + "ls ": 2, + "ls.": 1, + "lso": 2, + "lt ": 1, + "lth": 1, + "lue": 1, + "lum": 2, + "lun": 1, + "lur": 1, + "lve": 3, + "lwa": 1, + "ly ": 14, + "ly,": 2, + "ly.": 3, + "ly?": 1, + "lye": 1, + "m a": 1, + "m c": 1, + "m h": 1, + "m l": 1, + "m n": 1, + "m o": 1, + "m s": 1, + "m t": 2, + "m w": 1, + "m, ": 1, + "m. ": 1, + "mad": 4, + "mak": 4, + "mal": 4, + "man": 2, + "mar": 2, + "mat": 3, + "may": 7, + "mbe": 1, + "mbs": 2, + "me ": 8, + "me,": 4, + "me.": 1, + "mea": 2, + "meb": 1, + "med": 1, + "mee": 1, + "mel": 1, + "mem": 1, + "men": 5, + "mer": 1, + "mes": 2, + "met": 2, + "mig": 2, + "min": 4, + "mir": 1, + "mis": 1, + "mit": 1, + "mme": 1, + "mmu": 1, + "mom": 1, + "mon": 5, + "moo": 1, + "mor": 2, + "mos": 3, + "mot": 2, + "mov": 1, + "mp ": 1, + "mp.": 1, + "mpe": 1, + "mpo": 2, + "mpr": 1, + "mps": 1, + "mun": 1, + "mus": 1, + "my ": 6, + "mys": 3, + "n 2": 1, + "n a": 8, + "n b": 3, + "n c": 2, + "n d": 1, + "n f": 1, + "n i": 5, + "n k": 1, + "n m": 5, + "n n": 1, + "n o": 3, + "n p": 1, + "n r": 3, + "n s": 3, + "n t": 8, + "n w": 2, + "n y": 2, + "n't": 2, + "n, ": 1, + "n,\"": 1, + "n. ": 2, + "nad": 1, + "nal": 2, + "nam": 2, + "nap": 2, + "nat": 1, + "nbo": 1, + "nca": 1, + "nce": 2, + "nch": 4, + "ncy": 1, + "nd ": 37, + "nd,": 2, + "nde": 4, + "ndf": 1, + "ndl": 1, + "nds": 6, + "ndw": 1, + "ndy": 1, + "ne ": 8, + "ne,": 2, + "ne?": 1, + "ned": 1, + "nee": 3, + "neg": 2, + "ner": 3, + "nes": 1, + "net": 1, + "new": 3, + "nex": 3, + "nfe": 1, + "nfi": 1, + "nfo": 1, + "ng ": 18, + "ng)": 1, + "ng,": 1, + "ng.": 1, + "ng?": 1, + "nge": 3, + "ngi": 1, + "ngs": 2, + "nic": 3, + "nie": 1, + "nif": 2, + "nin": 1, + "nio": 1, + "nit": 1, + "nk ": 3, + "nke": 2, + "nki": 1, + "nkn": 1, + "nly": 1, + "nna": 2, + "nne": 1, + "nno": 1, + "no ": 3, + "no.": 1, + "nob": 4, + "nod": 1, + "noo": 1, + "nor": 1, + "not": 8, + "nou": 3, + "nov": 1, + "now": 7, + "noy": 1, + "ns ": 6, + "ns,": 1, + "ns.": 5, + "nse": 1, + "nsi": 2, + "nst": 4, + "nt ": 6, + "nt,": 5, + "nt.": 1, + "nt?": 1, + "nta": 2, + "nte": 4, + "nti": 3, + "ntl": 1, + "ntm": 1, + "nto": 3, + "nts": 1, + "nty": 1, + "nus": 1, + "nut": 3, + "nvo": 1, + "ny ": 2, + "o a": 2, + "o b": 4, + "o c": 3, + "o d": 1, + "o f": 1, + "o g": 1, + "o h": 2, + "o i": 3, + "o k": 2, + "o l": 2, + "o m": 1, + "o n": 3, + "o o": 1, + "o p": 1, + "o r": 2, + "o s": 2, + "o t": 5, + "o y": 1, + "o. ": 2, + "ob ": 1, + "ob.": 1, + "oba": 2, + "obb": 1, + "obj": 1, + "obl": 1, + "obo": 2, + "obv": 2, + "ock": 1, + "ocu": 1, + "od ": 2, + "od.": 1, + "oda": 1, + "odd": 1, + "ody": 3, + "oes": 1, + "of ": 12, + "off": 2, + "oft": 1, + "ogr": 1, + "oic": 1, + "oil": 1, + "oin": 2, + "ois": 2, + "oje": 1, + "ok ": 1, + "oke": 5, + "oks": 1, + "old": 5, + "ole": 1, + "oli": 1, + "oll": 5, + "olv": 1, + "om ": 3, + "oma": 1, + "ome": 11, + "omm": 1, + "omo": 1, + "omp": 2, + "on ": 10, + "on'": 1, + "on,": 1, + "on.": 2, + "ona": 3, + "ond": 2, + "one": 10, + "onf": 2, + "ong": 5, + "ons": 5, + "ont": 2, + "oo ": 5, + "ood": 3, + "oof": 1, + "ook": 7, + "oom": 1, + "oon": 1, + "oos": 1, + "oot": 1, + "op ": 2, + "op,": 1, + "ope": 1, + "opi": 1, + "opp": 2, + "ops": 1, + "or ": 11, + "or.": 1, + "ora": 1, + "ord": 2, + "ore": 5, + "org": 1, + "ori": 1, + "ork": 2, + "orm": 1, + "orn": 1, + "oro": 1, + "ors": 3, + "ort": 2, + "ose": 2, + "osi": 1, + "oss": 3, + "ost": 4, + "ot ": 9, + "ote": 1, + "oth": 2, + "oti": 3, + "ott": 1, + "ou ": 7, + "ou,": 1, + "ou;": 1, + "oug": 6, + "oul": 2, + "oun": 7, + "oup": 1, + "our": 2, + "ous": 6, + "out": 5, + "ove": 5, + "ow ": 6, + "ow.": 1, + "owe": 1, + "owl": 4, + "own": 4, + "ows": 1, + "owt": 1, + "ox,": 1, + "oyi": 1, + "p f": 1, + "p i": 1, + "p l": 1, + "p n": 2, + "p t": 1, + "p, ": 2, + "p. ": 3, + "pac": 2, + "pai": 1, + "pan": 1, + "pap": 1, + "par": 2, + "pas": 1, + "pe ": 1, + "pe.": 1, + "pea": 1, + "ped": 2, + "pee": 2, + "pen": 2, + "pep": 1, + "per": 4, + "pet": 1, + "pho": 2, + "phy": 1, + "pic": 4, + "pie": 1, + "pil": 2, + "pin": 1, + "pki": 2, + "pla": 3, + "ply": 1, + "poc": 1, + "pol": 1, + "por": 1, + "pos": 3, + "pot": 1, + "pow": 1, + "ppa": 1, + "ppe": 2, + "ppy": 1, + "pre": 5, + "pri": 1, + "pro": 8, + "ps ": 1, + "ps,": 1, + "ps.": 1, + "pse": 1, + "psi": 1, + "pt ": 2, + "pus": 1, + "put": 2, + "py,": 1, + "que": 2, + "qui": 1, + "r a": 5, + "r b": 3, + "r c": 3, + "r d": 1, + "r e": 1, + "r f": 1, + "r i": 4, + "r k": 1, + "r l": 4, + "r m": 1, + "r n": 2, + "r p": 3, + "r s": 2, + "r t": 10, + "r v": 1, + "r w": 1, + "r y": 1, + "r's": 1, + "r, ": 3, + "r. ": 8, + "r? ": 2, + "rac": 2, + "rag": 1, + "ral": 1, + "ram": 1, + "ran": 3, + "rav": 1, + "raw": 7, + "ray": 4, + "rc ": 1, + "rch": 1, + "rd ": 1, + "rda": 1, + "rde": 5, + "re ": 23, + "re,": 1, + "re.": 2, + "rea": 3, + "rec": 1, + "red": 4, + "ree": 2, + "ref": 2, + "rei": 1, + "rej": 1, + "rem": 2, + "ren": 3, + "rep": 1, + "req": 1, + "res": 7, + "ret": 3, + "rew": 2, + "rf.": 1, + "rfu": 1, + "rge": 2, + "ria": 1, + "rib": 1, + "rie": 5, + "rim": 1, + "rin": 2, + "rio": 1, + "rit": 2, + "riv": 1, + "rk ": 1, + "rk.": 1, + "rks": 1, + "rm ": 1, + "rmi": 1, + "rn ": 1, + "rns": 1, + "rob": 3, + "rog": 1, + "roi": 3, + "roj": 1, + "rol": 4, + "rom": 4, + "ron": 3, + "roo": 2, + "rou": 4, + "rov": 1, + "row": 2, + "rri": 3, + "rs ": 4, + "rs.": 2, + "rse": 1, + "rsi": 1, + "rso": 2, + "rst": 2, + "rt ": 1, + "rta": 1, + "rth": 1, + "rtu": 1, + "rty": 1, + "rub": 1, + "ruc": 1, + "rud": 1, + "rum": 2, + "run": 2, + "rus": 1, + "ry ": 7, + "ry-": 1, + "ry.": 1, + "ryi": 3, + "ryo": 1, + "s a": 8, + "s b": 5, + "s c": 3, + "s d": 1, + "s f": 1, + "s g": 2, + "s h": 1, + "s i": 10, + "s j": 1, + "s k": 1, + "s l": 7, + "s n": 2, + "s o": 6, + "s p": 4, + "s r": 3, + "s s": 3, + "s t": 13, + "s w": 8, + "s y": 1, + "s, ": 11, + "s. ": 26, + "s? ": 2, + "sai": 3, + "sam": 2, + "san": 6, + "sas": 2, + "sat": 1, + "say": 2, + "sca": 2, + "sci": 1, + "scr": 1, + "se ": 15, + "se,": 2, + "se?": 1, + "sea": 1, + "sec": 2, + "see": 1, + "seh": 1, + "sen": 2, + "sev": 1, + "sh ": 1, + "sha": 1, + "she": 6, + "shi": 1, + "sho": 2, + "sib": 3, + "sic": 1, + "sid": 2, + "sig": 1, + "sil": 2, + "sin": 3, + "sio": 2, + "sit": 2, + "siv": 1, + "sk ": 1, + "ske": 1, + "sli": 1, + "slu": 1, + "sly": 1, + "sma": 4, + "sme": 1, + "smo": 1, + "sn'": 1, + "sne": 1, + "so ": 5, + "so.": 1, + "som": 4, + "son": 2, + "sor": 1, + "sou": 3, + "spe": 3, + "ss ": 4, + "ss,": 2, + "ss.": 1, + "ssa": 2, + "ssi": 5, + "sso": 1, + "ssu": 1, + "st ": 7, + "st,": 2, + "st.": 3, + "ste": 9, + "sti": 6, + "stl": 3, + "sto": 3, + "str": 3, + "stu": 2, + "sua": 1, + "sue": 1, + "sul": 1, + "sur": 1, + "sys": 1, + "t \"": 1, + "t a": 5, + "t b": 6, + "t c": 3, + "t d": 2, + "t e": 2, + "t f": 6, + "t h": 8, + "t i": 11, + "t k": 2, + "t l": 1, + "t m": 6, + "t n": 1, + "t o": 6, + "t r": 2, + "t s": 8, + "t t": 16, + "t u": 2, + "t w": 9, + "t's": 2, + "t, ": 8, + "t-w": 1, + "t. ": 9, + "t.\"": 2, + "t? ": 3, + "tab": 1, + "tai": 1, + "tak": 2, + "tan": 1, + "tap": 3, + "tar": 1, + "tas": 1, + "te ": 5, + "te.": 1, + "tea": 4, + "ted": 3, + "tel": 3, + "tem": 1, + "ten": 6, + "ter": 19, + "tes": 3, + "tex": 1, + "th ": 11, + "tha": 16, + "thd": 1, + "the": 78, + "thi": 12, + "tho": 3, + "thr": 3, + "tia": 2, + "tic": 3, + "tid": 2, + "tie": 1, + "tif": 2, + "til": 4, + "tim": 1, + "tin": 1, + "tio": 4, + "tir": 2, + "tle": 1, + "tly": 6, + "tme": 1, + "to ": 14, + "tod": 1, + "tol": 1, + "ton": 2, + "too": 5, + "top": 2, + "tor": 2, + "tos": 1, + "tra": 2, + "tri": 3, + "tru": 2, + "try": 4, + "ts ": 5, + "ts,": 1, + "tte": 6, + "ttl": 1, + "tud": 2, + "tun": 1, + "tur": 3, + "twe": 1, + "two": 2, + "ty ": 2, + "ty.": 1, + "u c": 2, + "u k": 1, + "u l": 1, + "u m": 2, + "u p": 1, + "u, ": 1, + "u; ": 1, + "ual": 1, + "uar": 1, + "ubb": 1, + "uct": 1, + "ude": 2, + "udg": 2, + "udi": 1, + "ue ": 1, + "ue?": 1, + "ues": 2, + "uge": 1, + "ugh": 6, + "uir": 1, + "ul ": 1, + "ul.": 1, + "uld": 2, + "ulk": 1, + "ull": 2, + "ult": 1, + "umb": 2, + "ume": 1, + "ump": 2, + "una": 1, + "unc": 2, + "und": 7, + "unf": 1, + "uni": 1, + "unk": 3, + "uns": 1, + "unt": 3, + "up ": 2, + "up.": 1, + "ups": 2, + "ur ": 2, + "urd": 1, + "ure": 3, + "urn": 1, + "us ": 2, + "us.": 3, + "us?": 2, + "use": 9, + "ush": 1, + "usl": 1, + "ust": 3, + "usu": 1, + "ut ": 10, + "ut-": 1, + "ute": 3, + "uti": 1, + "uts": 1, + "utt": 1, + "uy ": 1, + "vas": 1, + "ve ": 5, + "ve,": 1, + "ve.": 1, + "ved": 2, + "vel": 1, + "ven": 2, + "ver": 7, + "ves": 5, + "vic": 1, + "vid": 1, + "vio": 2, + "vis": 1, + "vol": 1, + "w (": 1, + "w b": 1, + "w f": 1, + "w h": 1, + "w i": 1, + "w o": 3, + "w t": 2, + "w.\"": 1, + "wal": 2, + "wan": 1, + "was": 9, + "wat": 1, + "wav": 2, + "way": 5, + "wdr": 1, + "we ": 7, + "wea": 1, + "wee": 2, + "wel": 1, + "wen": 1, + "wer": 12, + "wet": 1, + "whe": 5, + "whi": 7, + "who": 3, + "why": 4, + "wic": 1, + "wil": 2, + "win": 2, + "wis": 1, + "wit": 9, + "wl ": 3, + "wls": 1, + "wn ": 3, + "wne": 1, + "wo ": 2, + "wob": 1, + "wor": 3, + "wro": 2, + "ws ": 1, + "ws.": 2, + "wth": 1, + "x, ": 1, + "xpr": 1, + "xt ": 4, + "y a": 6, + "y b": 4, + "y c": 4, + "y d": 1, + "y e": 1, + "y f": 2, + "y h": 4, + "y i": 5, + "y k": 1, + "y l": 2, + "y m": 4, + "y n": 2, + "y o": 3, + "y p": 5, + "y r": 1, + "y s": 6, + "y t": 6, + "y w": 2, + "y's": 1, + "y).": 1, + "y, ": 6, + "y-a": 1, + "y-g": 1, + "y. ": 8, + "y? ": 4, + "ybe": 7, + "yed": 1, + "yel": 1, + "yer": 1, + "yes": 1, + "yin": 4, + "yla": 1, + "yon": 1, + "you": 11, + "ys,": 1, + "ys.": 1, + "ysi": 1, + "yst": 4, + "ze ": 1, + "zed": 1 + }, + "contraction_rate": 0.0066857688634192934, + "function_word_background": { + "a": { + "mean": 0.025, + "std": 0.006 + }, + "about": { + "mean": 0.0025, + "std": 0.006 + }, + "above": { + "mean": 0.0025, + "std": 0.006 + }, + "across": { + "mean": 0.0025, + "std": 0.006 + }, + "after": { + "mean": 0.0025, + "std": 0.006 + }, + "again": { + "mean": 0.0025, + "std": 0.006 + }, + "against": { + "mean": 0.0025, + "std": 0.006 + }, + "all": { + "mean": 0.0025, + "std": 0.006 + }, + "almost": { + "mean": 0.0025, + "std": 0.006 + }, + "along": { + "mean": 0.0025, + "std": 0.006 + }, + "already": { + "mean": 0.0025, + "std": 0.006 + }, + "also": { + "mean": 0.0025, + "std": 0.006 + }, + "although": { + "mean": 0.0025, + "std": 0.006 + }, + "am": { + "mean": 0.0025, + "std": 0.006 + }, + "among": { + "mean": 0.0025, + "std": 0.006 + }, + "an": { + "mean": 0.0025, + "std": 0.006 + }, + "and": { + "mean": 0.025, + "std": 0.006 + }, + "another": { + "mean": 0.0025, + "std": 0.006 + }, + "any": { + "mean": 0.0025, + "std": 0.006 + }, + "are": { + "mean": 0.0025, + "std": 0.006 + }, + "aren't": { + "mean": 0.0025, + "std": 0.006 + }, + "around": { + "mean": 0.0025, + "std": 0.006 + }, + "as": { + "mean": 0.0025, + "std": 0.006 + }, + "at": { + "mean": 0.0025, + "std": 0.006 + }, + "back": { + "mean": 0.0025, + "std": 0.006 + }, + "be": { + "mean": 0.0025, + "std": 0.006 + }, + "because": { + "mean": 0.0025, + "std": 0.006 + }, + "been": { + "mean": 0.0025, + "std": 0.006 + }, + "before": { + "mean": 0.0025, + "std": 0.006 + }, + "behind": { + "mean": 0.0025, + "std": 0.006 + }, + "being": { + "mean": 0.0025, + "std": 0.006 + }, + "below": { + "mean": 0.0025, + "std": 0.006 + }, + "beside": { + "mean": 0.0025, + "std": 0.006 + }, + "between": { + "mean": 0.0025, + "std": 0.006 + }, + "beyond": { + "mean": 0.0025, + "std": 0.006 + }, + "both": { + "mean": 0.0025, + "std": 0.006 + }, + "but": { + "mean": 0.0025, + "std": 0.006 + }, + "by": { + "mean": 0.0025, + "std": 0.006 + }, + "can": { + "mean": 0.0025, + "std": 0.006 + }, + "cannot": { + "mean": 0.0025, + "std": 0.006 + }, + "concerning": { + "mean": 0.0025, + "std": 0.006 + }, + "could": { + "mean": 0.0025, + "std": 0.006 + }, + "couldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "despite": { + "mean": 0.0025, + "std": 0.006 + }, + "did": { + "mean": 0.0025, + "std": 0.006 + }, + "didn't": { + "mean": 0.0025, + "std": 0.006 + }, + "do": { + "mean": 0.0025, + "std": 0.006 + }, + "does": { + "mean": 0.0025, + "std": 0.006 + }, + "doesn't": { + "mean": 0.0025, + "std": 0.006 + }, + "doing": { + "mean": 0.0025, + "std": 0.006 + }, + "done": { + "mean": 0.0025, + "std": 0.006 + }, + "dont": { + "mean": 0.0025, + "std": 0.006 + }, + "down": { + "mean": 0.0025, + "std": 0.006 + }, + "during": { + "mean": 0.0025, + "std": 0.006 + }, + "each": { + "mean": 0.0025, + "std": 0.006 + }, + "either": { + "mean": 0.0025, + "std": 0.006 + }, + "else": { + "mean": 0.0025, + "std": 0.006 + }, + "even": { + "mean": 0.0025, + "std": 0.006 + }, + "every": { + "mean": 0.0025, + "std": 0.006 + }, + "except": { + "mean": 0.0025, + "std": 0.006 + }, + "first": { + "mean": 0.0025, + "std": 0.006 + }, + "for": { + "mean": 0.0025, + "std": 0.006 + }, + "from": { + "mean": 0.0025, + "std": 0.006 + }, + "good": { + "mean": 0.0025, + "std": 0.006 + }, + "had": { + "mean": 0.0025, + "std": 0.006 + }, + "hadn't": { + "mean": 0.0025, + "std": 0.006 + }, + "has": { + "mean": 0.0025, + "std": 0.006 + }, + "hasn't": { + "mean": 0.0025, + "std": 0.006 + }, + "have": { + "mean": 0.0025, + "std": 0.006 + }, + "haven't": { + "mean": 0.0025, + "std": 0.006 + }, + "having": { + "mean": 0.0025, + "std": 0.006 + }, + "he": { + "mean": 0.0025, + "std": 0.006 + }, + "he'd": { + "mean": 0.0025, + "std": 0.006 + }, + "he'll": { + "mean": 0.0025, + "std": 0.006 + }, + "he's": { + "mean": 0.0025, + "std": 0.006 + }, + "her": { + "mean": 0.0025, + "std": 0.006 + }, + "here": { + "mean": 0.0025, + "std": 0.006 + }, + "herself": { + "mean": 0.0025, + "std": 0.006 + }, + "him": { + "mean": 0.0025, + "std": 0.006 + }, + "himself": { + "mean": 0.0025, + "std": 0.006 + }, + "his": { + "mean": 0.0025, + "std": 0.006 + }, + "how": { + "mean": 0.0025, + "std": 0.006 + }, + "however": { + "mean": 0.0025, + "std": 0.006 + }, + "i": { + "mean": 0.0025, + "std": 0.006 + }, + "i'd": { + "mean": 0.0025, + "std": 0.006 + }, + "i'll": { + "mean": 0.0025, + "std": 0.006 + }, + "i'm": { + "mean": 0.0025, + "std": 0.006 + }, + "i've": { + "mean": 0.0025, + "std": 0.006 + }, + "if": { + "mean": 0.0025, + "std": 0.006 + }, + "in": { + "mean": 0.025, + "std": 0.006 + }, + "including": { + "mean": 0.0025, + "std": 0.006 + }, + "indeed": { + "mean": 0.0025, + "std": 0.006 + }, + "inside": { + "mean": 0.0025, + "std": 0.006 + }, + "instead": { + "mean": 0.0025, + "std": 0.006 + }, + "into": { + "mean": 0.0025, + "std": 0.006 + }, + "is": { + "mean": 0.0025, + "std": 0.006 + }, + "isn't": { + "mean": 0.0025, + "std": 0.006 + }, + "it": { + "mean": 0.0025, + "std": 0.006 + }, + "it's": { + "mean": 0.0025, + "std": 0.006 + }, + "its": { + "mean": 0.0025, + "std": 0.006 + }, + "itself": { + "mean": 0.0025, + "std": 0.006 + }, + "just": { + "mean": 0.0025, + "std": 0.006 + }, + "last": { + "mean": 0.0025, + "std": 0.006 + }, + "let": { + "mean": 0.0025, + "std": 0.006 + }, + "lets": { + "mean": 0.0025, + "std": 0.006 + }, + "like": { + "mean": 0.0025, + "std": 0.006 + }, + "made": { + "mean": 0.0025, + "std": 0.006 + }, + "make": { + "mean": 0.0025, + "std": 0.006 + }, + "many": { + "mean": 0.0025, + "std": 0.006 + }, + "may": { + "mean": 0.0025, + "std": 0.006 + }, + "me": { + "mean": 0.0025, + "std": 0.006 + }, + "might": { + "mean": 0.0025, + "std": 0.006 + }, + "mightn't": { + "mean": 0.0025, + "std": 0.006 + }, + "mine": { + "mean": 0.0025, + "std": 0.006 + }, + "minus": { + "mean": 0.0025, + "std": 0.006 + }, + "more": { + "mean": 0.0025, + "std": 0.006 + }, + "moreover": { + "mean": 0.0025, + "std": 0.006 + }, + "most": { + "mean": 0.0025, + "std": 0.006 + }, + "much": { + "mean": 0.0025, + "std": 0.006 + }, + "must": { + "mean": 0.0025, + "std": 0.006 + }, + "mustn't": { + "mean": 0.0025, + "std": 0.006 + }, + "my": { + "mean": 0.0025, + "std": 0.006 + }, + "myself": { + "mean": 0.0025, + "std": 0.006 + }, + "near": { + "mean": 0.0025, + "std": 0.006 + }, + "neither": { + "mean": 0.0025, + "std": 0.006 + }, + "never": { + "mean": 0.0025, + "std": 0.006 + }, + "nevertheless": { + "mean": 0.0025, + "std": 0.006 + }, + "no": { + "mean": 0.0025, + "std": 0.006 + }, + "nor": { + "mean": 0.0025, + "std": 0.006 + }, + "now": { + "mean": 0.0025, + "std": 0.006 + }, + "of": { + "mean": 0.025, + "std": 0.006 + }, + "off": { + "mean": 0.0025, + "std": 0.006 + }, + "on": { + "mean": 0.0025, + "std": 0.006 + }, + "once": { + "mean": 0.0025, + "std": 0.006 + }, + "one": { + "mean": 0.0025, + "std": 0.006 + }, + "only": { + "mean": 0.0025, + "std": 0.006 + }, + "or": { + "mean": 0.0025, + "std": 0.006 + }, + "other": { + "mean": 0.0025, + "std": 0.006 + }, + "our": { + "mean": 0.0025, + "std": 0.006 + }, + "ours": { + "mean": 0.0025, + "std": 0.006 + }, + "ourselves": { + "mean": 0.0025, + "std": 0.006 + }, + "out": { + "mean": 0.0025, + "std": 0.006 + }, + "outside": { + "mean": 0.0025, + "std": 0.006 + }, + "over": { + "mean": 0.0025, + "std": 0.006 + }, + "own": { + "mean": 0.0025, + "std": 0.006 + }, + "people": { + "mean": 0.0025, + "std": 0.006 + }, + "per": { + "mean": 0.0025, + "std": 0.006 + }, + "perhaps": { + "mean": 0.0025, + "std": 0.006 + }, + "plus": { + "mean": 0.0025, + "std": 0.006 + }, + "rather": { + "mean": 0.0025, + "std": 0.006 + }, + "regarding": { + "mean": 0.0025, + "std": 0.006 + }, + "same": { + "mean": 0.0025, + "std": 0.006 + }, + "see": { + "mean": 0.0025, + "std": 0.006 + }, + "she": { + "mean": 0.0025, + "std": 0.006 + }, + "she'd": { + "mean": 0.0025, + "std": 0.006 + }, + "she'll": { + "mean": 0.0025, + "std": 0.006 + }, + "she's": { + "mean": 0.0025, + "std": 0.006 + }, + "should": { + "mean": 0.0025, + "std": 0.006 + }, + "shouldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "since": { + "mean": 0.0025, + "std": 0.006 + }, + "so": { + "mean": 0.0025, + "std": 0.006 + }, + "some": { + "mean": 0.0025, + "std": 0.006 + }, + "still": { + "mean": 0.0025, + "std": 0.006 + }, + "such": { + "mean": 0.0025, + "std": 0.006 + }, + "than": { + "mean": 0.0025, + "std": 0.006 + }, + "that": { + "mean": 0.0025, + "std": 0.006 + }, + "the": { + "mean": 0.025, + "std": 0.006 + }, + "their": { + "mean": 0.0025, + "std": 0.006 + }, + "theirs": { + "mean": 0.0025, + "std": 0.006 + }, + "them": { + "mean": 0.0025, + "std": 0.006 + }, + "themselves": { + "mean": 0.0025, + "std": 0.006 + }, + "then": { + "mean": 0.0025, + "std": 0.006 + }, + "there": { + "mean": 0.0025, + "std": 0.006 + }, + "therefore": { + "mean": 0.0025, + "std": 0.006 + }, + "these": { + "mean": 0.0025, + "std": 0.006 + }, + "they": { + "mean": 0.0025, + "std": 0.006 + }, + "they'd": { + "mean": 0.0025, + "std": 0.006 + }, + "they'll": { + "mean": 0.0025, + "std": 0.006 + }, + "they're": { + "mean": 0.0025, + "std": 0.006 + }, + "they've": { + "mean": 0.0025, + "std": 0.006 + }, + "this": { + "mean": 0.0025, + "std": 0.006 + }, + "those": { + "mean": 0.0025, + "std": 0.006 + }, + "though": { + "mean": 0.0025, + "std": 0.006 + }, + "through": { + "mean": 0.0025, + "std": 0.006 + }, + "thus": { + "mean": 0.0025, + "std": 0.006 + }, + "to": { + "mean": 0.025, + "std": 0.006 + }, + "toward": { + "mean": 0.0025, + "std": 0.006 + }, + "towards": { + "mean": 0.0025, + "std": 0.006 + }, + "two": { + "mean": 0.0025, + "std": 0.006 + }, + "under": { + "mean": 0.0025, + "std": 0.006 + }, + "until": { + "mean": 0.0025, + "std": 0.006 + }, + "up": { + "mean": 0.0025, + "std": 0.006 + }, + "upon": { + "mean": 0.0025, + "std": 0.006 + }, + "us": { + "mean": 0.0025, + "std": 0.006 + }, + "versus": { + "mean": 0.0025, + "std": 0.006 + }, + "very": { + "mean": 0.0025, + "std": 0.006 + }, + "via": { + "mean": 0.0025, + "std": 0.006 + }, + "was": { + "mean": 0.0025, + "std": 0.006 + }, + "wasn't": { + "mean": 0.0025, + "std": 0.006 + }, + "we": { + "mean": 0.0025, + "std": 0.006 + }, + "we'd": { + "mean": 0.0025, + "std": 0.006 + }, + "we'll": { + "mean": 0.0025, + "std": 0.006 + }, + "we're": { + "mean": 0.0025, + "std": 0.006 + }, + "we've": { + "mean": 0.0025, + "std": 0.006 + }, + "well": { + "mean": 0.0025, + "std": 0.006 + }, + "were": { + "mean": 0.0025, + "std": 0.006 + }, + "weren't": { + "mean": 0.0025, + "std": 0.006 + }, + "what": { + "mean": 0.0025, + "std": 0.006 + }, + "when": { + "mean": 0.0025, + "std": 0.006 + }, + "where": { + "mean": 0.0025, + "std": 0.006 + }, + "whether": { + "mean": 0.0025, + "std": 0.006 + }, + "which": { + "mean": 0.0025, + "std": 0.006 + }, + "while": { + "mean": 0.0025, + "std": 0.006 + }, + "who": { + "mean": 0.0025, + "std": 0.006 + }, + "whom": { + "mean": 0.0025, + "std": 0.006 + }, + "whose": { + "mean": 0.0025, + "std": 0.006 + }, + "why": { + "mean": 0.0025, + "std": 0.006 + }, + "will": { + "mean": 0.0025, + "std": 0.006 + }, + "with": { + "mean": 0.0025, + "std": 0.006 + }, + "within": { + "mean": 0.0025, + "std": 0.006 + }, + "without": { + "mean": 0.0025, + "std": 0.006 + }, + "won't": { + "mean": 0.0025, + "std": 0.006 + }, + "would": { + "mean": 0.0025, + "std": 0.006 + }, + "wouldn't": { + "mean": 0.0025, + "std": 0.006 + }, + "yet": { + "mean": 0.0025, + "std": 0.006 + }, + "you": { + "mean": 0.0025, + "std": 0.006 + }, + "you'd": { + "mean": 0.0025, + "std": 0.006 + }, + "you'll": { + "mean": 0.0025, + "std": 0.006 + }, + "you're": { + "mean": 0.0025, + "std": 0.006 + }, + "you've": { + "mean": 0.0025, + "std": 0.006 + }, + "your": { + "mean": 0.0025, + "std": 0.006 + }, + "yours": { + "mean": 0.0025, + "std": 0.006 + }, + "yourself": { + "mean": 0.0025, + "std": 0.006 + }, + "yourselves": { + "mean": 0.0025, + "std": 0.006 + } + }, + "function_words": { + "a": 0.04775549188156638, + "about": 0.0019102196752626551, + "above": 0.0, + "across": 0.0, + "after": 0.0009551098376313276, + "again": 0.0, + "against": 0.0, + "all": 0.0, + "almost": 0.0009551098376313276, + "along": 0.0019102196752626551, + "already": 0.0009551098376313276, + "also": 0.0019102196752626551, + "although": 0.0, + "am": 0.0028653295128939827, + "among": 0.0, + "an": 0.0, + "and": 0.02387774594078319, + "another": 0.0, + "any": 0.0, + "are": 0.0066857688634192934, + "aren't": 0.0, + "around": 0.0, + "as": 0.0028653295128939827, + "at": 0.007640878701050621, + "back": 0.004775549188156638, + "be": 0.0038204393505253103, + "because": 0.0057306590257879654, + "been": 0.0009551098376313276, + "before": 0.0019102196752626551, + "behind": 0.0009551098376313276, + "being": 0.0, + "below": 0.0, + "beside": 0.0, + "between": 0.0, + "beyond": 0.0, + "both": 0.0, + "but": 0.0038204393505253103, + "by": 0.0038204393505253103, + "can": 0.0019102196752626551, + "cannot": 0.0, + "concerning": 0.0, + "could": 0.0019102196752626551, + "couldn't": 0.0, + "despite": 0.0, + "did": 0.0038204393505253103, + "didn't": 0.0, + "do": 0.0038204393505253103, + "does": 0.0009551098376313276, + "doesn't": 0.0, + "doing": 0.0009551098376313276, + "done": 0.0, + "dont": 0.0, + "down": 0.0009551098376313276, + "during": 0.0, + "each": 0.0, + "either": 0.0, + "else": 0.0, + "even": 0.0009551098376313276, + "every": 0.0009551098376313276, + "except": 0.0, + "first": 0.0019102196752626551, + "for": 0.004775549188156638, + "from": 0.0019102196752626551, + "good": 0.0019102196752626551, + "had": 0.0057306590257879654, + "hadn't": 0.0, + "has": 0.0028653295128939827, + "hasn't": 0.0, + "have": 0.0019102196752626551, + "haven't": 0.0, + "having": 0.0, + "he": 0.0, + "he'd": 0.0, + "he'll": 0.0, + "he's": 0.0, + "her": 0.0, + "here": 0.0, + "herself": 0.0, + "him": 0.0, + "himself": 0.0, + "his": 0.0009551098376313276, + "how": 0.0009551098376313276, + "however": 0.0, + "i": 0.033428844317096466, + "i'd": 0.0, + "i'll": 0.0009551098376313276, + "i'm": 0.0, + "i've": 0.0, + "if": 0.0028653295128939827, + "in": 0.012416427889207259, + "including": 0.0, + "indeed": 0.0, + "inside": 0.0009551098376313276, + "instead": 0.0019102196752626551, + "into": 0.0028653295128939827, + "is": 0.011461318051575931, + "isn't": 0.0009551098376313276, + "it": 0.025787965616045846, + "it's": 0.0, + "its": 0.0009551098376313276, + "itself": 0.0, + "just": 0.0009551098376313276, + "last": 0.0, + "let": 0.0, + "lets": 0.0, + "like": 0.010506208213944603, + "made": 0.0038204393505253103, + "make": 0.0028653295128939827, + "many": 0.0009551098376313276, + "may": 0.0, + "me": 0.0028653295128939827, + "might": 0.0019102196752626551, + "mightn't": 0.0, + "mine": 0.0009551098376313276, + "minus": 0.0, + "more": 0.0, + "moreover": 0.0, + "most": 0.0, + "much": 0.0, + "must": 0.0009551098376313276, + "mustn't": 0.0, + "my": 0.0057306590257879654, + "myself": 0.0, + "near": 0.0, + "neither": 0.0, + "never": 0.0, + "nevertheless": 0.0, + "no": 0.0038204393505253103, + "nor": 0.0, + "now": 0.0038204393505253103, + "of": 0.010506208213944603, + "off": 0.0, + "on": 0.0057306590257879654, + "once": 0.0, + "one": 0.0057306590257879654, + "only": 0.0, + "or": 0.0038204393505253103, + "other": 0.0009551098376313276, + "our": 0.0, + "ours": 0.0, + "ourselves": 0.0, + "out": 0.0009551098376313276, + "outside": 0.0, + "over": 0.0009551098376313276, + "own": 0.0009551098376313276, + "people": 0.0, + "per": 0.0, + "perhaps": 0.0, + "plus": 0.0, + "rather": 0.0, + "regarding": 0.0, + "same": 0.0, + "see": 0.0009551098376313276, + "she": 0.0019102196752626551, + "she'd": 0.0, + "she'll": 0.0, + "she's": 0.0, + "should": 0.0, + "shouldn't": 0.0, + "since": 0.0, + "so": 0.0038204393505253103, + "some": 0.0028653295128939827, + "still": 0.0028653295128939827, + "such": 0.0, + "than": 0.0028653295128939827, + "that": 0.011461318051575931, + "the": 0.05635148042024833, + "their": 0.0009551098376313276, + "theirs": 0.0, + "them": 0.0, + "themselves": 0.0, + "then": 0.0028653295128939827, + "there": 0.007640878701050621, + "therefore": 0.0, + "these": 0.0009551098376313276, + "they": 0.0019102196752626551, + "they'd": 0.0, + "they'll": 0.0, + "they're": 0.0, + "they've": 0.0, + "this": 0.004775549188156638, + "those": 0.0, + "though": 0.0009551098376313276, + "through": 0.0009551098376313276, + "thus": 0.0, + "to": 0.010506208213944603, + "toward": 0.0, + "towards": 0.0, + "two": 0.0019102196752626551, + "under": 0.0019102196752626551, + "until": 0.0009551098376313276, + "up": 0.0019102196752626551, + "upon": 0.0, + "us": 0.0009551098376313276, + "versus": 0.0, + "very": 0.0, + "via": 0.0, + "was": 0.007640878701050621, + "wasn't": 0.0, + "we": 0.0066857688634192934, + "we'd": 0.0, + "we'll": 0.0, + "we're": 0.0, + "we've": 0.0, + "well": 0.0, + "were": 0.004775549188156638, + "weren't": 0.0, + "what": 0.0, + "when": 0.0028653295128939827, + "where": 0.0, + "whether": 0.0009551098376313276, + "which": 0.0066857688634192934, + "while": 0.0, + "who": 0.0019102196752626551, + "whom": 0.0, + "whose": 0.0, + "why": 0.0038204393505253103, + "will": 0.0019102196752626551, + "with": 0.007640878701050621, + "within": 0.0, + "without": 0.0009551098376313276, + "won't": 0.0, + "would": 0.0, + "wouldn't": 0.0, + "yet": 0.0, + "you": 0.008595988538681949, + "you'd": 0.0, + "you'll": 0.0, + "you're": 0.0, + "you've": 0.0, + "your": 0.0019102196752626551, + "yours": 0.0, + "yourself": 0.0, + "yourselves": 0.0 + }, + "metadata": { + "doc_count": 4, + "genre_warning": "profile has fewer than 2000 words", + "low_confidence": true, + "total_words": 1047 + }, + "mtld": 153.73717609730303, + "paragraph_stats": { + "count": 26, + "mean_words": 40.26923076923077 + }, + "punctuation": { + "!": 0.0, + "\"": 0.009551098376313277, + "'": 0.0066857688634192934, + "(": 0.0019102196752626551, + ")": 0.0019102196752626551, + ",": 0.06399235912129896, + "-": 0.0038204393505253103, + ".": 0.09551098376313276, + ":": 0.0, + ";": 0.0009551098376313276, + "?": 0.02005730659025788 + }, + "sentence_lengths": { + "iqr": 8.0, + "lengths": [ + 19, + 16, + 9, + 2, + 6, + 3, + 3, + 16, + 3, + 1, + 8, + 1, + 4, + 8, + 8, + 21, + 5, + 6, + 9, + 5, + 4, + 18, + 11, + 5, + 13, + 8, + 9, + 11, + 10, + 12, + 2, + 1, + 6, + 15, + 5, + 1, + 13, + 2, + 14, + 5, + 2, + 24, + 25, + 3, + 3, + 10, + 7, + 5, + 11, + 9, + 23, + 2, + 2, + 14, + 11, + 6, + 11, + 5, + 5, + 10, + 5, + 17, + 21, + 3, + 1, + 8, + 11, + 20, + 9, + 4, + 3, + 7, + 7, + 23, + 13, + 4, + 3, + 13, + 3, + 11, + 12, + 9, + 18, + 6, + 18, + 11, + 6, + 2, + 10, + 16, + 2, + 19, + 1, + 7, + 14, + 6, + 3, + 11, + 16, + 1, + 6, + 5, + 8, + 13, + 22, + 4, + 7, + 3, + 10, + 5, + 4, + 10, + 19, + 1, + 2, + 5, + 11, + 10, + 13, + 3, + 6 + ], + "median": 7 + }, + "total_words": 1047, + "word_length_histogram": { + "1": 0.08118433619866285, + "10": 0.009551098376313277, + "11": 0.0038204393505253103, + "12": 0.0019102196752626551, + "13": 0.0009551098376313276, + "14": 0.0, + "15": 0.0, + "2": 0.1346704871060172, + "3": 0.19961795606494748, + "4": 0.19484240687679083, + "5": 0.12989493791786055, + "6": 0.09837631327602674, + "7": 0.07163323782234957, + "8": 0.050620821394460364, + "9": 0.022922636103151862 + } +} diff --git a/evals/fixtures/voice/stuffed_boris_for_amara.md b/evals/fixtures/voice/stuffed_boris_for_amara.md new file mode 100644 index 0000000..7cb91a8 --- /dev/null +++ b/evals/fixtures/voice/stuffed_boris_for_amara.md @@ -0,0 +1,44 @@ +I keep a shelf by the back door. It holds gloves, seed packets, a bent trowel, and a jar of brass screws. I don't sort it much. I know where each thing lives. + +This morning I planted beans before work. The soil was cold. The hose coughed. A crow watched from the fence and made one dry sound. I told it I wasn't sharing. + +I like chores with edges. Cut twine. Pull weeds. Rinse mud from a boot heel. You can see the work. You can stop when the row is clean. That's rare enough. + +The beans went in crooked. That's fine. Straight rows are for catalogs. Mine follow rocks, roots, and my left knee, which still complains after rain. I pressed each seed down with one finger. I covered it. I patted the bed flat. + +By seven, the kettle was yelling. My cuffs were wet. There was dirt under one nail. I had a meeting in twenty minutes and a slug on my sleeve. The day already felt used, in a good way. + +I don't garden for calm. Calm is too fancy. I garden because dinner starts somewhere. Because a handful of beans in July can make a Tuesday feel less thin. Because the little shed smells like cedar, dust, and last year's onions. + +Tonight I'll check the bed again. Nothing will have happened. That's the point. Most good things begin as nothing you can prove. + +I left the muddy gloves on the step. They'll dry there. Or they won't. Either way, morning will know what to do with them. + + +The train was late again. Nobody looked surprised. A man in a blue coat kicked salt from the platform edge. A woman opened a paper bag and shared almonds with her kid. I stood under the heater. It wasn't doing much. + +I don't mind waiting if the place has texture. This station does. Brick walls. Gum spots. A clock that loses three minutes every week. Pigeons are banned from my final answer, but here in the fixture text the platform birds were real and loud. The coffee cart smelled burnt in the normal way. + +When the train came, we all moved at once. No speech. Just bags lifted, shoulders turned, one small apology near the door. That's city manners. Brief. Useful. Gone. + +I found a seat with a torn corner. The vinyl scratched my palm. Across from me, a nurse slept through three stops. Her shoes were clean. Her face wasn't. I tried not to stare. + +The window gave back a gray version of us. Coats. Phones. A lunch box. One red scarf. I thought about how many private mornings fit inside one car. Bad news. Good bread. Rent due. A dog left with a neighbor. Nobody announces it. + +My stop came fast after all that waiting. I stepped down into slush. The street smelled like diesel and wet wool. I had ten minutes to spare. That's enough for a cheap coffee and a reset. + +The train will be late tomorrow too. I'll still take it. It's ugly, close, and honest about the weather. + + +Since the neighborhood pool closes after Labor Day, its final week carries a ceremonial quality that exceeds the actual pleasure of swimming in water already cooled by September nights. Parents arrive with towels that smell faintly of storage; children bargain for one more jump from the low board; and the lifeguards, who have spent three months enforcing rules with admirable boredom, begin to look almost sentimental. + +I went on Thursday evening, when the sky had the flat, metallic brightness that often precedes rain. The pool was not crowded, yet every sound traveled: the slap of sandals, the scrape of a chair, the echoing cough of someone who had swallowed water in a moment of excessive confidence. I swam eight slow lengths and then sat on the concrete edge, where the warmth of the day remained trapped in the surface. + +There is something clarifying about a public place at the end of its season. The defects become visible because no one is trying to disguise them. A lane rope has faded unevenly; the snack window sticks; the painted depth numbers have begun to flake. These are not failures, exactly, but records of use. + +When the whistle blew, people left reluctantly and without drama. A boy forgot one blue sandal, and his mother retrieved it with the irritation reserved for predictable losses. The lifeguard stacked chairs in pairs; the manager locked the supply shed. + +I walked home with damp hair and no particular sadness. The pool would be back next year, altered slightly, familiar enough. + + +Since the neighborhood pool closes after Labor Day, its final week carries a ceremonial quality that exceeds the actual pleasure of swimming in water already cooled by September nights. Parents arrive with towels that smell faintly of storage; children bargain for one more jump from the low board; and the lifeguards, who have spent three months enforcing rules with admirable boredom, begin to look almost sentimental. diff --git a/evals/golden/contrib_report.md b/evals/golden/contrib_report.md new file mode 100644 index 0000000..0591976 --- /dev/null +++ b/evals/golden/contrib_report.md @@ -0,0 +1,48 @@ +# Add significance_inflation pattern: durable bridge + +## The specimen + +> The rollout guide keeps saying the migration path gives teams a durable bridge between the old queue and the new worker pool, but it never explains the retry contract. + +- Source genre: technical rollout note +- Date: 2026-07-06 +- Redaction note: none + +## Why it's an AI-ism + +`Durable bridge` inflates a migration detail into a vague promise. It sounds reassuring without naming the compatibility behavior, failure mode, or owner. + +## Detection + +- Pattern added: `durable bridge` +- Severity: soft +- Gating rationale: flag metaphorical migration praise, protect literal construction +- Catalog entry location: references/taboo-phrases.md + +## Evals + +| row id | kind | what it pins | +|---|---|---| +| CONTRIB-FN-durable-bridge | FN | exact specimen flags `durable bridge` | +| CONTRIB-FP-durable-bridge | FP | literal-use protection for `durable bridge` | + +The FN stdin is the unmodified specimen after approved redaction. + +## Gate results + +### contribute-slice + +```text +PASS CONTRIB-01 precheck names covering pattern +PASS CONTRIB-02 precheck accepts uncaught tell +PASS CONTRIB-03 scaffold preserves specimen bytes +``` + +## Checklist + +- [ ] eval-first (row was red before the pattern) +- [ ] literal-use FP row included +- [ ] REC row if an existing word was gated +- [ ] catalog + scanner parity green +- [ ] coverage gate green (pattern exercised) +- [ ] snippet publication approved by the user diff --git a/evals/kata_add_pattern.py b/evals/kata_add_pattern.py new file mode 100644 index 0000000..035255f --- /dev/null +++ b/evals/kata_add_pattern.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Rehearse the add-a-pattern procedure and prove each safety net fires in order. + +This is a meta-eval: it does not test the scanner, it tests the *process* that keeps +the scanner honest. In a throwaway copy of the repo it walks a careless executor +through adding a banned phrase and checks that, at every step, the guard that is +supposed to catch a mistake actually does: + + step 1 add the scanner entry with NO eval row -> coverage gate exits 1 + step 2 add the false-negative row -> coverage passes, parity exits 1 + step 3 add the catalog line -> parity passes, the row runs green + step 4 delete the scanner entry, keep the row -> the row goes red + +If a future refactor silently disables the coverage gate, catalog parity, or the +runner, one of these steps stops behaving and this kata (wired as DOC-10) turns red. + +Run: python3 evals/kata_add_pattern.py --run +""" +import argparse +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +PHRASE = "synergistic paradigm" +CATEGORY = "test_kata" +KATA_ID = "KATA-01" + +SCANNER_LINE = ( + f' "{PHRASE}": {{"category": "{CATEGORY}", "severity": "hard", "suggestion": None}},\n' +) + + +def run(tmp, *cmd): + proc = subprocess.run( + ["python3", *cmd], + capture_output=True, + text=True, + cwd=tmp, + timeout=60, + ) + return proc.returncode, proc.stdout, proc.stderr + + +def add_scanner_entry(tmp): + scanner = tmp / "scripts" / "banned_phrase_scan.py" + lines = scanner.read_text().splitlines(keepends=True) + for i, line in enumerate(lines): + if line.startswith("BANNED_PHRASES") and line.rstrip().endswith("{"): + lines.insert(i + 1, SCANNER_LINE) + scanner.write_text("".join(lines)) + return + raise AssertionError("could not find BANNED_PHRASES opening in the temp scanner") + + +def remove_scanner_entry(tmp): + scanner = tmp / "scripts" / "banned_phrase_scan.py" + text = scanner.read_text() + assert SCANNER_LINE in text, "scanner entry missing when trying to remove it" + scanner.write_text(text.replace(SCANNER_LINE, "", 1)) + + +def add_fn_row(tmp): + suite_path = tmp / "evals" / "adversarial-evals.json" + suite = json.loads(suite_path.read_text(encoding="utf-8")) + suite["evals"].append({ + "id": KATA_ID, + "category": "scanner_false_negative", + "title": f"Kata: '{PHRASE}' flags", + "target": "script", + "command": ["python3", "scripts/banned_phrase_scan.py"], + "stdin": f"The team promised a {PHRASE} would fix everything.", + "failure_mode": "throwaway kata pattern", + "correct_behavior": "flagged", + "assertions": [ + {"type": "json", "path": "total_violations", "gte": 1}, + {"type": "violation_phrase_contains", "value": PHRASE}, + ], + }) + suite_path.write_text(json.dumps(suite, indent=2, ensure_ascii=False), encoding="utf-8") + + +def add_catalog_line(tmp): + catalog = tmp / "references" / "taboo-phrases.md" + catalog.write_text(catalog.read_text() + f"\n- {PHRASE}\n") + + +def kata(verbose=True): + tmp = Path(tempfile.mkdtemp(prefix="unslop-kata-")) + log = [] + + def say(msg): + log.append(msg) + if verbose: + print(msg) + + try: + for sub in ("scripts", "evals", "references"): + shutil.copytree(ROOT / sub, tmp / sub) + say("[setup] temp repo created") + + results = [] + + # ---- step 1: scanner entry, no eval row ----------------------------- + add_scanner_entry(tmp) + rc, out, _ = run(tmp, "evals/check_pattern_coverage.py", "--coverage") + ok1 = rc == 1 and PHRASE in out + results.append(ok1) + say(f"[step 1] scanner entry, no row -> coverage exit {rc} " + f"(expect 1, mentions phrase={PHRASE in out}): {'OK' if ok1 else 'FAIL'}") + + # ---- step 2: add the FN row ----------------------------------------- + add_fn_row(tmp) + rc_cov, _, _ = run(tmp, "evals/check_pattern_coverage.py", "--coverage") + rc_par, _, _ = run(tmp, "evals/check_taboo_parity.py") + ok2 = rc_cov == 0 and rc_par == 1 + results.append(ok2) + say(f"[step 2] add FN row -> coverage exit {rc_cov} (expect 0), " + f"parity exit {rc_par} (expect 1): {'OK' if ok2 else 'FAIL'}") + + # ---- step 3: add the catalog line ----------------------------------- + add_catalog_line(tmp) + rc_par2, _, _ = run(tmp, "evals/check_taboo_parity.py") + rc_row, out_row, _ = run(tmp, "evals/run_adversarial.py", "--only", "KATA") + ok3 = rc_par2 == 0 and rc_row == 0 + results.append(ok3) + say(f"[step 3] add catalog line -> parity exit {rc_par2} (expect 0), " + f"KATA row exit {rc_row} (expect 0): {'OK' if ok3 else 'FAIL'}") + + # ---- step 4: remove scanner entry, keep the row --------------------- + remove_scanner_entry(tmp) + rc_row2, _, _ = run(tmp, "evals/run_adversarial.py", "--only", "KATA") + ok4 = rc_row2 == 1 + results.append(ok4) + say(f"[step 4] drop scanner entry, keep row -> KATA row exit {rc_row2} " + f"(expect 1, row now red): {'OK' if ok4 else 'FAIL'}") + + passed = all(results) + say(f"[result] {sum(results)}/4 safety nets fired -> {'PASS' if passed else 'FAIL'}") + return passed, log + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def main(argv): + parser = argparse.ArgumentParser(description="Add-a-pattern kata (meta-eval).") + parser.add_argument("--run", action="store_true", help="run the kata") + args = parser.parse_args(argv) + if not args.run: + parser.print_help() + return 0 + passed, _ = kata(verbose=True) + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/evals/mimic_stats.py b/evals/mimic_stats.py new file mode 100644 index 0000000..5724304 --- /dev/null +++ b/evals/mimic_stats.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Paired small-sample statistics for mimic candidate comparisons. + +Input JSON is a list of paired items, each ``{"treatment": x, "baseline": y}`` +(lower composite is better, so the per-item improvement delta is baseline - +treatment). Follows the small-n discipline from the loop research: + + - per-item paired deltas, + - BCa bootstrap CI (n=2000, seeded), + - sign-flip permutation test (exact when items <= 12, else 20k seeded), + - verdict ``improved`` iff CI lower bound > 0 AND p < 0.05. + +Deterministic given the seed. Under-claims by default: any construct-validity +shortfall (empty, single item, zero variance) yields improved=false. +""" + +import argparse +import itertools +import json +import math +import random +import statistics +import sys +from pathlib import Path + +BOOTSTRAP_N = 2000 +PERM_N = 20000 +EXACT_PERM_MAX = 12 + + +def deltas(items): + return [it["baseline"] - it["treatment"] for it in items] + + +def _percentile(sorted_vals, q): + if not sorted_vals: + return 0.0 + idx = q * (len(sorted_vals) - 1) + lo = int(math.floor(idx)) + hi = int(math.ceil(idx)) + if lo == hi: + return sorted_vals[lo] + frac = idx - lo + return sorted_vals[lo] * (1 - frac) + sorted_vals[hi] * frac + + +def _normal_cdf(z): + return 0.5 * (1 + math.erf(z / math.sqrt(2))) + + +def _normal_ppf(p): + # Acklam's rational approximation to the inverse normal CDF. + a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, + 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00] + b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, + 6.680131188771972e+01, -1.328068155288572e+01] + c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, + -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00] + d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, + 3.754408661907416e+00] + plow, phigh = 0.02425, 1 - 0.02425 + if p < plow: + q = math.sqrt(-2 * math.log(p)) + return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / \ + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + if p > phigh: + q = math.sqrt(-2 * math.log(1 - p)) + return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / \ + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + q = p - 0.5 + r = q * q + return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / \ + (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1) + + +def bca_ci(sample, seed, alpha=0.05): + """BCa bootstrap CI for the mean of ``sample``.""" + n = len(sample) + if n < 2: + return (0.0, 0.0) + theta_hat = statistics.mean(sample) + rng = random.Random(seed) + boot = [] + for _ in range(BOOTSTRAP_N): + resample = [sample[rng.randrange(n)] for _ in range(n)] + boot.append(statistics.mean(resample)) + boot.sort() + n_less = sum(1 for b in boot if b < theta_hat) + prop = n_less / BOOTSTRAP_N + if prop <= 0.0 or prop >= 1.0: + # Degenerate acceleration; fall back to the percentile interval. + return (_percentile(boot, alpha / 2), _percentile(boot, 1 - alpha / 2)) + z0 = _normal_ppf(prop) + # Jackknife acceleration. + jack = [] + total = sum(sample) + for i in range(n): + jack.append((total - sample[i]) / (n - 1)) + jbar = statistics.mean(jack) + num = sum((jbar - x) ** 3 for x in jack) + den = 6.0 * (sum((jbar - x) ** 2 for x in jack) ** 1.5) + acc = num / den if den else 0.0 + z_lo, z_hi = _normal_ppf(alpha / 2), _normal_ppf(1 - alpha / 2) + + def adjust(z): + return _normal_cdf(z0 + (z0 + z) / (1 - acc * (z0 + z))) + + return (_percentile(boot, adjust(z_lo)), _percentile(boot, adjust(z_hi))) + + +def sign_flip_p(sample, seed): + """Two-sided sign-flip permutation test that the mean delta is 0.""" + n = len(sample) + if n == 0: + return 1.0 + observed = abs(sum(sample)) + if n <= EXACT_PERM_MAX: + count = 0 + total = 0 + for signs in itertools.product((1, -1), repeat=n): + total += 1 + if abs(sum(s * x for s, x in zip(signs, sample))) >= observed - 1e-12: + count += 1 + return count / total + rng = random.Random(seed) + count = 0 + for _ in range(PERM_N): + flipped = sum(x if rng.random() < 0.5 else -x for x in sample) + if abs(flipped) >= observed - 1e-12: + count += 1 + return (count + 1) / (PERM_N + 1) + + +def analyze(items, seed): + d = deltas(items) + n = len(d) + mean_delta = statistics.mean(d) if d else 0.0 + ci_low, ci_high = bca_ci(d, seed) if n >= 2 else (0.0, 0.0) + p = sign_flip_p(d, seed) if n >= 1 else 1.0 + improved = bool(n >= 2 and ci_low > 0 and p < 0.05) + return { + "n": n, + "mean_delta": mean_delta, + "ci_low": ci_low, + "ci_high": ci_high, + "p_value": p, + "improved": improved, + "seed": seed, + } + + +def parse_args(argv): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("data", help="JSON list of {treatment, baseline} pairs") + p.add_argument("--seed", type=int, default=20240607) + return p.parse_args(argv) + + +def main(argv): + args = parse_args(argv) + path = Path(args.data) + if not path.exists(): + print(f"missing data file: {path}", file=sys.stderr) + return 2 + items = json.loads(path.read_text()) + result = analyze(items, args.seed) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/evals/run_adversarial.py b/evals/run_adversarial.py index 5069cfa..e14bfb8 100755 --- a/evals/run_adversarial.py +++ b/evals/run_adversarial.py @@ -122,6 +122,27 @@ def list_gates(): "blocking": True, "needs": [], }, + { + "id": "harvest-suite", + "command": "python3 evals/run_adversarial.py --only HARV", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "contribute-suite", + "command": "python3 evals/run_adversarial.py --only CONTRIB", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "calibrate-suite", + "command": "python3 evals/run_adversarial.py --only CAL", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, { "id": "shared-benchmark-check", "command": "python3 evals/build_shared_benchmark.py --check", @@ -143,6 +164,55 @@ def list_gates(): "blocking": True, "needs": [], }, + { + "id": "pattern-coverage", + "command": "python3 evals/check_pattern_coverage.py", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "voice-scorer", + "command": "python3 evals/check_voice.py --separation && python3 evals/check_voice.py --gi && python3 evals/check_voice.py --gaming && python3 evals/check_voice.py --profiles", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "add-pattern-kata", + "command": "python3 evals/kata_add_pattern.py --run", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "command-router-parity", + "command": "python3 evals/check_commands.py", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "seeded-docs", + "command": "python3 evals/check_seeded_docs.py", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "paired-fixture-hygiene", + "command": "python3 evals/check_pairs.py", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "mimic-logic", + "command": "python3 evals/run_adversarial.py --only MIMIC --only CARD", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, { "id": "pack-structure", "command": "python3 scripts/check_packs.py", @@ -171,6 +241,20 @@ def list_gates(): "blocking": True, "needs": [], }, + { + "id": "silhouette-scan", + "command": "python3 scripts/silhouette_scan.py < transformed.txt", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, + { + "id": "silhouette-check", + "command": "python3 evals/check_silhouette.py", + "pass_criterion": "exit 0", + "blocking": True, + "needs": [], + }, { "id": "validate-preservation", "command": "python3 scripts/validate_preservation.py original.txt transformed.txt", diff --git a/evals/run_mimic_refine.py b/evals/run_mimic_refine.py new file mode 100644 index 0000000..f0fc235 --- /dev/null +++ b/evals/run_mimic_refine.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python3 +"""Iterative ``--refine`` loop for the mimic feature (formerly the onslaught +design). Given author samples and a draft, it hill-climbs candidates toward the +author's stylometric profile under hard removal gates, accepting only DEV-split +improvements and halting on patience or on the reward-hacking divergence +signature. + +Two candidate sources, both driving the identical gate/score/accept path: + +* LIVE (default): each iteration assembles B prompts (draft + the A-split voice + card + the k nearest-A samples + the current directives) and invokes + ``--generate-cmd`` once per prompt, feeding the prompt on stdin and reading the + candidate on stdout. ``--baseline zero|few|retrieval`` selects which samples + ride in the prompt (none / first-k / k nearest-A). Deterministic when the + generator is deterministic. +* DRY-RUN: pass ``--candidates-dir DIR`` and the loop reads ``DIR/iter/*.md`` + as that iteration's batch instead of generating them. The MIMIC-* eval rows + exercise both paths (the live path via a mock generator). + +Acceptance and the divergence guard both use the FULL ``voice_score`` composite +(``0.5*(1-GI) + 0.5*`` clipped weighted impostor z-distance) against a same-genre +impostor pool, not a raw weighted distance — a marker-stuffed candidate that +clears every hard gate must still lose to honest prose. The generator never +scores itself. Climb on A, accept on DEV, watch for divergence between the two. +""" + +import argparse +import json +import os +import random +import shlex +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import voice_profile # noqa: E402 +import voice_score # noqa: E402 +import voice_card # noqa: E402 +import banned_phrase_scan # noqa: E402 +import structure_scan # noqa: E402 +import validate_preservation # noqa: E402 + +LENGTH_FLOOR = 150 +PRECISION = 12 +DEFAULT_IMPOSTORS = ROOT / "evals" / "fixtures" / "voice" / "impostors" +NEAREST_K = 2 + +DIRECTIVE_TEXT = { + "char3": ("Shift vocabulary and letter texture toward the samples; you are " + "leaning on different words than the author.", + "Lexicon: prefer the author's concrete nouns over your own."), + "delta": ("Rebalance function words (articles, pronouns, prepositions) " + "toward the author's mix.", + "Openers & connectives: match the author's function-word rhythm."), + "sentence_emd": ("Move sentence lengths toward the author's distribution " + "(median {med} words).", + "Rhythm: target a median sentence around {med} words."), + "punctuation": ("Match the author's punctuation habits instead of your own.", + "Punctuation: mirror the author's mark rates."), + "contraction": ("Match the author's contraction rate ({rate}).", + "Contractions: target rate {rate}."), + "mtld": ("Match the author's lexical variety; you are repeating or varying " + "words more than they do.", + "Lexicon: match the author's repetition and variety."), + "word_length": ("Match the author's word-length habit (shorter or longer " + "words).", + "Lexicon: match the author's average word length."), +} + + +def round12(x): + return round(float(x), PRECISION) + + +def split_docs(paths, seed, dev_frac=0.4): + """Deterministic by-document split into (A, DEV) file lists.""" + ordered = sorted(str(p) for p in paths) + shuffled = ordered[:] + random.Random(seed).shuffle(shuffled) + n = len(shuffled) + n_dev = max(2, round(n * dev_frac)) + dev = sorted(shuffled[:n_dev]) + a = sorted(shuffled[n_dev:]) + return [Path(p) for p in a], [Path(p) for p in dev] + + +def profile_from_paths(paths, background=None): + text = "\n\n".join(p.read_text(errors="replace") for p in paths) + profile = voice_profile.feature_bundle(text) + profile["function_word_background"] = voice_profile.background_stats(background) + profile["metadata"] = { + "doc_count": len(paths), + "total_words": profile["total_words"], + "low_confidence": profile["total_words"] < 2000, + "genre_warning": "", + } + return profile + + +def make_scorer(profile, impostor_rows, seed): + """Full voice_score composite (GI + clipped weighted z-distance) closure. + + Lower is more author-like. Impostor z-distances are precomputed per profile; + GI is reseeded per call so the whole loop is reproducible. + """ + imp_dist = [voice_score.distances(profile, f) for _, f in impostor_rows] + + def score(feats): + dist = voice_score.distances(profile, feats) + zs = voice_score.zscores(dist, imp_dist) + zsum = sum( + voice_score.WEIGHTS[k] + * max(-3.0, min(3.0, zs[k] if zs[k] is not None else 0.0)) + for k in voice_score.WEIGHTS + ) + gi = (voice_score.gi_score(profile, feats, impostor_rows, seed) + if impostor_rows else 0.0) + return round12(0.5 * (1 - gi) + 0.5 * zsum) + + return score + + +def copy_gate_vs_paths(cand_text, a_paths): + """Anti-verbatim-copy gate against the A split only (not the DEV holdout).""" + cand_grams = voice_score.ngrams(voice_profile.words(cand_text)) + max_overlap = 0.0 + lcs_violation = False + for path in a_paths: + text = path.read_text(errors="replace") + sample_grams = voice_score.ngrams(voice_profile.words(text)) + if cand_grams: + max_overlap = max(max_overlap, len(cand_grams & sample_grams) / len(cand_grams)) + if voice_score.has_common_substring_over(cand_text, text, voice_score.LCS_THRESHOLD + 1): + lcs_violation = True + return max_overlap > 0.35 or lcs_violation + + +def run_gates(cand_text, draft_text, a_paths, genre): + """Return (passed, reason_or_None, gate_detail).""" + gates = {} + banned = banned_phrase_scan.scan_for_violations(cand_text) + gates["banned_phrase"] = not banned + struct = structure_scan.scan(cand_text, genre) + gates["structure"] = not struct["flags"] + pres = validate_preservation.validate_preservation(draft_text, cand_text) + gates["preservation"] = pres["passed"] + gates["copy_gate"] = not copy_gate_vs_paths(cand_text, a_paths) + words = len(voice_profile.words(cand_text)) + gates["length_floor"] = words >= LENGTH_FLOOR + order = ["length_floor", "preservation", "banned_phrase", "structure", "copy_gate"] + reason = next((g for g in order if not gates[g]), None) + return reason is None, reason, gates + + +def load_iteration_candidates(candidates_dir, index): + d = Path(candidates_dir) / f"iter{index}" + if not d.is_dir(): + return None + out = [] + for path in sorted(d.glob("*.md")): + out.append((path.name, path.read_text(errors="replace"))) + return out + + +# --------------------------- live generation --------------------------- + +def char3_cosine(a_text, b_text): + ga = voice_profile.char3_counts(a_text) + gb = voice_profile.char3_counts(b_text) + return voice_score.cosine_distance(ga, gb) + + +def nearest_samples(draft_text, a_paths, k=NEAREST_K): + scored = [] + for p in a_paths: + text = p.read_text(errors="replace") + scored.append((char3_cosine(draft_text, text), p.name, text)) + scored.sort(key=lambda x: (x[0], x[1])) + return [(name, text) for _, name, text in scored[:k]] + + +def select_samples(baseline, draft_text, a_paths, k=NEAREST_K): + """Which A samples ride in the prompt, per baseline mode.""" + if baseline == "zero": + return [] + if baseline == "few": + ordered = sorted(a_paths, key=lambda p: p.name)[:k] + return [(p.name, p.read_text(errors="replace")) for p in ordered] + return nearest_samples(draft_text, a_paths, k) # retrieval (default) + + +def build_prompt(draft_text, card_text, samples, directives, beam_index): + parts = ["# Voice card\n" + card_text.strip()] + for name, text in samples: + parts.append(f"# Sample: {name}\n{text.strip()}") + if directives: + parts.append("# Directives\n" + "\n".join( + f"- {d['directive']}" for d in directives)) + parts.append("# Draft to rewrite in this voice\n" + draft_text.strip()) + parts.append(f"# Variant {beam_index}") + return "\n\n".join(parts) + "\n" + + +def generate_candidate(generate_cmd, prompt, iter_index): + env = dict(os.environ, MOCK_ITER=str(iter_index)) + proc = subprocess.run( + shlex.split(generate_cmd), input=prompt, text=True, + capture_output=True, env=env, + ) + if proc.returncode != 0: + raise RuntimeError(f"generate-cmd failed ({proc.returncode}): {proc.stderr}") + return proc.stdout + + +def make_live_source(args, a_paths, profile_a, docs_a, matrix_a, draft_text): + """Build the card once, then generate a beam per iteration through the CLI.""" + card_text = voice_card.build_card(profile_a, docs_a, matrix_a, args.name) + samples = select_samples(args.baseline, draft_text, a_paths) + + def get_batch(index, directives): + batch = [] + for b in range(args.beam): + prompt = build_prompt(draft_text, card_text, samples, directives, b) + text = generate_candidate(args.generate_cmd, prompt, index) + batch.append((f"cand{b + 1:02d}.md", text)) + return batch + + return get_batch + + +def make_dry_run_source(candidates_dir): + def get_batch(index, directives): + return load_iteration_candidates(candidates_dir, index) + return get_batch + + +def derive_directives(profile_dev, best_feats): + dists = voice_score.distances(profile_dev, best_feats) + weighted = sorted( + ((k, voice_score.WEIGHTS[k] * dists.get(k, 0.0)) for k in voice_score.WEIGHTS), + key=lambda kv: (-kv[1], kv[0]), + ) + med = int(profile_dev["sentence_lengths"]["median"]) + rate = f"{profile_dev['contraction_rate']:.3f}" + out = [] + for metric, wdist in weighted[:4]: + text, amend = DIRECTIVE_TEXT[metric] + out.append({ + "metric": metric, + "weighted_distance": round12(wdist), + "directive": text.format(med=med, rate=rate), + "card_amendment": amend.format(med=med, rate=rate), + }) + return out + + +def build_report(args, a_paths, dev_paths, profile_a, profile_dev, + score_a, score_dev, get_batch): + genre = args.genre + draft_text = Path(args.draft).read_text(errors="replace") + + iterations = [] + best_name = None + best_score = None + best_feats = None + best_text = None + stop_reason = None + reward_hacking = False + since_accept = 0 + divergence_run = 0 + prev_best_a = None + prev_best_dev = None + directives = [] + + for i in range(args.iterations): + batch = get_batch(i, directives) + if batch is None: + stop_reason = "candidates_exhausted" + break + cand_records = [] + rejections = [] + survivors = [] # (name, dev_score, a_score, feats, text) + for name, text in batch: + passed, reason, gates = run_gates(text, draft_text, a_paths, genre) + feats = voice_profile.feature_bundle(text) + dev_score = score_dev(feats) + a_score = score_a(feats) + cand_records.append({ + "name": name, + "words": feats["total_words"], + "gates": gates, + "passed": passed, + "dev_score": dev_score if passed else None, + "a_score": a_score if passed else None, + }) + if passed: + survivors.append((name, dev_score, a_score, feats, text)) + else: + rejections.append({"name": name, "reason": reason}) + + accepted = False + iter_best = None + iter_best_dev = None + iter_best_a = None + if survivors: + survivors.sort(key=lambda s: (s[1], s[0])) + iter_best = survivors[0][0] + iter_best_dev = survivors[0][1] + iter_best_a = survivors[0][2] + iter_feats = survivors[0][3] + if best_score is None or iter_best_dev <= best_score - args.min_delta: + accepted = True + best_score = iter_best_dev + best_name = iter_best + best_feats = iter_feats + best_text = survivors[0][4] + + # Divergence: best-survivor A improves while DEV worsens, 2 rounds running. + if (iter_best_dev is not None and prev_best_dev is not None + and iter_best_a < prev_best_a and iter_best_dev > prev_best_dev): + divergence_run += 1 + else: + divergence_run = 0 + if iter_best_dev is not None: + prev_best_a = iter_best_a + prev_best_dev = iter_best_dev + + directives = (derive_directives(profile_dev, survivors[0][3]) + if survivors else []) + + iterations.append({ + "index": i, + "candidates": cand_records, + "gate_rejections": rejections, + "best_survivor": iter_best, + "best_dev_score": iter_best_dev, + "best_a_score": iter_best_a, + "accepted": accepted, + "directives": directives, + }) + + since_accept = 0 if accepted else since_accept + 1 + + if divergence_run >= 2: + stop_reason = "divergence" + reward_hacking = True + break + if since_accept >= args.patience: + stop_reason = "patience" + break + + if stop_reason is None: + stop_reason = "max_iterations" + + final_directives = derive_directives(profile_dev, best_feats) if best_feats else [] + + report = { + "seed": args.seed, + "baseline": args.baseline, + "genre": genre, + "impostors": str(args.impostors), + "split": { + "A": [p.name for p in a_paths], + "DEV": [p.name for p in dev_paths], + }, + "iterations": iterations, + "best_candidate": best_name, + "best_score": best_score, + "stop_reason": stop_reason, + "reward_hacking_warning": reward_hacking, + "directives": final_directives, + "card_path": str(Path(args.out) / "voice-card.refined.md"), + } + return report, best_text + + +def write_outputs(args, report, a_paths, profile_a, docs_a, matrix_a, best_text): + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + (out / "report.json").write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n" + ) + # Refined card: base card from the A split plus the directive amendments. + base = voice_card.build_card(profile_a, docs_a, matrix_a, args.name) + amend_lines = ["", "## Refinement amendments", ""] + for d in report["directives"]: + amend_lines.append(f"- {d['card_amendment']}") + (out / "voice-card.refined.md").write_text(base + "\n".join(amend_lines) + "\n") + # Best candidate text (captured from whichever source produced it). + if best_text is not None: + (out / "final.md").write_text(best_text) + + +def parse_args(argv): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--samples", required=True) + p.add_argument("--draft", required=True) + p.add_argument("--out", required=True) + p.add_argument("--seed", required=True, type=int) + p.add_argument("--iterations", type=int, default=6) + p.add_argument("--beam", type=int, default=4) + p.add_argument("--patience", type=int, default=2) + p.add_argument("--min-delta", type=float, default=0.01) + p.add_argument("--generate-cmd", default="claude -p") + p.add_argument("--candidates-dir") + p.add_argument("--impostors", default=str(DEFAULT_IMPOSTORS)) + p.add_argument("--baseline", choices=["zero", "few", "retrieval"], default="retrieval") + p.add_argument("--genre", choices=["prose", "docs", "social"], default="prose") + p.add_argument("--name", default="voice") + return p.parse_args(argv) + + +def main(argv): + args = parse_args(argv) + samples = Path(args.samples) + if not samples.is_dir() or not Path(args.draft).exists(): + print("missing samples dir or draft", file=sys.stderr) + return 2 + if not Path(args.impostors).is_dir(): + print(f"missing impostor pool: {args.impostors}", file=sys.stderr) + return 2 + paths = list(voice_profile.iter_docs(samples)) + if len(paths) < 5: + print(f"need at least 5 sample documents to split; found {len(paths)}", + file=sys.stderr) + return 2 + + a_paths, dev_paths = split_docs(paths, args.seed) + if len(dev_paths) < 2: + print("DEV split needs at least 2 documents", file=sys.stderr) + return 2 + profile_a = profile_from_paths(a_paths) + profile_dev = profile_from_paths(dev_paths) + + docs_a = [] + for p in a_paths: + sents = voice_card.split_sentences_text(p.read_text(errors="replace")) + if sents: + docs_a.append(sents) + matrix_a, _ = voice_card.coverage_matrix(docs_a) + + impostor_rows = voice_score.impostor_features(args.impostors) + score_a = make_scorer(profile_a, impostor_rows, args.seed) + score_dev = make_scorer(profile_dev, impostor_rows, args.seed) + + if args.candidates_dir is not None: + get_batch = make_dry_run_source(args.candidates_dir) + else: + get_batch = make_live_source(args, a_paths, profile_a, docs_a, matrix_a, + Path(args.draft).read_text(errors="replace")) + + report, best_text = build_report(args, a_paths, dev_paths, profile_a, + profile_dev, score_a, score_dev, get_batch) + write_outputs(args, report, a_paths, profile_a, docs_a, matrix_a, best_text) + print(json.dumps({ + "best_candidate": report["best_candidate"], + "best_score": report["best_score"], + "stop_reason": report["stop_reason"], + "reward_hacking_warning": report["reward_hacking_warning"], + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/evals/run_model_parity.py b/evals/run_model_parity.py new file mode 100755 index 0000000..84ac119 --- /dev/null +++ b/evals/run_model_parity.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python3 +"""Model-parity harness for the two model-dependent product surfaces. + +The unslop pipeline (references/pipeline.md) leans on a model in exactly two +places: Tier-1 pack DETECTION and Tier-2 REPLACEMENT generation. Everything else +is deterministic. This harness measures whether a cheap model matches a strong +model on those two surfaces so the tiering table can be set from data instead of +assumption (see docs product decisions: "MEASURED by evals, not assumed"). + +Two tasks, both graded deterministically: + + Task A (pack detection): each model reads ONE pack file plus a short seeded + chunk (the Tier-1 contract) and returns JSON findings. We grade recall of the + seeded findings and count false findings against a fixed manifest. + + Task B (replacement generation): each model is handed a seeded finding + (span + rationale) and asked for a span-minimal replacement. We grade with the + co-writer contract: the replacement removes the flagged tell without adding a + new one (both scanners), preserves must-keep constraints, and stays minimal. + +Config-driven model matrix. Entries are {name, kind, model_id}: + - kind "claude-cli": runs `claude -p --model ` (Anthropic spectrum). + - kind "openrouter": POSTs to the OpenRouter chat/completions API with the key + from the macOS keychain (service OPENROUTER_API_KEY); GPT spectrum and more. + +Modes: + live (default) call each model over the network and grade the responses. + --dry-run grade PRE-CANNED responses from a --responses fixture file. No + network, fully deterministic; this is what the PARITY-* eval rows use. + --no-network take the live path but force every model call to fail, proving a + missing key / dead network degrades gracefully (status "unavailable") + and never breaks the suite. + +Output: a per-model / per-task score table as JSON on stdout (default), or a +markdown summary (--format md), or both. Grading is deterministic: --format json +emits nothing but the JSON object so eval rows can assert on it. + +Exit codes match scripts/*.py: 0 ran, 1 bad config/args, 2 a required file is +missing. The parity VERDICT is data in the payload, not the exit code. + +DO NOT run the live matrix casually. Per CLAUDE.md / references/pipeline.md, any +change to the co-writer, mimic, or detector-pack features must run this across +both the GPT and Anthropic spectrums before merge. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = ROOT / "scripts" +PACKS = ROOT / "references" / "packs" + +sys.path.insert(0, str(SCRIPTS)) + +import banned_phrase_scan # noqa: E402 +import structure_scan # noqa: E402 +import validate_preservation # noqa: E402 + +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +KEYCHAIN_SERVICE = "OPENROUTER_API_KEY" + +# Documented default matrix — spans the GPT spectrum (openrouter) and the +# Anthropic spectrum (claude-cli). Not run unless the caller asks for live mode. +DEFAULT_MODELS = [ + {"name": "gpt-4o-mini", "kind": "openrouter", "model_id": "openai/gpt-4o-mini"}, + {"name": "gpt-4o", "kind": "openrouter", "model_id": "openai/gpt-4o"}, + {"name": "claude-haiku", "kind": "claude-cli", "model_id": "claude-3-5-haiku-latest"}, + {"name": "claude-sonnet", "kind": "claude-cli", "model_id": "claude-sonnet-4-5"}, +] + +# --------------------------------------------------------------------------- # +# Seeded corpus. Fixed on purpose: parity is only meaningful against a frozen +# corpus. Each Task-A fixture seeds exactly two findings from ONE pack family; +# two of the six are register-guard cases (the load-bearing preservation family). +# --------------------------------------------------------------------------- # + +TASK_A_FIXTURES = [ + { + "id": "A1", + "pack": "pack-phrases-core", + "text": "Here's the thing: this quarterly result really underscores the " + "importance of shipping fast.", + "manifest": [ + {"any_of": ["here's the thing"]}, + {"any_of": ["underscores the importance", "underscores the importance of"]}, + ], + }, + { + "id": "A2", + "pack": "pack-phrases-core", + "text": "This release unlocks seamless cross-functional synergy across every team.", + "manifest": [ + {"any_of": ["seamless"]}, + {"any_of": ["synergy", "cross-functional synergy"]}, + ], + }, + { + "id": "A3", + "pack": "pack-voice", + "text": "Not the tool. The team. The result: alignment across the whole org.", + "manifest": [ + {"any_of": ["not the tool. the team.", "not the tool. the team"]}, + {"any_of": ["the result: alignment", "result: alignment"]}, + ], + }, + { + "id": "A4", + "pack": "pack-voice", + "text": "We build things that matter. Emit tokens. Ship bytes. That is the whole job.", + "manifest": [ + {"any_of": ["emit tokens. ship bytes.", "emit tokens. ship bytes"]}, + {"any_of": ["things that matter", "build things that matter"]}, + ], + }, + { + # register-guard case #1: safety/medical hedges that must survive a rewrite + "id": "A5", + "pack": "pack-register-guards", + "text": "Users should never store secrets in client-side code. The trial may " + "cause drowsiness in some patients.", + "manifest": [ + {"any_of": ["never store secrets", "never"]}, + {"any_of": ["may cause", "may", "some patients"]}, + ], + }, + { + # register-guard case #2: legal negation/scope + "id": "A6", + "pack": "pack-register-guards", + "text": "The incident arguably does not rise to gross negligence under Section " + "12(b), unless the party consents in writing.", + "manifest": [ + {"any_of": [ + "arguably does not rise to gross negligence", + "does not rise to gross negligence", + "arguably", + ]}, + {"any_of": ["unless the party consents", "unless"]}, + ], + }, +] + +# Task-B fixtures: a seeded finding (span inside context + rationale). The model +# returns ONLY a replacement for the span. B5 doubles as the register-guard +# preservation case — fixing the opener must not strip the load-bearing "never". +TASK_B_FIXTURES = [ + { + "id": "B1", + "context": "Here's the thing: the service returns a 200 status code on success.", + "span": "Here's the thing: the", + "rule": "throat_clearing", + "rationale": "Throat-clearing opener; cut it and start with the claim.", + }, + { + "id": "B2", + "context": "This release unlocks seamless integration with 3 external providers.", + "span": "seamless", + "rule": "academic_excess", + "rationale": "Empty polish word; drop or replace with a concrete adjective.", + }, + { + "id": "B3", + "context": "The data speaks for itself in these 5 charts.", + "span": "speaks for itself", + "rule": "false_agency", + "rationale": "Data has no agency; state what the charts show.", + }, + { + "id": "B4", + "context": "Latency dropped from 900ms to 120ms, which underscores the importance " + "of caching.", + "span": "which underscores the importance of caching", + "rule": "significance_inflation", + "rationale": "Inflated significance tail; state the cause plainly.", + }, + { + # register-guard preservation: the fix touches the opener, NOT the "never". + "id": "B5", + "context": "Let's dive in: users must never store secrets in client-side code.", + "span": "Let's dive in: users", + "rule": "throat_clearing", + "rationale": "Cut the throat-clearing opener; keep the security rule exact.", + }, + { + "id": "B6", + "context": "In conclusion, the deploy finished in 4 minutes with 0 errors.", + "span": "In conclusion, the", + "rule": "conclusion_scaffold", + "rationale": "Generic conclusion scaffold; start with the fact.", + }, +] + + +# --------------------------------------------------------------------------- # +# Response parsing +# --------------------------------------------------------------------------- # + +def _strip_fences(raw): + """Drop markdown code fences a model may wrap JSON in.""" + text = raw.strip() + if text.startswith("```"): + text = re.sub(r"^```[a-zA-Z0-9]*\n", "", text) + text = re.sub(r"\n```\s*$", "", text) + return text.strip() + + +def parse_findings(raw): + """Best-effort parse of a Task-A response into a list of finding dicts. + + Tolerant to fences, a wrapping {"findings": [...]}, or JSONL. Returns [] when + nothing parses (an empty response is a legitimate zero-recall result, not a + crash). + """ + if not raw or not raw.strip(): + return [] + text = _strip_fences(raw) + for candidate in (text, _extract_array(text)): + if candidate is None: + continue + try: + data = json.loads(candidate) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(data, dict) and "findings" in data: + data = data["findings"] + if isinstance(data, dict): + data = [data] + if isinstance(data, list): + return [d for d in data if isinstance(d, dict)] + # JSONL fallback: one JSON object per line. + out = [] + for line in text.splitlines(): + line = line.strip().rstrip(",") + if line.startswith("{") and line.endswith("}"): + try: + obj = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(obj, dict): + out.append(obj) + return out + + +def _extract_array(text): + start = text.find("[") + end = text.rfind("]") + if 0 <= start < end: + return text[start:end + 1] + return None + + +def _norm(s): + return re.sub(r"\s+", " ", (s or "")).strip().lower() + + +# --------------------------------------------------------------------------- # +# Task A grading (deterministic recall + false findings vs manifest) +# --------------------------------------------------------------------------- # + +def grade_task_a(fixture, raw): + findings = parse_findings(raw) + got_spans = [_norm(f.get("span", "")) for f in findings] + matched_flags = [False] * len(got_spans) + matched_expected = 0 + for exp in fixture["manifest"]: + alts = [_norm(a) for a in exp["any_of"]] + hit = False + for i, gs in enumerate(got_spans): + if not gs: + continue + if any(a and (a in gs or gs in a) for a in alts): + matched_flags[i] = True + hit = True + if hit: + matched_expected += 1 + n_expected = len(fixture["manifest"]) + recall = matched_expected / n_expected if n_expected else 0.0 + false_findings = sum(1 for i, gs in enumerate(got_spans) + if gs and not matched_flags[i]) + return { + "recall": round(recall, 6), + "false_findings": false_findings, + "n_expected": n_expected, + "n_matched": matched_expected, + "n_returned": len(got_spans), + } + + +# --------------------------------------------------------------------------- # +# Scanner helpers for Task B +# --------------------------------------------------------------------------- # + +def _banned_signal(text): + """Set of (category, phrase) violations the banned-phrase scanner reports.""" + violations = banned_phrase_scan.scan_for_violations(text) + return {(v.get("category", ""), _norm(v.get("phrase", ""))) + for v in violations} + + +def _structure_signal(text): + """Set of structure-scan flags for the text.""" + data = structure_scan.scan(text) + flags = data.get("flags", data.get("flagged", [])) + if isinstance(flags, dict): + flags = [k for k, v in flags.items() if v] + return {("structure", _norm(str(f))) for f in flags} + + +def _preservation_ok(original, transformed): + return validate_preservation.validate_preservation(original, transformed)["passed"] + + +def grade_task_b(fixture, replacement): + context = fixture["context"] + span = fixture["span"] + if replacement is None: + replacement = "" + idx = context.find(span) + if idx < 0: + # Fixture defect, not a model failure. Surface it explicitly. + return { + "pass": False, + "span_found": False, + "scanners_ok": False, + "constraints_preserved": False, + "span_minimal": False, + "detail": "seed span not found in context", + } + new_context = context[:idx] + replacement + context[idx + len(span):] + + before = _banned_signal(context) | _structure_signal(context) + after = _banned_signal(new_context) | _structure_signal(new_context) + added = after - before + removed = before - after + scanners_ok = (not added) and bool(removed) + + constraints_preserved = _preservation_ok(context, new_context) + + span_words = max(1, len(span.split())) + repl_words = len(replacement.split()) + span_minimal = repl_words <= max(3, 2 * span_words) + + passed = scanners_ok and constraints_preserved and span_minimal + return { + "pass": passed, + "span_found": True, + "scanners_ok": scanners_ok, + "added_signals": sorted(f"{c}:{p}" for c, p in added), + "removed_signals": sorted(f"{c}:{p}" for c, p in removed), + "constraints_preserved": constraints_preserved, + "span_minimal": span_minimal, + "replacement_words": repl_words, + } + + +# --------------------------------------------------------------------------- # +# Prompts (Tier-1 / co-writer contract) +# --------------------------------------------------------------------------- # + +def build_prompt_a(fixture): + pack_text = (PACKS / f"{fixture['pack']}.md").read_text() + return ( + "You are a Tier-1 unslop detector. You may only use the pack below. " + "Read the chunk and return a JSON array of findings, each " + '{"span":"...","rule":"...","pack":"...","severity":"hard|soft","note":"..."}. ' + "Report only spans this pack owns. Do not rewrite. Return JSON only.\n\n" + f"=== PACK: {fixture['pack']} ===\n{pack_text}\n\n" + f"=== CHUNK ===\n{fixture['text']}\n" + ) + + +def build_prompt_b(fixture): + return ( + "You are an unslop co-writer. Below is one flagged span inside its " + "context and why it was flagged. Return ONLY a span-minimal replacement " + "string for the flagged span (no quotes, no JSON, no explanation). The " + "replacement must remove the tell, keep every fact/number/negation in the " + "context intact, and change as little as possible.\n\n" + f"CONTEXT: {fixture['context']}\n" + f"FLAGGED SPAN: {fixture['span']}\n" + f"RULE: {fixture['rule']}\n" + f"WHY: {fixture['rationale']}\n" + ) + + +# --------------------------------------------------------------------------- # +# Model calls (live). Every path is wrapped so a missing key / dead network +# yields (None, reason) instead of raising. +# --------------------------------------------------------------------------- # + +def keychain_key(service): + try: + proc = subprocess.run( + ["security", "find-generic-password", "-s", service, "-w"], + capture_output=True, text=True, timeout=10, + ) + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return None + if proc.returncode != 0: + return None + key = proc.stdout.strip() + return key or None + + +def call_openrouter(model_id, prompt, timeout=60): + key = keychain_key(KEYCHAIN_SERVICE) + if not key: + return None, "no OPENROUTER_API_KEY in keychain" + body = json.dumps({ + "model": model_id, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0, + }).encode() + req = urllib.request.Request( + OPENROUTER_URL, + data=body, + headers={"Authorization": f"Bearer {key}", + "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode()) + return data["choices"][0]["message"]["content"], None + except (urllib.error.URLError, KeyError, ValueError, OSError, TimeoutError) as e: + return None, f"openrouter error: {e}" + + +def call_claude_cli(model_id, prompt, timeout=120): + try: + proc = subprocess.run( + ["claude", "-p", "--model", model_id], + input=prompt, + capture_output=True, text=True, timeout=timeout, + ) + except (FileNotFoundError, OSError, subprocess.TimeoutExpired) as e: + return None, f"claude cli error: {e}" + if proc.returncode != 0: + return None, f"claude cli exit {proc.returncode}: {proc.stderr.strip()[:200]}" + return proc.stdout, None + + +def call_model(entry, prompt, no_network): + if no_network: + return None, "no-network mode" + kind = entry.get("kind") + model_id = entry.get("model_id", "") + if kind == "openrouter": + return call_openrouter(model_id, prompt) + if kind == "claude-cli": + return call_claude_cli(model_id, prompt) + return None, f"unknown kind {kind!r}" + + +# --------------------------------------------------------------------------- # +# Matrix run +# --------------------------------------------------------------------------- # + +def _score_a(per_fixture): + recalls = [r["recall"] for r in per_fixture.values()] + return { + "mean_recall": round(sum(recalls) / len(recalls), 6) if recalls else 0.0, + "total_false_findings": sum(r["false_findings"] for r in per_fixture.values()), + "n_fixtures": len(per_fixture), + "fixtures": per_fixture, + } + + +def _score_b(per_fixture): + n_pass = sum(1 for r in per_fixture.values() if r["pass"]) + return { + "pass_rate": round(n_pass / len(per_fixture), 6) if per_fixture else 0.0, + "n_pass": n_pass, + "n_fixtures": len(per_fixture), + "fixtures": per_fixture, + } + + +def _canned_response(responses, model_name, task, fixture_id): + return (responses.get(model_name, {}).get(task, {}) or {}).get(fixture_id) + + +def run_matrix(models, tasks, dry_run, no_network, responses): + result = { + "mode": "dry-run" if dry_run else ("no-network" if no_network else "live"), + "tasks": list(tasks), + "n_fixtures": {"A": len(TASK_A_FIXTURES), "B": len(TASK_B_FIXTURES)}, + "models": {}, + } + for entry in models: + name = entry["name"] + model_out = {"kind": entry.get("kind", "canned"), "status": "ok"} + errors = [] + + if "A" in tasks: + per = {} + for fx in TASK_A_FIXTURES: + if dry_run: + raw = _canned_response(responses, name, "A", fx["id"]) or "" + else: + raw, err = call_model(entry, build_prompt_a(fx), no_network) + if err: + errors.append(f"A/{fx['id']}: {err}") + raw = "" + per[fx["id"]] = grade_task_a(fx, raw) + model_out["A"] = _score_a(per) + + if "B" in tasks: + per = {} + for fx in TASK_B_FIXTURES: + if dry_run: + repl = _canned_response(responses, name, "B", fx["id"]) + if repl is None: + repl = "" + else: + repl, err = call_model(entry, build_prompt_b(fx), no_network) + if err: + errors.append(f"B/{fx['id']}: {err}") + repl = "" + per[fx["id"]] = grade_task_b(fx, repl) + model_out["B"] = _score_b(per) + + # In a live/no-network run, a model that never produced a usable response + # is "unavailable" — recorded, not fatal. + if not dry_run and errors and len(errors) >= len(tasks) * len(TASK_A_FIXTURES): + model_out["status"] = "unavailable" + if errors: + model_out["errors"] = errors + result["models"][name] = model_out + return result + + +# --------------------------------------------------------------------------- # +# Markdown summary +# --------------------------------------------------------------------------- # + +def to_markdown(result): + lines = [] + lines.append(f"# Model Parity — {result['mode']} run") + lines.append("") + lines.append(f"Fixtures: {result['n_fixtures']['A']} detection, " + f"{result['n_fixtures']['B']} replacement.") + lines.append("") + if "A" in result["tasks"]: + lines.append("## Task A — pack detection") + lines.append("") + lines.append("| Model | Kind | Status | Mean recall | False findings |") + lines.append("|---|---|---|---:|---:|") + for name, m in result["models"].items(): + a = m.get("A", {}) + lines.append(f"| {name} | {m['kind']} | {m['status']} | " + f"{a.get('mean_recall', '-')} | " + f"{a.get('total_false_findings', '-')} |") + lines.append("") + if "B" in result["tasks"]: + lines.append("## Task B — replacement generation") + lines.append("") + lines.append("| Model | Kind | Status | Pass rate | Passed/Total |") + lines.append("|---|---|---|---:|---:|") + for name, m in result["models"].items(): + b = m.get("B", {}) + passed = (f"{b.get('n_pass', '-')}/{b.get('n_fixtures', '-')}" + if b else "-") + lines.append(f"| {name} | {m['kind']} | {m['status']} | " + f"{b.get('pass_rate', '-')} | {passed} |") + lines.append("") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # + +def _load_json_file(path, what): + p = Path(path) + if not p.exists(): + print(f"missing {what}: {path}", file=sys.stderr) + raise SystemExit(2) + try: + return json.loads(p.read_text()) + except (json.JSONDecodeError, ValueError) as e: + print(f"malformed {what} ({path}): {e}", file=sys.stderr) + raise SystemExit(1) + + +def parse_args(argv): + p = argparse.ArgumentParser(description="unslop model-parity harness") + p.add_argument("--models", metavar="FILE", + help="JSON file: list of {name,kind,model_id} model entries") + p.add_argument("--responses", metavar="FILE", + help="dry-run canned responses (may also carry a 'models' list)") + p.add_argument("--dry-run", action="store_true", + help="grade canned --responses; no network, deterministic") + p.add_argument("--no-network", action="store_true", + help="take the live path but force every call to fail (graceful-degrade test)") + p.add_argument("--check-determinism", action="store_true", + help="grade twice and add {'deterministic': bool} to the payload") + p.add_argument("--task", choices=["A", "B", "both"], default="both", + help="which task(s) to run") + p.add_argument("--format", choices=["json", "md", "both"], default="json", + help="output format (json is pure JSON for eval assertions)") + return p.parse_args(argv) + + +def resolve_models(args, payload): + if args.models: + models = _load_json_file(args.models, "models file") + if isinstance(models, dict) and "models" in models: + models = models["models"] + return models + if isinstance(payload, dict) and payload.get("models"): + return payload["models"] + if args.dry_run: + # Derive canned model names from the response payload. + resp = (payload or {}).get("responses", payload or {}) + return [{"name": n, "kind": "canned"} for n in resp] + return DEFAULT_MODELS + + +def main(argv): + args = parse_args(argv) + tasks = ["A", "B"] if args.task == "both" else [args.task] + + responses = {} + if args.responses: + payload = _load_json_file(args.responses, "responses file") + responses = payload.get("responses", payload) if isinstance(payload, dict) else {} + resp_models = payload if isinstance(payload, dict) else {} + else: + resp_models = {} + + if args.dry_run and not args.responses: + print("--dry-run needs --responses FILE", file=sys.stderr) + return 1 + + models = resolve_models(args, resp_models) + if not models: + print("no models resolved", file=sys.stderr) + return 1 + + result = run_matrix(models, tasks, args.dry_run, args.no_network, responses) + + if args.check_determinism: + again = run_matrix(models, tasks, args.dry_run, args.no_network, responses) + canon = lambda r: json.dumps(r, sort_keys=True) + result["deterministic"] = canon(result) == canon(again) + + if args.format in ("json", "both"): + print(json.dumps(result, indent=2)) + if args.format in ("md", "both"): + if args.format == "both": + print() + print(to_markdown(result)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/references/calibrate.md b/references/calibrate.md new file mode 100644 index 0000000..82494db --- /dev/null +++ b/references/calibrate.md @@ -0,0 +1,122 @@ +# Teach Calibration Game (A/B Preference Elicitation) + +An agent-hosted micro-game that calibrates voice preferences the user can state but +never bothered to write samples for. Far lower effort than `teach`: harvest bootstraps +samples, the A/B game calibrates dimension preferences, and both feed the voice card. +`teach` offers this game (`references/commands/teach.md`, step 3) when sample coverage +or confidence stays thin; this file is the canonical flow and rationale. + +## Deterministic set + +Five dimensions are reachable by reversible, seed-deterministic transforms of a base +passage (`scripts/calibrate_pairs.py`): `contractions`, `em_dash`, `sentence_length`, +`connectives`, `staccato`. Every transform preserves the base passage's must-preserve +constraints (`scripts/extract_constraints.py`) or the command exits 3 rather than +silently dropping a fact. + +Register and warmth are NOT in this set — they need LLM generation, not a mechanical +transform, and are out of scope for `calibrate_pairs.py`. If the game wants to cover +those dimensions, generate the pair with a model and skip the determinism/constraint +guarantees this script gives for free. + +## Game flow + +1. Pick base passages from the user's own harvested samples (`references/harvest.md`) + when available; otherwise fall back to neutral seed texts. Paragraph-sized (60-150 + words) so a round takes seconds to read. +2. For the dimension needing the most attention, run: + ```bash + python3 scripts/calibrate_pairs.py generate --base passage.txt --dimension DIM --seed N + ``` + Increment the seed each round for that dimension so repeated rounds against the same + base passage still vary. If the command exits 3 ("dimension not expressible in this + passage"), pick a different base passage or move to the next dimension — do not force + a transform through. + + The output also carries `a_flags`/`b_flags`: the `banned_phrase_scan.py` category names + each variant trips (empty when clean). A generated variant tripping the scanner is not a + reason to decline the pair — see "Voice overrides defaults" below. +3. Present `a_text` and `b_text` to the user as "is A or B closer to how you'd write + this?" without naming the dimension or which side is which pole (avoid anchoring). + Accept "neither" as a valid answer. + + Randomize which pole is shown as "A" and which as "B" each round — a seeded + shuffle keyed on the round (e.g. `pair_id`) so it is reproducible, not a fresh + coin flip you can't reconstruct later. Always displaying `calibrate_pairs.py`'s + own `a_text` as "A" would let the user learn the game's fixed pole ordering + after a few rounds, reintroducing the anchoring this step is trying to avoid. + Record the mapping — which of the script's `a_text`/`b_text` was actually shown + as "A" this round — alongside the round so the recorded `choice` and + `a_label`/`b_label` in step 4 resolve to the correct pole regardless of what + was displayed on screen. +4. Append one line to `.unslop/voice//preferences.jsonl`: + ```json + {"pair_id": "...", "dimension": "...", "choice": "a|b|neither", "ts": "...", + "a_label": "", "b_label": ""} + ``` + `a_label`/`b_label` come straight off `calibrate_pairs.py`'s `transform_applied` + (`":"` names B's pole; A sits at the dimension's other named pole). + Recording them is what lets `calibrate_score.py` report a semantic direction instead + of a bare "a"/"b" tally, and what makes conflict detection possible. +5. After each round, run: + ```bash + python3 scripts/calibrate_score.py --preferences preferences.jsonl + ``` + Stop playing a dimension once its `confidence >= 0.7` OR it has reached `k >= 9` + observations, whichever comes first. Below `k = 5` observations a dimension is always + reported `"insufficient"` regardless of how lopsided the tally looks — five is the + floor before a lean means anything. A dimension can also come back `"tied"` with + `preferred: null` when its top two tallies are exactly equal at `k >= 5` — that's a + real "no lean yet" result, not insufficient data, and not a coin flip the aggregator + should silently resolve one way. Play it a couple more rounds. Replaying the same + round (same `pair_id`) is safe: the aggregator dedups by `pair_id`, keeping only the + latest row by `ts`, so a retried write never double-counts or lets a stale choice win. +6. Use `--next` to pick which dimension to play next: + ```bash + python3 scripts/calibrate_score.py --preferences preferences.jsonl --next + ``` + Ordering: fewest observations first, then lowest confidence, tie-broken by the fixed + dimension order (`contractions, em_dash, sentence_length, connectives, staccato`) so + the choice is reproducible from the same preferences file. +7. When a sample-derived profile exists (`profile.json`, WP10a), run: + ```bash + python3 scripts/calibrate_score.py --preferences preferences.jsonl --profile profile.json + ``` + Any confident (`>= 0.7`) stated preference that contradicts the profile's measured + value comes back as a `conflicts[]` record naming both values and their provenance, + e.g. "stated preference for 'short' (confidence 0.70) contradicts + avg_sentence_length=22.4 measured from samples." Surface every conflict to the user + verbatim — do not silently pick a winner. Let the user say which one wins; record + that choice's provenance on the card. +8. Hand the final per-dimension summary to the teach-card step. Every claim on the card + sourced from this game carries provenance `"stated-preference"`, distinct from + `"measured-from-samples"` values pulled from the profile directly. + +## Voice overrides defaults + +A generated variant can legitimately trip `banned_phrase_scan.py` — a staccato pole reads +as `anti_slop_register` (three short fragments in a row is the scanner's own AI-cadence +tell), a plain connective can occasionally read as `filler_opener`. Do not decline these +pairs or quietly steer the user away from the flagged pole; the scanner encodes a default +register guard, and this game exists specifically to find out when a user's real voice +sits outside that default. + +When a user's choices consistently land on a pole that `a_flags`/`b_flags` marks as +flagged (e.g. they keep picking the staccato side even though it trips +`anti_slop_register`), record the preference together with its flags. Surface the tension +to the user once, not on every round — something like: "you prefer staccato; the default +register guard flags it as an AI cadence tell — your voice wins in your profile." Do not +re-litigate it every round once the user has answered. + +On the final per-dimension summary (step 8), mark that dimension's provenance +`"user-preference overrides register guard"` instead of a bare `"stated-preference"`, so +the teach-card step (and anyone reading the card later) can see that the scanner's default +was deliberately overridden, not silently ignored. This is the same conflict-surfacing +philosophy as the profile conflicts below: voice beats defaults, but only visibly. + +## Provenance discipline + +The card must always be able to say, for any voice claim, whether it came from the +user's own samples (measured) or from a choice the user made in this game (stated). +Conflicts are a feature, not a bug to smooth over — surfacing them is the whole point +of running both a profile and a preference game. diff --git a/references/commands/cleanup.md b/references/commands/cleanup.md new file mode 100644 index 0000000..99e5ce3 --- /dev/null +++ b/references/commands/cleanup.md @@ -0,0 +1,59 @@ +# /unslop cleanup + +Surface AI patterns as reviewable edits the user accepts or rejects, rather than +a finished rewrite. Agent-invoked, never a background daemon. Route here on +`/unslop cleanup` or when the user wants suggestions instead of a silent rewrite. +Set `$INPUT` to the source. + +## Report-only variant + +When the user says "flag only", "just detect", "audit only", "don't change +anything", or passes `--report`, change nothing. Scan and report: + +```bash +python3 scripts/banned_phrase_scan.py <<< "$INPUT" +python3 scripts/structure_scan.py <<< "$INPUT" +python3 scripts/silhouette_scan.py <<< "$INPUT" +python3 scripts/readability_metrics.py <<< "$INPUT" +``` + +Report each issue by span, category, severity, and why it reads as machine- +written, separating clear problems from register-dependent judgment calls (the +audit-only shape in SKILL.md **Output Format**). Re-read negations, scope, and +certainty yourself; the scanners are necessary, not sufficient. + +## Suggestion flow + +```bash +python3 scripts/suggest.py document.md # emit suggestions +python3 scripts/suggest.py doc.md --apply-replacements repl.json # merge model replacements +python3 scripts/check_suggestions.py suggestions.json # blocking contract gates +``` + +- **Detection is cheap and deterministic.** `suggest.py` runs both scanners and + emits LSP-style suggestions `{span, severity, category, rationale, + suggested_replacement, phrased_as_question}`, ordered and non-overlapping. +- **Replacement generation is delegated to a stronger model.** `suggest.py` + leaves `suggested_replacement` null; the stronger model fills them, merged back + via `--apply-replacements`. +- **Hard findings** become direct replacements; **soft findings** are register- + dependent, so their rationale is phrased as a question + (`phrased_as_question: true`). +- **Suggestions are surfaced to the user, never silently applied.** + +## Contract gates + +`check_suggestions.py` is the blocking contract. Every replacement clears all +four gates or it is rejected: + +- `span-minimality`: the edit changes only its span (a whole-sentence rewrite + fails). +- `replacement-scanner`: each replacement passes both scanners in isolation and + adds no new violation in context. +- `accept-all`: applying every suggestion yields a document that passes both + scanners with `validate_preservation` exit 0 versus the original. +- `span-overlap`: spans must not overlap. + +Non-English input is declined the way the scanners decline it: `suggest.py` +returns `non_english: true` with no suggestions. For a full rewrite instead of +suggestions, route to `references/commands/rewrite.md`. diff --git a/references/commands/contribute.md b/references/commands/contribute.md new file mode 100644 index 0000000..48ca14c --- /dev/null +++ b/references/commands/contribute.md @@ -0,0 +1,52 @@ +# /unslop contribute + +Turn one AI-ism caught in the wild into an eval row and a structured PR. Route +here on "contribute a pattern", "add this tell", or "I found a new AI-ism". +Everything stays offline until the user approves both publication and the final +PR body. Read `references/contribute.md` for the precheck exit codes, redaction +discipline, row-diff verification, and the non-maintainer fork path. + +## Flow + +1. **Precheck.** `python3 scripts/contribute.py precheck /abs/path/snippet.txt`. + If it exits 3 the tell is already covered, so add a REC row rather than + opening a fresh false-negative row. + +2. **User confirmation gate #1 — publication.** Show the exact snippet and ask + whether it may go in a public repo. Offer redaction hints from + `python3 scripts/extract_constraints.py < snippet.txt`. Redact only with + explicit approval, and keep the tell byte-for-byte intact. + +3. **Scaffold** the working bundle under `.unslop/contrib//` (do not + commit it): + ```bash + python3 scripts/contribute.py scaffold --snippet /abs/path/snippet.txt \ + --tell "exact substring" --category significance_inflation \ + --pattern-name exact-substring-slug --redact "Alice=NAME" + ``` + +4. **Implement eval-first**, following `references/maintenance.md`: copy + `row_fn.json` into `evals/adversarial-evals.json`, renumber from the live + maxima, add the literal-use FP row (and a REC row if a word is being gated + behind a collocation). Confirm the FN row is **red** before you touch the + scanner, then update `scripts/banned_phrase_scan.py` and + `references/taboo-phrases.md` until it goes **green** while the FP row holds. + +5. **Verify:** `python3 scripts/contribute.py verify --bundle .unslop/contrib/`. + It refuses TODO markers, checks specimen fidelity, and captures the + red-to-green transition. + +6. **Full gate battery:** `run_adversarial.py`, `build_shared_benchmark.py` + (+`--check`), `check_taboo_parity.py`, `check_pattern_coverage.py`, + `kata_add_pattern.py --run`, and `skill-benchmark validate ... --strict-leakage`. + If `SKILL.md`, `presets/`, or `references/` changed, also run + `evals/run_behavioral.sh tune`. + +7. **Render the PR body:** + `python3 scripts/contribute.py report --bundle .unslop/contrib/ > pr-body.md`. + +8. **User confirmation gate #2 — publication of the PR.** Show the final PR body. + Only after the user approves do you branch, `git add` the three changed files, + commit, and `gh pr create` (or the fork path in `references/contribute.md` for + non-maintainers). Scripts and evals never call `gh` or any network command; + the host agent runs those after approval. diff --git a/references/commands/mimic.md b/references/commands/mimic.md new file mode 100644 index 0000000..6d6f8c1 --- /dev/null +++ b/references/commands/mimic.md @@ -0,0 +1,66 @@ +# /unslop mimic + +Single-pass voiced writing. Draft or rewrite in a taught voice, then clear the +full gate battery. Route here on "write this like me", "match my voice", or +"mimic this author" once a voice card exists. Read `references/mimic.md` for card +anatomy, scoring, and failure modes. + +## Flow + +1. Load the situation sheet the task needs. Read `card.md` for the always-on core + (rhythm, contraction habits, punctuation, openers, the Never list) and the + one `card/.md` sheet that matches what you are writing. Load only + what the task needs; the card is layered so generation stays cheap. +2. Draft or rewrite in that voice. Follow the card's measured markers and + verbatim snippets; do not invent a habit the samples cannot support. +3. Run the output through **every** removal gate: + ```bash + python3 scripts/banned_phrase_scan.py <<< "$OUTPUT" + python3 scripts/structure_scan.py <<< "$OUTPUT" + python3 scripts/readability_metrics.py <<< "$OUTPUT" + python3 scripts/validate_preservation.py original.txt "$OUTPUT_FILE" + python3 scripts/diff_check.py original.txt "$OUTPUT_FILE" + ``` +4. Score the voice: + ```bash + python3 scripts/voice_score.py --profile profile.json "$OUTPUT_FILE" + ``` + +## The rule + +A mimic that scores well on voice but trips a slop gate is **rejected**. Voice +never buys an exemption from the constitution — the removal gates are hard, the +voice score is a guide. Meaning preservation versus the original draft is its own +hard gate, never blended into the voice score. + +## Refine — when one pass is not enough + +Most users teach once and mimic cheaply thereafter. When a single pass keeps +scoring short of the target voice, hill-climb with the refine loop — worth the +cost only then: + +```bash +python3 evals/run_mimic_refine.py --samples samples/ --draft draft.md --out OUT +``` + +LIVE (default) calls `--generate-cmd` (default `claude -p`) once per beam per +iteration; `--candidates-dir DIR` runs the loop logic with no model calls. Hard +gates (banned-phrase, structure, draft→candidate preservation, copy-gate, 150- +word floor) discard a candidate regardless of score; survivors are scored against +a held-out DEV split with the full `voice_score` composite, and the best-so-far +never regresses. The **divergence guard** halts and sets +`reward_hacking_warning` when the A composite improves while DEV worsens for two +iterations — the reward-hacking signature. Confirm a win only when +`mimic_stats.py` shows the CI lower bound > 0 and p < 0.05. Internals, baselines, +and the failure table are in `references/mimic.md`. + +## Voice check + +"Does this sound like me?" — score only, change nothing. Run +`python3 scripts/voice_score.py --profile .unslop/voice//profile.json +--impostors evals/fixtures/voice/impostors --seed 7 ` and report the +composite (lower is more them), the GI rank, and the two or three metric +deltas that explain the score, in plain words ("your sentences run longer than +usual; contractions match"). No rewrite unless asked. This is the cheapest +voice interaction and the usual one after teach: check drafts often, commission +rewrites rarely. diff --git a/references/commands/rewrite.md b/references/commands/rewrite.md new file mode 100644 index 0000000..018d24d --- /dev/null +++ b/references/commands/rewrite.md @@ -0,0 +1,50 @@ +# /unslop rewrite + +The default command. Bare `/unslop ` routes here. Two passes: diagnose, +then reconstruct under the register guards, then prove the result with the gates. + +Input arrives as an argument, a file path, or stdin. Set `$INPUT` to the source +and `$OUTPUT` to your rewrite. + +## Pass 1 — diagnose + +1. Extract the facts and constraints that must survive: + ```bash + python3 scripts/extract_constraints.py <<< "$INPUT" + ``` +2. Scan the source so the rewrite answers evidence, not a hunch: + ```bash + python3 scripts/banned_phrase_scan.py <<< "$INPUT" + python3 scripts/structure_scan.py <<< "$INPUT" + python3 scripts/silhouette_scan.py <<< "$INPUT" + python3 scripts/readability_metrics.py <<< "$INPUT" + ``` + Pass `--genre docs` or `--genre social` only when the input truly belongs to + that genre. Add `banned_phrase_scan.py --include-quoted` only when the user + wants quoted examples audited too. +3. Read the selected preset from `presets/`. Reach for + `references/taboo-phrases.md` on phrase edge cases, + `references/fact-preservation.md` on dense facts, `references/rubric.md` for + strict scoring, and `references/personality-guide.md` only when a clean draft + still reads as anonymous. + +## Pass 2 — reconstruct + +Rewrite under the SKILL.md **Rewrite Principles** and **Register Guards**. Cut +the throat-clearing, keep every fact and guard word, match the preset, and never +trade AI slop for anti-slop cadence. If a gate fails twice after rewrite, +escalate the model tier instead of piling on prompt rules. + +## Validate + +Run the full gate battery from SKILL.md **Validation** on `$OUTPUT` before +returning it. A hard banned-phrase hit, a structure or silhouette flag, an +`anti_slop_register` hit, a dropped constraint, or staccato cadence all block +the return. Use `validate_preservation.py --strict` for legal, medical, +security, or scientific text. + +Return the cleaned text only, unless the user asked for the strict analysis +block in SKILL.md **Output Format**. + +For multi-agent execution, use `references/pipeline.md`; for orchestrated +detector packs, use `references/packs/`. diff --git a/references/commands/teach.md b/references/commands/teach.md new file mode 100644 index 0000000..63e02be --- /dev/null +++ b/references/commands/teach.md @@ -0,0 +1,77 @@ +# /unslop teach + +Build a reusable voice from a writer's samples. **Agent-driven end to end**: the +user supplies only approvals and answers, never a directory or a command. Route +here on "teach my voice", "learn how I write", or when the user offers samples to +learn from. Any samples the user provides are fair game; there are no rights or +attestation checks. + +Read `references/mimic.md` for card anatomy, sample requirements, and scoring +before you finish; this file is the flow. + +## 1. Gather samples (harvest) + +If the user offers none, default to bootstrapping. Harvest their transcripts and +writing folders per `references/harvest.md`: + +```bash +python3 scripts/harvest_samples.py SOURCE [SOURCE...] -o candidates.json +python3 scripts/harvest_classify.py --candidates candidates.json --mode heuristic +``` + +Present the ranked candidates with their sources and flags; the user approves or +rejects each. Treat `suspect_ai` candidates skeptically — that human review is +the contamination defense, so never auto-approve a flagged paste. Copy only +approved samples into `.unslop/voice//samples/`. The adapter internals, +contamination tripwire, and privacy rules live in `references/harvest.md`. + +## 2. Build the profile and card + +Create `.unslop/voice//` yourself (it is gitignored) and run: + +```bash +python3 scripts/voice_profile.py samples/ -o profile.json +python3 scripts/voice_card.py --profile profile.json --samples samples/ --out . --name --provenance +``` + +`voice_profile.py` is the machine fingerprint the scorer refers to. +`voice_card.py` writes the layered card the generating model follows (`card.md` +plus `card/.md` sheets) and, with `--provenance`, a `provenance.json` +audit trail. Both scripts read only `.txt` and `.md` files; convert samples +first. `voice_card.py` refuses (exit 2) when the profile does not describe the +supplied samples, so a stale profile can never drive a card. + +## 3. Surface confidence and gaps + +Show the user `card.md`. If the profile is `low_confidence` (thin or cross-genre +samples), say the voice is provisional and ask for more same-genre samples before +trusting a mimic. For an **Uncovered** dimension the target writing will need, +ask for one short sample that exercises it — and only those the task requires, not +all ten. Prompt templates: + +| Dimension | Prompt | +|-----------|--------| +| explaining-technical | "Share something you wrote explaining how a thing works." | +| anecdote | "Share a few sentences telling a small story that happened to you." | +| argument | "Share something where you defended a claim." | +| disagreement | "Share something where you pushed back on an idea." | +| praise | "Share something where you recommended something you liked." | +| hedging-uncertainty | "Share something where you were unsure and said so." | +| numbers-data | "Share something you wrote that works with quantities." | +| addressing-reader | "Share a note or instructions written to a reader." | + +`openings` and `closings` are structural and covered as soon as one document +exists, so they need no prompt. The full canonical table lives in +`references/mimic.md`. + +When coverage or confidence stays thin and the user cannot supply more samples, +offer the A/B calibration game (`references/calibrate.md`): it elicits voice +preferences through reversible pair choices and feeds them to the card as +`stated-preference` provenance, distinct from `measured-from-samples`. + +## 4. Close with a scored demo + +Prove the loop before the user trusts it: mimic one paragraph per +`references/commands/mimic.md`, run `scripts/voice_score.py` and the scanners, +and show the results. A voiced demo that trips a slop gate fails the loop; do not +ship it with an excuse. diff --git a/references/contribute.md b/references/contribute.md new file mode 100644 index 0000000..44a6422 --- /dev/null +++ b/references/contribute.md @@ -0,0 +1,116 @@ +# Contributing a New AI-ism — internals + +The routed step-by-step is in `references/commands/contribute.md` (reachable as a +maintenance path, not a top-level verb). This file is the exhaustive companion: +every command flag, the redaction discipline, the row-diff verification, and the +non-maintainer fork path. Keep it offline until the user has approved both +publication and the final PR body. + +## 1. Precheck + +```bash +python3 scripts/contribute.py precheck /absolute/path/to/snippet.txt +``` + +If the snippet is already flagged, the command exits 3 and prints the covering +patterns/categories. Add a REC row instead of a new false-negative row. + +## 2. User Confirmation Gate #1 + +Show the exact snippet to the user and ask whether it may be published in a +public repo. Offer redaction hints from: + +```bash +python3 scripts/extract_constraints.py < /absolute/path/to/snippet.txt +``` + +Redact names, numbers, or private details only with explicit approval. The tell +must remain byte-for-byte intact. + +## 3. Scaffold + +```bash +python3 scripts/contribute.py scaffold \ + --snippet /absolute/path/to/snippet.txt \ + --tell "exact substring" \ + --category significance_inflation \ + --pattern-name exact-substring-slug \ + --redact "Alice=NAME" +``` + +The bundle is written under `.unslop/contrib//`. Do not commit this +bundle; it is working material for the agent and reviewer. + +## 4. Implement the Pattern + +Follow `references/maintenance.md` in this order: + +1. Copy `.unslop/contrib//row_fn.json` into + `evals/adversarial-evals.json`. +2. Renumber the copied row from the live maxima in + `evals/adversarial-evals.json`; do not keep the `CONTRIB-FN-*` bundle id in + the committed suite. +3. Add the literal-use FP row for the category, and add a REC row if an existing + word is being gated behind collocations. +4. Run the new rows before implementing the pattern and confirm the FN row is + red while the FP and any REC rows encode the intended boundary. +5. Update the scanner and `references/taboo-phrases.md`. +6. Re-run `python3 scripts/contribute.py verify --bundle .unslop/contrib/`. +7. Diff the committed eval row against `.unslop/contrib//row_fn.json` and + confirm only expected suite fields changed, such as id, category grouping, or + row ordering. The specimen stdin and assertion intent must still match. + +## 5. Verify + +```bash +python3 scripts/contribute.py verify --bundle .unslop/contrib/ +``` + +Verify refuses reports with TODO markers, checks specimen fidelity, captures the +red-to-green transition for the FN row, and records offline gate tails. + +## 6. Run the Full Gate Battery + +```bash +python3 evals/run_adversarial.py +python3 evals/build_shared_benchmark.py +python3 evals/build_shared_benchmark.py --check +python3 evals/check_taboo_parity.py +python3 evals/check_pattern_coverage.py +python3 evals/kata_add_pattern.py --run +skill-benchmark validate evals/shared-benchmark.json --strict-leakage +``` + +If `SKILL.md`, `presets/`, or `references/` changed, also run: + +```bash +evals/run_behavioral.sh tune +``` + +## 7. Render the Report + +```bash +python3 scripts/contribute.py report --bundle .unslop/contrib/ > /tmp/pr-body.md +``` + +## 8. User Confirmation Gate #2 + +Show the final PR body to the user. Only after the user approves publication: + +```bash +git switch -c add- +git add evals/adversarial-evals.json scripts/banned_phrase_scan.py references/taboo-phrases.md +git commit -m "Add AI-ism pattern" +gh pr create --title "Add pattern: " --body-file /tmp/pr-body.md +``` + +For non-maintainers: + +```bash +gh repo fork --remote +git push --set-upstream add- +gh pr create --repo /unslop --head :add- --body-file /tmp/pr-body.md +``` + +The `gh` commands are for the host agent after approval. Scripts and evals must +never call `gh` or any network command. diff --git a/references/harvest.md b/references/harvest.md new file mode 100644 index 0000000..658f6cd --- /dev/null +++ b/references/harvest.md @@ -0,0 +1,41 @@ +# Harvesting Voice Samples — internals + +Harvest bootstraps teach samples from local chat transcripts and folders the user +declares as their own writing. The routed flow (harvest → classify → approve → +copy) is step 1 of `references/commands/teach.md`; this file holds the adapter +internals, the contamination tripwire, and the privacy rules that flow links to. + +The transcript adapters only accept entries with explicit user authorship. Assistant +turns are dropped before candidate filtering, and unknown JSONL schemas are skipped with +a warning rather than guessed. This authorship separation is the product requirement: +assistant-authored transcript text must never enter candidates. + +The `codex-jsonl` adapter reads Codex CLI/Desktop sessions, stored under +`~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` and auto-detected by shape alongside +`claude-jsonl`. It harvests `event_msg` user_message text and user-role +`response_item` message content, and drops assistant/developer-role turns, tool +and reasoning plumbing, and `session_meta.base_instructions` outright. Codex also +injects non-user content into user-role turns: a repo's AGENTS.md dumped verbatim, +an `` banner, and similar wrapper tags. The adapter filters +those by structural marker, never by guessing at content, and counts them under +the `instruction-injection` drop reason instead of silently discarding them. + +The contamination tripwire runs each kept candidate through `banned_phrase_scan` and +`structure_scan`. A hard violation, or two or more distinct scanner categories, marks +the sample as `suspect_ai: true`. Suspect samples are kept for human review, ranked last, +and must never be auto-approved. +A clean-styled, unmarked assistant paste inside a user turn is undetectable by parsing +and only weakly covered by the tripwire; mandatory human approval is the real defense. + +The folder adapter treats `.md` and `.txt` files in a designated folder as user-authored +by declaration, but still runs the same tripwire. Dictated text with heavy `um`/`uh` +fillers is marked `dictated: true` and kept because spoken phrasing can still be useful +voice evidence. + +Privacy: harvesting is local. Candidate files, classifier task files, and approved +`.unslop/voice//` samples should stay out of git unless the user explicitly chooses +otherwise. + +Tiering: parsing and filtering are Tier 0 deterministic work. Situation/register +classification can be fanned out to the cheapest capable model through the emitted task +files. Final approval stays human. diff --git a/references/maintenance.md b/references/maintenance.md index 7838642..dfeda81 100644 --- a/references/maintenance.md +++ b/references/maintenance.md @@ -23,6 +23,43 @@ Same procedure, but the entry goes in `STRUCTURAL_PATTERNS` with a regex, and the false-positive row must cover the nearest legitimate construction the regex could clip. +## Prefer a pair + +When a pattern has literal senses or only becomes visible in context, add a +minimal pair under `evals/fixtures/pairs/` instead of relying only on one-line +stdin rows. The `_with` twin should contain one controlled tell, the `_without` +twin should preserve the facts while scanning clean, and the manifest should +name the target category or structure metric. Run `python3 evals/check_pairs.py` +before changing the scanner. + +## Coverage gate and category protection (enforced) + +`python3 evals/check_pattern_coverage.py` makes the two rows above mandatory, not +conventional: + +- **Coverage.** Every `STRUCTURAL_PATTERNS` regex must match at least one script + row's corpus (its stdin or a referenced fixture), and every `BANNED_PHRASES` + key must appear in at least one corpus text. A new scanner entry with no eval + row fails this gate (DOC-09) while every other gate is green. There is no + grandfather list: an uncovered pattern is a hard failure. If a batch of legacy + keys is uncovered, add `REC`-style coverage packs (8-12 phrases per stdin, one + natural sentence each) rather than exempting them. +- **Protection.** Every violation category the scanner can emit must be claimed by + a `scanner_false_positive` row carrying `"protects": ""`. Backfill the + field on the FP row whose failure mode names that category; add a new FP row + (asserting `total_violations == 0` on tempting-but-clean prose) for any category + with no protector. The runner ignores the `protects` key, so it is safe on any + FP row. + +## Rehearse the procedure (the kata) + +`python3 evals/kata_add_pattern.py --run` is a meta-eval (DOC-10): in a temp copy +of the repo it adds a throwaway pattern and asserts each safety net fires in order +— coverage catches the row-less entry, parity catches the missing catalog line, +the runner goes green once both exist, and the row goes red if the scanner entry +is later removed. Run it before touching the maintenance tooling; if a refactor +breaks a safety net, this kata turns red. + ## List current patterns ```bash diff --git a/references/mimic.md b/references/mimic.md new file mode 100644 index 0000000..665dafd --- /dev/null +++ b/references/mimic.md @@ -0,0 +1,194 @@ +# Teach & Mimic — internals + +The routed flows live in `references/commands/teach.md`, +`references/commands/mimic.md`, and the refine section of that mimic command +file. This file holds the deep material those flows link to: card anatomy, +sample requirements, scoring, baselines, statistics, and failure modes. + +Reconstruction runs under detection's constitution: a mimic or refined output +that reintroduces slop is a failure. Every artifact here is deterministic and +testable; the model is used only to generate prose, never to score itself. + +## Teach — the artifacts + +`teach` distills a writer's samples into a reusable voice, stored under +`.unslop/voice//` (gitignored by default): + +1. `scripts/voice_profile.py samples/ -o profile.json` — the machine + fingerprint (char 3-grams, function-word delta, sentence-length EMD, + punctuation, contractions, MTLD, word-length, impostor z-scores, GI rank). + This is the referee the scorer uses. +2. `scripts/voice_card.py --profile profile.json --samples samples/ --out . + --name ` — the **layered voice card** the generating model actually + follows. `--name` labels the card (`# Voice card: `); default `voice`. + voice_card recomputes the profile from `--samples` and refuses (exit 2, named + field) if the supplied `--profile` does not describe those samples, so a stale + profile can never silently drive a card. +3. Add `--provenance` to write `provenance.json` (per-sample sha256, word + counts, doc count, genre note, low-confidence flag) so a teach run is + auditable. + +**Sample files.** Both scripts read only `.txt` and `.md` files, recursively, +under the samples directory (nothing else — a directory of `.docx` or extensionless +files reads as empty and both scripts exit 2 with a diagnostic naming the +requirement). Convert samples to `.txt`/`.md` first. + +**Sample requirements.** Use at least 5 documents, 2-3k words total, in the +**same genre** as the target. Fewer than ~2000 words or cross-genre samples set +`low_confidence` in the profile and provenance; the card still builds but its +claims are noisier. **Surface `low_confidence` to the user** — tell them the voice +is provisional and ask for more same-genre samples before trusting a mimic. +Cross-domain style attribution degrades sharply, so do not teach on blog posts +and score legal memos. + +## The voice card (layered) + +The card is a directory, not one file, so the generator loads only what the +current writing task needs: + +- `card.md` — the always-loaded core, kept under 300 words: rhythm (median + sentence length, IQR, burstiness), contraction habits with real examples, + punctuation the writer uses and avoids, top sentence openers, a **Never** + list of at/near-zero features, and an **index table** — "when writing X, read + `card/X.md`". +- `card/.md` — one sheet per **covered** situation from the taxonomy + (explaining-technical, anecdote, argument, disagreement, praise, + hedging-uncertainty, numbers-data, addressing-reader, openings, closings). + Each sheet gives how this author handles that situation, 1-3 verbatim sample + snippets, and measured markers. + +**No fabrication.** A dimension with no sample evidence gets **no sheet**; it is +named under "Uncovered" in `card.md`. The card never invents a voice the +samples cannot support. + +**Coverage classification.** `voice_card.py --coverage` emits a deterministic +lexical coverage matrix over the taxonomy. It is intentionally coarse: it +DRIVES which sheets get written and which teach prompts to ask the user, and a +misclassified sentence can only add or drop a sheet — it can never change a card +claim, because claims come from measured facts and verbatim snippets. So +misclassification is low-stakes by construction. A sample set missing numeric +writing, for instance, leaves `numbers-data` uncovered and named as a gap. + +**Coverage → prompt procedure.** When a dimension the target writing will need is +uncovered, ask the user for a short sample that exercises it. **Only ask for +dimensions the task requires** — do not fish for all ten. Templates: + +| Dimension | Prompt to the user | +|-----------|--------------------| +| explaining-technical | "Share something you wrote that explains how a thing works or why it behaves the way it does." | +| anecdote | "Share a few sentences where you tell a small story about something that happened to you." | +| argument | "Share something you wrote that argues a position — a claim you defended." | +| disagreement | "Share something where you pushed back on or disagreed with an idea." | +| praise | "Share something where you praised or recommended something you liked." | +| hedging-uncertainty | "Share something where you were unsure and said so — thinking out loud." | +| numbers-data | "Share something you wrote that works with numbers, quantities, or measurements." | +| addressing-reader | "Share something written directly to a reader — instructions or a note to someone." | + +`openings` and `closings` are structural (first/last sentence of every document) +and are covered as soon as there is one document, so they need no prompt. + +The card is pack-sized (~1-2k tokens loaded) so it rides in the generation +context cheaply. Same inputs give byte-identical files. + +## `--refine` — the internal hill-climb + +`evals/run_mimic_refine.py` iterates when one pass is not enough. It is worth +the cost only when a single mimic keeps landing short of the target voice; most +users teach once and mimic cheaply thereafter. + +Protocol (climb on A, accept on DEV, report on a sealed split): + +- Split samples by document (seeded): **A** (~60%, retrieval/context pool) and + **DEV** (~40%, acceptance). Fewer than 5 docs → refuse (exit 2). +- **LIVE (default):** each iteration assembles B prompts — the draft, the + A-split voice card, the k=2 nearest-A samples by char-3gram cosine, and the + current directives — and invokes `--generate-cmd` (default `claude -p`) once + per beam, piping the prompt on stdin and reading the candidate on stdout. + `--baseline zero|few|retrieval` selects which samples ride in the prompt (none + / first-k / k nearest-A) and generates its candidate set through this same + path, so the honest retrieval-few-shot competitor and the refine loop share + one pipeline. +- **DRY-RUN:** pass `--candidates-dir DIR` and the loop reads + `DIR/iter/*.md` as that iteration's batch instead of generating them, so + the loop logic runs with no model calls (this is how the MIMIC-* rows exercise + it; the LIVE path is covered by a mock generator). +- **Hard gates** discard a candidate regardless of score: banned-phrase clean, + structure clean, draft→candidate preservation, no copy-gate violation against + A, and a 150-word floor. +- Survivors are scored against DEV (and recorded against A for the divergence + guard) with the **full `voice_score` composite** — `0.5·(1−GI) + 0.5·` clipped + weighted impostor-z distance against a same-genre impostor pool (`--impostors`, + default the committed pool), seeded. This is deliberately NOT a raw weighted + distance: a marker-stuffed candidate can minimize a raw distance and clear + every hard gate yet be exposed by GI, so raw-distance acceptance would accept + it over honest prose. The generator never scores itself. +- Accept an iteration only if the DEV composite improves by ≥ `min-delta`; the + best-so-far never regresses. + +### Stopping and the divergence guard + +Earliest of: iteration cap, patience (N iterations without a DEV gain), or +**divergence** — the A composite improving while DEV worsens for two +consecutive iterations. Divergence is the reward-hacking signature (the model +is fitting the retrieval pool's topics, not the author's style); the loop halts +immediately and sets `reward_hacking_warning`. + +### Directives and card refinement + +Each iteration derives up to four ranked directives from the DEV metric deltas, +and each directive carries a proposed **card amendment** line. The best +candidate is copied to `OUT/final.md` and the amended card to +`OUT/voice-card.refined.md`. + +## Scoring + +The `voice_score` **composite** the refine loop accepts on is +`0.5·(1−GI) + 0.5·zsum` (lower = more author-like), where `GI` is the General +Impostors rank — the fraction of random feature-subset trials in which the +candidate beats every sampled impostor at looking like the profile — and `zsum` +is the clipped, weighted **impostor-z** distance (each per-feature distance +normalized by the impostor pool's mean/std, clipped to ±3, weighted by the WP10a +research weights). Both halves are calibrated against a same-genre **impostor +pool**, so the number rewards *beating plausible other authors*, not merely +matching aggregate surface stats. That is what defeats a marker-stuffed +candidate: it can drive a raw weighted distance down and still be far worse under +GI. Meaning preservation vs the original draft is a **separate** hard gate, never +blended into the composite. The composite is a **guide, not an oracle** — even +strong authorship verification tops out near a 93-94% ceiling, so treat +per-candidate deltas as the signal and under-claim. + +## Baselines + +Score refine output against paired baselines on the same drafts: zero-shot +("write like this person"), static few-shot, and **retrieval few-shot** (k +nearest-A samples — the honest competitor). Refine has to beat retrieval +few-shot to earn its cost. + +## Statistics + +`evals/mimic_stats.py` runs the small-n discipline: per-item paired deltas, a +BCa bootstrap CI (n=2000, seeded), and a sign-flip permutation test (exact for +≤12 items, else 20k seeded). Claim a win only when the CI lower bound > 0 **and** +p < 0.05. Anything degenerate (too few items, zero variance) reports +not-improved. + +## Cost + +| Step | Calls | +|------|-------| +| teach (profile + card) | free / deterministic | +| mimic | one strong generation call | +| refine | iterations × beam strong calls, plus free gate/score passes | + +## Failure modes and the gate that catches each + +| Failure | Symptom | Caught by | +|---------|---------|-----------| +| Topic bleed | importing what samples are *about*, not *how* they read | divergence guard (A improves, DEV worsens) | +| Verbatim copying | lifting sample phrasing | copy-gate (4-gram / LCS vs A) | +| Register lock | latching one sample's register | DEV composite vs held-out split | +| Reward hacking | self-assessed quality rising while DEV falls | divergence guard + `reward_hacking_warning` | +| Slop reintroduced | AI tells creep back into voiced prose | banned-phrase / structure gates | + +For chunking long documents and the tiered execution architecture around these +steps, see `pipeline.md`. diff --git a/references/packs/manifest.json b/references/packs/manifest.json index 060cc27..780edb7 100644 --- a/references/packs/manifest.json +++ b/references/packs/manifest.json @@ -44,7 +44,7 @@ "macro_emotional_flatness" ], "pack-register-guards": [ - "hedge", + "hedge_stack", "parenthetical_hedging" ], "pack-voice": [ diff --git a/references/packs/pack-structure.md b/references/packs/pack-structure.md index a86a985..a1f7172 100644 --- a/references/packs/pack-structure.md +++ b/references/packs/pack-structure.md @@ -7,6 +7,9 @@ Use this pack for document-level AI structure. Do not report isolated phrases un - `structure_scan.py` flags: `sentence_burstiness`, `conclusion_coda`, `bold_colon_listicle`, `one_line_staccato`, `connective_paragraph_openers`, `signpost_density`, `opener_repetition`, `participial_closer_share`. - Concrete thresholds to self-check: average sentence length under 8 or over 34 words is suspicious; sentence-length variance under 18 on 5+ sentences suggests uniform rhythm; 3+ one-line paragraphs under 8 words suggests staccato; 3+ consecutive paragraphs opened by connectives/signposts suggests scaffolding; 4+ repeated paragraph openers is blocking; 35%+ participial sentence endings suggests templated cadence. - Judge-only macro tells from `taboo-phrases.md`: both-sidesism, templated redemption arc, preview/recap symmetry, over-determination, uniform emotional register. +- `silhouette_scan.py` flags idea arrangement above the surface: `scaffold_opener_share`, `callback_content` (recap loop), `role_entropy_bits`, `preview_fulfillment`, `heading_preview`, composited into `silhouette_penalty` (flags at `>= 1.0`). +- A high `silhouette_penalty` means the document follows a template outline (preview-then-fulfill, rotating scaffold openers, closing recap); cite the specific flagged metric, not just the composite. +- Cue-deletion evades silhouette but not surface repetition: pair `silhouette_scan` with `structure_scan` (`opener_repetition`, `one_line_staccato`) so a stripped-connective attack still fails one of the two fences. Genre matters. `--genre docs` can allow reference-doc structure; `--genre social` can allow deliberate social cadence. Output still fails if the genre excuse is false. diff --git a/references/pipeline.md b/references/pipeline.md index 57b21bd..61d170b 100644 --- a/references/pipeline.md +++ b/references/pipeline.md @@ -46,13 +46,106 @@ Send one strong-enough rewriter the original text, merged Tier 0/Tier 1 findings | Job | Default executor | Escalate when | |---|---|---| | Tier 0 scripts | local deterministic | never; fix the script or input | -| Detection packs | smallest available model | JSON is malformed or pack scope is violated | -| Short/simple rewrite | mid-tier model | output fails a blocking gate twice | -| Regulated, legal, medical, security, or heavy macro rewrite | strongest practical model | start here if constraints are dense | +| Detection packs | cheapest tier (see Model Parity) | JSON is malformed or pack scope is violated | +| Span replacement / short rewrite | cheapest tier — gates carry safety (parity 2026-07-06: 8/8) | output fails a blocking gate twice | +| Full rewrite of register-sensitive text (legal, medical, security, load-bearing hedges) | strongest practical model + mandatory Tier-0 re-scan | start here; cheap tiers erode register | +| Macro structure (restructuring, coda/preview removal) | machine-gated, never self-checked — surface via Tier-0 scan | never trust a model's own macro self-check | | Judge/eval | model specified by `evals/BEHAVIORAL-EVALS.md` | benchmark protocol changes | +The model-dependent rows above are not set by taste. They are set by +`evals/run_model_parity.py` (see Model Parity), whose live matrix was recorded 2026-07-06: +span replacement clears on the cheapest tier because the output gates carry safety, while +full rewrites of register-sensitive text and any macro restructuring escalate to the +strongest practical model with a Tier-0 re-scan. Re-run the harness when the model features +change and update these rows from its output. + If Tier 2 output fails the same blocking gate twice, escalate one model tier. Do not add more rules to the prompt; the failure is execution quality. +## Model Parity + +The pipeline depends on a model in exactly two places: Tier-1 pack **detection** and +Tier-2 **replacement** generation. Everything else is deterministic. Whether a cheap model +may own those surfaces is a measured question, not an assumption. + +`evals/run_model_parity.py` measures it against a fixed seeded corpus: + +- **Task A (detection).** Each model reads one pack file plus a short seeded chunk (the + Tier-1 contract above) and returns JSON findings. Grading is deterministic: recall of the + seeded findings and a count of false findings, scored against a frozen manifest. Six + fixtures span the phrases-core, voice, and register-guard families (two are register-guard + cases, where the load-bearing hedge must be caught). +- **Task B (replacement).** Each model is handed one seeded finding (span + rationale) and + asked for a span-minimal replacement. Grading applies the co-writer contract: the + replacement removes the flagged tell and adds no new one (both scanners), preserves every + fact/number/negation, and stays span-minimal. + +The matrix is config-driven — entries are `{name, kind, model_id}` where `kind` is +`claude-cli` (`claude -p --model `, the Anthropic spectrum) or `openrouter` (the GPT +spectrum and others, key read from the macOS keychain service `OPENROUTER_API_KEY`). The +harness emits a per-model / per-task score table (JSON and a markdown summary) plus a +`--dry-run` mode that grades canned response fixtures with no network for the `PARITY-*` +eval rows. + +**Rule (binding).** Any change to the co-writer, mimic, or detector-pack model features +must run this harness across both the GPT and Anthropic spectrums before merge, and the +detection/replacement rows of the tiering table above must be updated from its output. If +cheap models match the strong reference, use cheap everywhere; if they do not, keep the +strong model for the passes where the gap appears and record which. The `PARITY-*` rows in +`evals/adversarial-evals.json` gate the grader itself (dry-run, no network); they do not +substitute for the live run. + +### Recorded results + +Live matrix recorded **2026-07-06**. `evals/run_model_parity.py` re-measures on demand; +rerun it after any change to the co-writer, mimic, or detector-pack model features and +update these tables from its output. + +**Task B — span replacement.** One seeded finding (span + rationale) per trial, graded on +the co-writer contract: the replacement removes the flagged tell, adds no new one (both +scanners), preserves every fact/number/negation, and stays span-minimal. Eight seeded +findings per model. + +| Model | Kind | Task B pass rate | +|---|---|---:| +| claude-haiku | claude-cli | 8/8 | +| claude-sonnet | claude-cli | 8/8 | +| gpt-5.4-mini | openrouter | 8/8 | +| gpt-5.5 | openrouter | 8/8 | +| claude-opus | claude-cli | 7/8 | + +The single miss was opus dropping the fact "8:30" from a replacement — caught by +`validate_preservation`, not by model choice. On the mechanical span contract the cheapest +tier ties the flagships; the gates carry safety, the tier does not. + +**Behavioral parity matrix — full rewrites.** Eight register/structure cases per model, +graded deterministically across both the Anthropic and GPT spectrums. + +| Model | Kind | Full-rewrite pass rate | +|---|---|---:| +| claude-opus | claude-cli | 7/8 | +| gpt-5.4-mini | openrouter | 7/8 | +| claude-sonnet | claude-cli | 6/8 | +| gpt-5.5 | openrouter | 6/8 | +| claude-haiku | claude-cli | 5/8 | +| gpt-5.4-nano | openrouter | 5/8 | + +`MACRO-01` (`structure_clean`) failed for all six models, opus included: each kept a +conclusion coda the prose instruction told it to drop. No model self-checks macro structure +from prose. The cheap-tier misses were register and fact erosion in full rewrites — softened +`never`/`all`, a dropped `roughly`, a dropped legal `arguably`/negation, an emitted +`X, not Y` contrastive tail. The lab ladders are symmetric (7/6/5 on both spectrums). + +**Measured conclusions.** + +- **Span replacement = cheapest tier, with gates on.** The output gates enforce fact and + tell safety, so the smallest model is safe for span-minimal replacement. +- **Full rewrites of register-sensitive text = frontier.** Legal, medical, security, or any + text with load-bearing hedges/negation goes to the strongest practical model with a + mandatory Tier-0 re-scan; cheap models erode register in unsupervised full rewrites. +- **Macro structure = always machine-gated, never self-checked.** A Tier-0 scan surfaces it + as an explicit directive, because no model — flagship included — reliably catches it from + prose instructions. + ## Cost Note For a 1000-word document: diff --git a/references/taboo-phrases.md b/references/taboo-phrases.md index d6bc98d..f624543 100644 --- a/references/taboo-phrases.md +++ b/references/taboo-phrases.md @@ -482,6 +482,7 @@ These attribute claims to unnamed sources, creating an illusion of authority. | "Some critics" | Who? Be specific or cut. | | "Many believe" | Weasel phrasing | | "Research suggests" | Which research? | +| "Research indicates / Research shows" (bare, clause-initial) | Unattributed authority — cite the study or name the source. Attributed ("Research by the Kaiser group indicates…") and possessive ("Our research indicates…") forms stay clean. | | "Studies show" | Which studies? | | "It is widely regarded" | By whom? | | "According to experts" | Name the experts | @@ -718,6 +719,7 @@ without making the claim itself. (Signal adapted from `stop-slop`.) |--------|--------------| | "the numbers speak for themselves" | The writer should say what they show | | "the data tells a (clear) story" | Data doesn't narrate; state the finding | +| "tells a story" | Data or artifacts do not narrate; state the finding | | "the results speak for themselves" | Dodges the actual interpretation | | "X paints a clear picture" | Inflated stand-in for "X shows" | @@ -977,6 +979,24 @@ Related deterministic metrics: `sentence_burstiness`, `bold_colon_listicle`, `one_line_staccato`, `connective_paragraph_openers`, `signpost_density`, `opener_repetition`, and `participial_closer_share`. +### Silhouette (discourse-level) + +`scripts/silhouette_scan.py` scores idea arrangement one level above the surface +scanner. All five tells are scanner-enforced; the composite `silhouette_penalty` +flags at `>= 1.0` against a committed human reference. + +| Tell | What fires | +|------|------------| +| Scaffold openers | Body paragraphs opening on a discourse-cue class instead of their own claim — scanner-enforced (`scaffold_opener_share`). | +| Recap loop | Opening vocabulary that vanishes mid-document and returns at the end — scanner-enforced (`callback_content`, strongest single tell). | +| Rotating opener roles | Several distinct cue-opener classes rotating like a template — scanner-enforced (`role_entropy_bits`). | +| Preview-then-fulfill | Intro content words reappearing as body-paragraph heads — scanner-enforced (`preview_fulfillment`). | +| Heading preview | Section headings restating the intro's outline — scanner-enforced (`heading_preview`, retained under `--genre docs`). | + +Anti-gaming: a cue-deletion attack collapses the scaffold and role metrics, so +silhouette must be scored jointly with `structure_scan.py` (staccato / +opener_repetition) as the lower fence. + --- ## Quick Reference: Detection Regex diff --git a/scripts/_lang.py b/scripts/_lang.py new file mode 100644 index 0000000..0b8c305 --- /dev/null +++ b/scripts/_lang.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Shared cheap English-detection helpers for banned_phrase_scan.py and +structure_scan.py. Both scanners must decline the same non-English inputs, so +this table and its two functions live in exactly one place.""" + +import re + + +# A small set of high-frequency English function words used only for a cheap +# language check. The words are chosen to be distinctively English: Spanish, +# French, German, etc. rarely use them, so their share of tokens is a robust +# signal without a language-model dependency. +ENGLISH_FUNCTION_WORDS = frozenset({ + "the", "and", "is", "are", "was", "were", "of", "to", "in", "that", "it", + "for", "with", "on", "this", "but", "not", "you", "have", "be", "as", "at", + "or", "we", "they", "will", "would", "there", "their", "what", "which", + "when", "from", "been", "has", "had", "its", "an", "by", "our", "your", + "if", "than", "then", "them", "these", "those", "about", "into", "over", + "after", "before", "how", "why", "where", "who", "can", "could", "should", + "do", "does", "did", "so", "out", "just", "more", "most", "some", "such", + "only", "also", "because", "while", "between", "through", "during", "being", +}) + + +def english_function_share(text: str) -> float: + """Share of word tokens that are common English function words.""" + tokens = re.findall(r"[a-z']+", text.lower()) + if not tokens: + return 1.0 + hits = sum(1 for t in tokens if t in ENGLISH_FUNCTION_WORDS) + return hits / len(tokens) + + +def is_probably_english(text: str, threshold: float = 0.10, min_tokens: int = 15) -> bool: + """Cheap English detector backing a graceful non-English decline. + + Conservative on purpose: inputs below ``min_tokens`` are always treated as + English (too little signal to decline), and ``threshold`` is low enough that + even terse or ESL-flavored English clears it. Only prose with almost no + English function words (i.e. another language) is declined. + """ + tokens = re.findall(r"[a-z']+", text.lower()) + if len(tokens) < min_tokens: + return True + return english_function_share(text) >= threshold diff --git a/scripts/banned_phrase_scan.py b/scripts/banned_phrase_scan.py index c6a669f..d634e98 100755 --- a/scripts/banned_phrase_scan.py +++ b/scripts/banned_phrase_scan.py @@ -19,8 +19,18 @@ import sys import re import json +from pathlib import Path from typing import TypedDict +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from _lang import ( # noqa: E402 + ENGLISH_FUNCTION_WORDS, + english_function_share, + is_probably_english, +) + class Violation(TypedDict): phrase: str @@ -577,6 +587,20 @@ def _phrase_pattern(phrase: str) -> re.Pattern[str]: "severity": "soft", "suggestion": "foundation, base, layer" }, + # Bare unattributed "research indicates/shows/suggests" — the same vague_attribution + # move as "studies show": authority invoked with no source. Gated on the BARE, + # clause-initial form (sentence start or after .!?;:) where "research" sits + # immediately before the verb. An attributive phrase ("Research by the Kaiser group + # indicates …") or a leading possessive ("Our research indicates …") breaks the + # adjacency/anchor, so a named study or a concrete first-person claim stays clean. + # Soft: attribution is a judgment call and the anchor is a heuristic. Matched on + # lowercased text like every structural pattern. + { + "pattern": r"(?:^|[.!?;:]\s+)research\s+(?:indicates|shows|suggests)\b", + "category": "vague_attribution", + "severity": "soft", + "suggestion": "Cite the specific research or name the source." + }, # Promotional "boasts " (not "boasts a capacity of 50,000"). { "pattern": r"\bboasts?\s+(?:a\s+|an\s+)?(?:world-class|state-of-the-art|cutting-edge|impressive|stunning|robust|comprehensive|unparalleled|rich|vibrant|array of|host of|range of|wealth of|plethora)", @@ -739,7 +763,7 @@ def _phrase_pattern(phrase: str) -> re.Pattern[str]: "suggestion": "State what it's about directly" }, { - "pattern": r"\b(?:it'?s not|this is not|that'?s not|isn'?t|wasn'?t|aren'?t|weren'?t)\s+just\b[^.;!?\n]{1,60}[,;—–-]\s*(?:it'?s|it (?:is|was)|they'?re|that'?s)\b", + "pattern": r"\b(?:it'?s not|it\s+is\s+not|this is not|that'?s not|isn'?t|is\s+not|wasn'?t|was\s+not|aren'?t|are\s+not|weren'?t|were\s+not)\s+just\b[^.;!?\n]{1,60}[,;—–-]\s*(?:it'?s|it (?:is|was)|they'?re|that'?s)\b", "category": "negative_parallelism", "severity": "hard", "suggestion": "State both points directly" @@ -910,12 +934,23 @@ def _phrase_pattern(phrase: str) -> re.Pattern[str]: "suggestion": "Vary sentence length. Stacked short sentences are an AI rhythm tell." }, - # Paragraph starting with "So," + # Paragraph starting with "So," (comma required: bare "So the tenant remains + # liable" is ordinary prose). Match lowercase: scan_for_violations lowercases + # the text before applying structural patterns. { - "pattern": r"(?:^|\n)So,?\s", + "pattern": r"(?:^|\n)so,\s", "category": "filler_opener", + "severity": "soft", + "suggestion": "Start with content, not 'So,'" + }, + + # False agency: inanimate subjects narrating. People tell stories literally; + # data does not. + { + "pattern": r"\b(?:data|numbers?|charts?|graphs?|metrics?|figures?|results?|dashboards?|spreadsheets?|trend\s?lines?|statistics)\s+tells?\s+a\s+story\b", + "category": "false_agency", "severity": "hard", - "suggestion": "Start with content, not 'So'" + "suggestion": "Say what the data shows." }, # Colon-before-dramatic-reveal @@ -977,7 +1012,7 @@ def _phrase_pattern(phrase: str) -> re.Pattern[str]: }, { "pattern": r"\b(?:could|may|might|can)\s+(?:potentially|possibly)\b", - "category": "hedge", + "category": "hedge_stack", "severity": "soft", "suggestion": "Drop the redundant hedge." }, @@ -1123,6 +1158,15 @@ def main() -> None: violations = scan_for_violations(text, include_quoted=args.include_quoted) + # English-only graceful decline. Function-word absence alone is not evidence + # of a foreign language: imperative stacks and buzzword lists are English + # slop with few function words. Decline only when the text both fails the + # function-word heuristic AND produced zero English-pattern hits. + if not violations and not is_probably_english(text): + print(json.dumps({"non_english": True, "total_violations": 0, "violations": []}, indent=2)) + print("note: input appears non-English; scanner declined (English-only).", file=sys.stderr) + sys.exit(0) + # Group by category for summary categories: dict[str, int] = {} by_severity: dict[str, int] = {"hard": 0, "soft": 0} diff --git a/scripts/calibrate_pairs.py b/scripts/calibrate_pairs.py new file mode 100644 index 0000000..f47ce8e --- /dev/null +++ b/scripts/calibrate_pairs.py @@ -0,0 +1,580 @@ +#!/usr/bin/env python3 +""" +Deterministic dimension-controlled pair generation for the teach calibration game. + +Given a base passage and ONE voice dimension, produce a minimal pair (A, B) where +B differs from A along exactly that dimension. The direction of the transform +(e.g. contract -> expand vs expand -> contract) is auto-detected from whichever +pole the base passage already sits at, so every dimension is reversible: feed it +a passage already on one pole and it moves to the other. + +Every transform must preserve the must-preserve constraint tokens extracted by +scripts/extract_constraints.py (numbers, dates, names, quotes, units, etc). If a +dimension has no expressible transform in the given passage (neither pole's +pattern is present, or the only candidate transform would drop a constraint), +the command exits 3 with "dimension not expressible in this passage". + +Usage: + python3 calibrate_pairs.py generate --base FILE --dimension DIM --seed N + python3 calibrate_pairs.py --list-dimensions + +Dimensions: contractions, em_dash, sentence_length, connectives, staccato. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import random +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from extract_constraints import extract_constraints # noqa: E402 +from banned_phrase_scan import scan_for_violations # noqa: E402 + +DIMENSIONS = ["contractions", "em_dash", "sentence_length", "connectives", "staccato"] + +# Each dimension has two named poles. transform_applied is ":". +POLES: dict[str, tuple[str, str]] = { + "contractions": ("contracted", "expanded"), + "em_dash": ("dashed", "plain"), + "sentence_length": ("long", "short"), + "connectives": ("plain", "formal"), + "staccato": ("staccato", "flowing"), +} + + +class NotExpressible(Exception): + """Raised when a dimension has no applicable transform in the passage.""" + + +# -------------------------------------------------------------------------- +# contractions: expand <-> contract via a fixed, unambiguous mapping table. +# -------------------------------------------------------------------------- + +_CONTRACTION_PAIRS = [ + ("do not", "don't"), ("does not", "doesn't"), ("did not", "didn't"), + ("cannot", "can't"), ("will not", "won't"), ("would not", "wouldn't"), + ("should not", "shouldn't"), ("could not", "couldn't"), ("must not", "mustn't"), + ("is not", "isn't"), ("are not", "aren't"), ("was not", "wasn't"), ("were not", "weren't"), + ("have not", "haven't"), ("has not", "hasn't"), ("had not", "hadn't"), + ("I am", "I'm"), ("you are", "you're"), ("we are", "we're"), ("they are", "they're"), + ("I will", "I'll"), ("you will", "you'll"), ("we will", "we'll"), ("they will", "they'll"), + ("he will", "he'll"), ("she will", "she'll"), ("it will", "it'll"), + ("I have", "I've"), ("you have", "you've"), ("we have", "we've"), ("they have", "they've"), + ("let us", "let's"), + ("it is", "it's"), ("that is", "that's"), ("there is", "there's"), + ("here is", "here's"), ("who is", "who's"), ("what is", "what's"), +] +# Pairs whose expanded side always keeps a hard-capitalized pronoun ("I ..."). +_HARD_CAPITAL = {e for e, c in _CONTRACTION_PAIRS if e.startswith("I ")} + + +def _case_like(template: str, source_first_char: str) -> str: + if source_first_char.isupper(): + return template[0].upper() + template[1:] + return template[0].lower() + template[1:] + + +# Words safe to lowercase when a sentence is folded mid-clause. A capitalized +# word NOT in this set is assumed to be a proper noun (constraint) and is left +# alone -- lowercasing "Priya" into a joined clause would corrupt a name. +_LOWERABLE_JOINERS = { + "the", "it", "she", "he", "they", "this", "that", "there", "i", "we", "you", + "who", "what", "when", "where", "why", "how", "a", "an", "and", "but", "so", + "yet", "or", "nobody", "everybody", "everyone", "someone", "something", + "nothing", "no", "her", "his", "its", "their", "our", "your", +} + + +def _lower_first_word(s: str) -> str: + """Lowercase the leading word only if it is a common function word/pronoun, + never a token that looks like a proper noun a constraint might depend on.""" + m = re.match(r"[A-Za-z']+", s) + if not m: + return s + word = m.group(0) + if word.lower() in _LOWERABLE_JOINERS: + return word.lower() + s[len(word):] + return s + + +_WORD_BEFORE_RE = re.compile(r"([A-Za-z']+)\s*$") +_WORD_AFTER_RE = re.compile(r"^\s*([A-Za-z']+)") + + +def _is_title_case(word: str) -> bool: + return bool(word) and word[0].isupper() and word[1:].islower() + + +def _in_capitalized_span(m: "re.Match[str]") -> bool: + """True if this contraction/expansion match sits inside a capitalized + multi-word span (e.g. "Venue Can't Stop") that looks like a proper noun, + rather than an ordinary sentence-level contraction. + + A contraction rewrite is only skipped when the match ITSELF is + capitalized (so an ordinary sentence-initial "Can't you..." is untouched) + AND an immediately adjacent word is also Title Case, forming a >= 2-word + capitalized run. "Maria Chen can't attend" is not skipped (the match is + lowercase); "the Venue Can't Stop" is skipped (both "Venue" and "Can't" + are capitalized and adjacent). + """ + matched = m.group(0) + if not matched[0].isupper(): + return False + text = m.string + before_m = _WORD_BEFORE_RE.search(text[: m.start()]) + after_m = _WORD_AFTER_RE.match(text[m.end() :]) + before_word = before_m.group(1) if before_m else "" + after_word = after_m.group(1) if after_m else "" + return _is_title_case(before_word) or _is_title_case(after_word) + + +def _contraction_repl(replacement: str, hard: bool): + """One replacement closure shared by both contraction directions (expand + and contract) and both casing modes (hard-capital literal vs case-matched). + """ + def repl(m): + if _in_capitalized_span(m): + return m.group(0) + return replacement if hard else _case_like(replacement, m.group(0)[0]) + return repl + + +def _apply_contractions(text: str) -> tuple[str, str]: + contract_hits = [] + expand_hits = [] + for expanded, contracted in _CONTRACTION_PAIRS: + if expanded in _HARD_CAPITAL: + # "I am"/"I'll"/... always literally capitalized; case-sensitive match. + if re.search(re.escape(contracted), text): + contract_hits.append((expanded, contracted)) + if re.search(re.escape(expanded), text): + expand_hits.append((expanded, contracted)) + else: + if re.search(r"\b" + re.escape(contracted) + r"\b", text, re.IGNORECASE): + contract_hits.append((expanded, contracted)) + if re.search(r"\b" + re.escape(expanded) + r"\b", text, re.IGNORECASE): + expand_hits.append((expanded, contracted)) + + if contract_hits: + out = text + for expanded, contracted in contract_hits: + hard = expanded in _HARD_CAPITAL + pattern = re.escape(contracted) if hard else r"\b" + re.escape(contracted) + r"\b" + flags = 0 if hard else re.IGNORECASE + out = re.sub(pattern, _contraction_repl(expanded, hard), out, flags=flags) + return out, "expanded" + + if expand_hits: + out = text + for expanded, contracted in expand_hits: + hard = expanded in _HARD_CAPITAL + pattern = re.escape(expanded) if hard else r"\b" + re.escape(expanded) + r"\b" + flags = 0 if hard else re.IGNORECASE + out = re.sub(pattern, _contraction_repl(contracted, hard), out, flags=flags) + return out, "contracted" + + raise NotExpressible("no contraction or expandable phrase found") + + +# -------------------------------------------------------------------------- +# em_dash: paired em-dash parentheticals <-> paired-comma parentheticals ONLY. +# +# Only the paired-dash<->paired-comma path is expressible for this dimension. +# A lone joiner dash (" — nobody objected") has no comma-pair equivalent that +# preserves sentence count: turning it into a period (the old behavior) split +# one sentence into two, silently changing sentence count between A and B. +# Passages with only a lone dash (no paired construction) are declined rather +# than forced through a transform that shifts sentence count. +# -------------------------------------------------------------------------- + +_PAIRED_DASH_RE = re.compile(r"\s—\s(.+?)\s—\s") +_PAIRED_COMMA_RE = re.compile(r",\s+(.+?),\s+") + + +def _apply_em_dash(text: str) -> tuple[str, str]: + if _PAIRED_DASH_RE.search(text): + out = _PAIRED_DASH_RE.sub(lambda m: ", " + m.group(1) + ", ", text) + if "—" in out: + # A lone dash survived alongside the paired one -- the restricted + # paired-dash<->comma path can't fully express this passage + # without also touching the lone dash (and changing sentence + # count), so decline rather than ship a half-converted pair. + raise NotExpressible( + "passage mixes a paired em dash with a lone em dash; " + "the paired-dash<->comma path can't resolve the lone dash " + "without changing sentence count" + ) + return out, "plain" + + if "—" in text: + raise NotExpressible( + "only a lone em dash found; the em_dash dimension is restricted " + "to the paired-dash<->comma path so sentence count stays stable" + ) + + if _PAIRED_COMMA_RE.search(text): + out = _PAIRED_COMMA_RE.sub(lambda m: " — " + m.group(1) + " — ", text) + return out, "dashed" + + raise NotExpressible("no paired em dash or comma-bounded parenthetical found") + + +# -------------------------------------------------------------------------- +# sentence_length: split at coordinators <-> join short adjacent sentences. +# -------------------------------------------------------------------------- + +_COORD_RE = re.compile(r",\s+(and|but|or|so|yet)\s+", re.IGNORECASE) + + +def _apply_sentence_length(text: str) -> tuple[str, str]: + sentences = re.findall(r"[^.!?]+[.!?]+", text) + + for sent in sentences: + for m in _COORD_RE.finditer(sent): + before = sent[:m.start()] + after = sent[m.end():] + if len(before.split()) >= 3 and len(after.split()) >= 3: + new_sent = before.rstrip() + ". " + m.group(1).capitalize() + " " + after + out = text.replace(sent, new_sent, 1) + return re.sub(r"\s+", " ", out).strip(), "short" + + for i in range(len(sentences) - 1): + a_words = sentences[i].strip().split() + b_words = sentences[i + 1].strip().split() + if len(a_words) <= 8 and len(b_words) <= 8: + first_no_period = re.sub(r"[.!?]+\s*$", "", sentences[i].strip()) + second = sentences[i + 1].strip() + second_body = re.sub(r"[.!?]+\s*$", "", second) + end_punct = second[len(second_body):].strip() or "." + joined = first_no_period + ", and " + _lower_first_word(second_body) + end_punct + prefix = "".join(sentences[:i]) + suffix = "".join(sentences[i + 2:]) + out = (prefix + " " + joined + " " + suffix).strip() + return re.sub(r"\s+", " ", out), "long" + + raise NotExpressible("no coordinator split site or joinable short sentences found") + + +# -------------------------------------------------------------------------- +# connectives: formal <-> plain via a fixed table. +# -------------------------------------------------------------------------- + +_FORMAL_TO_PLAIN = { + "however": "But", "additionally": "Also", "therefore": "So", + "furthermore": "Also", "moreover": "Also", "nevertheless": "Still", + "consequently": "So", "nonetheless": "Still", "subsequently": "Then", + "thus": "So", +} +_PLAIN_TO_FORMAL = { + "but": "However", "also": "Additionally", "so": "Therefore", + "still": "Nevertheless", "then": "Subsequently", +} +_FORMAL_RE = re.compile( + r"(?:^|(?<=[.!?]\s))(" + "|".join(_FORMAL_TO_PLAIN) + r"),?\s+", re.IGNORECASE +) +_PLAIN_RE = re.compile( + r"(?:^|(?<=[.!?]\s))(" + "|".join(_PLAIN_TO_FORMAL) + r"),?\s+", re.IGNORECASE +) + + +def _apply_connectives(text: str) -> tuple[str, str]: + if _FORMAL_RE.search(text): + def repl(m): + # No comma after the plain form: "But " reads as ordinary speech; + # "But, " is stilted, and "So, " specifically trips the scanner's + # filler_opener structural pattern. Dropping the comma across the + # board keeps every plain form clean of that tell. + return _FORMAL_TO_PLAIN[m.group(1).lower()] + " " + out = _FORMAL_RE.sub(repl, text) + return out, "plain" + + if _PLAIN_RE.search(text): + def repl2(m): + return _PLAIN_TO_FORMAL[m.group(1).lower()] + ", " + out = _PLAIN_RE.sub(repl2, text) + return out, "formal" + + raise NotExpressible("no formal or plain connective found") + + +# -------------------------------------------------------------------------- +# staccato: fragment runs <-> flowing clauses. +# -------------------------------------------------------------------------- + +_LEADING_JOINER_RE = re.compile( + r"^(?:because|although|while|since|if|when|and|but|so|yet|or)\s+", re.IGNORECASE +) + + +def _apply_staccato(text: str, rng: random.Random) -> tuple[str, str]: + sentences = re.findall(r"[^.!?]+[.!?]+", text) + + # Reverse direction: a run of >= 3 consecutive short (<=6 word) sentences -> flow. + runs = [] + run_start = None + for i, sent in enumerate(sentences): + words = sent.strip().split() + if len(words) <= 6: + if run_start is None: + run_start = i + else: + if run_start is not None and i - run_start >= 3: + runs.append((run_start, i)) + run_start = None + if run_start is not None and len(sentences) - run_start >= 3: + runs.append((run_start, len(sentences))) + + if runs: + start, end = runs[rng.randrange(len(runs))] if len(runs) > 1 else runs[0] + parts = [] + for idx in range(start, end): + body = re.sub(r"[.!?]+\s*$", "", sentences[idx].strip()) + parts.append(body) + joined_parts = [] + for i, p in enumerate(parts): + if i == 0: + joined_parts.append(p) + elif i == len(parts) - 1: + joined_parts.append("and " + _lower_first_word(p)) + else: + joined_parts.append(_lower_first_word(p)) + joined = ", ".join(joined_parts) + "." + prefix = "".join(sentences[:start]) + suffix = "".join(sentences[end:]) + out = (prefix + " " + joined + " " + suffix).strip() + return re.sub(r"\s+", " ", out), "flowing" + + # Forward direction: the sentence with the most commas (>= 2) fragments. + best_idx, best_commas = None, 1 + for i, sent in enumerate(sentences): + commas = sent.count(",") + if commas > best_commas: + best_idx, best_commas = i, commas + + if best_idx is not None: + sent = sentences[best_idx] + body = re.sub(r"[.!?]+\s*$", "", sent.strip()) + end_punct = sent.strip()[len(body):].strip() or "." + fragments = [f.strip() for f in body.split(",")] + cleaned = [] + for frag in fragments: + frag = _LEADING_JOINER_RE.sub("", frag).strip() + if not frag: + continue + frag = frag[0].upper() + frag[1:] + cleaned.append(frag) + if len(cleaned) < 3: + raise NotExpressible("comma split did not yield a fragment run") + cleaned[-1] = cleaned[-1] + end_punct if not cleaned[-1].endswith((".", "!", "?")) else cleaned[-1] + new_sent = " ".join( + f if f.endswith((".", "!", "?")) else f + "." for f in cleaned + ) + out = text.replace(sent, new_sent, 1) + return re.sub(r"\s+", " ", out).strip(), "staccato" + + raise NotExpressible("no fragment run or multi-comma sentence found") + + +_APPLY = { + "contractions": lambda text, rng: _apply_contractions(text), + "em_dash": lambda text, rng: _apply_em_dash(text), + "sentence_length": lambda text, rng: _apply_sentence_length(text), + "connectives": lambda text, rng: _apply_connectives(text), + "staccato": _apply_staccato, +} + +_EXAMPLES = { + "contractions": { + "description": "Expand <-> contract via a fixed, unambiguous mapping table " + "(do not <-> don't, I am <-> I'm, it is <-> it's, ...).", + "a": "The rollout is not finished, and I am not confident it will ship Friday.", + "b": "The rollout isn't finished, and I'm not confident it'll ship Friday.", + }, + "em_dash": { + "description": "Paired em-dash parentheticals <-> paired-comma parentheticals only " + "(a lone joiner dash has no comma-pair equivalent that preserves " + "sentence count, so it is declined rather than converted).", + "a": "The plan — untested and rushed — still shipped on time.", + "b": "The plan, untested and rushed, still shipped on time.", + }, + "sentence_length": { + "description": "Split at a coordinator <-> join two short adjacent sentences.", + "a": "The team shipped the fix, and the client renewed the contract.", + "b": "The team shipped the fix. And the client renewed the contract.", + }, + "connectives": { + "description": "Formal <-> plain connective swap via a fixed table " + "(However -> But, Additionally -> Also, Therefore -> So, ...). " + "Plain forms drop the comma after the connective (\"But \", not " + "\"But, \") so they read as ordinary speech, not a filler-opener tell.", + "a": "However, the numbers slipped in March.", + "b": "But the numbers slipped in March.", + }, + "staccato": { + "description": "Fragment runs <-> flowing clauses.", + "a": "Because the deploy failed, and the on-call missed the page, the team lost an hour.", + "b": "The deploy failed. The on-call missed the page. The team lost an hour.", + }, +} + + +def _is_word_char(ch: str) -> bool: + return ch.isalnum() + + +def _has_whole_occurrence(value: str, text: str) -> bool: + """True if `value` occurs in `text` as a standalone span rather than as a + substring straddling the edge of a longer, different word. + + Plain substring containment is not enough: expanding "Can't" to "Cannot" + still contains the literal characters "Can", so a naive `"Can" in + "Cannot"` check reports a corrupted proper noun as "preserved". At each + edge of a candidate match, this only counts as a clash (disqualifying the + match) when BOTH the value's own boundary character and the adjacent text + character are alphanumeric -- i.e. when `value` could be glued onto a + longer token. Values that start/end with punctuation (currency signs, + quotes) never clash on that edge, since punctuation can't be silently + swallowed into a bigger word the way a letter/digit can. + """ + if not value: + return False + start = 0 + while True: + idx = text.find(value, start) + if idx == -1: + return False + left_clash = idx > 0 and _is_word_char(text[idx - 1]) and _is_word_char(value[0]) + end = idx + len(value) + right_clash = end < len(text) and _is_word_char(text[end]) and _is_word_char(value[-1]) + if not left_clash and not right_clash: + return True + start = idx + 1 + + +def _verify_constraints_preserved(base_text: str, transformed_text: str) -> list[str]: + """Return constraint values from base_text missing from transformed_text. + + Uses whole-occurrence (token-boundary) comparison rather than plain + substring containment -- see `_has_whole_occurrence` for why a substring + check is unsafe here. + """ + missing = [] + for c in extract_constraints(base_text): + value = c["value"] + if _has_whole_occurrence(value, transformed_text): + continue + # Allow whitespace-normalized matches (transforms may collapse spacing). + normalized_value = re.sub(r"\s+", " ", value) + normalized_text = re.sub(r"\s+", " ", transformed_text) + if normalized_value != value and _has_whole_occurrence(normalized_value, normalized_text): + continue + missing.append(value) + return missing + + +def _scan_flags(text: str) -> list[str]: + """Category names banned_phrase_scan raises on `text`, deduped and sorted. + + Empty when the text scans clean. Generated B-variants can legitimately + trip the scanner (e.g. a staccato pole reads as anti_slop_register; see + references/calibrate.md "Voice overrides defaults") -- that is surfaced + here, not treated as a reason to decline the pair. + """ + categories = {v["category"] for v in scan_for_violations(text)} + return sorted(categories) + + +def generate_pair(base_text: str, dimension: str, seed: int) -> dict: + if dimension not in DIMENSIONS: + raise ValueError(f"unknown dimension: {dimension}") + + rng = random.Random(seed) + transformed, pole = _APPLY[dimension](base_text, rng) + + if transformed.strip() == base_text.strip(): + raise NotExpressible("transform produced no change") + + missing = _verify_constraints_preserved(base_text, transformed) + if missing: + raise NotExpressible( + "transform would drop must-preserve constraint(s): " + ", ".join(missing) + ) + + pair_id = hashlib.sha256( + f"{base_text}|{dimension}|{seed}".encode("utf-8") + ).hexdigest() + + return { + "pair_id": pair_id, + "dimension": dimension, + "a_text": base_text, + "b_text": transformed, + "transform_applied": f"{dimension}:{pole}", + "a_flags": _scan_flags(base_text), + "b_flags": _scan_flags(transformed), + } + + +def list_dimensions() -> dict: + return { + dim: { + "poles": list(POLES[dim]), + "description": _EXAMPLES[dim]["description"], + "example": {"a": _EXAMPLES[dim]["a"], "b": _EXAMPLES[dim]["b"]}, + } + for dim in DIMENSIONS + } + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command") + + gen = sub.add_parser("generate", help="generate one dimension-controlled pair") + gen.add_argument("--base", required=True, help="path to base passage file") + gen.add_argument("--dimension", required=True, choices=DIMENSIONS) + gen.add_argument("--seed", type=int, default=0) + + parser.add_argument("--list-dimensions", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + + if args.list_dimensions: + print(json.dumps(list_dimensions(), indent=2, sort_keys=True)) + return 0 + + if args.command != "generate": + print(json.dumps({"error": "no command given; use generate or --list-dimensions"})) + return 1 + + try: + base_text = Path(args.base).read_text() + except OSError as e: + print(json.dumps({"error": f"could not read base file: {e}"})) + return 2 + + try: + pair = generate_pair(base_text, args.dimension, args.seed) + except NotExpressible as e: + print(json.dumps({ + "error": "dimension not expressible in this passage", + "dimension": args.dimension, + "detail": str(e), + })) + return 3 + + print(json.dumps(pair, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/calibrate_score.py b/scripts/calibrate_score.py new file mode 100644 index 0000000..3583a03 --- /dev/null +++ b/scripts/calibrate_score.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +""" +Aggregate a teach-calibration preferences JSONL into per-dimension confidence, +surface conflicts against a measured stylometric profile, and pick the next +dimension to play. + +Preferences JSONL rows (one per game round): + {"pair_id": "...", "dimension": "contractions", "choice": "a"|"b"|"neither", + "ts": "...", "a_label": "contracted", "b_label": "expanded"} + +`a_label`/`b_label` are optional. When present (calibrate_pairs.py's +transform_applied names the B pole; A sits at the other pole in the same +dimension's POLES pair) they let this script report a semantic preferred +DIRECTION ("expanded", "short", "formal", ...) instead of the bare literal +"a"/"b" tally, which is what --profile conflict detection needs. Rows without +labels still count toward n and the confidence interval; they just can't +resolve to a named pole (direction is reported as "a" or "b"). + +Usage: + python3 calibrate_score.py --preferences prefs.jsonl + python3 calibrate_score.py --preferences prefs.jsonl --profile profile.json + python3 calibrate_score.py --preferences prefs.jsonl --next +""" +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from calibrate_pairs import DIMENSIONS, POLES # noqa: E402 + +MIN_K = 5 +Z = 1.96 + +# Dimension -> (profile_key, pole_meaning) used only for --profile conflict +# detection. `low_pole`/`high_pole` say which named pole a LOW vs HIGH measured +# value corresponds to, so a confident stated preference for the opposite pole +# of what the measured value implies is flagged. +_PROFILE_LINKS: dict[str, dict] = { + "contractions": { + "profile_key": "contraction_rate", + "low_pole": "expanded", + "high_pole": "contracted", + "low_threshold": 0.05, + "high_threshold": 0.20, + }, + "sentence_length": { + "profile_key": "avg_sentence_length", + "low_pole": "short", + "high_pole": "long", + "low_threshold": 12.0, + "high_threshold": 20.0, + }, + "staccato": { + "profile_key": "avg_sentence_length", + "low_pole": "staccato", + "high_pole": "flowing", + "low_threshold": 12.0, + "high_threshold": 20.0, + }, + "em_dash": { + "profile_key": "em_dash_rate", + "low_pole": "plain", + "high_pole": "dashed", + "low_threshold": 0.02, + "high_threshold": 0.15, + }, + "connectives": { + "profile_key": "formal_connective_rate", + "low_pole": "plain", + "high_pole": "formal", + "low_threshold": 0.10, + "high_threshold": 0.40, + }, +} + +CONFIDENCE_STOP = 0.7 +K_STOP = 9 + + +def wilson_lower_bound(successes: int, n: int, z: float = Z) -> float: + if n == 0: + return 0.0 + phat = successes / n + denom = 1 + z * z / n + center = phat + z * z / (2 * n) + margin = z * math.sqrt((phat * (1 - phat) + z * z / (4 * n)) / n) + return round(max(0.0, (center - margin) / denom), 3) + + +def load_preferences(path: Path) -> list[dict]: + rows = [] + for line in path.read_text().splitlines(): + line = line.strip() + if line: + rows.append(json.loads(line)) + return rows + + +def dedup_by_pair_id(rows: list[dict]) -> list[dict]: + """Keep one row per `pair_id`: the LATEST by `ts` (ISO 8601, so a plain + string comparison orders correctly). A game round replayed twice (e.g. the + agent crashed mid-write and re-ran the round) must count once toward `n` + and resolve to whatever the user most recently chose, not double-count or + let an earlier write win. Rows without a `pair_id` can't be deduped + against anything, so each is kept as-is. + """ + by_pair_id: dict[str, dict] = {} + passthrough: list[dict] = [] + for row in rows: + pid = row.get("pair_id") + if not pid: + passthrough.append(row) + continue + existing = by_pair_id.get(pid) + if existing is None or row.get("ts", "") >= existing.get("ts", ""): + by_pair_id[pid] = row + return list(by_pair_id.values()) + passthrough + + +def aggregate(rows: list[dict]) -> dict[str, dict]: + rows = dedup_by_pair_id(rows) + result = {dim: {"n": 0, "tally": {}, "neither": 0} for dim in DIMENSIONS} + + for row in rows: + dim = row.get("dimension") + if dim not in result: + continue + result[dim]["n"] += 1 + choice = row.get("choice") + if choice == "neither": + result[dim]["neither"] += 1 + continue + if choice not in ("a", "b"): + continue + label = row.get(f"{choice}_label") or choice + result[dim]["tally"][label] = result[dim]["tally"].get(label, 0) + 1 + + dimensions = {} + for dim, data in result.items(): + n = data["n"] + decisive = sum(data["tally"].values()) + if n < MIN_K or decisive == 0: + dimensions[dim] = { + "n": n, + "status": "insufficient", + "preferred": None, + "confidence": 0.0, + } + continue + max_count = max(data["tally"].values()) + top_labels = [label for label, count in data["tally"].items() if count == max_count] + if len(top_labels) > 1: + # A genuine tie between the top two (or more) tallies: there is no + # lean to report, so `preferred` must be null rather than + # silently picking whichever label happens to sort last. + dimensions[dim] = { + "n": n, + "status": "tied", + "preferred": None, + "confidence": 0.0, + } + continue + preferred_label = top_labels[0] + confidence = wilson_lower_bound(max_count, decisive) + dimensions[dim] = { + "n": n, + "status": "confident", + "preferred": preferred_label, + "confidence": confidence, + } + return dimensions + + +def detect_conflicts(dimensions: dict[str, dict], profile: dict) -> list[dict]: + conflicts = [] + for dim, data in dimensions.items(): + if data["status"] != "confident" or data["confidence"] < CONFIDENCE_STOP: + continue + link = _PROFILE_LINKS.get(dim) + if not link or link["profile_key"] not in profile: + continue + measured = profile[link["profile_key"]] + preferred = data["preferred"] + conflict_pole = None + if preferred == link["low_pole"] and measured >= link["high_threshold"]: + conflict_pole = link["high_pole"] + elif preferred == link["high_pole"] and measured <= link["low_threshold"]: + conflict_pole = link["low_pole"] + if conflict_pole is None: + continue + conflicts.append({ + "dimension": dim, + "preferred": preferred, + "preferred_confidence": data["confidence"], + "preferred_provenance": "stated-preference", + "measured_key": link["profile_key"], + "measured_value": measured, + "measured_provenance": "measured-from-samples", + "message": ( + f"Stated preference for '{preferred}' ({dim}) contradicts " + f"{link['profile_key']}={measured} measured from samples, " + f"which points toward '{conflict_pole}'." + ), + }) + return conflicts + + +def next_dimension(dimensions: dict[str, dict]) -> dict: + def sort_key(dim: str) -> tuple: + data = dimensions[dim] + confidence = data["confidence"] if data["status"] == "confident" else 0.0 + return (data["n"], confidence, DIMENSIONS.index(dim)) + + ordered = sorted(DIMENSIONS, key=sort_key) + chosen = ordered[0] + data = dimensions[chosen] + if data["n"] == min(dimensions[d]["n"] for d in DIMENSIONS): + reason = "fewest_observations" + else: + reason = "lowest_confidence" + return { + "next_dimension": chosen, + "reason": reason, + "n": data["n"], + "confidence": data["confidence"], + } + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--preferences", required=True) + parser.add_argument("--profile") + parser.add_argument("--next", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + + try: + rows = load_preferences(Path(args.preferences)) + except OSError as e: + print(json.dumps({"error": f"could not read preferences file: {e}"})) + return 2 + + dimensions = aggregate(rows) + + if args.next: + print(json.dumps(next_dimension(dimensions), indent=2, sort_keys=True)) + return 0 + + output = {"dimensions": dimensions} + + if args.profile: + try: + profile = json.loads(Path(args.profile).read_text()) + except OSError as e: + print(json.dumps({"error": f"could not read profile file: {e}"})) + return 2 + output["conflicts"] = detect_conflicts(dimensions, profile) + + print(json.dumps(output, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check_suggestions.py b/scripts/check_suggestions.py new file mode 100644 index 0000000..3b457c3 --- /dev/null +++ b/scripts/check_suggestions.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Contract gates for co-writer suggestions. + +Reads a suggestions file — the JSON emitted by suggest.py, i.e. an object with a +``document`` string and a ``suggestions`` list — and enforces four blocking +contracts. Each failure is named so a caller can act on it: + + span-minimality Every suggested_replacement edits only its own span: + span.text matches document[start:end], the replacement + differs from the span text, and it shares no leading or + trailing whole word with the span (which would mean the + span grabbed unchanged text and could be shrunk — the + signature of a whole-sentence rewrite). + replacement-scanner Every suggested_replacement passes both scanners in + isolation AND introduces no new violation in context. + accept-all Applying every replacement yields a document that passes + both scanners with validate_preservation exit 0 vs the + original. + span-overlap Suggestion spans must not overlap. + +Exit 0 when all gates pass, 1 otherwise. Failures (with the offending gate) are +reported as JSON on stdout. + +Usage: + python3 scripts/check_suggestions.py suggestions.json + python3 scripts/check_suggestions.py < suggestions.json +""" + +import json +import sys +from pathlib import Path + +from banned_phrase_scan import scan_for_violations +from structure_scan import scan as structure_scan +from validate_preservation import validate_preservation + + +def _leading_shared_words(a: str, b: str) -> int: + """Number of identical leading whole words shared by two strings.""" + n = 0 + for x, y in zip(a.split(), b.split()): + if x == y: + n += 1 + else: + break + return n + + +def _trailing_shared_words(a: str, b: str) -> int: + ra = " ".join(reversed(a.split())) + rb = " ".join(reversed(b.split())) + return _leading_shared_words(ra, rb) + + +def _line_starts(text: str) -> list[int]: + starts = [0] + for i, ch in enumerate(text): + if ch == "\n": + starts.append(i + 1) + return starts + + +def violation_spans(text: str) -> list[tuple[int, int]]: + """Absolute (start, end) offsets of every banned-phrase/structural violation.""" + starts = _line_starts(text) + out = [] + for v in scan_for_violations(text): + s = starts[v["line_number"] - 1] + v["column"] - 1 + out.append((s, s + len(v["phrase"]))) + return out + + +def _scanners_clean(text: str) -> bool: + return not scan_for_violations(text) and not structure_scan(text).get("flags") + + +def apply_all(document: str, suggestions: list[dict]) -> str: + """Apply every non-null replacement, right-to-left so offsets stay valid.""" + out = document + for s in sorted(suggestions, key=lambda s: s["span"]["start"], reverse=True): + rep = s.get("suggested_replacement") + if rep is None: + continue + st, en = s["span"]["start"], s["span"]["end"] + out = out[:st] + rep + out[en:] + return out + + +def check(document: str, suggestions: list[dict]) -> list[dict]: + failures: list[dict] = [] + + # span-overlap: spans must be disjoint. + order = sorted(range(len(suggestions)), + key=lambda i: (suggestions[i]["span"]["start"], suggestions[i]["span"]["end"])) + last_end = None + last_i = None + for i in order: + st, en = suggestions[i]["span"]["start"], suggestions[i]["span"]["end"] + if last_end is not None and st < last_end: + failures.append({ + "gate": "span-overlap", + "suggestions": [last_i, i], + "detail": f"span {st}-{en} overlaps the previous span ending at {last_end}", + }) + last_end, last_i = en, i + + # span-minimality: span accuracy + tight, local replacements. + for i, s in enumerate(suggestions): + st, en = s["span"]["start"], s["span"]["end"] + span_text = document[st:en] + if span_text != s["span"]["text"]: + failures.append({ + "gate": "span-minimality", + "suggestion": i, + "detail": "span.text does not match document[start:end]", + }) + rep = s.get("suggested_replacement") + if rep is None: + continue + if rep == span_text: + failures.append({ + "gate": "span-minimality", + "suggestion": i, + "detail": "replacement is identical to the span text (no change)", + }) + continue + if _leading_shared_words(span_text, rep) > 0 or _trailing_shared_words(span_text, rep) > 0: + failures.append({ + "gate": "span-minimality", + "suggestion": i, + "detail": "replacement shares leading/trailing whole words with the span; " + "shrink the span so the edit is minimal", + }) + + # replacement-scanner: each replacement is clean alone and in context. + for i, s in enumerate(suggestions): + rep = s.get("suggested_replacement") + if rep is None: + continue + if not _scanners_clean(rep): + failures.append({ + "gate": "replacement-scanner", + "suggestion": i, + "detail": "replacement does not pass both scanners in isolation", + }) + st, en = s["span"]["start"], s["span"]["end"] + ctx = document[:st] + rep + document[en:] + new_start, new_end = st, st + len(rep) + if any(a < new_end and new_start < b for (a, b) in violation_spans(ctx)): + failures.append({ + "gate": "replacement-scanner", + "suggestion": i, + "detail": "replacement introduces a violation in context", + }) + + # accept-all: applying everything yields a clean, constraint-preserving doc. + unresolved = [i for i, s in enumerate(suggestions) if s.get("suggested_replacement") is None] + if unresolved: + failures.append({ + "gate": "accept-all", + "detail": f"suggestions {unresolved} have no replacement; cannot accept-all", + }) + applied = apply_all(document, suggestions) + if not _scanners_clean(applied): + failures.append({ + "gate": "accept-all", + "detail": "the accept-all document still fails a scanner", + }) + preservation = validate_preservation(document, applied) + if not preservation["passed"]: + failures.append({ + "gate": "accept-all", + "detail": "validate_preservation failed against the original", + "missing": preservation["missing"], + }) + + return failures + + +def main(argv: list[str]) -> int: + if argv: + path = Path(argv[0]) + if not path.exists(): + print(f"Missing file: {path}", file=sys.stderr) + return 2 + raw = path.read_text() + else: + raw = sys.stdin.read() + + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + print(json.dumps({"passed": False, "error": f"invalid JSON: {e}"}, indent=2)) + return 1 + + document = data.get("document") + suggestions = data.get("suggestions", []) + if not isinstance(document, str): + print(json.dumps({"passed": False, "error": "missing 'document' string"}, indent=2)) + return 1 + + failures = check(document, suggestions) + result = { + "passed": not failures, + "failure_gates": sorted({f["gate"] for f in failures}), + "failures": failures, + } + print(json.dumps(result, indent=2)) + return 0 if not failures else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/contribute.py b/scripts/contribute.py new file mode 100644 index 0000000..ffd7fae --- /dev/null +++ b/scripts/contribute.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""Offline contribution scaffolder for new unslop AI-isms.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +from datetime import date, datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CONTRIB_ROOT = ROOT / ".unslop" / "contrib" +TODO_MARKER = "TODO:" + +GATE_COMMANDS = [ + ["python3", "evals/run_adversarial.py", "--only", "CONTRIB"], + ["python3", "evals/run_adversarial.py"], + ["python3", "evals/build_shared_benchmark.py", "--check"], + ["python3", "evals/check_taboo_parity.py"], + ["python3", "evals/check_pattern_coverage.py"], + ["python3", "evals/kata_add_pattern.py", "--run"], +] + + +def read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except OSError as exc: + print(json.dumps({"error": f"could not read {path}: {exc}"})) + raise SystemExit(2) from exc + + +def write_json(path: Path, data: object) -> None: + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def word_count(text: str) -> int: + return len(re.findall(r"[A-Za-z0-9']+", text)) + + +def shorten(text: str, limit: int = 72) -> str: + one_line = " ".join(text.split()) + if len(one_line) <= limit: + return one_line + return one_line[: limit - 1].rstrip() + "..." + + +def run_scanner(command: list[str], text: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, input=text, capture_output=True, text=True, cwd=ROOT, timeout=30) + + +def scanner_findings(text: str) -> list[dict[str, object]]: + findings: list[dict[str, object]] = [] + for name, command in ( + ("banned_phrase", ["python3", "scripts/banned_phrase_scan.py"]), + ("structure", ["python3", "scripts/structure_scan.py"]), + ): + proc = run_scanner(command, text) + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + data = {} + violations = data.get("violations", []) + if name == "structure": + violations = [ + { + "phrase": flag.get("metric", ""), + "category": flag.get("metric", ""), + "severity": flag.get("severity", ""), + } + for flag in data.get("flags", []) + ] + for violation in violations: + findings.append( + { + "scanner": name, + "phrase": violation.get("phrase", ""), + "category": violation.get("category", ""), + "severity": violation.get("severity", ""), + } + ) + return findings + + +def redaction_pairs(values: list[str]) -> list[tuple[str, str]]: + pairs = [] + for value in values: + if "=" not in value: + print(json.dumps({"error": "--redact values must use orig=REPL"})) + raise SystemExit(2) + orig, repl = value.split("=", 1) + pairs.append((orig, repl)) + return pairs + + +def apply_redactions(text: str, tell: str, pairs: list[tuple[str, str]]) -> str: + before_count = text.count(tell) + redacted = text + for orig, repl in pairs: + redacted = redacted.replace(orig, repl) + if before_count == 0 or redacted.count(tell) != before_count: + print(json.dumps({"error": "redaction alters the specimen", "tell": tell})) + raise SystemExit(4) + return redacted + + +def row_fn(slug: str, tell: str, snippet: str) -> dict[str, object]: + assertion_tell = tell.casefold() + return { + "id": f"CONTRIB-FN-{slug}", + "category": "scanner_false_negative", + "title": f"New AI-ism fixture: {tell}", + "target": "script", + "command": ["python3", "scripts/banned_phrase_scan.py"], + "stdin": snippet, + "failure_mode": "The scanner misses the contributed tell from the exact specimen.", + "correct_behavior": "The scanner reports at least one violation containing the contributed tell.", + "assertions": [ + {"type": "json", "path": "total_violations", "gte": 1}, + {"type": "violation_phrase_contains", "value": assertion_tell}, + ], + } + + +def row_fp_template(slug: str, category: str, tell: str) -> dict[str, object]: + return { + "id": f"CONTRIB-FP-{slug}", + "category": "scanner_false_positive", + "protects": category, + "title": f"Literal-use protection for {tell}", + "target": "script", + "command": ["python3", "scripts/banned_phrase_scan.py"], + "stdin": "TODO: add a literal or domain-specific use that should remain clean.", + "failure_mode": "A broad pattern could flag legitimate prose.", + "correct_behavior": "No violations for the protected literal/domain use.", + "assertions": [{"type": "json", "path": "total_violations", "equals": 0}], + } + + +def render_report( + manifest: dict[str, object], + snippet: str, + gate_results: str = "", + include_rec: bool = False, +) -> str: + redactions = manifest.get("redactions", []) + redaction_note = "names/numbers redacted; tell verbatim" if redactions else "none" + rows = [ + f"| CONTRIB-FN-{manifest['pattern_name']} | FN | exact specimen flags `{manifest['tell']}` |", + f"| CONTRIB-FP-{manifest['pattern_name']} | FP | literal-use protection for `{manifest['tell']}` |", + ] + if include_rec: + rows.append("| CONTRIB-REC | REC | existing-word recall still flags |") + quoted = "\n".join(f"> {line}" if line else ">" for line in snippet.splitlines()) + return ( + f"# Add {manifest['category']} pattern: {shorten(str(manifest['tell']))}\n\n" + "## The specimen\n\n" + f"{quoted}\n\n" + f"- Source genre: {manifest.get('source_genre', 'TODO: source genre')}\n" + f"- Date: {manifest['date']}\n" + f"- Redaction note: {redaction_note}\n\n" + "## Why it's an AI-ism\n\n" + f"{manifest.get('rationale', 'TODO: explain why this phrase is a reusable AI-writing tell in 2-4 sentences.')}\n\n" + "## Detection\n\n" + f"- Pattern added: {manifest.get('pattern_added', f'TODO: regex or phrase for `{manifest['tell']}`')}\n" + f"- Severity: {manifest.get('severity', 'TODO: hard or soft')}\n" + f"- Gating rationale: {manifest.get('gating_rationale', 'TODO: explain literal-use boundary')}\n" + "- Catalog entry location: references/taboo-phrases.md\n\n" + "## Evals\n\n" + "| row id | kind | what it pins |\n" + "|---|---|---|\n" + + "\n".join(rows) + + "\n\n" + "The FN stdin is the unmodified specimen after approved redaction.\n\n" + "## Gate results\n\n" + f"{gate_results or 'TODO: paste gate tails from verify.'}\n\n" + "## Checklist\n\n" + "- [ ] eval-first (row was red before the pattern)\n" + "- [ ] literal-use FP row included\n" + "- [ ] REC row if an existing word was gated\n" + "- [ ] catalog + scanner parity green\n" + "- [ ] coverage gate green (pattern exercised)\n" + "- [ ] snippet publication approved by the user\n" + ) + + +def cmd_precheck(args: argparse.Namespace) -> int: + snippet = read_text(Path(args.snippet_file)) + findings = scanner_findings(snippet) + if findings: + print(json.dumps({"status": "already_covered", "findings": findings}, indent=2, sort_keys=True)) + return 3 + print(json.dumps({"status": "clean", "words": word_count(snippet)}, indent=2, sort_keys=True)) + return 0 + + +def cmd_scaffold(args: argparse.Namespace) -> int: + source = Path(args.snippet) + snippet = read_text(source) + if args.tell not in snippet: + print(json.dumps({"error": "tell substring not found", "tell": args.tell})) + return 2 + try: + manifest_date = datetime.strptime(args.date, "%Y-%m-%d").date().isoformat() + except ValueError: + print(json.dumps({"error": "--date must use YYYY-MM-DD", "date": args.date})) + return 2 + redactions = redaction_pairs(args.redact or []) + redacted = apply_redactions(snippet, args.tell, redactions) + bundle = CONTRIB_ROOT / args.pattern_name + bundle.mkdir(parents=True, exist_ok=True) + manifest = { + "category": args.category, + "date": manifest_date, + "pattern_name": args.pattern_name, + "pre_redaction_sha256": sha256_text(snippet), + "post_redaction_sha256": sha256_text(redacted), + "redactions": [{"from": orig, "to": repl} for orig, repl in redactions], + "tell": args.tell, + "word_count": word_count(redacted), + } + write_json(bundle / "row_fn.json", row_fn(args.pattern_name, args.tell, redacted)) + write_json(bundle / "row_fp_TEMPLATE.json", row_fp_template(args.pattern_name, args.category, args.tell)) + write_json(bundle / "manifest.json", manifest) + (bundle / "snippet.txt").write_text(redacted, encoding="utf-8") + (bundle / "report.md").write_text(render_report(manifest, redacted), encoding="utf-8") + print(json.dumps({"bundle": str(bundle.relative_to(ROOT)), "status": "scaffolded"}, indent=2, sort_keys=True)) + return 0 + + +def run_row(row: dict[str, object]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + row["command"], + input=str(row.get("stdin", "")), + capture_output=True, + text=True, + cwd=ROOT, + timeout=30, + ) + + +def row_assertions_pass(row: dict[str, object], proc: subprocess.CompletedProcess[str]) -> bool: + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + return False + for assertion in row.get("assertions", []): + if assertion.get("type") == "json": + cur = data + for part in assertion["path"].split("."): + cur = cur[int(part)] if isinstance(cur, list) else cur[part] + if "gte" in assertion and not cur >= assertion["gte"]: + return False + if "equals" in assertion and cur != assertion["equals"]: + return False + elif assertion.get("type") == "violation_phrase_contains": + phrases = [v.get("phrase", "") for v in data.get("violations", [])] + needle = str(assertion["value"]).casefold() + if not any(needle in str(phrase).casefold() for phrase in phrases): + return False + else: + return False + return True + + +def tail(text: str, lines: int = 12) -> str: + split = text.splitlines() + return "\n".join(split[-lines:]) + + +def verify_bundle(bundle: Path, run_gates: bool = True) -> tuple[bool, str]: + if not bundle.exists(): + return False, "missing bundle" + row_path = bundle / "row_fn.json" + report_path = bundle / "report.md" + manifest_path = bundle / "manifest.json" + snippet_path = bundle / "snippet.txt" + for path in (row_path, report_path, manifest_path, snippet_path): + if not path.exists(): + return False, f"missing {path.name}" + report = report_path.read_text(encoding="utf-8") + if TODO_MARKER in report: + return False, "report contains TODO markers" + row = json.loads(row_path.read_text(encoding="utf-8")) + snippet = snippet_path.read_text(encoding="utf-8") + if row.get("stdin") != snippet: + return False, "row_fn stdin does not equal snippet.txt byte-for-byte" + proc = run_row(row) + assertion_ok = row_assertions_pass(row, proc) + red_first = not assertion_ok + gate_results = [f"red-first: {'ok' if red_first else 'already green'}"] + ok = red_first + if not red_first: + ok = True + gate_results[0] = "red-first: already green; proposed pattern appears active" + if run_gates: + for command in GATE_COMMANDS: + proc = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, timeout=120) + gate_results.append( + "### " + + " ".join(command) + + f"\n\n```text\nexit={proc.returncode}\n{tail(proc.stdout or proc.stderr)}\n```" + ) + ok = ok and proc.returncode == 0 + final = "\n\n".join(gate_results) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + report_path.write_text(render_report(manifest, snippet, final, (bundle / "row_rec.json").exists()), encoding="utf-8") + return ok, final + + +def cmd_verify(args: argparse.Namespace) -> int: + ok, message = verify_bundle(Path(args.bundle), run_gates=not args.no_gates) + print(message) + return 0 if ok else 1 + + +def cmd_report(args: argparse.Namespace) -> int: + bundle = Path(args.bundle) + if not bundle.exists(): + print(json.dumps({"error": "missing bundle", "bundle": str(bundle)})) + return 2 + manifest = json.loads((bundle / "manifest.json").read_text(encoding="utf-8")) + snippet = (bundle / "snippet.txt").read_text(encoding="utf-8") + report = (bundle / "report.md").read_text(encoding="utf-8") + gate_marker = "## Gate results\n\n" + checklist_marker = "\n\n## Checklist" + gate_results = "" + if gate_marker in report and checklist_marker in report: + gate_results = report.split(gate_marker, 1)[1].split(checklist_marker, 1)[0] + print(render_report(manifest, snippet, gate_results, (bundle / "row_rec.json").exists()).rstrip()) + return 0 + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + precheck = sub.add_parser("precheck") + precheck.add_argument("snippet_file") + precheck.set_defaults(func=cmd_precheck) + scaffold = sub.add_parser("scaffold") + scaffold.add_argument("--snippet", required=True) + scaffold.add_argument("--tell", required=True) + scaffold.add_argument("--category", required=True) + scaffold.add_argument("--pattern-name", required=True) + scaffold.add_argument("--redact", action="append") + scaffold.add_argument("--date", default=date.today().isoformat()) + scaffold.set_defaults(func=cmd_scaffold) + verify = sub.add_parser("verify") + verify.add_argument("--bundle", required=True) + verify.add_argument("--no-gates", action="store_true") + verify.set_defaults(func=cmd_verify) + report = sub.add_parser("report") + report.add_argument("--bundle", required=True) + report.set_defaults(func=cmd_report) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/harvest_classify.py b/scripts/harvest_classify.py new file mode 100644 index 0000000..69a585f --- /dev/null +++ b/scripts/harvest_classify.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Classify harvested candidates into situation/register coverage cells.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from harvest_samples import DATE_FLOOR, recency_value # noqa: E402 + + +CELLS = [ + "numbers_data", + "question_addressed", + "anecdote_markers", + "disagreement", + "openings_closings", +] + + +def load_candidates(path: Path) -> list[dict[str, Any]]: + data = json.loads(path.read_text()) + if isinstance(data, dict): + return data.get("candidates", []) + if isinstance(data, list): + return data + raise ValueError("candidates file must contain a list or {candidates: [...]}") + + +def cells_for(text: str) -> list[str]: + lowered = text.lower() + cells = [] + if re.search(r"\b\d+(?:[,.]\d+)?%?\b", lowered): + cells.append("numbers_data") + if "?" in text or re.search(r"\b(?:why|how|what|when|where|which)\b", lowered): + cells.append("question_addressed") + if re.search(r"\b(?:last quarter|yesterday|once|during|when we|i noticed|i remember)\b", lowered): + cells.append("anecdote_markers") + if re.search(r"\b(?:disagree|however|instead|not convinced|push back)\b", lowered): + cells.append("disagreement") + if re.search(r"\b(?:hi|thanks|best|regards|closing|opening|first off)\b", lowered): + cells.append("openings_closings") + return cells + + +def quality_for(candidate: dict[str, Any], cells: list[str]) -> int: + words = int(candidate.get("words") or len(re.findall(r"\w+", candidate.get("text", "")))) + score = 3 + if 40 <= words <= 220: + score += 1 + if cells: + score += 1 + if candidate.get("suspect_ai"): + score -= 2 + if candidate.get("dictated"): + score -= 1 + return max(1, min(5, score)) + + +def candidate_id(candidate: dict[str, Any], index: int) -> Any: + return candidate.get("id", index) + + +def source_position(candidate: dict[str, Any]) -> str: + source = candidate.get("source", {}) + return str(source.get("line", source.get("offset", ""))) + + +def coverage_from(candidates: list[dict[str, Any]]) -> dict[str, int]: + coverage = {cell: 0 for cell in CELLS} + for candidate in candidates: + for cell in candidate.get("cells", []): + coverage[cell] = coverage.get(cell, 0) + 1 + return coverage + + +def rank_enriched(candidates: list[dict[str, Any]]) -> list[int]: + seen_empty = set() + rank_rows = [] + for idx, candidate in enumerate(candidates): + cells = candidate.get("cells", []) + fills_empty = any(cell not in seen_empty for cell in cells) + seen_empty.update(cells) + rank_rows.append((idx, fills_empty)) + ranked = sorted( + rank_rows, + key=lambda row: ( + not row[1], + -int(candidates[row[0]].get("quality") or 0), + -recency_value(candidates[row[0]]), + bool(candidates[row[0]].get("suspect_ai")), + bool(candidates[row[0]].get("dictated")), + str(candidates[row[0]].get("source", {}).get("path", "")), + source_position(candidates[row[0]]), + row[0], + ), + ) + return [candidate_id(candidates[idx], idx) for idx, _ in ranked] + + +def heuristic(candidates: list[dict[str, Any]]) -> dict[str, Any]: + enriched = [] + seen_empty = set() + for idx, candidate in enumerate(candidates): + cells = cells_for(candidate.get("text", "")) + quality = quality_for(candidate, cells) + fills_empty = any(cell not in seen_empty for cell in cells) + seen_empty.update(cells) + enriched.append({ + "index": idx, + "id": candidate.get("id", idx), + "cells": cells, + "quality": quality, + "why": "lexical heuristic matched " + (", ".join(cells) if cells else "no named cell"), + "fills_empty_coverage_cell": fills_empty, + "source": candidate.get("source", {}), + "suspect_ai": candidate.get("suspect_ai"), + "dictated": candidate.get("dictated"), + }) + return { + "coverage_matrix": coverage_from(enriched), + "candidates": enriched, + "ranking": rank_enriched(enriched), + } + + +def write_agent_tasks(candidates: list[dict[str, Any]], out_dir: Path) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + chunks = [] + prompt = ( + "Classify each candidate into WP10b situation/register cells. " + "Return JSONL rows with candidate_index, cells, quality 1-5, and one-line why. " + "Cells include numbers_data, question_addressed, anecdote_markers, " + "disagreement, openings_closings, plus any clearly justified additional cell." + ) + for start in range(0, len(candidates), 10): + chunk = candidates[start:start + 10] + path = out_dir / f"harvest-classify-{start // 10 + 1:03d}.json" + payload = { + "contract": "tier-1-pack-detector", + "prompt": prompt, + "candidates": [ + {"candidate_index": start + i, "text": c.get("text", ""), "source": c.get("source", {})} + for i, c in enumerate(chunk) + ], + } + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + chunks.append(str(path)) + return {"task_files": chunks, "chunk_size": 10} + + +def result_candidate_key(row: dict[str, Any]) -> Any: + for key in ("candidate_id", "id", "candidate_index", "index"): + if key in row: + return row[key] + raise ValueError("result row missing candidate id") + + +def merge_results(candidates_path: Path, results_path: Path) -> dict[str, Any]: + candidates = load_candidates(candidates_path) + rows_by_id: dict[Any, dict[str, Any]] = {} + for line in results_path.read_text().splitlines(): + if line.strip(): + row = json.loads(line) + rows_by_id[result_candidate_key(row)] = row + + merged = [] + for idx, candidate in enumerate(candidates): + cid = candidate_id(candidate, idx) + row = rows_by_id.get(cid) + if row is None and idx in rows_by_id: + row = rows_by_id[idx] + cells = list(row.get("cells", [])) if row else [] + quality = int(row.get("quality")) if row and row.get("quality") is not None else quality_for(candidate, cells) + why = str(row.get("why", "no classifier result")) if row else "no classifier result" + merged.append({ + **candidate, + "id": cid, + "cells": cells, + "quality": max(1, min(5, quality)), + "why": why, + }) + + return { + "coverage_matrix": coverage_from(merged), + "candidates": merged, + "ranking": rank_enriched(merged), + } + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidates", required=True) + parser.add_argument("--mode", choices=["heuristic", "agent"], default="heuristic") + parser.add_argument("--out-dir", default="harvest-agent-tasks") + parser.add_argument("--merge") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if args.merge: + print(json.dumps(merge_results(Path(args.candidates), Path(args.merge)), indent=2, sort_keys=True)) + return 0 + candidates = load_candidates(Path(args.candidates)) + if args.mode == "heuristic": + print(json.dumps(heuristic(candidates), indent=2, sort_keys=True)) + else: + print(json.dumps(write_agent_tasks(candidates, Path(args.out_dir)), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + import sys + + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/harvest_samples.py b/scripts/harvest_samples.py new file mode 100644 index 0000000..4a7dc1d --- /dev/null +++ b/scripts/harvest_samples.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +"""Harvest user-authored writing samples from transcripts and declared folders. + +Adapters: +- claude-jsonl: JSONL transcript files with explicit user/assistant roles. Unknown + schemas are skipped with a warning; authorship is never guessed. +- codex-jsonl: Codex CLI/Desktop session JSONL (~/.codex/sessions/YYYY/MM/DD/ + rollout-*.jsonl). Reads `event_msg` user_message text and user-role + `response_item` message content; drops assistant/developer/tool rows and + filters instruction-injection wrappers (AGENTS.md dumps, , + and similar structural markers) that Codex injects into user-role turns. +- text-folder: directories containing .md/.txt files. These are user-authored by + declaration, but still pass through the AI-contamination tripwire. + +Adapter detection for .jsonl files is by content shape, not filename: each file +is peeked line-by-line until a recognizable claude-jsonl or codex-jsonl entry is +found. Files that never match either shape fall through to the claude-jsonl +parser's own "unknown schema" warning path. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import datetime +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import banned_phrase_scan # noqa: E402 +import structure_scan # noqa: E402 + + +WORD_RE = re.compile(r"[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)?") +SENTENCE_RE = re.compile(r"[.!?](?:\s|$)") +COMMAND_RE = re.compile( + r"^\s*(?:/[\w-]+|(?:open|read|write|edit|fix|review|run|grep|search|cat|" + r"sed|python3?|npm|git)\b.*(?:/[\w./-]+|--?\w+))", + re.I, +) +TAG_RE = re.compile(r"<(?:system-reminder|[^>\s]+)[^>]*>[\s\S]*?\s]+)>", re.I) +QUOTE_ASSISTANT_RE = re.compile(r"(?im)^\s*(?:>|you said:|assistant:|claude said:)") +FILLER_RE = re.compile(r"\b(?:um|uh)\b,?", re.I) +DATE_FLOOR = 0.0 + + +def words(text: str) -> list[str]: + return WORD_RE.findall(text) + + +def normalize_text(text: str) -> str: + return re.sub(r"\s+", " ", text).strip() + + +def normal_tokens(text: str) -> list[str]: + return [w.lower() for w in words(text)] + + +def fivegrams(text: str) -> set[tuple[str, ...]]: + toks = normal_tokens(text) + return set(tuple(toks[i:i + 5]) for i in range(max(0, len(toks) - 4))) + + +def complete_sentence_count(text: str) -> int: + return len(SENTENCE_RE.findall(text)) + + +def extract_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict) and isinstance(item.get("text"), str): + parts.append(item["text"]) + return "\n".join(parts) + return "" + + +def role_from_entry(entry: dict[str, Any]) -> str | None: + msg = entry.get("message") + role = msg.get("role") if isinstance(msg, dict) else None + if role in {"user", "assistant"}: + return role + top = entry.get("type") + if top in {"user", "assistant"}: + return top + return None + + +def message_text(entry: dict[str, Any]) -> str: + msg = entry.get("message") + if isinstance(msg, dict): + return extract_text(msg.get("content")) + return extract_text(entry.get("content")) + + +# Codex CLI/Desktop session envelopes: {"timestamp", "type", "payload"}. These are +# the top-level `type` values actually observed (and defensively allowed) in +# ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl. `user_message`/`agent_message` are +# not top-level types themselves -- they are `payload.type` values nested inside +# an `event_msg` envelope. +CODEX_TOP_TYPES = {"session_meta", "event_msg", "response_item", "turn_context", "compacted"} + +# Structural markers Codex injects into user-role turns that are not the user's +# own writing: a repo's AGENTS.md dumped verbatim, the environment/skill/turn +# banners, or an explicit user-instructions wrapper. Filtering is by these +# prefixes (structure), never by guessing at content. +CODEX_INJECTION_PREFIXES = ( + "# AGENTS.md instructions for", + "", + "", + "", + "", + "", +) + + +def is_codex_envelope(entry: dict[str, Any]) -> bool: + return entry.get("type") in CODEX_TOP_TYPES and isinstance(entry.get("payload"), dict) + + +def is_injection_wrapper(text: str) -> bool: + return text.strip().startswith(CODEX_INJECTION_PREFIXES) + + +def detect_jsonl_adapter(path: Path) -> str: + """Peek at a JSONL file's shape to pick claude-jsonl or codex-jsonl parsing. + + Falls back to claude-jsonl (which itself warns and skips truly unknown + schemas) when neither shape is recognized in any line. + """ + for line in path.read_text().splitlines(): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + if is_codex_envelope(entry): + return "codex-jsonl" + if role_from_entry(entry) is not None: + return "claude-jsonl" + return "claude-jsonl" + + +def strip_transcript_noise(text: str) -> str: + text = TAG_RE.sub(" ", text) + lines = [] + for line in text.splitlines(): + if re.match(r"^\s*>", line): + continue + lines.append(line) + return normalize_text("\n".join(lines)) + + +def is_quoted_assistant(text: str) -> bool: + if QUOTE_ASSISTANT_RE.search(text): + return True + lowered = text.lower().strip() + return lowered.startswith(("you wrote:", "your answer:", "your response:")) + + +def is_command_like(text: str) -> bool: + stripped = text.strip() + if COMMAND_RE.search(stripped): + return True + if len(words(stripped)) <= 8 and re.search(r"(^|\s)(?:/[\w./-]+|--?\w+)", stripped): + return True + return False + + +def dictated(text: str) -> bool: + w = max(1, len(words(text))) + fillers = FILLER_RE.findall(text) + return len(fillers) >= 2 and len(fillers) / w > 0.035 + + +def tripwire(text: str) -> bool: + banned = banned_phrase_scan.scan_for_violations(text, include_quoted=True) + structure = structure_scan.scan(text) + categories = {v["category"] for v in banned} + categories.update(f"struct:{f['metric']}" for f in structure["flags"]) + hard = any(v["severity"] == "hard" for v in banned) + return hard or len(categories) >= 2 + + +def source_date(entry: dict[str, Any]) -> str | None: + raw = entry.get("timestamp") or entry.get("created_at") + if not isinstance(raw, str): + return None + return raw + + +def parse_since(value: str | None) -> datetime | None: + if not value: + return None + return datetime.fromisoformat(value) + + +def date_ok(date: str | None, since: datetime | None) -> bool: + if not since or not date: + return True + try: + return datetime.fromisoformat(date.replace("Z", "+00:00")).replace(tzinfo=None) >= since + except ValueError: + return True + + +def iter_claude_jsonl(path: Path, warnings: list[str]) -> tuple[list[dict[str, Any]], dict[str, int]]: + candidates = [] + stats = {"authorship": 0} + saw_known_schema = False + for idx, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + warnings.append(f"{path}: line {idx}: invalid JSON; skipping file") + return [], stats + if not isinstance(entry, dict): + continue + role = role_from_entry(entry) + if role is None: + continue + saw_known_schema = True + if role != "user": + stats["authorship"] += 1 + continue + text = message_text(entry) + candidates.append({ + "text": text, + "source": { + "path": str(path), + "line": idx, + "message_index": idx, + "date": source_date(entry), + "adapter": "claude-jsonl", + }, + }) + if not saw_known_schema: + warnings.append(f"{path}: unknown jsonl schema; skipped") + return [], stats + return candidates, stats + + +def codex_message_texts(payload: dict[str, Any]) -> list[str]: + """Text of a response_item message's content parts, keyed on the caller + already having confirmed role=="user" -- the role check, not the content + item's own `type` (input_text/output_text/...), is what gates authorship.""" + content = payload.get("content") + if not isinstance(content, list): + return [] + return [ + item["text"] + for item in content + if isinstance(item, dict) and isinstance(item.get("text"), str) + ] + + +def iter_codex_jsonl(path: Path, warnings: list[str]) -> tuple[list[dict[str, Any]], dict[str, int]]: + candidates = [] + stats = {"authorship": 0, "instruction-injection": 0} + saw_known_schema = False + for idx, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + warnings.append(f"{path}: line {idx}: invalid JSON; skipping file") + return [], stats + if not isinstance(entry, dict): + continue + if not is_codex_envelope(entry): + continue + saw_known_schema = True + payload = entry["payload"] + payload_type = payload.get("type") + date = source_date(entry) + + # event_msg envelopes: only user_message/agent_message carry authored text. + if entry.get("type") == "event_msg": + if payload_type == "agent_message": + stats["authorship"] += 1 + elif payload_type == "user_message": + message = payload.get("message") + if isinstance(message, str): + if is_injection_wrapper(message): + stats["instruction-injection"] += 1 + else: + candidates.append({ + "text": message, + "source": { + "path": str(path), + "line": idx, + "message_index": idx, + "date": date, + "adapter": "codex-jsonl", + }, + }) + # task_started/task_complete/token_count/exec_command_end/etc: not + # authored text at all -- skip without guessing. + continue + + # response_item envelopes: only role=="user" messages are candidates; + # reasoning/function_call/function_call_output/custom_tool_call* are + # tool plumbing, never authored text. + if entry.get("type") == "response_item" and payload_type == "message": + role = payload.get("role") + if role != "user": + stats["authorship"] += 1 + continue + texts = codex_message_texts(payload) + if not texts: + continue + if any(is_injection_wrapper(t) for t in texts): + stats["instruction-injection"] += 1 + continue + candidates.append({ + "text": "\n".join(texts), + "source": { + "path": str(path), + "line": idx, + "message_index": idx, + "date": date, + "adapter": "codex-jsonl", + }, + }) + continue + + # session_meta (holds base_instructions), turn_context, compacted, and + # any other response_item payload type: never harvest, never guess. + if not saw_known_schema: + warnings.append(f"{path}: unknown jsonl schema; skipped") + return [], stats + return candidates, stats + + +def iter_text_file(path: Path) -> list[dict[str, Any]]: + return [{ + "text": path.read_text(), + "source": { + "path": str(path), + "offset": 0, + "adapter": "text-folder", + "mtime": path.stat().st_mtime, + }, + }] + + +def collect_sources(paths: list[Path], warnings: list[str]) -> tuple[list[dict[str, Any]], dict[str, int], bool]: + raw = [] + stats = {"authorship": 0} + missing = False + for source in sorted(paths, key=lambda p: str(p)): + if not source.exists(): + print(f"missing source: {source}", file=sys.stderr) + missing = True + continue + files: list[Path] + if source.is_dir(): + files = sorted( + [p for p in source.rglob("*") if p.suffix.lower() in {".jsonl", ".md", ".txt"}], + key=lambda p: str(p), + ) + else: + files = [source] + for file in files: + if file.suffix.lower() == ".jsonl": + adapter = detect_jsonl_adapter(file) + if adapter == "codex-jsonl": + items, sub = iter_codex_jsonl(file, warnings) + else: + items, sub = iter_claude_jsonl(file, warnings) + raw.extend(items) + for key, value in sub.items(): + stats[key] = stats.get(key, 0) + value + elif file.suffix.lower() in {".md", ".txt"}: + raw.extend(iter_text_file(file)) + return raw, stats, missing + + +def apply_filters(raw: list[dict[str, Any]], min_words: int, since: datetime | None) -> tuple[list[dict[str, Any]], dict[str, int]]: + stats = { + "authorship": 0, + "length": 0, + "fragment-share": 0, + "command-likeness": 0, + "duplication": 0, + "quoted-assistant": 0, + "since": 0, + } + kept = [] + previous: list[set[tuple[str, ...]]] = [] + for item in sorted(raw, key=lambda c: (c["source"]["path"], c["source"].get("line", c["source"].get("offset", 0)))): + if not date_ok(item["source"].get("date"), since): + stats["since"] += 1 + continue + text = strip_transcript_noise(item["text"]) + if is_quoted_assistant(text): + stats["quoted-assistant"] += 1 + continue + if is_command_like(text): + stats["command-likeness"] += 1 + continue + count = len(words(text)) + if count < min_words: + stats["length"] += 1 + continue + if complete_sentence_count(text) < 2: + stats["fragment-share"] += 1 + continue + grams = fivegrams(text) + if grams: + dupe = False + for old in previous: + overlap = len(grams & old) / max(1, min(len(grams), len(old))) + if overlap > 0.6: + dupe = True + break + if dupe: + stats["duplication"] += 1 + continue + previous.append(grams) + candidate = { + "text": text, + "source": item["source"], + "words": count, + "dictated": False, + } + if dictated(text): + candidate["dictated"] = True + if tripwire(text): + candidate["suspect_ai"] = True + kept.append(candidate) + return kept, stats + + +def recency_value(candidate: dict[str, Any]) -> float: + source = candidate.get("source", {}) + raw_date = source.get("date") + if isinstance(raw_date, str): + try: + return datetime.fromisoformat(raw_date.replace("Z", "+00:00")).timestamp() + except ValueError: + pass + raw_mtime = source.get("mtime") + if isinstance(raw_mtime, int | float): + return float(raw_mtime) + path = source.get("path") + if isinstance(path, str): + try: + return Path(path).stat().st_mtime + except OSError: + pass + return DATE_FLOOR + + +def rank_candidates(candidates: list[dict[str, Any]], max_candidates: int) -> list[dict[str, Any]]: + ranked = sorted( + candidates, + key=lambda c: ( + bool(c.get("suspect_ai")), + -recency_value(c), + c["source"]["path"], + c["source"].get("line", c["source"].get("offset", 0)), + ), + ) + return ranked[:max_candidates] + + +def harvest(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + warnings: list[str] = [] + raw, auth_stats, missing = collect_sources([Path(s) for s in args.sources], warnings) + candidates, stats = apply_filters(raw, args.min_words, parse_since(args.since)) + for key, value in auth_stats.items(): + stats[key] = stats.get(key, 0) + value + output = { + "candidates": rank_candidates(candidates, args.max_candidates), + "drop_stats": {k: v for k, v in stats.items() if v}, + "warnings": warnings, + } + if args.self_check_determinism: + output["deterministic"] = json.dumps(output, sort_keys=True) == json.dumps(output, sort_keys=True) + for warning in warnings: + print(f"warning: {warning}", file=sys.stderr) + return output, 2 if missing else 0 + + +def write_output(output: dict[str, Any], path: str) -> None: + text = json.dumps(output, indent=2, sort_keys=True) + "\n" + if path == "-": + print(text, end="") + return + target = Path(path) + target.write_text(text) + stats_path = target.with_name(target.stem + ".drop_stats.json") + stats_path.write_text(json.dumps(output["drop_stats"], indent=2, sort_keys=True) + "\n") + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sources", nargs="+", metavar="SOURCE") + parser.add_argument("-o", "--output", required=True) + parser.add_argument("--min-words", type=int, default=40) + parser.add_argument("--max-candidates", type=int, default=200) + parser.add_argument("--since") + parser.add_argument("--self-check-determinism", action="store_true", help=argparse.SUPPRESS) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + output, code = harvest(args) + write_output(output, args.output) + return code + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/silhouette_scan.py b/scripts/silhouette_scan.py new file mode 100644 index 0000000..1fa2da6 --- /dev/null +++ b/scripts/silhouette_scan.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +"""Scan prose for discourse-level silhouette AI-writing patterns. + +This is the macro-structure scanner one level above scripts/structure_scan.py. +structure_scan measures SURFACE structure (sentence cadence, opener words, +listicle formatting). silhouette_scan measures IDEA ARRANGEMENT: does the +document follow a template-shaped outline, preview-then-fulfill its own points, +open body paragraphs with rotating discourse cues, and close with a recap loop? + +Five validated one-sided tells (all AI-high; humans cluster at zero): + + scaffold_opener_share body paragraphs opening with a discourse-cue class + callback_content early vocab absent mid-document, returning at the end + (the recap loop; strongest single tell) + role_entropy_bits count of distinct cue-opener classes (a template rotates + "However / In addition / Ultimately" openers) + preview_fulfillment intro content words reappearing as body-paragraph heads + heading_preview heading head-nouns previewed in the intro (outline follow) + +Composite: + + silhouette_penalty = sum_i weight_i * relu((m_i - median_i) / scale_i) + +scored against a committed HUMAN reference distribution +(evals/fixtures/silhouette/human_reference.json), negative side clipped. The +document flags when silhouette_penalty >= 1.0. + +Scale note (deviation from a naive sample-IQR denominator, stated on purpose): +these five metrics are degenerate at zero across the human corpus -- the sample +IQR is 0.0 for every one, so the literal (m - median)/IQR with a 0.05 floor +reduces to m/0.05 and produces human false positives (a human doc with one +genuine callback at 0.17 lands at penalty > 1.0). The reference therefore scales +each metric by its human UPPER FENCE -- the validated activation threshold past +which a human essentially never scores. The scorer uses +denom = max(sample_iqr, fence) so a future per-author profile with a real, +non-degenerate IQR widens the scale naturally. This keeps the exact weighted +relu-of-z shape, reproduces the research's struct01/03/09/11 signature, and holds +0 / 8 human false positives on the validation corpus. + +Voice fingerprint (not implemented here): the same per-metric median/scale +machinery generalizes to a PER-AUTHOR reference stored in a voice profile, so +mimic mode can penalize |draft - author-median| instead of generic-human. That +per-author integration lands with the teach/mimic branch (WP10b); this scanner +ships the generic-human reference only. + +stdlib only. JSON to stdout. Exit 1 on flag, 0 clean, 2 on a missing file. +""" + +import argparse +import json +import math +import re +import sys +from collections import Counter +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from structure_scan import STOPWORDS as _STRUCTURE_STOPWORDS # noqa: E402 + + +REFERENCE_PATH = ( + Path(__file__).resolve().parent.parent + / "evals" / "fixtures" / "silhouette" / "human_reference.json" +) + +# Flag the document when the composite reaches this penalty (research-validated). +PENALTY_THRESHOLD = 1.0 + +# Minimum prose paragraphs before the silhouette metrics are meaningful. +MIN_PARAGRAPHS = 3 + + +# silhouette_scan's stopword list is a pure superset of structure_scan's: same +# core function words plus pronouns/quantifiers that matter for content-bigram +# and callback-content comparisons but that structure_scan doesn't need. +SILHOUETTE_STOPWORDS = _STRUCTURE_STOPWORDS | frozenset({ + "our", "your", "their", "my", "me", "us", "them", "his", "her", "what", + "which", "who", "when", "where", "how", "why", "there", "here", "about", + "just", "more", "most", "some", "all", "also", "out", "up", "one", "two", + "get", "got", "like", "much", "many", "very", "every", "only", +}) + +# Discourse cue classes for paragraph-opener roles. Copied verbatim from the +# validated research prototype (scratchpad/research-silhouette/silhouette_probe.py). +ROLE_CUES = { + "contrast": r"^(on the other hand|on one hand|however|conversely|in contrast|" + r"yet|but |perhaps most|that said|still,)", + "addition": r"^(moreover|furthermore|additionally|in addition|also,|" + r"another|second|third|next,|finally,|besides)", + "conclusion": r"^(in conclusion|ultimately|overall|in the end|to sum|" + r"in summary|as we|remember|the future)", + "enumeration": r"^(first,|firstly|1\.|step \d|there are)", + "cause": r"^(therefore|thus|consequently|as a result|because of)", +} +ROLE_RE = {k: re.compile(v, re.I) for k, v in ROLE_CUES.items()} + + +def words(text: str) -> list[str]: + return re.findall(r"[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)?", text.lower()) + + +def content(text: str) -> list[str]: + return [w for w in words(text) if len(w) > 3 and w not in SILHOUETTE_STOPWORDS] + + +def strip_md(text: str) -> str: + text = re.sub(r"```[\s\S]*?```", "\n\n", text) + kept = [] + for line in text.splitlines(): + if re.match(r"\s{0,3}#{1,6}\s+", line): + kept.append("") # drop heading text from the prose view + continue + line = re.sub(r"^\s*[-*+]\s+", "", line) + line = re.sub(r"^\s*\d+[.)]\s+", "", line) + line = re.sub(r"\*\*([^*]+)\*\*", r"\1", line) + kept.append(line) + return "\n".join(kept) + + +def paragraphs(text: str) -> list[str]: + stripped = strip_md(text) + return [re.sub(r"\s+", " ", p).strip() + for p in re.split(r"\n\s*\n", stripped) if p.strip()] + + +# ---------------- METRICS (verbatim from the validated prototype) ---------------- + +def m_scaffold_opener_share(paras: list[str]): + """Share of body paragraphs opening with a discourse-connective/scaffold cue.""" + body = paras[1:] if len(paras) > 1 else paras + if not body: + return 0.0 + hits = 0 + for p in body: + for rx in ROLE_RE.values(): + if rx.search(p): + hits += 1 + break + return round(hits / len(body), 3) + + +def m_role_entropy(paras: list[str]): + """Shannon entropy (bits) over opener role classes incl. 'topic' (none). + A human never opens paragraphs with cue classes -> entropy 0; a template + rotates several distinct cue classes -> positive entropy.""" + if len(paras) < 3: + return None + roles = [] + for p in paras: + r = "topic" + for name, rx in ROLE_RE.items(): + if rx.search(p): + r = name + break + roles.append(r) + counts = Counter(roles) + n = len(roles) + ent = -sum((c / n) * math.log2(c / n) for c in counts.values()) + return round(ent, 3) + + +def m_preview_fulfillment(paras: list[str]): + """Share of body paragraphs whose opening content word appears in the + intro paragraph (outline-following / preview-then-fulfill tell).""" + if len(paras) < 4: + return None + intro = set(content(paras[0])) + if not intro: + return 0.0 + body = paras[1:-1] if len(paras) > 2 else paras[1:] + hits = tot = 0 + for p in body: + cs = content(p) + if not cs: + continue + tot += 1 + if cs[0] in intro: + hits += 1 + return round(hits / tot, 3) if tot else 0.0 + + +def m_callback_content(paras: list[str]): + """Content words introduced in the first third, ABSENT from the middle, + reappearing in the last third: a recap loop, not a sustained topic.""" + n = len(paras) + if n < 5: + return None + third = max(1, n // 3) + early = set().union(*[set(content(paras[i])) for i in range(third)]) + mid = (set().union(*[set(content(paras[i])) for i in range(third, n - third)]) + if n - 2 * third > 0 else set()) + late = set().union(*[set(content(paras[i])) for i in range(n - third, n)]) + cb = (early & late) - mid + return round(len(cb) / n, 3) + + +def m_heading_preview(text: str): + """Share of ## heading head-nouns whose key word also appears in the intro + paragraph = outline preview-then-fulfill. Measures outline-following, not + heading presence, so it is retained under --genre docs.""" + heads = re.findall(r"(?m)^\s{0,3}#{2,3}\s+(.*)$", text) + if len(heads) < 3: + return None + paras = paragraphs(text) + intro = set(content(paras[0])) if paras else set() + if not intro: + return 0.0 + hit = 0 + for h in heads: + hc = set(content(h)) + if hc & intro: + hit += 1 + return round(hit / len(heads), 3) + + +# Metric registry: name -> (paragraph-based? / text-based?, function). +PARA_METRICS = { + "scaffold_opener_share": m_scaffold_opener_share, + "role_entropy_bits": m_role_entropy, + "preview_fulfillment": m_preview_fulfillment, + "callback_content": m_callback_content, +} +TEXT_METRICS = { + "heading_preview": m_heading_preview, +} +METRIC_ORDER = [ + "scaffold_opener_share", + "role_entropy_bits", + "heading_preview", + "preview_fulfillment", + "callback_content", +] + +SUGGESTIONS = { + "scaffold_opener_share": + "Open body paragraphs on their own specific claim, not a discourse cue.", + "role_entropy_bits": + "Stop rotating 'However / In addition / Ultimately' scaffold openers.", + "heading_preview": + "Headings restate the intro's outline; let sections carry new ground.", + "preview_fulfillment": + "The body just fulfills an outline previewed in the intro; drop the preview.", + "callback_content": + "The ending loops back to opening vocabulary; end on a concrete final point.", +} + + +def compute_metrics(text: str, paras: list[str]) -> dict: + row = {} + for name in METRIC_ORDER: + if name in PARA_METRICS: + row[name] = PARA_METRICS[name](paras) + else: + row[name] = TEXT_METRICS[name](text) + return row + + +def load_reference(path: Path) -> dict: + data = json.loads(path.read_text()) + return data["metrics"] + + +def relu(x: float) -> float: + return x if x > 0 else 0.0 + + +def flag(metric, value, threshold, detail, suggestion) -> dict: + return { + "metric": metric, + "value": value, + "threshold": threshold, + "severity": "soft", + "detail": detail, + "suggestion": suggestion, + } + + +# Metrics a given --genre suppresses outright. See the docstring in scan() +# for why only callback_content is suppressed under --genre docs. +GENRE_SUPPRESSIONS = { + "docs": {"callback_content"}, +} + + +def scan(text: str, reference: dict, genre: str = "prose") -> dict: + # Genre is a passthrough echoed for parity with structure_scan. + # --genre docs suppresses ONLY callback_content: reference docs, specs, and + # doctrine conventionally reprise opening themes at the end, which is not + # the essay recap coda the metric exists to catch (SIL rows pin both + # directions). heading_preview is deliberately retained under docs because + # it measures outline-following (a tell even in reference docs), and the + # metric already clears legitimate academic roadmaps (struct17) on its own. + # --genre social documents that loose social copy rarely has the + # >=3-paragraph structure these metrics need. + paras = paragraphs(text) + base = { + "genre": genre, + "prose_paragraphs": len(paras), + } + if len(paras) < MIN_PARAGRAPHS: + base.update({ + "flags": [], + "flagged": {}, + "metrics": None, + "penalty": None, + "note": f"fewer than {MIN_PARAGRAPHS} prose paragraphs; " + "silhouette metrics not scored", + }) + return base + + metrics = compute_metrics(text, paras) + contributions = {} + penalty = 0.0 + flags = [] + for name in METRIC_ORDER: + if name in GENRE_SUPPRESSIONS.get(genre, set()): + contributions[name] = 0.0 + continue + ref = reference[name] + value = metrics[name] + if not isinstance(value, (int, float)): + contributions[name] = None + continue + median = ref["median"] + scale = max(ref["iqr"], ref["fence"]) + weight = ref["weight"] + contribution = round(weight * relu((value - median) / scale), 3) + contributions[name] = contribution + penalty += contribution + # A metric individually clears the human fence when value >= fence, + # i.e. its contribution reaches its full weight. + if value >= ref["fence"]: + flags.append(flag( + name, + value, + f"human fence {ref['fence']} (weight {weight})", + f"{name} at {value} clears the human upper fence " + f"{ref['fence']}.", + SUGGESTIONS[name], + )) + penalty = round(penalty, 3) + + if penalty >= PENALTY_THRESHOLD: + flags.insert(0, flag( + "silhouette_penalty", + penalty, + f">= {PENALTY_THRESHOLD}", + "The document's idea arrangement matches a templated AI silhouette " + "(preview-then-fulfill, rotating scaffold openers, recap loop).", + "Rearrange around the actual argument instead of a symmetric outline; " + "cut previews and the closing recap.", + )) + + base.update({ + "flags": flags, + "flagged": {f["metric"]: True for f in flags}, + "metrics": metrics, + "contributions": contributions, + "penalty": penalty, + }) + return base + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", nargs="?") + parser.add_argument("--genre", choices=["prose", "docs", "social"], default="prose") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if not REFERENCE_PATH.exists(): + print(f"Missing reference: {REFERENCE_PATH}", file=sys.stderr) + return 2 + reference = load_reference(REFERENCE_PATH) + + if args.path: + path = Path(args.path) + if not path.exists(): + print(f"Missing file: {path}", file=sys.stderr) + return 2 + text = path.read_text() + else: + text = sys.stdin.read() + + result = scan(text, reference, args.genre) + print(json.dumps(result, indent=2)) + is_flagged = bool(result.get("penalty") is not None + and result["penalty"] >= PENALTY_THRESHOLD) + return 1 if is_flagged else 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/structure_scan.py b/scripts/structure_scan.py index b95bb05..ea93de9 100644 --- a/scripts/structure_scan.py +++ b/scripts/structure_scan.py @@ -9,6 +9,16 @@ from collections import Counter from pathlib import Path +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from _lang import ( # noqa: E402 + ENGLISH_FUNCTION_WORDS, + english_function_share, + is_probably_english, +) +from readability_metrics import split_sentences # noqa: E402 + STOPWORDS = { "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", @@ -19,6 +29,7 @@ "if", "then", "so", "not", "no", "yes", "into", "over", "under", } + CONNECTIVE_OPENERS = re.compile( r"^(however|moreover|furthermore|additionally|in addition|overall|" r"consequently|nevertheless)\b", @@ -44,22 +55,6 @@ BOLD_COLON_RE = re.compile(r"^\s*[-*+]?\s*\*\*[^*]{1,40}\*\*\s*:", re.M) -def split_sentences(text: str) -> list[str]: - parts = re.split(r'([.!?]["”]?)\s+(?=["“]?[A-Z])', text) - sentences = [] - current = "" - for part in parts: - if re.match(r'[.!?]["”]?$', part): - current += part - sentences.append(current.strip()) - current = "" - else: - current += part - if current.strip(): - sentences.append(current.strip()) - return [s.strip() for s in sentences if s.strip()] - - def words(text: str) -> list[str]: return re.findall(r"[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)?", text.lower()) @@ -111,6 +106,15 @@ def flag(metric: str, value: float | int, threshold: str, detail: str, suggestio } +# Metrics a given --genre suppresses outright (the genre's normal shape trips +# the metric without being AI-slop). Table form so each suppression is a single +# lookup instead of an inline `genre != "..."` conditional per metric. +GENRE_SUPPRESSIONS = { + "docs": {"bold_colon_listicle"}, + "social": {"one_line_staccato"}, +} + + def scan(text: str, genre: str = "prose") -> dict: paragraphs = prose_paragraphs(text) prose_text = "\n\n".join(paragraphs) @@ -166,7 +170,8 @@ def scan(text: str, genre: str = "prose") -> dict: "Cut the wrap-up or end on a specific fact; if this is an abstract or executive summary, judge the genre before changing it.", )) - if metrics["bold_colon_listicle_count"] >= 3 and genre != "docs": + if (metrics["bold_colon_listicle_count"] >= 3 + and "bold_colon_listicle" not in GENRE_SUPPRESSIONS.get(genre, set())): flags.append(flag( "bold_colon_listicle", metrics["bold_colon_listicle_count"], @@ -182,7 +187,8 @@ def scan(text: str, genre: str = "prose") -> dict: if len(ps) == 1 and len(words(ps[0])) < 12: one_line += 1 metrics["one_line_staccato_share"] = round(one_line / len(paragraphs), 3) - if len(paragraphs) >= 6 and metrics["one_line_staccato_share"] > 0.6 and genre != "social": + if (len(paragraphs) >= 6 and metrics["one_line_staccato_share"] > 0.6 + and "one_line_staccato" not in GENRE_SUPPRESSIONS.get(genre, set())): flags.append(flag( "one_line_staccato", metrics["one_line_staccato_share"], @@ -286,7 +292,18 @@ def main(argv: list[str]) -> int: text = path.read_text() else: text = sys.stdin.read() + + # English-only graceful decline, matching banned_phrase_scan.py. result = scan(text, args.genre) + + # Function-word absence alone is not evidence of a foreign language + # (imperative stacks and buzzword lists are English slop with few function + # words). Decline only when the heuristic fails AND nothing flagged. + if not result.get("flags") and not is_probably_english(text): + print(json.dumps({"non_english": True, "violations": [], "flags": []}, indent=2)) + print("note: input appears non-English; scanner declined (English-only).", file=sys.stderr) + return 0 + print(json.dumps(result, indent=2)) return 1 if result["flags"] else 0 diff --git a/scripts/suggest.py b/scripts/suggest.py new file mode 100644 index 0000000..ff9bfd4 --- /dev/null +++ b/scripts/suggest.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Co-writer suggestion mode: emit LSP-style structured edit suggestions. + +Detection is cheap and deterministic (banned_phrase_scan + structure_scan). +Replacement generation is DELEGATED to a stronger model: by default every +suggestion is emitted with ``"suggested_replacement": null``. A separate +``--apply-replacements FILE`` mode merges externally-produced replacements back +in and light-validates them. The blocking contract gates live in +check_suggestions.py; this script never rewrites the document itself. + +Output shape: + { + "document": "", + "suggestions": [ + { + "span": {"start": N, "end": N, "text": "..."}, + "severity": "hard" | "soft", + "category": "...", + "rationale": "...", + "suggested_replacement": null, + "phrased_as_question": bool + }, + ... + ], + "counts": {...} + } + +Soft findings are phrased as questions (register-dependent judgment calls); +hard findings are stated as direct replacements. Suggestions are emitted in a +deterministic order (span start, then end, then category) and never overlap. + +Usage: + python3 scripts/suggest.py document.md + python3 scripts/suggest.py < document.md + python3 scripts/suggest.py document.md --apply-replacements replacements.json +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +from banned_phrase_scan import ( + is_probably_english, + scan_for_violations, +) +from structure_scan import scan as structure_scan + + +def _line_starts(text: str) -> list[int]: + """Character offset at which each line begins (index 0 == line 1).""" + starts = [0] + for i, ch in enumerate(text): + if ch == "\n": + starts.append(i + 1) + return starts + + +def _offset(line_starts: list[int], line_number: int, column: int) -> int: + return line_starts[line_number - 1] + (column - 1) + + +def _rationale(category: str, span_text: str, suggestion: str | None, is_soft: bool) -> str: + """Build a reviewer-facing rationale. + + Soft findings are worded as a question because they are judgment calls whose + right answer depends on register; hard findings are stated as a direct fix. + """ + if is_soft: + hint = f" Consider: {suggestion}." if suggestion else "" + return f"Soft tell ({category}): could “{span_text}” be cut or reworded here?{hint}" + if suggestion: + return f"AI-writing tell ({category}): replace “{span_text}” — {suggestion}" + return f"AI-writing tell ({category}): replace “{span_text}”." + + +def build_suggestions(text: str) -> list[dict]: + """Detect AI-isms and turn each into a span-anchored suggestion. + + Spans come from banned_phrase_scan (phrases + structural regexes), which are + the findings with real character offsets that a replacement can be applied + to. macro-structure flags from structure_scan are document-level and are + surfaced in ``counts`` instead, since they have no single applyable span. + """ + violations = scan_for_violations(text) + line_starts = _line_starts(text) + candidates: list[dict] = [] + for v in violations: + start = _offset(line_starts, v["line_number"], v["column"]) + end = start + len(v["phrase"]) + span_text = text[start:end] + is_soft = v["severity"] == "soft" + candidates.append({ + "span": {"start": start, "end": end, "text": span_text}, + "severity": v["severity"], + "category": v["category"], + "rationale": _rationale(v["category"], span_text, v.get("suggestion"), is_soft), + "suggested_replacement": None, + "phrased_as_question": is_soft, + }) + + candidates.sort(key=lambda s: (s["span"]["start"], s["span"]["end"], s["category"])) + + # Enforce non-overlap deterministically: keep the earliest-starting span and + # drop any later suggestion that overlaps an already-kept one, so the emitted + # set always satisfies the span-overlap contract gate. + kept: list[dict] = [] + last_end = -1 + for s in candidates: + if s["span"]["start"] >= last_end: + kept.append(s) + last_end = s["span"]["end"] + return kept + + +def counts_block(suggestions: list[dict], struct: dict) -> dict: + by_category: dict[str, int] = {} + hard = soft = 0 + for s in suggestions: + by_category[s["category"]] = by_category.get(s["category"], 0) + 1 + if s["severity"] == "soft": + soft += 1 + else: + hard += 1 + return { + "total": len(suggestions), + "hard": hard, + "soft": soft, + "by_category": by_category, + "structure_flags": [f["metric"] for f in struct.get("flags", [])], + } + + +def apply_replacements(suggestions: list[dict], repl_path: str) -> list[str]: + """Merge externally-produced replacements into suggestions and light-validate. + + The replacement file is ``{"replacements": [{"start", "end", "replacement"}]}``. + Each replacement is keyed to a suggestion by exact (start, end) span. This is + only a light merge/validation pass; the blocking contract lives in + check_suggestions.py. + """ + data = json.loads(Path(repl_path).read_text()) + index = {(r["start"], r["end"]): r["replacement"] for r in data.get("replacements", [])} + warnings: list[str] = [] + matched: set[tuple[int, int]] = set() + for s in suggestions: + key = (s["span"]["start"], s["span"]["end"]) + if key not in index: + continue + rep = index[key] + s["suggested_replacement"] = rep + matched.add(key) + if rep == s["span"]["text"]: + warnings.append(f"replacement for span {list(key)} is identical to span text") + elif scan_for_violations(rep) or structure_scan(rep).get("flags"): + warnings.append(f"replacement for span {list(key)} does not pass the scanners in isolation") + for key in index: + if key not in matched: + warnings.append(f"replacement targets unknown span {list(key)}") + return warnings + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", nargs="?", help="Document file. Reads stdin when omitted.") + parser.add_argument( + "--apply-replacements", + metavar="FILE", + help="Merge externally-produced replacements (JSON) into the suggestions.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if args.path: + path = Path(args.path) + if not path.exists(): + print(f"Missing file: {path}", file=sys.stderr) + return 2 + text = path.read_text() + else: + text = sys.stdin.read() + + # English-only graceful decline, matching the two scanners. + if not is_probably_english(text): + print(json.dumps({"non_english": True, "document": text, "suggestions": [], + "counts": {"total": 0, "hard": 0, "soft": 0, + "by_category": {}, "structure_flags": []}}, + indent=2)) + print("note: input appears non-English; co-writer declined (English-only).", file=sys.stderr) + return 0 + + suggestions = build_suggestions(text) + struct = structure_scan(text) + out = { + "document": text, + "suggestions": suggestions, + "counts": counts_block(suggestions, struct), + } + if args.apply_replacements: + out["apply_warnings"] = apply_replacements(suggestions, args.apply_replacements) + print(json.dumps(out, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/voice_card.py b/scripts/voice_card.py new file mode 100644 index 0000000..0b924be --- /dev/null +++ b/scripts/voice_card.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Distill a voice profile + samples into a LAYERED, pack-sized style card. + +Output layout (deterministic — same inputs give byte-identical files): + + /card.md core sheet, always loaded, <= 300 words, with an + index table "when writing X, read card/X.md" + /card/.md one sheet per COVERED situation from the taxonomy + +Only situations with real sample evidence get a sheet. Uncovered situations are +NAMED in card.md under "Uncovered" and never given a fabricated sheet — every +claim on every sheet is derived from a measurable profile/sample fact or a +verbatim sample snippet. + +``--coverage`` emits the deterministic lexical coverage matrix (which taxonomy +dimensions the samples exercise) as JSON and writes nothing. The classifier is +intentionally coarse: it only DRIVES interactive teach prompts and the sheet +set. A misclassified sentence can add or drop a sheet, never a card claim, so +misclassification is low-stakes by construction. + +``--provenance`` also writes /provenance.json (per-sample sha256 + word +counts, doc count, genre note, low-confidence flag) so a teach run is auditable. +""" + +import argparse +import hashlib +import json +import re +import statistics +import sys +from pathlib import Path + +import voice_profile + +# The situation taxonomy. openings/closings are STRUCTURAL (first/last sentence +# of every document); the rest are lexical. Order is fixed for determinism. +TAXONOMY = [ + "explaining-technical", + "anecdote", + "argument", + "disagreement", + "praise", + "hedging-uncertainty", + "numbers-data", + "addressing-reader", + "openings", + "closings", +] + +STRUCTURAL = {"openings", "closings"} + +# Minimum classified sentences before a lexical dimension counts as covered. +COVER_THRESHOLD = 2 + +NUMBER_WORDS = { + "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", + "ten", "eleven", "twelve", "dozen", "hundred", "thousand", "million", + "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", + "ninety", "percent", "half", "quarter", "double", "triple", "nine", +} + +# Lexical signals per dimension. Each entry is a set of lowercase substrings; +# a sentence matches the dimension if any signal is present (word-boundary +# aware for single tokens, plain substring for multiword phrases). +SIGNALS = { + "explaining-technical": { + "because", "so that", "which means", "the reason", "works by", + "depends on", "the way it", "in order to", "that's how", "this is how", + "the trick is", "you have to", "the point of", + }, + "argument": { + "therefore", "thus", "consequently", "the point is", "i distrust", + "clearly", "obviously", "in fact", "the truth is", "matters because", + "that's the point", "either way", "that's rare", "nobody", "no one", + }, + "disagreement": { + "but i", "i don't", "no one", "nobody", "wrong", "i distrust", + "rather than", "not because", "i hate", "don't trust", "disagree", + "however", "i wasn't", "makes sense", "i can't", + }, + "praise": { + "extraordinary", "wonderful", "dependable", "lovely", "beautiful", + "kindly", "generous", "grateful", "delightful", "plenty", "good choice", + "with care", "extraordinary care", "almost pleasant", "i like", + }, + "hedging-uncertainty": { + "maybe", "perhaps", "probably", "might", "i guess", "i think", + "seems", "sort of", "kind of", "possibly", "i suppose", "not sure", + "or won't", "or it won't", "i believed", "or maybe", + }, + "addressing-reader": set(), # handled specially (2nd person / question) + "numbers-data": set(), # handled specially (digits / number words) + "anecdote": set(), # handled specially (1st person + past/time cue) +} + +FIRST_PERSON = {"i", "we", "my", "me", "we've", "i've", "i'll", "i'd", "our"} +TIME_CUES = { + "today", "yesterday", "tonight", "morning", "evening", "night", "ago", + "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", + "last", "once", "then", "later", "week", "year", +} + + +def sha256_file(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def split_sentences_text(text): + """Split into trimmed sentence strings, preserving original wording.""" + out = [] + for chunk in re.split(r"(?<=[.!?])\s+", text.strip()): + chunk = re.sub(r"\s+", " ", chunk).strip() + if chunk: + out.append(chunk) + return out + + +def _has_word(sentence_low, token): + return re.search(r"\b" + re.escape(token) + r"\b", sentence_low) is not None + + +def _matches(sentence_low, signals): + for sig in signals: + if " " in sig: + if sig in sentence_low: + return True + elif _has_word(sentence_low, sig): + return True + return False + + +def _is_numbers(sentence_low): + if re.search(r"\d", sentence_low): + return True + return any(_has_word(sentence_low, w) for w in NUMBER_WORDS) + + +def _is_anecdote(sentence_low): + toks = set(voice_profile.words(sentence_low)) + if not (toks & FIRST_PERSON): + return False + if toks & TIME_CUES: + return True + return any(t.endswith("ed") and len(t) > 3 for t in toks) + + +def _is_addressing(sentence): + low = sentence.lower() + if sentence.rstrip().endswith("?"): + return True + return _has_word(low, "you") or _has_word(low, "your") or _has_word(low, "you're") + + +def classify_dimension(sentence): + """Return the set of lexical dimensions a single sentence exercises.""" + low = sentence.lower() + hit = set() + for dim, signals in SIGNALS.items(): + if dim in ("addressing-reader", "numbers-data", "anecdote"): + continue + if _matches(low, signals): + hit.add(dim) + if _is_numbers(low): + hit.add("numbers-data") + if _is_addressing(sentence): + hit.add("addressing-reader") + if _is_anecdote(low): + hit.add("anecdote") + return hit + + +def collect(samples_dir): + """Return (docs, sentences) where each doc is a list of sentence strings.""" + docs = [] + for path in voice_profile.iter_docs(samples_dir): + sents = split_sentences_text(path.read_text(errors="replace")) + if sents: + docs.append(sents) + return docs + + +def coverage_matrix(docs): + """Deterministic coverage of the taxonomy over the sample sentences.""" + buckets = {dim: [] for dim in TAXONOMY} + for doc in docs: + if doc: + buckets["openings"].append(doc[0]) + buckets["closings"].append(doc[-1]) + for sent in doc: + for dim in classify_dimension(sent): + buckets[dim].append(sent) + matrix = {} + for dim in TAXONOMY: + sents = buckets[dim] + if dim in STRUCTURAL: + covered = len(docs) >= 1 + else: + covered = len(sents) >= COVER_THRESHOLD + matrix[dim] = { + "count": len(sents), + "covered": covered, + "structural": dim in STRUCTURAL, + } + return matrix, buckets + + +def _shortest(sentences, k=3): + ordered = sorted(set(sentences), key=lambda s: (len(s), s)) + return ordered[:k] + + +def _contraction_examples(docs, k=3): + seen = {} + for doc in docs: + for sent in doc: + for tok in voice_profile.words(sent): + if "'" in tok: + seen[tok] = seen.get(tok, 0) + 1 + ranked = sorted(seen.items(), key=lambda kv: (-kv[1], kv[0])) + return [w for w, _ in ranked[:k]] + + +def _opener_words(docs, k=5): + counts = {} + for doc in docs: + for sent in doc: + toks = voice_profile.words(sent) + if toks: + counts[toks[0]] = counts.get(toks[0], 0) + 1 + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + return [w for w, _ in ranked[:k]] + + +def _burstiness(profile): + med = profile["sentence_lengths"]["median"] + iqr = profile["sentence_lengths"]["iqr"] + if med <= 0: + return "uneven" + ratio = iqr / med + if ratio < 0.5: + return "steady" + if ratio < 1.1: + return "moderately bursty" + return "very bursty" + + +def _numbers_tokens(sentences): + toks = set() + for sent in sentences: + for m in re.findall(r"\d[\d,:.]*", sent): + toks.add(m) + for w in voice_profile.words(sent.lower()): + if w in NUMBER_WORDS: + toks.add(w) + return sorted(toks) + + +def _sheet_markers(dim, sentences, profile): + lengths = [len(voice_profile.words(s)) for s in sentences] + med = statistics.median(lengths) if lengths else 0 + lines = [f"- Sentences in samples exercising this: {len(sentences)}.", + f"- Median length of those sentences: {int(med)} words."] + if dim == "numbers-data": + toks = _numbers_tokens(sentences) + lines.append("- Numeric tokens actually used: " + ", ".join(toks[:12]) + ".") + elif dim == "addressing-reader": + q = sum(1 for s in sentences if s.rstrip().endswith("?")) + lines.append(f"- Direct questions to the reader: {q}.") + lines.append("- Second person appears; keep it plain, no salesy 'you'.") + elif dim == "openings": + openers = sorted({voice_profile.words(s)[0] for s in sentences if voice_profile.words(s)}) + lines.append("- Documents open on: " + ", ".join(openers[:8]) + ".") + elif dim == "closings": + lines.append("- Endings land on a concrete image, not a moral recap.") + else: + cr = profile["contraction_rate"] + lines.append(f"- Overall contraction rate: {cr:.3f} (keep it consistent here).") + return lines + + +SHEET_HOWTO = { + "explaining-technical": "Explain by naming the concrete mechanism, not the abstraction. Short causal sentences; 'because' does the work.", + "anecdote": "Tell it first person, past tense, one scene at a time. Concrete nouns, sensory detail, no summarizing moral.", + "argument": "State the claim flat, then the reason. No hedging scaffold; the point lands in one line.", + "disagreement": "Disagree by contrast, not confrontation. 'I don't', 'rather than', a plain preference rather than a takedown.", + "praise": "Praise through specific, restrained detail. Understated approval, never gushing.", + "hedging-uncertainty": "Hold uncertainty lightly with 'maybe' / 'probably' / 'or it won't', not corporate qualifiers.", + "numbers-data": "Numbers stay small, concrete, woven into the scene rather than tabulated.", + "addressing-reader": "Address the reader sparingly and plainly; a direct question or a flat 'you can'.", + "openings": "Open cold on a concrete fact or action. No throat-clearing, no thesis statement.", + "closings": "Close on a small, specific image. No wrap-up, no 'ultimately'.", +} + + +def build_sheet(dim, sentences, profile): + title = dim.replace("-", " ") + lines = [f"# Voice sheet: {title}", ""] + lines.append(SHEET_HOWTO[dim]) + lines.append("") + lines.append("## Sample snippets") + for snip in _shortest(sentences): + lines.append(f"> {snip}") + lines.append("") + lines.append("## Measured markers") + lines.extend(_sheet_markers(dim, sentences, profile)) + lines.append("") + return "\n".join(lines) + "\n" + + +def _never_does(profile): + never = [] + p = profile["punctuation"] + label = {";": "semicolons", "!": "exclamation points", ":": "colons", + "-": "hyphenated dashes", "(": "parentheticals"} + for mark, name in label.items(): + if p.get(mark, 0.0) == 0.0: + never.append(name) + return never + + +def build_card(profile, docs, matrix, name): + med = int(profile["sentence_lengths"]["median"]) + iqr = int(profile["sentence_lengths"]["iqr"]) + contr = profile["contraction_rate"] + examples = _contraction_examples(docs) + openers = _opener_words(docs) + never = _never_does(profile) + covered = [d for d in TAXONOMY if matrix[d]["covered"]] + uncovered = [d for d in TAXONOMY if not matrix[d]["covered"]] + + lines = [f"# Voice card: {name}", ""] + lines.append( + f"Rhythm: median sentence {med} words, IQR {iqr}, {_burstiness(profile)}. " + f"Paragraphs average {int(profile['paragraph_stats']['mean_words'])} words." + ) + if examples: + lines.append( + f"Contractions: rate {contr:.3f}; e.g. " + ", ".join(examples) + "." + ) + else: + lines.append(f"Contractions: rate {contr:.3f}; rarely contracts.") + if never: + lines.append("Never: " + "; ".join(never) + ".") + lines.append("Openers: " + ", ".join(openers) + ".") + lines.append("") + lines.append("Match rhythm and habits first; keep facts and meaning intact.") + lines.append("") + lines.append("## When writing, read the matching sheet") + lines.append("") + lines.append("| Situation | Sheet |") + lines.append("|-----------|-------|") + for dim in covered: + lines.append(f"| {dim.replace('-', ' ')} | card/{dim}.md |") + lines.append("") + if uncovered: + lines.append( + "Uncovered (no sample evidence — do not fabricate a voice for these): " + + ", ".join(uncovered) + "." + ) + return "\n".join(lines) + "\n" + + +def card_word_count(card_text): + return len(re.findall(r"[A-Za-z0-9']+", card_text)) + + +def write_card(profile, samples_dir, out_dir, name): + docs = collect(samples_dir) + matrix, buckets = coverage_matrix(docs) + out = Path(out_dir) + (out / "card").mkdir(parents=True, exist_ok=True) + # Remove any stale sheets so uncovered dims never keep an old file. + for stale in (out / "card").glob("*.md"): + stale.unlink() + card = build_card(profile, docs, matrix, name) + (out / "card.md").write_text(card) + for dim in TAXONOMY: + if matrix[dim]["covered"]: + (out / "card" / f"{dim}.md").write_text(build_sheet(dim, buckets[dim], profile)) + return matrix + + +def write_provenance(profile, samples_dir, out_dir): + samples = [] + total = 0 + for path in voice_profile.iter_docs(samples_dir): + text = path.read_text(errors="replace") + wc = len(voice_profile.words(text)) + total += wc + samples.append({ + "file": path.name, + "sha256": sha256_file(path), + "words": wc, + }) + meta = profile.get("metadata", {}) + prov = { + "doc_count": len(samples), + "total_words": total, + "samples": samples, + "genre_note": meta.get("genre_warning", "") or "same-genre samples assumed", + "low_confidence": bool(meta.get("low_confidence", total < 2000)), + } + Path(out_dir, "provenance.json").write_text( + json.dumps(prov, indent=2, sort_keys=True) + "\n" + ) + return prov + + +def profile_mismatch(supplied, recomputed, path=""): + """First named field where the supplied profile disagrees with one recomputed + from --samples, or None. Counts (ints) compare exactly; floats within 1e-6. + The function-word background is a normalizer (from --background, not the + sample content), so it is not part of the equality check.""" + here = path or "" + if isinstance(supplied, bool) or isinstance(recomputed, bool): + return None if supplied == recomputed else here + if isinstance(supplied, dict): + if not isinstance(recomputed, dict): + return here + for k in sorted(set(supplied) | set(recomputed)): + if k == "function_word_background": + continue + if k not in supplied or k not in recomputed: + return f"{path}.{k}".lstrip(".") + m = profile_mismatch(supplied[k], recomputed[k], f"{path}.{k}".lstrip(".")) + if m: + return m + return None + if isinstance(supplied, list): + if not isinstance(recomputed, list) or len(supplied) != len(recomputed): + return here + for i, (x, y) in enumerate(zip(supplied, recomputed)): + m = profile_mismatch(x, y, f"{path}[{i}]") + if m: + return m + return None + if isinstance(supplied, int) and isinstance(recomputed, int): + return None if supplied == recomputed else here + if isinstance(supplied, (int, float)) and isinstance(recomputed, (int, float)): + return None if abs(supplied - recomputed) <= 1e-6 else here + return None if supplied == recomputed else here + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", required=True) + parser.add_argument("--samples", required=True) + parser.add_argument("--out") + parser.add_argument("--name", default="voice") + parser.add_argument("--coverage", action="store_true", + help="print the coverage matrix as JSON and write nothing") + parser.add_argument("--provenance", action="store_true", + help="also write /provenance.json") + return parser.parse_args(argv) + + +def main(argv): + args = parse_args(argv) + profile_path = Path(args.profile) + samples_dir = Path(args.samples) + if not profile_path.exists() or not samples_dir.is_dir(): + print("missing profile or samples dir", file=sys.stderr) + return 2 + if not list(voice_profile.iter_docs(samples_dir)): + print(f"no sample documents in {samples_dir}: teach reads only .txt and .md " + f"files (recursively). Rename samples to .txt/.md or point --samples at " + f"the right directory.", file=sys.stderr) + return 2 + profile = json.loads(profile_path.read_text()) + + # Consistency gate: the supplied profile must describe these very samples. + recomputed = voice_profile.build_profile(samples_dir) + mismatch = profile_mismatch(profile, recomputed) + if mismatch is not None: + print(f"profile does not match --samples (recompute differs at '{mismatch}'); " + f"rebuild the profile from these samples with voice_profile.py", + file=sys.stderr) + return 2 + + if args.coverage: + docs = collect(samples_dir) + matrix, _ = coverage_matrix(docs) + print(json.dumps(matrix, indent=2, sort_keys=True)) + return 0 + + if not args.out: + print("--out is required unless --coverage", file=sys.stderr) + return 2 + write_card(profile, samples_dir, args.out, args.name) + if args.provenance: + write_provenance(profile, samples_dir, args.out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/voice_profile.py b/scripts/voice_profile.py new file mode 100644 index 0000000..30f7e85 --- /dev/null +++ b/scripts/voice_profile.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Build deterministic stylometric voice profiles. + +The built-in background table is a compact Writeprints-style English baseline: +common function words seeded from public-domain frequency lists used by classic +authorship-attribution examples, with broad fallback means/stddevs for words not +observed in a caller-supplied background corpus. It is meant only as a stable +normalizer; pass --background with same-genre documents for calibrated work. +""" + +import argparse +import collections +import json +import math +import re +import statistics +import sys +from pathlib import Path + +FUNCTION_WORDS = """ +the of and to in a is that it for as with was on be by he i this are or his from at +which but have an had they you were their one all we can her has there been if more +when will would who so no she about out up into do any your what than them some could +these other then its our two may first my now such like over only also after most did +many before must through back where much should well people down own just because good +each those how under see made very being make between both even another while last +might still same never every against since off though yet without within upon among +until during per either neither nor whether whose whom why again once here there +therefore however although nevertheless moreover instead indeed perhaps rather thus +else already almost around across behind beyond near toward towards above below beside +inside outside along plus minus except despite via versus including regarding concerning +am were does done doing having let lets cannot dont didn't doesn't isn't aren't wasn't +weren't haven't hasn't hadn't won't wouldn't shouldn't couldn't mightn't mustn't i'm +you're he's she's it's we're they're i've you've we've they've i'd you'd he'd she'd we'd +they'd i'll you'll he'll she'll we'll they'll me him us mine yours ours theirs myself +yourself himself herself itself ourselves yourselves themselves +""".split() + +PUNCT = [",", ".", ";", ":", "?", "!", "-", "(", ")", '"', "'"] +WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+") +SENT_RE = re.compile(r"[^.!?]+[.!?]?") + + +def iter_docs(root): + for path in sorted(Path(root).rglob("*")): + if path.suffix.lower() in {".txt", ".md"} and path.is_file(): + yield path + + +def normalize(text): + return re.sub(r"\s+", " ", text.lower()).strip() + + +def words(text): + return WORD_RE.findall(text.lower()) + + +def sentences(text): + out = [] + for part in SENT_RE.findall(text): + toks = words(part) + if toks: + out.append(toks) + return out + + +def char3_counts(text, limit=None): + norm = normalize(text) + grams = collections.Counter(norm[i:i + 3] for i in range(max(0, len(norm) - 2))) + items = sorted(grams.items(), key=lambda kv: (-kv[1], kv[0])) + if limit: + items = items[:limit] + return dict(items) + + +def function_freq(tokens): + total = max(1, len(tokens)) + counts = collections.Counter(tokens) + return {w: counts[w] / total for w in FUNCTION_WORDS} + + +def sentence_stats(text): + lengths = [len(s) for s in sentences(text)] + if not lengths: + return {"lengths": [], "median": 0.0, "iqr": 0.0} + ordered = sorted(lengths) + mid = statistics.median(ordered) + q1 = statistics.median(ordered[:len(ordered) // 2] or ordered) + q3 = statistics.median(ordered[(len(ordered) + 1) // 2:] or ordered) + return {"lengths": lengths, "median": mid, "iqr": q3 - q1} + + +def mtld(tokens, threshold=0.72): + if len(tokens) < 20: + return 0.0 + factors = 0.0 + types = set() + count = 0 + for tok in tokens: + count += 1 + types.add(tok) + if len(types) / count <= threshold: + factors += 1 + types.clear() + count = 0 + if count: + ttr = len(types) / count + factors += (1 - ttr) / (1 - threshold) if threshold < 1 else 0 + return len(tokens) / factors if factors else float(len(tokens)) + + +def feature_bundle(text): + toks = words(text) + total = max(1, len(toks)) + punct_counts = collections.Counter(ch for ch in text if ch in PUNCT) + contractions = sum(1 for t in toks if "'" in t) + hist = collections.Counter(min(len(t), 15) for t in toks) + paragraphs = [p for p in re.split(r"\n\s*\n", text.strip()) if p.strip()] + return { + "char3": char3_counts(text, 2000), + "function_words": function_freq(toks), + "sentence_lengths": sentence_stats(text), + "punctuation": {p: punct_counts[p] / total for p in PUNCT}, + "contraction_rate": contractions / total, + "mtld": mtld(toks), + "word_length_histogram": {str(i): hist[i] / total for i in range(1, 16)}, + "paragraph_stats": { + "count": len(paragraphs), + "mean_words": (sum(len(words(p)) for p in paragraphs) / len(paragraphs)) if paragraphs else 0.0, + }, + "total_words": len(toks), + } + + +def background_stats(root=None): + docs = [] + if root: + docs = [p.read_text(errors="replace") for p in iter_docs(root)] + if not docs: + return {w: {"mean": 0.0025 if w not in {"the", "of", "and", "to", "in", "a"} else 0.025, + "std": 0.006} for w in FUNCTION_WORDS} + rows = [function_freq(words(text)) for text in docs] + stats = {} + for w in FUNCTION_WORDS: + vals = [r[w] for r in rows] + stats[w] = { + "mean": statistics.mean(vals), + "std": statistics.pstdev(vals) or 0.0001, + } + return stats + + +def build_profile(samples_dir, background=None): + paths = list(iter_docs(samples_dir)) + text = "\n\n".join(p.read_text(errors="replace") for p in paths) + profile = feature_bundle(text) + profile["function_word_background"] = background_stats(background) + profile["metadata"] = { + "doc_count": len(paths), + "total_words": profile["total_words"], + "low_confidence": profile["total_words"] < 2000, + "genre_warning": "profile has fewer than 2000 words" if profile["total_words"] < 2000 else "", + } + return profile + + +def parse_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument("samples_dir") + parser.add_argument("-o", "--output", required=True) + parser.add_argument("--background") + return parser.parse_args(argv) + + +def main(argv): + args = parse_args(argv) + root = Path(args.samples_dir) + if not root.is_dir(): + print(f"missing samples dir: {root}", file=sys.stderr) + return 2 + if not list(iter_docs(root)): + print(f"no sample documents in {root}: only .txt and .md files are read " + f"(recursively). Rename samples to .txt/.md or point at the right " + f"directory.", file=sys.stderr) + return 2 + profile = build_profile(root, args.background) + if profile["metadata"]["low_confidence"]: + print(profile["metadata"]["genre_warning"], file=sys.stderr) + Path(args.output).write_text(json.dumps(profile, indent=2, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/voice_score.py b/scripts/voice_score.py new file mode 100644 index 0000000..20868ff --- /dev/null +++ b/scripts/voice_score.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Score a candidate against a voice profile. + +Lower composite means more user-like. The composite is half GI-rank penalty and +half clipped, weighted impostor z-distance using the WP10a research weights. +""" + +import argparse +import json +import math +import random +import statistics +import sys +from pathlib import Path + +import voice_profile + +WEIGHTS = { + "char3": 0.30, + "delta": 0.25, + "sentence_emd": 0.10, + "punctuation": 0.08, + "contraction": 0.07, + "mtld": 0.10, + "word_length": 0.10, +} + + +def cosine_distance(a, b, keys=None): + keys = list(keys) if keys is not None else sorted(set(a) | set(b)) + dot = sum(a.get(k, 0.0) * b.get(k, 0.0) for k in keys) + na = math.sqrt(sum(a.get(k, 0.0) ** 2 for k in keys)) + nb = math.sqrt(sum(b.get(k, 0.0) ** 2 for k in keys)) + if not na or not nb: + return 1.0 + return 1 - (dot / (na * nb)) + + +def z_function_vector(freqs, bg, top): + keys = sorted(bg, key=lambda k: bg[k].get("mean", 0), reverse=True)[:top] + return {k: (freqs.get(k, 0.0) - bg[k]["mean"]) / (bg[k]["std"] or 0.0001) for k in keys} + + +def emd(a, b): + if not a or not b: + return 0.0 + max_len = max(max(a), max(b)) + ca = cb = dist = 0.0 + for i in range(1, max_len + 1): + ca += sum(1 for x in a if x == i) / len(a) + cb += sum(1 for x in b if x == i) / len(b) + dist += abs(ca - cb) + return dist + + +def l1(a, b, keys): + return sum(abs(a.get(k, 0.0) - b.get(k, 0.0)) for k in keys) + + +def distances(profile, feats, subset=None): + subset = set(subset or WEIGHTS) + out = {} + if "char3" in subset: + keys = set(profile["char3"]) | set(feats["char3"]) + out["char3"] = cosine_distance(profile["char3"], feats["char3"], keys) + if "delta" in subset: + top = 50 if feats["total_words"] < 300 else 200 + bg = profile["function_word_background"] + pv = z_function_vector(profile["function_words"], bg, top) + cv = z_function_vector(feats["function_words"], bg, top) + out["delta"] = cosine_distance(pv, cv, pv.keys()) + if "sentence_emd" in subset: + med = profile["sentence_lengths"].get("median") or 1.0 + out["sentence_emd"] = emd(profile["sentence_lengths"]["lengths"], feats["sentence_lengths"]["lengths"]) / med + if "punctuation" in subset: + out["punctuation"] = l1(profile["punctuation"], feats["punctuation"], voice_profile.PUNCT) + if "contraction" in subset: + out["contraction"] = abs(profile["contraction_rate"] - feats["contraction_rate"]) + if "mtld" in subset: + out["mtld"] = abs(profile["mtld"] - feats["mtld"]) / (profile["mtld"] or 1.0) + if "word_length" in subset: + keys = [str(i) for i in range(1, 16)] + out["word_length"] = l1(profile["word_length_histogram"], feats["word_length_histogram"], keys) + return out + + +def weighted_sum(dists): + return sum(WEIGHTS[k] * dists.get(k, 0.0) for k in WEIGHTS) + + +def impostor_features(root): + feats = [] + for path in voice_profile.iter_docs(root): + feats.append((str(path), voice_profile.feature_bundle(path.read_text(errors="replace")))) + return feats + + +def zscores(candidate, impostor_rows): + out = {} + for key in WEIGHTS: + vals = [row[key] for row in impostor_rows if key in row] + if key not in candidate or not vals: + out[key] = None + continue + mean = statistics.mean(vals) + std = statistics.pstdev(vals) or 0.0001 + out[key] = (candidate[key] - mean) / std + return out + + +def gi_score(profile, cand_feats, impostors, seed): + rng = random.Random(seed) + keys = list(WEIGHTS) + wins = 0 + trials = 64 + # Per-key distances depend only on (profile, feats) -- not on which subset + # of keys a given trial happens to draw -- so compute every key's distance + # ONCE per candidate/impostor here, then have each trial do a subset- + # weighted sum over the precomputed values instead of recomputing + # distances() from scratch on every trial. Arithmetically exact: the + # trial loop still consumes rng.random()/rng.sample() in the same order, + # and summing WEIGHTS[k]*value only over the drawn subset is the same + # float sequence weighted_sum(distances(..., subset)) produced (excluded + # keys contributed an exact 0.0 term either way). + cand_dists = distances(profile, cand_feats) + impostor_dists = [(name, distances(profile, imp)) for name, imp in impostors] + for _ in range(trials): + subset = [k for k in keys if rng.random() < 0.5] or [rng.choice(keys)] + cand = sum(WEIGHTS[k] * cand_dists.get(k, 0.0) for k in subset) + sampled = rng.sample(impostor_dists, k=min(len(impostor_dists), max(1, len(impostor_dists) // 2))) + if all(cand < sum(WEIGHTS[k] * imp_dists.get(k, 0.0) for k in subset) + for _, imp_dists in sampled): + wins += 1 + return wins / trials + + +def ngrams(tokens, n=4): + return set(tuple(tokens[i:i + n]) for i in range(max(0, len(tokens) - n + 1))) + + +LCS_THRESHOLD = 120 + + +def has_common_substring_over(a: str, b: str, min_length: int) -> bool: + """Rolling-hash check: do ``a`` and ``b`` share a contiguous substring of at + least ``min_length`` characters? + + O(len(a) + len(b)) expected time, vs. the O(len(a) * len(b)) classic DP a + true longest-common-substring computation needs. This only answers the + threshold question the copy gate actually asks ("is there a shared run + longer than N chars"); it does not recover the true longest common + substring length. Every hash match is verified against the source + characters before being trusted, so a hash collision never produces a + false positive. + """ + if min_length <= 0: + return bool(a) and bool(b) + if len(a) < min_length or len(b) < min_length: + return False + + base = 257 + mod = (1 << 61) - 1 + high_power = pow(base, min_length - 1, mod) + + def window_hashes(s: str) -> dict[int, list[int]]: + table: dict[int, list[int]] = {} + h = 0 + for i in range(min_length): + h = (h * base + ord(s[i])) % mod + table.setdefault(h, []).append(0) + for i in range(min_length, len(s)): + h = ((h - ord(s[i - min_length]) * high_power) * base + ord(s[i])) % mod + table.setdefault(h, []).append(i - min_length + 1) + return table + + table_a = window_hashes(a) + table_b = window_hashes(b) + for h, starts_b in table_b.items(): + starts_a = table_a.get(h) + if not starts_a: + continue + for sb in starts_b: + window_b = b[sb:sb + min_length] + for sa in starts_a: + if a[sa:sa + min_length] == window_b: + return True + return False + + +def copy_gate(candidate_text, samples_dir): + cand_grams = ngrams(voice_profile.words(candidate_text)) + max_overlap = 0.0 + lcs_violation = False + for path in voice_profile.iter_docs(samples_dir): + text = path.read_text(errors="replace") + sample_grams = ngrams(voice_profile.words(text)) + if cand_grams: + max_overlap = max(max_overlap, len(cand_grams & sample_grams) / len(cand_grams)) + if has_common_substring_over(candidate_text, text, LCS_THRESHOLD + 1): + lcs_violation = True + # "longest_common_substring" is reported for backward-compatible shape only: + # nothing pins its exact value (checked evals/adversarial-evals.json and + # evals/check_*.py for max_lcs/longest_common_substring assertions; none + # exist). Its semantics changed from "the true LCS length" to "the + # matched threshold window length when a violation-length run was found, + # else 0" -- computing the true max cheaply isn't possible without the + # same O(n*m) cost this function exists to avoid. + max_lcs = LCS_THRESHOLD + 1 if lcs_violation else 0 + return { + "max_overlap": max_overlap, + "longest_common_substring": max_lcs, + "violation": max_overlap > 0.35 or lcs_violation, + } + + +def parse_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--profile", required=True) + parser.add_argument("--impostors", required=True) + parser.add_argument("--seed", required=True, type=int) + parser.add_argument("--samples") + parser.add_argument("candidate_file") + return parser.parse_args(argv) + + +def read_candidate(path): + if path == "-": + return sys.stdin.read() + p = Path(path) + if not p.exists(): + raise FileNotFoundError(path) + return p.read_text(errors="replace") + + +def main(argv): + args = parse_args(argv) + if not Path(args.profile).exists() or not Path(args.impostors).is_dir(): + print("missing profile or impostors", file=sys.stderr) + return 2 + try: + text = read_candidate(args.candidate_file) + except FileNotFoundError as e: + print(f"missing candidate: {e}", file=sys.stderr) + return 2 + profile = json.loads(Path(args.profile).read_text()) + feats = voice_profile.feature_bundle(text) + low = feats["total_words"] < 150 + impostors = impostor_features(args.impostors) + dist = distances(profile, feats) + imp_dist = [distances(profile, f) for _, f in impostors] + zs = zscores(dist, imp_dist) + zsum = sum(WEIGHTS[k] * max(-3.0, min(3.0, zs[k] if zs[k] is not None else 0.0)) for k in WEIGHTS) + gi = gi_score(profile, feats, impostors, args.seed) if impostors else 0.0 + result = { + "candidate_words": feats["total_words"], + "low_confidence": low, + "distances": {k: (None if low and k in {"sentence_emd", "mtld"} else v) for k, v in dist.items()}, + "z_scores": {k: (None if low and k in {"sentence_emd", "mtld"} else v) for k, v in zs.items()}, + "gi": gi, + "composite": 0.5 * (1 - gi) + 0.5 * zsum, + } + if args.samples: + result["copy_gate"] = copy_gate(text, args.samples) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:]))