From cd1964f059e6a5560618f9e5bf4c6b716856e7a3 Mon Sep 17 00:00:00 2001 From: Clayton Kim Date: Fri, 24 Jul 2026 09:37:38 -0700 Subject: [PATCH] Add title claim-function verifier --- .../rubrics/title-claim-function/clean.json | 6 +++++ .../rubrics/title-claim-function/fire.json | 12 +++++++++ evals/rubric-coverage.json | 5 ++-- evals/run-rubrics.mjs | 1 + .../scripts/verify/checks.json | 25 +++++++++++++++++-- .../scripts/verify/rubrics/pass-copy.md | 14 +++++++++++ 6 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 evals/fixtures/rubrics/title-claim-function/clean.json create mode 100644 evals/fixtures/rubrics/title-claim-function/fire.json diff --git a/evals/fixtures/rubrics/title-claim-function/clean.json b/evals/fixtures/rubrics/title-claim-function/clean.json new file mode 100644 index 0000000..5852e65 --- /dev/null +++ b/evals/fixtures/rubrics/title-claim-function/clean.json @@ -0,0 +1,6 @@ +{ + "judge_response": { + "pass": true, + "findings": [] + } +} diff --git a/evals/fixtures/rubrics/title-claim-function/fire.json b/evals/fixtures/rubrics/title-claim-function/fire.json new file mode 100644 index 0000000..9b35319 --- /dev/null +++ b/evals/fixtures/rubrics/title-claim-function/fire.json @@ -0,0 +1,12 @@ +{ + "judge_response": { + "pass": false, + "findings": [ + { + "check_id": "title-claim-function", + "evidence": "Slide s3-12 is titled “Week 1 ends with a calibrated eval,” assigning outcome-producing agency to a calendar container instead of naming the completion evidence.", + "fix": "Name the criterion directly: “The Week 1 capstone is a clean run with recorded agreement.”" + } + ] + } +} diff --git a/evals/rubric-coverage.json b/evals/rubric-coverage.json index a2bbee1..6b62f4a 100644 --- a/evals/rubric-coverage.json +++ b/evals/rubric-coverage.json @@ -1,15 +1,16 @@ { - "note": "Tracks executable-fixture coverage for the 31 llm-pass/transcript catalog checks that evals/run.mjs cannot exercise (see plans/007-DESIGN-NOTE.md). Separate from evals/parity-allowlist.json, which is scoped to deterministic-stage (static-text/static-dom/browser) checks only. operating-model-fit also has a JSONL routing set covering slide and long-form section decisions.", + "note": "Tracks executable-fixture coverage for the 32 llm-pass/transcript catalog checks that evals/run.mjs cannot exercise (see plans/007-DESIGN-NOTE.md). Separate from evals/parity-allowlist.json, which is scoped to deterministic-stage (static-text/static-dom/browser) checks only. operating-model-fit also has a JSONL routing set covering slide and long-form section decisions.", "behavioral_fixture_sets": { "operating-model-fit": "evals/fixtures/rubrics/operating-model-fit/routing-cases.jsonl" }, - "total_llm_pass_and_transcript_checks": 31, + "total_llm_pass_and_transcript_checks": 32, "covered_by_evals_run_rubrics": [ "text-visibly-clipped", "fixed-ui-obscures-content", "diagram-type-coherent", "preset-both-mode-visual", "deck-content-completeness", + "title-claim-function", "operating-model-fit", "reel-caption-matches-transcript" ], diff --git a/evals/run-rubrics.mjs b/evals/run-rubrics.mjs index 07a12de..6f3567d 100644 --- a/evals/run-rubrics.mjs +++ b/evals/run-rubrics.mjs @@ -56,6 +56,7 @@ const SLICE = [ { id: 'diagram-type-coherent', mode: 'stub' }, { id: 'preset-both-mode-visual', mode: 'stub' }, { id: 'deck-content-completeness', mode: 'stub' }, + { id: 'title-claim-function', mode: 'stub' }, { id: 'operating-model-fit', mode: 'stub' }, { id: 'reel-caption-matches-transcript', mode: 'algorithm' }, ]; diff --git a/plugins/visual-explainer/scripts/verify/checks.json b/plugins/visual-explainer/scripts/verify/checks.json index 2db0067..f811c1f 100644 --- a/plugins/visual-explainer/scripts/verify/checks.json +++ b/plugins/visual-explainer/scripts/verify/checks.json @@ -1,7 +1,7 @@ { "version": 2, "generated": "2026-07-04", - "note": "v2: +37 AI-tell/copy checks from impeccable+unslop distillation. Stage counts: {\"static-text\":70,\"static-dom\":69,\"llm-pass\":29,\"browser\":37,\"transcript\":2}", + "note": "v2: AI-tell/copy checks from impeccable+unslop distillation. Stage counts: {\"static-text\":70,\"static-dom\":69,\"llm-pass\":30,\"browser\":37,\"transcript\":2}", "checks": [ { "family": "F1_slop_global", @@ -4054,6 +4054,27 @@ "ai-default-accent-hue" ] }, + { + "family": "F9_copy", + "id": "title-claim-function", + "title": "Slide and section titles state a test, mechanism, result, or decision", + "profiles": [ + "page", + "slides", + "magazine" + ], + "stage": "llm-pass", + "severity": "warn", + "applies_when": "The artifact has slide titles or section headings that do narrative work. Covers, literal dividers, agendas, quotes, and reference labels are reviewed under the exception rules.", + "spec": "pass: pass-copy. For each narrative title, ask whether it names the concrete test, mechanism, result, decision, action, or completion criterion the unit earns. Warn on calendar/program containers made to deliver outcomes ('Week 1 ends with a calibrated eval'), tool/data/score/package personification ('the judge lies', 'the score hides', 'the package outlives'), abstract uplift transformations ('taste becomes infrastructure'), journey/paradigm/pillars framing, generic topic handles when a stronger claim is present, or counting titles that describe deck structure instead of the point.", + "fixture_violation": "A slide titled 'Week 1 ends with a calibrated eval' when the body actually defines the completion evidence: a clean package run with judge agreement recorded.", + "fp_guards": "Do not ban inanimate subjects categorically. Accurate causal mechanisms ('Examples calibrate the judge', 'The gate fails the build'), literal temporal facts ('Week 1 ends on Friday'), exact technical noun phrases, short questions that state the decision, and cover/divider/agenda labels may pass. Judge title function against the unit's narrative job, not a fixed grammar template.", + "source_rules": [ + "title-states-test-mechanism-result-decision", + "no-category-error-title-agency", + "no-generic-title-handles" + ] + }, { "family": "F9_copy", "id": "copy-redundancy", @@ -4068,7 +4089,7 @@ "applies_when": "Page has section headings each followed by prose.", "spec": "pass: pass-copy. Rubric question: 'Does any section heading merely rephrase its own first sentence with no added fact/number/mechanism, or is the same explanatory point repeated across two or more sections?' YES = warn. A heading that just names its topic ('Architecture' over an architecture diagram) is fine — flag same information content restated, not a shared topic word.", "fixture_violation": "Heading 'Faster deploys' followed by 'Deploys are now faster.' with no new specifics.", - "fp_guards": "A topic sentence that states the claim AND immediately substantiates it with a number/mechanism passes. Only zero-added-specificity restatement or cross-section duplication fires.", + "fp_guards": "A topic sentence that states the claim AND immediately substantiates it with a number/mechanism passes. Only zero-added-specificity restatement or cross-section duplication fires here; a generic topic handle may still be flagged separately by title-claim-function.", "source_rules": [ "ai-redundant-heading-body-echo", "ai-redundant-restated-copy" diff --git a/plugins/visual-explainer/scripts/verify/rubrics/pass-copy.md b/plugins/visual-explainer/scripts/verify/rubrics/pass-copy.md index 14ab67a..a808c74 100644 --- a/plugins/visual-explainer/scripts/verify/rubrics/pass-copy.md +++ b/plugins/visual-explainer/scripts/verify/rubrics/pass-copy.md @@ -3,6 +3,7 @@ Inputs: - Exclude code, identifiers, filenames, table headers, labels, counters, timestamps, status strings, Mermaid labels, and square-bracket system messages. Questions: +- [title-claim-function] For each narrative slide title or section heading, does it name the concrete test, mechanism, result, decision, action, or completion criterion the unit earns? Flag calendar/program containers delivering outcomes, tool/data/score/package personification, abstract uplift transformations, journey/paradigm/pillars framing, generic topic handles when the body contains a stronger claim, and counting titles that describe deck structure instead of the point. - [unslop-prose-style] Beyond fixed phrases, does this prose show AI-slop stylistic tells such as manufactured emphasis, predictable three-item list rhythm, hollow however/moreover transitions, or empty summary sentences that `/unslop` would rewrite? - [copy-redundancy] Does any section heading merely rephrase its own first sentence with no added fact, number, or mechanism, or is the same explanatory point repeated across two or more sections? - [jargon-undefined] For a broad or onboarding audience, does the copy rely on two or more acronyms or internal system names that are load-bearing yet never expanded on first use, glossed, or otherwise defined? @@ -11,12 +12,25 @@ Verdict JSON schema: `{"pass":true,"findings":[{"check_id":"","evidence":"paragraph beginning ...","fix":"Run /unslop on the prose block and replace the rendered copy."}]}` Fail examples: +- A capstone slide says "Week 1 ends with a calibrated eval" instead of naming the evidence that completes the capstone. +- A bias slide says "Your judge is lying to someone" instead of naming order, verbosity, and self-preference as the mechanisms. +- A handoff slide says "The package outlives the cohort" instead of naming the routes or owner decision. +- A substantive slide is titled only "Architecture", "Proof plan", "Three pillars", or "What the budget buys" when its body contains a specific result or decision. - "In today's fast-paced landscape" opens a technical explainer. - Three adjacent cards all use the same "X, Y, and Z" rhythm. - A section ends with an empty recap sentence that adds no information. - A heading says "Faster deploys" and the first sentence only says "Deploys are now faster." - An onboarding explainer uses CRDT, WAL, and HLC without expanding or defining them. +Pass examples: +- "Examples calibrate the judge to your bar" describes a causal mechanism. +- "Week 1 ends on Friday" is a literal temporal fact. +- "Can one packet reconstruct both products?" states the decision test. +- A cover, divider, agenda, or exact technical reference label may use a concise literal title. + +Title review is semantic, not a blanket subject-verb ban. The title is part of +the explanatory model: it tells the viewer what to inspect. + ## De-Slop Rubric Addendum Instruction hierarchy: if `/unslop` is available, invoke it on drafted prose before writing HTML. This rubric is the fallback when `/unslop` is unavailable and the verification layer for rendered copy. Deterministic literals live in `copy-slop-phrases`; this pass covers judgment.