From 5ead78c558a6506e330c28248395dbfb2523a4ae Mon Sep 17 00:00:00 2001 From: Clayton Kim Date: Fri, 24 Jul 2026 06:39:25 -0700 Subject: [PATCH 1/2] Add conditional operating-model judge --- .../rubrics/operating-model-fit/clean.json | 36 +++++ .../rubrics/operating-model-fit/fire.json | 23 ++++ .../operating-model-fit/routing-cases.jsonl | 16 +++ evals/rubric-coverage.json | 8 +- evals/run-rubrics.mjs | 128 +++++++++++++++++- .../agents/ve-verifier-operating-model.md | 30 ++++ .../references/verification.md | 18 ++- .../scripts/verify/checks.json | 23 +++- .../scripts/verify/lib/report.mjs | 14 ++ .../verify/rubrics/pass-operating-model.md | 54 ++++++++ 10 files changed, 343 insertions(+), 7 deletions(-) create mode 100644 evals/fixtures/rubrics/operating-model-fit/clean.json create mode 100644 evals/fixtures/rubrics/operating-model-fit/fire.json create mode 100644 evals/fixtures/rubrics/operating-model-fit/routing-cases.jsonl create mode 100644 plugins/visual-explainer/.claude/agents/ve-verifier-operating-model.md create mode 100644 plugins/visual-explainer/scripts/verify/rubrics/pass-operating-model.md diff --git a/evals/fixtures/rubrics/operating-model-fit/clean.json b/evals/fixtures/rubrics/operating-model-fit/clean.json new file mode 100644 index 0000000..01cb437 --- /dev/null +++ b/evals/fixtures/rubrics/operating-model-fit/clean.json @@ -0,0 +1,36 @@ +{ + "judge_response": { + "pass": true, + "routing": [ + { + "unit_id": "cover", + "unit_type": "slide", + "treatment": "none", + "confidence": "high", + "reason": "The slide introduces the deck with one title." + }, + { + "unit_id": "review-economics", + "unit_type": "slide", + "treatment": "relational", + "confidence": "high", + "reason": "The claim is a repeated-cost versus one-time-cost comparison." + }, + { + "unit_id": "review-routing", + "unit_type": "slide", + "treatment": "operating-model", + "confidence": "high", + "reason": "The claim depends on policy-based routing." + }, + { + "unit_id": "clinical-ide", + "unit_type": "slide", + "treatment": "simulated-surface", + "confidence": "high", + "reason": "The physician's interaction with evidence and hypotheses is the claim." + } + ], + "findings": [] + } +} diff --git a/evals/fixtures/rubrics/operating-model-fit/fire.json b/evals/fixtures/rubrics/operating-model-fit/fire.json new file mode 100644 index 0000000..652accb --- /dev/null +++ b/evals/fixtures/rubrics/operating-model-fit/fire.json @@ -0,0 +1,23 @@ +{ + "judge_response": { + "pass": false, + "routing": [ + { + "unit_id": "review-routing", + "unit_type": "slide", + "treatment": "operating-model", + "confidence": "high", + "reason": "The claim depends on policy-based routing into distinct clinician queues." + } + ], + "findings": [ + { + "check_id": "operating-model-fit", + "unit_id": "review-routing", + "unit_type": "slide", + "evidence": "Four equal cards name review jobs but show no selection rule, mandatory path, protected probability sample, or deduplication.", + "fix": "Render the universal checks and policy router, then separate mandatory, representative, calibration, and frontier lanes." + } + ] + } +} diff --git a/evals/fixtures/rubrics/operating-model-fit/routing-cases.jsonl b/evals/fixtures/rubrics/operating-model-fit/routing-cases.jsonl new file mode 100644 index 0000000..e3ef33a --- /dev/null +++ b/evals/fixtures/rubrics/operating-model-fit/routing-cases.jsonl @@ -0,0 +1,16 @@ +{"id":"section-hero-single-claim","unit_type":"section","narrative_job":"Open a report with one proposition.","visible_claim":"Clinical reasoning is the scarce asset.","expected_treatment":"none","expected_applicable":false,"rationale":"A dominant claim does not need a relational model."} +{"id":"section-quote","unit_type":"section","narrative_job":"Let one expert observation carry the argument.","visible_claim":"A physician quote with attribution.","expected_treatment":"none","expected_applicable":false,"rationale":"A quote is evidence, not a system."} +{"id":"section-single-kpi","unit_type":"section","narrative_job":"Establish one baseline number.","visible_claim":"Universal review consumes five minutes per task.","expected_treatment":"none","expected_applicable":false,"rationale":"One number and one qualifier need hierarchy, not a diagram."} +{"id":"section-faq","unit_type":"section","narrative_job":"Answer implementation questions in plain language.","visible_claim":"Four independent questions and answers.","expected_treatment":"none","expected_applicable":false,"rationale":"Independent answers remain a list."} +{"id":"section-pricing-compare","unit_type":"section","narrative_job":"Compare internal annotation cost with external licensing value.","visible_claim":"Two price bases and their ratio.","expected_treatment":"relational","expected_applicable":true,"rationale":"Aligned quantities make the comparison inspectable."} +{"id":"section-before-after","unit_type":"section","narrative_job":"Show how review allocation changes after launch.","visible_claim":"Universal stamping becomes sampled review.","expected_treatment":"relational","expected_applicable":true,"rationale":"A paired composition exposes the change without implying a process."} +{"id":"section-benchmark-matrix","unit_type":"section","narrative_job":"Compare benchmark coverage and saturation.","visible_claim":"Tasks mapped against modalities and measured abilities.","expected_treatment":"relational","expected_applicable":true,"rationale":"A matrix is the faithful form for exact mappings."} +{"id":"section-review-router","unit_type":"section","narrative_job":"Explain which work reaches clinicians and why.","visible_claim":"Policy routes cases into mandatory, representative, calibration, and frontier queues.","expected_treatment":"operating-model","expected_applicable":true,"rationale":"The selection rules and separate queues are the claim."} +{"id":"section-state-transition","unit_type":"section","narrative_job":"Explain how a case moves from evidence collection to adjudication.","visible_claim":"Case states have explicit gates and abstention paths.","expected_treatment":"operating-model","expected_applicable":true,"rationale":"State and gate semantics must remain visible."} +{"id":"section-causal-feedback","unit_type":"section","narrative_job":"Show how clinician labels improve future sampling.","visible_claim":"Errors update the judge, which changes the next frontier sample.","expected_treatment":"operating-model","expected_applicable":true,"rationale":"The feedback loop is causal, not a list of peers."} +{"id":"section-resource-to-gate","unit_type":"section","narrative_job":"Connect clinician hours to evidence and release decisions.","visible_claim":"Annotation creates eval evidence that controls model release.","expected_treatment":"operating-model","expected_applicable":true,"rationale":"The resource-to-decision chain is the strategic argument."} +{"id":"section-clinical-workspace","unit_type":"section","narrative_job":"Show how a physician investigates a multimodal case.","visible_claim":"A physician manipulates evidence, hypotheses, and next actions in one workspace.","expected_treatment":"simulated-surface","expected_applicable":true,"rationale":"The interaction model is more important than an abstract workflow."} +{"id":"slide-cover","unit_type":"slide","narrative_job":"Name the argument.","visible_claim":"Evals are strategic IP.","expected_treatment":"none","expected_applicable":false,"rationale":"The cover needs one focal statement."} +{"id":"slide-break-even","unit_type":"slide","narrative_job":"Make the review-cost threshold legible.","visible_claim":"Per-task cost times volume compared with benchmark investment.","expected_treatment":"relational","expected_applicable":true,"rationale":"An equation and paired totals are sufficient."} +{"id":"slide-review-router","unit_type":"slide","narrative_job":"Explain the new clinician allocation policy.","visible_claim":"Only selected cases reach distinct expert queues.","expected_treatment":"operating-model","expected_applicable":true,"rationale":"Policy routing is the claim."} +{"id":"slide-clinical-ide","unit_type":"slide","narrative_job":"Make the future training environment concrete.","visible_claim":"Physicians reason over multimodal evidence and probability-bearing hypotheses.","expected_treatment":"simulated-surface","expected_applicable":true,"rationale":"The product surface is the most faithful explanation."} diff --git a/evals/rubric-coverage.json b/evals/rubric-coverage.json index 9677a05..a2bbee1 100644 --- a/evals/rubric-coverage.json +++ b/evals/rubric-coverage.json @@ -1,12 +1,16 @@ { - "note": "Tracks executable-fixture coverage for the 30 llm-pass/transcript catalog checks that evals/run.mjs cannot exercise (see plans/007-DESIGN-NOTE.md). Separate from evals/parity-allowlist.json, which is scoped to deterministic-stage (static-text/static-dom/browser) checks only.", - "total_llm_pass_and_transcript_checks": 30, + "note": "Tracks executable-fixture coverage for the 31 llm-pass/transcript catalog checks that evals/run.mjs cannot exercise (see plans/007-DESIGN-NOTE.md). Separate from evals/parity-allowlist.json, which is scoped to deterministic-stage (static-text/static-dom/browser) checks only. operating-model-fit also has a JSONL routing set covering slide and long-form section decisions.", + "behavioral_fixture_sets": { + "operating-model-fit": "evals/fixtures/rubrics/operating-model-fit/routing-cases.jsonl" + }, + "total_llm_pass_and_transcript_checks": 31, "covered_by_evals_run_rubrics": [ "text-visibly-clipped", "fixed-ui-obscures-content", "diagram-type-coherent", "preset-both-mode-visual", "deck-content-completeness", + "operating-model-fit", "reel-caption-matches-transcript" ], "remaining_uncovered": [ diff --git a/evals/run-rubrics.mjs b/evals/run-rubrics.mjs index 770ee76..07a12de 100644 --- a/evals/run-rubrics.mjs +++ b/evals/run-rubrics.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Regression suite for the 30 llm-pass/transcript catalog checks that +// Regression suite for the 31 llm-pass/transcript catalog checks that // evals/run.mjs cannot exercise (it filters to static-text/static-dom/browser // stages by design — see plans/007-DESIGN-NOTE.md for why). // @@ -28,10 +28,16 @@ import { readFileSync, readdirSync, existsSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { buildReport } from '../plugins/visual-explainer/scripts/verify/lib/report.mjs'; const EVAL_ROOT = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(EVAL_ROOT, '..'); const RUBRICS_FIXTURES_ROOT = join(EVAL_ROOT, 'fixtures', 'rubrics'); +const OPERATING_MODEL_ROUTING_CASES = join( + RUBRICS_FIXTURES_ROOT, + 'operating-model-fit', + 'routing-cases.jsonl', +); const checksCatalog = JSON.parse( readFileSync(resolve(REPO_ROOT, 'plugins/visual-explainer/scripts/verify/checks.json'), 'utf8'), @@ -39,7 +45,8 @@ const checksCatalog = JSON.parse( const severityById = new Map(checksCatalog.map((check) => [check.id, check.severity])); -// The first implementation slice from plan 007 step 3: 6 checks, the +// The first implementation slice from plan 007 step 3, extended with the +// operating-model routing contract: 7 checks. The // 13 error-severity uncovered checks are the priority pool this was drawn // from. `mode: 'stub'` checks are asserted via deriveStatus() against a // canned judge_response fixture; `mode: 'algorithm'` runs the real scorer. @@ -49,6 +56,7 @@ const SLICE = [ { id: 'diagram-type-coherent', mode: 'stub' }, { id: 'preset-both-mode-visual', mode: 'stub' }, { id: 'deck-content-completeness', mode: 'stub' }, + { id: 'operating-model-fit', mode: 'stub' }, { id: 'reel-caption-matches-transcript', mode: 'algorithm' }, ]; @@ -154,6 +162,118 @@ function runTranscriptCheck(id) { return rows; } +// --- Operating-model contract: routing set + section pass selection -------- + +function readJsonl(path) { + return readFileSync(path, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line); + } catch (error) { + throw new Error(`${path}:${index + 1}: ${error.message}`); + } + }); +} + +function runOperatingModelRoutingCases() { + const allowedTreatments = new Set([ + 'none', + 'relational', + 'operating-model', + 'simulated-surface', + ]); + const allowedUnitTypes = new Set(['slide', 'section']); + const cases = readJsonl(OPERATING_MODEL_ROUTING_CASES); + const seenIds = new Set(); + const rows = []; + + for (const fixture of cases) { + const errors = []; + if (!fixture.id || seenIds.has(fixture.id)) errors.push('id must be present and unique'); + if (!allowedUnitTypes.has(fixture.unit_type)) errors.push('unit_type must be slide or section'); + if (!allowedTreatments.has(fixture.expected_treatment)) errors.push('invalid expected_treatment'); + if (typeof fixture.expected_applicable !== 'boolean') errors.push('expected_applicable must be boolean'); + if (fixture.expected_applicable !== (fixture.expected_treatment !== 'none')) { + errors.push('expected_applicable must be false only for treatment=none'); + } + if (!fixture.narrative_job || !fixture.visible_claim || !fixture.rationale) { + errors.push('narrative_job, visible_claim, and rationale are required'); + } + seenIds.add(fixture.id); + rows.push({ + id: 'operating-model-routing-case', + caseName: fixture.id || 'missing-id', + status: errors.length ? 'FAIL' : 'pass', + message: errors.length + ? errors.join('; ') + : `${fixture.unit_type} -> ${fixture.expected_treatment}`, + }); + } + + const coverageAssertions = [ + ['covers-slide-units', cases.some((fixture) => fixture.unit_type === 'slide')], + ['covers-section-units', cases.some((fixture) => fixture.unit_type === 'section')], + ...Array.from(allowedTreatments, (treatment) => [ + `covers-${treatment}`, + cases.some((fixture) => fixture.expected_treatment === treatment), + ]), + ]; + for (const [caseName, pass] of coverageAssertions) { + rows.push({ + id: 'operating-model-routing-coverage', + caseName, + status: pass ? 'pass' : 'FAIL', + message: pass ? 'covered' : 'missing', + }); + } + return rows; +} + +function runOperatingModelPassSelectionCases() { + const operatingModelCheck = { + id: 'operating-model-fit', + stage: 'llm-pass', + status: 'llm-required', + severity: 'warn', + }; + const cases = [ + { + name: 'sectioned-page', + ctx: { profile: 'page', preset: 'custom', presetHint: '', html: '
Case review
', filePath: 'sectioned.html' }, + expected: true, + }, + { + name: 'simple-page', + ctx: { profile: 'page', preset: 'custom', presetHint: '', html: '
One statement
', filePath: 'simple.html' }, + expected: false, + }, + { + name: 'vertical-deck', + ctx: { profile: 'slides', preset: 'custom', presetHint: '', html: '
', filePath: 'slides.html' }, + expected: true, + }, + { + name: 'presentation-stage', + ctx: { profile: 'page', preset: 'custom', presetHint: '', html: '
', filePath: 'stage.html' }, + expected: true, + }, + ]; + + return cases.map(({ name, ctx, expected }) => { + const report = buildReport(ctx, [operatingModelCheck]); + const actual = report.llm_passes_required.includes('operating-model'); + return { + id: 'operating-model-pass-selection', + caseName: name, + status: actual === expected ? 'pass' : 'FAIL', + message: `expected=${expected} actual=${actual}`, + }; + }); +} + // --- Driver ------------------------------------------------------------ const catalogIds = new Set(checksCatalog.map((check) => check.id)); @@ -168,6 +288,8 @@ for (const entry of SLICE) { const rows = entry.mode === 'algorithm' ? runTranscriptCheck(entry.id) : runStubCheck(entry.id); allRows.push(...rows); } +allRows.push(...runOperatingModelRoutingCases()); +allRows.push(...runOperatingModelPassSelectionCases()); console.log('check_id,case,status,detail'); for (const row of allRows) { @@ -184,4 +306,4 @@ if (failures.length) { } const checksCovered = new Set(allRows.map((row) => row.id)).size; -console.log(`\nAll ${allRows.length} assertions passed across ${checksCovered} checks (fire + no-fire each).`); +console.log(`\nAll ${allRows.length} assertions passed across ${checksCovered} check families.`); diff --git a/plugins/visual-explainer/.claude/agents/ve-verifier-operating-model.md b/plugins/visual-explainer/.claude/agents/ve-verifier-operating-model.md new file mode 100644 index 0000000..1b49dab --- /dev/null +++ b/plugins/visual-explainer/.claude/agents/ve-verifier-operating-model.md @@ -0,0 +1,30 @@ +--- +name: ve-verifier-operating-model +description: Route each slide or coherent page section to the minimum explanatory treatment and judge operating-model fidelity only where applicable. Returns only verdict JSON. +tools: Read +--- + +# ve-verifier-operating-model + +Run one verification pass. Do not fix files. + +## Read + +1. `plugins/visual-explainer/scripts/verify/rubrics/pass-operating-model.md` +2. One screenshot per review unit: slide, presented stage, or coherent page section. +3. The review map supplied by the orchestrator: unit id, unit type, visible title, and one-sentence narrative job. +4. Only the brief excerpts needed to resolve a unit's intended decision. + +Route every unit before judging it. Do not penalize `none` units for lacking a +diagram. Do not infer intent from implementation source. + +## Output + +Return only one valid JSON object matching the rubric schema: + +```json +{"pass":true,"routing":[],"findings":[]} +``` + +Do not include prose, markdown, code fences, summaries, or recommendations +outside `routing` and `findings`. diff --git a/plugins/visual-explainer/references/verification.md b/plugins/visual-explainer/references/verification.md index 99c2892..64e45a0 100644 --- a/plugins/visual-explainer/references/verification.md +++ b/plugins/visual-explainer/references/verification.md @@ -58,7 +58,8 @@ In single-agent mode, YOU produce the declared inputs before running each rubric 2. Build the P-completeness source-vs-render inventory from the source material and the rendered artifact: headings, bullets, table rows, cards, collapsible details, footnotes, and demo-frame summaries. Do not use screenshots for this pass. 3. Capture P-diagram inputs one figure at a time. Scroll to each element carrying `data-diagram-role` or `.mermaid`; screenshot that element's bounding region; extract its visible labels; write a one-line content brief for the figure. -4. Use candidate element lists from each deterministic check's `evidence` and `where` fields when a pass asks for candidate extracts. +4. For P-operating-model, build a review map with one row per slide or coherent long-form section: stable unit id, unit type (`slide` or `section`), visible title, one-sentence narrative job, and screenshot path. If the source has no narrative job, use the visible claim and mark the route low-confidence. Do not use implementation source to infer intent. +5. Use candidate element lists from each deterministic check's `evidence` and `where` fields when a pass asks for candidate extracts. ## 2. Run LLM Passes @@ -86,6 +87,7 @@ Use these passes: | P-layout | `llm_passes_required` includes `hierarchy`, or any layout candidate exists | `report.json`; the 4 standard screenshots; candidate lists for clipping, fixed chrome, div-grid tables, slide screenshots, demo frames when referenced | `{{skill_dir}}/scripts/verify/rubrics/pass-layout.md` | | P-aesthetic | `llm_passes_required` includes any `aesthetic-*` token, a preset is detected or declared, or any preset candidate exists | `report.json`; active preset name; light/dark screenshots; candidate extracts named in the report | `{{skill_dir}}/scripts/verify/rubrics/pass-aesthetic.md` | | P-diagram | diagrams are present | `report.json`; one screenshot per figure; extracted diagram labels; one-line content brief for each figure | `{{skill_dir}}/scripts/verify/rubrics/pass-diagram.md` | +| P-operating-model | `llm_passes_required` includes `operating-model` | one screenshot per review unit; review map with unit id, unit type, visible title, and one-sentence narrative job; only the brief excerpts needed to resolve intent | `{{skill_dir}}/scripts/verify/rubrics/pass-operating-model.md` | | P-completeness | source material or demo evidence exists | source inventory; extracted rendered headings, bullets, table rows, cards, and demo-frame summary; no page screenshots | `{{skill_dir}}/scripts/verify/rubrics/pass-completeness.md` | | P-copy | extracted prose exists | excluded-filtered prose text only | `{{skill_dir}}/scripts/verify/rubrics/pass-copy.md` | | P-visual-tells | `llm_passes_required` includes `visual-tells` (page/slides/magazine) | `report.json`; light/dark screenshots; candidate extracts | `{{skill_dir}}/scripts/verify/rubrics/pass-visual-tells.md` | @@ -103,6 +105,20 @@ Load only the active preset section in `pass-aesthetic.md`. Do not judge Nothing Run only for rendered figures. Use per-figure screenshots, not one full-page screenshot. Apply the diagram Removal Test before emit; after emit, inspect screenshots. If a hand-authored SVG still fails after 2 repair attempts, replace the figure with Mermaid instead of continuing manual repair. +### P-operating-model Questions + +Route every slide or coherent page section before judging it: + +- `none`: cover, divider, quote, image, agenda, simple CTA, single statistic, or one claim with one supporting fact; +- `relational`: comparison, before/after, formula, aligned evidence, or exact mappings; +- `operating-model`: sequence, routing, state, causality, provenance, uncertainty, dependencies, feedback, or resource flow; and +- `simulated-surface`: the workflow or interface itself is the claim. + +Apply model-fidelity questions only to the final three routes. Do not require a +diagram when a table, formula, paired contrast, or aligned rows tell the truth. +Return a finding only when the chosen or missing structure materially weakens +or misstates the unit's narrative job. + ### P-completeness Questions Compare source inventory to extracted rendered content. Do not use screenshots. Missing source sections, decision cards, table rows, collapsible details, footnotes, or demonstrably low-value demo embeds fail this pass. diff --git a/plugins/visual-explainer/scripts/verify/checks.json b/plugins/visual-explainer/scripts/verify/checks.json index 12ef184..2db0067 100644 --- a/plugins/visual-explainer/scripts/verify/checks.json +++ b/plugins/visual-explainer/scripts/verify/checks.json @@ -1,7 +1,7 @@ { "version": 2, "generated": "2026-07-04", - "note": "v2: +37 AI-tell/copy checks from impeccable+unslop distillation. Stage counts: {\"static-text\":70,\"static-dom\":69,\"llm-pass\":28,\"browser\":37,\"transcript\":2}", + "note": "v2: +37 AI-tell/copy checks from impeccable+unslop distillation. Stage counts: {\"static-text\":70,\"static-dom\":69,\"llm-pass\":29,\"browser\":37,\"transcript\":2}", "checks": [ { "family": "F1_slop_global", @@ -2450,6 +2450,27 @@ "presentation-focal-point-and-contrast" ] }, + { + "family": "F5_slides", + "id": "operating-model-fit", + "title": "Review units use the minimum faithful explanatory treatment", + "profiles": [ + "page", + "slides", + "magazine" + ], + "stage": "llm-pass", + "severity": "warn", + "applies_when": "The artifact is a vertical deck, magazine deck, fixed-stage PresentationDeck, or long-form page with coherent sections. The pass routes every slide or section before judging model fidelity.", + "spec": "pass: pass-operating-model. First route each review unit to none, relational, operating-model, or simulated-surface from its narrative job. Judge only relational, operating-model, and simulated-surface units. Warn when the composition materially hides or misstates a required comparison, sequence, routing policy, state transition, causal/evidence relationship, uncertainty, dependency, feedback loop, or resource flow; also warn when an elaborate diagram is imposed on a unit that needs only a table, sentence, quote, image, or single number.", + "fixture_violation": "A clinician-routing policy rendered as four equal cards with no visible selection rule or separation between mandatory review, representative measurement, calibration, and frontier discovery.", + "fp_guards": "Do not require this treatment on covers, heroes, dividers, quotes, full-bleed images, agendas, FAQs, simple CTAs, single statistics, or one-claim/one-fact units. Lists and genuine peer sets may remain cards. A table, formula, or paired contrast may satisfy a relational unit. Route vague or underspecified units downward rather than inventing an operating model.", + "source_rules": [ + "operating-model-first", + "layout-is-a-domain-claim", + "minimum-faithful-treatment" + ] + }, { "family": "F5_slides", "id": "composition-variety-visual", diff --git a/plugins/visual-explainer/scripts/verify/lib/report.mjs b/plugins/visual-explainer/scripts/verify/lib/report.mjs index 4cefe8f..0436820 100644 --- a/plugins/visual-explainer/scripts/verify/lib/report.mjs +++ b/plugins/visual-explainer/scripts/verify/lib/report.mjs @@ -49,5 +49,19 @@ function llmPassesFor(ctx, checks) { required.add('copy'); if (['page', 'slides', 'magazine'].includes(ctx.profile)) required.add('visual-tells'); } + const hasReviewUnits = + ['slides', 'magazine'].includes(ctx.profile) || + /data-ve-presentation/i.test(ctx.html) || + /)/i.test(ctx.html); + if ( + hasReviewUnits && + checks.some( + (check) => + check.id === 'operating-model-fit' && + check.status === 'llm-required', + ) + ) { + required.add('operating-model'); + } return Array.from(required); } diff --git a/plugins/visual-explainer/scripts/verify/rubrics/pass-operating-model.md b/plugins/visual-explainer/scripts/verify/rubrics/pass-operating-model.md new file mode 100644 index 0000000..596e0aa --- /dev/null +++ b/plugins/visual-explainer/scripts/verify/rubrics/pass-operating-model.md @@ -0,0 +1,54 @@ +Inputs: +- One screenshot per review unit: a slide, presented stage, or coherent long-form page section. +- A review map containing, for each unit: stable unit id, unit type (`slide` or `section`), visible title, and one-sentence narrative job. If no narrative job exists, use the visible claim and mark routing confidence `low`. +- Optional source brief excerpts that define a unit's intended decision. Do not read implementation source. + +Route before judging: + +Assign exactly one treatment to every review unit. + +- `none`: The unit is a cover, hero, divider, quote, full-bleed image, agenda, FAQ, simple CTA, single statistic, or one claim with one supporting fact. It does not need a relational model. +- `relational`: The claim depends on comparison, before/after, a formula, aligned evidence, or a small set of exact mappings. A table, paired composition, or annotated equation may be sufficient. +- `operating-model`: The claim depends on sequence, routing, state transitions, causality, provenance, uncertainty or probability, dependencies, feedback, or a resource-to-output flow. +- `simulated-surface`: The workflow or interface itself is the claim, so a product/environment view is more faithful than an abstract diagram. Use this rarely. + +Routing rules: + +- Ask whether the viewer must understand relationships among multiple elements to accept the unit's claim. If no, route `none`. +- Do not escalate because a unit contains three or more items. Lists and genuine peer sets may remain cards or rows. +- Do not require a diagram when a table, formula, or paired contrast is more faithful. +- Do not infer an elaborate system from a vague topic title. When intent is missing, prefer the lower treatment and mark confidence `low`. +- The treatment names explanatory depth, not visual ambition. A well-made `none` unit is not lesser work. + +Questions for units routed `relational`, `operating-model`, or `simulated-surface`: + +- [operating-model-fit] Does the composition faithfully expose the relationships the viewer needs for the unit's narrative job? +- Does the visual grammar make a truthful claim? Cards imply independent peers; a pipeline implies ordered transformation; a funnel implies attrition; a tree implies hierarchy or prerequisites; a graph implies meaningful relationships; a control plane implies policy routing; a trace implies sequence and provenance. +- Can the viewer reconstruct the relevant comparison, sequence, routing rule, state change, evidence path, or dependency without presenter narration? +- Does each load-bearing position, connector, line style, size, color, or animation have one stable meaning? +- When uncertainty, abstention, residual probability, or an unsupported state is material, does the unit keep it visible? +- Is the treatment proportionate? Flag both generic-card flattening and a full systems diagram used for a claim that only needs a table or sentence. + +Failure threshold: + +Return a finding only when the selected or missing structure materially weakens or misstates the unit's claim. Do not fail a unit merely because it has no diagram, uses cards, or chooses a lower treatment than another unit. + +Verdict JSON schema: + +`{"pass":true,"routing":[{"unit_id":"s01","unit_type":"slide|section","treatment":"none|relational|operating-model|simulated-surface","confidence":"low|medium|high","reason":"..."}],"findings":[{"check_id":"operating-model-fit","unit_id":"s04","unit_type":"slide|section","evidence":"...","fix":"..."}]}` + +Pass examples: + +- A cover slide routes `none` and uses one dominant title. +- A report hero or FAQ section routes `none`; neither needs a system diagram. +- A benchmark-matrix section routes `relational` and uses aligned rows. +- A review-cost slide routes `relational` and shows the repeated unit, equation, and break-even without a systems diagram. +- A clinician-review slide routes `operating-model` and shows the policy router separating mandatory review, representative sampling, calibration, and frontier queues. +- A clinical IDE slide routes `simulated-surface` because the physician's interaction with evidence, hypotheses, and next actions is the claim. + +Fail examples: + +- A routing policy appears as four equal cards, leaving no selection rule or separation between mandatory review and sampled measurement. +- A causal feedback loop is drawn as a one-way pipeline. +- A probability-bearing diagnostic graph presents named diagnoses without residual uncertainty. +- A quote slide is forced into a node graph even though the relationship adds no information. From 0fefccc352d33e8644413c535f15a6ccb304d8ff Mon Sep 17 00:00:00 2001 From: Clayton Kim Date: Fri, 24 Jul 2026 08:46:24 -0700 Subject: [PATCH 2/2] Deduplicate React in external deck exports --- package-lock.json | 18 +++--- package.json | 6 +- .../ve-mdx/export-react-singleton.test.mjs | 58 +++++++++++++++++++ scripts/ve-mdx/export.mjs | 7 +++ 4 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 scripts/ve-mdx/export-react-singleton.test.mjs diff --git a/package-lock.json b/package-lock.json index 9e218d4..f94a407 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,8 @@ "@mdx-js/rollup": "^3.1.1", "@tailwindcss/vite": "^4.3.0", "@vitejs/plugin-react": "^6.0.1", - "react": "^19.2.6", - "react-dom": "^19.2.6", + "react": "19.2.7", + "react-dom": "19.2.7", "tailwindcss": "^4.3.0", "typescript": "^6.0.3", "vite": "^8.0.12" @@ -3293,24 +3293,24 @@ } }, "node_modules/react": { - "version": "19.2.6", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.6.tgz", - "integrity": "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q==", + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/react-dom": { - "version": "19.2.6", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.6.tgz", - "integrity": "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g==", + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", "license": "MIT", "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^19.2.6" + "react": "^19.2.7" } }, "node_modules/recma-build-jsx": { diff --git a/package.json b/package.json index 22e1a0d..8d27840 100644 --- a/package.json +++ b/package.json @@ -21,14 +21,14 @@ "ve:eval-rubrics": "node evals/run-rubrics.mjs", "check:manifests": "node scripts/check-manifests.mjs", "ve:learn": "node scripts/ve-mdx/learn.mjs", - "test": "node --test visual-explainer-mdx/diagram-layout.test.mjs visual-explainer-mdx/presentation-core.test.mjs scripts/ve-mdx/design-systems.test.mjs scripts/ve-mdx/learn-extractors.test.mjs" + "test": "node --test visual-explainer-mdx/diagram-layout.test.mjs visual-explainer-mdx/presentation-core.test.mjs scripts/ve-mdx/design-systems.test.mjs scripts/ve-mdx/export-react-singleton.test.mjs scripts/ve-mdx/learn-extractors.test.mjs" }, "dependencies": { "@mdx-js/rollup": "^3.1.1", "@tailwindcss/vite": "^4.3.0", "@vitejs/plugin-react": "^6.0.1", - "react": "^19.2.6", - "react-dom": "^19.2.6", + "react": "19.2.7", + "react-dom": "19.2.7", "tailwindcss": "^4.3.0", "typescript": "^6.0.3", "vite": "^8.0.12" diff --git a/scripts/ve-mdx/export-react-singleton.test.mjs b/scripts/ve-mdx/export-react-singleton.test.mjs new file mode 100644 index 0000000..e775e4e --- /dev/null +++ b/scripts/ve-mdx/export-react-singleton.test.mjs @@ -0,0 +1,58 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); + +test('client export dedupes React when the source lives in another package tree', () => { + const externalRoot = mkdtempSync(path.join(os.tmpdir(), 'artifacture-react-singleton-')); + try { + mkdirSync(path.join(externalRoot, 'node_modules'), { recursive: true }); + cpSync( + path.join(REPO_ROOT, 'node_modules/react'), + path.join(externalRoot, 'node_modules/react'), + { recursive: true }, + ); + + const source = path.join(externalRoot, 'external-deck.tsx'); + const output = path.join(externalRoot, 'external-deck.html'); + writeFileSync( + source, + `import React, { useState } from 'react'; + +export default function ExternalDeck() { + const [slide] = useState('one'); + return
External deck
; +} +`, + ); + + const result = spawnSync( + process.execPath, + ['scripts/ve-mdx/export.mjs', source, '--out', output, '--draft'], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + + const html = readFileSync(output, 'utf8'); + const hookImplementations = html.match(/useState=function/g) ?? []; + assert.equal( + hookImplementations.length, + 1, + `expected one React hook implementation, found ${hookImplementations.length}`, + ); + } finally { + rmSync(externalRoot, { recursive: true, force: true }); + } +}); diff --git a/scripts/ve-mdx/export.mjs b/scripts/ve-mdx/export.mjs index 16b1a59..ff7b4af 100644 --- a/scripts/ve-mdx/export.mjs +++ b/scripts/ve-mdx/export.mjs @@ -97,6 +97,13 @@ async function main() { base: './', logLevel: 'warn', plugins: [veMdxPreflightPlugin(source, draft), mdx(), react(), tailwindcss()], + // A source outside the Artifacture package can have its own physical + // React install. Without dedupe, the generated bundle renders with + // Artifacture's ReactDOM while hooks import the source package's React, + // leaving that second copy's dispatcher null at runtime. + resolve: { + dedupe: ['react', 'react-dom', 'react/jsx-runtime', 'react/jsx-dev-runtime'], + }, server: { fs: { allow: [repoRoot, tmp],