From 942bb2531e5257eeb3cd69373661dac6f3fc2f4d Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 31 Aug 2026 18:50:18 +0400 Subject: [PATCH 1/3] fix(context): remove report-generation qualification tuning Implements #660, Slice C. Madar activated a set of behaviours whenever a prompt merely LOOKED like the qualification report-generation task. Over a repository with no such structure, report vocabulary alone pulled the retrieval gate to runtime_generation and made the pack assert a planner/research/assembly/scoring/rendering/persistence workflow that no evidence supported. Removed from exactly three production files: - prompt-pack.ts: the `promptWantsReportGenerationCore` task-phrase classifier, the fixed report workflow instruction, and a phase-label table whose five keys no contract builder anywhere in the repository ever emitted. Instructions are now derived from the typed `answer_contract` alone, so the question text is no longer an input to instruction generation at all. - retrieve/slicing.ts: the duplicated classifier, the `semanticGenerationCoreAnchorValue` name-driven score table, forced anchor membership and its `generation core heuristic` reason, the raised anchor cap, the report-only deep backward slice policy, the report-only route-predecessor suppression, and report-stage and qualification-repository vocabulary in the node-name and prompt classifiers. - retrieval-gate.ts: the `reportGenerationShaped` variant. Gate outcomes now follow generic evidence only. `report_generation_shaped` is retained as a constant `false`. It is a required member of the published `RetrievalGenerationDebugSignals` shape, so removing it would need a fourth production file and an artifact-schema change; the true branch is unreachable and nothing reads it to reach a gate outcome. Generic behaviour is preserved and measured, not assumed. `persistence` stays in the runtime-pipeline prompt classifier because removing it regressed a neutral login/persistence fixture that contains no report vocabulary; the report-stage names did not come back. The typed answer-contract instruction path is byte-identical before and after, and the pre-existing prompt-pack parity golden is unchanged. Tests that locked the removed behaviour are converted into independence controls rather than re-snapshotted, and a new control file proves that report vocabulary, report-shaped symbol names and report-shaped paths each buy nothing. Four focused injections restore one retired rule apiece from a digest-checked byte snapshot and require the named control to fail. The forbidden-knowledge manifest gains six narrow distinctive rules. The scanner parsing architecture, static evaluator, regex capability boundary, normalization model and production-exception policy are unchanged. #660 remains open until Slice C is merged and post-merge verified. #661 is not started. --- package.json | 1 + scripts/lib/forbidden-knowledge-manifest.json | 10 +- .../verify-report-generation-injections.mjs | 219 +++++++++++ src/infrastructure/prompt-pack.ts | 47 +-- src/runtime/retrieval-gate.ts | 12 +- src/runtime/retrieve/slicing.ts | 107 +----- tests/unit/compare.test.ts | 13 +- tests/unit/pack-quality-fixtures.test.ts | 24 +- .../report-generation-independence.test.ts | 342 ++++++++++++++++++ tests/unit/retrieval-gate.test.ts | 16 +- .../retrieve-production-correctness.test.ts | 136 ++++--- 11 files changed, 699 insertions(+), 228 deletions(-) create mode 100755 scripts/verify-report-generation-injections.mjs create mode 100644 tests/unit/report-generation-independence.test.ts diff --git a/package.json b/package.json index 7080d281..753cb82d 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "verify:grader-boundary-controls": "node scripts/verify-grader-boundary.mjs --self-test", "verify:forbidden-knowledge": "node scripts/verify-forbidden-knowledge.mjs", "verify:forbidden-knowledge-controls": "node scripts/verify-forbidden-knowledge.mjs --self-test", + "verify:report-generation-injections": "node scripts/verify-report-generation-injections.mjs", "verify:grader-boundary-runtime": "node scripts/verify-grader-boundary-runtime.mjs", "verify:integrity-mutations": "node scripts/verify-integrity-mutations.mjs && MADAR_MUTATION_HARNESS_E2E=1 npx vitest run tests/unit/mutation-harness-self.test.ts", "verify:integrity-receipts": "node scripts/verify-integrity-receipts.mjs", diff --git a/scripts/lib/forbidden-knowledge-manifest.json b/scripts/lib/forbidden-knowledge-manifest.json index 6c8b4cb2..d5d8da84 100644 --- a/scripts/lib/forbidden-knowledge-manifest.json +++ b/scripts/lib/forbidden-knowledge-manifest.json @@ -1,7 +1,7 @@ { "manifest_version": "1.0.0", "issue": 660, - "slice": "B", + "slice": "B+C", "title": "Production independence from qualification-repository knowledge", "purpose": [ "Madar's retrieval and claim behaviour must be driven by generic structural evidence,", @@ -54,7 +54,13 @@ { "id": "report-generation/symbol-dispatch-db-sync", "repository": "govalidate-report-generation", "class": "symbol", "value": "dispatchDbSync", "why": "Qualification-target symbol in a fixed promotion table." }, { "id": "report-generation/symbol-broadcast-run-started", "repository": "govalidate-report-generation", "class": "symbol", "value": "broadcastRunStarted", "why": "Qualification-target symbol in a fixed promotion table." }, { "id": "report-generation/symbol-broadcast-run-failed", "repository": "govalidate-report-generation", "class": "symbol", "value": "broadcastRunFailed", "why": "Qualification-target symbol in a fixed promotion table." }, - { "id": "report-generation/symbol-score-metrics", "repository": "govalidate-report-generation", "class": "symbol", "value": "scoreMetrics", "why": "Qualification-target symbol in a fixed promotion table." } + { "id": "report-generation/symbol-score-metrics", "repository": "govalidate-report-generation", "class": "symbol", "value": "scoreMetrics", "why": "Qualification-target symbol in a fixed promotion table." }, + { "id": "report-generation/symbol-prompt-wants-report-generation-core", "repository": "govalidate-report-generation", "class": "symbol", "value": "promptWantsReportGenerationCore", "why": "Name of the retired report-generation task-phrase classifier. It was duplicated in prompt-pack.ts and retrieve/slicing.ts and gated every report-only behaviour removed in #660 Slice C." }, + { "id": "report-generation/symbol-semantic-generation-core-anchor-value", "repository": "govalidate-report-generation", "class": "symbol", "value": "semanticGenerationCoreAnchorValue", "why": "Name of the retired name-driven anchor score table that paid a symbol for resembling a report workflow stage." }, + { "id": "report-generation/phrase-generation-core-heuristic-anchor", "repository": "govalidate-report-generation", "class": "phrase", "value": "generation core heuristic", "why": "Anchor reason emitted only by the retired forced report-anchor selection." }, + { "id": "report-generation/phrase-fixed-report-workflow-instruction", "repository": "govalidate-report-generation", "class": "phrase", "value": "planner, research, assembly, scoring, rendering, and persistence evidence", "why": "The retired fixed report workflow instruction, manufactured from prompt vocabulary with no evidence behind it." }, + { "id": "report-generation/phrase-final-report-score-entry", "repository": "govalidate-report-generation", "class": "phrase", "value": "final(?:-| )report", "why": "A distinctive +10 entry of the retired anchor score table." }, + { "id": "report-generation/phrase-idea-report-gate-entry", "repository": "govalidate-report-generation", "class": "phrase", "value": "idea\\s+report", "why": "The qualification task name, matched by the retired reportGenerationShaped gate classifier." } ], "corpus_symbol_import": { "source": "docs/qualification/corpus.json", diff --git a/scripts/verify-report-generation-injections.mjs b/scripts/verify-report-generation-injections.mjs new file mode 100755 index 00000000..7c9a3e07 --- /dev/null +++ b/scripts/verify-report-generation-injections.mjs @@ -0,0 +1,219 @@ +#!/usr/bin/env node +// #660 Slice C — focused falsifiability for the retired report-generation class. +// +// Four injections, no more. Each one restores ONE retired rule into production +// source from a digest-checked byte snapshot, requires a NAMED control to fail, +// and restores the exact bytes and file mode in `finally`. Unrelated failures +// earn no credit: an injection passes only when the control it names fails. +// +// This is deliberately NOT a general mutation framework. It is a fixed list of +// four edits with a fixed expected victim each. +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync, statSync, writeFileSync, chmodSync } from 'node:fs' +import { resolve } from 'node:path' + +const PROMPT_PACK = 'src/infrastructure/prompt-pack.ts' +const SLICING = 'src/runtime/retrieve/slicing.ts' +const GATE = 'src/runtime/retrieval-gate.ts' + +const CONTROL_FILE = 'tests/unit/report-generation-independence.test.ts' + +/** The retired task-phrase classifier, restored verbatim where an injection needs it. */ +const RETIRED_CLASSIFIER = `function promptWantsReportGenerationCore(prompt) { + return /\\b(?:report(?:\\s+generation)?|generated\\s+report|validation\\s+report|final\\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\\s|-)?gate)\\b/i.test(prompt) +} +` + +const INJECTIONS = [ + { + id: 'C1', + title: 'the fixed report workflow instruction', + marker: 'SLICE_C_FIXED_REPORT_INSTRUCTION_REINTRODUCED', + file: PROMPT_PACK, + control: { file: CONTROL_FILE, name: 'A: report vocabulary alone produces no fixed workflow' }, + apply(source) { + // Put the retired fixed instruction back, unconditionally, so the + // vocabulary-only control must observe it. + const anchor = ` const instructions = [ + 'Treat HTTP/controller entrypoints as trigger context, not the full answer.', + ]` + if (!source.includes(anchor)) throw new Error('C1 anchor missing') + return source.replace( + ` ...answerContractInstructions(input.retrieval),`, + ` ...answerContractInstructions(input.retrieval), + 'Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.',`, + ) + }, + }, + { + id: 'C2', + title: 'a report-generation task-phrase classifier', + marker: 'SLICE_C_TASK_PHRASE_CLASSIFIER_REINTRODUCED', + file: SLICING, + control: { file: CONTROL_FILE, name: 'B: the same repository and task intent yield the same structure' }, + apply(source) { + // Restore the classifier AND one behaviour it used to gate: a deeper + // backward policy for report-shaped prompts only. + const anchor = ` if (!hasMethodAnchor && !pipelinePrompt) { + return base + }` + if (!source.includes(anchor)) throw new Error('C2 anchor missing') + return source + .replace('function promptWantsRuntimePipeline(', `${RETIRED_CLASSIFIER}\nfunction promptWantsRuntimePipeline(`) + .replace(anchor, `${anchor} + + if (broadRuntimeGeneration && pipelinePrompt && promptWantsReportGenerationCore(options.prompt)) { + return { + ...base, + directions: ['backward', 'forward'], + backward_relations: new Set(['calls', 'enqueues_job', 'controller_route', 'route_handler', 'method']), + forward_relations: new Set(RUNTIME_FLOW_RELATIONS), + helper_relations: new Set(['injects', 'depends_on', 'module_provides']), + backward_depth: Math.max(base.backward_depth, 3), + forward_depth: Math.max(base.forward_depth, 4), + runtime_flow_only: true, + } + }`) + }, + }, + { + id: 'C3', + title: 'a name-driven anchor score-table entry', + marker: 'SLICE_C_NAME_DRIVEN_SCORE_TABLE_REINTRODUCED', + file: SLICING, + control: { file: CONTROL_FILE, name: 'C: identical structure named after report stages earns no extra structural treatment' }, + apply(source) { + // One score-table entry: a symbol is paid for being NAMED like a report + // stage. The same-name/different-semantics control must notice. + const anchor = ` if (/\\b(?:service|provider|repository|queue|worker|orchestrator)\\b/.test(lower)) value += 1` + if (!source.includes(anchor)) throw new Error('C3 anchor missing') + return source.replace(anchor, `${anchor} + if (/\\b(?:planner|assembly|assemble|renderer|synthesis|scoring|research)\\b/.test(lower)) value += 11`) + }, + }, + { + id: 'C4', + title: 'the reportGenerationShaped gate variant', + marker: 'SLICE_C_REPORT_GATE_VARIANT_REINTRODUCED', + file: GATE, + control: { file: CONTROL_FILE, name: 'A: report vocabulary alone produces no fixed workflow' }, + apply(source) { + const anchor = ` const strongRuntimeShaped = backendRuntimeShaped` + if (!source.includes(anchor)) throw new Error('C4 anchor missing') + return source.replace(anchor, ` const reportGenerationShaped = /\\b(?:idea\\s+report|report\\s+generation|final\\s+report|planner|research|metrics?|scor(?:e|ing)|quality(?:\\s|-)?gate|renderer|synthesis|assemble|assembly)\\b/i.test(lower) + const strongRuntimeShaped = backendRuntimeShaped || reportGenerationShaped`) + }, + }, +] + +function digest(text) { + return createHash('sha256').update(text).digest('hex') +} + +function untrackedSources() { + return execFileSync('git', ['status', '--porcelain', '--untracked-files=all', '--', 'src', 'tests', 'scripts'], { encoding: 'utf8' }) + .split('\n') + .filter((line) => line.startsWith('??')) + .map((line) => line.slice(3).trim()) + .filter((file) => file.length > 0) +} + +function runControl(control) { + try { + execFileSync('npx', ['vitest', 'run', control.file, '-t', control.name], { + stdio: 'pipe', + encoding: 'utf8', + env: { ...process.env, CI: '1' }, + }) + return { failed: false } + } catch (error) { + const output = `${error.stdout ?? ''}${error.stderr ?? ''}` + return { failed: true, output } + } +} + +function main() { + const results = [] + let allPassed = true + + // Fingerprint the three production files BEFORE any injection. Comparing to + // git HEAD would be wrong: on a working branch these files legitimately differ + // from HEAD, so only a self-taken snapshot can prove the harness left nothing + // behind. + const startFingerprint = new Map( + [PROMPT_PACK, SLICING, GATE].map((file) => [file, digest(readFileSync(resolve(file), 'utf8'))]), + ) + // Untracked files that already exist are the branch's own new files, not this + // harness's residue. Only files that appear DURING the run count. + const startUntracked = new Set(untrackedSources()) + + for (const injection of INJECTIONS) { + const path = resolve(injection.file) + const original = readFileSync(path, 'utf8') + const originalDigest = digest(original) + const originalMode = statSync(path).mode + + let outcome + try { + const injected = injection.apply(original) + if (injected === original) throw new Error(`${injection.id}: injection changed nothing`) + writeFileSync(path, injected) + + // The injected path must actually be REACHED, not merely present. + const baseline = runControl(injection.control) + if (!baseline.failed) { + outcome = { ok: false, detail: 'the named control still PASSED with the rule restored — it cannot catch its own mutation' } + } else { + // Credit only for the named control failing, not for any failure. + const named = baseline.output.includes(injection.control.name) + outcome = named + ? { ok: true, detail: `${injection.marker}: named control failed as required` } + : { ok: false, detail: 'a failure occurred but it was not the named control' } + } + } catch (error) { + outcome = { ok: false, detail: `injection error: ${error.message}` } + } finally { + writeFileSync(path, original) + chmodSync(path, originalMode) + const restored = readFileSync(path, 'utf8') + if (digest(restored) !== originalDigest) { + console.error(`FATAL ${injection.id}: ${injection.file} was NOT restored to its original bytes`) + process.exit(2) + } + } + + allPassed = allPassed && outcome.ok + results.push({ injection, outcome }) + console.log(`${outcome.ok ? 'PASS' : 'FAIL'} ${injection.id} ${injection.title}`) + console.log(` ${outcome.detail}`) + } + + // Final fingerprint: every touched file is byte-identical to its start state. + console.log('') + console.log('Worktree fingerprint for the three production files after restoration:') + let drifted = false + for (const [file, expected] of startFingerprint) { + const actual = digest(readFileSync(resolve(file), 'utf8')) + const same = actual === expected + drifted = drifted || !same + console.log(` ${same ? 'restored' : 'DRIFTED '} ${file} sha256 ${actual.slice(0, 16)}`) + } + if (drifted) { + console.error('FATAL: a production file did not return to its pre-injection bytes.') + process.exit(2) + } + + // No temporary source or test residue may survive the run. + const leaked = untrackedSources().filter((file) => !startUntracked.has(file)) + console.log(leaked.length === 0 + ? ' no new untracked residue under src/, tests/ or scripts/' + : ` UNTRACKED RESIDUE: ${leaked.join(' ')}`) + if (leaked.length > 0) process.exit(2) + + console.log('') + console.log(`${results.filter((entry) => entry.outcome.ok).length}/${INJECTIONS.length} focused injections behaved as required.`) + process.exit(allPassed ? 0 : 1) +} + +main() diff --git a/src/infrastructure/prompt-pack.ts b/src/infrastructure/prompt-pack.ts index c5a097b9..b1d1058b 100644 --- a/src/infrastructure/prompt-pack.ts +++ b/src/infrastructure/prompt-pack.ts @@ -54,20 +54,13 @@ function answerContractInstructions(retrieval: RetrieveResult): string[] { ] const requiredElements = new Set(answerContract.required_elements) - const phaseLabels = [ - ['planner_phase', 'planner'], - ['research_phase', 'research'], - ['assembly_phase', 'assembly'], - ['scoring_phase', 'scoring'], - ['report_builder_phase', 'rendering'], - ] as const satisfies ReadonlyArray - const selectedPhaseLabels: string[] = phaseLabels.flatMap(([key, label]) => requiredElements.has(key) ? [label] : []) - if (selectedPhaseLabels.length > 0 || requiredElements.has('persistence_or_artifact_storage')) { - const segments = [...selectedPhaseLabels] - if (requiredElements.has('persistence_or_artifact_storage')) { - segments.push('persistence') - } - instructions.push(`Follow ${segments.join(', ')} evidence before concluding the flow.`) + // Every instruction below is keyed on a typed `required_elements` entry that + // `runtimeGenerationContractPhaseElements` derives from execution-slice + // evidence (phase coverage, an `enqueues_job` boundary, slice status). The + // question text is not an input here, so prompt vocabulary cannot manufacture + // a workflow that the repository evidence does not show. + if (requiredElements.has('persistence_or_artifact_storage')) { + instructions.push('Follow persistence evidence before concluding the flow.') } else if (requiredElements.has('main_pipeline_phases')) { instructions.push('Cover the main runtime pipeline phases instead of stopping at the entrypoint.') } @@ -91,30 +84,6 @@ function answerContractInstructions(retrieval: RetrieveResult): string[] { return instructions } -function promptWantsReportGenerationCore(prompt: string): boolean { - return /\b(?:report(?:\s+generation)?|generated\s+report|validation\s+report|final\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate)\b/i.test(prompt) -} - -function generationCoreInstructions(question: string, retrieval: RetrieveResult): string[] { - const contractInstructions = answerContractInstructions(retrieval) - if (contractInstructions.length > 0) { - return contractInstructions - } - - if ( - retrieval.retrieval_gate?.signals.generation_intent !== 'runtime_generation' - || retrieval.retrieval_gate?.signals.target_domain_hint !== 'backend_runtime' - || !promptWantsReportGenerationCore(question) - ) { - return [] - } - - return [ - 'Treat HTTP/controller entrypoints as trigger context, not the full answer, when downstream generation-core evidence is present.', - 'Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.', - ] -} - export function buildMadarPromptPack(input: BuildMadarPromptPackInput): ComparePromptPack { const explainPayloadCore = buildAnswerReadyPackSchema( buildExplainPackPayloadCore(compactRetrieveResult(input.retrieval), input.retrieval), @@ -126,7 +95,7 @@ export function buildMadarPromptPack(input: BuildMadarPromptPackInput): CompareP instructions: [ 'Answer the question using only the provided graph-guided retrieval output.', 'If the retrieval does not contain the answer, say so.', - ...generationCoreInstructions(input.question, input.retrieval), + ...answerContractInstructions(input.retrieval), ], stable_prefix_title: 'Retrieved graph context', stable_sections: [ diff --git a/src/runtime/retrieval-gate.ts b/src/runtime/retrieval-gate.ts index 6f972859..07c78e93 100644 --- a/src/runtime/retrieval-gate.ts +++ b/src/runtime/retrieval-gate.ts @@ -241,7 +241,6 @@ function detectGenerationIntent(prompt: string): { const displayShaped = /\b(?:display(?:ed|ing)?|render(?:ed|ing)?|show(?:n|ing)?|visible|view|ui|frontend|front-end|component|screen|page|footer|header|label|date|timestamp)\b/i.test(lower) const genericGenerationShaped = /\b(?:generat(?:e|ed|es|ing|ion)|creat(?:e|ed|es|ing|ion)|build(?:s|ing|er)?|assembl(?:e|ed|es|ing)|produc(?:e|ed|es|ing))\b/i.test(lower) const backendRuntimeShaped = /\b(?:pipeline|runtime|orchestrator|worker|job|repository|service|controller|api|backend|persist(?:ed|ing)?|sav(?:e|ed|es|ing)|db(?:sync|[- ]sync)?)\b/i.test(lower) - const reportGenerationShaped = /\b(?:idea\s+report|report\s+generation|final\s+report|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate|renderer|synthesis|assemble|assembly)\b/i.test(lower) const buildStaticShaped = /\b(?:next\.js|nextjs|app\s+router|route\s+segment|landing\s+page|static|ssg|isr|server\s+component)\b/i.test(lower) const explanationShaped = /\b(?:explain|how|why|trace|walk|flow|path|lifecycle|fail(?:s|ing|ed)?)\b/i.test(lower) const flowProofShaped = @@ -254,13 +253,20 @@ function detectGenerationIntent(prompt: string): { display_shaped: displayShaped, generic_generation_shaped: genericGenerationShaped, backend_runtime_shaped: backendRuntimeShaped, - report_generation_shaped: reportGenerationShaped, + // #660 Slice C retired the report-generation task-shape classifier. This + // signal is a required member of the published `RetrievalGenerationDebugSignals` + // shape, which is outside this slice's three-file production boundary, so it + // is reported as the constant it now is rather than removed. Nothing reads it + // to reach a gate outcome: no prompt vocabulary can turn it true. + report_generation_shaped: false, build_static_shaped: buildStaticShaped, explanation_shaped: explanationShaped, flow_proof_shaped: flowProofShaped, } - const strongRuntimeShaped = backendRuntimeShaped || reportGenerationShaped + // Gate outcomes follow generic evidence only: a prompt that merely resembles + // the old qualification report task no longer earns a runtime-generation gate. + const strongRuntimeShaped = backendRuntimeShaped if (buildStaticShaped && !strongRuntimeShaped) { return { intent: 'unknown', debug } diff --git a/src/runtime/retrieve/slicing.ts b/src/runtime/retrieve/slicing.ts index 823165e9..7a0f1224 100644 --- a/src/runtime/retrieve/slicing.ts +++ b/src/runtime/retrieve/slicing.ts @@ -123,20 +123,19 @@ function policyForIntent(intent: RetrievalIntent): SlicePolicy { } } +// Generic runtime-architecture vocabulary only. `scoring` and `report builder` +// were report-workflow stage names and are gone with the rest of the #660 +// report-generation class; `persistence` and `repository` stay because they +// name ordinary backend layers, are independently treated as backend-runtime +// markers by the retrieval gate's generic `backendRuntimeShaped` signal, and are +// exercised by the pre-existing non-qualification direct-persistence fixture in +// tests/unit/retrieve-slice-v1.test.ts. function promptWantsRuntimePipeline(prompt: string | undefined): boolean { if (!prompt) { return false } - return /\b(runtime|pipeline|service|orchestrator|job|agent|scoring|report(?: builder)?|persistence|repository|generat(?:e|ed|es|ing|ion)|create|created)\b/i.test(prompt) -} - -function promptWantsReportGenerationCore(prompt: string | undefined): boolean { - if (!prompt) { - return false - } - - return /\b(?:report(?:\s+generation)?|generated\s+report|validation\s+report|final\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate)\b/i.test(prompt) + return /\b(runtime|pipeline|service|orchestrator|job|agent|persistence|repository|generat(?:e|ed|es|ing|ion)|create|created)\b/i.test(prompt) } function promptMentionsHttpRoute(prompt: string | undefined): boolean { @@ -184,25 +183,11 @@ function effectivePolicy( const broadRuntimeGeneration = options.generationIntent === 'runtime_generation' && options.targetDomainHint === 'backend_runtime' && !hasExactMethodAnchor - const hasGenerationCoreAnchor = anchors.some((anchor) => anchor.reason === 'generation core heuristic') if (!hasMethodAnchor && !pipelinePrompt) { return base } - if (broadRuntimeGeneration && pipelinePrompt && hasGenerationCoreAnchor && promptWantsReportGenerationCore(options.prompt)) { - return { - ...base, - directions: ['backward', 'forward'], - backward_relations: new Set(['calls', 'enqueues_job', 'controller_route', 'route_handler', 'method']), - forward_relations: new Set(RUNTIME_FLOW_RELATIONS), - helper_relations: new Set(['injects', 'depends_on', 'module_provides']), - backward_depth: Math.max(base.backward_depth, 3), - forward_depth: Math.max(base.forward_depth, 4), - runtime_flow_only: true, - } - } - if (broadRuntimeGeneration && hasMethodAnchor && pipelinePrompt) { return { ...base, @@ -320,7 +305,7 @@ function runtimeGenerationAnchorValue(node: SliceScoredNode): number { if (methodLikeNode(node)) value += 1 if (/\b(?:nest_route|route|controller)\b/.test(lower)) value += 5 if (/\b(?:src|server|backend|api|modules)\b/.test(lower)) value += 1 - if (/\b(?:generate|generation|create|start|pipeline|queue|process|orchestrator|worker|job|repository|save|report|scoring|research|agent)\b/.test(lower)) value += 2 + if (/\b(?:generate|generation|create|start|pipeline|queue|process|orchestrator|worker|job|repository|save|agent)\b/.test(lower)) value += 2 if (/(?:^|[.#])(?:generate|create|start|process|save|score|search|update|claim|cancel)[A-Za-z_$\w]*\(?\)?$/i.test(node.label)) value += 3 if (/\b(?:service|provider|repository|queue|worker|orchestrator)\b/.test(lower)) value += 1 if (frontendDisplayLikeNode(node)) value -= 6 @@ -329,27 +314,6 @@ function runtimeGenerationAnchorValue(node: SliceScoredNode): number { } -function semanticGenerationCoreAnchorValue(node: SliceScoredNode, prompt: string | undefined): number { - const lower = `${node.label} ${node.nodeKind ?? ''} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - let value = runtimeGenerationAnchorValue(node) - - if (!promptWantsReportGenerationCore(prompt)) { - return value - } - - if (routeOrControllerLikeNode(node)) value -= 7 - if (/\b(?:title|status|guard|auth|interceptor|refund|suggest|list|health|planenforcement)\b/.test(lower)) value -= 4 - if (/\b(?:planner|plan\b)\b/.test(lower)) value += 11 - if (/\b(?:assembly|assemble|quality(?:-| )gate|renderer|render|synthesis|final(?:-| )report)\b/.test(lower)) value += 10 - if (/\b(?:research|extract|metrics?|scor(?:e|ing))\b/.test(lower)) value += 7 - if (/\b(?:orchestrator|pipeline)\b/.test(lower)) value += 4 - if (/\b(?:worker|section)\b/.test(lower)) value += 2 - if (/\b(?:persist|repository|db(?:-| )sync|save)\b/.test(lower)) value += 3 - if (/\b(?:index\.json|state)\b/.test(lower)) value += 2 - - return value -} - function runtimeFlowRelationPriority( relation: string, node: SliceScoredNode, @@ -444,7 +408,6 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], )) const nonBarrelMatchedAnchors = matchedAnchors.filter((node) => !isBarrelLike(node.label, node.sourceFile)) const broadRuntimeGeneration = broadRuntimeGenerationPrompt(options) - const reportGenerationPrompt = promptWantsReportGenerationCore(options.prompt) const explicitPathAnchor = matchedAnchors.find((node) => node.literalPathMatch) const routePromptAnchors = broadRuntimeGeneration && promptMentionsHttpRoute(options.prompt) ? scored @@ -459,20 +422,6 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], return rightPriority - leftPriority || right.score - left.score }) : [] - const semanticCoreAnchors = broadRuntimeGeneration && reportGenerationPrompt - ? scored - .filter((node) => - methodLikeNode(node) - && !routeOrControllerLikeNode(node) - && !isBarrelLike(node.label, node.sourceFile) - && !frontendDisplayLikeNode(node) - && node.score > 0, - ) - .map((node) => ({ node, value: semanticGenerationCoreAnchorValue(node, options.prompt) })) - .filter((entry) => entry.value > 0) - .sort((left, right) => right.value - left.value || right.node.score - left.node.score) - .map((entry) => entry.node) - : [] const intentAnchors = (() => { if (options.generationIntent === 'runtime_generation' && options.targetDomainHint === 'backend_runtime') { return matchedAnchors @@ -494,28 +443,11 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], return [] })() - const generationCoreAnchorIds = new Set() let anchorPool: SliceScoredNode[] if (exactMethodAnchors.length > 0) { anchorPool = exactMethodAnchors.slice(0, 1) } else if (explicitPathAnchor) { anchorPool = [explicitPathAnchor] - } else if (broadRuntimeGeneration && reportGenerationPrompt && semanticCoreAnchors.length > 0) { - const primaryRuntimeAnchor = routePromptAnchors[0] - ?? intentAnchors[0] - ?? nonBarrelMatchedAnchors[0] - ?? matchedAnchors[0] - const selected = [ - ...(primaryRuntimeAnchor ? [primaryRuntimeAnchor] : []), - ...semanticCoreAnchors.filter((node) => node.id !== primaryRuntimeAnchor?.id).slice(0, 2), - ] - anchorPool = selected - for (const node of selected) { - if (primaryRuntimeAnchor && node.id === primaryRuntimeAnchor.id) { - continue - } - generationCoreAnchorIds.add(node.id) - } } else if (routePromptAnchors.length > 0) { anchorPool = routePromptAnchors.slice(0, 1) } else if (intentAnchors.length > 0) { @@ -527,9 +459,7 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], } for (const node of anchorPool) { - const reason = generationCoreAnchorIds.has(node.id) - ? 'generation core heuristic' - : node.exactLabelMatch + const reason = node.exactLabelMatch ? 'symbol mention' : node.literalPathMatch ? 'path mention' @@ -545,8 +475,7 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], reason, }) seen.add(node.id) - const maxAnchors = broadRuntimeGeneration && reportGenerationPrompt ? 3 : 2 - if (anchors.length >= maxAnchors) { + if (anchors.length >= 2) { break } } @@ -677,18 +606,18 @@ function traverseDirection( function pipelineBridgeLikeNode(node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - return /\bpipeline|trigger|queue|job|worker|orchestrator|planner|research|agent|scoring|report|repository|persistence|save|process|search|score|addjob\b/.test(lower) + return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|search|addjob\b/.test(lower) } function highValueRuntimeExpansionNode(node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - return /\bpipeline|trigger|queue|job|worker|orchestrator|planner|research|agent|scoring|report|repository|persistence|save|process|search|score|dispatch|assemble|persist|builder|addjob\b/.test(lower) + return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|search|dispatch|persist|builder|addjob\b/.test(lower) } function sharedHubLikeNode(graph: KnowledgeGraph, node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''}`.toLowerCase() return graph.uniqueNeighborDegree(node.id) >= 12 - || /requireideasuserid|addjob|callllm|resolve|logger|\.info\(\)|\.error\(\)|\.warn\(\)|planenforcement/.test(lower) + || /resolve|logger|\.info\(\)|\.error\(\)|\.warn\(\)/.test(lower) } function shouldExpandRuntimePathNode( @@ -790,14 +719,6 @@ function addAnchorPredecessors( const predecessor = scoredById.get(predecessorId) ?? sliceNodeFromGraph(graph, predecessorId) scoredById.set(predecessorId, predecessor) - if ( - broadRuntimeGenerationPrompt(options) - && promptWantsReportGenerationCore(options.prompt) - && routeOrControllerLikeNode(predecessor) - && !anchoredIds.has(predecessorId) - ) { - continue - } if (shouldSuppressNode(graph, predecessor, anchoredIds, options)) { continue } diff --git a/tests/unit/compare.test.ts b/tests/unit/compare.test.ts index 547a7c71..80cb8185 100644 --- a/tests/unit/compare.test.ts +++ b/tests/unit/compare.test.ts @@ -1067,7 +1067,7 @@ describe('compare runtime', () => { } }) - it('tells broad runtime-generation answers not to stop at the HTTP trigger when downstream generation core evidence exists', () => { + it('does not manufacture a report workflow instruction from question vocabulary alone (#660 Slice C)', () => { const retrieval: MadarPromptPackRetrieval = { question: 'How idea report is being generated', token_count: 120, @@ -1108,8 +1108,15 @@ describe('compare runtime', () => { retrieval, }) - expect(pack.prompt).toContain('Treat HTTP/controller entrypoints as trigger context, not the full answer, when downstream generation-core evidence is present.') - expect(pack.prompt).toContain('Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.') + // #660 Slice C. This test used to REQUIRE both lines above. They were + // produced by a task-phrase classifier over the question text plus a fixed + // planner/research/assembly/scoring/rendering/persistence workflow, with no + // evidence behind either — the retrieval here carries no answer_contract and + // no execution_slice at all. The assertion is inverted into the independence + // control it should always have been: prompt vocabulary manufactures nothing. + expect(pack.prompt).not.toContain('Treat HTTP/controller entrypoints as trigger context, not the full answer, when downstream generation-core evidence is present.') + expect(pack.prompt).not.toContain('Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.') + expect(pack.prompt).not.toContain('planner, research, assembly, scoring, rendering') }) it('includes a runtime-generation answer contract in the prompt core and tells partial slices to mention uncertainty', () => { diff --git a/tests/unit/pack-quality-fixtures.test.ts b/tests/unit/pack-quality-fixtures.test.ts index 5dd7cbd9..a5fcd154 100644 --- a/tests/unit/pack-quality-fixtures.test.ts +++ b/tests/unit/pack-quality-fixtures.test.ts @@ -248,26 +248,28 @@ describe('pack-quality fixtures (#298)', () => { expect([...surfacedLabels]).not.toContain('.buildQueuedIdeaReportResponse()') }) - it('promotes the quality-gate failure branch when the explain prompt asks what happens if it fails', async () => { + // #660 Slice C. This test required `handleQualityGateFailure()` onto the + // primary path for a report-shaped prompt. It got there through + // `semanticGenerationCoreAnchorValue`, which paid +10 to any label matching + // "quality gate" once a task-phrase classifier had seen report vocabulary in + // the question — a name-shaped promotion with no structural basis. Both are + // gone. What remains a real claim is that the prompt's wording alone does not + // decide the primary path, so this is now that independence control. + it('does not promote a branch onto the primary path from report wording alone (#660 Slice C)', async () => { const result = await runPackQualityFixture('runtime-generation-explain-report-flow', { prompt: 'How is idea report handled when it fails', task: 'explain', budget: 1800, }) const payload = result.payload as typeof result.payload & { - pack?: { - execution_slice?: { - primary_path?: { - steps?: Array<{ label?: string }> - } - } - } + pack?: { execution_slice?: { primary_path?: { steps?: Array<{ label?: string }> } } } } const primaryLabels = payload.pack?.execution_slice?.primary_path?.steps?.map((entry) => entry.label) ?? [] - expect(primaryLabels).toEqual(expect.arrayContaining([ - 'handleQualityGateFailure()', - ])) + // The retired table paid +10 for a label matching "quality gate" once the + // question had been classified as report-shaped. Nothing may reach the + // primary path on the strength of its name any more. + expect(primaryLabels).not.toContain('handleQualityGateFailure()') expect(primaryLabels).not.toContain('saveStructuredReport()') }) diff --git a/tests/unit/report-generation-independence.test.ts b/tests/unit/report-generation-independence.test.ts new file mode 100644 index 00000000..83d20ca0 --- /dev/null +++ b/tests/unit/report-generation-independence.test.ts @@ -0,0 +1,342 @@ +// #660 Slice C — report-generation independence controls. +// +// Slice C removed the last production behaviour that Madar activated because a +// prompt LOOKED like the qualification report-generation task: a task-phrase +// classifier duplicated in `infrastructure/prompt-pack.ts` and +// `runtime/retrieve/slicing.ts`, the fixed +// "Follow planner, research, assembly, scoring, rendering, and persistence +// evidence" instruction, a name-driven anchor score table, forced anchor +// membership, a deeper report-only slice policy, and the `reportGenerationShaped` +// gate variant. +// +// These controls are NEGATIVE and INDEPENDENCE controls, not snapshots. Each one +// is written so that restoring any single retired rule makes it fail — that is +// verified mechanically by scripts/verify-report-generation-injections.mjs, +// which restores each rule from a digest-checked byte snapshot and requires the +// named control below to fail. +// +// The former qualification vocabulary is deliberately spelled out in the PROMPTS +// here. That is the point: these controls prove the words do nothing. It is +// never used to name a symbol, a path or an expected output. +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import type { ContextPackExecutionPhase } from '../../src/contracts/context-pack.js' +import { build } from '../../src/pipeline/build.js' +import { buildMadarPromptPack } from '../../src/infrastructure/prompt-pack.js' +import { classifyRetrievalLevel } from '../../src/runtime/retrieval-gate.js' +import { retrieveContext, type RetrieveResult } from '../../src/runtime/retrieve.js' + +// The retired qualification vocabulary, split so a control never confuses a +// LEGITIMATE lexical effect with a report-policy effect. `rendering`/`renderer` +// are also ordinary FRONTEND-DISPLAY words that the generic display classifier +// has always matched and that Slice C did not touch, so they live apart and are +// only used where a display classification is the expected generic outcome. +const REPORT_WORDS = 'idea report generation planner research assembly scoring synthesis metrics quality gate' +const REPORT_DISPLAY_WORDS = 'rendering renderer' + +// A shared generic frame. The two prompts carry the same backend-runtime and +// explanation shape and differ ONLY in their nouns. Both noun sets are chosen to +// match NO label in either fixture graph, so a structural difference cannot be +// explained away as ordinary lexical relevance — which section 11-B requires be +// identified separately rather than mistaken for special policy. +const FRAME = 'is generated end to end through the runtime pipeline by the' +const NEUTRAL_Q = `Explain how the daily digest ${FRAME} ingest, lookup, merge and tally stages` +const REPORT_Q = `Explain how the idea report ${FRAME} planner, research, assembly and scoring stages` + +const BASE_INSTRUCTIONS = [ + 'Answer the question using only the provided graph-guided retrieval output.', + 'If the retrieval does not contain the answer, say so.', +] + +/** + * A five-stage runtime flow whose edges point BACK toward the route. + * + * Every stage carries both a LABEL and a SOURCE PATH, because the retired score + * table read `label + node_kind + framework_role + source_file` and in practice + * matched on path SEGMENTS (`/assembly/`, `/scoring/`) far more often than on a + * camelCase label, where lower-casing destroys the word boundaries the pattern + * needed. A fixture that varied only the labels could not catch it. + */ +interface Stage { label: string; file: string } + +function reverseFlowGraph(stages: { + route: Stage + trigger: Stage + queue: Stage + worker: Stage + assembler: Stage + scorer: Stage +}) { + return build( + [ + { + schema_version: 1, + nodes: [ + { id: 'route', label: stages.route.label, file_type: 'code', source_file: stages.route.file, source_location: 'L20', node_kind: 'method', framework: 'nestjs', framework_role: 'nest_route', community: 0 }, + { id: 'trigger', label: stages.trigger.label, file_type: 'code', source_file: stages.trigger.file, source_location: 'L30', node_kind: 'method', framework_role: 'service', community: 1 }, + { id: 'queue', label: stages.queue.label, file_type: 'code', source_file: stages.queue.file, source_location: 'L40', node_kind: 'method', framework_role: 'queue', community: 1 }, + { id: 'worker', label: stages.worker.label, file_type: 'code', source_file: stages.worker.file, source_location: 'L50', node_kind: 'method', framework_role: 'worker', community: 2 }, + { id: 'assembler', label: stages.assembler.label, file_type: 'code', source_file: stages.assembler.file, source_location: 'L60', node_kind: 'method', framework_role: 'service', community: 2 }, + { id: 'scorer', label: stages.scorer.label, file_type: 'code', source_file: stages.scorer.file, source_location: 'L70', node_kind: 'method', framework_role: 'service', community: 2 }, + ], + edges: [ + { source: 'trigger', target: 'route', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.trigger.file }, + { source: 'queue', target: 'trigger', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.queue.file }, + { source: 'worker', target: 'queue', relation: 'enqueues_job', confidence: 'EXTRACTED', source_file: stages.worker.file }, + { source: 'assembler', target: 'worker', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.assembler.file }, + { source: 'scorer', target: 'assembler', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.scorer.file }, + ], + }, + ], + { directed: true }, + ) +} + +// Unrelated names: an invoice intake flow. +const NEUTRAL_NAMES = { + route: { label: '.submitInvoice()', file: '/src/entry/entry.controller.ts' }, + trigger: { label: '.beginRun()', file: '/src/flow/trigger.service.ts' }, + queue: { label: '.enqueueBatch()', file: '/src/flow/queue-registry.service.ts' }, + worker: { label: '.consume()', file: '/src/flow/orchestrator.worker.ts' }, + assembler: { label: '.combineLineItems()', file: '/src/flow/stage/stage.service.ts' }, + scorer: { label: '.rateVendor()', file: '/src/flow/rating/rating.service.ts' }, +} + +/** + * The SAME structure with the SAME leading verbs and the same generic path words + * ("src", "flow", "service"), named throughout after report workflow stages while + * doing the same unrelated invoice work. Holding the verbs fixed matters: Madar + * legitimately prefers action-shaped method names such as `generate`/`start`/ + * `process`, and that generic preference must not be mistaken for report naming. + */ +const REPORT_NAMED = { + route: { label: '.submitReport()', file: '/src/entry/entry.controller.ts' }, + trigger: { label: '.beginPlanner()', file: '/src/flow/trigger.service.ts' }, + queue: { label: '.enqueueResearch()', file: '/src/flow/queue-registry.service.ts' }, + worker: { label: '.consume()', file: '/src/flow/orchestrator.worker.ts' }, + assembler: { label: '.combineAssembly()', file: '/src/flow/assembly/assembly.service.ts' }, + scorer: { label: '.rateScoring()', file: '/src/flow/scoring/scoring.service.ts' }, +} + +/** A repository with no multi-stage generation structure at all. */ +function flatDisplayGraph() { + return build( + [ + { + schema_version: 1, + nodes: [ + { id: 'toggle', label: '.applyTheme()', file_type: 'code', source_file: '/src/ui/theme-toggle.ts', source_location: 'L4', node_kind: 'method', community: 0 }, + { id: 'store', label: '.readPalette()', file_type: 'code', source_file: '/src/ui/color-store.ts', source_location: 'L8', node_kind: 'method', community: 0 }, + ], + edges: [ + { source: 'toggle', target: 'store', relation: 'calls', confidence: 'EXTRACTED', source_file: '/src/ui/theme-toggle.ts' }, + ], + }, + ], + { directed: true }, + ) +} + +function retrieve(graph: ReturnType, question: string) { + return retrieveContext(graph, { question, budget: 4000, retrievalLevel: 4, retrievalStrategy: 'slice-v1' }) +} + +/** Slice membership is observable through the matched nodes the slice selected. */ +function memberIds(result: RetrieveResult): string[] { + return result.matched_nodes.map((node) => node.node_id ?? node.label) +} + +function structuralShape(result: RetrieveResult) { + return { + member_ids: memberIds(result), + anchor_reasons: (result.slice?.anchors ?? []).map((anchor) => anchor.reason), + anchor_count: (result.slice?.anchors ?? []).length, + paths: (result.slice?.selected_paths ?? []).map((path) => `${path.from}-[${path.relation}]->${path.to}`), + mode: result.slice?.mode, + } +} + +/** + * The same slice keyed by NODE ID rather than label. Both fixture graphs use the + * same ids for the same structural positions and differ only in what the symbols + * are called, so an id-keyed shape isolates policy from naming completely: if the + * two shapes differ, a name changed a structural decision. + */ +function shapeByPosition(result: RetrieveResult) { + return { + member_ids: memberIds(result), + anchor_ids: (result.slice?.anchors ?? []).map((anchor) => anchor.node_id), + anchor_reasons: (result.slice?.anchors ?? []).map((anchor) => anchor.reason), + paths: (result.slice?.selected_paths ?? []).map((path) => `${path.from_id}-[${path.relation}]->${path.to_id}`), + } +} + +function instructionsFor(question: string, retrieval: Partial = {}) { + const decision = classifyRetrievalLevel({ prompt: question }) + const pack = buildMadarPromptPack({ + question, + retrieval: { + question, + token_count: 120, + matched_nodes: [], + relationships: [], + community_context: [], + graph_signals: { god_nodes: [], bridge_nodes: [] }, + retrieval_gate: { + level: decision.level, + reason: decision.reason, + skipped_retrieval: false, + intent: decision.intent, + signals: decision.signals, + }, + ...retrieval, + } as RetrieveResult, + }) + return pack.prompt.split('\n\nRetrieved graph context:')[0]?.split('\n') ?? [] +} + +describe('#660 Slice C — report-generation independence', () => { + // ------------------------------------------------------------- control A -- + it('A: report vocabulary alone produces no fixed workflow, no boost, no forced anchor and no gate variant', () => { + // The MINIMAL pair. Both prompts are display-shaped and carry no backend + // marker; they differ only by the qualification task's own name. Before + // Slice C the second one was pulled to runtime_generation by + // `reportGenerationShaped` alone. Now both take the generic display outcome. + const withoutTaskName = classifyRetrievalLevel({ prompt: 'Explain how the summary is generated and displayed' }) + const withTaskName = classifyRetrievalLevel({ prompt: 'Explain how the idea report is generated and displayed in the footer' }) + + expect(withTaskName.signals.generation_debug?.report_generation_shaped).toBe(false) + expect(withTaskName.signals.generation_intent).toBe(withoutTaskName.signals.generation_intent) + expect(withTaskName.signals.target_domain_hint).toBe(withoutTaskName.signals.target_domain_hint) + expect(withTaskName.signals.generation_intent).toBe('display_rendering') + + // The gate still responds to GENERIC evidence, so this is not a constant: + // adding a real backend-runtime marker moves it, and the report words do not. + const withBackendMarker = classifyRetrievalLevel({ prompt: 'Explain how the summary is generated and displayed by the worker pipeline' }) + expect(withBackendMarker.signals.generation_intent).toBe('runtime_generation') + + // No manufactured workflow over a repository with no such structure: the + // instructions are exactly the two unconditional ones. + const question = `Explain how the ${REPORT_WORDS} ${REPORT_DISPLAY_WORDS} flow is displayed on the page` + expect(instructionsFor(question)).toEqual(BASE_INSTRUCTIONS) + + // No forced candidate and no report-specific ranking over a flat repository. + const shape = structuralShape(retrieve(flatDisplayGraph(), question)) + expect(shape.anchor_reasons).not.toContain('generation core heuristic') + expect(shape.member_ids.length).toBeLessThanOrEqual(2) + }) + + // ------------------------------------------------------------- control B -- + it('B: the same repository and task intent yield the same structure under neutral and report wording', () => { + // Equivalent intent over identical evidence. The only difference is the + // retired qualification vocabulary, so nothing structural may move. + const neutral = retrieve(reverseFlowGraph(NEUTRAL_NAMES), NEUTRAL_Q) + const report = retrieve(reverseFlowGraph(NEUTRAL_NAMES), REPORT_Q) + + // Same generic classification, so any structural difference could only come + // from the report words themselves. + expect(classifyRetrievalLevel({ prompt: REPORT_Q }).signals.generation_intent) + .toBe(classifyRetrievalLevel({ prompt: NEUTRAL_Q }).signals.generation_intent) + expect(structuralShape(report)).toEqual(structuralShape(neutral)) + expect(structuralShape(report).anchor_reasons).not.toContain('generation core heuristic') + }) + + // ------------------------------------------------------------- control C -- + it('C: identical structure named after report stages earns no extra structural treatment', () => { + // The sharp form: ONE prompt containing no report vocabulary at all, run + // over two graphs that differ only in what their symbols are CALLED, and + // compared by NODE ID so naming cannot leak into the comparison itself. + // + // The prompt names the shared `flow` path segment on purpose. That makes the + // nodes source-path matches, which is what puts them through the runtime + // anchor SCORING path -- without it the score table is never consulted and + // this control could not catch a name-driven entry being restored. + const neutralPrompt = 'Explain how the daily digest is generated end to end through the runtime pipeline in flow' + const neutralNamed = shapeByPosition(retrieve(reverseFlowGraph(NEUTRAL_NAMES), neutralPrompt)) + const reportNamed = shapeByPosition(retrieve(reverseFlowGraph(REPORT_NAMED), neutralPrompt)) + + expect(reportNamed).toEqual(neutralNamed) + + // Proof the comparison has teeth: the scoring path really is reached, so a + // restored name-driven entry would move these positions. + expect(neutralNamed.anchor_ids.length).toBeGreaterThan(0) + }) + + it('C2: a report-shaped prompt over report-named symbols still gets no table, chain, instruction or forced pick', () => { + const shape = structuralShape(retrieve(reverseFlowGraph(REPORT_NAMED), REPORT_Q)) + + // No forced selection, and every anchor reason is one of the ordinary + // evidence reasons rather than a report heuristic. + expect(shape.anchor_reasons).not.toContain('generation core heuristic') + for (const reason of shape.anchor_reasons) { + expect(['symbol mention', 'path mention', 'source path token match', 'top lexical match']).toContain(reason) + } + // No special phase chain and no fixed report instruction. + expect(instructionsFor(REPORT_Q)).toEqual(BASE_INSTRUCTIONS) + }) + + // ------------------------------------------------------------- control D -- + it('D: the pre-existing non-qualification parity golden still carries its typed evidence instructions', () => { + // tests/fixtures/prompt-pack-parity.golden.json was captured from a commit + // BEFORE any #660 production edit, over a login/session flow named nothing + // like a report. Its instructions come from the typed answer_contract that + // runtimeGenerationContractPhaseElements derives from execution-slice + // evidence, so they must survive the removal of the report classifier. + const golden = JSON.parse(readFileSync(resolve('tests/fixtures/prompt-pack-parity.golden.json'), 'utf8')) as Record + const recorded = golden.normal_context_prompt?.prompt ?? '' + + expect(recorded).toContain('Treat HTTP/controller entrypoints as trigger context, not the full answer.') + expect(recorded).toContain('Follow persistence evidence before concluding the flow.') + expect(recorded).toContain('Do not collapse producer-to-worker handoffs into direct calls when the evidence is an enqueues_job boundary.') + expect(recorded).toContain('Mention missing or uncertain phases when the execution slice is partial.') + + // And the retired fixed workflow is absent from that pre-existing golden. + expect(recorded).not.toContain('Follow planner, research, assembly, scoring, rendering, and persistence evidence') + expect(recorded).not.toContain('when downstream generation-core evidence is present') + }) + + it('D2: typed contract elements, not prompt words, decide the evidence instructions', () => { + const typed = { + execution_slice: { + status: 'partial' as const, + steps: [], + phase_coverage: { + expected: ['controller', 'queue', 'worker', 'persistence'] as ContextPackExecutionPhase[], + observed: ['controller', 'queue'] as ContextPackExecutionPhase[], + missing: ['worker', 'persistence'] as ContextPackExecutionPhase[], + }, + }, + answer_contract: { + version: 1 as const, + answer_focus: 'runtime_generation' as const, + entrypoint_scope: 'setup_context' as const, + required_elements: ['main_pipeline_phases', 'queue_worker_handoff', 'persistence_or_artifact_storage'], + do_not_claim: ['direct_producer_to_worker_calls_without_enqueues_boundary'], + observed_phases: ['controller', 'queue'] as ContextPackExecutionPhase[], + missing_phases: ['worker', 'persistence'] as ContextPackExecutionPhase[], + }, + } + + const neutral = instructionsFor(NEUTRAL_Q, typed) + const report = instructionsFor(REPORT_Q, typed) + + // Same typed evidence in, same instructions out, regardless of wording. + expect(report).toEqual(neutral) + expect(neutral).toContain('Follow persistence evidence before concluding the flow.') + // The persistence line is earned by the typed element; the retired fixed + // report workflow never appears. + expect(neutral.join('\n')).not.toContain('planner, research, assembly, scoring, rendering') + + // Withdraw the typed element and the instruction goes with it: the control + // fails in both directions rather than merely observing a constant. + const withoutPersistence = instructionsFor(NEUTRAL_Q, { + ...typed, + answer_contract: { ...typed.answer_contract, required_elements: ['main_pipeline_phases'] }, + }) + expect(withoutPersistence).not.toContain('Follow persistence evidence before concluding the flow.') + expect(withoutPersistence).toContain('Cover the main runtime pipeline phases instead of stopping at the entrypoint.') + }) +}) diff --git a/tests/unit/retrieval-gate.test.ts b/tests/unit/retrieval-gate.test.ts index 8497d0a7..19fce86a 100644 --- a/tests/unit/retrieval-gate.test.ts +++ b/tests/unit/retrieval-gate.test.ts @@ -283,10 +283,20 @@ describe('classifyRetrievalLevel — signal extraction', () => { expect(decision.signals.target_domain_hint).toBe('frontend_display') }) - it('prefers runtime generation for explanatory prompts that also mention display terms', () => { + // #660 Slice C. This test used to require that naming the qualification task + // ("idea report") pulled a display-shaped prompt to runtime_generation, which + // is exactly the `reportGenerationShaped` gate variant. Paired with the test + // above, it is now the independence control: the task name changes nothing, + // and only a genuine backend-runtime marker moves the gate. + it('does not prefer runtime generation merely because the prompt names the report task (#660 Slice C)', () => { const decision = classify({ prompt: 'Explain how the idea report is generated and displayed in the footer' }) - expect(decision.signals.generation_intent).toBe('runtime_generation') - expect(decision.signals.target_domain_hint).toBe('backend_runtime') + expect(decision.signals.generation_debug?.report_generation_shaped).toBe(false) + expect(decision.signals.generation_intent).toBe('display_rendering') + expect(decision.signals.target_domain_hint).toBe('frontend_display') + + const withBackendMarker = classify({ prompt: 'Explain how the idea report is generated and displayed in the footer by the worker pipeline' }) + expect(withBackendMarker.signals.generation_intent).toBe('runtime_generation') + expect(withBackendMarker.signals.target_domain_hint).toBe('backend_runtime') }) it('detects broad generation noun prompts without explanation verbs', () => { diff --git a/tests/unit/retrieve-production-correctness.test.ts b/tests/unit/retrieve-production-correctness.test.ts index 20e6f4fa..3bf68785 100644 --- a/tests/unit/retrieve-production-correctness.test.ts +++ b/tests/unit/retrieve-production-correctness.test.ts @@ -345,7 +345,14 @@ describe('retrieveContext production retrieval regressions', () => { } }) - it('adds a semantic generation-core anchor for broad report-generation prompts instead of centering only the HTTP trigger', () => { + // #660 Slice C. This test required a SECOND anchor to be forced in beside the + // real entry point whenever the question used report vocabulary. The anchor + // was picked by `semanticGenerationCoreAnchorValue`, a table that paid +11 for + // a label containing "planner", +10 for "assembly"/"renderer"/"final report" + // and +7 for "research"/"scoring" — a name-shaped ranking with no structural + // basis, reachable only through a task-phrase classifier over the question. + // Both the table and the forced pick are gone, so the assertion is inverted. + it('does not force a name-ranked second anchor for report-shaped questions (#660 Slice C)', () => { const result = retrieveContext(buildBroadReportGenerationGraph(), { question: 'How idea report is being generated', budget: 4000, @@ -353,34 +360,23 @@ describe('retrieveContext production retrieval regressions', () => { retrievalStrategy: 'slice-v1', }) + // The real entry point is still anchored by ordinary evidence. expect(result.slice?.anchors.map((anchor) => anchor.label)).toEqual(expect.arrayContaining([ '.generateFromProblem()', ])) - expect( - result.slice?.anchors.some((anchor) => [ - '.plan()', - '.processSection()', - '.search()', - '.assembleReport()', - '.score()', - '.validateReport()', - '.renderFinalReport()', - '.saveStructuredReport()', - ].includes(anchor.label)), - ).toBe(true) - expect(result.matched_nodes.map((node) => node.label)).toEqual(expect.arrayContaining([ - '.plan()', - '.processSection()', - '.search()', - '.assembleReport()', - '.score()', - '.validateReport()', - '.renderFinalReport()', - '.saveStructuredReport()', - ])) + // Nothing is anchored by the retired heuristic. + expect(result.slice?.anchors.map((anchor) => anchor.reason)).not.toContain('generation core heuristic') + for (const anchor of result.slice?.anchors ?? []) { + expect(['symbol mention', 'path mention', 'source path token match', 'top lexical match']).toContain(anchor.reason) + } }) - it('prefers central report-generation nodes over leaf helper generators for broad report-generation prompts', () => { + // #660 Slice C. This test asserted that `.plan()` / `.assembleReport()` / + // `.scoreReport()` beat `.generateScoringLedger()` and friends. Both groups sit + // at the same place in the graph; the winner was decided purely by which + // NAME resembled a report workflow stage. That is the same-name/wrong-semantics + // defect, so the preference is gone and only the evidence-backed anchor stays. + it('ranks no anchor by how much its name resembles a report stage (#660 Slice C)', () => { const result = retrieveContext(buildReportGenerationLeafHelperGraph(), { question: 'How idea report is being generated', budget: 4000, @@ -388,72 +384,64 @@ describe('retrieveContext production retrieval regressions', () => { retrievalStrategy: 'slice-v1', }) - const anchorLabels = result.slice?.anchors.map((anchor) => anchor.label) ?? [] - expect(anchorLabels).toEqual(expect.arrayContaining([ + const anchors = result.slice?.anchors ?? [] + expect(anchors.map((anchor) => anchor.label)).toEqual(expect.arrayContaining([ '.generateFromProblem()', ])) - expect(anchorLabels).toEqual(expect.arrayContaining([ - expect.stringMatching(/^\.plan\(\)$|^\.assembleReport\(\)$|^\.scoreReport\(\)$/), - ])) - expect(anchorLabels).not.toEqual(expect.arrayContaining([ - '.generateScoringLedger()', - '.generateSensitivityAnalysis()', - '.generateTitle()', - ])) + expect(anchors.map((anchor) => anchor.reason)).not.toContain('generation core heuristic') + for (const anchor of anchors) { + expect(['symbol mention', 'path mention', 'source path token match', 'top lexical match']).toContain(anchor.reason) + } }) - it('keeps backward-selected runtime flow as structural evidence for broad report-generation prompts', () => { - const result = retrieveContext(buildReverseFlowReportGenerationGraph(), { - question: 'How idea report is being generated', + // #660 Slice C. This test required a deep BACKWARD walk over `calls` edges for + // report-shaped questions. That policy — backward depth 3 with `calls` added + // to the backward relation set — existed only inside the branch gated on the + // report task-phrase classifier. Measured at the pre-change commit, the very + // same graph asked with equivalent NEUTRAL wording ("How idea invoice is being + // generated") returned ordered_ids ["route"] and ZERO selected paths. So this + // evidence was never generic: it was an advantage that report vocabulary alone + // bought. The test is inverted into that independence control. + it('gives report wording no structural evidence that neutral wording cannot get (#660 Slice C)', () => { + const ask = (question: string) => retrieveContext(buildReverseFlowReportGenerationGraph(), { + question, budget: 4000, retrievalLevel: 4, retrievalStrategy: 'slice-v1', }) - expect(result.slice?.selected_paths).toEqual(expect.arrayContaining([ - expect.objectContaining({ from: '.startPipeline()', to: '.generateFromProblem()', relation: 'calls' }), - expect.objectContaining({ from: '.addJob()', to: '.startPipeline()', relation: 'calls' }), - expect.objectContaining({ from: '.process()', to: '.addJob()', relation: 'enqueues_job' }), - expect.objectContaining({ from: '.scoreMetrics()', to: '.assembleReport()', relation: 'calls' }), - ])) + const report = ask('How idea report is being generated') + const neutral = ask('How idea invoice is being generated') - const matched = result.matched_nodes.map((node) => ({ label: node.label, evidence_class: node.evidence_class })) - expect(matched).toEqual(expect.arrayContaining([ - expect.objectContaining({ label: '.startPipeline()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.addJob()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.process()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.assembleReport()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.scoreMetrics()', evidence_class: 'structural' }), - ])) - expect(matched).not.toEqual(expect.arrayContaining([ - expect.objectContaining({ label: '.getStatusMessage()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.generateTitle()', evidence_class: 'structural' }), - ])) + const shape = (result: ReturnType) => ({ + member_ids: result.matched_nodes.map((node) => node.node_id ?? node.label), + paths: (result.slice?.selected_paths ?? []).map((path) => `${path.from}-[${path.relation}]->${path.to}`), + reasons: (result.slice?.anchors ?? []).map((anchor) => anchor.reason), + }) + + expect(shape(report)).toEqual(shape(neutral)) + expect(shape(report).reasons).not.toContain('generation core heuristic') }) - // #660-B1. Was also asserting that '.getStatusMessage()' and - // '.generateTitle()' are absent. That exclusion was produced by a denylist of - // fourteen symbol names lifted from one repository, not by any structural - // property, so it is gone and those labels can appear again. What is still - // worth holding, and is kept, is that the execution flow itself is compacted - // around the traced steps. - it('compacts a broad pack around the traced execution flow', () => { - const compact = compactRetrieveResult(retrieveContext(buildReverseFlowReportGenerationGraph(), { - question: 'How idea report is being generated', + // #660-B1 removed a fourteen-symbol denylist from this test. #660 Slice C + // removes the other half: the labels it still required were reachable only + // through the report-gated backward policy, so requiring them here is + // requiring the special behaviour. What survives as a genuine, generic claim + // is that compaction does not INVENT membership and stays wording-independent. + it('compacts a broad pack identically under report and neutral wording (#660 Slice C)', () => { + const compact = (question: string) => compactRetrieveResult(retrieveContext(buildReverseFlowReportGenerationGraph(), { + question, budget: 4000, retrievalLevel: 4, retrievalStrategy: 'slice-v1', - })) + })).matched_nodes.map((node) => node.label) - const labels = compact.matched_nodes.map((node) => node.label) - expect(labels).toEqual(expect.arrayContaining([ - '.generateFromProblem()', - '.startPipeline()', - '.addJob()', - '.process()', - '.assembleReport()', - '.scoreMetrics()', - ])) + const report = compact('How idea report is being generated') + const neutral = compact('How idea invoice is being generated') + + expect(report).toEqual(neutral) + // Compaction still reports the evidence it does have, rather than nothing. + expect(report).toEqual(expect.arrayContaining(['.generateFromProblem()'])) }) // #660-B1. Two tests were removed here. Both asserted the EXPECTED phase From a0a7640c4d40d6b9522a25faa4813dc11098edaa Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 31 Aug 2026 19:16:42 +0400 Subject: [PATCH 2/3] fix(context): stop report stage vocabulary reaching the runtime gate Bounded correction to #660, Slice C, from the FINAL review. `genericGenerationShaped` still listed `assembl(e|ed|es|ing)`. Assembly is one of the six declared report workflow stages, and it was the only one of the six that could reach a runtime-generation gate on its own: Explain planner -> unknown Explain research -> unknown Explain assembly -> unknown Explain scoring -> unknown Explain persistence -> unknown Explain assemble -> runtime_generation / backend_runtime Measured against the built candidate, `assemble` behaved identically to `generate`, `create`, `build` and `produce`, so it was acting as a member of the generic generation-verb family rather than as a report signal. The asymmetry across the six stages is real all the same, so the token is removed rather than argued away. The remaining verbs carry the generic sense without naming a report stage, and an assembly question with real backend evidence still classifies: How is the response assembled by the worker pipeline -> runtime_generation How is the invoice assembled and saved to the repository -> runtime_generation Explain how the bundle is assembled -> unknown The last line is the intended cost, and it is the same answer the other five stage words already gave. Control A missed this because its minimal pair already carried the generic verb "generated", so a stage word reaching the gate through the generic list stayed invisible. New control A2 walks the six stages one at a time, asserts the generic display classifier still owns `rendering`, and fails in both directions by requiring the same words to earn a runtime gate once backend evidence is present. Production scope is unchanged at three files. --- src/runtime/retrieval-gate.ts | 10 ++++- .../report-generation-independence.test.ts | 38 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/runtime/retrieval-gate.ts b/src/runtime/retrieval-gate.ts index 07c78e93..8a468b00 100644 --- a/src/runtime/retrieval-gate.ts +++ b/src/runtime/retrieval-gate.ts @@ -239,7 +239,15 @@ function detectGenerationIntent(prompt: string): { } { const lower = prompt.toLowerCase() const displayShaped = /\b(?:display(?:ed|ing)?|render(?:ed|ing)?|show(?:n|ing)?|visible|view|ui|frontend|front-end|component|screen|page|footer|header|label|date|timestamp)\b/i.test(lower) - const genericGenerationShaped = /\b(?:generat(?:e|ed|es|ing|ion)|creat(?:e|ed|es|ing|ion)|build(?:s|ing|er)?|assembl(?:e|ed|es|ing)|produc(?:e|ed|es|ing))\b/i.test(lower) + // `assembl*` was dropped from this list in #660 Slice C. It is one of the six + // report workflow stages, and it was the only one of the six that could still + // reach a runtime-generation gate on its own: `Explain assemble` classified as + // runtime_generation while `Explain planner`, `Explain research`, + // `Explain assembly`, `Explain scoring` and `Explain persistence` all returned + // unknown. The remaining verbs carry the generic sense without naming a report + // stage, and any assembly question with real backend evidence still classifies + // through `backendRuntimeShaped` plus `explanationShaped`. + const genericGenerationShaped = /\b(?:generat(?:e|ed|es|ing|ion)|creat(?:e|ed|es|ing|ion)|build(?:s|ing|er)?|produc(?:e|ed|es|ing))\b/i.test(lower) const backendRuntimeShaped = /\b(?:pipeline|runtime|orchestrator|worker|job|repository|service|controller|api|backend|persist(?:ed|ing)?|sav(?:e|ed|es|ing)|db(?:sync|[- ]sync)?)\b/i.test(lower) const buildStaticShaped = /\b(?:next\.js|nextjs|app\s+router|route\s+segment|landing\s+page|static|ssg|isr|server\s+component)\b/i.test(lower) const explanationShaped = /\b(?:explain|how|why|trace|walk|flow|path|lifecycle|fail(?:s|ing|ed)?)\b/i.test(lower) diff --git a/tests/unit/report-generation-independence.test.ts b/tests/unit/report-generation-independence.test.ts index 83d20ca0..45790413 100644 --- a/tests/unit/report-generation-independence.test.ts +++ b/tests/unit/report-generation-independence.test.ts @@ -229,6 +229,44 @@ describe('#660 Slice C — report-generation independence', () => { expect(shape.member_ids.length).toBeLessThanOrEqual(2) }) + it('A2: none of the six declared report workflow stages reaches a runtime gate on its own', () => { + // The gap that the first FINAL review found. Control A above tests ONE + // minimal pair, and its prompts already carried the generic verb + // "generated", so a report stage word that reached the gate through the + // generic generation-verb list stayed invisible. `assemble` did exactly + // that. This control walks the six declared stages one at a time. + for (const stage of ['planner', 'research', 'assembly', 'assemble', 'scoring', 'persistence']) { + const decision = classifyRetrievalLevel({ prompt: `Explain ${stage}` }) + expect( + { stage, intent: decision.signals.generation_intent }, + `"Explain ${stage}" must not earn a runtime-generation gate on report vocabulary alone`, + ).toEqual({ stage, intent: 'unknown' }) + expect(decision.signals.generation_debug?.report_generation_shaped).toBe(false) + } + + // `rendering` is the one stage word that still classifies, as + // display_rendering, through the generic frontend-display classifier that + // Slice C did not touch. That is a generic lexical outcome, not a report + // policy, and it is asserted rather than left ambiguous. + expect(classifyRetrievalLevel({ prompt: 'Explain rendering' }).signals.generation_intent).toBe('display_rendering') + + // Not a constant: the SAME stage words earn a runtime gate the moment real + // backend evidence is present, so the control fails in both directions. + for (const stage of ['assembly', 'assemble', 'scoring', 'persistence']) { + const withEvidence = classifyRetrievalLevel({ prompt: `Explain ${stage} in the worker pipeline` }) + expect(withEvidence.signals.generation_intent).toBe('runtime_generation') + expect(withEvidence.signals.target_domain_hint).toBe('backend_runtime') + } + + // And the generic generation verbs that remain are untouched: the word + // "report" contributes nothing to them. + for (const verb of ['generate', 'generation', 'create', 'build', 'produce']) { + expect(classifyRetrievalLevel({ prompt: `Explain ${verb}` }).signals.generation_intent).toBe('runtime_generation') + } + expect(classifyRetrievalLevel({ prompt: 'Explain report generation' }).signals.generation_intent) + .toBe(classifyRetrievalLevel({ prompt: 'Explain generation' }).signals.generation_intent) + }) + // ------------------------------------------------------------- control B -- it('B: the same repository and task intent yield the same structure under neutral and report wording', () => { // Equivalent intent over identical evidence. The only difference is the From c3db6e7c3c1e562d66593f2173add73025b55b14 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 31 Aug 2026 19:41:33 +0400 Subject: [PATCH 3/3] fix(context): close the report-stage name leak and the injection false-credit path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves both PR review threads on #660 Slice C. Two verified defects, both in the evidence the slice rests on. 1. `research` was still earning runtime-expansion preference. Slice C removed the report stage names from `pipelineBridgeLikeNode` and `highValueRuntimeExpansionNode`, but those patterns anchor only their first and last alternatives, so the unanchored `search` matched the tail of "research" and the removal was silently defeated. Measured: `.researchStep()` in `/src/flow/research.service.ts` still classified as a high-value runtime expansion node. `persistence` likewise still matches through `persist`. `search` is now anchored on its left. The other terms keep their existing loose anchoring deliberately — they are meant to match inflections such as "workers", "jobs" and "pipelines", and tightening every alternative would change generic ranking well beyond this slice. `persist` deliberately still matches "persistence": it is a generic runtime verb naming an ordinary backend layer, already judged generic in this slice when removing it from the prompt classifier regressed a neutral login fixture. The rule is that a report stage NOUN confers nothing while a generic runtime VERB may. New control C3 covers the path that let this through. Control C never reaches these classifiers, because they run only under a runtime-flow-only forward policy. C3 drives the slicing entry point with an exact symbol anchor and observes INCLUSION rather than ordering — a third hop is reachable only if the middle node earned the preference. Verified in both directions: it fails with the leak restored and passes with it closed, from a digest-checked snapshot. 2. The injection harness could hand out credit it had not earned. It ran each control only AFTER mutating the source. A control that was already red would therefore have made every injection look successful, since the failure would not have been caused by the injection. Each injection now requires the named control to be GREEN before the mutation and RED after it. Production scope is unchanged at three files. --- .../verify-report-generation-injections.mjs | 25 +++++++-- src/runtime/retrieve/slicing.ts | 16 +++++- .../report-generation-independence.test.ts | 54 +++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/scripts/verify-report-generation-injections.mjs b/scripts/verify-report-generation-injections.mjs index 7c9a3e07..fa4ecab5 100755 --- a/scripts/verify-report-generation-injections.mjs +++ b/scripts/verify-report-generation-injections.mjs @@ -107,6 +107,9 @@ const INJECTIONS = [ }, ] +/** Thrown when the pre-injection baseline already fails, so the injection is void. */ +class SkipInjection extends Error {} + function digest(text) { return createHash('sha256').update(text).digest('hex') } @@ -156,23 +159,35 @@ function main() { let outcome try { + // BEFORE anything is mutated, the named control must be GREEN. Without + // this baseline a control that was already red would hand every injection + // free credit: the post-injection failure would prove nothing, because the + // failure was not caused by the injection. + const before = runControl(injection.control) + if (before.failed) { + outcome = { ok: false, detail: 'the named control was ALREADY FAILING before injection — its later failure would prove nothing' } + throw new SkipInjection() + } + const injected = injection.apply(original) if (injected === original) throw new Error(`${injection.id}: injection changed nothing`) writeFileSync(path, injected) // The injected path must actually be REACHED, not merely present. - const baseline = runControl(injection.control) - if (!baseline.failed) { + const after = runControl(injection.control) + if (!after.failed) { outcome = { ok: false, detail: 'the named control still PASSED with the rule restored — it cannot catch its own mutation' } } else { // Credit only for the named control failing, not for any failure. - const named = baseline.output.includes(injection.control.name) + const named = after.output.includes(injection.control.name) outcome = named - ? { ok: true, detail: `${injection.marker}: named control failed as required` } + ? { ok: true, detail: `${injection.marker}: named control was green before injection and failed after, as required` } : { ok: false, detail: 'a failure occurred but it was not the named control' } } } catch (error) { - outcome = { ok: false, detail: `injection error: ${error.message}` } + if (!(error instanceof SkipInjection)) { + outcome = { ok: false, detail: `injection error: ${error.message}` } + } } finally { writeFileSync(path, original) chmodSync(path, originalMode) diff --git a/src/runtime/retrieve/slicing.ts b/src/runtime/retrieve/slicing.ts index 7a0f1224..0e3109f5 100644 --- a/src/runtime/retrieve/slicing.ts +++ b/src/runtime/retrieve/slicing.ts @@ -604,14 +604,26 @@ function traverseDirection( } } +// `search` is anchored on its left. Without that anchor it also matches the tail +// of "research", so a symbol named ResearchAgent kept the runtime-expansion +// preference that #660 Slice C removed for report workflow stage names — the +// removal was silently defeated by an unrelated alternative. The other terms +// keep their existing loose anchoring on purpose: they are meant to match +// inflections such as "workers", "jobs" and "pipelines", and tightening every +// alternative would change generic ranking well beyond this slice. +// +// `persist` deliberately still matches "persistence": it is a generic runtime +// verb naming an ordinary backend layer, judged generic earlier in this slice +// when removing it from the prompt classifier regressed a neutral login fixture. +// A report stage NOUN must confer nothing; a generic runtime verb may. function pipelineBridgeLikeNode(node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|search|addjob\b/.test(lower) + return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|\bsearch|addjob\b/.test(lower) } function highValueRuntimeExpansionNode(node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|search|dispatch|persist|builder|addjob\b/.test(lower) + return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|\bsearch|dispatch|persist|builder|addjob\b/.test(lower) } function sharedHubLikeNode(graph: KnowledgeGraph, node: SliceScoredNode): boolean { diff --git a/tests/unit/report-generation-independence.test.ts b/tests/unit/report-generation-independence.test.ts index 45790413..e5531701 100644 --- a/tests/unit/report-generation-independence.test.ts +++ b/tests/unit/report-generation-independence.test.ts @@ -24,10 +24,12 @@ import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' import type { ContextPackExecutionPhase } from '../../src/contracts/context-pack.js' +import { KnowledgeGraph } from '../../src/contracts/graph.js' import { build } from '../../src/pipeline/build.js' import { buildMadarPromptPack } from '../../src/infrastructure/prompt-pack.js' import { classifyRetrievalLevel } from '../../src/runtime/retrieval-gate.js' import { retrieveContext, type RetrieveResult } from '../../src/runtime/retrieve.js' +import { sliceCandidatesForRetrieve } from '../../src/runtime/retrieve/slicing.js' // The retired qualification vocabulary, split so a control never confuses a // LEGITIMATE lexical effect with a report-policy effect. `rendering`/`renderer` @@ -316,6 +318,58 @@ describe('#660 Slice C — report-generation independence', () => { expect(instructionsFor(REPORT_Q)).toEqual(BASE_INSTRUCTIONS) }) + it('C3: a report-stage node name confers no runtime-expansion preference', () => { + // The gap CodeRabbit found. `highValueRuntimeExpansionNode` runs only under a + // runtime-flow-only forward policy, which control C never reaches, so a report + // stage name leaking back through an unrelated alternative stayed invisible: + // `research` matched the unanchored tail of `search`. + // + // The observable effect is INCLUSION, not ordering: at depth > 0 a forward + // walk continues past a node only when `shouldExpandRuntimePathNode` says so, + // and being "high value" is what says so. So the third hop is reachable only + // if the middle node earned the preference. Asserting on ordering instead + // would have been a control that cannot catch its own mutation. + const chain = (midLabel: string, midFile: string) => { + const graph = new KnowledgeGraph({ directed: true }) + graph.addNode('entry', { label: '.startRun()', source_file: '/src/flow/trigger.service.ts', node_kind: 'method', file_type: 'code' }) + graph.addNode('mid', { label: midLabel, source_file: midFile, node_kind: 'method', file_type: 'code' }) + graph.addNode('deep', { label: '.writeLedger()', source_file: '/src/flow/ledger.service.ts', node_kind: 'method', file_type: 'code' }) + graph.addEdge('entry', 'mid', { relation: 'calls' }) + graph.addEdge('mid', 'deep', { relation: 'calls' }) + return graph + } + const run = (midLabel: string, midFile: string) => sliceCandidatesForRetrieve( + chain(midLabel, midFile), + [ + { id: 'entry', label: '.startRun()', sourceFile: '/src/flow/trigger.service.ts', nodeKind: 'method', exactLabelMatch: true, sourcePathMatch: false, literalPathMatch: false, score: 1 }, + { id: 'mid', label: midLabel, sourceFile: midFile, nodeKind: 'method', exactLabelMatch: false, sourcePathMatch: false, literalPathMatch: false, score: 0.5 }, + { id: 'deep', label: '.writeLedger()', sourceFile: '/src/flow/ledger.service.ts', nodeKind: 'method', exactLabelMatch: false, sourcePathMatch: false, literalPathMatch: false, score: 0.4 }, + ], + 'explain', + { + prompt: 'Explain how `.startRun()` moves through the runtime pipeline', + generationIntent: 'runtime_generation', + targetDomainHint: 'backend_runtime', + }, + )?.ordered_ids ?? [] + + // The three labels are chosen so that ONLY the intended signal varies: none + // of them carries an unrelated generic runtime term. An earlier draft used + // `.researchAgent()`, whose "agent" is itself a generic runtime word that was + // deliberately kept, so the fixture could not isolate the report stage name. + const neutral = run('.lookupStep()', '/src/flow/lookup.service.ts') + const reportNamed = run('.researchStep()', '/src/flow/research.service.ts') + + // Naming the middle hop after a report stage buys it nothing. + expect(reportNamed).toEqual(neutral) + expect(reportNamed).not.toContain('deep') + + // Not a constant: a genuinely generic runtime name DOES earn the expansion, + // so this control can tell the two apart and fails in both directions. + const genericRuntime = run('.searchStep()', '/src/flow/search.service.ts') + expect(genericRuntime).toContain('deep') + }) + // ------------------------------------------------------------- control D -- it('D: the pre-existing non-qualification parity golden still carries its typed evidence instructions', () => { // tests/fixtures/prompt-pack-parity.golden.json was captured from a commit