diff --git a/package.json b/package.json index 7080d281..753cb82d 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "verify:grader-boundary-controls": "node scripts/verify-grader-boundary.mjs --self-test", "verify:forbidden-knowledge": "node scripts/verify-forbidden-knowledge.mjs", "verify:forbidden-knowledge-controls": "node scripts/verify-forbidden-knowledge.mjs --self-test", + "verify:report-generation-injections": "node scripts/verify-report-generation-injections.mjs", "verify:grader-boundary-runtime": "node scripts/verify-grader-boundary-runtime.mjs", "verify:integrity-mutations": "node scripts/verify-integrity-mutations.mjs && MADAR_MUTATION_HARNESS_E2E=1 npx vitest run tests/unit/mutation-harness-self.test.ts", "verify:integrity-receipts": "node scripts/verify-integrity-receipts.mjs", diff --git a/scripts/lib/forbidden-knowledge-manifest.json b/scripts/lib/forbidden-knowledge-manifest.json index 6c8b4cb2..d5d8da84 100644 --- a/scripts/lib/forbidden-knowledge-manifest.json +++ b/scripts/lib/forbidden-knowledge-manifest.json @@ -1,7 +1,7 @@ { "manifest_version": "1.0.0", "issue": 660, - "slice": "B", + "slice": "B+C", "title": "Production independence from qualification-repository knowledge", "purpose": [ "Madar's retrieval and claim behaviour must be driven by generic structural evidence,", @@ -54,7 +54,13 @@ { "id": "report-generation/symbol-dispatch-db-sync", "repository": "govalidate-report-generation", "class": "symbol", "value": "dispatchDbSync", "why": "Qualification-target symbol in a fixed promotion table." }, { "id": "report-generation/symbol-broadcast-run-started", "repository": "govalidate-report-generation", "class": "symbol", "value": "broadcastRunStarted", "why": "Qualification-target symbol in a fixed promotion table." }, { "id": "report-generation/symbol-broadcast-run-failed", "repository": "govalidate-report-generation", "class": "symbol", "value": "broadcastRunFailed", "why": "Qualification-target symbol in a fixed promotion table." }, - { "id": "report-generation/symbol-score-metrics", "repository": "govalidate-report-generation", "class": "symbol", "value": "scoreMetrics", "why": "Qualification-target symbol in a fixed promotion table." } + { "id": "report-generation/symbol-score-metrics", "repository": "govalidate-report-generation", "class": "symbol", "value": "scoreMetrics", "why": "Qualification-target symbol in a fixed promotion table." }, + { "id": "report-generation/symbol-prompt-wants-report-generation-core", "repository": "govalidate-report-generation", "class": "symbol", "value": "promptWantsReportGenerationCore", "why": "Name of the retired report-generation task-phrase classifier. It was duplicated in prompt-pack.ts and retrieve/slicing.ts and gated every report-only behaviour removed in #660 Slice C." }, + { "id": "report-generation/symbol-semantic-generation-core-anchor-value", "repository": "govalidate-report-generation", "class": "symbol", "value": "semanticGenerationCoreAnchorValue", "why": "Name of the retired name-driven anchor score table that paid a symbol for resembling a report workflow stage." }, + { "id": "report-generation/phrase-generation-core-heuristic-anchor", "repository": "govalidate-report-generation", "class": "phrase", "value": "generation core heuristic", "why": "Anchor reason emitted only by the retired forced report-anchor selection." }, + { "id": "report-generation/phrase-fixed-report-workflow-instruction", "repository": "govalidate-report-generation", "class": "phrase", "value": "planner, research, assembly, scoring, rendering, and persistence evidence", "why": "The retired fixed report workflow instruction, manufactured from prompt vocabulary with no evidence behind it." }, + { "id": "report-generation/phrase-final-report-score-entry", "repository": "govalidate-report-generation", "class": "phrase", "value": "final(?:-| )report", "why": "A distinctive +10 entry of the retired anchor score table." }, + { "id": "report-generation/phrase-idea-report-gate-entry", "repository": "govalidate-report-generation", "class": "phrase", "value": "idea\\s+report", "why": "The qualification task name, matched by the retired reportGenerationShaped gate classifier." } ], "corpus_symbol_import": { "source": "docs/qualification/corpus.json", diff --git a/scripts/verify-report-generation-injections.mjs b/scripts/verify-report-generation-injections.mjs new file mode 100755 index 00000000..fa4ecab5 --- /dev/null +++ b/scripts/verify-report-generation-injections.mjs @@ -0,0 +1,234 @@ +#!/usr/bin/env node +// #660 Slice C — focused falsifiability for the retired report-generation class. +// +// Four injections, no more. Each one restores ONE retired rule into production +// source from a digest-checked byte snapshot, requires a NAMED control to fail, +// and restores the exact bytes and file mode in `finally`. Unrelated failures +// earn no credit: an injection passes only when the control it names fails. +// +// This is deliberately NOT a general mutation framework. It is a fixed list of +// four edits with a fixed expected victim each. +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync, statSync, writeFileSync, chmodSync } from 'node:fs' +import { resolve } from 'node:path' + +const PROMPT_PACK = 'src/infrastructure/prompt-pack.ts' +const SLICING = 'src/runtime/retrieve/slicing.ts' +const GATE = 'src/runtime/retrieval-gate.ts' + +const CONTROL_FILE = 'tests/unit/report-generation-independence.test.ts' + +/** The retired task-phrase classifier, restored verbatim where an injection needs it. */ +const RETIRED_CLASSIFIER = `function promptWantsReportGenerationCore(prompt) { + return /\\b(?:report(?:\\s+generation)?|generated\\s+report|validation\\s+report|final\\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\\s|-)?gate)\\b/i.test(prompt) +} +` + +const INJECTIONS = [ + { + id: 'C1', + title: 'the fixed report workflow instruction', + marker: 'SLICE_C_FIXED_REPORT_INSTRUCTION_REINTRODUCED', + file: PROMPT_PACK, + control: { file: CONTROL_FILE, name: 'A: report vocabulary alone produces no fixed workflow' }, + apply(source) { + // Put the retired fixed instruction back, unconditionally, so the + // vocabulary-only control must observe it. + const anchor = ` const instructions = [ + 'Treat HTTP/controller entrypoints as trigger context, not the full answer.', + ]` + if (!source.includes(anchor)) throw new Error('C1 anchor missing') + return source.replace( + ` ...answerContractInstructions(input.retrieval),`, + ` ...answerContractInstructions(input.retrieval), + 'Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.',`, + ) + }, + }, + { + id: 'C2', + title: 'a report-generation task-phrase classifier', + marker: 'SLICE_C_TASK_PHRASE_CLASSIFIER_REINTRODUCED', + file: SLICING, + control: { file: CONTROL_FILE, name: 'B: the same repository and task intent yield the same structure' }, + apply(source) { + // Restore the classifier AND one behaviour it used to gate: a deeper + // backward policy for report-shaped prompts only. + const anchor = ` if (!hasMethodAnchor && !pipelinePrompt) { + return base + }` + if (!source.includes(anchor)) throw new Error('C2 anchor missing') + return source + .replace('function promptWantsRuntimePipeline(', `${RETIRED_CLASSIFIER}\nfunction promptWantsRuntimePipeline(`) + .replace(anchor, `${anchor} + + if (broadRuntimeGeneration && pipelinePrompt && promptWantsReportGenerationCore(options.prompt)) { + return { + ...base, + directions: ['backward', 'forward'], + backward_relations: new Set(['calls', 'enqueues_job', 'controller_route', 'route_handler', 'method']), + forward_relations: new Set(RUNTIME_FLOW_RELATIONS), + helper_relations: new Set(['injects', 'depends_on', 'module_provides']), + backward_depth: Math.max(base.backward_depth, 3), + forward_depth: Math.max(base.forward_depth, 4), + runtime_flow_only: true, + } + }`) + }, + }, + { + id: 'C3', + title: 'a name-driven anchor score-table entry', + marker: 'SLICE_C_NAME_DRIVEN_SCORE_TABLE_REINTRODUCED', + file: SLICING, + control: { file: CONTROL_FILE, name: 'C: identical structure named after report stages earns no extra structural treatment' }, + apply(source) { + // One score-table entry: a symbol is paid for being NAMED like a report + // stage. The same-name/different-semantics control must notice. + const anchor = ` if (/\\b(?:service|provider|repository|queue|worker|orchestrator)\\b/.test(lower)) value += 1` + if (!source.includes(anchor)) throw new Error('C3 anchor missing') + return source.replace(anchor, `${anchor} + if (/\\b(?:planner|assembly|assemble|renderer|synthesis|scoring|research)\\b/.test(lower)) value += 11`) + }, + }, + { + id: 'C4', + title: 'the reportGenerationShaped gate variant', + marker: 'SLICE_C_REPORT_GATE_VARIANT_REINTRODUCED', + file: GATE, + control: { file: CONTROL_FILE, name: 'A: report vocabulary alone produces no fixed workflow' }, + apply(source) { + const anchor = ` const strongRuntimeShaped = backendRuntimeShaped` + if (!source.includes(anchor)) throw new Error('C4 anchor missing') + return source.replace(anchor, ` const reportGenerationShaped = /\\b(?:idea\\s+report|report\\s+generation|final\\s+report|planner|research|metrics?|scor(?:e|ing)|quality(?:\\s|-)?gate|renderer|synthesis|assemble|assembly)\\b/i.test(lower) + const strongRuntimeShaped = backendRuntimeShaped || reportGenerationShaped`) + }, + }, +] + +/** Thrown when the pre-injection baseline already fails, so the injection is void. */ +class SkipInjection extends Error {} + +function digest(text) { + return createHash('sha256').update(text).digest('hex') +} + +function untrackedSources() { + return execFileSync('git', ['status', '--porcelain', '--untracked-files=all', '--', 'src', 'tests', 'scripts'], { encoding: 'utf8' }) + .split('\n') + .filter((line) => line.startsWith('??')) + .map((line) => line.slice(3).trim()) + .filter((file) => file.length > 0) +} + +function runControl(control) { + try { + execFileSync('npx', ['vitest', 'run', control.file, '-t', control.name], { + stdio: 'pipe', + encoding: 'utf8', + env: { ...process.env, CI: '1' }, + }) + return { failed: false } + } catch (error) { + const output = `${error.stdout ?? ''}${error.stderr ?? ''}` + return { failed: true, output } + } +} + +function main() { + const results = [] + let allPassed = true + + // Fingerprint the three production files BEFORE any injection. Comparing to + // git HEAD would be wrong: on a working branch these files legitimately differ + // from HEAD, so only a self-taken snapshot can prove the harness left nothing + // behind. + const startFingerprint = new Map( + [PROMPT_PACK, SLICING, GATE].map((file) => [file, digest(readFileSync(resolve(file), 'utf8'))]), + ) + // Untracked files that already exist are the branch's own new files, not this + // harness's residue. Only files that appear DURING the run count. + const startUntracked = new Set(untrackedSources()) + + for (const injection of INJECTIONS) { + const path = resolve(injection.file) + const original = readFileSync(path, 'utf8') + const originalDigest = digest(original) + const originalMode = statSync(path).mode + + let outcome + try { + // BEFORE anything is mutated, the named control must be GREEN. Without + // this baseline a control that was already red would hand every injection + // free credit: the post-injection failure would prove nothing, because the + // failure was not caused by the injection. + const before = runControl(injection.control) + if (before.failed) { + outcome = { ok: false, detail: 'the named control was ALREADY FAILING before injection — its later failure would prove nothing' } + throw new SkipInjection() + } + + const injected = injection.apply(original) + if (injected === original) throw new Error(`${injection.id}: injection changed nothing`) + writeFileSync(path, injected) + + // The injected path must actually be REACHED, not merely present. + const after = runControl(injection.control) + if (!after.failed) { + outcome = { ok: false, detail: 'the named control still PASSED with the rule restored — it cannot catch its own mutation' } + } else { + // Credit only for the named control failing, not for any failure. + const named = after.output.includes(injection.control.name) + outcome = named + ? { ok: true, detail: `${injection.marker}: named control was green before injection and failed after, as required` } + : { ok: false, detail: 'a failure occurred but it was not the named control' } + } + } catch (error) { + if (!(error instanceof SkipInjection)) { + outcome = { ok: false, detail: `injection error: ${error.message}` } + } + } finally { + writeFileSync(path, original) + chmodSync(path, originalMode) + const restored = readFileSync(path, 'utf8') + if (digest(restored) !== originalDigest) { + console.error(`FATAL ${injection.id}: ${injection.file} was NOT restored to its original bytes`) + process.exit(2) + } + } + + allPassed = allPassed && outcome.ok + results.push({ injection, outcome }) + console.log(`${outcome.ok ? 'PASS' : 'FAIL'} ${injection.id} ${injection.title}`) + console.log(` ${outcome.detail}`) + } + + // Final fingerprint: every touched file is byte-identical to its start state. + console.log('') + console.log('Worktree fingerprint for the three production files after restoration:') + let drifted = false + for (const [file, expected] of startFingerprint) { + const actual = digest(readFileSync(resolve(file), 'utf8')) + const same = actual === expected + drifted = drifted || !same + console.log(` ${same ? 'restored' : 'DRIFTED '} ${file} sha256 ${actual.slice(0, 16)}`) + } + if (drifted) { + console.error('FATAL: a production file did not return to its pre-injection bytes.') + process.exit(2) + } + + // No temporary source or test residue may survive the run. + const leaked = untrackedSources().filter((file) => !startUntracked.has(file)) + console.log(leaked.length === 0 + ? ' no new untracked residue under src/, tests/ or scripts/' + : ` UNTRACKED RESIDUE: ${leaked.join(' ')}`) + if (leaked.length > 0) process.exit(2) + + console.log('') + console.log(`${results.filter((entry) => entry.outcome.ok).length}/${INJECTIONS.length} focused injections behaved as required.`) + process.exit(allPassed ? 0 : 1) +} + +main() diff --git a/src/infrastructure/prompt-pack.ts b/src/infrastructure/prompt-pack.ts index c5a097b9..b1d1058b 100644 --- a/src/infrastructure/prompt-pack.ts +++ b/src/infrastructure/prompt-pack.ts @@ -54,20 +54,13 @@ function answerContractInstructions(retrieval: RetrieveResult): string[] { ] const requiredElements = new Set(answerContract.required_elements) - const phaseLabels = [ - ['planner_phase', 'planner'], - ['research_phase', 'research'], - ['assembly_phase', 'assembly'], - ['scoring_phase', 'scoring'], - ['report_builder_phase', 'rendering'], - ] as const satisfies ReadonlyArray - const selectedPhaseLabels: string[] = phaseLabels.flatMap(([key, label]) => requiredElements.has(key) ? [label] : []) - if (selectedPhaseLabels.length > 0 || requiredElements.has('persistence_or_artifact_storage')) { - const segments = [...selectedPhaseLabels] - if (requiredElements.has('persistence_or_artifact_storage')) { - segments.push('persistence') - } - instructions.push(`Follow ${segments.join(', ')} evidence before concluding the flow.`) + // Every instruction below is keyed on a typed `required_elements` entry that + // `runtimeGenerationContractPhaseElements` derives from execution-slice + // evidence (phase coverage, an `enqueues_job` boundary, slice status). The + // question text is not an input here, so prompt vocabulary cannot manufacture + // a workflow that the repository evidence does not show. + if (requiredElements.has('persistence_or_artifact_storage')) { + instructions.push('Follow persistence evidence before concluding the flow.') } else if (requiredElements.has('main_pipeline_phases')) { instructions.push('Cover the main runtime pipeline phases instead of stopping at the entrypoint.') } @@ -91,30 +84,6 @@ function answerContractInstructions(retrieval: RetrieveResult): string[] { return instructions } -function promptWantsReportGenerationCore(prompt: string): boolean { - return /\b(?:report(?:\s+generation)?|generated\s+report|validation\s+report|final\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate)\b/i.test(prompt) -} - -function generationCoreInstructions(question: string, retrieval: RetrieveResult): string[] { - const contractInstructions = answerContractInstructions(retrieval) - if (contractInstructions.length > 0) { - return contractInstructions - } - - if ( - retrieval.retrieval_gate?.signals.generation_intent !== 'runtime_generation' - || retrieval.retrieval_gate?.signals.target_domain_hint !== 'backend_runtime' - || !promptWantsReportGenerationCore(question) - ) { - return [] - } - - return [ - 'Treat HTTP/controller entrypoints as trigger context, not the full answer, when downstream generation-core evidence is present.', - 'Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.', - ] -} - export function buildMadarPromptPack(input: BuildMadarPromptPackInput): ComparePromptPack { const explainPayloadCore = buildAnswerReadyPackSchema( buildExplainPackPayloadCore(compactRetrieveResult(input.retrieval), input.retrieval), @@ -126,7 +95,7 @@ export function buildMadarPromptPack(input: BuildMadarPromptPackInput): CompareP instructions: [ 'Answer the question using only the provided graph-guided retrieval output.', 'If the retrieval does not contain the answer, say so.', - ...generationCoreInstructions(input.question, input.retrieval), + ...answerContractInstructions(input.retrieval), ], stable_prefix_title: 'Retrieved graph context', stable_sections: [ diff --git a/src/runtime/retrieval-gate.ts b/src/runtime/retrieval-gate.ts index 6f972859..8a468b00 100644 --- a/src/runtime/retrieval-gate.ts +++ b/src/runtime/retrieval-gate.ts @@ -239,9 +239,16 @@ function detectGenerationIntent(prompt: string): { } { const lower = prompt.toLowerCase() const displayShaped = /\b(?:display(?:ed|ing)?|render(?:ed|ing)?|show(?:n|ing)?|visible|view|ui|frontend|front-end|component|screen|page|footer|header|label|date|timestamp)\b/i.test(lower) - const genericGenerationShaped = /\b(?:generat(?:e|ed|es|ing|ion)|creat(?:e|ed|es|ing|ion)|build(?:s|ing|er)?|assembl(?:e|ed|es|ing)|produc(?:e|ed|es|ing))\b/i.test(lower) + // `assembl*` was dropped from this list in #660 Slice C. It is one of the six + // report workflow stages, and it was the only one of the six that could still + // reach a runtime-generation gate on its own: `Explain assemble` classified as + // runtime_generation while `Explain planner`, `Explain research`, + // `Explain assembly`, `Explain scoring` and `Explain persistence` all returned + // unknown. The remaining verbs carry the generic sense without naming a report + // stage, and any assembly question with real backend evidence still classifies + // through `backendRuntimeShaped` plus `explanationShaped`. + const genericGenerationShaped = /\b(?:generat(?:e|ed|es|ing|ion)|creat(?:e|ed|es|ing|ion)|build(?:s|ing|er)?|produc(?:e|ed|es|ing))\b/i.test(lower) const backendRuntimeShaped = /\b(?:pipeline|runtime|orchestrator|worker|job|repository|service|controller|api|backend|persist(?:ed|ing)?|sav(?:e|ed|es|ing)|db(?:sync|[- ]sync)?)\b/i.test(lower) - const reportGenerationShaped = /\b(?:idea\s+report|report\s+generation|final\s+report|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate|renderer|synthesis|assemble|assembly)\b/i.test(lower) const buildStaticShaped = /\b(?:next\.js|nextjs|app\s+router|route\s+segment|landing\s+page|static|ssg|isr|server\s+component)\b/i.test(lower) const explanationShaped = /\b(?:explain|how|why|trace|walk|flow|path|lifecycle|fail(?:s|ing|ed)?)\b/i.test(lower) const flowProofShaped = @@ -254,13 +261,20 @@ function detectGenerationIntent(prompt: string): { display_shaped: displayShaped, generic_generation_shaped: genericGenerationShaped, backend_runtime_shaped: backendRuntimeShaped, - report_generation_shaped: reportGenerationShaped, + // #660 Slice C retired the report-generation task-shape classifier. This + // signal is a required member of the published `RetrievalGenerationDebugSignals` + // shape, which is outside this slice's three-file production boundary, so it + // is reported as the constant it now is rather than removed. Nothing reads it + // to reach a gate outcome: no prompt vocabulary can turn it true. + report_generation_shaped: false, build_static_shaped: buildStaticShaped, explanation_shaped: explanationShaped, flow_proof_shaped: flowProofShaped, } - const strongRuntimeShaped = backendRuntimeShaped || reportGenerationShaped + // Gate outcomes follow generic evidence only: a prompt that merely resembles + // the old qualification report task no longer earns a runtime-generation gate. + const strongRuntimeShaped = backendRuntimeShaped if (buildStaticShaped && !strongRuntimeShaped) { return { intent: 'unknown', debug } diff --git a/src/runtime/retrieve/slicing.ts b/src/runtime/retrieve/slicing.ts index 823165e9..0e3109f5 100644 --- a/src/runtime/retrieve/slicing.ts +++ b/src/runtime/retrieve/slicing.ts @@ -123,20 +123,19 @@ function policyForIntent(intent: RetrievalIntent): SlicePolicy { } } +// Generic runtime-architecture vocabulary only. `scoring` and `report builder` +// were report-workflow stage names and are gone with the rest of the #660 +// report-generation class; `persistence` and `repository` stay because they +// name ordinary backend layers, are independently treated as backend-runtime +// markers by the retrieval gate's generic `backendRuntimeShaped` signal, and are +// exercised by the pre-existing non-qualification direct-persistence fixture in +// tests/unit/retrieve-slice-v1.test.ts. function promptWantsRuntimePipeline(prompt: string | undefined): boolean { if (!prompt) { return false } - return /\b(runtime|pipeline|service|orchestrator|job|agent|scoring|report(?: builder)?|persistence|repository|generat(?:e|ed|es|ing|ion)|create|created)\b/i.test(prompt) -} - -function promptWantsReportGenerationCore(prompt: string | undefined): boolean { - if (!prompt) { - return false - } - - return /\b(?:report(?:\s+generation)?|generated\s+report|validation\s+report|final\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate)\b/i.test(prompt) + return /\b(runtime|pipeline|service|orchestrator|job|agent|persistence|repository|generat(?:e|ed|es|ing|ion)|create|created)\b/i.test(prompt) } function promptMentionsHttpRoute(prompt: string | undefined): boolean { @@ -184,25 +183,11 @@ function effectivePolicy( const broadRuntimeGeneration = options.generationIntent === 'runtime_generation' && options.targetDomainHint === 'backend_runtime' && !hasExactMethodAnchor - const hasGenerationCoreAnchor = anchors.some((anchor) => anchor.reason === 'generation core heuristic') if (!hasMethodAnchor && !pipelinePrompt) { return base } - if (broadRuntimeGeneration && pipelinePrompt && hasGenerationCoreAnchor && promptWantsReportGenerationCore(options.prompt)) { - return { - ...base, - directions: ['backward', 'forward'], - backward_relations: new Set(['calls', 'enqueues_job', 'controller_route', 'route_handler', 'method']), - forward_relations: new Set(RUNTIME_FLOW_RELATIONS), - helper_relations: new Set(['injects', 'depends_on', 'module_provides']), - backward_depth: Math.max(base.backward_depth, 3), - forward_depth: Math.max(base.forward_depth, 4), - runtime_flow_only: true, - } - } - if (broadRuntimeGeneration && hasMethodAnchor && pipelinePrompt) { return { ...base, @@ -320,7 +305,7 @@ function runtimeGenerationAnchorValue(node: SliceScoredNode): number { if (methodLikeNode(node)) value += 1 if (/\b(?:nest_route|route|controller)\b/.test(lower)) value += 5 if (/\b(?:src|server|backend|api|modules)\b/.test(lower)) value += 1 - if (/\b(?:generate|generation|create|start|pipeline|queue|process|orchestrator|worker|job|repository|save|report|scoring|research|agent)\b/.test(lower)) value += 2 + if (/\b(?:generate|generation|create|start|pipeline|queue|process|orchestrator|worker|job|repository|save|agent)\b/.test(lower)) value += 2 if (/(?:^|[.#])(?:generate|create|start|process|save|score|search|update|claim|cancel)[A-Za-z_$\w]*\(?\)?$/i.test(node.label)) value += 3 if (/\b(?:service|provider|repository|queue|worker|orchestrator)\b/.test(lower)) value += 1 if (frontendDisplayLikeNode(node)) value -= 6 @@ -329,27 +314,6 @@ function runtimeGenerationAnchorValue(node: SliceScoredNode): number { } -function semanticGenerationCoreAnchorValue(node: SliceScoredNode, prompt: string | undefined): number { - const lower = `${node.label} ${node.nodeKind ?? ''} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - let value = runtimeGenerationAnchorValue(node) - - if (!promptWantsReportGenerationCore(prompt)) { - return value - } - - if (routeOrControllerLikeNode(node)) value -= 7 - if (/\b(?:title|status|guard|auth|interceptor|refund|suggest|list|health|planenforcement)\b/.test(lower)) value -= 4 - if (/\b(?:planner|plan\b)\b/.test(lower)) value += 11 - if (/\b(?:assembly|assemble|quality(?:-| )gate|renderer|render|synthesis|final(?:-| )report)\b/.test(lower)) value += 10 - if (/\b(?:research|extract|metrics?|scor(?:e|ing))\b/.test(lower)) value += 7 - if (/\b(?:orchestrator|pipeline)\b/.test(lower)) value += 4 - if (/\b(?:worker|section)\b/.test(lower)) value += 2 - if (/\b(?:persist|repository|db(?:-| )sync|save)\b/.test(lower)) value += 3 - if (/\b(?:index\.json|state)\b/.test(lower)) value += 2 - - return value -} - function runtimeFlowRelationPriority( relation: string, node: SliceScoredNode, @@ -444,7 +408,6 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], )) const nonBarrelMatchedAnchors = matchedAnchors.filter((node) => !isBarrelLike(node.label, node.sourceFile)) const broadRuntimeGeneration = broadRuntimeGenerationPrompt(options) - const reportGenerationPrompt = promptWantsReportGenerationCore(options.prompt) const explicitPathAnchor = matchedAnchors.find((node) => node.literalPathMatch) const routePromptAnchors = broadRuntimeGeneration && promptMentionsHttpRoute(options.prompt) ? scored @@ -459,20 +422,6 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], return rightPriority - leftPriority || right.score - left.score }) : [] - const semanticCoreAnchors = broadRuntimeGeneration && reportGenerationPrompt - ? scored - .filter((node) => - methodLikeNode(node) - && !routeOrControllerLikeNode(node) - && !isBarrelLike(node.label, node.sourceFile) - && !frontendDisplayLikeNode(node) - && node.score > 0, - ) - .map((node) => ({ node, value: semanticGenerationCoreAnchorValue(node, options.prompt) })) - .filter((entry) => entry.value > 0) - .sort((left, right) => right.value - left.value || right.node.score - left.node.score) - .map((entry) => entry.node) - : [] const intentAnchors = (() => { if (options.generationIntent === 'runtime_generation' && options.targetDomainHint === 'backend_runtime') { return matchedAnchors @@ -494,28 +443,11 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], return [] })() - const generationCoreAnchorIds = new Set() let anchorPool: SliceScoredNode[] if (exactMethodAnchors.length > 0) { anchorPool = exactMethodAnchors.slice(0, 1) } else if (explicitPathAnchor) { anchorPool = [explicitPathAnchor] - } else if (broadRuntimeGeneration && reportGenerationPrompt && semanticCoreAnchors.length > 0) { - const primaryRuntimeAnchor = routePromptAnchors[0] - ?? intentAnchors[0] - ?? nonBarrelMatchedAnchors[0] - ?? matchedAnchors[0] - const selected = [ - ...(primaryRuntimeAnchor ? [primaryRuntimeAnchor] : []), - ...semanticCoreAnchors.filter((node) => node.id !== primaryRuntimeAnchor?.id).slice(0, 2), - ] - anchorPool = selected - for (const node of selected) { - if (primaryRuntimeAnchor && node.id === primaryRuntimeAnchor.id) { - continue - } - generationCoreAnchorIds.add(node.id) - } } else if (routePromptAnchors.length > 0) { anchorPool = routePromptAnchors.slice(0, 1) } else if (intentAnchors.length > 0) { @@ -527,9 +459,7 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], } for (const node of anchorPool) { - const reason = generationCoreAnchorIds.has(node.id) - ? 'generation core heuristic' - : node.exactLabelMatch + const reason = node.exactLabelMatch ? 'symbol mention' : node.literalPathMatch ? 'path mention' @@ -545,8 +475,7 @@ function buildAnchors(graph: KnowledgeGraph, scored: readonly SliceScoredNode[], reason, }) seen.add(node.id) - const maxAnchors = broadRuntimeGeneration && reportGenerationPrompt ? 3 : 2 - if (anchors.length >= maxAnchors) { + if (anchors.length >= 2) { break } } @@ -675,20 +604,32 @@ function traverseDirection( } } +// `search` is anchored on its left. Without that anchor it also matches the tail +// of "research", so a symbol named ResearchAgent kept the runtime-expansion +// preference that #660 Slice C removed for report workflow stage names — the +// removal was silently defeated by an unrelated alternative. The other terms +// keep their existing loose anchoring on purpose: they are meant to match +// inflections such as "workers", "jobs" and "pipelines", and tightening every +// alternative would change generic ranking well beyond this slice. +// +// `persist` deliberately still matches "persistence": it is a generic runtime +// verb naming an ordinary backend layer, judged generic earlier in this slice +// when removing it from the prompt classifier regressed a neutral login fixture. +// A report stage NOUN must confer nothing; a generic runtime verb may. function pipelineBridgeLikeNode(node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - return /\bpipeline|trigger|queue|job|worker|orchestrator|planner|research|agent|scoring|report|repository|persistence|save|process|search|score|addjob\b/.test(lower) + return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|\bsearch|addjob\b/.test(lower) } function highValueRuntimeExpansionNode(node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''} ${node.sourceFile}`.toLowerCase() - return /\bpipeline|trigger|queue|job|worker|orchestrator|planner|research|agent|scoring|report|repository|persistence|save|process|search|score|dispatch|assemble|persist|builder|addjob\b/.test(lower) + return /\bpipeline|trigger|queue|job|worker|orchestrator|agent|repository|save|process|\bsearch|dispatch|persist|builder|addjob\b/.test(lower) } function sharedHubLikeNode(graph: KnowledgeGraph, node: SliceScoredNode): boolean { const lower = `${node.label} ${node.frameworkRole ?? ''}`.toLowerCase() return graph.uniqueNeighborDegree(node.id) >= 12 - || /requireideasuserid|addjob|callllm|resolve|logger|\.info\(\)|\.error\(\)|\.warn\(\)|planenforcement/.test(lower) + || /resolve|logger|\.info\(\)|\.error\(\)|\.warn\(\)/.test(lower) } function shouldExpandRuntimePathNode( @@ -790,14 +731,6 @@ function addAnchorPredecessors( const predecessor = scoredById.get(predecessorId) ?? sliceNodeFromGraph(graph, predecessorId) scoredById.set(predecessorId, predecessor) - if ( - broadRuntimeGenerationPrompt(options) - && promptWantsReportGenerationCore(options.prompt) - && routeOrControllerLikeNode(predecessor) - && !anchoredIds.has(predecessorId) - ) { - continue - } if (shouldSuppressNode(graph, predecessor, anchoredIds, options)) { continue } diff --git a/tests/unit/compare.test.ts b/tests/unit/compare.test.ts index 547a7c71..80cb8185 100644 --- a/tests/unit/compare.test.ts +++ b/tests/unit/compare.test.ts @@ -1067,7 +1067,7 @@ describe('compare runtime', () => { } }) - it('tells broad runtime-generation answers not to stop at the HTTP trigger when downstream generation core evidence exists', () => { + it('does not manufacture a report workflow instruction from question vocabulary alone (#660 Slice C)', () => { const retrieval: MadarPromptPackRetrieval = { question: 'How idea report is being generated', token_count: 120, @@ -1108,8 +1108,15 @@ describe('compare runtime', () => { retrieval, }) - expect(pack.prompt).toContain('Treat HTTP/controller entrypoints as trigger context, not the full answer, when downstream generation-core evidence is present.') - expect(pack.prompt).toContain('Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.') + // #660 Slice C. This test used to REQUIRE both lines above. They were + // produced by a task-phrase classifier over the question text plus a fixed + // planner/research/assembly/scoring/rendering/persistence workflow, with no + // evidence behind either — the retrieval here carries no answer_contract and + // no execution_slice at all. The assertion is inverted into the independence + // control it should always have been: prompt vocabulary manufactures nothing. + expect(pack.prompt).not.toContain('Treat HTTP/controller entrypoints as trigger context, not the full answer, when downstream generation-core evidence is present.') + expect(pack.prompt).not.toContain('Follow planner, research, assembly, scoring, rendering, and persistence evidence before concluding the flow.') + expect(pack.prompt).not.toContain('planner, research, assembly, scoring, rendering') }) it('includes a runtime-generation answer contract in the prompt core and tells partial slices to mention uncertainty', () => { diff --git a/tests/unit/pack-quality-fixtures.test.ts b/tests/unit/pack-quality-fixtures.test.ts index 5dd7cbd9..a5fcd154 100644 --- a/tests/unit/pack-quality-fixtures.test.ts +++ b/tests/unit/pack-quality-fixtures.test.ts @@ -248,26 +248,28 @@ describe('pack-quality fixtures (#298)', () => { expect([...surfacedLabels]).not.toContain('.buildQueuedIdeaReportResponse()') }) - it('promotes the quality-gate failure branch when the explain prompt asks what happens if it fails', async () => { + // #660 Slice C. This test required `handleQualityGateFailure()` onto the + // primary path for a report-shaped prompt. It got there through + // `semanticGenerationCoreAnchorValue`, which paid +10 to any label matching + // "quality gate" once a task-phrase classifier had seen report vocabulary in + // the question — a name-shaped promotion with no structural basis. Both are + // gone. What remains a real claim is that the prompt's wording alone does not + // decide the primary path, so this is now that independence control. + it('does not promote a branch onto the primary path from report wording alone (#660 Slice C)', async () => { const result = await runPackQualityFixture('runtime-generation-explain-report-flow', { prompt: 'How is idea report handled when it fails', task: 'explain', budget: 1800, }) const payload = result.payload as typeof result.payload & { - pack?: { - execution_slice?: { - primary_path?: { - steps?: Array<{ label?: string }> - } - } - } + pack?: { execution_slice?: { primary_path?: { steps?: Array<{ label?: string }> } } } } const primaryLabels = payload.pack?.execution_slice?.primary_path?.steps?.map((entry) => entry.label) ?? [] - expect(primaryLabels).toEqual(expect.arrayContaining([ - 'handleQualityGateFailure()', - ])) + // The retired table paid +10 for a label matching "quality gate" once the + // question had been classified as report-shaped. Nothing may reach the + // primary path on the strength of its name any more. + expect(primaryLabels).not.toContain('handleQualityGateFailure()') expect(primaryLabels).not.toContain('saveStructuredReport()') }) diff --git a/tests/unit/report-generation-independence.test.ts b/tests/unit/report-generation-independence.test.ts new file mode 100644 index 00000000..e5531701 --- /dev/null +++ b/tests/unit/report-generation-independence.test.ts @@ -0,0 +1,434 @@ +// #660 Slice C — report-generation independence controls. +// +// Slice C removed the last production behaviour that Madar activated because a +// prompt LOOKED like the qualification report-generation task: a task-phrase +// classifier duplicated in `infrastructure/prompt-pack.ts` and +// `runtime/retrieve/slicing.ts`, the fixed +// "Follow planner, research, assembly, scoring, rendering, and persistence +// evidence" instruction, a name-driven anchor score table, forced anchor +// membership, a deeper report-only slice policy, and the `reportGenerationShaped` +// gate variant. +// +// These controls are NEGATIVE and INDEPENDENCE controls, not snapshots. Each one +// is written so that restoring any single retired rule makes it fail — that is +// verified mechanically by scripts/verify-report-generation-injections.mjs, +// which restores each rule from a digest-checked byte snapshot and requires the +// named control below to fail. +// +// The former qualification vocabulary is deliberately spelled out in the PROMPTS +// here. That is the point: these controls prove the words do nothing. It is +// never used to name a symbol, a path or an expected output. +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import type { ContextPackExecutionPhase } from '../../src/contracts/context-pack.js' +import { KnowledgeGraph } from '../../src/contracts/graph.js' +import { build } from '../../src/pipeline/build.js' +import { buildMadarPromptPack } from '../../src/infrastructure/prompt-pack.js' +import { classifyRetrievalLevel } from '../../src/runtime/retrieval-gate.js' +import { retrieveContext, type RetrieveResult } from '../../src/runtime/retrieve.js' +import { sliceCandidatesForRetrieve } from '../../src/runtime/retrieve/slicing.js' + +// The retired qualification vocabulary, split so a control never confuses a +// LEGITIMATE lexical effect with a report-policy effect. `rendering`/`renderer` +// are also ordinary FRONTEND-DISPLAY words that the generic display classifier +// has always matched and that Slice C did not touch, so they live apart and are +// only used where a display classification is the expected generic outcome. +const REPORT_WORDS = 'idea report generation planner research assembly scoring synthesis metrics quality gate' +const REPORT_DISPLAY_WORDS = 'rendering renderer' + +// A shared generic frame. The two prompts carry the same backend-runtime and +// explanation shape and differ ONLY in their nouns. Both noun sets are chosen to +// match NO label in either fixture graph, so a structural difference cannot be +// explained away as ordinary lexical relevance — which section 11-B requires be +// identified separately rather than mistaken for special policy. +const FRAME = 'is generated end to end through the runtime pipeline by the' +const NEUTRAL_Q = `Explain how the daily digest ${FRAME} ingest, lookup, merge and tally stages` +const REPORT_Q = `Explain how the idea report ${FRAME} planner, research, assembly and scoring stages` + +const BASE_INSTRUCTIONS = [ + 'Answer the question using only the provided graph-guided retrieval output.', + 'If the retrieval does not contain the answer, say so.', +] + +/** + * A five-stage runtime flow whose edges point BACK toward the route. + * + * Every stage carries both a LABEL and a SOURCE PATH, because the retired score + * table read `label + node_kind + framework_role + source_file` and in practice + * matched on path SEGMENTS (`/assembly/`, `/scoring/`) far more often than on a + * camelCase label, where lower-casing destroys the word boundaries the pattern + * needed. A fixture that varied only the labels could not catch it. + */ +interface Stage { label: string; file: string } + +function reverseFlowGraph(stages: { + route: Stage + trigger: Stage + queue: Stage + worker: Stage + assembler: Stage + scorer: Stage +}) { + return build( + [ + { + schema_version: 1, + nodes: [ + { id: 'route', label: stages.route.label, file_type: 'code', source_file: stages.route.file, source_location: 'L20', node_kind: 'method', framework: 'nestjs', framework_role: 'nest_route', community: 0 }, + { id: 'trigger', label: stages.trigger.label, file_type: 'code', source_file: stages.trigger.file, source_location: 'L30', node_kind: 'method', framework_role: 'service', community: 1 }, + { id: 'queue', label: stages.queue.label, file_type: 'code', source_file: stages.queue.file, source_location: 'L40', node_kind: 'method', framework_role: 'queue', community: 1 }, + { id: 'worker', label: stages.worker.label, file_type: 'code', source_file: stages.worker.file, source_location: 'L50', node_kind: 'method', framework_role: 'worker', community: 2 }, + { id: 'assembler', label: stages.assembler.label, file_type: 'code', source_file: stages.assembler.file, source_location: 'L60', node_kind: 'method', framework_role: 'service', community: 2 }, + { id: 'scorer', label: stages.scorer.label, file_type: 'code', source_file: stages.scorer.file, source_location: 'L70', node_kind: 'method', framework_role: 'service', community: 2 }, + ], + edges: [ + { source: 'trigger', target: 'route', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.trigger.file }, + { source: 'queue', target: 'trigger', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.queue.file }, + { source: 'worker', target: 'queue', relation: 'enqueues_job', confidence: 'EXTRACTED', source_file: stages.worker.file }, + { source: 'assembler', target: 'worker', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.assembler.file }, + { source: 'scorer', target: 'assembler', relation: 'calls', confidence: 'EXTRACTED', source_file: stages.scorer.file }, + ], + }, + ], + { directed: true }, + ) +} + +// Unrelated names: an invoice intake flow. +const NEUTRAL_NAMES = { + route: { label: '.submitInvoice()', file: '/src/entry/entry.controller.ts' }, + trigger: { label: '.beginRun()', file: '/src/flow/trigger.service.ts' }, + queue: { label: '.enqueueBatch()', file: '/src/flow/queue-registry.service.ts' }, + worker: { label: '.consume()', file: '/src/flow/orchestrator.worker.ts' }, + assembler: { label: '.combineLineItems()', file: '/src/flow/stage/stage.service.ts' }, + scorer: { label: '.rateVendor()', file: '/src/flow/rating/rating.service.ts' }, +} + +/** + * The SAME structure with the SAME leading verbs and the same generic path words + * ("src", "flow", "service"), named throughout after report workflow stages while + * doing the same unrelated invoice work. Holding the verbs fixed matters: Madar + * legitimately prefers action-shaped method names such as `generate`/`start`/ + * `process`, and that generic preference must not be mistaken for report naming. + */ +const REPORT_NAMED = { + route: { label: '.submitReport()', file: '/src/entry/entry.controller.ts' }, + trigger: { label: '.beginPlanner()', file: '/src/flow/trigger.service.ts' }, + queue: { label: '.enqueueResearch()', file: '/src/flow/queue-registry.service.ts' }, + worker: { label: '.consume()', file: '/src/flow/orchestrator.worker.ts' }, + assembler: { label: '.combineAssembly()', file: '/src/flow/assembly/assembly.service.ts' }, + scorer: { label: '.rateScoring()', file: '/src/flow/scoring/scoring.service.ts' }, +} + +/** A repository with no multi-stage generation structure at all. */ +function flatDisplayGraph() { + return build( + [ + { + schema_version: 1, + nodes: [ + { id: 'toggle', label: '.applyTheme()', file_type: 'code', source_file: '/src/ui/theme-toggle.ts', source_location: 'L4', node_kind: 'method', community: 0 }, + { id: 'store', label: '.readPalette()', file_type: 'code', source_file: '/src/ui/color-store.ts', source_location: 'L8', node_kind: 'method', community: 0 }, + ], + edges: [ + { source: 'toggle', target: 'store', relation: 'calls', confidence: 'EXTRACTED', source_file: '/src/ui/theme-toggle.ts' }, + ], + }, + ], + { directed: true }, + ) +} + +function retrieve(graph: ReturnType, question: string) { + return retrieveContext(graph, { question, budget: 4000, retrievalLevel: 4, retrievalStrategy: 'slice-v1' }) +} + +/** Slice membership is observable through the matched nodes the slice selected. */ +function memberIds(result: RetrieveResult): string[] { + return result.matched_nodes.map((node) => node.node_id ?? node.label) +} + +function structuralShape(result: RetrieveResult) { + return { + member_ids: memberIds(result), + anchor_reasons: (result.slice?.anchors ?? []).map((anchor) => anchor.reason), + anchor_count: (result.slice?.anchors ?? []).length, + paths: (result.slice?.selected_paths ?? []).map((path) => `${path.from}-[${path.relation}]->${path.to}`), + mode: result.slice?.mode, + } +} + +/** + * The same slice keyed by NODE ID rather than label. Both fixture graphs use the + * same ids for the same structural positions and differ only in what the symbols + * are called, so an id-keyed shape isolates policy from naming completely: if the + * two shapes differ, a name changed a structural decision. + */ +function shapeByPosition(result: RetrieveResult) { + return { + member_ids: memberIds(result), + anchor_ids: (result.slice?.anchors ?? []).map((anchor) => anchor.node_id), + anchor_reasons: (result.slice?.anchors ?? []).map((anchor) => anchor.reason), + paths: (result.slice?.selected_paths ?? []).map((path) => `${path.from_id}-[${path.relation}]->${path.to_id}`), + } +} + +function instructionsFor(question: string, retrieval: Partial = {}) { + const decision = classifyRetrievalLevel({ prompt: question }) + const pack = buildMadarPromptPack({ + question, + retrieval: { + question, + token_count: 120, + matched_nodes: [], + relationships: [], + community_context: [], + graph_signals: { god_nodes: [], bridge_nodes: [] }, + retrieval_gate: { + level: decision.level, + reason: decision.reason, + skipped_retrieval: false, + intent: decision.intent, + signals: decision.signals, + }, + ...retrieval, + } as RetrieveResult, + }) + return pack.prompt.split('\n\nRetrieved graph context:')[0]?.split('\n') ?? [] +} + +describe('#660 Slice C — report-generation independence', () => { + // ------------------------------------------------------------- control A -- + it('A: report vocabulary alone produces no fixed workflow, no boost, no forced anchor and no gate variant', () => { + // The MINIMAL pair. Both prompts are display-shaped and carry no backend + // marker; they differ only by the qualification task's own name. Before + // Slice C the second one was pulled to runtime_generation by + // `reportGenerationShaped` alone. Now both take the generic display outcome. + const withoutTaskName = classifyRetrievalLevel({ prompt: 'Explain how the summary is generated and displayed' }) + const withTaskName = classifyRetrievalLevel({ prompt: 'Explain how the idea report is generated and displayed in the footer' }) + + expect(withTaskName.signals.generation_debug?.report_generation_shaped).toBe(false) + expect(withTaskName.signals.generation_intent).toBe(withoutTaskName.signals.generation_intent) + expect(withTaskName.signals.target_domain_hint).toBe(withoutTaskName.signals.target_domain_hint) + expect(withTaskName.signals.generation_intent).toBe('display_rendering') + + // The gate still responds to GENERIC evidence, so this is not a constant: + // adding a real backend-runtime marker moves it, and the report words do not. + const withBackendMarker = classifyRetrievalLevel({ prompt: 'Explain how the summary is generated and displayed by the worker pipeline' }) + expect(withBackendMarker.signals.generation_intent).toBe('runtime_generation') + + // No manufactured workflow over a repository with no such structure: the + // instructions are exactly the two unconditional ones. + const question = `Explain how the ${REPORT_WORDS} ${REPORT_DISPLAY_WORDS} flow is displayed on the page` + expect(instructionsFor(question)).toEqual(BASE_INSTRUCTIONS) + + // No forced candidate and no report-specific ranking over a flat repository. + const shape = structuralShape(retrieve(flatDisplayGraph(), question)) + expect(shape.anchor_reasons).not.toContain('generation core heuristic') + expect(shape.member_ids.length).toBeLessThanOrEqual(2) + }) + + it('A2: none of the six declared report workflow stages reaches a runtime gate on its own', () => { + // The gap that the first FINAL review found. Control A above tests ONE + // minimal pair, and its prompts already carried the generic verb + // "generated", so a report stage word that reached the gate through the + // generic generation-verb list stayed invisible. `assemble` did exactly + // that. This control walks the six declared stages one at a time. + for (const stage of ['planner', 'research', 'assembly', 'assemble', 'scoring', 'persistence']) { + const decision = classifyRetrievalLevel({ prompt: `Explain ${stage}` }) + expect( + { stage, intent: decision.signals.generation_intent }, + `"Explain ${stage}" must not earn a runtime-generation gate on report vocabulary alone`, + ).toEqual({ stage, intent: 'unknown' }) + expect(decision.signals.generation_debug?.report_generation_shaped).toBe(false) + } + + // `rendering` is the one stage word that still classifies, as + // display_rendering, through the generic frontend-display classifier that + // Slice C did not touch. That is a generic lexical outcome, not a report + // policy, and it is asserted rather than left ambiguous. + expect(classifyRetrievalLevel({ prompt: 'Explain rendering' }).signals.generation_intent).toBe('display_rendering') + + // Not a constant: the SAME stage words earn a runtime gate the moment real + // backend evidence is present, so the control fails in both directions. + for (const stage of ['assembly', 'assemble', 'scoring', 'persistence']) { + const withEvidence = classifyRetrievalLevel({ prompt: `Explain ${stage} in the worker pipeline` }) + expect(withEvidence.signals.generation_intent).toBe('runtime_generation') + expect(withEvidence.signals.target_domain_hint).toBe('backend_runtime') + } + + // And the generic generation verbs that remain are untouched: the word + // "report" contributes nothing to them. + for (const verb of ['generate', 'generation', 'create', 'build', 'produce']) { + expect(classifyRetrievalLevel({ prompt: `Explain ${verb}` }).signals.generation_intent).toBe('runtime_generation') + } + expect(classifyRetrievalLevel({ prompt: 'Explain report generation' }).signals.generation_intent) + .toBe(classifyRetrievalLevel({ prompt: 'Explain generation' }).signals.generation_intent) + }) + + // ------------------------------------------------------------- control B -- + it('B: the same repository and task intent yield the same structure under neutral and report wording', () => { + // Equivalent intent over identical evidence. The only difference is the + // retired qualification vocabulary, so nothing structural may move. + const neutral = retrieve(reverseFlowGraph(NEUTRAL_NAMES), NEUTRAL_Q) + const report = retrieve(reverseFlowGraph(NEUTRAL_NAMES), REPORT_Q) + + // Same generic classification, so any structural difference could only come + // from the report words themselves. + expect(classifyRetrievalLevel({ prompt: REPORT_Q }).signals.generation_intent) + .toBe(classifyRetrievalLevel({ prompt: NEUTRAL_Q }).signals.generation_intent) + expect(structuralShape(report)).toEqual(structuralShape(neutral)) + expect(structuralShape(report).anchor_reasons).not.toContain('generation core heuristic') + }) + + // ------------------------------------------------------------- control C -- + it('C: identical structure named after report stages earns no extra structural treatment', () => { + // The sharp form: ONE prompt containing no report vocabulary at all, run + // over two graphs that differ only in what their symbols are CALLED, and + // compared by NODE ID so naming cannot leak into the comparison itself. + // + // The prompt names the shared `flow` path segment on purpose. That makes the + // nodes source-path matches, which is what puts them through the runtime + // anchor SCORING path -- without it the score table is never consulted and + // this control could not catch a name-driven entry being restored. + const neutralPrompt = 'Explain how the daily digest is generated end to end through the runtime pipeline in flow' + const neutralNamed = shapeByPosition(retrieve(reverseFlowGraph(NEUTRAL_NAMES), neutralPrompt)) + const reportNamed = shapeByPosition(retrieve(reverseFlowGraph(REPORT_NAMED), neutralPrompt)) + + expect(reportNamed).toEqual(neutralNamed) + + // Proof the comparison has teeth: the scoring path really is reached, so a + // restored name-driven entry would move these positions. + expect(neutralNamed.anchor_ids.length).toBeGreaterThan(0) + }) + + it('C2: a report-shaped prompt over report-named symbols still gets no table, chain, instruction or forced pick', () => { + const shape = structuralShape(retrieve(reverseFlowGraph(REPORT_NAMED), REPORT_Q)) + + // No forced selection, and every anchor reason is one of the ordinary + // evidence reasons rather than a report heuristic. + expect(shape.anchor_reasons).not.toContain('generation core heuristic') + for (const reason of shape.anchor_reasons) { + expect(['symbol mention', 'path mention', 'source path token match', 'top lexical match']).toContain(reason) + } + // No special phase chain and no fixed report instruction. + expect(instructionsFor(REPORT_Q)).toEqual(BASE_INSTRUCTIONS) + }) + + it('C3: a report-stage node name confers no runtime-expansion preference', () => { + // The gap CodeRabbit found. `highValueRuntimeExpansionNode` runs only under a + // runtime-flow-only forward policy, which control C never reaches, so a report + // stage name leaking back through an unrelated alternative stayed invisible: + // `research` matched the unanchored tail of `search`. + // + // The observable effect is INCLUSION, not ordering: at depth > 0 a forward + // walk continues past a node only when `shouldExpandRuntimePathNode` says so, + // and being "high value" is what says so. So the third hop is reachable only + // if the middle node earned the preference. Asserting on ordering instead + // would have been a control that cannot catch its own mutation. + const chain = (midLabel: string, midFile: string) => { + const graph = new KnowledgeGraph({ directed: true }) + graph.addNode('entry', { label: '.startRun()', source_file: '/src/flow/trigger.service.ts', node_kind: 'method', file_type: 'code' }) + graph.addNode('mid', { label: midLabel, source_file: midFile, node_kind: 'method', file_type: 'code' }) + graph.addNode('deep', { label: '.writeLedger()', source_file: '/src/flow/ledger.service.ts', node_kind: 'method', file_type: 'code' }) + graph.addEdge('entry', 'mid', { relation: 'calls' }) + graph.addEdge('mid', 'deep', { relation: 'calls' }) + return graph + } + const run = (midLabel: string, midFile: string) => sliceCandidatesForRetrieve( + chain(midLabel, midFile), + [ + { id: 'entry', label: '.startRun()', sourceFile: '/src/flow/trigger.service.ts', nodeKind: 'method', exactLabelMatch: true, sourcePathMatch: false, literalPathMatch: false, score: 1 }, + { id: 'mid', label: midLabel, sourceFile: midFile, nodeKind: 'method', exactLabelMatch: false, sourcePathMatch: false, literalPathMatch: false, score: 0.5 }, + { id: 'deep', label: '.writeLedger()', sourceFile: '/src/flow/ledger.service.ts', nodeKind: 'method', exactLabelMatch: false, sourcePathMatch: false, literalPathMatch: false, score: 0.4 }, + ], + 'explain', + { + prompt: 'Explain how `.startRun()` moves through the runtime pipeline', + generationIntent: 'runtime_generation', + targetDomainHint: 'backend_runtime', + }, + )?.ordered_ids ?? [] + + // The three labels are chosen so that ONLY the intended signal varies: none + // of them carries an unrelated generic runtime term. An earlier draft used + // `.researchAgent()`, whose "agent" is itself a generic runtime word that was + // deliberately kept, so the fixture could not isolate the report stage name. + const neutral = run('.lookupStep()', '/src/flow/lookup.service.ts') + const reportNamed = run('.researchStep()', '/src/flow/research.service.ts') + + // Naming the middle hop after a report stage buys it nothing. + expect(reportNamed).toEqual(neutral) + expect(reportNamed).not.toContain('deep') + + // Not a constant: a genuinely generic runtime name DOES earn the expansion, + // so this control can tell the two apart and fails in both directions. + const genericRuntime = run('.searchStep()', '/src/flow/search.service.ts') + expect(genericRuntime).toContain('deep') + }) + + // ------------------------------------------------------------- control D -- + it('D: the pre-existing non-qualification parity golden still carries its typed evidence instructions', () => { + // tests/fixtures/prompt-pack-parity.golden.json was captured from a commit + // BEFORE any #660 production edit, over a login/session flow named nothing + // like a report. Its instructions come from the typed answer_contract that + // runtimeGenerationContractPhaseElements derives from execution-slice + // evidence, so they must survive the removal of the report classifier. + const golden = JSON.parse(readFileSync(resolve('tests/fixtures/prompt-pack-parity.golden.json'), 'utf8')) as Record + const recorded = golden.normal_context_prompt?.prompt ?? '' + + expect(recorded).toContain('Treat HTTP/controller entrypoints as trigger context, not the full answer.') + expect(recorded).toContain('Follow persistence evidence before concluding the flow.') + expect(recorded).toContain('Do not collapse producer-to-worker handoffs into direct calls when the evidence is an enqueues_job boundary.') + expect(recorded).toContain('Mention missing or uncertain phases when the execution slice is partial.') + + // And the retired fixed workflow is absent from that pre-existing golden. + expect(recorded).not.toContain('Follow planner, research, assembly, scoring, rendering, and persistence evidence') + expect(recorded).not.toContain('when downstream generation-core evidence is present') + }) + + it('D2: typed contract elements, not prompt words, decide the evidence instructions', () => { + const typed = { + execution_slice: { + status: 'partial' as const, + steps: [], + phase_coverage: { + expected: ['controller', 'queue', 'worker', 'persistence'] as ContextPackExecutionPhase[], + observed: ['controller', 'queue'] as ContextPackExecutionPhase[], + missing: ['worker', 'persistence'] as ContextPackExecutionPhase[], + }, + }, + answer_contract: { + version: 1 as const, + answer_focus: 'runtime_generation' as const, + entrypoint_scope: 'setup_context' as const, + required_elements: ['main_pipeline_phases', 'queue_worker_handoff', 'persistence_or_artifact_storage'], + do_not_claim: ['direct_producer_to_worker_calls_without_enqueues_boundary'], + observed_phases: ['controller', 'queue'] as ContextPackExecutionPhase[], + missing_phases: ['worker', 'persistence'] as ContextPackExecutionPhase[], + }, + } + + const neutral = instructionsFor(NEUTRAL_Q, typed) + const report = instructionsFor(REPORT_Q, typed) + + // Same typed evidence in, same instructions out, regardless of wording. + expect(report).toEqual(neutral) + expect(neutral).toContain('Follow persistence evidence before concluding the flow.') + // The persistence line is earned by the typed element; the retired fixed + // report workflow never appears. + expect(neutral.join('\n')).not.toContain('planner, research, assembly, scoring, rendering') + + // Withdraw the typed element and the instruction goes with it: the control + // fails in both directions rather than merely observing a constant. + const withoutPersistence = instructionsFor(NEUTRAL_Q, { + ...typed, + answer_contract: { ...typed.answer_contract, required_elements: ['main_pipeline_phases'] }, + }) + expect(withoutPersistence).not.toContain('Follow persistence evidence before concluding the flow.') + expect(withoutPersistence).toContain('Cover the main runtime pipeline phases instead of stopping at the entrypoint.') + }) +}) diff --git a/tests/unit/retrieval-gate.test.ts b/tests/unit/retrieval-gate.test.ts index 8497d0a7..19fce86a 100644 --- a/tests/unit/retrieval-gate.test.ts +++ b/tests/unit/retrieval-gate.test.ts @@ -283,10 +283,20 @@ describe('classifyRetrievalLevel — signal extraction', () => { expect(decision.signals.target_domain_hint).toBe('frontend_display') }) - it('prefers runtime generation for explanatory prompts that also mention display terms', () => { + // #660 Slice C. This test used to require that naming the qualification task + // ("idea report") pulled a display-shaped prompt to runtime_generation, which + // is exactly the `reportGenerationShaped` gate variant. Paired with the test + // above, it is now the independence control: the task name changes nothing, + // and only a genuine backend-runtime marker moves the gate. + it('does not prefer runtime generation merely because the prompt names the report task (#660 Slice C)', () => { const decision = classify({ prompt: 'Explain how the idea report is generated and displayed in the footer' }) - expect(decision.signals.generation_intent).toBe('runtime_generation') - expect(decision.signals.target_domain_hint).toBe('backend_runtime') + expect(decision.signals.generation_debug?.report_generation_shaped).toBe(false) + expect(decision.signals.generation_intent).toBe('display_rendering') + expect(decision.signals.target_domain_hint).toBe('frontend_display') + + const withBackendMarker = classify({ prompt: 'Explain how the idea report is generated and displayed in the footer by the worker pipeline' }) + expect(withBackendMarker.signals.generation_intent).toBe('runtime_generation') + expect(withBackendMarker.signals.target_domain_hint).toBe('backend_runtime') }) it('detects broad generation noun prompts without explanation verbs', () => { diff --git a/tests/unit/retrieve-production-correctness.test.ts b/tests/unit/retrieve-production-correctness.test.ts index 20e6f4fa..3bf68785 100644 --- a/tests/unit/retrieve-production-correctness.test.ts +++ b/tests/unit/retrieve-production-correctness.test.ts @@ -345,7 +345,14 @@ describe('retrieveContext production retrieval regressions', () => { } }) - it('adds a semantic generation-core anchor for broad report-generation prompts instead of centering only the HTTP trigger', () => { + // #660 Slice C. This test required a SECOND anchor to be forced in beside the + // real entry point whenever the question used report vocabulary. The anchor + // was picked by `semanticGenerationCoreAnchorValue`, a table that paid +11 for + // a label containing "planner", +10 for "assembly"/"renderer"/"final report" + // and +7 for "research"/"scoring" — a name-shaped ranking with no structural + // basis, reachable only through a task-phrase classifier over the question. + // Both the table and the forced pick are gone, so the assertion is inverted. + it('does not force a name-ranked second anchor for report-shaped questions (#660 Slice C)', () => { const result = retrieveContext(buildBroadReportGenerationGraph(), { question: 'How idea report is being generated', budget: 4000, @@ -353,34 +360,23 @@ describe('retrieveContext production retrieval regressions', () => { retrievalStrategy: 'slice-v1', }) + // The real entry point is still anchored by ordinary evidence. expect(result.slice?.anchors.map((anchor) => anchor.label)).toEqual(expect.arrayContaining([ '.generateFromProblem()', ])) - expect( - result.slice?.anchors.some((anchor) => [ - '.plan()', - '.processSection()', - '.search()', - '.assembleReport()', - '.score()', - '.validateReport()', - '.renderFinalReport()', - '.saveStructuredReport()', - ].includes(anchor.label)), - ).toBe(true) - expect(result.matched_nodes.map((node) => node.label)).toEqual(expect.arrayContaining([ - '.plan()', - '.processSection()', - '.search()', - '.assembleReport()', - '.score()', - '.validateReport()', - '.renderFinalReport()', - '.saveStructuredReport()', - ])) + // Nothing is anchored by the retired heuristic. + expect(result.slice?.anchors.map((anchor) => anchor.reason)).not.toContain('generation core heuristic') + for (const anchor of result.slice?.anchors ?? []) { + expect(['symbol mention', 'path mention', 'source path token match', 'top lexical match']).toContain(anchor.reason) + } }) - it('prefers central report-generation nodes over leaf helper generators for broad report-generation prompts', () => { + // #660 Slice C. This test asserted that `.plan()` / `.assembleReport()` / + // `.scoreReport()` beat `.generateScoringLedger()` and friends. Both groups sit + // at the same place in the graph; the winner was decided purely by which + // NAME resembled a report workflow stage. That is the same-name/wrong-semantics + // defect, so the preference is gone and only the evidence-backed anchor stays. + it('ranks no anchor by how much its name resembles a report stage (#660 Slice C)', () => { const result = retrieveContext(buildReportGenerationLeafHelperGraph(), { question: 'How idea report is being generated', budget: 4000, @@ -388,72 +384,64 @@ describe('retrieveContext production retrieval regressions', () => { retrievalStrategy: 'slice-v1', }) - const anchorLabels = result.slice?.anchors.map((anchor) => anchor.label) ?? [] - expect(anchorLabels).toEqual(expect.arrayContaining([ + const anchors = result.slice?.anchors ?? [] + expect(anchors.map((anchor) => anchor.label)).toEqual(expect.arrayContaining([ '.generateFromProblem()', ])) - expect(anchorLabels).toEqual(expect.arrayContaining([ - expect.stringMatching(/^\.plan\(\)$|^\.assembleReport\(\)$|^\.scoreReport\(\)$/), - ])) - expect(anchorLabels).not.toEqual(expect.arrayContaining([ - '.generateScoringLedger()', - '.generateSensitivityAnalysis()', - '.generateTitle()', - ])) + expect(anchors.map((anchor) => anchor.reason)).not.toContain('generation core heuristic') + for (const anchor of anchors) { + expect(['symbol mention', 'path mention', 'source path token match', 'top lexical match']).toContain(anchor.reason) + } }) - it('keeps backward-selected runtime flow as structural evidence for broad report-generation prompts', () => { - const result = retrieveContext(buildReverseFlowReportGenerationGraph(), { - question: 'How idea report is being generated', + // #660 Slice C. This test required a deep BACKWARD walk over `calls` edges for + // report-shaped questions. That policy — backward depth 3 with `calls` added + // to the backward relation set — existed only inside the branch gated on the + // report task-phrase classifier. Measured at the pre-change commit, the very + // same graph asked with equivalent NEUTRAL wording ("How idea invoice is being + // generated") returned ordered_ids ["route"] and ZERO selected paths. So this + // evidence was never generic: it was an advantage that report vocabulary alone + // bought. The test is inverted into that independence control. + it('gives report wording no structural evidence that neutral wording cannot get (#660 Slice C)', () => { + const ask = (question: string) => retrieveContext(buildReverseFlowReportGenerationGraph(), { + question, budget: 4000, retrievalLevel: 4, retrievalStrategy: 'slice-v1', }) - expect(result.slice?.selected_paths).toEqual(expect.arrayContaining([ - expect.objectContaining({ from: '.startPipeline()', to: '.generateFromProblem()', relation: 'calls' }), - expect.objectContaining({ from: '.addJob()', to: '.startPipeline()', relation: 'calls' }), - expect.objectContaining({ from: '.process()', to: '.addJob()', relation: 'enqueues_job' }), - expect.objectContaining({ from: '.scoreMetrics()', to: '.assembleReport()', relation: 'calls' }), - ])) + const report = ask('How idea report is being generated') + const neutral = ask('How idea invoice is being generated') - const matched = result.matched_nodes.map((node) => ({ label: node.label, evidence_class: node.evidence_class })) - expect(matched).toEqual(expect.arrayContaining([ - expect.objectContaining({ label: '.startPipeline()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.addJob()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.process()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.assembleReport()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.scoreMetrics()', evidence_class: 'structural' }), - ])) - expect(matched).not.toEqual(expect.arrayContaining([ - expect.objectContaining({ label: '.getStatusMessage()', evidence_class: 'structural' }), - expect.objectContaining({ label: '.generateTitle()', evidence_class: 'structural' }), - ])) + const shape = (result: ReturnType) => ({ + member_ids: result.matched_nodes.map((node) => node.node_id ?? node.label), + paths: (result.slice?.selected_paths ?? []).map((path) => `${path.from}-[${path.relation}]->${path.to}`), + reasons: (result.slice?.anchors ?? []).map((anchor) => anchor.reason), + }) + + expect(shape(report)).toEqual(shape(neutral)) + expect(shape(report).reasons).not.toContain('generation core heuristic') }) - // #660-B1. Was also asserting that '.getStatusMessage()' and - // '.generateTitle()' are absent. That exclusion was produced by a denylist of - // fourteen symbol names lifted from one repository, not by any structural - // property, so it is gone and those labels can appear again. What is still - // worth holding, and is kept, is that the execution flow itself is compacted - // around the traced steps. - it('compacts a broad pack around the traced execution flow', () => { - const compact = compactRetrieveResult(retrieveContext(buildReverseFlowReportGenerationGraph(), { - question: 'How idea report is being generated', + // #660-B1 removed a fourteen-symbol denylist from this test. #660 Slice C + // removes the other half: the labels it still required were reachable only + // through the report-gated backward policy, so requiring them here is + // requiring the special behaviour. What survives as a genuine, generic claim + // is that compaction does not INVENT membership and stays wording-independent. + it('compacts a broad pack identically under report and neutral wording (#660 Slice C)', () => { + const compact = (question: string) => compactRetrieveResult(retrieveContext(buildReverseFlowReportGenerationGraph(), { + question, budget: 4000, retrievalLevel: 4, retrievalStrategy: 'slice-v1', - })) + })).matched_nodes.map((node) => node.label) - const labels = compact.matched_nodes.map((node) => node.label) - expect(labels).toEqual(expect.arrayContaining([ - '.generateFromProblem()', - '.startPipeline()', - '.addJob()', - '.process()', - '.assembleReport()', - '.scoreMetrics()', - ])) + const report = compact('How idea report is being generated') + const neutral = compact('How idea invoice is being generated') + + expect(report).toEqual(neutral) + // Compaction still reports the evidence it does have, rather than nothing. + expect(report).toEqual(expect.arrayContaining(['.generateFromProblem()'])) }) // #660-B1. Two tests were removed here. Both asserted the EXPECTED phase