diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca930090..51fd3a79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,19 @@ jobs: - name: Verify grader/runtime structural boundary run: npm run verify:grader-boundary-controls + # #660-B. Runs on every lane on purpose: production independence from a + # qualification repository is a property of the source, and a guarantee + # that only holds on one platform is not one. The self-test reintroduces + # real qualification knowledge into real production files -- in each + # encoding the manifest claims to normalize -- requires the exact + # attributable rejection, then restores the bytes and proves they went + # back. It also injects a task-phrase ranking rule and requires the + # behavioural control that owns that class to fail on its own assertion, + # because a text scanner cannot see that class and must not pretend to. + # Pure local file I/O, no network and no spend. + - name: Verify production independence from qualification repositories + run: npm run verify:forbidden-knowledge-controls + # The old-reader proof extracts the released v0.32.1 loader from the # in-repository tag. The default shallow checkout carries no tags, so the # proof cannot run without this. One pinned ref is fetched, not full diff --git a/package.json b/package.json index ed3d053a..7080d281 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,8 @@ "qualify:validate": "node .github/scripts/validate-qualification-contract.mjs", "verify:grader-boundary": "node scripts/verify-grader-boundary.mjs", "verify:grader-boundary-controls": "node scripts/verify-grader-boundary.mjs --self-test", + "verify:forbidden-knowledge": "node scripts/verify-forbidden-knowledge.mjs", + "verify:forbidden-knowledge-controls": "node scripts/verify-forbidden-knowledge.mjs --self-test", "verify:grader-boundary-runtime": "node scripts/verify-grader-boundary-runtime.mjs", "verify:integrity-mutations": "node scripts/verify-integrity-mutations.mjs && MADAR_MUTATION_HARNESS_E2E=1 npx vitest run tests/unit/mutation-harness-self.test.ts", "verify:integrity-receipts": "node scripts/verify-integrity-receipts.mjs", diff --git a/scripts/lib/forbidden-knowledge-manifest.json b/scripts/lib/forbidden-knowledge-manifest.json new file mode 100644 index 00000000..6c8b4cb2 --- /dev/null +++ b/scripts/lib/forbidden-knowledge-manifest.json @@ -0,0 +1,74 @@ +{ + "manifest_version": "1.0.0", + "issue": 660, + "slice": "B", + "title": "Production independence from qualification-repository knowledge", + "purpose": [ + "Madar's retrieval and claim behaviour must be driven by generic structural evidence,", + "never by knowledge of a specific benchmark or qualification repository. This manifest", + "enumerates names, paths and shapes that belong to qualification targets. Their presence", + "anywhere in production source (src/**) means production behaviour has been shaped around", + "a qualification repository rather than around evidence.", + "", + "SCOPE LIMIT (deliberate, do not remove): this manifest and its scanner own LITERAL and", + "NORMALIZED contamination only. They cannot prove the absence of semantic overfitting --", + "a rule keyed on prompt vocabulary or a forced selection encodes the qualification task", + "without containing any name listed here. That class is owned by direct behavioural tests", + "(see tests/unit/production-independence.test.ts), not by this scanner." + ], + "scope": { + "production_roots": ["src"], + "note": "Enumerated by productionSourceFiles() -- every src/**/*.ts that is not a .d.ts." + }, + "normalization": { + "forms": ["tokens", "squashed"], + "tokens": "camel/Pascal boundaries split, backslashes to forward slashes, every run of non-alphanumerics collapsed to one space, lowercased, matched as a contiguous whole-token run. Catches camelCase, snake_case, kebab-case, dotted and path-separated spellings.", + "squashed": "every non-alphanumeric removed, lowercased, matched as a plain substring. Catches the case-flattened spelling a rule uses when it lowercases a label before testing it -- the form in which most of the removed demotion table was actually written." + }, + "rules": [ + { "id": "openstatus/path-router-status-page", "repository": "openstatus", "class": "path", "value": "packages/api/src/router/statusPage.ts", "why": "Exact file path of a qualification target's public status router." }, + { "id": "openstatus/path-status-page-utils", "repository": "openstatus", "class": "path", "value": "statusPage.utils", "why": "Qualification-target module name used to demote a candidate by path." }, + { "id": "openstatus/path-lib-http-etag", "repository": "openstatus", "class": "path", "value": "lib/http/etag", "why": "Qualification-target file path used to demote a candidate by path." }, + { "id": "openstatus/path-content-markdown", "repository": "openstatus", "class": "path", "value": "content/markdown", "why": "Qualification-target directory used to demote a candidate by path." }, + { "id": "openstatus/symbol-status-page", "repository": "openstatus", "class": "symbol", "value": "statusPage", "why": "Qualification-target router symbol." }, + { "id": "openstatus/symbol-http-checker-handler", "repository": "openstatus", "class": "symbol", "value": "HTTPCheckerHandler", "why": "Qualification-target Go handler symbol." }, + { "id": "openstatus/symbol-update-status", "repository": "openstatus", "class": "symbol", "value": "UpdateStatus", "why": "Qualification-target Go symbol used to key a fixed claim." }, + { "id": "openstatus/symbol-cloudtasks", "repository": "openstatus", "class": "symbol", "value": "cloudtasks", "why": "Queue client package used by a qualification target." }, + { "id": "openstatus/symbol-new-client", "repository": "openstatus", "class": "symbol", "value": "NewClient", "why": "Queue client constructor used by a qualification target to key a fixed claim." }, + { "id": "openstatus/symbol-create-task", "repository": "openstatus", "class": "symbol", "value": "CreateTask", "why": "Queue enqueue call used by a qualification target to key a fixed claim." }, + { "id": "openstatus/symbol-page-indicator", "repository": "openstatus", "class": "symbol", "value": "pageIndicator", "why": "Qualification-target status projection symbol." }, + { "id": "openstatus/symbol-status-reports", "repository": "openstatus", "class": "symbol", "value": "statusReports", "why": "Qualification-target model field used to key a fixed claim." }, + { "id": "openstatus/symbol-bar-type", "repository": "openstatus", "class": "symbol", "value": "barType", "why": "Qualification-target model field used to key a fixed claim." }, + { "id": "openstatus/symbol-status-glyph", "repository": "openstatus", "class": "symbol", "value": "statusGlyph", "why": "Qualification-target presentation symbol used to demote a candidate." }, + { "id": "openstatus/symbol-compute-etag", "repository": "openstatus", "class": "symbol", "value": "computeETag", "why": "Qualification-target utility symbol used to demote a candidate." }, + { "id": "report-generation/symbol-generate-scoring-ledger", "repository": "govalidate-report-generation", "class": "symbol", "value": "generateScoringLedger", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-generate-sensitivity-analysis", "repository": "govalidate-report-generation", "class": "symbol", "value": "generateSensitivityAnalysis", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-generate-suggested-next-steps", "repository": "govalidate-report-generation", "class": "symbol", "value": "generateSuggestedNextSteps", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-score-metric-batch", "repository": "govalidate-report-generation", "class": "symbol", "value": "scoreMetricBatch", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-deduplicate-evidence-refs", "repository": "govalidate-report-generation", "class": "symbol", "value": "deduplicateEvidenceRefs", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-map-composite-to-recommendation", "repository": "govalidate-report-generation", "class": "symbol", "value": "mapCompositeToRecommendation", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-normalize-metric", "repository": "govalidate-report-generation", "class": "symbol", "value": "normalizeMetric", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-metric-human-prompt", "repository": "govalidate-report-generation", "class": "symbol", "value": "metricHumanPrompt", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-fallback-metric", "repository": "govalidate-report-generation", "class": "symbol", "value": "fallbackMetric", "why": "Qualification-target symbol in a fixed demotion table." }, + { "id": "report-generation/symbol-dispatch-wave", "repository": "govalidate-report-generation", "class": "symbol", "value": "dispatchWave", "why": "Qualification-target symbol in a fixed promotion table." }, + { "id": "report-generation/symbol-dispatch-db-sync", "repository": "govalidate-report-generation", "class": "symbol", "value": "dispatchDbSync", "why": "Qualification-target symbol in a fixed promotion table." }, + { "id": "report-generation/symbol-broadcast-run-started", "repository": "govalidate-report-generation", "class": "symbol", "value": "broadcastRunStarted", "why": "Qualification-target symbol in a fixed promotion table." }, + { "id": "report-generation/symbol-broadcast-run-failed", "repository": "govalidate-report-generation", "class": "symbol", "value": "broadcastRunFailed", "why": "Qualification-target symbol in a fixed promotion table." }, + { "id": "report-generation/symbol-score-metrics", "repository": "govalidate-report-generation", "class": "symbol", "value": "scoreMetrics", "why": "Qualification-target symbol in a fixed promotion table." } + ], + "corpus_symbol_import": { + "source": "docs/qualification/corpus.json", + "pointer": "forbidden_target_symbols", + "why": "The live frozen contract already names the distinctive symbols of the currently pinned targets. Importing them keeps this manifest in step with the contract instead of drifting from it. The contract file is read, never written." + }, + "exceptions": [], + "exception_policy": { + "required_fields": ["id", "rule_id", "file", "why", "expires"], + "forbidden": [ + "an exception for any known qualification-specific production behaviour", + "a wildcard file pattern, or any pattern matching more than one file", + "an exception that matches nothing (a stale exception hides a rule that no longer fires)", + "an exception whose expiry date has passed" + ] + } +} diff --git a/scripts/lib/forbidden-knowledge-selftest.mjs b/scripts/lib/forbidden-knowledge-selftest.mjs new file mode 100644 index 00000000..0b8e9f36 --- /dev/null +++ b/scripts/lib/forbidden-knowledge-selftest.mjs @@ -0,0 +1,521 @@ +/** + * #660-B falsifiability harness for the production independence scan. + * + * A scan that has never been shown to fail is decorative. These controls put + * real qualification-repository knowledge back into a real production file, in + * each of the encodings the manifest claims to normalize, and require the + * scanner to reject it naming the exact file, line and rule. Then they put the + * bytes back and prove they went back. + * + * Each control declares a PREMISE -- the observable fact the injection was + * supposed to create -- checked before the verdict is read. Without it a + * silently no-op injection is indistinguishable from a working control, which + * is the failure mode that makes a guard look green while proving nothing. + * + * Restoration is by byte snapshot, never by `git checkout`/`reset`/`clean`: + * the worktree may carry other uncommitted work, and a git-based "restore" + * would destroy it. Every restore is digest-verified, and a file that cannot + * be restored is reported loudly. + * + * Runs standalone (never inside the vitest worker pool) because it mutates + * source files on disk. + */ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { ByteSnapshot } from './grader-boundary-selftest.mjs' +import { + analyzeForbiddenKnowledge, + loadForbiddenKnowledgeManifest, + FORBIDDEN_KNOWLEDGE_IN_PRODUCTION, + FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID, +} from './forbidden-knowledge.mjs' + +const MANIFEST = 'scripts/lib/forbidden-knowledge-manifest.json' + +// A production file that is decontaminated and stays in scope, so an injection +// here is a true reintroduction rather than a change to an unscanned file. +const TARGET = 'src/runtime/retrieve.ts' + +/** The scanner reported at least one violation at this file for this rule. */ +const expectViolation = (file, ruleId) => (result) => ( + (result.violations ?? []).some((violation) => ( + violation.file === file + && violation.rule === ruleId + && Number.isInteger(violation.line) + && violation.line > 0 + && typeof violation.raw === 'string' + && violation.raw.length > 0 + && typeof violation.normalized === 'string' + && violation.normalized.length > 0 + )) +) + +/** The scanner refused the manifest itself rather than scanning the tree. */ +const expectManifestProblem = (fragment) => (result) => ( + result.reason === FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID + && (result.manifestProblems ?? []).some((problem) => problem.includes(fragment)) +) + +function injectionCases() { + return [ + { + id: 'F1', + title: 'exact forbidden repository path reintroduced as a string literal', + expectRule: 'openstatus/path-router-status-page', + verdict: expectViolation(TARGET, 'openstatus/path-router-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F1 injection', + "const F1_PREFERRED_FILES = ['packages/api/src/router/statusPage.ts']", + 'void F1_PREFERRED_FILES', + '', + ].join('\n')) + }, + }, + { + id: 'F2', + title: 'forbidden symbol reintroduced in a normalized (snake_case) spelling', + expectRule: 'openstatus/symbol-http-checker-handler', + verdict: expectViolation(TARGET, 'openstatus/symbol-http-checker-handler'), + inject(snapshot) { + // Deliberately NOT the original spelling. If the scanner only matched + // the literal it was given, this passes unnoticed. + snapshot.append(TARGET, [ + '', + '// #660-B F2 injection', + "const F2_ROLE = 'http_checker_handler'", + 'void F2_ROLE', + '', + ].join('\n')) + }, + }, + { + id: 'F3', + title: 'forbidden symbol reintroduced in a case-flattened score table', + expectRule: 'report-generation/symbol-generate-scoring-ledger', + verdict: expectViolation(TARGET, 'report-generation/symbol-generate-scoring-ledger'), + inject(snapshot) { + // The encoding the removed demotion table actually used: all lowercase, + // no separators, inside a regex alternation. + snapshot.append(TARGET, [ + '', + '// #660-B F3 injection', + 'const F3_DEMOTIONS = /(?:generatescoringledger|somethingelse)/i', + 'void F3_DEMOTIONS', + '', + ].join('\n')) + }, + }, + { + id: 'F4', + title: 'forbidden path fragment reintroduced inside a template span', + expectRule: 'openstatus/path-status-page-utils', + verdict: expectViolation(TARGET, 'openstatus/path-status-page-utils'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F4 injection', + 'const F4_SUFFIX = `${String(1)}/statusPage.utils.ts`', + 'void F4_SUFFIX', + '', + ].join('\n')) + }, + }, + { + id: 'F12', + title: 'a hex-escaped string literal is decoded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F12 injection', + "const F12_NAME = '\\x73tatusPage'", + 'void F12_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F13', + title: 'a hex-escaped regex literal is decoded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F13 injection', + 'const F13_PATTERN = /status\\x50age/i', + 'void F13_PATTERN', + '', + ].join('\n')) + }, + }, + { + id: 'F14', + title: 'a name split across a static concatenation is folded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F14 injection', + "const F14_NAME = 'status' + 'Page'", + 'void F14_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F15', + title: 'a name split across a static template is folded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F15 injection', + 'const F15_NAME = `status${\'\'}Page`', + 'void F15_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F16', + title: 'a unicode-escaped string literal is decoded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F16 injection', + "const F16_NAME = '\\u0073tatusPage'", + 'void F16_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F19', + title: 'a legacy octal regex escape is decoded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + // /\\163tatusPage/ executes as /statusPage/. A scanner that reads the + // raw spelling sees nothing forbidden at all. + snapshot.append(TARGET, [ + '', + '// #660-B1 F19 injection', + 'const F19_PATTERN = /\\163tatusPage/i', + 'void F19_PATTERN', + '', + ].join('\n')) + }, + }, + { + id: 'F20', + title: 'a preferred-file list entry written as a static concatenation is folded', + expectRule: 'openstatus/path-router-status-page', + verdict: expectViolation(TARGET, 'openstatus/path-router-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B1 F20 injection', + "const F20_PREFERRED = ['packages/api/src/router/' + 'statusPage.ts']", + 'void F20_PREFERRED', + '', + ].join('\n')) + }, + }, + { + id: 'F24', + title: 'a name assembled through a spread argument is folded', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B1 F24 injection', + "const F24_NAME = 'status'.concat(...['Page'])", + 'void F24_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F25', + title: 'a name assembled across an array hole is folded', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + // `['status',, 'Page'].join('')` is `statusPage`: the elided element is + // undefined and join renders it as the empty string. + snapshot.append(TARGET, [ + '', + '// #660-B1 F25 injection', + "const F25_NAME = ['status',, 'Page'].join('')", + 'void F25_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F26', + title: 'a name assembled through a spread inside an array is folded', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B1 F26 injection', + "const F26_NAME = ['sta', ...['tus'], 'Page'].join('')", + 'void F26_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F22', + title: 'a name assembled with .concat() is folded before matching', + expectRule: 'openstatus/symbol-status-page', + verdict: expectViolation(TARGET, 'openstatus/symbol-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B1 F22 injection', + "const F22_NAME = 'status'.concat('Page')", + 'void F22_NAME', + '', + ].join('\n')) + }, + }, + { + id: 'F23', + title: 'a path assembled with array join is folded before matching', + expectRule: 'openstatus/path-router-status-page', + verdict: expectViolation(TARGET, 'openstatus/path-router-status-page'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B1 F23 injection', + "const F23_PATH = ['packages/api', 'src/router', 'statusPage.ts'].join('/')", + 'void F23_PATH', + '', + ].join('\n')) + }, + }, + { + id: 'F5', + title: 'forbidden symbol reintroduced as an identifier, not a string', + expectRule: 'openstatus/symbol-page-indicator', + verdict: expectViolation(TARGET, 'openstatus/symbol-page-indicator'), + inject(snapshot) { + snapshot.append(TARGET, [ + '', + '// #660-B F5 injection', + 'const pageIndicator = (value: string): string => value', + 'void pageIndicator', + '', + ].join('\n')) + }, + }, + ] +} + +function manifestCases() { + const base = () => JSON.parse(readFileSync(resolve(process.cwd(), MANIFEST), 'utf8')) + return [ + { + id: 'F6', + title: 'a malformed rule is refused rather than skipped', + verdict: expectManifestProblem('must be a non-empty string'), + mutate(manifest) { + manifest.rules.push({ id: 'broken/rule', repository: 'x', class: 'symbol', value: '', why: 'malformed on purpose' }) + return manifest + }, + }, + { + id: 'F7', + title: 'a duplicate rule id is refused', + verdict: expectManifestProblem('duplicates an earlier rule id'), + mutate(manifest) { + manifest.rules.push({ ...manifest.rules[0] }) + return manifest + }, + }, + { + id: 'F8', + title: 'a wildcard exception is refused', + verdict: expectManifestProblem('must be one exact repo-relative path'), + mutate(manifest) { + manifest.exceptions = [{ + id: 'wildcard', rule_id: manifest.rules[0].id, file: 'src/**/*.ts', + why: 'a boundary-less exemption', expires: '2099-01-01', + }] + return manifest + }, + }, + { + id: 'F9', + title: 'an expired exception is refused', + verdict: expectManifestProblem('expired on'), + mutate(manifest) { + manifest.exceptions = [{ + id: 'expired', rule_id: manifest.rules[0].id, file: TARGET, + why: 'stale on purpose', expires: '2020-01-01', + }] + return manifest + }, + }, + { + id: 'F17', + title: 'an impossible calendar date is refused, not merely shape-checked', + verdict: expectManifestProblem('must be a real ISO calendar date'), + mutate(manifest) { + manifest.exceptions = [{ + id: 'impossible-date', rule_id: manifest.rules[0].id, file: TARGET, + why: 'shaped like a date but is not one', expires: '2099-13-45', + }] + return manifest + }, + }, + { + id: 'F18', + title: 'two exemptions covering the same rule and file are refused', + verdict: expectManifestProblem('one exemption per rule per file'), + mutate(manifest) { + manifest.exceptions = [ + { id: 'first', rule_id: manifest.rules[0].id, file: TARGET, why: 'first', expires: '2099-01-01' }, + { id: 'second', rule_id: manifest.rules[0].id, file: TARGET, why: 'duplicate scope', expires: '2099-01-01' }, + ] + return manifest + }, + }, + { + id: 'F10', + title: 'an exception that matches nothing is reported as unused', + verdict: (result) => (result.unusedExceptions ?? []).some((entry) => entry.id === 'unused') && result.ok === false, + mutate(manifest) { + manifest.exceptions = [{ + id: 'unused', rule_id: manifest.rules[0].id, file: TARGET, + why: 'matches nothing because the tree is clean', expires: '2099-01-01', + }] + return manifest + }, + }, + ] + void base +} + +export function runForbiddenKnowledgeSelfTest({ root = process.cwd(), log = console.log } = {}) { + const results = [] + + // Read the untouched tree first, so a dirty baseline is reported as such + // rather than as a failed injection. + const baseline = analyzeForbiddenKnowledge({ root }) + results.push({ + id: 'F0', + title: 'the untouched tree is clean, so every injection below starts from zero', + passed: baseline.ok === true && baseline.violations.length === 0 && baseline.rulesApplied > 0, + detail: baseline.ok + ? `clean; ${baseline.filesScanned} file(s), ${baseline.rulesApplied} rule(s)` + : `baseline is NOT clean: ${baseline.violations.map((v) => `${v.file}:${v.line} [${v.rule}]`).join(', ')}`, + }) + + for (const testCase of injectionCases()) { + const snapshot = new ByteSnapshot(root) + let passed = false + let detail = '' + try { + testCase.inject(snapshot) + + // PREMISE: the injected text is really on disk. Without this a no-op + // injection would look exactly like a working control. + const injected = readFileSync(resolve(root, TARGET), 'utf8') + const premiseHolds = injected.includes(`${testCase.id} injection`) + if (!premiseHolds) { + detail = 'injection did not reach the file; the control proves nothing' + } else { + const result = analyzeForbiddenKnowledge({ root }) + passed = result.ok === false + && result.reason === FORBIDDEN_KNOWLEDGE_IN_PRODUCTION + && testCase.verdict(result) + const hit = (result.violations ?? []).find((v) => v.rule === testCase.expectRule) + detail = passed + ? `rejected at ${hit.file}:${hit.line} via ${hit.matchForms.join('+')}; raw ${JSON.stringify(hit.raw.trim())}` + : `expected rule ${testCase.expectRule} to fire; got ${ + (result.violations ?? []).map((v) => `${v.file}:${v.line} [${v.rule}]`).join(', ') || 'no violations'}` + } + } catch (error) { + detail = `control threw: ${error?.message ?? String(error)}` + } finally { + const unrestored = snapshot.restore() + if (unrestored.length > 0) { + passed = false + detail += ` | RESTORE FAILED: ${unrestored.join(', ')}` + } + } + results.push({ id: testCase.id, title: testCase.title, passed, detail }) + } + + for (const testCase of manifestCases()) { + const snapshot = new ByteSnapshot(root) + let passed = false + let detail = '' + try { + const mutated = testCase.mutate(JSON.parse(readFileSync(resolve(root, MANIFEST), 'utf8'))) + snapshot.write(MANIFEST, `${JSON.stringify(mutated, null, 2)}\n`) + + // PREMISE: the manifest on disk really changed. + const onDisk = JSON.parse(readFileSync(resolve(root, MANIFEST), 'utf8')) + const premiseHolds = JSON.stringify(onDisk) === JSON.stringify(mutated) + if (!premiseHolds) { + detail = 'manifest mutation did not reach disk; the control proves nothing' + } else { + const result = analyzeForbiddenKnowledge({ root }) + passed = result.ok === false && testCase.verdict(result) + detail = passed + ? `refused: ${(result.manifestProblems ?? []).concat( + (result.unusedExceptions ?? []).map((e) => `unused exception ${e.id}`), + ).join(' | ')}` + : `expected a refusal; got ok=${result.ok} reason=${result.reason} problems=${ + JSON.stringify(result.manifestProblems ?? [])}` + } + } catch (error) { + detail = `control threw: ${error?.message ?? String(error)}` + } finally { + const unrestored = snapshot.restore() + if (unrestored.length > 0) { + passed = false + detail += ` | RESTORE FAILED: ${unrestored.join(', ')}` + } + } + results.push({ id: testCase.id, title: testCase.title, passed, detail }) + } + + // The tree must be exactly as clean afterwards as it was before, or a later + // run inherits a mystery. + const afterAll = analyzeForbiddenKnowledge({ root }) + const manifestAfter = loadForbiddenKnowledgeManifest(root) + results.push({ + id: 'F11', + title: 'the tree and the manifest are restored exactly', + passed: afterAll.ok === true + && afterAll.violations.length === 0 + && manifestAfter.ok === true + && manifestAfter.rules.length === (baseline.rulesApplied ?? 0), + detail: afterAll.ok && manifestAfter.ok + ? `clean; ${manifestAfter.rules.length} rule(s) restored` + : `NOT restored: ${afterAll.violations.map((v) => `${v.file}:${v.line}`).join(', ')} ${manifestAfter.problems.join(' | ')}`, + }) + + for (const entry of results) { + log(` ${entry.passed ? 'PASS' : 'FAIL'} ${entry.id} ${entry.title}`) + log(` ${entry.detail}`) + } + + return { ok: results.every((entry) => entry.passed), results } +} diff --git a/scripts/lib/forbidden-knowledge.d.mts b/scripts/lib/forbidden-knowledge.d.mts new file mode 100644 index 00000000..bd7f49ae --- /dev/null +++ b/scripts/lib/forbidden-knowledge.d.mts @@ -0,0 +1,114 @@ +/** The scanner's declared capability boundary; asserted by its contract test. */ +export declare const SCANNER_CAPABILITIES: { + readonly literal_and_static_detection: true + readonly regex_semantic_evaluation: false + readonly runtime_constructed_value_proof: false + readonly semantic_overfitting_proof: false +} + +export declare const FORBIDDEN_KNOWLEDGE_IN_PRODUCTION: 'FORBIDDEN_QUALIFICATION_KNOWLEDGE_IN_PRODUCTION' +export declare const FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID: 'FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID' + +/** One entry in the versioned manifest, or one imported from the frozen contract. */ +export interface ForbiddenKnowledgeRule { + readonly id: string + readonly repository: string + readonly class: 'path' | 'symbol' | 'phrase' + readonly value: string + readonly why: string + /** The file this rule came from: the manifest, or the frozen contract. */ + readonly origin: string +} + +/** A narrow, dated, single-file exemption from one rule. */ +export interface ForbiddenKnowledgeException { + readonly id: string + readonly ruleId: string + readonly file: string + readonly why: string + readonly expires: string +} + +export interface ForbiddenKnowledgeManifest { + readonly ok: boolean + readonly problems: readonly string[] + readonly rules: readonly ForbiddenKnowledgeRule[] + readonly exceptions: readonly ForbiddenKnowledgeException[] + readonly manifestVersion?: string +} + +/** One occurrence of qualification-repository knowledge in production source. */ +export interface ForbiddenKnowledgeViolation { + readonly file: string + readonly line: number + /** Where in the file the match sits: a literal, a regex, a template span, an identifier or a comment. */ + readonly site: 'string' | 'regex' | 'template' | 'identifier' | 'comment' | 'folded' + readonly rule: string + readonly repository: string + readonly ruleClass: 'path' | 'symbol' | 'phrase' + readonly ruleValue: string + readonly why: string + readonly raw: string + /** The site's value after static decoding, so an escaped spelling is legible. */ + readonly decoded: string + readonly normalized: string + /** Which normalization form matched: whole tokens, the case-flattened form, or both. */ + readonly matchForms: readonly ('tokens' | 'squashed')[] +} + +/** Proof that the one-pass index did what it claims. Read by the controls. */ +export interface ForbiddenKnowledgeStats { + readonly indexedFiles: number + readonly parseCalls: number + readonly siteCount: number +} + +export interface ProductionSourceIndex { + readonly byFile: ReadonlyMap + readonly stats: ForbiddenKnowledgeStats +} + +export interface ForbiddenKnowledgeResult { + readonly ok: boolean + readonly reason: string | null + readonly manifestProblems: readonly string[] + readonly manifestVersion: string | null + readonly violations: readonly ForbiddenKnowledgeViolation[] + readonly filesScanned: number + readonly rulesApplied: number + readonly unusedExceptions: readonly ForbiddenKnowledgeException[] + readonly stats: ForbiddenKnowledgeStats +} + +export interface ForbiddenKnowledgeInput { + readonly root?: string + readonly files?: readonly string[] + readonly manifest?: ForbiddenKnowledgeManifest + readonly readFile?: (file: string) => string + /** A prebuilt index, so a caller can prove the sources are parsed once. */ + readonly index?: ProductionSourceIndex + /** Overrides "now" for expiry checks; ISO date, tests only. */ + readonly today?: string +} + +/** One place in a source file where a name, path or shape can be written down. */ +export interface KnowledgeBearingSite { + readonly kind: 'string' | 'regex' | 'template' | 'identifier' | 'comment' | 'folded' + readonly text: string + readonly line: number +} + +export declare function tokenForm(value: string): string +export declare function squashForm(value: string): string +export declare function decodeEscapes(value: string): string +export declare function knowledgeBearingSites(sourceText: string, fileName?: string): KnowledgeBearingSite[] +export declare function buildProductionSourceIndex(input: { + readonly files: readonly string[] + readonly readFile: (file: string) => string +}): ProductionSourceIndex +export declare function loadForbiddenKnowledgeManifest( + root?: string, + options?: { readonly today?: string }, +): ForbiddenKnowledgeManifest +export declare function analyzeForbiddenKnowledge(input?: ForbiddenKnowledgeInput): ForbiddenKnowledgeResult +export declare function formatForbiddenKnowledgeReport(result: ForbiddenKnowledgeResult): string diff --git a/scripts/lib/forbidden-knowledge.mjs b/scripts/lib/forbidden-knowledge.mjs new file mode 100644 index 00000000..94092413 --- /dev/null +++ b/scripts/lib/forbidden-knowledge.mjs @@ -0,0 +1,826 @@ +/** + * #660-B -- production independence from qualification-repository knowledge. + * + * Madar's retrieval and claim behaviour must follow generic structural + * evidence. A name, path or code shape lifted from a qualification target that + * reaches production means behaviour was shaped around that repository instead + * of around evidence, and a benchmark result produced that way measures the + * tuning rather than the tool. + * + * This module owns the LITERAL and NORMALIZED half of that guarantee. It + * parses every production source file with the TypeScript compiler and + * inspects the places repository knowledge can actually be written down: + * + * - string literals, no-substitution templates, and every fixed span of a + * template expression (preferred-file arrays, score-table keys, paths) + * - regular-expression literals (the form most contaminated rules used) + * - identifiers and property names (symbols, preferred-symbol lists) + * - comments (a target named in prose is still that knowledge in production) + * + * Encodings ARE decoded before matching, because a rule spelled + * `'\x73tatusPage'`, `/\163tatusPage/`, `'status' + 'Page'` or a static + * template is the same knowledge as the plain spelling. String literals are + * read through the compiler, which decodes escapes for us; regex SOURCE is + * decoded here as text; statically foldable concatenations, templates, + * `.concat` and `[].join` are folded before they are matched. + * + * WHAT IT IS NOT + * + * It does not execute, compile, simulate, or decide the semantic language of an + * arbitrary regular expression, and it must not start. An earlier version + * compiled patterns to ask whether they COULD match a forbidden value: that + * produced 1662 false positives on a clean tree, needed five rounds of + * narrowing to reach zero, and still failed OPEN whenever its own safety bounds + * were exceeded. Deciding what a regex can match is not a job a literal scanner + * can finish, and a guard that fails open is worse than one with a stated edge. + * + * THE EDGE, STATED RATHER THAN IMPLIED + * + * A pattern whose forbidden value exists only in its MATCHING SEMANTICS is + * outside this contract. `/statusP{1}age/` and `/^(?=statusP{1}age$)/` both + * match `statusPage` at runtime and are NOT detected here, because no decoded + * textual run of the source spells the value. Some semantically clever patterns + * are caught anyway when their source happens to contain the run -- that is a + * coincidence of spelling, not a capability, and is not claimed as one. + * + * Only STATICALLY foldable expressions fold. A name assembled at runtime is + * beyond any static scanner, and this one does not pretend otherwise. + * + * That class, and semantic overfitting generally -- a rule keyed on prompt + * vocabulary, a forced selection, a reserved result slot -- is owned by direct + * behavioural independence tests, unrelated-name controls, + * renamed-implementation controls, independent holdout evaluation, and code + * review. Pretending a text scanner covers it would be the more dangerous + * error. + */ +import { readFileSync, existsSync } from 'node:fs' +import { resolve } from 'node:path' +import ts from 'typescript' +import { productionSourceFiles } from './grader-boundary.mjs' + +/** + * What this scanner claims to do, declared rather than implied. + * + * The capability boundary is data so it can be asserted by a test instead of + * living in prose that drifts. A future implementation may widen it only by + * deliberately changing this declaration and the test that pins it -- which is + * the point: an accidental re-introduction of pattern evaluation cannot pass + * silently. + */ +/** + * One elided array element, kept distinct from the empty string. + * + * `Array.prototype.join` renders a hole as '', while spreading one into + * `String.prototype.concat` yields `undefined`, which stringifies as the text + * 'undefined'. Collapsing the two at collection time reported + * `'status'.concat(...[, 'Page'])` -- really `'statusundefinedPage'` -- as the + * forbidden name `statusPage`. + */ +const HOLE = Symbol('array-hole') + +/** How `Array.prototype.join` renders an element. */ +const renderForJoin = (part) => (part === HOLE ? '' : part) + +/** How `String.prototype.concat` renders an argument. */ +const renderForConcat = (part) => (part === HOLE ? 'undefined' : part) + +export const SCANNER_CAPABILITIES = Object.freeze({ + /** Literals, decoded escapes, normalized forms, bounded static folding. */ + literal_and_static_detection: true, + /** Deciding what an arbitrary regular expression can match. Never. */ + regex_semantic_evaluation: false, + /** Proving anything about values assembled at runtime. */ + runtime_constructed_value_proof: false, + /** Proving the absence of semantic overfitting; owned by behavioural tests. */ + semantic_overfitting_proof: false, +}) + +export const FORBIDDEN_KNOWLEDGE_IN_PRODUCTION = 'FORBIDDEN_QUALIFICATION_KNOWLEDGE_IN_PRODUCTION' +export const FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID = 'FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID' + +const MANIFEST_PATH = 'scripts/lib/forbidden-knowledge-manifest.json' +const CORPUS_PATH = 'docs/qualification/corpus.json' + +/* ------------------------------------------------------------------ * + * Normalization + * ------------------------------------------------------------------ */ + +/** + * Whole-token form. Splits camel/Pascal humps, turns every run of + * non-alphanumerics into one space, lowercases, and pads with spaces so a + * needle matches as a contiguous run of WHOLE tokens rather than as a bare + * substring. `statusPage`, `status_page`, `status-page`, `status.page` and + * `status/page` all reduce to the same thing. + */ +export function tokenForm(value) { + const spaced = String(value) + .replaceAll('\\', '/') + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() + return spaced.length === 0 ? '' : ` ${spaced} ` +} + +/** + * Case-flattened form. Removes every separator, so `generateScoringLedger` and + * the `generatescoringledger` spelling a lowercasing rule actually uses reduce + * to one string. Without this the manifest would miss the encoding in which + * most of the removed demotion tables were written. + */ +export function squashForm(value) { + return String(value).toLowerCase().replace(/[^a-z0-9]+/g, '') +} + +/** + * Decode the escape forms a name can hide behind in regex source. TypeScript + * already decodes escapes inside string literals for us; regex source arrives + * raw, so \x50 and P have to be resolved here or /status\x50age/i reads as + * innocent text. A stray backslash before an ordinary word character is dropped + * for the same reason. + */ +export function decodeEscapes(value) { + return String(value) + .replace(/\\u\{([0-9a-fA-F]+)\}/g, (_, hex) => safeFromCodePoint(hex, 16)) + .replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => safeFromCodePoint(hex, 16)) + .replace(/\\x([0-9a-fA-F]{2})/g, (_, hex) => safeFromCodePoint(hex, 16)) + // Legacy ECMAScript octal escapes, which are still executable inside a + // regex: /\163tatusPage/ runs as /statusPage/. Decoded BEFORE the generic + // stray-backslash rule below, which would otherwise turn \163 into 163 and + // destroy the evidence. + .replace(/\\([0-7]{1,3})/g, (whole, digits) => decodeLegacyOctal(whole, digits)) + .replace(/\\([A-Za-z0-9])/g, '$1') +} + +function safeFromCodePoint(digits, radix) { + const code = Number.parseInt(digits, radix) + return Number.isInteger(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : '' +} + +/** + * `\NNN` is ambiguous in a regex: it is a BACKREFERENCE when capture groups + * exist and a legacy octal escape otherwise, and a static scanner cannot always + * tell. The split used here is the one that matters for hiding a name: only + * escapes that decode to a PRINTABLE ASCII character are treated as octal. + * + * That resolves the ambiguity in the safe direction. `\163` decodes to `s` and + * is treated as text; `\1`..`\9` decode to control characters, are left alone, + * and continue to read as backreferences. Anything outside printable ASCII is + * returned unchanged rather than silently deleted, so an unsupported spelling + * stays visible in the decoded value instead of being called clean. + */ +function decodeLegacyOctal(whole, digits) { + const code = Number.parseInt(digits, 8) + if (!Number.isInteger(code) || code < 0x20 || code > 0x7e) { + return whole + } + return String.fromCodePoint(code) +} + +/** + * Both normalized forms of one value, computed ONCE. + * + * The forms used to be recomputed for every (site, rule) pair, which made the + * scan cost sites x rules string transformations: measured at 8.7s for 36 + * rules and 4.4s for 18, i.e. linear in the rule count, which is what tripped + * the protected 15s control. Precomputing on both sides makes matching a plain + * substring test and the rule count stops driving the cost. + */ +function normalizedForms(value) { + const decoded = decodeEscapes(value) + return { decoded, tokens: tokenForm(decoded), squashed: squashForm(decoded) } +} + +/** + * Which precomputed forms of a site contain the precomputed forms of a rule. + * + * Purely textual, and deliberately so. EVERY rule class is tested against EVERY + * site kind, including regex sources: there is no per-class or per-kind + * exclusion left to hide behind, and the result cannot depend on the order the + * rules were declared in. + */ +function matchPrecomputedForms(site, rule) { + const forms = [] + if (rule.tokens.length > 0 && site.tokens.includes(rule.tokens)) { + forms.push('tokens') + } + if (rule.squashed.length > 0 && site.squashed.includes(rule.squashed)) { + forms.push('squashed') + } + return forms +} + +/* ------------------------------------------------------------------ * + * Manifest + * ------------------------------------------------------------------ */ + +const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/ +const RULE_CLASSES = new Set(['path', 'symbol', 'phrase']) +const MINIMUM_DISTINCTIVE_LENGTH = 4 + +/** `2099-13-45` matches the shape but is not a date. Reject it as malformed. */ +function isRealCalendarDate(value) { + const [year, month, day] = value.split('-').map(Number) + if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) { + return false + } + const date = new Date(Date.UTC(year, month - 1, day)) + return date.getUTCFullYear() === year + && date.getUTCMonth() === month - 1 + && date.getUTCDate() === day +} + +/** + * Fail-closed manifest validation. A guard whose configuration is not itself + * checked can be switched off by a typo, so a malformed manifest is a failure + * of the check, never a warning beside a green result. + */ +export function loadForbiddenKnowledgeManifest(root = process.cwd(), options = {}) { + const problems = [] + const manifestPath = resolve(root, MANIFEST_PATH) + if (!existsSync(manifestPath)) { + return { ok: false, problems: [`manifest missing at ${MANIFEST_PATH}`], rules: [], exceptions: [] } + } + + let raw + try { + raw = JSON.parse(readFileSync(manifestPath, 'utf8')) + } catch (error) { + return { ok: false, problems: [`manifest is not valid JSON: ${error.message}`], rules: [], exceptions: [] } + } + + if (typeof raw.manifest_version !== 'string' || raw.manifest_version.trim().length === 0) { + problems.push('manifest_version must be a non-empty string') + } + + const rules = [] + const seenIds = new Set() + const seenNormalized = new Map() + + if (!Array.isArray(raw.rules) || raw.rules.length === 0) { + problems.push('rules must be a non-empty array') + } else { + for (const [index, entry] of raw.rules.entries()) { + const where = `rules[${index}]` + if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) { + problems.push(`${where} must be an object`) + continue + } + + const { id, repository, class: ruleClass, value, why } = entry + let malformed = false + for (const [field, fieldValue] of Object.entries({ id, repository, value, why })) { + if (typeof fieldValue !== 'string' || fieldValue.trim().length === 0) { + problems.push(`${where}.${field} must be a non-empty string`) + malformed = true + } + } + if (!RULE_CLASSES.has(ruleClass)) { + problems.push(`${where}.class must be one of ${[...RULE_CLASSES].join(', ')}`) + malformed = true + } + if (malformed) { + continue + } + + if (seenIds.has(id)) { + problems.push(`${where}.id duplicates an earlier rule id: ${id}`) + continue + } + seenIds.add(id) + + // Two rules that normalize identically can never both fire, so the + // second is dead configuration that makes the manifest look broader + // than the protection it actually provides. + const normalizedKey = `${squashForm(value)}${tokenForm(value)}` + if (seenNormalized.has(normalizedKey)) { + problems.push(`${where} (${id}) normalizes identically to ${seenNormalized.get(normalizedKey)}; remove the duplicate`) + continue + } + seenNormalized.set(normalizedKey, id) + + if (squashForm(value).length < MINIMUM_DISTINCTIVE_LENGTH) { + problems.push(`${where} (${id}) value ${JSON.stringify(value)} is too short to be distinctive; a rule this broad would fire on ordinary code`) + continue + } + + rules.push({ id, repository, class: ruleClass, value, why, origin: MANIFEST_PATH }) + } + } + + // The frozen contract already names the distinctive symbols of the pinned + // targets. Importing them keeps this manifest in step with the contract + // instead of drifting from it. The contract file is read, never written. + const importSpec = raw.corpus_symbol_import + if (importSpec !== undefined) { + if (importSpec === null || typeof importSpec !== 'object' || Array.isArray(importSpec)) { + problems.push('corpus_symbol_import must be an object when present') + } else { + const corpusRelative = importSpec.source ?? CORPUS_PATH + const corpusPath = resolve(root, corpusRelative) + if (!existsSync(corpusPath)) { + problems.push(`corpus_symbol_import.source missing at ${corpusRelative}`) + } else { + try { + const corpus = JSON.parse(readFileSync(corpusPath, 'utf8')) + const pointer = importSpec.pointer ?? 'forbidden_target_symbols' + const groups = corpus[pointer] + let imported = 0 + if (groups !== null && typeof groups === 'object' && !Array.isArray(groups)) { + for (const [target, symbols] of Object.entries(groups)) { + if (!Array.isArray(symbols)) { + continue + } + for (const symbol of symbols) { + if (typeof symbol !== 'string' || squashForm(symbol).length < MINIMUM_DISTINCTIVE_LENGTH) { + continue + } + const normalizedKey = `${squashForm(symbol)}${tokenForm(symbol)}` + if (seenNormalized.has(normalizedKey)) { + continue + } + seenNormalized.set(normalizedKey, `corpus/${target}/${symbol}`) + rules.push({ + id: `corpus/${target}/${symbol}`, + repository: target, + class: 'symbol', + value: symbol, + why: `Declared distinctive symbol of pinned qualification target "${target}" in the frozen contract.`, + origin: corpusRelative, + }) + imported += 1 + } + } + } + if (imported === 0) { + problems.push(`corpus_symbol_import matched no symbols at ${corpusRelative}#${pointer}; the pointer or the contract shape changed`) + } + } catch (error) { + problems.push(`corpus_symbol_import unreadable: ${error.message}`) + } + } + } + } + + const exceptions = [] + const ruleIds = new Set(rules.map((rule) => rule.id)) + const rawExceptions = raw.exceptions ?? [] + if (!Array.isArray(rawExceptions)) { + problems.push('exceptions must be an array') + } else { + const seenExceptionIds = new Set() + const seenExceptionScopes = new Map() + const today = options.today ?? new Date().toISOString().slice(0, 10) + for (const [index, entry] of rawExceptions.entries()) { + const where = `exceptions[${index}]` + if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) { + problems.push(`${where} must be an object`) + continue + } + + const { id, rule_id: ruleId, file, why, expires } = entry + let malformed = false + for (const [field, fieldValue] of Object.entries({ id, rule_id: ruleId, file, why, expires })) { + if (typeof fieldValue !== 'string' || fieldValue.trim().length === 0) { + problems.push(`${where}.${field} must be a non-empty string`) + malformed = true + } + } + if (malformed) { + continue + } + + if (seenExceptionIds.has(id)) { + problems.push(`${where}.id duplicates an earlier exception id: ${id}`) + continue + } + seenExceptionIds.add(id) + + if (!ruleIds.has(ruleId)) { + problems.push(`${where}.rule_id ${JSON.stringify(ruleId)} matches no rule`) + continue + } + + // A wildcard exception is an exemption with no boundary: it keeps + // approving files nobody reviewed. One exception, one exact file. + if (/[*?[\]]/.test(file) || file.includes('..')) { + problems.push(`${where}.file must be one exact repo-relative path, not a pattern: ${file}`) + continue + } + + if (!ISO_DATE.test(expires) || !isRealCalendarDate(expires)) { + problems.push(`${where}.expires must be a real ISO calendar date (YYYY-MM-DD): ${expires}`) + continue + } + if (expires < today) { + problems.push(`${where} (${id}) expired on ${expires}; renew it with a reason or delete it`) + continue + } + + const scope = `${ruleId}\u0000${file}` + if (seenExceptionScopes.has(scope)) { + problems.push(`${where} (${id}) covers the same rule and file as ${seenExceptionScopes.get(scope)}; one exemption per rule per file`) + continue + } + seenExceptionScopes.set(scope, id) + + exceptions.push({ id, ruleId, file, why, expires }) + } + } + + return { + ok: problems.length === 0, + problems, + rules, + exceptions, + manifestVersion: raw.manifest_version, + } +} + +/* ------------------------------------------------------------------ * + * Scanning + * ------------------------------------------------------------------ */ + +/** + * Every place a name, path or shape can be written down in a source file, with + * the line it sits on. Literals and regexes carry the encoded forms; + * identifiers carry symbol references; comments carry prose knowledge. + */ +export function knowledgeBearingSites(sourceText, fileName = 'file.ts') { + const source = ts.createSourceFile(fileName, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS) + const sites = [] + const seen = new Set() + + const record = (kind, text, position) => { + if (typeof text !== 'string' || text.trim().length === 0) { + return + } + const line = source.getLineAndCharacterOfPosition(position).line + 1 + const key = `${kind}${line}${text}` + if (seen.has(key)) { + return + } + seen.add(key) + sites.push({ kind, text, line }) + } + + /** + * The string a statically foldable expression evaluates to, or null. + * + * `'status' + 'Page'` and `` `status${''}Page` `` are the same name as the + * plain spelling, split across nodes that individually match nothing. Folding + * them is what stops "write it in two pieces" from being a way through. + * Only fully static expressions fold; anything computed at runtime returns + * null and is left to the behavioural controls. + */ + /** + * A list of expressions flattened to strings, or null if any element is not + * statically known. + * + * Two shapes are handled that a plain per-element fold misses, and both + * evaluate at runtime to exactly the plain spelling: + * + * - SPREAD of a statically known array: `.concat(...['Page'])` + * - a HOLE in an array literal: `['status',, 'Page'].join('')`, where the + * elided element is `undefined` and `join` renders it as the empty + * string, exactly as `Array.prototype.join` specifies. + */ + /** + * Whether an expression is a STRING at runtime, as opposed to an array. + * + * `.concat` exists on both, and the two have different semantics: a string + * receiver concatenates to a string, an array receiver produces an array. + * Only the string form may fold. This is a deliberately narrow shape test, + * not type inference: anything it cannot recognise as a string literal form + * returns false and the expression is left unfolded. + */ + const isStringValuedExpression = (node) => { + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node) || ts.isTemplateExpression(node)) { + return true + } + if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isSatisfiesExpression(node)) { + return isStringValuedExpression(node.expression) + } + if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.PlusToken) { + return isStringValuedExpression(node.left) || isStringValuedExpression(node.right) + } + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression)) { + const method = node.expression.name.text + if (method === 'join') { + return ts.isArrayLiteralExpression(node.expression.expression) + } + if (method === 'concat') { + return isStringValuedExpression(node.expression.expression) + } + } + return false + } + + /** + * Array ELEMENTS, or null if any is not statically known. Used only by the + * operations whose JavaScript semantics are modelled -- `.join(separator)` on + * an array literal, and static spread operands -- never to give a bare array + * a string value of its own. + * + * A hole is yielded as the HOLE sentinel rather than pre-rendered, because + * its rendering depends on who consumes it, and getting that wrong produced a + * false positive: + * + * ['status', , 'Page'].join('') -> 'statusPage' hole is '' + * 'status'.concat(...[, 'Page']) -> 'statusundefinedPage' hole is undefined + * + * Spreading a hole materialises a real `undefined` element, which `join` + * still renders as '' and string `.concat` renders as the text 'undefined'. + * Both are the same sentinel here; only the consumer differs. + */ + const staticList = (elements) => { + const parts = [] + for (const element of elements) { + if (ts.isOmittedExpression(element)) { + parts.push(HOLE) + continue + } + if (ts.isSpreadElement(element)) { + if (!ts.isArrayLiteralExpression(element.expression)) { + return null + } + const inner = staticList(element.expression.elements) + if (inner === null) { + return null + } + parts.push(...inner) + continue + } + const value = staticValue(element) + if (value === null) { + return null + } + parts.push(value) + } + return parts + } + + const staticValue = (node) => { + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) { + return node.text + } + if (ts.isParenthesizedExpression(node)) { + return staticValue(node.expression) + } + if (ts.isAsExpression(node) || ts.isSatisfiesExpression(node) || ts.isTypeAssertionExpression?.(node)) { + return staticValue(node.expression) + } + if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.PlusToken) { + const left = staticValue(node.left) + const right = staticValue(node.right) + return left === null || right === null ? null : left + right + } + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression)) { + const method = node.expression.name.text + // `'status'.concat('Page')`, and `'status'.concat(...['Page'])`. + // + // Only a STRING receiver folds. `['status'].concat('Page')` is + // Array.prototype.concat and evaluates to the ARRAY ['status','Page'], + // not to the string 'statusPage'; folding it produced a false positive. + if (method === 'concat') { + if (!isStringValuedExpression(node.expression.expression)) { + return null + } + const receiver = staticValue(node.expression.expression) + if (receiver === null) { + return null + } + const parts = staticList(node.arguments) + // String.prototype.concat stringifies undefined as 'undefined'. + return parts === null ? null : receiver + parts.map(renderForConcat).join('') + } + // `['status', 'Page'].join('')`, including spreads and holes. + if (method === 'join' && ts.isArrayLiteralExpression(node.expression.expression)) { + const separator = node.arguments.length === 0 ? ',' : staticValue(node.arguments[0]) + if (separator === null) { + return null + } + const parts = staticList(node.expression.expression.elements) + // Array.prototype.join renders holes and undefined as the empty string. + return parts === null ? null : parts.map(renderForJoin).join(separator) + } + } + if (ts.isTemplateExpression(node)) { + let folded = node.head.text + for (const span of node.templateSpans) { + const inner = staticValue(span.expression) + if (inner === null) { + return null + } + folded += inner + span.literal.text + } + return folded + } + return null + } + + const visit = (node) => { + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) { + record('string', node.text, node.getStart(source)) + } else if (ts.isRegularExpressionLiteral(node)) { + record('regex', node.text, node.getStart(source)) + } else if (ts.isTemplateHead(node) || ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { + // Fixed spans of a template: the interpolations vary, the spans do not. + record('template', node.text, node.getStart(source)) + } else if (ts.isIdentifier(node) || ts.isPrivateIdentifier(node)) { + record('identifier', node.text, node.getStart(source)) + } + + // Fold whole static expressions too, so a name split across several nodes + // is matched as the one string it actually denotes. + if (ts.isBinaryExpression(node) || ts.isTemplateExpression(node) || ts.isCallExpression(node)) { + const folded = staticValue(node) + if (folded !== null) { + record('folded', folded, node.getStart(source)) + } + } + + ts.forEachChild(node, visit) + } + visit(source) + + // Comments are not reachable from forEachChild, and a qualification target + // named in prose is still that knowledge sitting in production. + for (const match of sourceText.matchAll(/\/\/[^\n]*|\/\*[\s\S]*?\*\//g)) { + record('comment', match[0], match.index) + } + + return sites +} + +/** + * Read, decode, parse and normalize every production file exactly ONCE. + * + * The index is the scanner's only view of the tree; rules are applied to it + * afterwards, so adding a rule costs one substring test per site rather than + * another pass over the sources. `stats` is not decoration: the controls read + * it to prove each file was parsed exactly once and that the parse count does + * not move when the rule count does. + */ +export function buildProductionSourceIndex({ files, readFile }) { + const byFile = new Map() + const stats = { indexedFiles: 0, parseCalls: 0, siteCount: 0 } + + for (const file of files) { + if (byFile.has(file)) { + // A duplicate entry in the file list must not become a second parse. + continue + } + const text = readFile(file) + stats.parseCalls += 1 + const sites = knowledgeBearingSites(text, file).map((site) => ({ + ...site, + ...normalizedForms(site.text), + })) + byFile.set(file, sites) + stats.indexedFiles += 1 + stats.siteCount += sites.length + } + + return { byFile, stats } +} + +function isExcepted(exceptions, ruleId, file) { + return exceptions.some((exception) => exception.ruleId === ruleId && exception.file === file) +} + +export function analyzeForbiddenKnowledge(input = {}) { + const root = input.root ?? process.cwd() + const manifest = input.manifest ?? loadForbiddenKnowledgeManifest(root, input) + + if (!manifest.ok) { + return { + ok: false, + reason: FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID, + manifestProblems: manifest.problems, + manifestVersion: manifest.manifestVersion ?? null, + violations: [], + filesScanned: 0, + rulesApplied: 0, + unusedExceptions: [], + stats: { indexedFiles: 0, parseCalls: 0, siteCount: 0 }, + } + } + + const files = input.files ?? productionSourceFiles(root) + const readFile = input.readFile ?? ((file) => readFileSync(resolve(root, file), 'utf8')) + + // Every file is read, decoded, parsed and normalized exactly once, BEFORE any + // rule is consulted. There is deliberately no raw-text pre-filter: it would + // read the file before escapes are decoded and before split literals are + // folded, so an escaped or split name would skip the AST walk entirely and + // the scan would report clean. + const index = input.index ?? buildProductionSourceIndex({ files, readFile }) + + // Rule needles are normalized once as well, not once per site. + const rules = manifest.rules.map((rule) => ({ ...rule, ...normalizedForms(rule.value) })) + + const violations = [] + const usedExceptions = new Set() + + for (const [file, sites] of index.byFile) { + for (const site of sites) { + for (const rule of rules) { + const forms = matchPrecomputedForms(site, rule) + if (forms.length === 0) { + continue + } + if (isExcepted(manifest.exceptions, rule.id, file)) { + usedExceptions.add(`${rule.id}${file}`) + continue + } + violations.push({ + file, + line: site.line, + site: site.kind, + rule: rule.id, + repository: rule.repository, + ruleClass: rule.class, + ruleValue: rule.value, + why: rule.why, + raw: site.text.length > 200 ? `${site.text.slice(0, 200)}...` : site.text, + decoded: site.decoded.length > 200 ? `${site.decoded.slice(0, 200)}...` : site.decoded, + normalized: forms.includes('tokens') ? site.tokens.trim() : site.squashed, + matchForms: forms, + }) + } + } + } + + // A stale exception is worse than none: it silently keeps approving a rule + // that no longer fires, and hides the day that rule starts firing again. + const unusedExceptions = manifest.exceptions.filter( + (exception) => !usedExceptions.has(`${exception.ruleId}${exception.file}`), + ) + + violations.sort((left, right) => ( + left.file.localeCompare(right.file) + || left.line - right.line + || left.rule.localeCompare(right.rule) + || left.site.localeCompare(right.site) + )) + + return { + ok: violations.length === 0 && unusedExceptions.length === 0, + reason: violations.length > 0 ? FORBIDDEN_KNOWLEDGE_IN_PRODUCTION : null, + manifestProblems: [], + manifestVersion: manifest.manifestVersion, + violations, + filesScanned: index.stats.indexedFiles, + rulesApplied: rules.length, + unusedExceptions, + stats: index.stats, + } +} + +export function formatForbiddenKnowledgeReport(result) { + const lines = [] + + if (result.reason === FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID) { + lines.push(`${FORBIDDEN_KNOWLEDGE_MANIFEST_INVALID}: the manifest is not usable, so the scan proves nothing.`) + for (const entry of result.manifestProblems) { + lines.push(` - ${entry}`) + } + return lines.join('\n') + } + + lines.push( + `Production independence scan: ${result.filesScanned} production files, ` + + `${result.rulesApplied} manifest rules (manifest ${result.manifestVersion}).`, + ) + + if (result.violations.length > 0) { + lines.push('') + lines.push(`${FORBIDDEN_KNOWLEDGE_IN_PRODUCTION}: ${result.violations.length} occurrence(s).`) + for (const violation of result.violations) { + lines.push(` ${violation.file}:${violation.line} [${violation.rule}]`) + lines.push(` site ${violation.site}, matched via ${violation.matchForms.join('+')}`) + lines.push(` rule ${violation.ruleClass} ${JSON.stringify(violation.ruleValue)} (${violation.repository}) -- ${violation.why}`) + lines.push(` raw ${violation.raw}`) + lines.push(` normalized ${violation.normalized}`) + } + } + + if (result.unusedExceptions.length > 0) { + lines.push('') + lines.push(`${result.unusedExceptions.length} exception(s) matched nothing and must be deleted:`) + for (const exception of result.unusedExceptions) { + lines.push(` - ${exception.id} (rule ${exception.ruleId}, file ${exception.file})`) + } + } + + if (result.ok) { + lines.push('') + lines.push('No qualification-repository knowledge found in production source.') + lines.push('NOTE: this scan owns literal and normalized contamination only. Task-phrase') + lines.push('ranking and forced-selection contamination are owned by behavioural tests.') + } + + return lines.join('\n') +} diff --git a/scripts/lib/semantic-independence-selftest.mjs b/scripts/lib/semantic-independence-selftest.mjs new file mode 100644 index 00000000..6762a838 --- /dev/null +++ b/scripts/lib/semantic-independence-selftest.mjs @@ -0,0 +1,368 @@ +/** + * #660-B falsifiability harness for the SEMANTIC half of production independence. + * + * The forbidden-knowledge scanner cannot see this class. A rule keyed on prompt + * vocabulary, or one that forces a candidate into the result, encodes a + * qualification task while containing no name any manifest could list. That + * class is owned by behavioural tests -- so those tests have to be shown to + * fail when the contamination comes back, or they are decoration. + * + * Each control puts one real task-phrase or forced-selection rule back into a + * real production file, runs the specific behavioural test that owns it as a + * child process, and requires that test to FAIL. It then restores the bytes and + * proves they went back, and finally requires the same test to pass again, so a + * control that "passed" because the suite was broken all along is caught. + * + * Runs standalone (never inside the vitest worker pool) because it mutates + * source files on disk and spawns vitest. + */ +import { spawnSync } from 'node:child_process' +import { createRequire } from 'node:module' +import { existsSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' + +import { ByteSnapshot } from './grader-boundary-selftest.mjs' + +/** + * Resolved from vitest's own package manifest rather than shelled out to, so + * this works identically on Windows, where `npx` is a `.cmd` shim that + * `spawnSync` cannot execute directly. + */ +function resolveVitestBin() { + const require_ = createRequire(import.meta.url) + const manifestPath = require_.resolve('vitest/package.json') + const { bin } = require_(manifestPath) + const entry = typeof bin === 'string' ? bin : bin?.vitest + if (typeof entry !== 'string') { + throw new Error('vitest package.json declares no bin entry; cannot run the owning control') + } + return join(dirname(manifestPath), entry) +} + +const VITEST_BIN = resolveVitestBin() + +const FALLBACK = 'src/runtime/retrieve/conceptual-fallback.ts' +const RETRIEVE = 'src/runtime/retrieve.ts' + +/** + * Apply an injection whose anchor is written with LF, to a file that may be + * checked out with CRLF. + * + * Only `docs/qualification/**` and `*.madar` are pinned to LF in .gitattributes, + * so `src/**` arrives with CRLF on Windows. A multi-line LF anchor then matches + * nothing, `replaceOnce` throws "injection target not found", and every control + * built on it fails for a reason that has nothing to do with the property under + * test. The anchor and its replacement are converted to the file's own line + * ending before matching, so the control tests behaviour on every platform. + */ +function replaceAnchor(snapshot, root, file, anchor, replacement) { + const text = readFileSync(resolve(root, file), 'utf8') + const usesCrlf = /\r\n/.test(text) + const toFileEol = (value) => (usesCrlf ? value.replaceAll('\r\n', '\n').replaceAll('\n', '\r\n') : value) + snapshot.replaceOnce(file, toFileEol(anchor), toFileEol(replacement)) +} + +const OWNER_D = 'D. ranks a one-for-one substituted repository' +const OWNER_D2 = 'D2. selects the same nodes end-to-end' +const OWNING_TEST_FILE = 'tests/unit/production-independence.test.ts' + +/** + * The anchor every injection attaches to: the end of the per-obligation + * reservation loop. If this text ever moves, the injection fails loudly rather + * than silently doing nothing. + */ +const RESERVATION_ANCHOR = ` for (const nodeId of preferredObligationAnchorIds) { + boosts.set(nodeId, Math.max(boosts.get(nodeId) ?? 0, CONCEPTUAL_WORKFLOW_RESERVATION_BOOST)) + } +` + +/** + * The retrieval-path anchor. Mutations attached here run in `retrieveContext` + * itself, so they are owned by the END-TO-END control rather than by the + * fallback planner's boost map. A control that only ever watched boosts would + * miss a forced selection applied after ranking. + */ +const RETRIEVE_ANCHOR = ` let orderedCandidates = inclusionOrder + let sliceMetadata: ContextPackSliceMetadata | undefined +` + +/** + * Where final membership is actually fixed. `orderedCandidates` is upstream of + * the pack's own budgeted selection, so an injection there can be discarded and + * prove nothing -- which is exactly what the membership premise caught. + */ +const MEMBERSHIP_ANCHOR = ` const matchedNodes = pack.nodes as RetrieveMatchedNode[] +` + +function runOwningTest(root, testNameFilter, extraEnv = {}) { + // `npx` is `npx.cmd` on Windows and spawnSync cannot execute it directly: + // without a shell the call returns status null and every control below would + // read as "the owning test failed", which is a false pass. Run the vitest + // entry with the current node binary instead, which is portable and also + // avoids resolving a shim on every platform. + const result = spawnSync( + process.execPath, + [VITEST_BIN, 'run', OWNING_TEST_FILE, '-t', testNameFilter, '--reporter=dot'], + { cwd: root, encoding: 'utf8', env: { ...process.env, CI: '1', ...extraEnv } }, + ) + if (result.error) { + return { failed: true, status: null, output: `spawn failed: ${result.error.message}`, spawnFailed: true } + } + if (result.status === null) { + return { failed: true, status: null, output: `vitest did not run to completion (signal ${result.signal})`, spawnFailed: true } + } + return { + failed: result.status !== 0, + status: result.status, + output: `${result.stdout ?? ''}${result.stderr ?? ''}`, + } +} + +/** + * The membership D2 itself measured, read back through a file seam. + * + * The premise these controls need is not "the file changed" but "the selected + * SET changed". A reorder satisfies the first and not the second, and a control + * that accepted the first would pass while proving nothing. Reusing the real + * control rather than a generated spec means the observation cannot drift from + * what the control asserts. + */ +function measureD2Membership(root) { + const out = join(tmpdir(), `madar-d2-membership-${process.pid}-${Math.random().toString(36).slice(2)}.json`) + try { + runOwningTest(root, OWNER_D2, { MADAR_D2_MEMBERSHIP_OUT: out }) + if (!existsSync(out)) { + return { ok: false, membership: [], detail: 'D2 did not emit a membership measurement' } + } + const parsed = JSON.parse(readFileSync(out, 'utf8')) + return { ok: true, membership: parsed.qualification?.membership ?? [], detail: '' } + } catch (error) { + return { ok: false, membership: [], detail: `membership read failed: ${error?.message ?? String(error)}` } + } finally { + if (existsSync(out)) { + rmSync(out, { force: true }) + } + } +} + +const run0Detail = (run) => run.output.slice(0, 300).replace(/\s+/g, ' ') + +function cases() { + return [ + { + id: 'H1', + title: 'a restored public+page task-phrase forced selection is caught by control D', + file: FALLBACK, + testFilter: OWNER_D, + ownerPattern: /ranks a one-for-one substituted repository/, + inject(snapshot, root) { + // A task-phrase score adjustment: exactly the shape §5 names. It fires + // only when the prompt carries both words, so it moves the original + // fixture and leaves the substituted one alone. + replaceAnchor(snapshot, root, FALLBACK, RESERVATION_ANCHOR, `${RESERVATION_ANCHOR} + // #660-B H1 injection -- an obligation-term score adjustment. + if (obligations.some((entry) => entry.terms.includes('public') && entry.terms.includes('page'))) { + for (const anchor of anchors) { + if (anchor.sourceFile.includes('/router/')) { + boosts.set(anchor.id, (boosts.get(anchor.id) ?? 0) + 25) + } + } + } +`) + }, + }, + { + id: 'H2', + title: 'a restored repository-path ranking boost is caught by control D', + file: FALLBACK, + testFilter: OWNER_D, + ownerPattern: /ranks a one-for-one substituted repository/, + inject(snapshot, root) { + // A repository-path boost with no prompt vocabulary at all, so a + // prompt-shaped control would not see it either. + replaceAnchor(snapshot, root, FALLBACK, RESERVATION_ANCHOR, `${RESERVATION_ANCHOR} + // #660-B H2 injection -- a repository-path score adjustment. + for (const anchor of anchors) { + if (anchor.sourceFile.includes('/checker/')) { + boosts.set(anchor.id, (boosts.get(anchor.id) ?? 0) + 25) + } + } +`) + }, + }, + { + id: 'H4', + title: 'a forced final-membership change on the retrieval path is caught by D2', + file: RETRIEVE, + testFilter: OWNER_D2, + ownerPattern: /selects the same nodes end-to-end/, + // A reorder of already-selected candidates is NOT a sufficient premise: + // D2 asserts membership, so shuffling the same set proves nothing. This + // injection promotes a repository-path candidate the clean tree does not + // select at all, which is a genuine membership change. + membershipNode: 'h4-forced-membership', + inject(snapshot, root) { + replaceAnchor(snapshot, root, RETRIEVE, MEMBERSHIP_ANCHOR, `${MEMBERSHIP_ANCHOR} + // #660-B1 H4 injection -- a payload entry pinned in because a repository + // path is present, which is the shape the removed claim-pinning path had. + // The pinned entry is one the clean tree never selects, so its appearance is + // a genuine final-membership change rather than a reordering. + if (orderedCandidates.some((entry) => entry.sourceFile.includes('/checker/'))) { + matchedNodes.push({ + node_id: 'h4-forced-membership', + label: 'h4ForcedMembership()', + source_file: '/apps/checker/forced.go', + line_number: 1, + snippet: null, + relevance_band: 'direct', + } as RetrieveMatchedNode) + } +`) + }, + }, + { + id: 'H5', + title: 'a question-shaped bypass inside the requested slice is caught by D2', + file: RETRIEVE, + testFilter: OWNER_D2, + ownerPattern: /selects the same nodes end-to-end/, + // Removes a candidate the clean tree DOES select, so membership shrinks + // on one side only. Gated on the requested slice, so it exercises the + // path D2 now requests rather than a strategy nothing asked for. + membershipNode: 'n2', + inject(snapshot, root) { + replaceAnchor(snapshot, root, RETRIEVE, MEMBERSHIP_ANCHOR, `${MEMBERSHIP_ANCHOR} + // #660-B1 H5 injection -- a question-shaped bypass applied where membership + // is final, gated on a slice actually having been requested. + if (options.retrievalStrategy === 'slice-v1' && /\\bstatus\\b/i.test(options.question ?? '')) { + for (let h5i = matchedNodes.length - 1; h5i >= 0; h5i -= 1) { + if ((matchedNodes[h5i]?.source_file ?? '').includes('/checker/')) { + matchedNodes.splice(h5i, 1) + } + } + } +`) + }, + }, + ] +} + +/** + * True when the injection actually moved the SELECTED SET, not just its order. + * `membershipNode` names the node whose selection state must flip. + */ +function membershipPremiseHolds(root, testCase, reportFailure) { + const probe = measureD2Membership(root) + if (!probe.ok) { + reportFailure(`membership probe failed, so the premise is unproven: ${probe.detail}`) + return false + } + const node = testCase.membershipNode + const present = probe.membership.includes(node) + // H4 adds a node the clean tree lacks; H5 removes one it has. + const flipped = testCase.id === 'H4' ? present : !present + if (!flipped) { + reportFailure( + `injection did not change final membership (${node} ${present ? 'present' : 'absent'}, ` + + `set ${JSON.stringify(probe.membership)}); a reorder is not a sufficient premise`, + ) + return false + } + return true +} + +export function runSemanticIndependenceSelfTest({ root = process.cwd(), log = console.log } = {}) { + const results = [] + + // The owning test must pass on the untouched tree, or "it failed after the + // injection" proves nothing at all. + // Vitest -t is a substring match, so 'D. ' does NOT select 'D2. '. The two + // controls have to be run as two filters or the D2 baseline is never + // established -- which is exactly what a previous version of this harness + // claimed and did not do. + const runBothOwners = () => ({ + d: runOwningTest(root, OWNER_D), + d2: runOwningTest(root, OWNER_D2), + }) + const baseline = runBothOwners() + const baselineOk = !baseline.d.failed && !baseline.d2.failed + results.push({ + id: 'H0', + title: 'controls D and D2 each pass on the untouched tree, run as separate filters', + passed: baselineOk, + detail: baselineOk + ? 'D green and D2 green before any injection' + : `baseline already failing -- D exit ${baseline.d.status}, D2 exit ${baseline.d2.status}; nothing below proves anything: ${ + run0Detail(baseline.d.failed ? baseline.d : baseline.d2)}`, + }) + + if (!baselineOk) { + for (const entry of results) { + log(` ${entry.passed ? 'PASS' : 'FAIL'} ${entry.id} ${entry.title}`) + log(` ${entry.detail}`) + } + return { ok: false, results } + } + + for (const testCase of cases()) { + const snapshot = new ByteSnapshot(root) + let passed = false + let detail = '' + try { + testCase.inject(snapshot, root) + + // PREMISE 1: the injected rule is really on disk. + const injected = readFileSync(resolve(root, testCase.file), 'utf8') + const marker = `${testCase.id} injection` + if (!injected.includes(marker)) { + detail = 'injection did not reach the file; the control proves nothing' + } else if (testCase.membershipNode !== undefined + && !membershipPremiseHolds(root, testCase, (message) => { detail = message })) { + // PREMISE 2 handled inside membershipPremiseHolds, which writes detail. + } else { + const run = runOwningTest(root, testCase.testFilter) + // A crash, a type error or a missing test would also be a non-zero + // exit. Only the control's OWN assertion counts as ownership. + const failedOnItsAssertion = run.failed + && !run.spawnFailed + && /AssertionError/.test(run.output) + && testCase.ownerPattern.test(run.output) + passed = failedOnItsAssertion + detail = passed + ? `owning control failed on its own assertion (exit ${run.status})` + : run.failed + ? `control failed for an UNRELATED reason, which proves nothing: ${run.output.slice(0, 400).replace(/\s+/g, ' ')}` + : 'owning control still PASSED with the contamination present; it does not own this behaviour' + } + } catch (error) { + detail = `control threw: ${error?.message ?? String(error)}` + } finally { + const unrestored = snapshot.restore() + if (unrestored.length > 0) { + passed = false + detail += ` | RESTORE FAILED: ${unrestored.join(', ')}` + } + } + results.push({ id: testCase.id, title: testCase.title, passed, detail }) + } + + // The tree must be exactly as green afterwards as before. + const after = runBothOwners() + const afterOk = !after.d.failed && !after.d2.failed + results.push({ + id: 'H3', + title: 'the tree is restored and controls D and D2 are each green again', + passed: afterOk, + detail: afterOk + ? 'D green and D2 green after restore' + : `still failing after restore -- D exit ${after.d.status}, D2 exit ${after.d2.status}`, + }) + + for (const entry of results) { + log(` ${entry.passed ? 'PASS' : 'FAIL'} ${entry.id} ${entry.title}`) + log(` ${entry.detail}`) + } + + return { ok: results.every((entry) => entry.passed), results } +} diff --git a/scripts/verify-forbidden-knowledge.mjs b/scripts/verify-forbidden-knowledge.mjs new file mode 100644 index 00000000..9c2102e6 --- /dev/null +++ b/scripts/verify-forbidden-knowledge.mjs @@ -0,0 +1,50 @@ +#!/usr/bin/env node +/** + * #660-B -- CLI wrapper for the production independence scan. + * + * npm run verify:forbidden-knowledge + * npm run verify:forbidden-knowledge-controls (adds the falsifiability run) + * + * The same engine runs as a vitest control (tests/unit/forbidden-knowledge.test.ts) + * so the boundary is enforced on every protected CI lane, not only on the lane + * that happens to run extra scripts. + */ +import { analyzeForbiddenKnowledge, formatForbiddenKnowledgeReport } from './lib/forbidden-knowledge.mjs' +import { runForbiddenKnowledgeSelfTest } from './lib/forbidden-knowledge-selftest.mjs' +import { runSemanticIndependenceSelfTest } from './lib/semantic-independence-selftest.mjs' + +const selfTest = process.argv.includes('--self-test') + +const result = analyzeForbiddenKnowledge() +const report = formatForbiddenKnowledgeReport(result) + +if (!result.ok) { + console.error(report) + process.exit(1) +} + +console.log(report) + +if (!selfTest) { + process.exit(0) +} + +// Falsifiability: a guard that has never been shown to fail proves nothing. +console.log('\nProduction independence falsifiability controls:') +const self = runForbiddenKnowledgeSelfTest() +if (!self.ok) { + console.error('\nLiteral-scan controls FAILED. The scan is not trustworthy until these pass.') + process.exit(1) +} + +// The scanner cannot see task-phrase or forced-selection contamination, so the +// behavioural tests that DO own it have to be shown to fail when it returns. +console.log('\nSemantic independence falsifiability controls:') +const semantic = runSemanticIndependenceSelfTest() +if (!semantic.ok) { + console.error('\nSemantic controls FAILED. The behavioural tests do not own the contamination they claim to.') + process.exit(1) +} + +console.log('\nAll production independence controls passed.') +process.exit(0) diff --git a/src/runtime/context-pack.ts b/src/runtime/context-pack.ts index 4d1dba72..8993f5bd 100644 --- a/src/runtime/context-pack.ts +++ b/src/runtime/context-pack.ts @@ -423,101 +423,113 @@ function buildInputProvenanceClaims(nodes: readonly ContextPackNode[]): ContextP return claims } -function buildStatusProjectionSplitClaims(nodes: readonly ContextPackNode[]): ContextPackClaim[] { - const projection = nodes.find((node) => ( - node.snippet?.includes('pageIndicator(page.status)') === true - && node.snippet.includes('page.statusReports') - )) - const incidentRollup = nodes.find((node) => ( - /e\.type\s*===\s*["']incident["']/.test(node.snippet ?? '') - && /barType\s*!==\s*["']manual["']/.test(node.snippet ?? '') - )) - if (!projection || !incidentRollup) { - return [] - } - - return [{ - evidence_class: projection.evidence_class ?? 'supporting', - text: `public payload divergence: when barType is not manual, an open incident event can make page.status "error" in ${incidentRollup.source_file}; ${projection.source_file} builds unresolved incident entries only from page.statusReports, so an auto-created incident without a status report can yield an error indicator with an empty incidents list`, - node_labels: [projection.label, incidentRollup.label], - }] +/** + * Typed execution relationships worth stating in prose, with the noun used to + * introduce the claim. Every key is a relation an extractor emits from + * framework structure -- a route registration, a queue handoff, a render edge + * -- never a name or a phrase read out of a snippet. + */ +const EXECUTION_RELATION_CLAIMS = new Map([ + ['enqueues_job', 'queue handoff'], + ['handles_route', 'route handling'], + ['registers_route', 'route registration'], + ['controller_route', 'route handling'], + ['route_handler', 'route handling'], + ['declares_controller', 'controller declaration'], + ['loads_route', 'route loading'], + ['submits_route', 'route submission'], + ['renders', 'render path'], +]) + +function nodeByLabelOrId( + nodes: readonly ContextPackNode[], + label: string, + nodeId: string | undefined, +): ContextPackNode | undefined { + if (nodeId !== undefined) { + const byId = nodes.find((node) => node.node_id === nodeId) + if (byId) { + return byId + } + } + return nodes.find((node) => node.label === label) } -function buildPublicStatusRuntimeProvenanceClaims(nodes: readonly ContextPackNode[]): ContextPackClaim[] { - const boundary = nodes.find((node) => ( - /\btrpc\s*\.\s*statusPage\s*\.\s*get\s*\.\s*queryOptions\b/i.test(node.snippet ?? '') - && /\bto(?:Status|Summary|UnresolvedIncidents)\s*\(\s*data\b/.test(node.snippet ?? '') - )) - const publicRouter = nodes.find((node) => ( - /(?:^|\/)packages\/api\/src\/router\/statusPage\.ts$/i.test(node.source_file.replaceAll('\\', '/')) - && /e\.type\s*===\s*["']incident["']/.test(node.snippet ?? '') - )) - const incidentAwarePublicRouter = publicRouter && ( - /e\.type\s*===\s*["']incident["']/.test(publicRouter.snippet ?? '') - && /barType\s*!==\s*["']manual["']/.test(publicRouter.snippet ?? '') - ) - // Prefer a source-range excerpt that proves a distinct status decision over - // a symbol-name-only fallback. This intentionally relies on the selected - // evidence, not a repository-specific path or an absence-of-query claim. - const semanticAlternate = nodes.find((node) => ( - node !== publicRouter - && /overall\s*status/i.test(node.snippet ?? '') - && /status\s*report/i.test(node.snippet ?? '') - && /maintenance/i.test(node.snippet ?? '') - )) - const alternate = semanticAlternate ?? nodes.find((node) => ( - node !== publicRouter && /(?:compute|derive|resolve).*status/i.test(node.label) - )) - if (!boundary || !publicRouter) { - return [] - } - - const alternateClaim = semanticAlternate && incidentAwarePublicRouter - ? `; ${publicRouter.source_file} treats an open incident event as "error" outside manual mode, while ${semanticAlternate.source_file} ${semanticAlternate.label} derives overall status from active status reports and maintenance` - : alternate - ? `; ${alternate.source_file} ${alternate.label} is a separate computation path` - : '' - - return [{ - evidence_class: boundary.evidence_class ?? 'supporting', - text: `public runtime provenance: ${boundary.source_file} ${boundary.label} fetches trpc.statusPage.get and passes that data to the public status-json serializers backed by ${publicRouter.source_file}${alternateClaim}`, - node_labels: [boundary.label, publicRouter.label, ...(alternate ? [alternate.label] : [])], - }] -} +/** + * Claims earned from structure the extractors actually recorded. + * + * A relationship claim is emitted only when a typed edge connects two nodes + * that are both in the pack, so the claim cites two real source locations and + * names the relation that produced it. Two files mentioning the same word are + * not a relationship, and no amount of matching text will make one here: this + * builder never reads a snippet. + * + * A framework-role claim is emitted only when an extractor declared the role + * from framework structure, which is why it survives a repository renaming + * every identifier. + */ +function buildStructuralEvidenceClaims( + nodes: readonly ContextPackNode[], + relationships: readonly ContextPackRelationship[], +): ContextPackClaim[] { + const claims: ContextPackClaim[] = [] + const seen = new Set() + + for (const relationship of relationships) { + const kind = EXECUTION_RELATION_CLAIMS.get(relationship.relation) + if (kind === undefined) { + continue + } + + const from = nodeByLabelOrId(nodes, relationship.from, relationship.from_id) + const to = nodeByLabelOrId(nodes, relationship.to, relationship.to_id) + if (!from || !to || from === to) { + continue + } + + const dedupeKey = `${kind}\u0000${from.label}\u0000${to.label}` + if (seen.has(dedupeKey)) { + continue + } + seen.add(dedupeKey) -function buildFailureHandoffClaims(nodes: readonly ContextPackNode[]): ContextPackClaim[] { - const detector = nodes.find((node) => ( - /HTTPCheckerHandler/.test(node.label) - && /\bUpdateStatus\s*\(/.test(node.snippet ?? '') - && /\bStatus\s*:\s*["']error["']/.test(node.snippet ?? '') - )) - const handoff = nodes.find((node) => ( - node.label === 'UpdateStatus()' - && /\bcloudtasks\.NewClient\s*\(/.test(node.snippet ?? '') - && /\.CreateTask\s*\(/.test(node.snippet ?? '') - )) - if (!detector || !handoff) { - return [] + claims.push({ + evidence_class: from.evidence_class ?? 'supporting', + text: `${kind}: ${from.source_file} ${from.label} ${relationship.relation} ${to.source_file} ${to.label}`, + node_labels: [from.label, to.label], + }) } - return [ - { - evidence_class: detector.evidence_class ?? 'supporting', - text: `failure detection: ${detector.source_file} ${detector.label} sends Status "error" to UpdateStatus`, - node_labels: [detector.label], - }, - { - evidence_class: handoff.evidence_class ?? 'supporting', - text: `cross-runtime handoff: ${handoff.source_file} ${handoff.label} enqueues the checker status update with Cloud Tasks`, - node_labels: [handoff.label], - }, - ] + for (const node of nodes) { + const role = node.framework_role + if (role === undefined || role.trim().length === 0) { + continue + } + if (!/(?:route|controller|handler|middleware|job|queue|worker)/i.test(role)) { + continue + } + + const dedupeKey = `role\u0000${node.label}\u0000${role}` + if (seen.has(dedupeKey)) { + continue + } + seen.add(dedupeKey) + + claims.push({ + evidence_class: node.evidence_class ?? 'supporting', + text: `runtime boundary: ${node.source_file} ${node.label} is a framework-declared ${role}`, + node_labels: [node.label], + }) + } + + return claims } function buildClaims( taskContract: ContextPackTaskContract, labelsByEvidence: ReadonlyMap, nodes: readonly ContextPackNode[], + relationships: readonly ContextPackRelationship[], ): ContextPackClaim[] { const evidenceOrder = orderedEvidence(taskContract, labelsByEvidence.keys()) @@ -535,9 +547,7 @@ function buildClaims( }) return [ - ...buildPublicStatusRuntimeProvenanceClaims(nodes), - ...buildStatusProjectionSplitClaims(nodes), - ...buildFailureHandoffClaims(nodes), + ...buildStructuralEvidenceClaims(nodes, relationships), ...buildInputProvenanceClaims(nodes), ...evidenceClaims, ] @@ -895,7 +905,7 @@ function looksScriptMigration(sourceFile: string, label: string): boolean { } function promptAllowsScriptMigration(prompt: string | undefined): boolean { - return /\b(?:scripts?|migrat(?:e|ed|es|ing|ion)|backfill|cli|one-off|repair|old pipeline|seed(?:ing|ers?)|seeds?\s+(?:data|db|database|scripts?|files?))\b/i.test(prompt ?? '') + return /\b(?:scripts?|migrat(?:e|ed|es|ing|ion)|backfill|cli|one-off|repair|seed(?:ing|ers?)|seeds?\s+(?:data|db|database|scripts?|files?))\b/i.test(prompt ?? '') } function sourceDomainPenalty(view: CandidateScoringView, taskContract: ContextPackTaskContract): number { @@ -1402,7 +1412,7 @@ export function compileContextPack< nodes: renderedNodes.nodes, relationships, community_context: (input.community_context ?? []).filter((community) => selectedCommunities.has(community.id)), - claims: buildClaims(input.task_contract, selectedLabelsByEvidence, renderedNodes.nodes), + claims: buildClaims(input.task_contract, selectedLabelsByEvidence, renderedNodes.nodes, relationships), expandable: buildExpandableRefs(input.task_contract, omittedNodes), coverage: coverageEntriesForCandidates( input.task_contract, diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 78b12109..820b0414 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -691,20 +691,13 @@ const QUERY_EVIDENCE_OPERATION_PATTERN = /(?:\b(?:await|case|catch|delete|dispat const QUERY_EVIDENCE_STATE_MUTATION_PATTERN = /(?:\b(?:create|insert|transition|upsert)\w*\s*\(|\.(?:create|insert|upsert)\s*\(|\bnew\s+\w+)/i const QUERY_EVIDENCE_LOW_VALUE_LINE_PATTERN = /^\s*(?:(?:import|package)\b|(?:export\s+)?type\b|interface\b|\/\/|\/\*|\*|[{}()[\],;]+\s*$)/i const QUERY_EVIDENCE_LOG_LINE_PATTERN = /\b(?:logger|log)\.\w+\s*\(/i -const QUERY_EVIDENCE_HANDOFF_PATTERN = /(?:\b(?:createTask|dispatch|emit|enqueue|insert|publish|send|update|upsert)\w*\s*\(|\.(?:createTask|dispatch|emit|enqueue|insert|publish|send\w*|update|upsert)\s*\()/i -const QUERY_EVIDENCE_DELIVERY_HANDOFF_PATTERN = /(?:\b(?:createTask|dispatch|emit|enqueue|publish|send|trigger)\w*\s*\(|\.(?:createTask|dispatch|emit|enqueue|publish|send\w*)\s*\()/i +const QUERY_EVIDENCE_HANDOFF_PATTERN = /(?:\b(?:dispatch|emit|enqueue|insert|publish|send|update|upsert)\w*\s*\(|\.(?:dispatch|emit|enqueue|insert|publish|send\w*|update|upsert)\s*\()/i +const QUERY_EVIDENCE_DELIVERY_HANDOFF_PATTERN = /(?:\b(?:dispatch|emit|enqueue|publish|send|trigger)\w*\s*\(|\.(?:dispatch|emit|enqueue|publish|send\w*)\s*\()/i const QUERY_EVIDENCE_DELIVERY_OPERATION_PATTERN = /(?:\b(?:deliver|dispatch|emit|enqueue|publish|send)\w*\s*\(|\.(?:deliver|dispatch|emit|enqueue|publish|send)\w*\s*\()/i const QUERY_EVIDENCE_RETRY_PATTERN = /\b(?:backoff|exponential|retr(?:y|ied|ies))\b/i -const QUERY_EVIDENCE_OVERALL_RESULT_PATTERN = /\boverall\w*(?:result|state|status)\w*\s*=/i const QUERY_EVIDENCE_DECISION_PATTERN = /(?:\b\w*(?:result|state|status)\w*\s*=.*(?:\?|\.some\s*\()|\?\s*[\w.]+\s*:)/i const QUERY_EVIDENCE_DECISION_ASSIGNMENT_PATTERN = /\b\w*(?:result|state|status)\w*\s*=/i -const QUERY_EVIDENCE_PAGE_RESULT_PATTERN = /(?:\bpage\w*(?:indicator|status)\w*\s*\(|\bstatus\s*:\s*page\w*\s*\()/i -const QUERY_EVIDENCE_PAGE_COLLECTION_PATTERN = /\breturn\s+page\.\w+/i const QUERY_EVIDENCE_COMPUTATION_PATTERN = /(?:\b(?:compute|derive|resolve)\w*\s*\(|\b\w*(?:indicator|result|state|status)\w*\s*=|\b\w*(?:indicator|status)\w*\s*\()/i -const QUERY_EVIDENCE_INCIDENT_STATUS_PATTERN = /(?:\b\w+\.type\s*===?\s*["']incident["'][^\n]*!\w+\.to|!\w+\.to[^\n]*\b\w+\.type\s*===?\s*["']incident["'])/i -const QUERY_EVIDENCE_MONITOR_ROLLUP_PATTERN = /\bstatus\s*=\s*monitors\.some\s*\(/i -const QUERY_EVIDENCE_INPUT_PROVENANCE_PATTERN = /\bRouterOutputs\b.*\[\s*["']statusPage["']\s*\].*\[\s*["']get["']\s*\]/i -const QUERY_EVIDENCE_PUBLIC_ROUTER_FETCH_PATTERN = /\btrpc\s*\.\s*statusPage\s*\.\s*get\s*\.\s*queryOptions\b/i const QUERY_EVIDENCE_DECLARATION_PATTERN = /^\s*(?:(?:export\s+)?(?:async\s+)?(?:function\b|const\s+\w+\s*=\s*async\b)|func\b)/i function boundedSourceRange( @@ -725,10 +718,9 @@ interface QueryEvidenceFragment { const QUERY_EVIDENCE_CONTINUATION_START_PATTERN = /^\s*(?:[.?:]|&&|\|\|)/ const QUERY_EVIDENCE_CONTINUATION_END_PATTERN = /(?:=>|&&|\|\||[=?:])\s*$/ const QUERY_EVIDENCE_STRUCTURED_CALL_PATTERN = /(?:\b\w+\s*\(|:=\s*&?[\w.]+)[^;]*\{\s*$/ -const QUERY_EVIDENCE_DISCRIMINANT_PROPERTY_PATTERN = /^\s*([\w]*(?:action|event|incident|method|mode|monitor|notification|queue|route|status|type|url)[\w]*)\s*:/i -const QUERY_EVIDENCE_PROVIDER_HANDOFF_PATTERN = /\b([A-Za-z_]\w*)\.(?:createTask|deliver\w*|dispatch\w*|emit\w*|enqueue\w*|publish\w*|send\w*|trigger\w*)\s*\(/i -const QUERY_EVIDENCE_PROVIDER_SETUP_PATTERN = /(?:\bnew\s+\w*(?:client|provider|queue|transport)\w*\s*\(|\b\w*newClient\s*\(|\b\w*(?:client|provider|queue|transport)\w*\.)/i -const QUERY_EVIDENCE_CONTAINER_DECLARATION_PATTERN = /^\s*(?:export\s+const\s+\w*(?:route|router)\w*\s*=|\w+\s*:\s*\w*Procedure\.query\s*\()/i +const QUERY_EVIDENCE_DISCRIMINANT_PROPERTY_PATTERN = /^\s*([\w]*(?:action|event|method|mode|queue|route|status|type|url)[\w]*)\s*:/i +const QUERY_EVIDENCE_PROVIDER_HANDOFF_PATTERN = /\b([A-Za-z_]\w*)\.(?:deliver\w*|dispatch\w*|emit\w*|enqueue\w*|publish\w*|send\w*|trigger\w*)\s*\(/i +const QUERY_EVIDENCE_PROVIDER_SETUP_PATTERN = /(?:\bnew\s*\w*(?:client|provider|queue|transport)\w*\s*\(|\b\w*(?:client|provider|queue|transport)\w*\.)/i function braceDelta(value: string): number { return (value.match(/\{/g)?.length ?? 0) - (value.match(/\}/g)?.length ?? 0) @@ -778,7 +770,7 @@ function structuredCallFragment( } const priority = (key: string): number => { if (/(?:action|event|mode|status|type)/.test(key)) return 0 - if (/(?:incident|method|monitor|notification|queue|route|url)/.test(key)) return 1 + if (/(?:method|queue|route|url)/.test(key)) return 1 return 2 } const discriminants = properties @@ -820,94 +812,6 @@ function providerHandoffFragment( return null } -function publicRouterFetchFragment( - lines: readonly string[], - lineNumber: number, - rangeStart: number, -): { start: number; end: number; text: string } | null { - const queryOptions = lines[lineNumber - 1] ?? '' - if (!QUERY_EVIDENCE_PUBLIC_ROUTER_FETCH_PATTERN.test(queryOptions)) { - return null - } - - for (let candidate = lineNumber - 1; candidate >= Math.max(rangeStart, lineNumber - 3); candidate -= 1) { - const fetch = lines[candidate - 1] ?? '' - if (!/\b(?:const|let)\s+\w+\s*=\s*await\s+\w+\.fetchQuery\s*\(/.test(fetch)) { - continue - } - return { - start: candidate, - end: lineNumber, - text: `${fetch.trim()} ${queryOptions.trim()}`, - } - } - - return null -} - -function incidentStatusOwnerFragment( - lines: readonly string[], - lineNumber: number, - rangeStart: number, - rangeEnd: number, -): { start: number; end: number; text: string } | null { - const first = lines[lineNumber - 1] ?? '' - if (!/^\s*const\s+status\s*=/.test(first)) { - return null - } - let decisionEnd = lineNumber - while (decisionEnd < rangeEnd && decisionEnd - lineNumber < 5) { - const current = lines[decisionEnd - 1] ?? '' - if (/;\s*$/.test(current)) { - break - } - decisionEnd += 1 - } - const decision = lines.slice(lineNumber - 1, decisionEnd).join(' ') - if (!QUERY_EVIDENCE_INCIDENT_STATUS_PATTERN.test(decision)) { - return null - } - - let ownerNumber: number | null = null - for (let candidate = lineNumber - 1; candidate >= Math.max(rangeStart, lineNumber - 16); candidate -= 1) { - if (/^\s*const\s+\w+\s*=.*\.map\s*\(/.test(lines[candidate - 1] ?? '')) { - ownerNumber = candidate - break - } - } - if (ownerNumber === null) { - return null - } - - const returnEvidence: string[] = [] - let returnEnd = decisionEnd - for (let candidate = decisionEnd + 1; candidate <= Math.min(rangeEnd, decisionEnd + 20); candidate += 1) { - const value = lines[candidate - 1] ?? '' - if (returnEvidence.length === 0 && /^\s*return\s*\{/.test(value)) { - returnEvidence.push(value.trim()) - returnEnd = candidate - continue - } - if (returnEvidence.length > 0 && /^\s*(?:\.\.\.\w+(?:\.\w+)?,|status,|events,)/.test(value)) { - returnEvidence.push(value.trim()) - returnEnd = candidate - if (returnEvidence.some((entry) => /^status,$/.test(entry))) { - break - } - } - } - - return { - start: ownerNumber, - end: Math.max(decisionEnd, returnEnd), - text: [ - (lines[ownerNumber - 1] ?? '').trim(), - ...returnEvidence, - `L${lineNumber}: ${decision.trim()}`, - ].join(' '), - } -} - function queryEvidenceScoringText(value: string): string { return value .replace(/\/\*[\s\S]*?\*\//g, ' ') @@ -935,20 +839,10 @@ function queryEvidenceFragments( ) { start -= 1 } - const incidentStatusOwner = incidentStatusOwnerFragment(lines, lineNumber, range.start, range.end) - const publicRouterFetch = incidentStatusOwner ? null : publicRouterFetchFragment(lines, lineNumber, range.start) - const providerHandoff = incidentStatusOwner || publicRouterFetch ? null : providerHandoffFragment(lines, lineNumber, range.start) - const structured = incidentStatusOwner || publicRouterFetch || providerHandoff ? null : structuredCallFragment(lines, lineNumber, range.end) + const providerHandoff = providerHandoffFragment(lines, lineNumber, range.start) + const structured = providerHandoff ? null : structuredCallFragment(lines, lineNumber, range.end) let text: string | null = null - if (incidentStatusOwner) { - start = incidentStatusOwner.start - end = incidentStatusOwner.end - text = incidentStatusOwner.text - } else if (publicRouterFetch) { - start = publicRouterFetch.start - end = publicRouterFetch.end - text = publicRouterFetch.text - } else if (providerHandoff) { + if (providerHandoff) { start = providerHandoff.start end = providerHandoff.end text = providerHandoff.text @@ -1091,18 +985,6 @@ function queryEvidenceRange( if (QUERY_EVIDENCE_COMPUTATION_PATTERN.test(scoringText)) { score += 1.2 } - if (QUERY_EVIDENCE_INCIDENT_STATUS_PATTERN.test(scoringText)) { - score += 2.4 - } - if (QUERY_EVIDENCE_MONITOR_ROLLUP_PATTERN.test(scoringText)) { - score += 2 - } - if (QUERY_EVIDENCE_INPUT_PROVENANCE_PATTERN.test(scoringText)) { - score += 2.5 - } - if (QUERY_EVIDENCE_PUBLIC_ROUTER_FETCH_PATTERN.test(scoringText)) { - score += 4 - } if (QUERY_EVIDENCE_DECLARATION_PATTERN.test(scoringText)) { score -= 1.25 } @@ -1132,10 +1014,7 @@ function queryEvidenceRange( } function selectQueryEvidenceLines(range: QueryEvidenceRange): QueryEvidenceLine[] { - const valuableLines = range.lines.filter((line) => ( - !QUERY_EVIDENCE_LOW_VALUE_LINE_PATTERN.test(line.text) - || QUERY_EVIDENCE_INPUT_PROVENANCE_PATTERN.test(line.text) - )) + const valuableLines = range.lines.filter((line) => !QUERY_EVIDENCE_LOW_VALUE_LINE_PATTERN.test(line.text)) const remaining = [...(valuableLines.length > 0 ? valuableLines : range.lines)] const selected: QueryEvidenceLine[] = [] const coveredTerms = new Set() @@ -1203,17 +1082,13 @@ function selectQueryEvidenceLines(range: QueryEvidenceRange): QueryEvidenceLine[ if (best) addSelected(best) } + // Reserve one line per generic evidence shape. Every entry is an ordinary + // programming construct -- a handoff call, a retry, a branch, a computation + // -- so this pass cannot prefer one repository's code over another's. for (const semanticPattern of [ QUERY_EVIDENCE_DELIVERY_HANDOFF_PATTERN, QUERY_EVIDENCE_RETRY_PATTERN, - QUERY_EVIDENCE_PUBLIC_ROUTER_FETCH_PATTERN, - QUERY_EVIDENCE_OVERALL_RESULT_PATTERN, QUERY_EVIDENCE_DECISION_PATTERN, - QUERY_EVIDENCE_PAGE_RESULT_PATTERN, - QUERY_EVIDENCE_PAGE_COLLECTION_PATTERN, - QUERY_EVIDENCE_INCIDENT_STATUS_PATTERN, - QUERY_EVIDENCE_MONITOR_ROLLUP_PATTERN, - QUERY_EVIDENCE_INPUT_PROVENANCE_PATTERN, QUERY_EVIDENCE_COMPUTATION_PATTERN, ]) { if (selected.length >= QUERY_EVIDENCE_SNIPPET_MAX_LINES) { @@ -1245,14 +1120,7 @@ function selectQueryEvidenceLines(range: QueryEvidenceRange): QueryEvidenceLine[ const addsSemanticEvidence = [ QUERY_EVIDENCE_DELIVERY_HANDOFF_PATTERN, QUERY_EVIDENCE_RETRY_PATTERN, - QUERY_EVIDENCE_PUBLIC_ROUTER_FETCH_PATTERN, - QUERY_EVIDENCE_OVERALL_RESULT_PATTERN, QUERY_EVIDENCE_DECISION_PATTERN, - QUERY_EVIDENCE_PAGE_RESULT_PATTERN, - QUERY_EVIDENCE_PAGE_COLLECTION_PATTERN, - QUERY_EVIDENCE_INCIDENT_STATUS_PATTERN, - QUERY_EVIDENCE_MONITOR_ROLLUP_PATTERN, - QUERY_EVIDENCE_INPUT_PROVENANCE_PATTERN, QUERY_EVIDENCE_COMPUTATION_PATTERN, ].some((pattern) => ( pattern.test(queryEvidenceScoringText(line.text)) @@ -1360,13 +1228,7 @@ export function readQueryEvidenceSnippet( } } - let selectedLines = selectQueryEvidenceLines(selectedRange) - if ( - options.fileNodeLike - && selectedLines.some((line) => QUERY_EVIDENCE_INCIDENT_STATUS_PATTERN.test(queryEvidenceScoringText(line.text))) - ) { - selectedLines = selectedLines.filter((line) => !QUERY_EVIDENCE_CONTAINER_DECLARATION_PATTERN.test(line.text)) - } + const selectedLines = selectQueryEvidenceLines(selectedRange) const snippet = renderQueryEvidenceLines(selectedLines) if (!snippet || selectedLines.length === 0) { return null @@ -2011,14 +1873,12 @@ function excludedTermMatches(value: string, excludedTerms: readonly string[], ex .some((termToken) => valueTokens.has(termToken)) } -function promptAllowsSourceDomain(domain: SourceDomain, intent: string, prompt: string, questionTokens: readonly string[]): boolean { - const lowerPrompt = prompt.toLowerCase() +function promptAllowsSourceDomain(domain: SourceDomain, intent: string, questionTokens: readonly string[]): boolean { switch (domain) { case 'test': return intent === 'test' || includesAnyToken(questionTokens, ['test', 'tests', 'spec', 'coverage', 'e2e']) case 'benchmark': return includesAnyToken(questionTokens, ['bench', 'benchmark', 'benchmarks', 'perf', 'performance']) - || /\b(html reporter|reporter utilities?)\b/i.test(lowerPrompt) case 'fixture': return includesAnyToken(questionTokens, ['fixture', 'fixtures', 'mock', 'mocks']) case 'generated': @@ -2041,7 +1901,7 @@ function defaultSourceDomainPenalty( prompt: string, questionTokens: readonly string[], ): number { - if (promptAllowsSourceDomain(domain, intent, prompt, questionTokens)) { + if (promptAllowsSourceDomain(domain, intent, questionTokens)) { return 0 } @@ -2138,10 +1998,10 @@ function runtimeGenerationNodeValue( if (/\b(?:nest_route|nest_controller|nest_provider|controller|service|repository|worker|orchestrator)\b/.test(lower)) { value += 1.75 } - if (/\b(?:generate|generation|create|start|process|pipeline|queue|job|research|agent|scoring|score|report|repository|persist|save|builder)\b/.test(lower)) { + if (/\b(?:generate|generation|create|start|process|pipeline|queue|job|repository|persist|save)\b/.test(lower)) { value += 1.5 } - if (/(?:^|[.#])(?:generate|create|start|process|save|score|search|update|claim|cancel)[A-Za-z_$\w]*\(?\)?$/i.test(node.label)) { + if (/(?:^|[.#])(?:generate|create|start|process|save|search|update)[A-Za-z_$\w]*\(?\)?$/i.test(node.label)) { value += 1 } @@ -2165,9 +2025,6 @@ function frontendDisplayNodePenalty( if (/\b(?:display|render|shown?|visible|footer|header|label|date|timestamp|component)\b/.test(lower)) { penalty += 1.5 } - if (/^pick[A-Z]/.test(node.label)) { - penalty += 1.25 - } return penalty } @@ -2228,7 +2085,7 @@ function runtimeGenerationSourceDomainPenalty( } function promptAllowsScriptMigration(question: string): boolean { - return /\b(?:scripts?|migrat(?:e|ed|es|ing|ion)|backfill|cli|one-off|repair|old pipeline|seed(?:ing|ers?)|seeds?\s+(?:data|db|database|scripts?|files?))\b/i.test(question) + return /\b(?:scripts?|migrat(?:e|ed|es|ing|ion)|backfill|cli|one-off|repair|seed(?:ing|ers?)|seeds?\s+(?:data|db|database|scripts?|files?))\b/i.test(question) } function scriptMigrationPathPenalty( @@ -2463,20 +2320,12 @@ function promptWantsRuntimePipeline(question: string): boolean { return /\b(runtime|pipeline|service|orchestrator|job|agent|scoring|report(?: builder)?|persistence|repository|queue|worker)\b/i.test(question) } -function promptWantsReportGenerationCore(question: string): boolean { - return /\b(?:report(?:\s+generation)?|generated\s+report|validation\s+report|final\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate)\b/i.test(question) -} - function promptHasExplicitExecutionAnchor(question: string): boolean { return containsUrlLikeRoutePath(question) || /`[^`]+`/.test(question) || /\b[A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*\b/.test(question) } -function promptWantsDetailedReportGenerationPhases(question: string): boolean { - return promptWantsReportGenerationCore(question) && !promptHasExplicitExecutionAnchor(question) -} - function promptWantsAuthGuardPhase(question: string): boolean { return /\b(?:auth|guard|guards|authorize|authorized|authorization|permissions?|roles?|plan enforcement)\b/i.test(question) } @@ -2502,8 +2351,7 @@ function promptExplicitlyWantsRuntimeHandoff(question: string): boolean { } function promptUsesExpandedExecutionTaxonomy(question: string): boolean { - return promptWantsDetailedReportGenerationPhases(question) - || promptWantsAuthGuardPhase(question) + return promptWantsAuthGuardPhase(question) || promptWantsValidationPhase(question) || promptWantsNotificationOrEventPhase(question) } @@ -2536,7 +2384,7 @@ function supportingPolicyOrLoggerNode( node: Pick, ): boolean { const lower = `${node.label} ${node.framework_role ?? ''} ${node.source_file}`.toLowerCase() - return /planenforcement|guard|interceptor|swagger|apioperation|apiresponse|apitags|logger|\.info\(\)|\.error\(\)|\.warn\(\)|\.debug\(\)/.test(lower) + return /guard|interceptor|swagger|apioperation|apiresponse|apitags|logger|\.info\(\)|\.error\(\)|\.warn\(\)|\.debug\(\)/.test(lower) } function pipelineBridgeNode( @@ -2549,7 +2397,15 @@ function runtimeFlowRelation(relation: string): boolean { return relation === 'calls' || relation === 'enqueues_job' } -function lowValueReportGenerationCompactNode( +/** + * Whether a node is a low-value step for compaction. + * + * This asks the generic execution-step test and nothing else. It used to carry + * a list of symbol names observed in particular repositories, which demoted + * those exact symbols wherever they appeared and demoted nothing equivalent in + * a repository that had named things differently. + */ +function lowValueCompactNode( node: { label: string source_file: string @@ -2557,74 +2413,12 @@ function lowValueReportGenerationCompactNode( framework_role?: string | undefined }, ): boolean { - const lower = `${node.label} ${node.source_file} ${node.node_kind ?? ''} ${node.framework_role ?? ''}`.toLowerCase() return lowValueExecutionStep({ label: node.label, source_file: node.source_file, ...(node.node_kind !== undefined ? { node_kind: node.node_kind } : {}), ...(node.framework_role !== undefined ? { framework_role: node.framework_role } : {}), }) - || /\b(?:title|status|suggest|guard|auth|interceptor|refund|claim|cancel|signedurl|buildperspective|letsbuild|publish|delete)\b/.test(lower) - || /(?:^|[.#])(?:generatefallbacktitle|generatetitle|getstatusmessage|claimqueuedpipelinerun|releasequeuedpipelineclaim|releaseunusedcreditreservation|generatebuildperspective|generatesignedurl|generateletsbuild|publishidea|getidea|listideas|deleteidea|suggestimprovements)[A-Za-z_$\w]*\(?\)?$/i.test(node.label) -} - -function recoveredReportGenerationNoise( - question: string, - node: Pick, -): boolean { - if (!promptWantsRuntimePipeline(question) || !promptWantsReportGenerationCore(question)) { - return false - } - - return lowValueReportGenerationCompactNode({ - label: node.label, - source_file: node.sourceFile, - node_kind: node.nodeKind, - framework_role: node.frameworkRole, - }) -} - -function reportGenerationCompactApplies(result: RetrieveResult): boolean { - if ( - result.retrieval_strategy !== 'slice-v1' - || !promptWantsRuntimePipeline(result.question) - || !promptWantsReportGenerationCore(result.question) - || !result.slice - || promptExpectsPersistenceStep(result.question) - ) { - return false - } - - if (result.slice.anchors.some((anchor) => anchor.reason === 'symbol mention' || anchor.reason === 'path mention')) { - return false - } - - return result.slice.anchors.some((anchor) => anchor.reason === 'generation core heuristic') - || (result.execution_slice?.steps.length ?? 0) >= 3 -} - -function reportGenerationCompactPriority( - node: Pick, - executionStepLabels: ReadonlySet, -): number { - const lower = `${node.label} ${node.source_file} ${node.node_kind ?? ''} ${node.framework_role ?? ''}`.toLowerCase() - let value = 0 - - if (executionStepLabels.has(node.label)) value += 80 - if (/\b(?:planner|plan\b|research|assembly|assemble|renderer|render|synth|quality(?:-| )gate|dispatchwave|dispatchdbsync|broadcastrunstarted|broadcastrunfailed|queue|job|worker|orchestrator|process|persist|save)\b/.test(lower)) value += 40 - if (/\bscoremetrics\b/.test(lower)) value += 35 - if (pipelineBridgeNode(node)) value += 20 - if (node.relevance_band === 'direct') value += 5 - - if ( - /(?:^|[.#])(?:generatescoringledger|generatesensitivityanalysis|generatesuggestednextsteps|scoremetricbatch|deduplicateevidencerefs|mapcompositetorecommendation|normalizemetric|parsejson)\(?\)?$/i.test(node.label) - || /\b(?:metrichumanprompt|fallbackmetric)\b/.test(lower) - ) { - value -= 35 - } - if (lowValueReportGenerationCompactNode(node)) value -= 100 - - return value } function compactSlicePromotionApplies(result: RetrieveResult): boolean { @@ -2632,10 +2426,6 @@ function compactSlicePromotionApplies(result: RetrieveResult): boolean { return false } - if (reportGenerationCompactApplies(result)) { - return true - } - return (result.slice?.anchors ?? []).some((anchor) => anchor.reason === 'symbol mention' && methodLikeLabel(anchor.label), ) @@ -2657,91 +2447,6 @@ function promotedSliceCompactNodeIds(result: RetrieveResult): string[] { return [] } - if (reportGenerationCompactApplies(result)) { - const matchedById = new Map( - result.matched_nodes - .map((node) => (typeof node.node_id === 'string' && node.node_id.length > 0 ? [node.node_id, node] as const : null)) - .filter((entry): entry is readonly [string, RetrieveMatchedNode] => entry !== null), - ) - const executionStepLabels = new Set() - const executionStepFiles = new Set() - const promoted = new Set() - const addPromoted = (nodeId: string | undefined): void => { - if (typeof nodeId !== 'string' || nodeId.length === 0) { - return - } - const node = matchedById.get(nodeId) - if (!node || supportingPolicyOrLoggerNode(node) || lowValueReportGenerationCompactNode(node)) { - return - } - promoted.add(nodeId) - } - const addPromotedByLabel = (label: string): void => { - for (const node of result.matched_nodes) { - if (node.label !== label) { - continue - } - addPromoted(node.node_id) - } - } - - for (const anchor of result.slice.anchors) { - addPromoted(anchor.node_id) - } - - for (const step of result.execution_slice?.steps ?? []) { - executionStepLabels.add(step.label) - executionStepFiles.add(step.source_file) - addPromoted(step.node_id) - addPromotedByLabel(step.label) - } - for (const step of result.execution_slice?.primary_path?.steps ?? []) { - executionStepLabels.add(step.label) - executionStepFiles.add(step.source_file) - addPromoted(step.node_id) - addPromotedByLabel(step.label) - } - - for (const path of result.slice.selected_paths) { - if (!runtimeFlowRelation(path.relation)) { - continue - } - - const fromNode = typeof path.from_id === 'string' ? matchedById.get(path.from_id) : undefined - const toNode = typeof path.to_id === 'string' ? matchedById.get(path.to_id) : undefined - if (fromNode && pipelineBridgeNode(fromNode)) { - addPromoted(path.from_id) - } - if (toNode && pipelineBridgeNode(toNode)) { - addPromoted(path.to_id) - } - if (fromNode && toNode && fromNode.source_file === toNode.source_file) { - if (executionStepFiles.has(fromNode.source_file)) { - addPromoted(path.from_id) - addPromoted(path.to_id) - } - } - } - - return result.matched_nodes - .filter((node) => typeof node.node_id === 'string' && promoted.has(node.node_id)) - .sort((left, right) => { - const priorityDelta = reportGenerationCompactPriority(right, executionStepLabels) - - reportGenerationCompactPriority(left, executionStepLabels) - if (priorityDelta !== 0) { - return priorityDelta - } - const leftPipeline = pipelineBridgeNode(left) ? 1 : 0 - const rightPipeline = pipelineBridgeNode(right) ? 1 : 0 - if (leftPipeline !== rightPipeline) { - return rightPipeline - leftPipeline - } - return right.match_score - left.match_score - }) - .flatMap((node) => (typeof node.node_id === 'string' ? [node.node_id] : [])) - .slice(0, 24) - } - const promoted = new Set( result.slice.anchors .map((anchor) => anchor.node_id) @@ -2784,83 +2489,6 @@ function promotedSliceCompactLabels(result: RetrieveResult): string[] { return [] } - if (reportGenerationCompactApplies(result)) { - const matchedById = new Map( - result.matched_nodes - .map((node) => (typeof node.node_id === 'string' && node.node_id.length > 0 ? [node.node_id, node] as const : null)) - .filter((entry): entry is readonly [string, RetrieveMatchedNode] => entry !== null), - ) - const representativeByLabel = new Map() - for (const node of result.matched_nodes) { - if (!representativeByLabel.has(node.label)) { - representativeByLabel.set(node.label, node) - } - } - - const executionStepLabels = new Set() - const executionStepFiles = new Set() - const promotedLabels = new Set() - const addPromotedLabel = (label: string | undefined, node?: RetrieveMatchedNode): void => { - if (typeof label !== 'string' || label.length === 0) { - return - } - const candidate = node ?? representativeByLabel.get(label) - if (!candidate || supportingPolicyOrLoggerNode(candidate) || lowValueReportGenerationCompactNode(candidate)) { - return - } - promotedLabels.add(label) - } - - for (const anchor of result.slice.anchors) { - addPromotedLabel(anchor.label, typeof anchor.node_id === 'string' ? matchedById.get(anchor.node_id) : undefined) - } - - for (const step of result.execution_slice?.steps ?? []) { - executionStepLabels.add(step.label) - executionStepFiles.add(step.source_file) - addPromotedLabel(step.label, typeof step.node_id === 'string' ? matchedById.get(step.node_id) : undefined) - } - for (const step of result.execution_slice?.primary_path?.steps ?? []) { - executionStepLabels.add(step.label) - executionStepFiles.add(step.source_file) - addPromotedLabel(step.label, typeof step.node_id === 'string' ? matchedById.get(step.node_id) : undefined) - } - - for (const path of result.slice.selected_paths) { - if (!runtimeFlowRelation(path.relation)) { - continue - } - - const fromNode = typeof path.from_id === 'string' ? matchedById.get(path.from_id) : representativeByLabel.get(path.from) - const toNode = typeof path.to_id === 'string' ? matchedById.get(path.to_id) : representativeByLabel.get(path.to) - if (fromNode && pipelineBridgeNode(fromNode)) { - addPromotedLabel(fromNode.label, fromNode) - } - if (toNode && pipelineBridgeNode(toNode)) { - addPromotedLabel(toNode.label, toNode) - } - if (fromNode && toNode && fromNode.source_file === toNode.source_file) { - if (executionStepFiles.has(fromNode.source_file)) { - addPromotedLabel(fromNode.label, fromNode) - addPromotedLabel(toNode.label, toNode) - } - } - } - - return [...promotedLabels] - .sort((left, right) => { - const leftNode = representativeByLabel.get(left) - const rightNode = representativeByLabel.get(right) - const priorityDelta = (rightNode ? reportGenerationCompactPriority(rightNode, executionStepLabels) : 0) - - (leftNode ? reportGenerationCompactPriority(leftNode, executionStepLabels) : 0) - if (priorityDelta !== 0) { - return priorityDelta - } - return left.localeCompare(right) - }) - .slice(0, 24) - } - return result.slice.anchors.map((anchor) => anchor.label) } @@ -3422,15 +3050,6 @@ function executionPhaseOrder(question: string): ExecutionPhase[] { const enabled = new Set(['controller', 'service', 'queue', 'worker', 'persistence']) if (promptWantsAuthGuardPhase(question)) enabled.add('auth_guard') if (promptWantsValidationPhase(question)) enabled.add('validation') - if (promptWantsDetailedReportGenerationPhases(question)) { - enabled.add('orchestrator') - enabled.add('planner') - enabled.add('external_research_or_api') - enabled.add('report_builder') - enabled.add('scoring') - enabled.add('quality_gate') - enabled.add('renderer_or_synthesis') - } if (promptWantsNotificationOrEventPhase(question)) enabled.add('notification_or_event') const phaseOrder: ExecutionPhase[] = [ @@ -3477,20 +3096,10 @@ function expectedExecutionPhases( if (promptWantsServiceStep(question)) { phases.push('service') } - if (promptWantsDetailedReportGenerationPhases(question)) { - if (scopeHasExecutionPhase(scopeSteps, 'orchestrator')) phases.push('orchestrator') - if (scopeHasExecutionPhase(scopeSteps, 'planner')) phases.push('planner') - if (scopeHasExecutionPhase(scopeSteps, 'external_research_or_api')) phases.push('external_research_or_api') - if (scopeHasExecutionPhase(scopeSteps, 'report_builder')) phases.push('report_builder') - if (scopeHasExecutionPhase(scopeSteps, 'scoring')) phases.push('scoring') - if (scopeHasExecutionPhase(scopeSteps, 'quality_gate')) phases.push('quality_gate') - if (scopeHasExecutionPhase(scopeSteps, 'renderer_or_synthesis')) phases.push('renderer_or_synthesis') - if (scopeHasExecutionPhase(scopeSteps, 'persistence')) phases.push('persistence') - } const scopeHasRuntimeHandoff = scopeHasExecutionPhase(scopeSteps, 'queue') || scopeHasExecutionPhase(scopeSteps, 'worker') if ( promptExplicitlyWantsRuntimeHandoff(question) - || (!promptWantsDetailedReportGenerationPhases(question) && promptWantsRuntimePipeline(question) && scopeHasRuntimeHandoff) + || (promptWantsRuntimePipeline(question) && scopeHasRuntimeHandoff) ) { phases.push('queue', 'worker') } @@ -4063,10 +3672,6 @@ function walkExecutionSlice( return orderedPathIds } -function runtimeGenerationAnswerContractWantsReportGenerationCore(question: string): boolean { - return /\b(?:report(?:\s+generation)?|generated\s+report|validation\s+report|final\s+report|assembly|assemble|synthesis|renderer|render|planner|research|metrics?|scor(?:e|ing)|quality(?:\s|-)?gate)\b/i.test(question) -} - function runtimeGenerationStepText( value: Pick, ): string { @@ -4104,24 +3709,6 @@ function runtimeGenerationContractPhaseElements( elements.add('missing_or_uncertain_phases') } - if (runtimeGenerationAnswerContractWantsReportGenerationCore(question)) { - if (/\b(?:planner|\.plan\(|plan\(\)|planning)\b/i.test(evidenceText)) { - elements.add('planner_phase') - } - if (/\b(?:research|search\(\)|processsection|processsection\(\)|section[-_\s]?research)\b/i.test(evidenceText)) { - elements.add('research_phase') - } - if (/\b(?:assembly|assemble|synthesis)\b/i.test(evidenceText)) { - elements.add('assembly_phase') - } - if (/\b(?:score|scoring|metrics?)\b/i.test(evidenceText)) { - elements.add('scoring_phase') - } - if (/\b(?:render|renderer|report builder|reportbuilder|final report)\b/i.test(evidenceText)) { - elements.add('report_builder_phase') - } - } - return [...elements] } @@ -5252,7 +4839,6 @@ function retrieveContextPass( graph: KnowledgeGraph, options: RetrieveOptions, conceptualNodeBoosts: ReadonlyMap = new Map(), - preserveConceptualObligationOrder = false, ): RetrieveResult { // Guard before candidate expansion, which also reads directional adjacency. // sliceCandidatesForRetrieve repeats the guard to protect its direct callers. @@ -5912,20 +5498,16 @@ function retrieveContextPass( const inclusionOrder = expansionPolicy.include_peripheral ? frameworkOrderedCandidates : frameworkOrderedCandidates.filter((node) => node.relevanceBand !== 'peripheral') - // Cross-domain conceptual recovery can intentionally keep disconnected - // evidence outside one slice. For a report-generation workflow, exclude - // known side actions from that preserved set so controller siblings such - // as list/status/health do not displace the execution path. - let orderedCandidates = preserveConceptualObligationOrder - ? inclusionOrder.filter((node) => !recoveredReportGenerationNoise(question, node)) - : inclusionOrder + let orderedCandidates = inclusionOrder let sliceMetadata: ContextPackSliceMetadata | undefined - // A multi-obligation conceptual fallback can deliberately assemble - // cross-service and cross-language owners that do not form one local - // slice. Only that explicitly selected recovery mode bypasses slice-v1; - // ordinary conceptual reranking and symbol/path-anchored questions keep - // the established slice contract. - if (options.retrievalStrategy === 'slice-v1' && !preserveConceptualObligationOrder) { + // A requested slice is always applied. The conceptual-recovery bypass + // that used to sit here was measured: instrumenting its trigger across + // the retrieval, conceptual-fallback, pack-quality and production + // correctness suites showed it firing on exactly three questions, all of + // them qualification-shaped, and on no independent fixture. A mode only + // the benchmark reaches is benchmark tuning, whatever its condition + // looks like, so it is gone rather than re-argued. + if (options.retrievalStrategy === 'slice-v1') { const sliced = sliceCandidatesForRetrieve( graph, scored.map((node) => ({ @@ -6135,14 +5717,7 @@ function retrieveContextWithConceptualFallback(graph: KnowledgeGraph, options: R return { ...initial, retrieval_plan: proposal.plan } } - const preserveConceptualObligationOrder = initialQuality.explicit_anchors === 0 - && (proposal.plan.query_obligations?.total ?? 0) >= 4 - const recovered = retrieveContextPass( - graph, - options, - proposal.nodeBoosts, - preserveConceptualObligationOrder, - ) + const recovered = retrieveContextPass(graph, options, proposal.nodeBoosts) const finalized = finalizeConceptualFallbackPlan( proposal, retrievalQualitySnapshot(graph, recovered), diff --git a/src/runtime/retrieve/conceptual-fallback.ts b/src/runtime/retrieve/conceptual-fallback.ts index 63fc1b63..f93f12c3 100644 --- a/src/runtime/retrieve/conceptual-fallback.ts +++ b/src/runtime/retrieve/conceptual-fallback.ts @@ -41,7 +41,6 @@ const QUERY_DIRECTIVE_TERMS = new Set([ 'note', 'relevant', 'remaining', 'path', 'paths', 'state', 'symbols', 'trace', 'uncertainty', ]) -const DIVERGENCE_SCOPE_NOISE = new Set(['across', 'logic', 'these']) const CHANGE_LIFECYCLE_TERMS = new Set([ 'change', 'changed', 'changes', 'changing', @@ -106,11 +105,11 @@ const TRANSITION_PREFIXES = ['becom', 'creat', 'insert', 'open', 'transition', ' const PRESENTATION_QUERY_PATTERN = /\b(?:component|dashboard|frontend|render|screen|ui|visual|widget)\b/i const PRESENTATION_PATH_PATTERN = /(?:\.(?:jsx|tsx)$|\/(?:components?|dashboard|views?|widgets?)\/)/i const PRESENTATION_LABEL_PATTERN = /^(?:page\s+\/|.*(?:badge|card|component|screen|widget).*)$/i -const RUNTIME_PATH_PATTERN = /\/(?:api|checker|content|db|handlers?|persistence|routes?|schema|server|services?|workflows?)\//i -const CORE_BEHAVIOR_OWNER_PATH_PATTERN = /\/(?:api|checker|content|handlers?|services?|workflows?)\//i +const RUNTIME_PATH_PATTERN = /\/(?:api|db|handlers?|persistence|routes?|schema|server|services?|workflows?)\//i +const CORE_BEHAVIOR_OWNER_PATH_PATTERN = /\/(?:api|handlers?|services?|workflows?)\//i const PERSISTENCE_PATH_PATTERN = /\/(?:db|persistence|repositories?|schema)(?:\/|\.)/i -const LOW_VALUE_OWNER_PATH_PATTERN = /(?:\.pb\.go$|(?:_pb|\.pb)\.ts$|\/(?:errors?|limits)\.[^/]+$|\/lib\/http\/etag\.[^/]+$|statusPage\.utils\.[^/]+$|\/content\/markdown\/)/i -const LOW_VALUE_OWNER_LABEL_PATTERN = /(?:Error\(\)?$|(?:create)?ErrorResponse\(\)?$|ErrorResponse$|Limits?\(\)?$|(?:assert|check)?\w*Quota\(\)?$|computeETag\(\)?$|validate\w*Access\(\)?$|(?:statusLabel|statusGlyph|generate\w*)\(\)?$)/i +const LOW_VALUE_OWNER_PATH_PATTERN = /(?:\.pb\.go$|(?:_pb|\.pb)\.ts$|\/(?:errors?|limits)\.[^/]+$)/i +const LOW_VALUE_OWNER_LABEL_PATTERN = /(?:Error\(\)?$|(?:create)?ErrorResponse\(\)?$|ErrorResponse$|Limits?\(\)?$)/i const EXTERNAL_SCOPE_PATTERN = /(?:^|[\/_-])external(?:[\/_-]|$)/i const FLOW_TEST_SOURCE_PATTERN = /(?:^|\/)(?:tests?|__tests__)(?:\/|$)|(?:^|\/)(?:test|tests?[-_.][^/]*)\.[^/]+$|(?:\.test\.[^/]+$|\.spec\.[^/]+$|_test\.go$)/i const FLOW_TYPE_SOURCE_PATTERN = /(?:^|\/)(?:types?|interfaces?)(?:\/|\.[^/]+$)|(?:^|\/)(?:types?|interfaces?)\.[^/]+$/i @@ -121,7 +120,6 @@ const QUERY_EVIDENCE_STATE_MUTATION_PATTERN = /(?:\b(?:create|insert|transition| const QUERY_EVIDENCE_DELIVERY_OPERATION_PATTERN = /(?:\b(?:deliver|dispatch|emit|enqueue|notify|publish|send|trigger)\w*\s*\(|\.(?:deliver|dispatch|emit|enqueue|notify|publish|send|trigger)\w*\s*\()/i const QUERY_EVIDENCE_COMPUTATION_OPERATION_PATTERN = /(?:\b(?:compute|derive|resolve)\w*\s*\(|\b\w*(?:indicator|result|state|status)\w*\s*=|\b\w*(?:indicator|status)\w*\s*\()/i const EXPLICIT_ERROR_QUERY_PATTERN = /\b(?:error|exception|throw|throws|thrown)\b/i -const FLOW_OUTCOME_TERMS = new Set(['error', 'fail', 'failed', 'failure', 'result', 'response', 'status']) const QUERY_STOP_WORDS = new Set([ 'a', 'about', 'after', 'again', 'agent', 'also', 'an', 'and', 'are', 'be', @@ -187,9 +185,6 @@ interface AnchorCandidate { persistenceShaped: boolean lowValueOwner: boolean behaviorOwner: boolean - fileOwner: boolean - publicBoundaryOwner: boolean - runtimeScope: string } export interface QueryEvidenceObligation { @@ -315,9 +310,7 @@ function divergenceScopeTerms(obligation: QueryEvidenceObligation | undefined): if (!obligation) { return [] } - const literalTerms = obligation.terms.filter((term) => !term.startsWith('@')) - const scopedTerms = literalTerms.filter((term) => !DIVERGENCE_SCOPE_NOISE.has(term)) - return scopedTerms.length > 0 ? scopedTerms : literalTerms + return obligation.terms.filter((term) => !term.startsWith('@')) } function lexicalTermsMatch(left: string, right: string): boolean { @@ -816,34 +809,9 @@ function presentationShapedNode(node: VocabularyNode): boolean { || /^(?:component|page|screen|view|widget)$/.test(node.frameworkRole) } -function runtimeScopeForSource(sourceFile: string): string { - const normalized = sourceFile.replaceAll('\\', '/') - const match = normalized.match(/(?:^|\/)(apps|packages)\/([^/]+)/i) - if (match?.[1] && match[2]) { - return `${match[1].toLowerCase()}/${match[2].toLowerCase()}` - } - const parts = normalized.split('/').filter(Boolean) - return parts.slice(-3, -1).join('/') || normalized -} -function fileOwnerNode(node: Pick): boolean { - const basename = node.sourceFile.replaceAll('\\', '/').split('/').at(-1)?.toLowerCase() ?? '' - const normalizedLabel = node.label.replaceAll('\\', '/').split('/').at(-1)?.toLowerCase() ?? '' - return normalizedLabel === basename - || (node.nodeKind.trim().length === 0 && normalizedLabel.replace(/\.[^.]+$/, '') === basename.replace(/\.[^.]+$/, '')) -} -function publicBoundaryOwnerNode(node: Pick): boolean { - const normalizedSource = node.sourceFile.replaceAll('\\', '/') - const routePath = /\/(?:app\/)?api\//i.test(normalizedSource) - && /\/route\.[^/]+$/i.test(normalizedSource) - const routeLabel = /^(?:GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)(?:\(\)|\s|$)|^route\.[^.]+$/i.test(node.label) - return (routePath && routeLabel) || /(?:route|request)_handler/i.test(node.frameworkRole) -} -function runtimeLanguageForSource(sourceFile: string): string { - return sourceFile.replaceAll('\\', '/').split('.').at(-1)?.toLowerCase() ?? '' -} function vocabularyDocumentFrequency(index: RepositoryVocabularyIndex, queryTerm: string): number { let count = 0 @@ -947,9 +915,6 @@ function anchorForNode( lowValueOwner: LOW_VALUE_OWNER_PATH_PATTERN.test(node.sourceFile) || LOW_VALUE_OWNER_LABEL_PATTERN.test(node.label), behaviorOwner: CORE_BEHAVIOR_OWNER_PATH_PATTERN.test(node.sourceFile), - fileOwner: fileOwnerNode(node), - publicBoundaryOwner: publicBoundaryOwnerNode(node), - runtimeScope: runtimeScopeForSource(node.sourceFile), } } @@ -1056,9 +1021,6 @@ function diversifyAnchors( || (obligation.terms.includes(TRANSITION_CONCEPT) ? Number(right.transitionOwner) - Number(left.transitionOwner) : 0) - || (obligation.terms.includes('public') && obligation.terms.includes('page') - ? Number(right.fileOwner) - Number(left.fileOwner) - : 0) || obligationSymbolMatchCount(right, obligation) - obligationSymbolMatchCount(left, obligation) || (right.obligationMatches.get(obligation.index) ?? 0) - (left.obligationMatches.get(obligation.index) ?? 0) || obligationSpecificMatchCount(right, obligation) - obligationSpecificMatchCount(left, obligation) @@ -1186,19 +1148,6 @@ function diversifyAnchors( if (behaviorOrder !== 0) { return behaviorOrder } - const publicFileOwnerOrder = obligation.terms.includes('public') && obligation.terms.includes('page') - ? Number(right.fileOwner) - Number(left.fileOwner) - : 0 - if (publicFileOwnerOrder !== 0) { - return publicFileOwnerOrder - } - const publicOwnerGrounding = obligation.terms.includes('public') && obligation.terms.includes('page') - ? obligationSymbolMatchCount(right, obligation) - obligationSymbolMatchCount(left, obligation) - || obligationSpecificMatchCount(right, obligation) - obligationSpecificMatchCount(left, obligation) - : 0 - if (publicOwnerGrounding !== 0) { - return publicOwnerGrounding - } const workflowContextOrder = crossObligationContextMatchCount(right, obligation) - crossObligationContextMatchCount(left, obligation) if (workflowContextOrder !== 0) { @@ -1227,34 +1176,6 @@ function diversifyAnchors( } } - // A public status computation is incomplete without the HTTP boundary that - // fetches and serializes it. Reserve that owner separately from the status - // implementation so runtime provenance is explicit rather than inferred - // from a shared output type. - for (const obligation of obligations.filter((candidate) => ( - candidate.terms.includes('public') && candidate.terms.includes('page') - ))) { - const publicTerms = obligation.terms.filter((term) => !term.startsWith('@')) - const boundary = ranked - .filter((anchor) => ( - anchor.publicBoundaryOwner - && publicTerms.filter((term) => anchor.matchedQueryTerms.has(term)).length >= 2 - && !anchor.lowValueOwner - )) - .sort((left, right) => ( - Number(/\(\)$/.test(right.label)) - Number(/\(\)$/.test(left.label)) - || obligationSymbolMatchCount(right, obligation) - obligationSymbolMatchCount(left, obligation) - || obligationSpecificMatchCount(right, obligation) - obligationSpecificMatchCount(left, obligation) - || right.structuralDegree - left.structuralDegree - || right.score - left.score - || left.id.localeCompare(right.id) - ))[0] - if (boundary) { - if (!selectedIds.has(boundary.id)) add(boundary) - reservedByObligation.add(boundary.id) - } - } - // A divergence request requires at least two distinct implementations to // compare. Reserve a second scoped computation owner instead of hoping it // survives the global top-k. @@ -1305,89 +1226,6 @@ function diversifyAnchors( } } - // Cross-language transports are often not linked statically. Keep one - // lifecycle-shaped owner from a second runtime scope for the first stage so - // the pack can expose that boundary and state the remaining uncertainty. - const firstObligation = obligations[0] - const firstPrimaryId = firstObligation ? preferredByObligation.get(firstObligation.index) : undefined - const firstPrimaryScope = firstPrimaryId ? ranked.find((anchor) => anchor.id === firstPrimaryId)?.runtimeScope : undefined - const firstPrimaryLanguage = firstPrimaryId - ? runtimeLanguageForSource(ranked.find((anchor) => anchor.id === firstPrimaryId)?.sourceFile ?? '') - : undefined - if (firstObligation) { - const headTerm = firstObligation.terms.at(-1) - const outcomeLabel = (anchor: AnchorCandidate): boolean => ( - tokenize(anchor.label).some((term) => FLOW_OUTCOME_TERMS.has(term)) - ) - const boundaryOwner = ranked - .filter((anchor) => ( - anchor.id !== firstPrimaryId - && anchor.runtimeScope !== firstPrimaryScope - && anchor.transitionOwner - && !anchor.lowValueOwner - && (anchor.symbolQueryTerms.size > 0 || anchor.pathQueryTerms.size > 0) - && firstObligation.terms.some((term) => anchor.matchedQueryTerms.has(term)) - )) - .sort((left, right) => ( - Number( - firstPrimaryLanguage !== undefined - && runtimeLanguageForSource(right.sourceFile) !== firstPrimaryLanguage, - ) - Number( - firstPrimaryLanguage !== undefined - && runtimeLanguageForSource(left.sourceFile) !== firstPrimaryLanguage, - ) - || Number(outcomeLabel(right)) - Number(outcomeLabel(left)) - || Number(headTerm !== undefined && right.pathQueryTerms.has(headTerm)) - - Number(headTerm !== undefined && left.pathQueryTerms.has(headTerm)) - || firstObligation.terms.filter((term) => right.matchedQueryTerms.has(term)).length - - firstObligation.terms.filter((term) => left.matchedQueryTerms.has(term)).length - || right.structuralDegree - left.structuralDegree - || right.score - left.score - || left.id.localeCompare(right.id) - ))[0] - if (boundaryOwner) { - if (!selectedIds.has(boundaryOwner.id)) add(boundaryOwner) - reservedByObligation.add(boundaryOwner.id) - - const rankedById = new Map(ranked.map((anchor) => [anchor.id, anchor])) - const boundaryCaller = graph.predecessors(boundaryOwner.id) - .flatMap((nodeId) => { - const anchor = rankedById.get(nodeId) - if (!anchor) return [] - const isBoundaryCall = graph.relationsBetween(nodeId, boundaryOwner.id) - .some((relation) => /^(?:calls|dispatches|emits|enqueues|invokes|publishes|triggers)$/.test(relation)) - return isBoundaryCall - ? [anchor] - : [] - }) - .filter((anchor) => ( - anchor.sourceFile !== boundaryOwner.sourceFile - && anchor.behaviorOwner - && !anchor.lowValueOwner - && firstObligation.terms.some((term) => anchor.matchedQueryTerms.has(term)) - )) - .sort((left, right) => ( - Number(right.obligationMatches.has(firstObligation.index)) - - Number(left.obligationMatches.has(firstObligation.index)) - || firstObligation.terms.filter((term) => right.symbolQueryTerms.has(term)).length - - firstObligation.terms.filter((term) => left.symbolQueryTerms.has(term)).length - || firstObligation.terms.filter((term) => right.matchedQueryTerms.has(term)).length - - firstObligation.terms.filter((term) => left.matchedQueryTerms.has(term)).length - || right.structuralDegree - left.structuralDegree - || right.score - left.score - || left.id.localeCompare(right.id) - ))[0] - if (boundaryCaller) { - if (!selectedIds.has(boundaryCaller.id)) add(boundaryCaller) - if (firstPrimaryId && preferredByObligation.get(firstObligation.index) === firstPrimaryId) { - reservedByObligation.delete(firstPrimaryId) - } - preferredByObligation.set(firstObligation.index, boundaryCaller.id) - reservedByObligation.add(boundaryCaller.id) - } - } - } - for (const anchor of ranked) { if (selectedIds.has(anchor.id)) { continue diff --git a/tests/unit/context-pack-command.test.ts b/tests/unit/context-pack-command.test.ts index c33cae76..1ce494a0 100644 --- a/tests/unit/context-pack-command.test.ts +++ b/tests/unit/context-pack-command.test.ts @@ -1070,85 +1070,16 @@ describe('context-pack-command', () => { ]) }) - it('does not preserve a ready verdict when the serialized pack omits prompt obligations', async () => { - const prompt = 'Explain the exact end-to-end path from a failed HTTP monitor check to incident creation, notification delivery, and the public status-page result. Compare every distinct overall-status computation. Read-only: do not modify files.' - const graph = buildCrossLayerMonitorFlowFixture() - const retrieval = retrieveContext(graph, { - question: prompt, - budget: 1_800, - taskKind: 'explain', - retrievalStrategy: 'slice-v1', - }) - const compact = compactRetrieveResult(retrieval) - const omittedNodeIds = new Set( - compact.matched_nodes - .filter((node) => /apps\/workflows\/src\/checker\/(?:index|alerting|utils)\.ts$/.test(node.source_file)) - .flatMap((node) => node.node_id ? [node.node_id] : []), - ) - const serialized = { - ...compact, - matched_nodes: compact.matched_nodes.filter((node) => !node.node_id || !omittedNodeIds.has(node.node_id)), - relationships: compact.relationships.filter((relationship) => ( - (!relationship.from_id || !omittedNodeIds.has(relationship.from_id)) - && (!relationship.to_id || !omittedNodeIds.has(relationship.to_id)) - )), - } - const optimisticRetrieval: RetrieveResult = { - ...retrieval, - recovery: { - ...retrieval.recovery!, - status: 'not_needed', - initial_state: 'ready', - final_state: 'ready', - attempts: [], - improved: false, - }, - } - const dependencies: ContextPackCommandDependencies = { - loadGraph: vi.fn().mockReturnValue(graph), - retrieveContext: vi.fn().mockReturnValue(optimisticRetrieval), - compactRetrieveResult: vi.fn().mockReturnValue(serialized), - analyzePrImpact: vi.fn(), - compactPrImpactResult: vi.fn(), - analyzeImpact: vi.fn(), - compactImpactResult: vi.fn(), - } + // #660-B1. Removed. This asserted a multi-service selection that only the + // conceptual-recovery slice bypass produced. Instrumenting that bypass's + // trigger across the retrieval, conceptual-fallback, pack-quality and + // production-correctness suites showed it firing on exactly three + // questions, all qualification-shaped, and on no independent fixture -- so + // it was removed rather than defended, and with it the selection this + // asserted. Recorded here rather than deleted silently: a requested slice + // is now always applied, and a cross-service question with no explicit + // anchor is sliced to a local slice. - const payload = JSON.parse(await runContextPackCommand({ - prompt, - budget: 1_800, - task: 'explain', - graphPath: 'out/graph.json', - graphPathIntent: 'explicit' as const, - retrievalStrategy: 'slice-v1', - format: 'json', - }, dependencies)) as { - evidence?: { - coverage?: string - coverage_detail?: { missing_obligations?: string[] } - answerability?: { state?: string; broad_search_fallback?: string } - agent_directive?: string - } - pack?: { - matched_nodes?: Array<{ label: string; source_file: string; snippet?: string | null }> - retrieval_plan?: { query_obligations?: { total?: number; finally_covered?: number } } - } - } - - expect(optimisticRetrieval.recovery?.final_state).toBe('ready') - expect(payload.evidence).toMatchObject({ - coverage: 'partial', - answerability: { - state: 'verify_targets', - broad_search_fallback: 'targeted_only', - }, - agent_directive: 'verify_one_targeted_file', - }) - expect(payload.evidence?.coverage_detail?.missing_obligations).toEqual(expect.arrayContaining([ - 'query:obligation:2', - 'query:obligation:3', - ])) - }) it('reconciles a retained retrieval-plan receipt to the serialized snippets', () => { const prompt = 'Explain the exact end-to-end path from a failed HTTP monitor check to incident creation, notification delivery, and the public status-page result. Compare every distinct overall-status computation. Read-only: do not modify files.' @@ -1192,73 +1123,16 @@ describe('context-pack-command', () => { }) }) - it('keeps a late unique evidence owner when the eight-node response cap would otherwise drop it', () => { - const prompt = 'Explain the exact end-to-end path from a failed HTTP monitor check to incident creation, notification delivery, and the public status-page result. Compare every distinct overall-status computation.' - const retrieval = retrieveContext(buildCrossLayerMonitorFlowFixture(), { - question: prompt, - budget: 1_800, - taskKind: 'explain', - retrievalStrategy: 'slice-v1', - }) - const compact = compactRetrieveResult(retrieval) - const incidentOwner = compact.matched_nodes.find((node) => node.snippet?.includes('insert(incidentTable)')) - expect(incidentOwner).toBeDefined() - const nonIncidentNodes = compact.matched_nodes - .filter((node) => node.node_id !== incidentOwner?.node_id) - .map((node) => node.snippet?.includes('insert(incidentTable)') - ? { ...node, snippet: 'const incident = await findOpenIncident(monitorId)' } - : node) - const duplicateSource = nonIncidentNodes.find((node) => node.source_file.includes('status-json')) - ?? nonIncidentNodes[0]! - const pressuredNodes = [ - ...nonIncidentNodes, - { ...duplicateSource, node_id: 'duplicate-status-owner-1', label: 'toUnresolvedIncidents' }, - { ...duplicateSource, node_id: 'duplicate-status-owner-2', label: 'unresolvedIncidents' }, - incidentOwner!, - ] - const { score: _score, ...evidence } = assessMadarResponseEvidence({ - evidencePlan: buildRetrievalEvidencePlanFromResult(retrieval), - question: prompt, - recovery: retrieval.recovery, - }) - - const payload = buildAnswerReadyPackSchema({ - schema_version: 1, - task: 'explain', - prompt, - budget: 5_000, - evidence, - expandable: retrieval.expandable ?? [], - pack: { - ...compact, - matched_nodes: pressuredNodes, - }, - }, 5_000, retrieval.selection_diagnostics) - const selectedNodes = (payload.pack as { - matched_nodes: Array<{ node_id?: string; label: string; source_file: string; snippet?: string }> - retrieval_plan?: { query_obligations?: { total?: number; finally_covered?: number } } - }).matched_nodes - const retrievalPlan = (payload.pack as { - retrieval_plan?: { query_obligations?: { total?: number; finally_covered?: number } } - }).retrieval_plan - const serializedEvidence = payload.evidence as { - answerability?: { state?: string } - agent_directive?: string - } + // #660-B1. Removed. This asserted a multi-service selection that only the + // conceptual-recovery slice bypass produced. Instrumenting that bypass's + // trigger across the retrieval, conceptual-fallback, pack-quality and + // production-correctness suites showed it firing on exactly three + // questions, all qualification-shaped, and on no independent fixture -- so + // it was removed rather than defended, and with it the selection this + // asserted. Recorded here rather than deleted silently: a requested slice + // is now always applied, and a cross-service question with no explicit + // anchor is sliced to a local slice. - expect(selectedNodes).toHaveLength(8) - expect(selectedNodes.some((node) => node.node_id === incidentOwner?.node_id)).toBe(true) - expect(selectedNodes.map((node) => node.snippet ?? '').join('\n')).toMatch(/insert\(incidentTable\)/) - expect(serializedEvidence).toMatchObject({ - answerability: { state: expect.stringMatching(/^ready(?:_with_caveat)?$/) }, - agent_directive: 'answer_from_pack', - }) - const serializedCoverage = evaluateQueryEvidenceCoverage(prompt, selectedNodes) - expect(retrievalPlan?.query_obligations).toMatchObject({ - total: serializedCoverage.total, - finally_covered: serializedCoverage.covered, - }) - }) it('keeps every cited supporting node when one falls beyond the answer-ready node cap', () => { const matchedNodes = Array.from({ length: 9 }, (_, index) => ({ diff --git a/tests/unit/context-pack.test.ts b/tests/unit/context-pack.test.ts index f50dd9ae..fd1a91ff 100644 --- a/tests/unit/context-pack.test.ts +++ b/tests/unit/context-pack.test.ts @@ -346,156 +346,134 @@ describe('context-pack', () => { ]) }) - it('makes a split public-status projection explicit and deduplicates file-level provenance', () => { + // #660-B. Was three tests pinning fixed claim strings keyed on one + // qualification repository's paths and symbols. Those builders are gone; + // what remains is asserted on generic, evidence-backed behaviour. + + // Retained subject: per-file deduplication of router-output provenance. + // Every identifier is unrelated to any qualification target, so passing + // proves the builder reads the framework type helper, not a remembered name. + it('deduplicates router-output input provenance per source file', () => { const pack = compileContextPack({ task_contract: classifyTaskContract('explain', { budget: 160, - prompt: 'Compare public status computation paths.', + prompt: 'Compare catalog summary computation paths.', }), nodes: [ nodeCandidate({ - node_id: 'unresolved_incidents', - label: 'unresolvedIncidents()', - source_file: 'apps/status-page/src/content/status-json.ts', + node_id: 'pending_entries', + label: 'pendingEntries()', + source_file: 'apps/portal/src/content/summary-json.ts', line_number: 50, file_type: 'code', snippet: [ - 'type Page = NonNullable;', - 'status: pageIndicator(page.status),', - 'return page.statusReports.filter((report) => report.status !== "resolved")', + 'type Listing = NonNullable;', + 'return listing.entries.filter((entry) => entry.state !== "closed")', ].join('\n'), match_score: 9, relevance_band: 'direct', community: 0, }, 'primary', 20), nodeCandidate({ - node_id: 'status_json', - label: 'status-json.ts', - source_file: 'apps/status-page/src/content/status-json.ts', + node_id: 'summary_json', + label: 'summary-json.ts', + source_file: 'apps/portal/src/content/summary-json.ts', line_number: 1, file_type: 'code', - snippet: 'type Page = NonNullable;', + snippet: 'type Listing = NonNullable;', match_score: 8, relevance_band: 'direct', community: 0, }, 'primary', 10), - nodeCandidate({ - node_id: 'status_page_router', - label: 'statusPage.ts', - source_file: 'packages/api/src/router/statusPage.ts', - line_number: 226, - file_type: 'code', - snippet: 'events.some((e) => e.type === "incident" && !e.to) && barType !== "manual" ? "error" : activeReportStatus(events)', - match_score: 8, - relevance_band: 'direct', - community: 1, - }, 'primary', 20), ], }) - expect(pack.claims.filter((claim) => claim.text.startsWith('input provenance:'))).toHaveLength(1) - expect(pack.claims[0]).toEqual({ - evidence_class: 'primary', - text: 'public payload divergence: when barType is not manual, an open incident event can make page.status "error" in packages/api/src/router/statusPage.ts; apps/status-page/src/content/status-json.ts builds unresolved incident entries only from page.statusReports, so an auto-created incident without a status report can yield an error indicator with an empty incidents list', - node_labels: ['unresolvedIncidents()', 'statusPage.ts'], - }) - expect(pack.claims[1]?.text).toBe( - 'input provenance: unresolvedIncidents() consumes data typed as the RouterOutputs["statusPage"]["get"] router output', + const provenance = pack.claims.filter((claim) => claim.text.startsWith('input provenance:')) + expect(provenance).toHaveLength(1) + expect(provenance[0]?.text).toBe( + 'input provenance: pendingEntries() consumes data typed as the RouterOutputs["catalog"]["list"] router output', ) }) - it('states the public runtime router provenance separately from an alternate computation', () => { + // Replaces the fixed "cross-runtime handoff" claim. The handoff is now + // recovered from the typed `enqueues_job` edge an extractor recorded, so + // the claim cites two real source locations and names the relation that + // produced it -- and no string in either snippet can conjure it. + it('states a queue handoff from the typed edge rather than from matched text', () => { const pack = compileContextPack({ task_contract: classifyTaskContract('explain', { - budget: 180, - prompt: 'Identify inconsistent public status computation paths.', + budget: 200, + prompt: 'Trace a failed check into the workflow.', }), nodes: [ nodeCandidate({ - node_id: 'public_status_route', - label: 'GET()', - source_file: 'apps/status-page/src/app/api/status/[[...path]]/route.ts', - line_number: 34, - file_type: 'code', - snippet: 'const data = await queryClient.fetchQuery(trpc.statusPage.get.queryOptions({ slug })); const payload = toStatus(data, baseUrl)', - match_score: 10, - relevance_band: 'direct', - community: 0, - }, 'primary', 20), - nodeCandidate({ - node_id: 'status_page_router', - label: 'statusPage.ts', - source_file: 'packages/api/src/router/statusPage.ts', - line_number: 226, + node_id: 'producer', + label: 'SyncRecord()', + source_file: 'apps/ledger/sync.go', + line_number: 12, file_type: 'code', - snippet: 'events.some((e) => e.type === "incident" && !e.to) && barType !== "manual"', + snippet: 'queueClient.Dispatch(ctx, req)', match_score: 9, relevance_band: 'direct', - community: 1, + community: 0, }, 'primary', 20), nodeCandidate({ - node_id: 'alternate_status', - label: 'computeOverallStatus()', - source_file: 'apps/server/src/routes/rpc/handlers/status-page/index.ts', - line_number: 360, + node_id: 'consumer', + label: 'RecordSyncWorker.run()', + source_file: 'apps/workflows/src/record-sync.ts', + line_number: 30, file_type: 'code', - snippet: 'const overallStatus = hasActiveStatusReport ? DEGRADED : hasActiveMaintenance ? MAINTENANCE : OPERATIONAL', + snippet: 'await persistRecord(payload)', match_score: 8, relevance_band: 'direct', - community: 2, + community: 1, }, 'primary', 20), ], + relationships: [ + { from_id: 'producer', from: 'SyncRecord()', to_id: 'consumer', to: 'RecordSyncWorker.run()', relation: 'enqueues_job' }, + ], }) - expect(pack.claims[0]).toEqual({ - evidence_class: 'primary', - text: 'public runtime provenance: apps/status-page/src/app/api/status/[[...path]]/route.ts GET() fetches trpc.statusPage.get and passes that data to the public status-json serializers backed by packages/api/src/router/statusPage.ts; packages/api/src/router/statusPage.ts treats an open incident event as "error" outside manual mode, while apps/server/src/routes/rpc/handlers/status-page/index.ts computeOverallStatus() derives overall status from active status reports and maintenance', - node_labels: ['GET()', 'statusPage.ts', 'computeOverallStatus()'], - }) + expect(pack.claims.some((claim) => claim.text === ( + 'queue handoff: apps/ledger/sync.go SyncRecord() enqueues_job apps/workflows/src/record-sync.ts RecordSyncWorker.run()' + ))).toBe(true) }) - it('states the failed-check transport handoff across Go owners', () => { + // The negative half of the same control: identical nodes, no typed edge. + // Nothing in the text may be allowed to produce the handoff claim. + it('states no queue handoff when no typed edge connects the two owners', () => { const pack = compileContextPack({ task_contract: classifyTaskContract('explain', { - budget: 120, - prompt: 'Trace a failed monitor check into the workflow.', + budget: 200, + prompt: 'Trace a failed check into the workflow.', }), nodes: [ nodeCandidate({ - node_id: 'http_checker_handler', - label: '.HTTPCheckerHandler()', - source_file: 'apps/checker/handlers/checker.go', - line_number: 47, + node_id: 'producer', + label: 'SyncRecord()', + source_file: 'apps/ledger/sync.go', + line_number: 12, file_type: 'code', - snippet: 'checker.UpdateStatus(ctx, checker.UpdateData{ Status: "error", MonitorId: req.MonitorID })', - match_score: 10, + snippet: 'queueClient.Dispatch(ctx, req)', + match_score: 9, relevance_band: 'direct', community: 0, }, 'primary', 20), nodeCandidate({ - node_id: 'update_status', - label: 'UpdateStatus()', - source_file: 'apps/checker/checker/update.go', - line_number: 29, + node_id: 'consumer', + label: 'RecordSyncWorker.run()', + source_file: 'apps/workflows/src/record-sync.ts', + line_number: 30, file_type: 'code', - snippet: 'client, err := cloudtasks.NewClient(ctx); _, err = client.CreateTask(ctx, req)', - match_score: 9, + snippet: 'await persistRecord(payload)', + match_score: 8, relevance_band: 'direct', - community: 0, + community: 1, }, 'primary', 20), ], }) - expect(pack.claims[0]).toEqual({ - evidence_class: 'primary', - text: 'failure detection: apps/checker/handlers/checker.go .HTTPCheckerHandler() sends Status "error" to UpdateStatus', - node_labels: ['.HTTPCheckerHandler()'], - }) - expect(pack.claims[1]).toEqual({ - evidence_class: 'primary', - text: 'cross-runtime handoff: apps/checker/checker/update.go UpdateStatus() enqueues the checker status update with Cloud Tasks', - node_labels: ['UpdateStatus()'], - }) + expect(pack.claims.some((claim) => claim.text.startsWith('queue handoff:'))).toBe(false) }) it('keeps the same selected labels while task-aware rendering changes representation cost after selection', () => { diff --git a/tests/unit/pack-quality-fixtures.test.ts b/tests/unit/pack-quality-fixtures.test.ts index 338f1c52..5dd7cbd9 100644 --- a/tests/unit/pack-quality-fixtures.test.ts +++ b/tests/unit/pack-quality-fixtures.test.ts @@ -217,17 +217,17 @@ describe('pack-quality fixtures (#298)', () => { 'saveStructuredReport()', ]), ) + // #660-B1. Was asserting the planner / external_research_or_api / + // report_builder vocabulary. Those phases were only ever enabled when a + // prompt classifier recognised one benchmark task; the classifier is gone, + // so they are neither expected nor observed. The phases below are derived + // structurally from the execution steps and are what remains true. expect(payload.pack?.execution_slice?.phase_coverage).toEqual(expect.objectContaining({ - expected: expect.arrayContaining([ - 'planner', - 'external_research_or_api', - 'report_builder', - 'persistence', - ]), observed: expect.arrayContaining([ - 'planner', - 'external_research_or_api', - 'report_builder', + 'controller', + 'service', + 'queue', + 'worker', 'persistence', ]), missing: [], diff --git a/tests/unit/production-independence.test.ts b/tests/unit/production-independence.test.ts new file mode 100644 index 00000000..8686e9f8 --- /dev/null +++ b/tests/unit/production-independence.test.ts @@ -0,0 +1,725 @@ +/** + * #660-B -- production independence from qualification-repository knowledge. + * + * The forbidden-knowledge scanner owns the literal half of this guarantee and + * cannot own the rest: a rule keyed on prompt vocabulary, or one that forces a + * candidate into the result, encodes a qualification task without containing + * any name a scanner could look for. That class is owned here, behaviourally. + * + * The sharpest control in this file is the pair C/D: two graphs with identical + * structure and completely different names must rank identically, and two + * graphs with identical names and different structure must not. Together they + * say retrieval follows the evidence, not the vocabulary. + */ +import { readFileSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import type { ContextPackNode } from '../../src/contracts/context-pack.js' +import { KnowledgeGraph } from '../../src/contracts/graph.js' +import { + classifyTaskContract, + compileContextPack, + type ContextPackNodeCandidate, +} from '../../src/runtime/context-pack.js' +import { planConceptualFallback } from '../../src/runtime/retrieve/conceptual-fallback.js' +import { retrieveContext } from '../../src/runtime/retrieve.js' +import { + analyzeForbiddenKnowledge, + buildProductionSourceIndex, + decodeEscapes, + knowledgeBearingSites, + loadForbiddenKnowledgeManifest, + SCANNER_CAPABILITIES, +} from '../../scripts/lib/forbidden-knowledge.mjs' +import { productionSourceFiles } from '../../scripts/lib/grader-boundary.mjs' + +/** + * The production index, built ONCE for this suite. + * + * Every assertion below reads the same index rather than re-parsing the tree, + * which is the point of the one-pass design and also what keeps this file + * inside the protected control's time budget under coverage instrumentation. + */ +let cachedIndex: ReturnType | null = null +const readProductionFile = (file: string): string => readFileSync(resolve(process.cwd(), file), 'utf8') +function sharedIndex(): ReturnType { + cachedIndex ??= buildProductionSourceIndex({ + files: productionSourceFiles(process.cwd()), + readFile: readProductionFile, + }) + return cachedIndex +} + +/** Claim prefixes that were produced by fixed, repository-keyed builders. */ +const REMOVED_FIXED_CLAIM_PREFIXES = [ + 'public runtime provenance:', + 'public payload divergence:', + 'failure detection:', + 'cross-runtime handoff:', +] + +interface NodeSpec { + id: string + label: string + source: string + snippet?: string + frameworkRole?: string +} + +interface EdgeSpec { + from: string + to: string + relation: string +} + +function buildGraph(nodes: readonly NodeSpec[], edges: readonly EdgeSpec[]): KnowledgeGraph { + const graph = new KnowledgeGraph({ directed: true }) + for (const node of nodes) { + graph.addNode(node.id, { + label: node.label, + source_file: node.source, + source_location: 'L1-L3', + file_type: 'code', + node_kind: 'function', + type: 'function', + ...(node.snippet !== undefined ? { snippet: node.snippet } : {}), + ...(node.frameworkRole !== undefined ? { framework_role: node.frameworkRole } : {}), + }) + } + for (const edge of edges) { + graph.addEdge(edge.from, edge.to, { relation: edge.relation }) + } + return graph +} + +const LOW_QUALITY = { + selected_nodes: 0, + workflow_coherence: 0, + missing_required_evidence: 2, + missing_semantic_categories: 2, + direct_nodes: 0, + source_files: 0, +} as never + +function boostOrder(graph: KnowledgeGraph, question: string): string[] { + const proposal = planConceptualFallback(graph, { + question, + initialQuality: LOW_QUALITY, + selectedNodes: [], + }) + return [...(proposal?.nodeBoosts ?? new Map())] + .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])) + .map(([id]) => id) +} + +function candidate( + entry: ContextPackNode, + tokenCost: number, +): ContextPackNodeCandidate { + return { + label: entry.label, + ...(typeof entry.node_id === 'string' ? { node_id: entry.node_id } : {}), + community: entry.community ?? null, + source_file: entry.source_file, + line_number: entry.line_number, + ...(typeof entry.file_type === 'string' ? { file_type: entry.file_type } : {}), + ...(typeof entry.snippet === 'string' ? { snippet: entry.snippet } : {}), + evidence_class: 'primary', + estimate_tokens: () => tokenCost, + build_entry: () => ({ ...entry, evidence_class: 'primary' }), + } +} + +function packFor( + nodes: ReadonlyArray<{ id: string; label: string; source: string; snippet: string; frameworkRole?: string }>, + relationships: ReadonlyArray<{ from: string; to: string; relation: string }>, + prompt: string, +) { + return compileContextPack({ + task_contract: classifyTaskContract('explain', { budget: 240, prompt }), + nodes: nodes.map((node, index) => candidate({ + node_id: node.id, + label: node.label, + source_file: node.source, + line_number: 10 + index, + snippet: node.snippet, + file_type: 'code', + match_score: 9 - index, + relevance_band: 'direct', + community: 0, + ...(node.frameworkRole !== undefined ? { framework_role: node.frameworkRole } : {}), + }, 20)), + relationships: relationships.map((edge) => ({ + from_id: edge.from, + from: nodes.find((node) => node.id === edge.from)?.label ?? edge.from, + to_id: edge.to, + to: nodes.find((node) => node.id === edge.to)?.label ?? edge.to, + relation: edge.relation, + })), + }) +} + +/* ------------------------------------------------------------------ * + * The two structures used by controls C and D. + * + * `QUALIFICATION_NAMES` uses the exact vocabulary of a qualification target. + * `UNRELATED_NAMES` is the same structure with every identifier replaced. + * `SAME_NAMES_OTHER_STRUCTURE` keeps the qualification names but wires them + * into a different shape. + * ------------------------------------------------------------------ */ + +/* ------------------------------------------------------------------ * + * The fixtures used by controls C and D. + * + * D is an exact isomorphism: every identifier AND every question word is + * substituted one-for-one (monitor->probe, check->scan, status->health, + * incident->entry, notification->alert, page->panel), so the lexical + * relationship between the question and the code is preserved. Ordinary + * query-term matching is therefore held constant and the only thing that could + * make the two rank differently is a rule that knows one of these vocabularies. + * ------------------------------------------------------------------ */ + +const QUALIFICATION_NAMES: NodeSpec[] = [ + { id: 'n1', label: '.HTTPCheckerHandler()', source: '/apps/checker/handlers/checker.go' }, + { id: 'n2', label: 'UpdateStatus()', source: '/apps/checker/checker/update.go' }, + { id: 'n3', label: 'createIncident()', source: '/apps/workflows/src/checker/incident.ts' }, + { id: 'n4', label: 'triggerNotifications()', source: '/apps/workflows/src/checker/alerting.ts' }, + { id: 'n5', label: 'statusPage.ts', source: '/packages/api/src/router/statusPage.ts' }, +] + +const SUBSTITUTED_NAMES: NodeSpec[] = [ + { id: 'n1', label: '.HTTPScannerHandler()', source: '/apps/scanner/handlers/scanner.go' }, + { id: 'n2', label: 'UpdateHealth()', source: '/apps/scanner/scanner/update.go' }, + { id: 'n3', label: 'createEntry()', source: '/apps/workflows/src/scanner/entry.ts' }, + { id: 'n4', label: 'triggerAlerts()', source: '/apps/workflows/src/scanner/alerting.ts' }, + { id: 'n5', label: 'healthPanel.ts', source: '/packages/api/src/router/healthPanel.ts' }, +] + +const SHARED_EDGES: EdgeSpec[] = [ + { from: 'n1', to: 'n2', relation: 'calls' }, + { from: 'n2', to: 'n3', relation: 'calls' }, + { from: 'n3', to: 'n4', relation: 'calls' }, + { from: 'n3', to: 'n5', relation: 'updates_slice' }, +] + +const QUALIFICATION_QUESTION = 'Trace how a failed monitor check becomes an incident, triggers notifications, and affects the public status-page status.' +// `state` is deliberately avoided as a substitute: it is already in the +// pre-existing QUERY_DIRECTIVE_TERMS list, so using it would strip a term on +// one side only and break the isomorphism for a reason unrelated to naming. +const SUBSTITUTED_QUESTION = 'Trace how a failed probe scan becomes an entry, triggers alerts, and affects the public health-panel health.' + +describe('production independence from qualification repositories', () => { + /* ---------------- A. unrelated repository with similar names ------------- */ + + it('A. gives a repository with qualification-like names but no typed evidence no fixed claim', () => { + // Every name below is one the removed rules keyed on. None of them may + // produce a claim, because nothing structural connects these nodes. + const pack = packFor( + [ + { id: 'a', label: 'statusPage', source: 'src/statusPage.ts', snippet: 'export const statusPage = 1' }, + { id: 'b', label: 'publicPage', source: 'src/publicPage.ts', snippet: 'export const publicPage = 2' }, + { id: 'c', label: 'UpdateStatus', source: 'src/UpdateStatus.ts', snippet: 'export function UpdateStatus() {}' }, + { id: 'd', label: 'CreateTask', source: 'src/CreateTask.ts', snippet: 'export function CreateTask() {}' }, + { id: 'e', label: 'HTTPCheckerHandler', source: 'src/checker.ts', snippet: 'export function HTTPCheckerHandler() {}' }, + { id: 'f', label: 'incident', source: 'src/incident.ts', snippet: 'export const incident = 3' }, + ], + [], + 'Explain the router, checker and incident handling.', + ) + + for (const prefix of REMOVED_FIXED_CLAIM_PREFIXES) { + expect( + pack.claims.some((claim) => claim.text.startsWith(prefix)), + `a repository that merely uses these names received the claim "${prefix}"`, + ).toBe(false) + } + // With no typed relationship and no framework role, no structural claim is + // earned at all. + expect(pack.claims.some((claim) => claim.text.startsWith('queue handoff:'))).toBe(false) + expect(pack.claims.some((claim) => claim.text.startsWith('runtime boundary:'))).toBe(false) + }) + + /* ---------------- B. renamed implementation ----------------------------- */ + + it('B. recovers the producer/consumer relationship after every name is changed', () => { + const pack = packFor( + [ + { id: 'producer', label: 'SyncRecord()', source: 'apps/ledger/sync.go', snippet: 'queueClient.Dispatch(ctx, req)' }, + { id: 'consumer', label: 'RecordSyncWorker.run()', source: 'apps/workflows/src/record-sync.ts', snippet: 'await persistRecord(payload)' }, + ], + [{ from: 'producer', to: 'consumer', relation: 'enqueues_job' }], + 'Trace a failed check into the workflow.', + ) + + expect(pack.claims.some((claim) => claim.text === ( + 'queue handoff: apps/ledger/sync.go SyncRecord() enqueues_job apps/workflows/src/record-sync.ts RecordSyncWorker.run()' + ))).toBe(true) + }) + + it('B2. recovers a framework-declared route handler after every name is changed', () => { + // POSITIVE half. Every identifier is unrelated to any qualification target + // and the role is one an extractor declares from framework structure, so a + // passing assertion means the claim was earned by that declared role rather + // than by any name. Asserting the exact text matters: an earlier version of + // this control supplied no role and asserted only the negative below, which + // meant deleting the claim builder outright left it green. + const pack = packFor( + [ + { + id: 'route', + label: 'listCatalogEntries()', + source: 'apps/portal/src/routes/catalog.ts', + snippet: 'export async function listCatalogEntries() {}', + frameworkRole: 'fastify_route', + }, + ], + [], + 'Explain the public catalog endpoint.', + ) + + expect(pack.claims.map((claim) => claim.text)).toContain( + 'runtime boundary: apps/portal/src/routes/catalog.ts listCatalogEntries() is a framework-declared fastify_route', + ) + }) + + it('B2b. claims no runtime boundary from a route-shaped path with no declared role', () => { + // NEGATIVE half, kept separate. A path that looks like a route is not + // evidence; the role has to come from an extractor. + const pack = packFor( + [ + { id: 'route', label: 'GET()', source: 'apps/portal/src/app/api/summary/route.ts', snippet: 'export async function GET() {}' }, + ], + [], + 'Explain the public summary endpoint.', + ) + expect(pack.claims.some((claim) => claim.text.startsWith('runtime boundary:'))).toBe(false) + }) + + /* ---------------- C / D. names vs structure ----------------------------- */ + + it('D. ranks a one-for-one substituted repository exactly as it ranks the original', () => { + const qualification = boostOrder(buildGraph(QUALIFICATION_NAMES, SHARED_EDGES), QUALIFICATION_QUESTION) + const substituted = boostOrder(buildGraph(SUBSTITUTED_NAMES, SHARED_EDGES), SUBSTITUTED_QUESTION) + + // Node ids are shared between the two fixtures on purpose: only the labels, + // paths and question words differ, and they differ by the same substitution. + // An identical ordering means ranking followed structure and generic + // question-to-code matching. Any rule that recognises one vocabulary breaks + // this, which is exactly what the removed rules did. + expect(substituted, `qualification ${JSON.stringify(qualification)} vs substituted ${JSON.stringify(substituted)}`) + .toEqual(qualification) + }) + + it('C. does not give the qualification names their former treatment under a different structure', () => { + const connected = boostOrder(buildGraph(QUALIFICATION_NAMES, SHARED_EDGES), QUALIFICATION_QUESTION) + // Same names, no edges at all: a name-driven rule would rank them the same + // way regardless. A structural one cannot. + const disconnected = boostOrder(buildGraph(QUALIFICATION_NAMES, []), QUALIFICATION_QUESTION) + + expect(disconnected).not.toEqual(connected) + }) + + /* ---------------- D2. the same isomorphism, end to end ------------------ */ + + it('D2. selects the same nodes end-to-end for a one-for-one substituted repository', () => { + // D observes conceptual boosts. This observes what retrieval actually + // RETURNS, under a requested slice-v1, so a forced selection or a slice + // bypass added anywhere on the retrieval path is caught here and not only + // in the fallback planner. + // + // The assertion is on final MEMBERSHIP, not ordering. A reorder that leaves + // the selected set intact is not evidence of contamination, and treating it + // as such would make this control fire on noise. + const observe = (nodes: readonly NodeSpec[], question: string) => { + const result = retrieveContext(buildGraph(nodes, SHARED_EDGES), { + question, + budget: 5000, + retrievalStrategy: 'slice-v1', + }) + return { + ids: result.matched_nodes.map((node) => node.node_id ?? node.label), + membership: [...new Set(result.matched_nodes.map((node) => node.node_id ?? node.label))].sort(), + strategy: result.retrieval_strategy, + obligations: result.retrieval_plan?.query_obligations?.total ?? 0, + covered: result.retrieval_plan?.query_obligations?.finally_covered ?? 0, + } + } + + const qualification = observe(QUALIFICATION_NAMES, QUALIFICATION_QUESTION) + const substituted = observe(SUBSTITUTED_NAMES, SUBSTITUTED_QUESTION) + + // Measurement seam for the semantic falsifiability harness. It needs the + // SET this control actually observed, so that "the injection changed final + // membership" is a measured premise rather than an assumption. Written + // before the assertions so a failing run still reports what it saw. + const membershipOut = process.env.MADAR_D2_MEMBERSHIP_OUT + if (membershipOut !== undefined && membershipOut.length > 0) { + writeFileSync(membershipOut, JSON.stringify({ qualification, substituted }), 'utf8') + } + + // Explicit, truthful baseline. These are the measured values on a clean + // tree; they are asserted so that a mutation cannot quietly satisfy the + // comparison below by emptying both sides. + expect(qualification.strategy).toBe('slice-v1') + expect(substituted.strategy).toBe('slice-v1') + // Measured, not hoped for. n5 (the cross-package router node) is dropped by + // the requested slice now that no recovery mode opts out of slicing, and n1 + // is never selected. Both are asserted so the comparison below cannot be + // satisfied by a mutation that empties both sides. + expect(qualification.membership).toEqual(['n2', 'n3', 'n4']) + expect(qualification.obligations).toBe(4) + expect(qualification.membership).not.toContain('n1') + expect(substituted.membership).not.toContain('n1') + expect(qualification.membership).not.toContain('n5') + // Nothing may pin an entry into the payload because of a repository path. + // The falsifiability harness injects exactly this id, so a clean tree must + // not contain it and the injected tree must. + expect(qualification.membership).not.toContain('h4-forced-membership') + + expect( + substituted.membership, + `qualification ${JSON.stringify(qualification)} vs substituted ${JSON.stringify(substituted)}`, + ).toEqual(qualification.membership) + expect(substituted.obligations).toBe(qualification.obligations) + expect(substituted.covered).toBe(qualification.covered) + }) + + /* ---------------- E. qualification strings alone ------------------------ */ + + it('E. produces no fixed claim from the historical strings without supporting evidence', () => { + // The exact snippets the removed builders matched on, with no relationships. + const pack = packFor( + [ + { + id: 'x', + label: 'unresolvedIncidents()', + source: 'apps/status-page/src/content/status-json.ts', + snippet: [ + 'type Page = NonNullable;', + 'status: pageIndicator(page.status),', + 'return page.statusReports.filter((report) => report.status !== "resolved")', + ].join('\n'), + }, + { + id: 'y', + label: 'statusPage.ts', + source: 'packages/api/src/router/statusPage.ts', + snippet: 'events.some((e) => e.type === "incident" && !e.to) && barType !== "manual" ? "error" : activeReportStatus(events)', + }, + { + id: 'z', + label: 'UpdateStatus()', + source: 'apps/checker/checker/update.go', + snippet: 'client, err := cloudtasks.NewClient(ctx)\n_, err = client.CreateTask(ctx, req)', + }, + ], + [], + 'Identify inconsistent public status computation paths.', + ) + + for (const prefix of REMOVED_FIXED_CLAIM_PREFIXES) { + expect( + pack.claims.some((claim) => claim.text.startsWith(prefix)), + `the historical strings alone still produced "${prefix}"`, + ).toBe(false) + } + }) + + /* ---------------- F. source attribution --------------------------------- */ + + it('F. cites a node the pack actually carries for every claim it emits', () => { + const pack = packFor( + [ + { id: 'producer', label: 'SyncRecord()', source: 'apps/ledger/sync.go', snippet: 'queueClient.Dispatch(ctx, req)' }, + { id: 'consumer', label: 'RecordSyncWorker.run()', source: 'apps/workflows/src/record-sync.ts', snippet: 'await persistRecord(payload)' }, + ], + [{ from: 'producer', to: 'consumer', relation: 'enqueues_job' }], + 'Trace a failed check into the workflow.', + ) + + const packLabels = new Set(pack.nodes.map((node) => node.label)) + expect(pack.claims.length).toBeGreaterThan(0) + for (const claim of pack.claims) { + expect(claim.node_labels.length, `claim cites nothing: ${claim.text}`).toBeGreaterThan(0) + for (const label of claim.node_labels) { + expect(packLabels, `claim cites a node absent from the pack: ${claim.text}`).toContain(label) + } + } + }) + + /* ---------------- G. the literal scanner runs on every lane ------------- */ + + describe('G. forbidden-knowledge scanner', () => { + it('accepts a valid manifest and reports rules from both the file and the frozen contract', () => { + const manifest = loadForbiddenKnowledgeManifest(process.cwd()) + expect(manifest.problems).toEqual([]) + expect(manifest.ok).toBe(true) + expect(manifest.rules.length).toBeGreaterThan(0) + // Rules imported from the frozen contract keep the manifest in step with + // the pinned corpus instead of drifting from it. + expect(manifest.rules.some((rule: { origin: string }) => rule.origin.includes('corpus.json'))).toBe(true) + expect(manifest.exceptions).toEqual([]) + }) + + // #660-B1. The scan used to renormalize every site for every rule, which + // cost sites x rules string transformations and timed the control out at + // 15s on CI. These four assertions are the standing proof that the one-pass + // index is real: sources are read, parsed and normalized once, and the rule + // count no longer drives the work. + it('parses each production file exactly once, whatever the rule count', () => { + const files = productionSourceFiles(process.cwd()) + const index = sharedIndex() + const manifest = loadForbiddenKnowledgeManifest(process.cwd()) + + expect(new Set(files).size).toBe(files.length) + expect(index.stats.parseCalls).toBe(files.length) + expect(index.stats.indexedFiles).toBe(files.length) + + // The same index answers both scans, and halving the rules changes + // neither the source work done nor the verdict. Rules are applied to the + // index; they never drive another pass over the tree. + const full = analyzeForbiddenKnowledge({ root: process.cwd(), index }) + const halved = analyzeForbiddenKnowledge({ + root: process.cwd(), + index, + manifest: { ...manifest, rules: manifest.rules.slice(0, Math.floor(manifest.rules.length / 2)) }, + }) + expect(halved.stats).toEqual(full.stats) + expect(halved.stats.parseCalls).toBe(files.length) + expect(full.violations).toEqual([]) + expect(halved.violations).toEqual([]) + }) + + it('never parses a file twice, even when the file list repeats one', () => { + // A small slice is enough: de-duplication is a property of the index, not + // of how many files it was handed. + const files = productionSourceFiles(process.cwd()).slice(0, 5) + const duplicated = [...files, ...files] + const index = buildProductionSourceIndex({ files: duplicated, readFile: readProductionFile }) + expect(duplicated.length).toBe(files.length * 2) + expect(index.stats.parseCalls).toBe(files.length) + expect(index.byFile.size).toBe(files.length) + }) + + it('represents every production file in the index', () => { + const files = productionSourceFiles(process.cwd()) + const index = sharedIndex() + for (const file of files) { + expect(index.byFile.has(file), `missing from index: ${file}`).toBe(true) + } + }) + + it('reports the same result whatever order the rules are declared in', () => { + const manifest = loadForbiddenKnowledgeManifest(process.cwd()) + const reversed = { ...manifest, rules: [...manifest.rules].reverse() } + const index = sharedIndex() + const forward = analyzeForbiddenKnowledge({ root: process.cwd(), index, manifest }) + const backward = analyzeForbiddenKnowledge({ root: process.cwd(), index, manifest: reversed }) + expect(backward.ok).toBe(forward.ok) + expect(backward.violations).toEqual(forward.violations) + }) + + // #660-B1. A legacy octal escape is executable inside a regex: + // /\163tatusPage/ runs as /statusPage/. Both directions are asserted, so a + // decoder that silently stopped working could not pass this. + it('decodes a legacy octal escape that the raw spelling hides', () => { + const raw = String.raw`/\163tatusPage/i` + expect(raw).not.toContain('statusPage') + expect(decodeEscapes(raw)).toContain('statusPage') + + const result = analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => `const probe = ${raw}\n`, + }) + const hit = result.violations.find((violation: { rule: string }) => violation.rule === 'openstatus/symbol-status-page') + expect(hit, 'legacy octal escape was not classified').toBeDefined() + expect(hit?.raw).toContain('163') + expect(hit?.decoded).toContain('statusPage') + }) + + it('leaves a regex backreference alone rather than inventing a name', () => { + // \1..\9 decode to control characters, not printable text, so they stay + // backreferences. Resolving the ambiguity the other way would fabricate + // matches in ordinary regexes. + expect(decodeEscapes(String.raw`(status)\1`)).not.toContain('statusPage') + }) + + // #660-B1. The scanner is a bounded decoded-literal detector. These four + // assertions pin that contract from both sides: the encodings it must + // catch, the rule classes it must treat alike, and -- explicitly -- the + // edge it does not cover, so nobody later reads more into a green scan + // than it means. + + it('catches a forbidden name written in every decoded escape form', () => { + const forms = { + plain: String.raw`/statusPage/i`, + hex: String.raw`/\x73tatusPage/i`, + unicode: String.raw`/\u0073tatusPage/i`, + codePoint: String.raw`/\u{73}tatusPage/iu`, + legacyOctal: String.raw`/\163tatusPage/i`, + } + for (const [name, literal] of Object.entries(forms)) { + const result = analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => `const probe = ${literal}\n`, + }) + const hit = result.violations.find((violation: { rule: string }) => violation.rule === 'openstatus/symbol-status-page') + expect(hit, `${name} escape form was not classified`).toBeDefined() + // Both spellings are retained, so a reader can see what was written and + // what it means without re-deriving the decode. + expect(hit?.raw).toContain('statusPage'.slice(-4)) + expect(hit?.decoded).toContain('statusPage') + expect(hit?.site).toBe('regex') + expect(hit?.line).toBe(1) + } + }) + + it('applies path rules and symbol rules alike to regex source', () => { + // The earlier implementation evaluated regex sites for symbol rules only. + // That exclusion must not survive: a regex source is just text, and every + // rule class is tested against it. + const pathHit = analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => `const probe = ${String.raw`/packages\/api\/src\/router\/statusPage\.ts/i`}\n`, + }).violations.find((violation: { rule: string }) => violation.rule === 'openstatus/path-router-status-page') + expect(pathHit, 'a path rule was not applied to a regex source site').toBeDefined() + + const escapedPathHit = analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => `const probe = ${String.raw`/lib\/http\/\x65tag/i`}\n`, + }).violations.find((violation: { rule: string }) => violation.rule === 'openstatus/path-lib-http-etag') + expect(escapedPathHit, 'an escaped path rule was not applied to a regex source site').toBeDefined() + }) + + it('declares arbitrary regex semantics outside the decoded-literal scanner contract', () => { + // The scanner's capability boundary, machine-enforced rather than + // documented. It is a bounded decoded-literal detector: it reads regex + // SOURCE as text and decodes escapes, and it does not interpret what a + // pattern can match. Changing that requires deliberately changing this + // declaration and this test. + expect(SCANNER_CAPABILITIES).toEqual({ + literal_and_static_detection: true, + regex_semantic_evaluation: false, + runtime_constructed_value_proof: false, + semantic_overfitting_proof: false, + }) + + const classifies = (literal: string): boolean => analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => `const probe = ${literal}\n`, + }).violations.some((violation: { rule: string }) => violation.rule === 'openstatus/symbol-status-page') + + // Detected: the forbidden text is present in the source once decoded. + expect(classifies(String.raw`/statusPage/i`), 'direct literal source').toBe(true) + expect(classifies(String.raw`/\x73tatusPage/i`), 'hex-escaped source').toBe(true) + expect(classifies(String.raw`/\163tatusPage/i`), 'legacy-octal source').toBe(true) + + // Not interpreted as equivalent: the value exists only in what the + // pattern MATCHES, and no decoded run of the source spells it. These two + // are asserted individually rather than as a general claim about complex + // patterns -- a pattern may coincidentally contain the text directly, and + // would then be detected, correctly. + expect(classifies(String.raw`/statusP{1}age/i`), 'quantifier pattern').toBe(false) + expect(classifies(String.raw`/^(?=statusP{1}age$)/i`), 'lookahead pattern').toBe(false) + }) + + it('folds only the constructions whose JavaScript semantics are modelled', () => { + // Every row carries the source the scanner sees AND the same expression + // evaluated for real, so the fold is compared against what JavaScript + // actually produces rather than against what would be convenient. Two + // false positives were found this way and are pinned here: + // + // ['status', 'Page'] is an ARRAY, coercing to 'status,Page' + // 'status'.concat(...[, 'Page']) is 'statusundefinedPage', because a + // spread hole is undefined and + // String.prototype.concat stringifies it + // + // A hole's rendering depends on its consumer: join renders it as '', + // string concat renders it as the text 'undefined'. + const expectations: Array<{ source: string; evaluate: () => unknown; foldsToString: boolean }> = [ + { source: "'status' + 'Page'", evaluate: () => 'status' + 'Page', foldsToString: true }, + { source: "'status'.concat('Page')", evaluate: () => 'status'.concat('Page'), foldsToString: true }, + { source: "'status'.concat(...['Page'])", evaluate: () => 'status'.concat(...['Page']), foldsToString: true }, + { source: "['status', 'Page'].join('')", evaluate: () => ['status', 'Page'].join(''), foldsToString: true }, + { source: "['sta', ...['tus'], 'Page'].join('')", evaluate: () => ['sta', ...['tus'], 'Page'].join(''), foldsToString: true }, + { source: "['status', , 'Page'].join('')", evaluate: () => ['status', , 'Page'].join(''), foldsToString: true }, + { source: "['status', ...[, 'Page']].join('')", evaluate: () => ['status', ...[, 'Page']].join(''), foldsToString: true }, + { source: "[, 'Page'].join('')", evaluate: () => [, 'Page'].join(''), foldsToString: true }, + // Folds, but to what JavaScript really produces -- so the forbidden + // name is simply not present in the result. + { source: "'status'.concat(...[, 'Page'])", evaluate: () => 'status'.concat(...([, 'Page'] as unknown as string[])), foldsToString: true }, + // Not strings at all, so no fold. + { source: "['status', 'Page']", evaluate: () => ['status', 'Page'], foldsToString: false }, + { source: "['status'].concat('Page')", evaluate: () => ['status'].concat('Page'), foldsToString: false }, + ] + + for (const { source, evaluate, foldsToString } of expectations) { + const folded = knowledgeBearingSites(`const probe = ${source}\n`, 'probe.ts') + .filter((site: { kind: string }) => site.kind === 'folded') + .map((site: { text: string }) => site.text) + const runtime = evaluate() + + if (!foldsToString) { + expect(Array.isArray(runtime), `${source} should evaluate to an array`).toBe(true) + expect(folded, `${source} evaluates to an array and must not fold to a string`).toEqual([]) + continue + } + + expect(typeof runtime, `${source} should evaluate to a string`).toBe('string') + // The decisive assertion: the scanner's folded value must be exactly + // what JavaScript produces. A fold that disagrees is a false positive, + // whichever direction it errs in. + expect(folded, `${source} folded to something JavaScript does not produce`).toContain(runtime) + } + }) + + it('does not report a forbidden name for a construction that does not produce it', () => { + // The reproduction from the FINAL review, kept as a standing control. + const source = "'status'.concat(...[, 'Page'])" + // The assertion is type-level only -- TypeScript types a sparse spread + // element as string | undefined, which concat rejects. The runtime array + // keeps its hole, which is the whole point of the case. + expect('status'.concat(...([, 'Page'] as unknown as string[]))).toBe('statusundefinedPage') + + const result = analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => `const probe = ${source}\n`, + }) + expect(result.violations, 'a construction that never spells the name was reported').toEqual([]) + }) + + it('still scans literal components of an array it refuses to fold', () => { + // Declining to fold the array must not make its members invisible: a + // preferred-file list is exactly this shape. + const result = analyzeForbiddenKnowledge({ + root: process.cwd(), + files: ['probe.ts'], + readFile: () => "const preferred = ['statusPage', 'unrelated']\n", + }) + const hit = result.violations.find((violation: { rule: string }) => violation.rule === 'openstatus/symbol-status-page') + expect(hit, 'a literal inside an unfolded array escaped detection').toBeDefined() + expect(hit?.site).toBe('string') + }) + + it('finds no qualification-repository knowledge in production source', () => { + const result = analyzeForbiddenKnowledge({ root: process.cwd(), index: sharedIndex() }) + const detail = result.violations + .map((violation: { file: string; line: number; rule: string; raw: string }) => ( + `${violation.file}:${violation.line} [${violation.rule}] ${violation.raw}` + )) + .join('\n') + expect(result.violations.length, detail).toBe(0) + expect(result.unusedExceptions).toEqual([]) + expect(result.ok).toBe(true) + expect(result.filesScanned).toBeGreaterThan(0) + }) + }) +}) diff --git a/tests/unit/retrieve-conceptual-fallback.test.ts b/tests/unit/retrieve-conceptual-fallback.test.ts index c60f8fca..7e7237a7 100644 --- a/tests/unit/retrieve-conceptual-fallback.test.ts +++ b/tests/unit/retrieve-conceptual-fallback.test.ts @@ -407,34 +407,15 @@ describe('conceptual-query fallback planner', () => { expect(labels.filter((label) => label !== 'SearchStatusBadge').length / labels.length).toBeGreaterThanOrEqual(0.75) }) - it('reserves the cross-runtime caller that owns the first failure transition', () => { - const graph = new KnowledgeGraph({ directed: true }) - addNode(graph, 'failure-log', 'FailedMonitorLog', '/packages/services/monitor/failure-log.ts') - addNode(graph, 'status-update', 'UpdateStatus()', '/apps/checker/checker/update.go') - addNode(graph, 'http-checker', '.HTTPCheckerHandler()', '/apps/checker/handlers/checker.go', { - framework: 'gin', - framework_role: 'gin_handler', - }) - addNode(graph, 'incident', 'createIncident()', '/apps/workflows/checker/incident.ts') - addNode(graph, 'notifications', 'triggerNotifications()', '/apps/workflows/checker/alerting.ts') - addNode(graph, 'public-status', 'statusPage.ts', '/packages/api/router/statusPage.ts') - addNode(graph, 'alternate-status', 'computeOverallStatus()', '/apps/server/status-page/index.ts') - graph.addEdge('http-checker', 'status-update', { relation: 'calls' }) - graph.addEdge('status-update', 'incident', { relation: 'calls' }) - graph.addEdge('incident', 'notifications', { relation: 'calls' }) - graph.addEdge('public-status', 'incident', { relation: 'uses' }) - graph.addEdge('alternate-status', 'public-status', { relation: 'references' }) - - const proposal = planConceptualFallback(graph, { - question: 'Trace how a failed monitor check becomes an incident, triggers notifications, and affects the public status-page status. Identify inconsistent status-computation paths.', - initialQuality: lowQuality(), - selectedNodes: [], - }) - - expect(proposal.preferredObligationAnchors?.get(0)).toBe('http-checker') - expect(proposal.nodeBoosts.get('http-checker')).toBeGreaterThanOrEqual(9) - expect(proposal.nodeBoosts.get('status-update')).toBeGreaterThanOrEqual(9) - }) + // #660-B. Two tests were removed here. One pinned a reservation gated on the + // prompt containing both 'public' and 'page'; the other pinned a forced + // cross-runtime selection that also overwrote an obligation's preferred + // anchor. Both are gone from production. Measured consequence, recorded + // rather than papered over: with no generic signal reserving it, a + // framework-declared route handler is no longer force-selected into the + // proposal. That is the intended removal of an unsupported selection, not a + // replacement -- see tests/unit/production-independence.test.ts for the + // relationships that ARE recovered from typed evidence after renaming. it('reserves workflow-local creation and delivery owners for natural wording', () => { const graph = new KnowledgeGraph({ directed: true }) @@ -465,34 +446,6 @@ describe('conceptual-query fallback planner', () => { expect(proposal.nodeBoosts.has('test-delivery')).toBe(false) }) - it('reserves the public HTTP boundary separately from status computation owners', () => { - const graph = new KnowledgeGraph({ directed: true }) - addNode(graph, 'public-json-route', 'GET()', '/apps/status-page/src/app/api/status/[[...path]]/route.ts', { - framework_role: 'next_route_handler', - }) - addNode(graph, 'status-json', 'status-json.ts', '/apps/status-page/src/content/status-json.ts') - addNode(graph, 'public-status', 'statusPage.ts', '/packages/api/src/router/statusPage.ts') - addNode(graph, 'overall-status', 'computeOverallStatus()', '/apps/server/src/routes/status-page/index.ts') - addNode(graph, 'failed-check', '.HTTPCheckerHandler()', '/apps/checker/handlers/checker.go') - addNode(graph, 'incident-owner', 'createIncident()', '/apps/workflows/src/checker/incident.ts') - addNode(graph, 'notification-owner', 'triggerNotifications()', '/apps/workflows/src/checker/alerting.ts') - graph.addEdge('public-json-route', 'status-json', { relation: 'calls' }) - graph.addEdge('status-json', 'public-status', { relation: 'provides' }) - graph.addEdge('overall-status', 'public-status', { relation: 'references' }) - graph.addEdge('failed-check', 'incident-owner', { relation: 'calls' }) - graph.addEdge('incident-owner', 'notification-owner', { relation: 'calls' }) - graph.addEdge('incident-owner', 'public-status', { relation: 'updates_slice' }) - - const proposal = planConceptualFallback(graph, { - question: 'Trace how a failed monitor check becomes an incident, triggers notifications, and affects the public status-page status. Identify inconsistent status-computation paths.', - initialQuality: lowQuality(), - selectedNodes: [], - }) - - expect(proposal.nodeBoosts.get('public-json-route')).toBeGreaterThanOrEqual(9) - expect(proposal.nodeBoosts.has('public-status')).toBe(true) - expect(proposal.nodeBoosts.has('overall-status')).toBe(true) - }) it('caps every BFS neighbor read on a hub-heavy graph', () => { const graph = new KnowledgeGraph({ directed: true }) diff --git a/tests/unit/retrieve-cross-layer-flow.test.ts b/tests/unit/retrieve-cross-layer-flow.test.ts index 5c1afe72..c1c4767c 100644 --- a/tests/unit/retrieve-cross-layer-flow.test.ts +++ b/tests/unit/retrieve-cross-layer-flow.test.ts @@ -32,64 +32,16 @@ function writeCrossLayerGraphFixture(root: string): string { } describe('cross-layer flow retrieval', () => { - it('covers every flow obligation without letting presentation vocabulary dominate', () => { - const started = performance.now() - const result = retrieveContext(buildCrossLayerMonitorFlowFixture(), { - question: QUESTION, - budget: 1_800, - retrievalStrategy: 'slice-v1', - }) - const elapsedMs = performance.now() - started - const selectedFiles = new Set(result.matched_nodes.map((node) => node.source_file)) - const relevantSelected = [...selectedFiles].filter((file) => ( - CROSS_LAYER_MONITOR_FLOW_FILES.includes(file as typeof CROSS_LAYER_MONITOR_FLOW_FILES[number]) - )) - const precision = relevantSelected.length / Math.max(selectedFiles.size, 1) - const evidence = assessMadarResponseEvidence({ - evidencePlan: buildRetrievalEvidencePlanFromResult(result), - question: QUESTION, - recovery: result.recovery, - }) - const snippetCoverage = evaluateQueryEvidenceCoverage(QUESTION, result.matched_nodes) + // #660-B1. Removed. This asserted a multi-service selection that only the + // conceptual-recovery slice bypass produced. Instrumenting that bypass's + // trigger across the retrieval, conceptual-fallback, pack-quality and + // production-correctness suites showed it firing on exactly three + // questions, all qualification-shaped, and on no independent fixture -- so + // it was removed rather than defended, and with it the selection this + // asserted. Recorded here rather than deleted silently: a requested slice + // is now always applied, and a cross-service question with no explicit + // anchor is sliced to a local slice. - expect( - CROSS_LAYER_MONITOR_FLOW_FILES.every((file) => selectedFiles.has(file)), - JSON.stringify({ - selected: [...selectedFiles], - labels: result.matched_nodes.map((node) => node.label), - relationships: result.relationships, - retrievalPlan: result.retrieval_plan, - recovery: result.recovery, - }, null, 2), - ).toBe(true) - expect(precision).toBeGreaterThanOrEqual(0.7) - expect(result.matched_nodes.map((node) => node.label)).not.toContain('computeEffectiveStatus') - expect(result.relationships.length).toBeGreaterThanOrEqual(5) - expect(result.retrieval_plan).toMatchObject({ - status: 'recovered', - reasons: expect.arrayContaining(['missing_query_obligations']), - query_obligations: { - total: 5, - finally_covered: 5, - }, - attempts: [expect.objectContaining({ - status: 'applied', - promoted_communities: expect.arrayContaining([1, 2, 3, 4, 5, 6]), - })], - }) - expect(result.retrieval_plan?.query_obligations?.initially_covered).toBeLessThan( - result.retrieval_plan?.query_obligations?.finally_covered ?? 0, - ) - expect(result.retrieval_plan?.query_obligations).toMatchObject({ - total: snippetCoverage.total, - finally_covered: snippetCoverage.covered, - }) - expect(evidence.answerability.state).toMatch(/^ready(?:_with_caveat)?$/) - expect(evidence.answerability.broad_search_fallback).toBe('not_needed') - expect(evidence.agent_directive).toBe('answer_from_pack') - expect(result.token_count).toBeLessThanOrEqual(1_800) - expect(elapsedMs).toBeLessThan(750) - }) it('returns an answer-ready workflow through one context_pack MCP call', async () => { const fixtureParent = resolve('out', 'test-runtime') @@ -118,7 +70,7 @@ describe('cross-layer flow retrieval', () => { matched_nodes?: Array<{ label: string; source_file: string }> relationships?: unknown[] } - claims?: Array<{ text: string }> + claims?: Array<{ text: string; node_labels: string[] }> evidence?: { answerability?: { state?: string; broad_search_fallback?: string } agent_directive?: string @@ -140,10 +92,17 @@ describe('cross-layer flow retrieval', () => { 'packages/api/src/router/external-service/effective-status.ts', ) expect(payload.pack?.matched_nodes?.map((node) => node.label)).toContain('computeOverallStatus') - expect(payload.claims?.some((claim) => ( - claim.text.includes('treats an open incident event as "error" outside manual mode') - && claim.text.includes('derives overall status from active status reports and maintenance') - ))).toBe(true) + // #660-B. Was an assertion on one fixed claim string keyed on this + // repository's wording. What is worth holding is that every claim cites + // a node the pack actually carries -- source attribution, not prose. + const packLabels = new Set(payload.pack?.matched_nodes?.map((node) => node.label) ?? []) + expect((payload.claims?.length ?? 0)).toBeGreaterThan(0) + for (const claim of payload.claims ?? []) { + expect(claim.node_labels.length).toBeGreaterThan(0) + for (const label of claim.node_labels) { + expect(packLabels, `claim cites a node absent from the pack: ${claim.text}`).toContain(label) + } + } expect( payload.pack?.relationships?.length ?? 0, JSON.stringify(payload, null, 2), @@ -260,84 +219,11 @@ describe('cross-layer flow retrieval', () => { } }) - it('keeps the derived monitor-status owner attached to the public page rollup', () => { - const fixtureParent = resolve('out', 'test-runtime') - mkdirSync(fixtureParent, { recursive: true }) - const root = mkdtempSync(join(fixtureParent, 'madar-status-rollup-')) - const sourceFile = join(root, 'statusPage.ts') - try { - writeFileSync(sourceFile, [ - 'export const statusPageRouter = createTRPCRouter({', - ' get: publicProcedure.query(async () => {', - ' const monitorComponents = page.components.filter(isMonitorComponent)', - ' const monitors = monitorComponents.map((c) => {', - ' const events = getEvents({ incidents: c.monitor.incidents })', - ' const status =', - ' events.some((e) => e.type === "incident" && !e.to) &&', - ' barType !== "manual"', - ' ? "error"', - ' : "success";', - ' return {', - ' ...c.monitor,', - ' status,', - ' events,', - ' }', - ' })', - ' const status = monitors.some((m) => m.status === "error")', - ' ? "error"', - ' : "success"', - ' return { ...page, monitors, status }', - ' }),', - '})', - ].join('\n'), 'utf8') - - const evidence = readQueryEvidenceSnippet(sourceFile, 1, { - question: QUESTION, - label: 'statusPage.ts', - sourceLocation: 'L1-L22', - fileNodeLike: true, - }) - - expect(evidence?.snippet).toContain('monitorComponents.map') - expect(evidence?.snippet).toContain('...c.monitor') - expect(evidence?.snippet).toContain('e.type === "incident"') - expect(evidence?.snippet).toContain('monitors.some') - } finally { - rmSync(root, { recursive: true, force: true }) - } - }) - - it('keeps the tRPC output provenance attached to machine-status serialization', () => { - const fixtureParent = resolve('out', 'test-runtime') - mkdirSync(fixtureParent, { recursive: true }) - const root = mkdtempSync(join(fixtureParent, 'madar-status-json-')) - const sourceFile = join(root, 'status-json.ts') - try { - writeFileSync(sourceFile, [ - 'import type { RouterOutputs } from "@openstatus/api"', - 'type Page = NonNullable', - 'export function toStatus(page: Page) {', - ' return { status: pageIndicator(page.status) }', - '}', - 'export function unresolvedIncidents(page: Page) {', - ' return page.statusReports.filter((report) => report.status !== "resolved")', - '}', - ].join('\n'), 'utf8') - - const evidence = readQueryEvidenceSnippet(sourceFile, 1, { - question: QUESTION, - label: 'status-json.ts', - sourceLocation: 'L1-L8', - fileNodeLike: true, - }) - - expect(evidence?.snippet).toContain('RouterOutputs["statusPage"]["get"]') - expect(evidence?.snippet).toContain('pageIndicator(page.status)') - expect(evidence?.snippet).toContain('page.statusReports') - } finally { - rmSync(root, { recursive: true, force: true }) - } - }) + // #660-B. Two tests were removed here. Their only subjects were + // `incidentStatusOwnerFragment` and `QUERY_EVIDENCE_INPUT_PROVENANCE_PATTERN`, + // which recognised one qualification repository's exact source lines. Both + // are gone from production, so a test asserting they fire would assert a + // behaviour that no longer exists rather than a property worth holding. it('keeps the public route fetch beside the status JSON serializer dispatch', () => { const fixtureParent = resolve('out', 'test-runtime') @@ -411,33 +297,37 @@ describe('cross-layer flow retrieval', () => { } }) + // #660-B. The subject is the generic provider-handoff shape: a transport is + // constructed, then a delivery call is made on it. Every identifier here is + // deliberately unrelated to any qualification target, so passing proves the + // fragment extractor follows the code shape rather than remembered names. it('keeps the concrete transport provider beside a delivery handoff', () => { const fixtureParent = resolve('out', 'test-runtime') mkdirSync(fixtureParent, { recursive: true }) const root = mkdtempSync(join(fixtureParent, 'madar-query-provider-')) - const sourceFile = join(root, 'update.go') + const sourceFile = join(root, 'sync.go') try { writeFileSync(sourceFile, [ - 'package checker', + 'package ledger', '', - 'func UpdateStatus(ctx context.Context, updateData UpdateData) error {', - ' url := "https://workflows.example/updateStatus"', - ' client, err := cloudtasks.NewClient(ctx, option.WithAuthCredentials(creds))', + 'func SyncRecord(ctx context.Context, payload RecordPayload) error {', + ' url := "https://workflows.example/syncRecord"', + ' queueClient, err := messaging.NewQueueClient(ctx, option.WithAuthCredentials(creds))', ' if err != nil { return err }', - ' req := &taskspb.CreateTaskRequest{Parent: queuePath}', - ' _, err = client.CreateTask(ctx, req)', + ' req := &messagingpb.DispatchRequest{Parent: queuePath}', + ' _, err = queueClient.Dispatch(ctx, req)', ' return err', '}', ].join('\n'), 'utf8') const evidence = readQueryEvidenceSnippet(sourceFile, 3, { question: QUESTION, - label: 'UpdateStatus()', + label: 'SyncRecord()', sourceLocation: 'L3', }) - expect(evidence?.snippet).toContain('cloudtasks.NewClient') - expect(evidence?.snippet).toContain('client.CreateTask') + expect(evidence?.snippet).toContain('messaging.NewQueueClient') + expect(evidence?.snippet).toContain('queueClient.Dispatch') } finally { rmSync(root, { recursive: true, force: true }) } diff --git a/tests/unit/retrieve-production-correctness.test.ts b/tests/unit/retrieve-production-correctness.test.ts index f3169a4e..20e6f4fa 100644 --- a/tests/unit/retrieve-production-correctness.test.ts +++ b/tests/unit/retrieve-production-correctness.test.ts @@ -431,7 +431,13 @@ describe('retrieveContext production retrieval regressions', () => { ])) }) - it('compacts broad report-generation packs around the execution flow instead of status/title noise', () => { + // #660-B1. Was also asserting that '.getStatusMessage()' and + // '.generateTitle()' are absent. That exclusion was produced by a denylist of + // fourteen symbol names lifted from one repository, not by any structural + // property, so it is gone and those labels can appear again. What is still + // worth holding, and is kept, is that the execution flow itself is compacted + // around the traced steps. + it('compacts a broad pack around the traced execution flow', () => { const compact = compactRetrieveResult(retrieveContext(buildReverseFlowReportGenerationGraph(), { question: 'How idea report is being generated', budget: 4000, @@ -448,85 +454,19 @@ describe('retrieveContext production retrieval regressions', () => { '.assembleReport()', '.scoreMetrics()', ])) - expect(labels).not.toEqual(expect.arrayContaining([ - '.getStatusMessage()', - '.generateTitle()', - ])) }) - it('keeps low-confidence report-generation slices honest when no runtime handoff was traced', () => { - const compact = compactRetrieveResult(retrieveContext(buildBroadReportGenerationGraph(), { - question: 'How idea report is being generated', - budget: 4000, - retrievalLevel: 4, - retrievalStrategy: 'slice-v1', - })) + // #660-B1. Two tests were removed here. Both asserted the EXPECTED phase + // vocabulary of one benchmark task -- planner, external_research_or_api, + // report_builder, scoring, quality_gate -- which production only enabled + // when a prompt classifier recognised that task. The classifier and the + // phase list it gated are gone, so the tests asserted a vocabulary that no + // longer exists rather than a property worth holding. + // + // The phases that ARE still derived structurally from execution steps + // (controller, service, queue, worker, persistence) remain covered by the + // surrounding tests. The remaining half of the report-generation class + // lives in the Slice-C files and is untouched here; Slice C re-derives any + // expectation that should survive it. - expect(compact.execution_slice?.confidence).toBe('low') - expect(compact.execution_slice?.confidence_reasons).toEqual(expect.arrayContaining([ - 'no_runtime_handoff', - ])) - expect(compact.execution_slice?.status).toBe('partial') - expect(compact.execution_slice?.phase_coverage).toEqual(expect.objectContaining({ - missing: expect.arrayContaining([ - 'external_research_or_api', - 'report_builder', - 'scoring', - 'quality_gate', - 'renderer_or_synthesis', - 'persistence', - ]), - })) - expect(compact.execution_slice?.phase_coverage?.observed).not.toContain('external_research_or_api') - expect(compact.execution_slice?.phase_coverage?.observed).not.toContain('report_builder') - expect(compact.execution_slice?.phase_coverage?.observed).not.toContain('scoring') - expect(compact.execution_slice?.phase_coverage?.observed).not.toContain('quality_gate') - expect(compact.execution_slice?.phase_coverage?.observed).not.toContain('renderer_or_synthesis') - expect(compact.execution_slice?.phase_coverage?.observed).not.toContain('persistence') - expect(compact.answer_contract).toEqual(expect.objectContaining({ - observed_phases: compact.execution_slice?.phase_coverage?.observed, - missing_phases: compact.execution_slice?.phase_coverage?.missing, - do_not_claim: expect.arrayContaining([ - 'full_runtime_certainty_when_slice_is_partial', - ]), - uncertainty_notes: expect.arrayContaining([ - expect.stringMatching(/not enough evidence; missing .*persistence/), - ]), - })) - }) - - it('retains richer report-generation expectations without overclaiming untraced phases', () => { - const compact = compactRetrieveResult(retrieveContext(buildBroadReportGenerationGraph(), { - question: 'How idea report is being generated', - budget: 4000, - retrievalLevel: 4, - retrievalStrategy: 'slice-v1', - })) - - expect(compact.execution_slice?.phase_coverage).toEqual(expect.objectContaining({ - expected: [ - 'planner', - 'external_research_or_api', - 'report_builder', - 'scoring', - 'quality_gate', - 'renderer_or_synthesis', - 'persistence', - ], - observed: expect.arrayContaining([ - 'controller', - 'service', - 'queue', - ]), - missing: expect.arrayContaining([ - 'planner', - 'external_research_or_api', - 'report_builder', - 'scoring', - 'quality_gate', - 'renderer_or_synthesis', - 'persistence', - ]), - })) - }) }) diff --git a/tests/unit/spi-nest-di-runtime-calls-realistic.test.ts b/tests/unit/spi-nest-di-runtime-calls-realistic.test.ts index 3de3d19e..2fc2ba13 100644 --- a/tests/unit/spi-nest-di-runtime-calls-realistic.test.ts +++ b/tests/unit/spi-nest-di-runtime-calls-realistic.test.ts @@ -515,16 +515,17 @@ describe('SPI realistic Nest DI runtime-call fixture', () => { ]), }), ]), + // #660-B1. Was asserting the orchestrator / external_research_or_api / + // scoring vocabulary. Those phases were only enabled when a prompt + // classifier recognised one benchmark task; that classifier is gone, so + // they are neither expected nor observed. What remains asserted is the + // architectural spine this fixture really models. phase_coverage: expect.objectContaining({ - expected: ['orchestrator', 'external_research_or_api', 'scoring', 'persistence'], observed: expect.arrayContaining([ 'controller', 'service', - 'orchestrator', 'queue', 'worker', - 'external_research_or_api', - 'scoring', 'persistence', ]), missing: [],