From e19591c684b95902b6adcd7c7798c1fcb2e24473 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 18:20:39 +0400 Subject: [PATCH 01/27] fix(retrieve): retain fitting owner declaration evidence --- src/runtime/query-evidence-dependencies.ts | 588 ++++++++++++++++ src/runtime/retrieve.ts | 213 +++++- .../unit/retrieve-owner-declarations.test.ts | 659 ++++++++++++++++++ 3 files changed, 1450 insertions(+), 10 deletions(-) create mode 100644 src/runtime/query-evidence-dependencies.ts create mode 100644 tests/unit/retrieve-owner-declarations.test.ts diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts new file mode 100644 index 00000000..ef8bd6e9 --- /dev/null +++ b/src/runtime/query-evidence-dependencies.ts @@ -0,0 +1,588 @@ +import { extname } from 'node:path' + +import * as ts from 'typescript' + +export interface RepresentedQueryEvidenceSource { + startLine: number + endLine: number + text: string +} + +export interface OwnerDeclarationEvidenceLine { + lineNumber: number + text: string +} + +export interface OwnerDeclarationEvidence { + startLine: number + endLine: number + lines: OwnerDeclarationEvidenceLine[] +} + +export interface OwnerDeclarationCompletionInput { + sourceFilePath: string + sourceLines: readonly string[] + ownerRange: { start: number; end: number } + representedSource: readonly RepresentedQueryEvidenceSource[] +} + +interface ParsedSourceSnapshot { + sourceFilePath: string + sourceText: string + sourceFile: ts.SourceFile +} + +interface ConstBinding { + declaration: ts.VariableDeclaration + statement: ts.VariableStatement +} + +type FunctionOwner = ( + | ts.FunctionDeclaration + | ts.FunctionExpression + | ts.ArrowFunction + | ts.MethodDeclaration + | ts.GetAccessorDeclaration + | ts.SetAccessorDeclaration + | ts.ConstructorDeclaration +) & { body: ts.ConciseBody } + +const parsedSourceSnapshots = new WeakMap() +const JS_TS_EXTENSIONS = new Set(['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs']) + +function scriptKindForPath(sourceFilePath: string): ts.ScriptKind | null { + switch (extname(sourceFilePath).toLowerCase()) { + case '.ts': + case '.mts': + case '.cts': + return ts.ScriptKind.TS + case '.tsx': + return ts.ScriptKind.TSX + case '.js': + case '.mjs': + case '.cjs': + return ts.ScriptKind.JS + case '.jsx': + return ts.ScriptKind.JSX + default: + return null + } +} + +function parsedSourceForSnapshot( + sourceFilePath: string, + sourceLines: readonly string[], +): ts.SourceFile | null { + const extension = extname(sourceFilePath).toLowerCase() + if (!JS_TS_EXTENSIONS.has(extension)) { + return null + } + const scriptKind = scriptKindForPath(sourceFilePath) + if (scriptKind === null) { + return null + } + + const sourceText = sourceLines.join('\n') + const cached = parsedSourceSnapshots.get(sourceLines) + if ( + cached + && cached.sourceFilePath === sourceFilePath + && cached.sourceText === sourceText + ) { + return cached.sourceFile + } + + const sourceFile = ts.createSourceFile( + sourceFilePath, + sourceText, + ts.ScriptTarget.Latest, + true, + scriptKind, + ) + parsedSourceSnapshots.set(sourceLines, { sourceFilePath, sourceText, sourceFile }) + return sourceFile +} + +function lineRangeOf(node: ts.Node, sourceFile: ts.SourceFile): { start: number; end: number } { + const start = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + 1 + const finalPosition = Math.max(node.getStart(sourceFile), node.getEnd() - 1) + const end = sourceFile.getLineAndCharacterOfPosition(finalPosition).line + 1 + return { start, end } +} + +function functionOwnerWithBody(node: ts.Node): FunctionOwner | null { + if ( + !ts.isFunctionDeclaration(node) + && !ts.isFunctionExpression(node) + && !ts.isArrowFunction(node) + && !ts.isMethodDeclaration(node) + && !ts.isGetAccessorDeclaration(node) + && !ts.isSetAccessorDeclaration(node) + && !ts.isConstructorDeclaration(node) + ) { + return null + } + return node.body ? node as FunctionOwner : null +} + +function identifyingRangeOf(owner: FunctionOwner, sourceFile: ts.SourceFile): { start: number; end: number } { + let wrapped: ts.Node = owner + while ( + (ts.isParenthesizedExpression(wrapped.parent) + || ts.isAsExpression(wrapped.parent) + || ts.isSatisfiesExpression(wrapped.parent) + || ts.isTypeAssertionExpression(wrapped.parent) + || ts.isNonNullExpression(wrapped.parent)) + && wrapped.parent.expression === wrapped + ) { + wrapped = wrapped.parent + } + if ( + ts.isVariableDeclaration(wrapped.parent) + && wrapped.parent.initializer === wrapped + && ts.isVariableDeclarationList(wrapped.parent.parent) + && ts.isVariableStatement(wrapped.parent.parent.parent) + ) { + return lineRangeOf(wrapped.parent.parent.parent, sourceFile) + } + return lineRangeOf(owner, sourceFile) +} + +function uniquelyIdentifiedOwner( + sourceFile: ts.SourceFile, + ownerRange: { start: number; end: number }, +): FunctionOwner | null { + const owners: FunctionOwner[] = [] + const visit = (node: ts.Node): void => { + const owner = functionOwnerWithBody(node) + if (owner) { + const range = identifyingRangeOf(owner, sourceFile) + if (range.start === ownerRange.start && range.end === ownerRange.end) { + owners.push(owner) + } + } + ts.forEachChild(node, visit) + } + visit(sourceFile) + return owners.length === 1 ? owners[0]! : null +} + +function normalizedSource(value: string): string { + return value.replace(/\s+/g, ' ').trim() +} + +function representedStatements( + owner: FunctionOwner, + representedSource: readonly RepresentedQueryEvidenceSource[], + sourceFile: ts.SourceFile, +): ts.Statement[] { + const representedByRange = new Map>() + for (const represented of representedSource) { + const key = `${represented.startLine}:${represented.endLine}` + const texts = representedByRange.get(key) ?? new Set() + texts.add(normalizedSource(represented.text)) + representedByRange.set(key, texts) + } + + const statements: ts.Statement[] = [] + const visit = (node: ts.Node): void => { + if (node !== owner) { + if (functionOwnerWithBody(node) || ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + return + } + if (ts.isStatement(node)) { + const range = lineRangeOf(node, sourceFile) + const texts = representedByRange.get(`${range.start}:${range.end}`) + if (texts?.has(normalizedSource(node.getText(sourceFile)))) { + statements.push(node) + } + } + } + ts.forEachChild(node, visit) + } + visit(owner) + return statements +} + +function isNestedFunctionOrClass(node: ts.Node): boolean { + return functionOwnerWithBody(node) !== null + || ts.isClassDeclaration(node) + || ts.isClassExpression(node) +} + +function isValueReference(identifier: ts.Identifier): boolean { + const parent = identifier.parent + if (ts.isShorthandPropertyAssignment(parent) && parent.name === identifier) { + return true + } + if ( + (ts.isPropertyAccessExpression(parent) && parent.name === identifier) + || (ts.isLabeledStatement(parent) && parent.label === identifier) + || ((ts.isBreakStatement(parent) || ts.isContinueStatement(parent)) && parent.label === identifier) + ) { + return false + } + if ('name' in parent && (parent as ts.NamedDeclaration).name === identifier) { + return false + } + return true +} + +function collectValueReferences( + root: ts.Node, + rejectNestedFunctions: boolean, +): { identifiers: ts.Identifier[]; supported: boolean } { + const identifiers: ts.Identifier[] = [] + let supported = true + const visit = (node: ts.Node): void => { + if (!supported) { + return + } + if (node !== root && isNestedFunctionOrClass(node)) { + supported = !rejectNestedFunctions + return + } + if (ts.isTypeNode(node)) { + return + } + if ( + ts.isYieldExpression(node) + || (ts.isDeleteExpression(node)) + || ( + ts.isBinaryExpression(node) + && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ) + || ( + (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) + && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) + ) + ) { + supported = false + return + } + if (ts.isIdentifier(node) && isValueReference(node)) { + identifiers.push(node) + } + ts.forEachChild(node, visit) + } + visit(root) + return { identifiers, supported } +} + +function bindingNames(name: ts.BindingName): string[] { + if (ts.isIdentifier(name)) { + return [name.text] + } + return name.elements.flatMap((element) => ( + ts.isBindingElement(element) ? bindingNames(element.name) : [] + )) +} + +function assignmentTargetNames(node: ts.Node): string[] { + if (ts.isIdentifier(node)) { + return [node.text] + } + if (ts.isParenthesizedExpression(node)) { + return assignmentTargetNames(node.expression) + } + if (ts.isArrayLiteralExpression(node)) { + return node.elements.flatMap((element) => assignmentTargetNames(element)) + } + if (ts.isObjectLiteralExpression(node)) { + return node.properties.flatMap((property) => { + if (ts.isShorthandPropertyAssignment(property)) { + return [property.name.text] + } + if (ts.isPropertyAssignment(property)) { + return assignmentTargetNames(property.initializer) + } + if (ts.isSpreadAssignment(property)) { + return assignmentTargetNames(property.expression) + } + return [] + }) + } + return [] +} + +function writtenBindingNames(owner: FunctionOwner): Set { + const names = new Set() + const visit = (node: ts.Node): void => { + if ( + ts.isBinaryExpression(node) + && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ) { + for (const name of assignmentTargetNames(node.left)) names.add(name) + } else if ( + (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) + && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) + ) { + for (const name of assignmentTargetNames(node.operand)) names.add(name) + } else if (ts.isForInStatement(node) || ts.isForOfStatement(node)) { + if (!ts.isVariableDeclarationList(node.initializer)) { + for (const name of assignmentTargetNames(node.initializer)) names.add(name) + } + } + ts.forEachChild(node, visit) + } + visit(owner) + return names +} + +function functionScopedVarBindingNames(owner: FunctionOwner): Set { + const names = new Set() + const visit = (node: ts.Node): void => { + if (node !== owner && isNestedFunctionOrClass(node)) { + return + } + if ( + ts.isVariableDeclarationList(node) + && (node.flags & ts.NodeFlags.BlockScoped) === 0 + ) { + for (const declaration of node.declarations) { + for (const name of bindingNames(declaration.name)) names.add(name) + } + } + ts.forEachChild(node, visit) + } + visit(owner) + return names +} + +type LexicalScope = + | ts.Block + | ts.CaseBlock + | ts.CatchClause + | ts.ForStatement + | ts.ForInStatement + | ts.ForOfStatement + +function lexicalScopeChain(node: ts.Node, owner: FunctionOwner): LexicalScope[] | null { + const scopes: LexicalScope[] = [] + let current: ts.Node | undefined = node.parent + while (current && current !== owner) { + if ( + ts.isBlock(current) + || ts.isCaseBlock(current) + || ts.isCatchClause(current) + || ts.isForStatement(current) + || ts.isForInStatement(current) + || ts.isForOfStatement(current) + ) { + scopes.push(current) + } + if (isNestedFunctionOrClass(current)) { + return null + } + current = current.parent + } + return ts.isBlock(owner.body) && scopes.includes(owner.body) ? scopes : null +} + +function hasBindingName(name: ts.BindingName, expected: string): boolean { + return bindingNames(name).includes(expected) +} + +function constBindingForReference( + identifier: ts.Identifier, + owner: FunctionOwner, + writtenNames: ReadonlySet, + functionVarNames: ReadonlySet, +): { binding: ConstBinding | null; invalidDependency: boolean } { + const scopes = lexicalScopeChain(identifier, owner) + if (!scopes || functionVarNames.has(identifier.text)) { + return { binding: null, invalidDependency: false } + } + + for (const scope of scopes) { + if (scope === owner.body && owner.parameters.some((parameter) => hasBindingName(parameter.name, identifier.text))) { + return { binding: null, invalidDependency: false } + } + if ( + ts.isCatchClause(scope) + && scope.variableDeclaration + && hasBindingName(scope.variableDeclaration.name, identifier.text) + ) { + return { binding: null, invalidDependency: false } + } + + const declarations: Array<{ + declaration: ts.VariableDeclaration + statement: ts.VariableStatement + declarationKind: 'const' | 'mutable' + }> = [] + let unsupportedBinding = false + const statements = ts.isBlock(scope) + ? scope.statements + : ts.isCaseBlock(scope) + ? scope.clauses.flatMap((clause) => [...clause.statements]) + : [] + for (const statement of statements) { + if (ts.isVariableStatement(statement)) { + const declarationKind = (statement.declarationList.flags & ts.NodeFlags.Const) !== 0 + ? 'const' + : 'mutable' + for (const declaration of statement.declarationList.declarations) { + if (!hasBindingName(declaration.name, identifier.text)) { + continue + } + declarations.push({ declaration, statement, declarationKind }) + } + } else if ( + ((ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement) || ts.isEnumDeclaration(statement)) + && statement.name?.text === identifier.text) + ) { + unsupportedBinding = true + } + } + if (ts.isForStatement(scope) || ts.isForInStatement(scope) || ts.isForOfStatement(scope)) { + const initializer = scope.initializer + if (initializer && ts.isVariableDeclarationList(initializer)) { + unsupportedBinding ||= initializer.declarations.some((declaration) => ( + hasBindingName(declaration.name, identifier.text) + )) + } + } + + if (declarations.length === 0 && !unsupportedBinding) { + continue + } + if (declarations.length !== 1 || unsupportedBinding) { + return { binding: null, invalidDependency: true } + } + + const candidate = declarations[0]! + if ( + candidate.declarationKind !== 'const' + || !ts.isIdentifier(candidate.declaration.name) + || !candidate.declaration.initializer + || writtenNames.has(identifier.text) + ) { + return { binding: null, invalidDependency: false } + } + if (candidate.declaration.getStart() >= identifier.getStart()) { + return { binding: null, invalidDependency: true } + } + if (candidate.statement.declarationList.declarations.some((declaration) => ( + !ts.isIdentifier(declaration.name) || !declaration.initializer + ))) { + return { binding: null, invalidDependency: true } + } + return { + binding: { declaration: candidate.declaration, statement: candidate.statement }, + invalidDependency: false, + } + } + return { binding: null, invalidDependency: false } +} + +function declarationClosure( + owner: FunctionOwner, + seedStatements: readonly ts.Statement[], +): ts.VariableStatement[] | null { + const writtenNames = writtenBindingNames(owner) + const functionVarNames = functionScopedVarBindingNames(owner) + const statements = new Set() + const completed = new Set() + const visiting = new Set() + + const completeBinding = (binding: ConstBinding): boolean => { + if (completed.has(binding.declaration)) { + return true + } + if (visiting.has(binding.declaration)) { + return false + } + visiting.add(binding.declaration) + const references = collectValueReferences(binding.declaration.initializer!, true) + if (!references.supported) { + return false + } + for (const reference of references.identifiers) { + const resolved = constBindingForReference(reference, owner, writtenNames, functionVarNames) + if (resolved.invalidDependency) { + return false + } + if (resolved.binding && !completeBinding(resolved.binding)) { + return false + } + } + visiting.delete(binding.declaration) + completed.add(binding.declaration) + statements.add(binding.statement) + return true + } + + for (const statement of seedStatements) { + const references = collectValueReferences(statement, false) + for (const reference of references.identifiers) { + const resolved = constBindingForReference(reference, owner, writtenNames, functionVarNames) + if (resolved.invalidDependency) { + continue + } + if (resolved.binding && !completeBinding(resolved.binding)) { + return null + } + } + } + return [...statements] +} + +/** + * Returns a complete, source-local const-declaration closure for exact source + * statements already represented in a query excerpt. It intentionally does + * not perform type checking, value inference, repository traversal, or reads. + */ +export function ownerLocalDeclarationEvidence( + input: OwnerDeclarationCompletionInput, +): OwnerDeclarationEvidence[] { + try { + const sourceFile = parsedSourceForSnapshot(input.sourceFilePath, input.sourceLines) + if (!sourceFile) { + return [] + } + const parseDiagnostics = (sourceFile as ts.SourceFile & { + parseDiagnostics?: readonly ts.Diagnostic[] + }).parseDiagnostics + if (parseDiagnostics && parseDiagnostics.length > 0) { + return [] + } + + const owner = uniquelyIdentifiedOwner(sourceFile, input.ownerRange) + if (!owner || !ts.isBlock(owner.body)) { + return [] + } + const seeds = representedStatements(owner, input.representedSource, sourceFile) + if (seeds.length === 0) { + return [] + } + const closure = declarationClosure(owner, seeds) + if (!closure || closure.length === 0) { + return [] + } + + const representedRanges = input.representedSource.map((represented) => ({ + start: represented.startLine, + end: represented.endLine, + })) + return closure + .map((statement): OwnerDeclarationEvidence => { + const range = lineRangeOf(statement, sourceFile) + return { + startLine: range.start, + endLine: range.end, + lines: input.sourceLines + .slice(range.start - 1, range.end) + .map((text, offset) => ({ lineNumber: range.start + offset, text })), + } + }) + .filter((declaration) => !representedRanges.some((range) => ( + range.start <= declaration.startLine && declaration.endLine <= range.end + ))) + .sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine) + } catch { + return [] + } +} diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index cf74b390..42bc9e3a 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -85,6 +85,10 @@ import { type RetrievalStageObserver, } from './retrieve/pipeline.js' import { communitiesFromGraph, estimateQueryTokens } from './serve.js' +import { + ownerLocalDeclarationEvidence, + type RepresentedQueryEvidenceSource, +} from './query-evidence-dependencies.js' export { tokenizeLabel, tokenizeQuestion } from './retrieve/pipeline.js' @@ -751,6 +755,7 @@ interface QueryEvidenceLine { index: number endIndex: number text: string + representedSource: RepresentedQueryEvidenceSource[] score: number matchedTerms: Set matchedObligations: Set @@ -789,6 +794,7 @@ interface QueryEvidenceFragment { index: number endIndex: number text: string + representedSource: RepresentedQueryEvidenceSource[] } const QUERY_EVIDENCE_CONTINUATION_START_PATTERN = /^\s*(?:[.?:]|&&|\|\|)/ @@ -806,7 +812,7 @@ function structuredCallFragment( lines: readonly string[], lineNumber: number, rangeEnd: number, -): { end: number; text: string } | null { +): { end: number; text: string; representedSource: RepresentedQueryEvidenceSource[] } | null { const first = lines[lineNumber - 1] ?? '' if ( QUERY_EVIDENCE_DECLARATION_PATTERN.test(first) @@ -856,6 +862,22 @@ function structuredCallFragment( return { end, text: [first, ...nestedHandoffs.slice(0, 1).map((handoff) => handoff.text), ...discriminants].join(' '), + representedSource: [ + { startLine: lineNumber, endLine: lineNumber, text: first }, + ...nestedHandoffs.slice(0, 1).map((handoff) => ({ + startLine: handoff.index, + endLine: handoff.index, + text: lines[handoff.index - 1] ?? '', + })), + ...properties + .sort((left, right) => priority(left.key) - priority(right.key) || left.index - right.index) + .slice(0, 3) + .map((property) => ({ + startLine: property.index, + endLine: property.index, + text: lines[property.index - 1] ?? '', + })), + ], } } @@ -863,7 +885,12 @@ function providerHandoffFragment( lines: readonly string[], lineNumber: number, rangeStart: number, -): { start: number; end: number; text: string } | null { +): { + start: number + end: number + text: string + representedSource: RepresentedQueryEvidenceSource[] +} | null { const handoff = lines[lineNumber - 1] ?? '' const match = handoff.match(QUERY_EVIDENCE_PROVIDER_HANDOFF_PATTERN) const receiver = match?.[1] @@ -883,6 +910,10 @@ function providerHandoffFragment( start: candidateNumber, end: lineNumber, text: `${candidate.trim()} L${lineNumber}: ${handoff.trim()}`, + representedSource: [ + { startLine: candidateNumber, endLine: candidateNumber, text: candidate }, + { startLine: lineNumber, endLine: lineNumber, text: handoff }, + ], } } return null @@ -948,6 +979,13 @@ function queryEvidenceFragments( index: start, endIndex: end, text: text ?? lines.slice(start - 1, end).join(' '), + representedSource: providerHandoff?.representedSource + ?? structured?.representedSource + ?? [{ + startLine: start, + endLine: end, + text: lines.slice(start - 1, end).join(' '), + }], }) } return fragments @@ -1075,6 +1113,7 @@ function queryEvidenceRange( index: candidate.index, endIndex: candidate.endIndex, text, + representedSource: candidate.representedSource, score, matchedTerms, matchedObligations, @@ -1230,8 +1269,13 @@ function selectQueryEvidenceLines(range: QueryEvidenceRange): QueryEvidenceLine[ return selected.sort((left, right) => left.index - right.index) } -function renderQueryEvidenceLines(lines: readonly QueryEvidenceLine[]): string | null { - const rendered: string[] = [] +interface RenderedQueryEvidenceLine { + source: QueryEvidenceLine + text: string +} + +function renderQueryEvidenceLineEntries(lines: readonly QueryEvidenceLine[]): RenderedQueryEvidenceLine[] { + const rendered: RenderedQueryEvidenceLine[] = [] let usedChars = 0 for (const line of lines) { const normalized = line.text.replace(/\s+/g, ' ').trim() @@ -1245,10 +1289,146 @@ function renderQueryEvidenceLines(lines: readonly QueryEvidenceLine[]): string | break } const bounded = `${prefix}${content}`.slice(0, remaining).trimEnd() - rendered.push(bounded) + rendered.push({ + source: line, + text: bounded, + }) usedChars += bounded.length + separatorChars } - return rendered.length > 0 ? rendered.join('\n') : null + return rendered +} + +function renderQueryEvidenceLines(lines: readonly QueryEvidenceLine[]): string | null { + const rendered = renderQueryEvidenceLineEntries(lines) + return rendered.length > 0 ? rendered.map((line) => line.text).join('\n') : null +} + +function explicitOwnerRange( + sourceLocation: string | null | undefined, + parsedRange: { start: number; end: number } | null, + lineNumber: number, + lineCount: number, + derived: boolean | undefined, + fileNodeLike: boolean | undefined, +): { start: number; end: number } | null { + if (derived || fileNodeLike || !parsedRange || typeof sourceLocation !== 'string') { + return null + } + const match = /^L(\d+)(?:-L?(\d+))?$/.exec(sourceLocation) + if (!match?.[1]) { + return null + } + const start = Number.parseInt(match[1], 10) + const end = Number.parseInt(match[2] ?? match[1], 10) + if ( + start <= 0 + || end < start + || end > lineCount + || parsedRange.start !== start + || parsedRange.end !== end + || lineNumber < start + || lineNumber > end + ) { + return null + } + return { start, end } +} + +function completeOwnerDeclarationEvidence( + sourceFile: string, + sourceLines: readonly string[], + ownerRange: { start: number; end: number }, + baseline: readonly RenderedQueryEvidenceLine[], +): { snippet: string; lineNumber: number } | null { + try { + const representedSource = baseline.flatMap((rendered) => { + const fullContent = rendered.source.text.replace(/\s+/g, ' ').trim() + const prefix = `L${rendered.source.index}: ` + const renderedContent = rendered.text.startsWith(prefix) + ? rendered.text.slice(prefix.length) + : '' + let searchFrom = 0 + return rendered.source.representedSource.flatMap((represented) => { + const representedText = represented.text.replace(/\s+/g, ' ').trim() + const index = representedText.length > 0 + ? fullContent.indexOf(representedText, searchFrom) + : -1 + if (index < 0) { + return [] + } + const end = index + representedText.length + searchFrom = end + return renderedContent.slice(index, end) === fullContent.slice(index, end) + ? [represented] + : [] + }) + }) + if (representedSource.length === 0) { + return null + } + const declarations = ownerLocalDeclarationEvidence({ + sourceFilePath: sourceFile, + sourceLines, + ownerRange, + representedSource, + }) + if (declarations.length === 0) { + return null + } + + const additions = declarations.flatMap((declaration) => declaration.lines.map((line) => { + const content = line.text.replace(/\s+/g, ' ').trim() + return { + lineNumber: line.lineNumber, + content, + text: `L${line.lineNumber}: ${content}`, + } + })) + if ( + additions.some((line) => ( + line.content.length === 0 || line.content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + )) + ) { + return null + } + + const merged = [ + ...baseline.map((line, baselineIndex) => ({ + lineNumber: line.source.index, + text: line.text, + baselineIndex, + })), + ...additions.map((line) => ({ + lineNumber: line.lineNumber, + text: line.text, + baselineIndex: null, + })), + ].sort((left, right) => ( + left.lineNumber - right.lineNumber + || (left.baselineIndex === null ? -1 : left.baselineIndex) + - (right.baselineIndex === null ? -1 : right.baselineIndex) + )) + if (merged.length > QUERY_EVIDENCE_SNIPPET_MAX_LINES) { + return null + } + const snippet = merged.map((line) => line.text).join('\n') + if (snippet.length > QUERY_EVIDENCE_SNIPPET_CHAR_CAP) { + return null + } + const preservedBaseline = merged + .filter((line) => line.baselineIndex !== null) + .sort((left, right) => left.baselineIndex! - right.baselineIndex!) + .map((line) => line.text) + if ( + preservedBaseline.length !== baseline.length + || preservedBaseline.some((line, index) => line !== baseline[index]?.text) + ) { + return null + } + return { snippet, lineNumber: merged[0]!.lineNumber } + } catch { + return null + } } /** @@ -1334,6 +1514,7 @@ export function readQueryEvidenceSnippet( index: symbolFragment.index, endIndex: symbolFragment.endIndex, text: symbolFragment.text, + representedSource: symbolFragment.representedSource, score: 0, matchedTerms: new Set(), matchedObligations: new Set(), @@ -1362,13 +1543,25 @@ export function readQueryEvidenceSnippet( selectedLines = mergedLines } } - const snippet = renderQueryEvidenceLines(selectedLines) - if (!snippet || selectedLines.length === 0) { + const baseline = renderQueryEvidenceLineEntries(selectedLines) + if (baseline.length === 0) { return null } + const baselineSnippet = baseline.map((line) => line.text).join('\n') + const ownerRange = explicitOwnerRange( + options.sourceLocation, + parsedRange, + lineNumber, + lines.length, + options.derived, + options.fileNodeLike, + ) + const completed = ownerRange + ? completeOwnerDeclarationEvidence(sourceFile, lines, ownerRange, baseline) + : null return { - snippet, - lineNumber: selectedLines[0]!.index, + snippet: completed?.snippet ?? baselineSnippet, + lineNumber: completed?.lineNumber ?? baseline[0]!.source.index, scope, } } catch { diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts new file mode 100644 index 00000000..36476d90 --- /dev/null +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -0,0 +1,659 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { join, resolve } from 'node:path' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('typescript', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + createSourceFile: vi.fn(actual.createSourceFile), + } +}) + +import * as ts from 'typescript' + +import { readQueryEvidenceSnippet, type QueryEvidenceSnippet } from '../../src/runtime/retrieve.js' + +const QUESTION = 'How does dispatch outcome return a validated result with retry handling?' +const roots: string[] = [] + +function sourceFixture( + sourceLines: readonly string[], + extension = '.ts', +): { sourceFile: string; sourceLocation: string } { + const fixtureParent = resolve('out', 'test-runtime') + mkdirSync(fixtureParent, { recursive: true }) + const root = mkdtempSync(join(fixtureParent, 'owner-declaration-')) + roots.push(root) + const sourceFile = join(root, `sample${extension}`) + writeFileSync(sourceFile, sourceLines.join('\n'), 'utf8') + return { + sourceFile, + sourceLocation: `L1-L${sourceLines.length}`, + } +} + +function evidenceFor( + sourceLines: readonly string[], + options: { + extension?: string + label?: string + lineNumber?: number + question?: string + sourceLocation?: string | null + derived?: boolean + fileCache?: Map + } = {}, +): QueryEvidenceSnippet | null { + const fixture = sourceFixture(sourceLines, options.extension) + return readQueryEvidenceSnippet(fixture.sourceFile, options.lineNumber ?? 1, { + question: options.question ?? QUESTION, + label: options.label ?? 'executeSample', + sourceLocation: options.sourceLocation === undefined + ? fixture.sourceLocation + : options.sourceLocation, + ...(options.derived === undefined ? {} : { derived: options.derived }), + ...(options.fileCache ? { fileCache: options.fileCache } : {}), + }) +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } + vi.mocked(ts.createSourceFile).mockClear() +}) + +describe('owner-local declaration completion', () => { + it('includes a direct preceding const used by a selected return', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence).toEqual({ + snippet: 'L2: const datum = 12.5\nL3: return dispatchOutcome(datum)', + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('includes a separated declaration without inventing an interval excerpt', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'ready'", + ' observeUnrelatedWork()', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe("L2: const datum = 'ready'\nL4: return dispatchOutcome(datum)") + expect(evidence?.snippet).not.toContain('observeUnrelatedWork') + }) + + it('includes an acyclic const alias closure once in source order', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const origin = 12.5', + ' const alias = origin', + ' return dispatchOutcome(alias)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const origin = 12.5', + 'L3: const alias = origin', + 'L4: return dispatchOutcome(alias)', + ].join('\n')) + }) + + it('deduplicates repeated uses and resolves multiple simple declarators by identity', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const origin = 12.5, alias = origin', + ' return dispatchOutcome(alias, alias)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const origin = 12.5, alias = origin', + 'L3: return dispatchOutcome(alias, alias)', + ].join('\n')) + expect(evidence?.snippet.match(/const origin/g)).toHaveLength(1) + }) + + it.each(['.js', '.mjs'])( + 'supports JavaScript function owners in %s files', + (extension) => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ], { extension }) + + expect(evidence?.snippet).toContain('L2: const datum = 12.5') + }, + ) + + it.each([ + { + name: 'function expression', + source: [ + 'export const executeSample = function () {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ], + lineNumber: 1, + sourceLocation: 'L1-L4', + }, + { + name: 'arrow function', + source: [ + 'export const executeSample = () => {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ], + lineNumber: 1, + sourceLocation: 'L1-L4', + }, + { + name: 'line-split arrow variable owner', + source: [ + 'export const executeSample =', + ' () => {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + ' }', + ], + lineNumber: 1, + sourceLocation: 'L1-L5', + }, + { + name: 'class method', + source: [ + 'export class SampleProcessor {', + ' executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + ' }', + '}', + ], + lineNumber: 2, + sourceLocation: 'L2-L5', + }, + ])('supports a complete $name owner', ({ source, lineNumber, sourceLocation }) => { + const evidence = evidenceFor(source, { lineNumber, sourceLocation }) + + expect(evidence?.snippet).toContain('const datum = 12.5') + expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') + }) + + it('supports TypeScript syntax and gives every multiline initializer line its physical prefix', () => { + const evidence = evidenceFor([ + 'export function executeSample(): unknown {', + ' const datum: number = buildDatum(', + ' 12.5,', + ' )', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const datum: number = buildDatum(', + 'L3: 12.5,', + 'L4: )', + 'L5: return dispatchOutcome(datum)', + ].join('\n')) + }) + + it('uses a selected multiline statement only when its complete text is represented', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'ready'", + ' return datum', + ' ? dispatchOutcome(datum)', + ' : retryOutcome()', + '}', + ]) + + expect(evidence?.snippet).toContain("L2: const datum = 'ready'") + expect(evidence?.snippet).toContain('return datum ? dispatchOutcome(datum) : retryOutcome()') + }) + + it('resolves a closer block const and never attributes the outer binding', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'outer'", + ' {', + " const datum = 'inner'", + ' return dispatchOutcome(datum)', + ' }', + '}', + ]) + + expect(evidence?.snippet).toContain("L4: const datum = 'inner'") + expect(evidence?.snippet).not.toContain("const datum = 'outer'") + }) + + it('resolves an ancestor-block const when no closer lexical binding exists', () => { + const evidence = evidenceFor([ + 'export function executeSample(flag: boolean) {', + " const datum = 'owner'", + ' if (flag) {', + ' return dispatchOutcome(datum)', + ' }', + ' return retryOutcome()', + '}', + ]) + + expect(evidence?.snippet).toContain("L2: const datum = 'owner'") + expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') + }) + + it('does not cross a parameter binding or a nested-function owner boundary', () => { + const parameterEvidence = evidenceFor([ + "const datum = 'outside'", + 'export function executeSample(datum: string) {', + ' return dispatchOutcome(datum)', + '}', + ], { lineNumber: 2, sourceLocation: 'L2-L4' }) + const nestedEvidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'outer'", + ' function nestedTask() {', + ' return dispatchOutcome(datum)', + ' }', + ' return finishOuterWork()', + '}', + ]) + + expect(parameterEvidence?.snippet).not.toContain("const datum = 'outside'") + expect(nestedEvidence?.snippet).not.toContain("const datum = 'outer'") + }) + + it('does not cross a for-loop binding to an outer const with the same name', () => { + const evidence = evidenceFor([ + 'export function executeSample(values: string[]) {', + " const datum = 'outer'", + ' for (const datum of values) {', + ' return dispatchOutcome(datum)', + ' }', + ' return retryOutcome()', + '}', + ]) + + expect(evidence?.snippet).not.toContain("const datum = 'outer'") + }) + + it.each([ + { + name: 'let binding', + source: ['export function executeSample() {', ' let datum = 12.5', ' return dispatchOutcome(datum)', '}'], + forbidden: 'let datum', + }, + { + name: 'var binding', + source: ['export function executeSample() {', ' var datum = 12.5', ' return dispatchOutcome(datum)', '}'], + forbidden: 'var datum', + }, + { + name: 'written const binding', + source: ['export function executeSample() {', ' const datum = 12.5', ' datum = 14', ' return dispatchOutcome(datum)', '}'], + forbidden: 'const datum', + }, + { + name: 'destructured binding', + source: ['export function executeSample(input: Input) {', ' const { datum } = input', ' return dispatchOutcome(datum)', '}'], + forbidden: 'const { datum }', + }, + { + name: 'declaration after use', + source: ['export function executeSample() {', ' dispatchOutcome(datum)', ' const datum = 12.5', ' return retryOutcome()', '}'], + forbidden: 'const datum', + }, + { + name: 'foreign lexical block', + source: ['export function executeSample(flag: boolean) {', ' if (flag) {', ' const datum = 12.5', ' }', ' return dispatchOutcome(datum)', '}'], + forbidden: 'const datum', + }, + { + name: 'cyclic or forward alias closure', + source: ['export function executeSample() {', ' const first = second', ' const second = first', ' return dispatchOutcome(second)', '}'], + forbidden: 'const first', + }, + { + name: 'initializer containing a nested function', + source: ['export function executeSample() {', ' const datum = (() => 12.5)()', ' return dispatchOutcome(datum)', '}'], + forbidden: 'const datum', + }, + ])('does not guess through a $name', ({ source, forbidden }) => { + expect(evidenceFor(source)?.snippet).not.toContain(forbidden) + }) + + it('does not use a declaration from a sibling function', () => { + const evidence = evidenceFor([ + 'function siblingTask() {', + " const datum = 'sibling'", + ' return datum', + '}', + 'export function executeSample() {', + ' return dispatchOutcome(datum)', + '}', + ], { lineNumber: 5, sourceLocation: 'L5-L7' }) + + expect(evidence?.snippet).not.toContain("const datum = 'sibling'") + }) + + it('ignores non-value identifier positions but keeps shorthand and computed value uses', () => { + const propertyOnly = evidenceFor([ + 'export function executeSample(service: Service) {', + " const datum = 'not-a-value-use'", + ' return dispatchOutcome({ datum: service.datum }, "datum")', + '}', + ]) + const typeOnly = evidenceFor([ + 'export function executeSample() {', + ' const DatumType = 12.5', + ' return dispatchOutcome(createValue())', + '}', + ]) + const shorthand = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome({ datum })', + '}', + ]) + const computed = evidenceFor([ + 'export function executeSample() {', + " const datum = 'key'", + ' return dispatchOutcome({ [datum]: true })', + '}', + ]) + + expect(propertyOnly?.snippet).not.toContain("const datum = 'not-a-value-use'") + expect(typeOnly?.snippet).not.toContain('const DatumType') + expect(shorthand?.snippet).toContain('L2: const datum = 12.5') + expect(computed?.snippet).toContain("L2: const datum = 'key'") + }) + + it('ignores labels, comments, and string text as dependency uses', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'not-a-value-use'", + ' datum: dispatchOutcome(12.5) // datum is only text here', + ' return retryOutcome("datum")', + '}', + ]) + + expect(evidence?.snippet).not.toContain("const datum = 'not-a-value-use'") + }) + + it('preserves no-dependency and unsupported-language output byte for byte', () => { + const expected = { + snippet: 'L2: return dispatchOutcome(12.5)', + lineNumber: 2, + scope: 'symbol' as const, + } + const source = [ + 'export function executeSample() {', + ' return dispatchOutcome(12.5)', + '}', + ] + + expect(evidenceFor(source)).toEqual(expected) + expect(evidenceFor(source, { extension: '.py' })).toEqual(expected) + }) + + it.each([ + { name: 'missing range', sourceLocation: null, derived: false, expected: 'return' }, + { name: 'malformed range', sourceLocation: 'not-a-range', derived: false, expected: 'return' }, + { name: 'clamped range', sourceLocation: 'L1-L99', derived: false, expected: 'return' }, + { + name: 'incomplete range', + sourceLocation: 'L1-L2', + derived: false, + expected: { + snippet: 'L2: const datum = 12.5\nL3: return dispatchOutcome(datum)', + lineNumber: 2, + scope: 'source_file' as const, + }, + }, + { + name: 'declaration-only range', + sourceLocation: 'L1', + derived: false, + expected: { + snippet: 'L1: export function executeSample() {\nL3: return dispatchOutcome(datum)', + lineNumber: 1, + scope: 'source_file' as const, + }, + }, + { name: 'derived range', sourceLocation: 'L1-L4', derived: true, expected: 'return' }, + ])('preserves the baseline for a $name', ({ sourceLocation, derived, expected }) => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ], { sourceLocation, derived }) + + if (expected === 'return') { + expect(evidence).toEqual({ + snippet: 'L3: return dispatchOutcome(datum)', + lineNumber: 3, + scope: 'symbol', + }) + } else { + expect(evidence).toEqual(expected) + } + }) + + it('preserves an ambiguous same-line function range byte for byte', () => { + const sourceLine = 'export const executeSample = () => (() => dispatchOutcome(12.5))()' + expect(evidenceFor([sourceLine], { sourceLocation: 'L1' })).toEqual({ + snippet: `L1: ${sourceLine}`, + lineNumber: 1, + scope: 'symbol', + }) + }) + + it('contains parser failure and preserves malformed-source output', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + ]) + + expect(evidence).toEqual({ + snippet: 'L3: return dispatchOutcome(datum)', + lineNumber: 3, + scope: 'symbol', + }) + }) + + it('does not infer dependencies from a synthesized noncontiguous fragment', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'hidden'", + ' dispatchOutcome({', + " eventType: 'ready',", + ' hiddenPayload: datum,', + ' })', + '}', + ], { question: 'How does dispatch outcome use the ready event type?' }) + + expect(evidence?.snippet).toContain("eventType: 'ready'") + expect(evidence?.snippet).not.toContain("const datum = 'hidden'") + }) + + it('does not mistake duplicate string text for a represented statement truncated later', () => { + const statementText = 'return dispatchOutcome(datum)' + const evidence = evidenceFor([ + `export const executeSample = function (label = "${statementText}${'x'.repeat(220)}") {`, + " const datum = 'hidden'", + ` ${statementText}`, + '}', + ]) + + expect(evidence?.snippet).toContain(statementText) + expect(evidence?.snippet).not.toContain('L3:') + expect(evidence?.snippet).not.toContain("const datum = 'hidden'") + }) + + it('completes only protected owner evidence during source-file supplementation', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + " const datum = 'owner'", + ' return computeWidget(datum)', + '}', + '', + 'async function workflowTask() {', + " const remote = 'foreign'", + ' return retryDelivery(await dispatchRecord(validateRequest(remote)))', + '}', + ], { + sourceLocation: 'L1-L4', + question: 'How does compute widget validate and dispatch a record with retry delivery?', + }) + + expect(evidence?.scope).toBe('source_file') + expect(evidence?.snippet).toContain("L2: const datum = 'owner'") + expect(evidence?.snippet).toContain('return computeWidget(datum)') + expect(evidence?.snippet).not.toContain("const remote = 'foreign'") + }) +}) + +describe('owner declaration completion budgets and atomic preservation', () => { + it('admits a declaration closure that exactly fills the 300-character cap', () => { + const returnSource = ` return dispatchOutcome(datum, '${'r'.repeat(48)}')` + const renderedReturn = `L3: ${returnSource.trim()}` + const declarationShell = "L2: const datum = ''" + const fillLength = 300 - renderedReturn.length - declarationShell.length - 1 + const declarationSource = ` const datum = '${'d'.repeat(fillLength)}'` + const renderedDeclaration = `L2: ${declarationSource.trim()}` + + const evidence = evidenceFor([ + 'export function executeSample() {', + declarationSource, + returnSource, + '}', + ]) + + expect(renderedDeclaration.slice(4).length).toBeLessThanOrEqual(220) + expect(evidence?.snippet).toBe(`${renderedDeclaration}\n${renderedReturn}`) + expect(evidence?.snippet).toHaveLength(300) + }) + + it('rejects a declaration whose physical fragment exceeds 220 characters', () => { + const declaration = ` const datum = '${'d'.repeat(205)}'` + const baseline = 'L3: return dispatchOutcome(datum)' + expect(declaration.trim()).toHaveLength(221) + + const evidence = evidenceFor([ + 'export function executeSample() {', + declaration, + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe(baseline) + }) + + it('rejects an atomic closure that would exceed four rendered fragments', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const first = 1', + ' const second = first', + ' const third = second', + ' const fourth = third', + ' return dispatchOutcome(fourth)', + '}', + ]) + + expect(evidence?.snippet).toBe('L6: return dispatchOutcome(fourth)') + }) + + it('rejects total overflow without removing or truncating prior selected evidence', () => { + const declaration = ` const datum = '${'d'.repeat(190)}'` + const source = [ + 'export function executeSample() {', + declaration, + ' await dispatchOutcome(datum)', + ' await retryOutcome(datum)', + ' return validateResult(datum)', + '}', + ] + const baseline = [ + 'L3: await dispatchOutcome(datum)', + 'L4: await retryOutcome(datum)', + 'L5: return validateResult(datum)', + ].join('\n') + + const evidence = evidenceFor(source) + + expect(evidence?.snippet).toBe(baseline) + expect(evidence?.snippet).toContain('L5: return validateResult(datum)') + }) +}) + +describe('owner declaration parse snapshot cache', () => { + it('does not reuse a parsed source after content changes at the same path', () => { + const firstSource = [ + 'export function executeSample() {', + " const datum = 'first'", + ' return dispatchOutcome(datum)', + '}', + ] + const secondSource = [ + 'export function executeSample() {', + " const datum = 'second'", + ' return dispatchOutcome(datum)', + '}', + ] + const { sourceFile, sourceLocation } = sourceFixture(firstSource) + const fileCache = new Map() + const options = { + question: QUESTION, + label: 'executeSample', + sourceLocation, + fileCache, + } + + const first = readQueryEvidenceSnippet(sourceFile, 1, options) + writeFileSync(sourceFile, secondSource.join('\n'), 'utf8') + fileCache.set(sourceFile, [...secondSource]) + const second = readQueryEvidenceSnippet(sourceFile, 1, options) + + expect(first?.snippet).toContain("const datum = 'first'") + expect(second?.snippet).toContain("const datum = 'second'") + expect(second?.snippet).not.toContain("const datum = 'first'") + }) + + it('parses an unchanged cached source snapshot at most once', () => { + const source = [ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ] + const { sourceFile, sourceLocation } = sourceFixture(source) + const fileCache = new Map() + const options = { + question: QUESTION, + label: 'executeSample', + sourceLocation, + fileCache, + } + + readQueryEvidenceSnippet(sourceFile, 1, options) + readQueryEvidenceSnippet(sourceFile, 1, options) + + expect(ts.createSourceFile).toHaveBeenCalledTimes(1) + }) +}) From c7381d0facd8146e84ac3a289b64b02695cda7af Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 19:21:19 +0400 Subject: [PATCH 02/27] fix(retrieve): preserve declaration bytes and reject resource bindings --- src/runtime/query-evidence-dependencies.ts | 3 +- src/runtime/retrieve.ts | 4 +- .../unit/retrieve-owner-declarations.test.ts | 97 +++++++++++++++---- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index ef8bd6e9..71b510a4 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -421,7 +421,8 @@ function constBindingForReference( : [] for (const statement of statements) { if (ts.isVariableStatement(statement)) { - const declarationKind = (statement.declarationList.flags & ts.NodeFlags.Const) !== 0 + const blockScopedFlags = statement.declarationList.flags & ts.NodeFlags.BlockScoped + const declarationKind = blockScopedFlags === ts.NodeFlags.Const ? 'const' : 'mutable' for (const declaration of statement.declarationList.declarations) { diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 42bc9e3a..94306133 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -1377,7 +1377,7 @@ function completeOwnerDeclarationEvidence( } const additions = declarations.flatMap((declaration) => declaration.lines.map((line) => { - const content = line.text.replace(/\s+/g, ' ').trim() + const content = line.text return { lineNumber: line.lineNumber, content, @@ -1386,7 +1386,7 @@ function completeOwnerDeclarationEvidence( })) if ( additions.some((line) => ( - line.content.length === 0 || line.content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + line.content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP )) ) { return null diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 36476d90..1d57df44 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -75,7 +75,7 @@ describe('owner-local declaration completion', () => { ]) expect(evidence).toEqual({ - snippet: 'L2: const datum = 12.5\nL3: return dispatchOutcome(datum)', + snippet: 'L2: const datum = 12.5\nL3: return dispatchOutcome(datum)', lineNumber: 2, scope: 'symbol', }) @@ -90,7 +90,7 @@ describe('owner-local declaration completion', () => { '}', ]) - expect(evidence?.snippet).toBe("L2: const datum = 'ready'\nL4: return dispatchOutcome(datum)") + expect(evidence?.snippet).toBe("L2: const datum = 'ready'\nL4: return dispatchOutcome(datum)") expect(evidence?.snippet).not.toContain('observeUnrelatedWork') }) @@ -104,8 +104,8 @@ describe('owner-local declaration completion', () => { ]) expect(evidence?.snippet).toBe([ - 'L2: const origin = 12.5', - 'L3: const alias = origin', + 'L2: const origin = 12.5', + 'L3: const alias = origin', 'L4: return dispatchOutcome(alias)', ].join('\n')) }) @@ -119,7 +119,7 @@ describe('owner-local declaration completion', () => { ]) expect(evidence?.snippet).toBe([ - 'L2: const origin = 12.5, alias = origin', + 'L2: const origin = 12.5, alias = origin', 'L3: return dispatchOutcome(alias, alias)', ].join('\n')) expect(evidence?.snippet.match(/const origin/g)).toHaveLength(1) @@ -135,7 +135,7 @@ describe('owner-local declaration completion', () => { '}', ], { extension }) - expect(evidence?.snippet).toContain('L2: const datum = 12.5') + expect(evidence?.snippet).toContain('L2: const datum = 12.5') }, ) @@ -205,13 +205,60 @@ describe('owner-local declaration completion', () => { ]) expect(evidence?.snippet).toBe([ - 'L2: const datum: number = buildDatum(', - 'L3: 12.5,', - 'L4: )', + 'L2: const datum: number = buildDatum(', + 'L3: 12.5,', + 'L4: )', 'L5: return dispatchOutcome(datum)', ].join('\n')) }) + it('preserves meaningful whitespace in a newly added string-literal declaration', () => { + const evidence = evidenceFor([ + 'export function assemble() {', + " const input = 'alpha beta';", + ' return { displayedText: input };', + '}', + ], { question: 'What is the displayed text?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + "L2: const input = 'alpha beta';", + 'L3: return { displayedText: input };', + ].join('\n')) + }) + + it('preserves a literal tab in a string and repeated spaces in a regex', () => { + const declaration = " const input = ['alpha\tbeta', /alpha beta/];" + const evidence = evidenceFor([ + 'export function executeSample() {', + declaration, + ' return dispatchOutcome(input)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + `L2: ${declaration}`, + 'L3: return dispatchOutcome(input)', + ].join('\n')) + }) + + it('preserves physical multiline template content, including a blank line', () => { + const evidence = evidenceFor([ + 'export function assemble() {', + ' const input = ` alpha ', + '', + ' beta `', + ' return { displayedText: input };', + '}', + ], { question: 'What is the displayed text?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const input = ` alpha ', + 'L3: ', + 'L4: beta `', + 'L5: return { displayedText: input };', + ].join('\n')) + }) + it('uses a selected multiline statement only when its complete text is represented', () => { const evidence = evidenceFor([ 'export function executeSample() {', @@ -222,7 +269,7 @@ describe('owner-local declaration completion', () => { '}', ]) - expect(evidence?.snippet).toContain("L2: const datum = 'ready'") + expect(evidence?.snippet).toContain("L2: const datum = 'ready'") expect(evidence?.snippet).toContain('return datum ? dispatchOutcome(datum) : retryOutcome()') }) @@ -237,7 +284,7 @@ describe('owner-local declaration completion', () => { '}', ]) - expect(evidence?.snippet).toContain("L4: const datum = 'inner'") + expect(evidence?.snippet).toContain("L4: const datum = 'inner'") expect(evidence?.snippet).not.toContain("const datum = 'outer'") }) @@ -252,7 +299,7 @@ describe('owner-local declaration completion', () => { '}', ]) - expect(evidence?.snippet).toContain("L2: const datum = 'owner'") + expect(evidence?.snippet).toContain("L2: const datum = 'owner'") expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') }) @@ -291,6 +338,20 @@ describe('owner-local declaration completion', () => { expect(evidence?.snippet).not.toContain("const datum = 'outer'") }) + it.each([ + { declaration: 'using datum = acquireDatum()' }, + { declaration: 'await using datum = acquireDatum()' }, + ])('does not complete an unsupported $declaration binding', ({ declaration }) => { + const evidence = evidenceFor([ + 'export async function executeSample() {', + ` ${declaration}`, + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe('L3: return dispatchOutcome(datum)') + }) + it.each([ { name: 'let binding', @@ -378,8 +439,8 @@ describe('owner-local declaration completion', () => { expect(propertyOnly?.snippet).not.toContain("const datum = 'not-a-value-use'") expect(typeOnly?.snippet).not.toContain('const DatumType') - expect(shorthand?.snippet).toContain('L2: const datum = 12.5') - expect(computed?.snippet).toContain("L2: const datum = 'key'") + expect(shorthand?.snippet).toContain('L2: const datum = 12.5') + expect(computed?.snippet).toContain("L2: const datum = 'key'") }) it('ignores labels, comments, and string text as dependency uses', () => { @@ -523,7 +584,7 @@ describe('owner-local declaration completion', () => { }) expect(evidence?.scope).toBe('source_file') - expect(evidence?.snippet).toContain("L2: const datum = 'owner'") + expect(evidence?.snippet).toContain("L2: const datum = 'owner'") expect(evidence?.snippet).toContain('return computeWidget(datum)') expect(evidence?.snippet).not.toContain("const remote = 'foreign'") }) @@ -533,10 +594,10 @@ describe('owner declaration completion budgets and atomic preservation', () => { it('admits a declaration closure that exactly fills the 300-character cap', () => { const returnSource = ` return dispatchOutcome(datum, '${'r'.repeat(48)}')` const renderedReturn = `L3: ${returnSource.trim()}` - const declarationShell = "L2: const datum = ''" + const declarationShell = "L2: const datum = ''" const fillLength = 300 - renderedReturn.length - declarationShell.length - 1 const declarationSource = ` const datum = '${'d'.repeat(fillLength)}'` - const renderedDeclaration = `L2: ${declarationSource.trim()}` + const renderedDeclaration = `L2: ${declarationSource}` const evidence = evidenceFor([ 'export function executeSample() {', @@ -553,7 +614,7 @@ describe('owner declaration completion budgets and atomic preservation', () => { it('rejects a declaration whose physical fragment exceeds 220 characters', () => { const declaration = ` const datum = '${'d'.repeat(205)}'` const baseline = 'L3: return dispatchOutcome(datum)' - expect(declaration.trim()).toHaveLength(221) + expect(declaration).toHaveLength(223) const evidence = evidenceFor([ 'export function executeSample() {', From 4072b5bf9424cdfdfc57968d85c4464e82674543 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 19:35:21 +0400 Subject: [PATCH 03/27] fix(generate): preserve verified source evidence in auto composition --- src/infrastructure/generate.ts | 265 ++++++++++++++- src/pipeline/extract.ts | 2 +- .../generate-auto-source-composition.test.ts | 305 ++++++++++++++++++ tests/unit/generate-spi-flag.test.ts | 12 + 4 files changed, 580 insertions(+), 4 deletions(-) create mode 100644 tests/unit/generate-auto-source-composition.test.ts diff --git a/src/infrastructure/generate.ts b/src/infrastructure/generate.ts index edee2d78..994ef904 100644 --- a/src/infrastructure/generate.ts +++ b/src/infrastructure/generate.ts @@ -1,5 +1,5 @@ import { existsSync, mkdirSync, readFileSync } from 'node:fs' -import { join, resolve } from 'node:path' +import { isAbsolute, join, relative, resolve, sep } from 'node:path' import { KnowledgeGraph } from '../contracts/graph.js' import { rebindEvidenceOccurrence } from '../contracts/semantic-identity.js' @@ -40,6 +40,7 @@ import { createIndexingManifest, indexingStrictViolations, localIndexingPath } f import { buildSpiCached, type SpiCacheStats } from '../pipeline/spi/cache.js' import { isSpiSupportedSourceFile } from '../pipeline/spi/build.js' import { projectSpiToExtraction } from '../pipeline/spi/projector.js' +import type { SemanticProgramIndex, SpiSymbol, SpiSymbolKind } from '../pipeline/spi/types.js' import { generate as generateReport } from '../pipeline/report.js' import { toWiki } from '../pipeline/wiki.js' import { loadGraph } from '../runtime/serve.js' @@ -279,6 +280,255 @@ function mergeExtractions(extractions: ExtractionData[]): ExtractionData { }, emptyExtraction()) } +const AUTO_SOURCE_LOCATION = /^L([1-9]\d*)(?:-L([1-9]\d*))?$/ +const AUTO_SOURCE_MAX_LINES = 25 +const AUTO_SOURCE_MAX_CHARS = 2_000 +const AUTO_SOURCE_MAX_TRUNCATED_CHARS = AUTO_SOURCE_MAX_CHARS + 3 +const AUTO_SOURCE_PROJECTABLE_KINDS: ReadonlySet = new Set([ + 'function', + 'class', + 'interface', + 'type-alias', + 'enum', + 'method', + 'constant', + 'variable', + 'namespace', +]) + +type AutoSourceRange = { + start: number + end: number +} + +type AutoSourceOwner = { + file: string + start: number + end: number + symbol: SpiSymbol +} + +function canonicalCorpusPath(rootPath: string, candidate: unknown): string | null { + if (typeof candidate !== 'string' || candidate.length === 0 || candidate.includes('\0')) { + return null + } + + const absolutePath = resolve(rootPath, candidate) + const fromRoot = relative(rootPath, absolutePath) + if (fromRoot === '' || fromRoot === '..' || fromRoot.startsWith(`..${sep}`) || isAbsolute(fromRoot)) { + return null + } + return absolutePath.replaceAll('\\', '/') +} + +function parseAutoSourceLocation(value: unknown): AutoSourceRange | null { + if (typeof value !== 'string') return null + const match = AUTO_SOURCE_LOCATION.exec(value) + if (!match) return null + const start = Number(match[1]) + const end = match[2] === undefined ? start : Number(match[2]) + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start <= 0 || end < start) { + return null + } + return { start, end } +} + +function validAutoSourceSnippet(value: unknown, symbol: SpiSymbol): value is string { + if (typeof value !== 'string' || value.length === 0 || value !== value.trim() || value.includes('\r')) { + return false + } + if (value.split('\n').length > AUTO_SOURCE_MAX_LINES || value.length > AUTO_SOURCE_MAX_TRUNCATED_CHARS) { + return false + } + if (value.length > AUTO_SOURCE_MAX_CHARS && !value.endsWith('...')) { + return false + } + + const declarationName = symbol.kind === 'method' + ? symbol.name.slice(symbol.name.lastIndexOf('.') + 1) + : symbol.name + if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(declarationName)) { + return false + } + const escapedName = declarationName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + return new RegExp(`(^|[^A-Za-z0-9_$])${escapedName}([^A-Za-z0-9_$]|$)`).test(value) +} + +function validSpiOwnerRange(symbol: SpiSymbol): AutoSourceRange | null { + const coordinates = [ + symbol.range?.start?.line, + symbol.range?.start?.column, + symbol.range?.end?.line, + symbol.range?.end?.column, + ] + if (!coordinates.every((coordinate) => Number.isSafeInteger(coordinate) && coordinate > 0)) { + return null + } + + const [startLine, startColumn, endLine, endColumn] = coordinates as [number, number, number, number] + if (endLine < startLine || (endLine === startLine && endColumn <= startColumn)) { + return null + } + const inclusiveEnd = endColumn === 1 && endLine > startLine ? endLine - 1 : endLine + if (inclusiveEnd < startLine) return null + return { start: startLine, end: inclusiveEnd } +} + +function expectedSpiLabel(symbol: SpiSymbol): string | null { + if (!AUTO_SOURCE_PROJECTABLE_KINDS.has(symbol.kind) || symbol.framework_metadata?.external_call === true) { + return null + } + if (symbol.kind === 'method') { + const dotAt = symbol.name.lastIndexOf('.') + return dotAt > 0 && dotAt < symbol.name.length - 1 ? `.${symbol.name.slice(dotAt + 1)}()` : null + } + if (symbol.kind === 'function') return `${symbol.name}()` + return symbol.name +} + +function compatibleSpiNodeKind(symbol: SpiSymbol, node: ExtractionNode): boolean { + const kind = node.node_kind + if (kind === undefined) return true + if (symbol.kind === 'function') { + return kind !== 'class' && kind !== 'controller' && kind !== 'method' + } + if (symbol.kind === 'method') { + return kind === 'method' || kind === 'function' || kind === 'route' + } + if (symbol.kind === 'class') { + return kind === 'class' || kind === 'controller' + } + return false +} + +function provenanceMatchesOwner(node: ExtractionNode, rootPath: string, owner: AutoSourceOwner): boolean { + if (node.provenance === undefined) return true + if (!Array.isArray(node.provenance) || node.provenance.length === 0) return false + + return node.provenance.every((entry) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return false + if ('source_file' in entry && entry.source_file !== undefined) { + if (canonicalCorpusPath(rootPath, entry.source_file) !== owner.file) return false + } + if ('source_location' in entry && entry.source_location !== undefined) { + const location = parseAutoSourceLocation(entry.source_location) + if (!location || location.start !== owner.start) return false + } + return true + }) +} + +/** + * Auto mode deliberately combines two views of supported source. SPI owns + * identity and metadata; legacy owns its already-normalized bounded source + * evidence. Join those views only when the raw SPI declaration uniquely + * proves that both extraction nodes describe the same in-corpus declaration. + */ +function composeDefaultAutoSourceEvidence(args: { + rootPath: string + spiCodeFiles: readonly string[] + spi: SemanticProgramIndex + legacy: ExtractionData + projectedSpi: ExtractionData +}): { legacy: ExtractionData; projectedSpi: ExtractionData } { + const rootPath = resolve(args.rootPath) + const allowedFiles = new Set( + args.spiCodeFiles.flatMap((filePath) => { + const canonical = canonicalCorpusPath(rootPath, filePath) + return canonical ? [canonical] : [] + }), + ) + + const spiFilesById = new Map() + const spiFilesByPath = new Map() + for (const file of args.spi.files) { + const byId = spiFilesById.get(file.id) ?? [] + byId.push(file) + spiFilesById.set(file.id, byId) + const filePath = canonicalCorpusPath(rootPath, file.path) + if (filePath) { + const byPath = spiFilesByPath.get(filePath) ?? [] + byPath.push(file) + spiFilesByPath.set(filePath, byPath) + } + } + + const symbolsByFile = new Map() + for (const symbol of args.spi.symbols) { + const files = spiFilesById.get(symbol.file_id) + if (!files || files.length !== 1) continue + const filePath = canonicalCorpusPath(rootPath, files[0]?.path) + if (!filePath || !allowedFiles.has(filePath) || spiFilesByPath.get(filePath)?.length !== 1) continue + const symbols = symbolsByFile.get(filePath) ?? [] + symbols.push(symbol) + symbolsByFile.set(filePath, symbols) + } + + const legacyById = new Map() + for (const node of args.legacy.nodes) { + const nodes = legacyById.get(node.id) ?? [] + nodes.push(node) + legacyById.set(node.id, nodes) + } + const projectedById = new Map() + for (const node of args.projectedSpi.nodes) { + const nodes = projectedById.get(node.id) ?? [] + nodes.push(node) + projectedById.set(node.id, nodes) + } + + const composedNodeIds = new Set() + const composedProjectedNodes = args.projectedSpi.nodes.map((spiNode) => { + const legacyNodes = legacyById.get(spiNode.id) + if (!legacyNodes || legacyNodes.length !== 1 || projectedById.get(spiNode.id)?.length !== 1) return spiNode + const legacyNode = legacyNodes[0]! + if (legacyNode.label !== spiNode.label || legacyNode.file_type !== 'code' || spiNode.file_type !== 'code') return spiNode + if (legacyNode.virtual === true || spiNode.virtual === true) return spiNode + + const spiFile = canonicalCorpusPath(rootPath, spiNode.source_file) + const legacyFile = canonicalCorpusPath(rootPath, legacyNode.source_file) + if (!spiFile || legacyFile !== spiFile || !allowedFiles.has(spiFile)) return spiNode + + const projectedStart = parseAutoSourceLocation(spiNode.source_location) + if (!projectedStart || projectedStart.start !== projectedStart.end) return spiNode + + const fileSymbols = symbolsByFile.get(spiFile) ?? [] + const compatibleSymbols = fileSymbols.filter((symbol) => + expectedSpiLabel(symbol) === spiNode.label && compatibleSpiNodeKind(symbol, spiNode), + ) + if (compatibleSymbols.length !== 1) return spiNode + const symbol = compatibleSymbols[0]! + const ownerRange = validSpiOwnerRange(symbol) + if (!ownerRange || ownerRange.start !== projectedStart.start) return spiNode + if (fileSymbols.some((other) => other !== symbol && validSpiOwnerRange(other)?.start === ownerRange.start)) return spiNode + + const owner: AutoSourceOwner = { file: spiFile, start: ownerRange.start, end: ownerRange.end, symbol } + const legacySourceLocation = legacyNode.source_location + const legacyRange = parseAutoSourceLocation(legacySourceLocation) + if (!legacyRange || legacyRange.start !== owner.start || legacyRange.end !== owner.end) return spiNode + if (!validAutoSourceSnippet(legacyNode.snippet, symbol)) return spiNode + if (!provenanceMatchesOwner(spiNode, rootPath, owner) || !provenanceMatchesOwner(legacyNode, rootPath, owner)) return spiNode + + composedNodeIds.add(spiNode.id) + return { + ...spiNode, + source_location: legacySourceLocation!, + snippet: legacyNode.snippet, + } + }) + + return { + legacy: { + ...args.legacy, + nodes: args.legacy.nodes.filter((node) => !composedNodeIds.has(node.id)), + }, + projectedSpi: { + ...args.projectedSpi, + nodes: composedProjectedNodes, + }, + } +} + function sourceFileKey(sourceFile: unknown): string | null { return typeof sourceFile === 'string' && sourceFile.length > 0 ? resolve(sourceFile) : null } @@ -821,10 +1071,19 @@ export function generateGraph(rootPath = '.', options: GenerateGraphOptions = {} onFileOutcome: recordExtractionOutcome('legacy_fallback', 'spi_unsupported_language'), }), 'legacy_fallback') : emptyExtraction() + const composedAutoExtractions = extractionMode === 'auto' && built + ? composeDefaultAutoSourceEvidence({ + rootPath: resolvedRootPath, + spiCodeFiles, + spi: built.spi, + legacy: legacyAugmentationExtraction, + projectedSpi: spiSupplementalExtraction, + }) + : null const codeExtraction = extractionMode === 'auto' ? mergeExtractions([ - legacyAugmentationExtraction, - spiSupplementalExtraction, + composedAutoExtractions?.legacy ?? legacyAugmentationExtraction, + composedAutoExtractions?.projectedSpi ?? spiSupplementalExtraction, legacyFallbackExtraction, ]) : spiExtraction diff --git a/src/pipeline/extract.ts b/src/pipeline/extract.ts index 74cb85c9..48fe4215 100644 --- a/src/pipeline/extract.ts +++ b/src/pipeline/extract.ts @@ -67,7 +67,7 @@ export type { ExtractionStageObserver, } from './extract/pipeline.js' -export const EXTRACTOR_CACHE_VERSION = 68 +export const EXTRACTOR_CACHE_VERSION = 69 const PYTHON_KEYWORDS = new Set(['if', 'elif', 'else', 'for', 'while', 'return', 'class', 'def', 'lambda', 'with', 'print', 'sum']) const GENERIC_CODE_EXTENSIONS = new Set(['.go', '.rs', '.java', '.kt', '.kts', '.scala', '.cs', '.c', '.cc', '.cpp', '.cxx', '.h', '.hpp', '.swift', '.php', '.zig']) const RUBY_KEYWORDS = new Set(['if', 'elsif', 'else', 'unless', 'while', 'until', 'return', 'super', 'yield', 'class', 'def']) diff --git a/tests/unit/generate-auto-source-composition.test.ts b/tests/unit/generate-auto-source-composition.test.ts new file mode 100644 index 00000000..b9ba8fab --- /dev/null +++ b/tests/unit/generate-auto-source-composition.test.ts @@ -0,0 +1,305 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ExtractionData } from '../../src/contracts/types.js' +import type { SemanticProgramIndex } from '../../src/pipeline/spi/types.js' + +const compositionControls = vi.hoisted(() => ({ + legacyTransform: undefined as ((extraction: ExtractionData) => ExtractionData) | undefined, + spiTransform: undefined as ((spi: SemanticProgramIndex) => SemanticProgramIndex) | undefined, +})) + +vi.mock('../../src/pipeline/extract.js', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + extract: (...args: Parameters) => { + const extraction = actual.extract(...args) + return compositionControls.legacyTransform + ? compositionControls.legacyTransform(structuredClone(extraction)) + : extraction + }, + } +}) + +vi.mock('../../src/pipeline/spi/cache.js', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + buildSpiCached: (options: Parameters[0]) => { + const built = actual.buildSpiCached(options) + return compositionControls.spiTransform + ? { ...built, spi: compositionControls.spiTransform(structuredClone(built.spi)) } + : built + }, + } +}) + +import { generateGraph } from '../../src/infrastructure/generate.js' +import { readGeneratedGraphJson } from './helpers/generated-graph.js' + +type GraphNode = Record & { + id: string + label: string + source_file: string + source_location?: string + snippet?: string +} + +function writeFile(root: string, relativePath: string, content: string): void { + const filePath = join(root, relativePath) + mkdirSync(dirname(filePath), { recursive: true }) + writeFileSync(filePath, content, 'utf8') +} + +function generatedNodes(root: string, mode: 'auto' | 'legacy' | 'spi' = 'auto'): GraphNode[] { + const result = generateGraph(root, { extractionMode: mode, noHtml: true }) + return (readGeneratedGraphJson(result.graphPath) as unknown as { nodes: GraphNode[] }).nodes +} + +function nodeByLabel(nodes: GraphNode[], label: string): GraphNode { + const matches = nodes.filter((node) => node.label === label) + expect(matches, `expected one generated node labelled ${label}`).toHaveLength(1) + return matches[0]! +} + +describe('default-auto source composition', () => { + let sandbox: string + + beforeEach(() => { + sandbox = mkdtempSync(join(tmpdir(), 'generate-auto-source-')) + compositionControls.legacyTransform = undefined + compositionControls.spiTransform = undefined + }) + + afterEach(() => { + compositionControls.legacyTransform = undefined + compositionControls.spiTransform = undefined + rmSync(sandbox, { recursive: true, force: true }) + }) + + it('preserves verified generated source on the SPI node', () => { + writeFile(sandbox, 'src/server.ts', [ + 'import express from "express"', + 'export const app = express()', + 'export function owned(): number {', + ' return 1', + '}', + 'app.get("/owned", owned)', + ].join('\n') + '\n') + + const nodes = generatedNodes(sandbox) + const owned = nodeByLabel(nodes, 'owned()') + expect(owned).toMatchObject({ + framework: 'express', + framework_role: 'express_route', + route_path: '/owned', + extraction_strategy: 'spi', + source_location: 'L3-L5', + snippet: 'export function owned(): number {\n return 1\n}', + }) + expect(owned.provenance).toEqual([ + expect.objectContaining({ + source_file: join(sandbox, 'src/server.ts'), + source_location: 'L3', + }), + ]) + }) + + it('rejects wrong-path legacy source ownership', () => { + writeFile(sandbox, 'src/owner.ts', 'export function owned(): number { return 1 }\n') + compositionControls.legacyTransform = (extraction) => ({ + ...extraction, + nodes: extraction.nodes.map((node) => node.label === 'owned()' + ? { + ...node, + source_file: join(sandbox, '..', 'foreign.ts'), + snippet: 'export function owned(): number { return 999 }', + } + : node), + }) + + const owned = nodeByLabel(generatedNodes(sandbox), 'owned()') + expect(owned.source_file).toBe(join(sandbox, 'src/owner.ts')) + expect(owned.source_location).toBe('L1') + expect(owned.snippet).toBeUndefined() + }) + + it.each([ + ['wrong start', 'L2'], + ['reversed range', 'L2-L1'], + ['out-of-anchor range', 'L1-L9'], + ['malformed range', 'line 1'], + ])('rejects a %s in legacy source evidence', (_label, sourceLocation) => { + writeFile(sandbox, 'src/owner.ts', 'export function owned(): number { return 1 }\n') + compositionControls.legacyTransform = (extraction) => ({ + ...extraction, + nodes: extraction.nodes.map((node) => node.label === 'owned()' + ? { ...node, source_location: sourceLocation, snippet: 'export function owned(): number { return 999 }' } + : node), + }) + + const owned = nodeByLabel(generatedNodes(sandbox), 'owned()') + expect(owned.source_location).toBe('L1') + expect(owned.snippet).toBeUndefined() + }) + + it('rejects duplicate legacy declarations and unnormalized snippets', () => { + writeFile(sandbox, 'src/owner.ts', 'export function owned(): number { return 1 }\n') + compositionControls.legacyTransform = (extraction) => { + const owned = extraction.nodes.find((node) => node.label === 'owned()')! + return { + ...extraction, + nodes: [ + ...extraction.nodes, + { ...owned, snippet: 'export function owned(): number { return 999 }' }, + ], + } + } + expect(nodeByLabel(generatedNodes(sandbox), 'owned()').snippet).toBeUndefined() + + rmSync(join(sandbox, '.madar'), { recursive: true, force: true }) + compositionControls.legacyTransform = (extraction) => ({ + ...extraction, + nodes: extraction.nodes.map((node) => node.label === 'owned()' + ? { ...node, snippet: `export function owned() {${'x'.repeat(1_980)}...x` } + : node), + }) + expect(nodeByLabel(generatedNodes(sandbox), 'owned()').snippet).toBeUndefined() + + rmSync(join(sandbox, '.madar'), { recursive: true, force: true }) + compositionControls.legacyTransform = (extraction) => ({ + ...extraction, + nodes: extraction.nodes.map((node) => node.label === 'owned()' + ? { ...node, snippet: 'export function owned(): number {\r\n return 1\r\n}' } + : node), + }) + expect(nodeByLabel(generatedNodes(sandbox), 'owned()').snippet).toBeUndefined() + }) + + it('rejects overload and same-line declaration ambiguity', () => { + writeFile(sandbox, 'src/ambiguous.ts', [ + 'export function overloaded(value: string): string', + 'export function overloaded(value: number): number', + 'export function overloaded(value: string | number): string | number { return value }', + 'export const alpha = 1, beta = 2', + ].join('\n') + '\n') + + const nodes = generatedNodes(sandbox) + expect(nodeByLabel(nodes, 'overloaded()').snippet).toBeUndefined() + expect(nodeByLabel(nodes, 'alpha').snippet).toBeUndefined() + expect(nodeByLabel(nodes, 'beta').snippet).toBeUndefined() + }) + + it('uses SPI exclusive ends without guessing invalid coordinates', () => { + writeFile(sandbox, 'src/owner.ts', [ + 'export function owned(): number {', + ' return 1', + '}', + ].join('\n') + '\n') + compositionControls.spiTransform = (spi) => ({ + ...spi, + symbols: spi.symbols.map((symbol) => symbol.name === 'owned' + ? { ...symbol, range: { ...symbol.range, end: { line: 4, column: 1 } } } + : symbol), + }) + expect(nodeByLabel(generatedNodes(sandbox), 'owned()')).toMatchObject({ + source_location: 'L1-L3', + snippet: 'export function owned(): number {\n return 1\n}', + }) + + rmSync(join(sandbox, '.madar'), { recursive: true, force: true }) + compositionControls.spiTransform = (spi) => ({ + ...spi, + symbols: spi.symbols.map((symbol) => symbol.name === 'owned' + ? { ...symbol, range: { ...symbol.range, end: { ...symbol.range.start } } } + : symbol), + }) + const invalid = nodeByLabel(generatedNodes(sandbox), 'owned()') + expect(invalid.source_location).toBe('L1') + expect(invalid.snippet).toBeUndefined() + }) + + it('preserves first/last, one-line, CRLF, 25/26-line, and 2003-character legacy bytes', () => { + const cases: Array<{ name: string; content: string }> = [ + { name: 'oneLine', content: 'export function oneLine(): number { return 1 }' }, + { name: 'crlfOwner', content: 'export function crlfOwner(): number {\r\n return 1\r\n}' }, + { + name: 'lines25', + content: ['export function lines25(): number {', ...Array.from({ length: 23 }, (_, index) => ` const value${index} = ${index}`), '}'].join('\n'), + }, + { + name: 'lines26', + content: ['export function lines26(): number {', ...Array.from({ length: 24 }, (_, index) => ` const value${index} = ${index}`), '}'].join('\n'), + }, + { + name: 'chars2003', + content: `export function chars2003(): string { return '${'x'.repeat(2_050)}' }`, + }, + ] + + for (const testCase of cases) { + const autoRoot = join(sandbox, `auto-${testCase.name}`) + const legacyRoot = join(sandbox, `legacy-${testCase.name}`) + writeFile(autoRoot, 'src/input.ts', testCase.content) + writeFile(legacyRoot, 'src/input.ts', testCase.content) + const autoNode = nodeByLabel(generatedNodes(autoRoot), `${testCase.name}()`) + const legacyNode = nodeByLabel(generatedNodes(legacyRoot, 'legacy'), `${testCase.name}()`) + expect({ location: autoNode.source_location, snippet: autoNode.snippet }).toEqual({ + location: legacyNode.source_location, + snippet: legacyNode.snippet, + }) + } + + const lines25 = nodeByLabel(generatedNodes(join(sandbox, 'auto-lines25')), 'lines25()') + const lines26 = nodeByLabel(generatedNodes(join(sandbox, 'auto-lines26')), 'lines26()') + const chars2003 = nodeByLabel(generatedNodes(join(sandbox, 'auto-chars2003')), 'chars2003()') + expect(lines25.snippet?.split('\n')).toHaveLength(25) + expect(lines26.source_location).toBe('L1-L26') + expect(lines26.snippet?.split('\n')).toHaveLength(25) + expect(chars2003.snippet).toHaveLength(2_003) + expect(chars2003.snippet?.endsWith('...')).toBe(true) + }, 60_000) + + it('does not enrich a synthetic external call or alter explicit SPI behavior', () => { + const source = [ + 'import { readFileSync } from "node:fs"', + 'export function owned(): string {', + ' return readFileSync("missing", "utf8")', + '}', + ].join('\n') + '\n' + writeFile(sandbox, 'src/owner.ts', source) + const autoNodes = generatedNodes(sandbox) + const external = autoNodes.find((node) => node.label === 'readFileSync()') + expect(external?.snippet).toBeUndefined() + + const spiRoot = join(sandbox, 'explicit-spi') + writeFile(spiRoot, 'src/owner.ts', source) + const explicitSpi = nodeByLabel(generatedNodes(spiRoot, 'spi'), 'owned()') + expect(explicitSpi.source_location).toBe('L2') + expect(explicitSpi.snippet).toBeUndefined() + }) + + it('persists identical source evidence across cold, warm, and canonical reloads', () => { + writeFile(sandbox, 'src/owner.ts', [ + 'export function owned(): number {', + ' return 1', + '}', + ].join('\n') + '\n') + const cold = generateGraph(sandbox, { extractionMode: 'auto', noHtml: true }) + const coldGraph = readGeneratedGraphJson(cold.graphPath) as unknown as { nodes: GraphNode[] } + const warm = generateGraph(sandbox, { extractionMode: 'auto', noHtml: true }) + const warmGraph = readGeneratedGraphJson(warm.graphPath) as unknown as { nodes: GraphNode[] } + expect(cold.cache?.hit).toBe(false) + expect(warm.cache?.hit).toBe(true) + expect(nodeByLabel(warmGraph.nodes, 'owned()')).toMatchObject({ + source_location: 'L1-L3', + snippet: 'export function owned(): number {\n return 1\n}', + }) + expect(warmGraph.nodes.map(({ id, source_file, source_location, snippet }) => ({ id, source_file, source_location, snippet }))) + .toEqual(coldGraph.nodes.map(({ id, source_file, source_location, snippet }) => ({ id, source_file, source_location, snippet }))) + }) +}) diff --git a/tests/unit/generate-spi-flag.test.ts b/tests/unit/generate-spi-flag.test.ts index 0ac308cf..3c7fbbc3 100644 --- a/tests/unit/generate-spi-flag.test.ts +++ b/tests/unit/generate-spi-flag.test.ts @@ -102,6 +102,8 @@ describe('generateGraph capability-aware auto extraction', () => { framework_role: 'express_route', route_path: '/users', extraction_strategy: 'spi', + source_location: 'L3', + snippet: 'export function listUsers(): void {}', }) expect(goMain).toMatchObject({ extraction_strategy: 'legacy_fallback' }) expect(graph.nodes).toEqual(expect.arrayContaining([ @@ -144,6 +146,9 @@ describe('generateGraph capability-aware auto extraction', () => { writeMixedWorkspace() const first = generateGraph(sandbox, { extractionMode: 'auto', noHtml: true }) + const firstGraph = readGeneratedGraphJson(first.graphPath) as { + nodes: Array> + } const second = generateGraph(sandbox, { extractionMode: 'auto', noHtml: true }) const secondGraph = readGeneratedGraphJson(second.graphPath) as { nodes: Array> @@ -156,6 +161,13 @@ describe('generateGraph capability-aware auto extraction', () => { expect(secondGraph.nodes).toEqual(expect.arrayContaining([ expect.objectContaining({ label: 'main()', source_file: expect.stringMatching(/cmd\/main\.go$/) }), ])) + const sourceFields = (nodes: Array>) => nodes.map((node) => ({ + id: node.id, + source_file: node.source_file, + source_location: node.source_location, + snippet: node.snippet, + })) + expect(sourceFields(secondGraph.nodes)).toEqual(sourceFields(firstGraph.nodes)) }) it('does not use the fallback in explicit SPI mode', () => { From 5aaeea965fa23d2bee604cc2ab6983f26b7551ba Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 20:06:45 +0400 Subject: [PATCH 04/27] fix: resolve owner evidence writes by lexical binding identity --- src/runtime/query-evidence-dependencies.ts | 295 +++++++++++++----- .../unit/retrieve-owner-declarations.test.ts | 142 +++++++++ 2 files changed, 355 insertions(+), 82 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 71b510a4..cf2c9283 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -279,60 +279,41 @@ function bindingNames(name: ts.BindingName): string[] { )) } -function assignmentTargetNames(node: ts.Node): string[] { +function assignmentTargetIdentifiers(node: ts.Node): ts.Identifier[] { if (ts.isIdentifier(node)) { - return [node.text] + return [node] } if (ts.isParenthesizedExpression(node)) { - return assignmentTargetNames(node.expression) + return assignmentTargetIdentifiers(node.expression) } if (ts.isArrayLiteralExpression(node)) { - return node.elements.flatMap((element) => assignmentTargetNames(element)) + return node.elements.flatMap((element) => assignmentTargetIdentifiers(element)) } if (ts.isObjectLiteralExpression(node)) { return node.properties.flatMap((property) => { if (ts.isShorthandPropertyAssignment(property)) { - return [property.name.text] + return [property.name] } if (ts.isPropertyAssignment(property)) { - return assignmentTargetNames(property.initializer) + return assignmentTargetIdentifiers(property.initializer) } if (ts.isSpreadAssignment(property)) { - return assignmentTargetNames(property.expression) + return assignmentTargetIdentifiers(property.expression) } return [] }) } - return [] -} - -function writtenBindingNames(owner: FunctionOwner): Set { - const names = new Set() - const visit = (node: ts.Node): void => { - if ( - ts.isBinaryExpression(node) - && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment - && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment - ) { - for (const name of assignmentTargetNames(node.left)) names.add(name) - } else if ( - (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) - && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) - ) { - for (const name of assignmentTargetNames(node.operand)) names.add(name) - } else if (ts.isForInStatement(node) || ts.isForOfStatement(node)) { - if (!ts.isVariableDeclarationList(node.initializer)) { - for (const name of assignmentTargetNames(node.initializer)) names.add(name) - } - } - ts.forEachChild(node, visit) + if (ts.isSpreadElement(node)) { + return assignmentTargetIdentifiers(node.expression) } - visit(owner) - return names + return [] } -function functionScopedVarBindingNames(owner: FunctionOwner): Set { - const names = new Set() +function functionScopedVarDeclarations( + owner: FunctionOwner, + expected: string, +): ts.VariableDeclaration[] { + const declarations: ts.VariableDeclaration[] = [] const visit = (node: ts.Node): void => { if (node !== owner && isNestedFunctionOrClass(node)) { return @@ -342,13 +323,15 @@ function functionScopedVarBindingNames(owner: FunctionOwner): Set { && (node.flags & ts.NodeFlags.BlockScoped) === 0 ) { for (const declaration of node.declarations) { - for (const name of bindingNames(declaration.name)) names.add(name) + if (hasBindingName(declaration.name, expected)) { + declarations.push(declaration) + } } } ts.forEachChild(node, visit) } visit(owner) - return names + return declarations } type LexicalScope = @@ -381,31 +364,132 @@ function lexicalScopeChain(node: ts.Node, owner: FunctionOwner): LexicalScope[] return ts.isBlock(owner.body) && scopes.includes(owner.body) ? scopes : null } +type LexicalEnvironment = ( + | LexicalScope + | FunctionOwner + | ts.ClassDeclaration + | ts.ClassExpression +) + +interface ResolvedLexicalBinding { + binding: ConstBinding | null + identity: ts.Node | null + invalidDependency: boolean +} + +function lexicalEnvironmentChain( + node: ts.Node, + owner: FunctionOwner, + crossNestedOwners: boolean, +): LexicalEnvironment[] | null { + const environments: LexicalEnvironment[] = [] + let foundOwner = false + let current: ts.Node | undefined = node.parent + while (current) { + const functionOwner = functionOwnerWithBody(current) + if (functionOwner) { + if (functionOwner !== owner && !crossNestedOwners) { + return null + } + const preceding = environments.length - 1 + if (preceding >= 0 && environments[preceding] === functionOwner.body) { + environments.splice(preceding, 0, functionOwner) + } else { + environments.push(functionOwner) + } + if (functionOwner === owner) { + foundOwner = true + break + } + } else if (ts.isClassDeclaration(current) || ts.isClassExpression(current)) { + environments.push(current) + } else if ( + ts.isBlock(current) + || ts.isCaseBlock(current) + || ts.isCatchClause(current) + || ts.isForStatement(current) + || ts.isForInStatement(current) + || ts.isForOfStatement(current) + ) { + environments.push(current) + } + current = current.parent + } + return foundOwner ? environments : null +} + function hasBindingName(name: ts.BindingName, expected: string): boolean { return bindingNames(name).includes(expected) } -function constBindingForReference( +function resolveLexicalBinding( identifier: ts.Identifier, owner: FunctionOwner, - writtenNames: ReadonlySet, - functionVarNames: ReadonlySet, -): { binding: ConstBinding | null; invalidDependency: boolean } { - const scopes = lexicalScopeChain(identifier, owner) - if (!scopes || functionVarNames.has(identifier.text)) { - return { binding: null, invalidDependency: false } + crossNestedOwners: boolean, +): ResolvedLexicalBinding { + let environments: LexicalEnvironment[] | null + if (crossNestedOwners) { + environments = lexicalEnvironmentChain(identifier, owner, true) + } else { + const scopes = lexicalScopeChain(identifier, owner) + environments = scopes + ? scopes.flatMap((scope) => ( + scope === owner.body ? [owner, scope] : [scope] + )) + : null + } + if (!environments) { + return { binding: null, identity: null, invalidDependency: false } } - for (const scope of scopes) { - if (scope === owner.body && owner.parameters.some((parameter) => hasBindingName(parameter.name, identifier.text))) { - return { binding: null, invalidDependency: false } + for (const environment of environments) { + const functionOwner = functionOwnerWithBody(environment) + if (functionOwner) { + const parameter = functionOwner.parameters.find((candidate) => ( + hasBindingName(candidate.name, identifier.text) + )) + const functionName = ( + (ts.isFunctionDeclaration(functionOwner) || ts.isFunctionExpression(functionOwner)) + && functionOwner.name?.text === identifier.text + ) + ? functionOwner + : null + const varDeclarations = functionScopedVarDeclarations(functionOwner, identifier.text) + const identity = parameter ?? functionName ?? varDeclarations[0] ?? null + if (identity) { + return { binding: null, identity, invalidDependency: false } + } + continue + } + + if ( + (ts.isClassDeclaration(environment) || ts.isClassExpression(environment)) + && environment.name?.text === identifier.text + ) { + return { binding: null, identity: environment, invalidDependency: false } } if ( - ts.isCatchClause(scope) - && scope.variableDeclaration - && hasBindingName(scope.variableDeclaration.name, identifier.text) + ts.isCatchClause(environment) + && environment.variableDeclaration + && hasBindingName(environment.variableDeclaration.name, identifier.text) ) { - return { binding: null, invalidDependency: false } + return { + binding: null, + identity: environment.variableDeclaration, + invalidDependency: false, + } + } + if (ts.isForStatement(environment) || ts.isForInStatement(environment) || ts.isForOfStatement(environment)) { + const initializer = environment.initializer + if (initializer && ts.isVariableDeclarationList(initializer)) { + const declaration = initializer.declarations.find((candidate) => ( + hasBindingName(candidate.name, identifier.text) + )) + if (declaration) { + return { binding: null, identity: declaration, invalidDependency: false } + } + } + continue } const declarations: Array<{ @@ -413,11 +497,11 @@ function constBindingForReference( statement: ts.VariableStatement declarationKind: 'const' | 'mutable' }> = [] - let unsupportedBinding = false - const statements = ts.isBlock(scope) - ? scope.statements - : ts.isCaseBlock(scope) - ? scope.clauses.flatMap((clause) => [...clause.statements]) + const unsupportedBindings: ts.Node[] = [] + const statements = ts.isBlock(environment) + ? environment.statements + : ts.isCaseBlock(environment) + ? environment.clauses.flatMap((clause) => [...clause.statements]) : [] for (const statement of statements) { if (ts.isVariableStatement(statement)) { @@ -435,23 +519,19 @@ function constBindingForReference( ((ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement) || ts.isEnumDeclaration(statement)) && statement.name?.text === identifier.text) ) { - unsupportedBinding = true - } - } - if (ts.isForStatement(scope) || ts.isForInStatement(scope) || ts.isForOfStatement(scope)) { - const initializer = scope.initializer - if (initializer && ts.isVariableDeclarationList(initializer)) { - unsupportedBinding ||= initializer.declarations.some((declaration) => ( - hasBindingName(declaration.name, identifier.text) - )) + unsupportedBindings.push(statement) } } - if (declarations.length === 0 && !unsupportedBinding) { + if (declarations.length === 0 && unsupportedBindings.length === 0) { continue } - if (declarations.length !== 1 || unsupportedBinding) { - return { binding: null, invalidDependency: true } + if (declarations.length !== 1 || unsupportedBindings.length > 0) { + return { + binding: null, + identity: declarations[0]?.declaration ?? unsupportedBindings[0] ?? environment, + invalidDependency: true, + } } const candidate = declarations[0]! @@ -459,32 +539,83 @@ function constBindingForReference( candidate.declarationKind !== 'const' || !ts.isIdentifier(candidate.declaration.name) || !candidate.declaration.initializer - || writtenNames.has(identifier.text) ) { - return { binding: null, invalidDependency: false } - } - if (candidate.declaration.getStart() >= identifier.getStart()) { - return { binding: null, invalidDependency: true } - } - if (candidate.statement.declarationList.declarations.some((declaration) => ( - !ts.isIdentifier(declaration.name) || !declaration.initializer - ))) { - return { binding: null, invalidDependency: true } + return { + binding: null, + identity: candidate.declaration, + invalidDependency: false, + } } return { binding: { declaration: candidate.declaration, statement: candidate.statement }, + identity: candidate.declaration, invalidDependency: false, } } - return { binding: null, invalidDependency: false } + return { binding: null, identity: null, invalidDependency: false } +} + +function writtenBindings(owner: FunctionOwner): Set { + const bindings = new Set() + const visit = (node: ts.Node): void => { + let targets: ts.Identifier[] = [] + if ( + ts.isBinaryExpression(node) + && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ) { + targets = assignmentTargetIdentifiers(node.left) + } else if ( + (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) + && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) + ) { + targets = assignmentTargetIdentifiers(node.operand) + } else if (ts.isForInStatement(node) || ts.isForOfStatement(node)) { + if (!ts.isVariableDeclarationList(node.initializer)) { + targets = assignmentTargetIdentifiers(node.initializer) + } + } + for (const target of targets) { + const resolved = resolveLexicalBinding(target, owner, true) + if (resolved.identity) { + bindings.add(resolved.identity) + } + } + ts.forEachChild(node, visit) + } + visit(owner) + return bindings +} + +function constBindingForReference( + identifier: ts.Identifier, + owner: FunctionOwner, + written: ReadonlySet, +): { binding: ConstBinding | null; invalidDependency: boolean } { + const resolved = resolveLexicalBinding(identifier, owner, false) + if (!resolved.binding || resolved.invalidDependency) { + return { binding: null, invalidDependency: resolved.invalidDependency } + } + const candidate = resolved.binding + if (candidate.declaration.getStart() >= identifier.getStart()) { + return { binding: null, invalidDependency: true } + } + if (candidate.statement.declarationList.declarations.some((declaration) => ( + !ts.isIdentifier(declaration.name) || !declaration.initializer + ))) { + return { binding: null, invalidDependency: true } + } + if (written.has(candidate.declaration)) { + return { binding: null, invalidDependency: false } + } + return { binding: candidate, invalidDependency: false } } function declarationClosure( owner: FunctionOwner, seedStatements: readonly ts.Statement[], ): ts.VariableStatement[] | null { - const writtenNames = writtenBindingNames(owner) - const functionVarNames = functionScopedVarBindingNames(owner) + const written = writtenBindings(owner) const statements = new Set() const completed = new Set() const visiting = new Set() @@ -502,7 +633,7 @@ function declarationClosure( return false } for (const reference of references.identifiers) { - const resolved = constBindingForReference(reference, owner, writtenNames, functionVarNames) + const resolved = constBindingForReference(reference, owner, written) if (resolved.invalidDependency) { return false } @@ -519,7 +650,7 @@ function declarationClosure( for (const statement of seedStatements) { const references = collectValueReferences(statement, false) for (const reference of references.identifiers) { - const resolved = constBindingForReference(reference, owner, writtenNames, functionVarNames) + const resolved = constBindingForReference(reference, owner, written) if (resolved.invalidDependency) { continue } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 1d57df44..1f0fe412 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -590,6 +590,148 @@ describe('owner-local declaration completion', () => { }) }) +describe('owner-local binding identity correction', () => { + it.each([ + { + barrier: 'function-scoped var', + source: [ + 'export function executeSample() {', + " var datum = 'function binding'", + ' {', + " const datum = 'block binding'", + ' return dispatchOutcome(datum)', + ' }', + '}', + ], + forbidden: 'var datum', + }, + { + barrier: 'owner parameter', + source: [ + 'export function executeSample(datum: string) {', + ' {', + " const datum = 'block binding'", + ' return dispatchOutcome(datum)', + ' }', + '}', + ], + forbidden: 'executeSample(datum', + }, + ])('resolves a closer block const before a $barrier barrier', ({ source, forbidden }) => { + const evidence = evidenceFor(source) + + expect(evidence?.snippet).toContain("const datum = 'block binding'") + expect(evidence?.snippet).not.toContain(forbidden) + }) + + it.each([ + { + location: 'sibling block write', + source: [ + 'export function executeSample() {', + " const datum = 'owner binding'", + " { let datum = 'sibling binding'; datum = 'changed' }", + ' return dispatchOutcome(datum)', + '}', + ], + }, + { + location: 'nested-owner write', + source: [ + 'export function executeSample() {', + " const datum = 'owner binding'", + " function changeNested() { let datum = 'nested binding'; datum = 'changed' }", + ' return dispatchOutcome(datum)', + '}', + ], + }, + ])('does not let a $location to a different binding poison the owner const', ({ source }) => { + const evidence = evidenceFor(source) + + expect(evidence?.snippet).toContain("L2: const datum = 'owner binding'") + expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') + }) + + it('resolves a transitive const chain through sibling and nested-owner shadow writes', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const origin = 12.5', + ' const alias = origin', + ' { let origin = 99; origin = 100 }', + " function changeNested() { let alias = 'nested'; alias = 'changed' }", + ' return dispatchOutcome(alias)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const origin = 12.5', + 'L3: const alias = origin', + 'L6: return dispatchOutcome(alias)', + ].join('\n')) + }) + + it.each([ + { + location: 'sibling block', + source: [ + 'export function executeSample() {', + " const datum = 'owner binding'", + " { datum = 'changed' }", + ' return dispatchOutcome(datum)', + '}', + ], + }, + { + location: 'nested owner', + source: [ + 'export function executeSample() {', + " const datum = 'owner binding'", + " function changeNested() { datum = 'changed' }", + ' return dispatchOutcome(datum)', + '}', + ], + }, + ])('rejects the owner const when a $location writes that same binding', ({ source }) => { + const evidence = evidenceFor(source) + + expect(evidence?.snippet).not.toContain("const datum = 'owner binding'") + expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') + }) + + it.each([ + { + barrier: 'for-loop binding', + source: [ + 'export function executeSample(values: string[]) {', + " const datum = 'owner binding'", + ' for (const datum of values) {', + ' return dispatchOutcome(datum)', + ' }', + ' return retryOutcome()', + '}', + ], + }, + { + barrier: 'catch binding', + source: [ + 'export function executeSample() {', + " const datum = 'owner binding'", + ' try {', + " throw new Error('failed')", + ' } catch (datum) {', + ' return dispatchOutcome(datum)', + ' }', + '}', + ], + }, + ])('keeps a nearer $barrier as a barrier to the outer const', ({ source }) => { + const evidence = evidenceFor(source) + + expect(evidence?.snippet).not.toContain("const datum = 'owner binding'") + expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') + }) +}) + describe('owner declaration completion budgets and atomic preservation', () => { it('admits a declaration closure that exactly fills the 300-character cap', () => { const returnSource = ` return dispatchOutcome(datum, '${'r'.repeat(48)}')` From b44b86eb9eedda71163877735c8a10ec37af5f7f Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 20:16:46 +0400 Subject: [PATCH 05/27] fix: identify auto source owners by qualified method and location --- src/infrastructure/generate.ts | 14 ++- .../generate-auto-source-composition.test.ts | 105 ++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/src/infrastructure/generate.ts b/src/infrastructure/generate.ts index 994ef904..031dffb1 100644 --- a/src/infrastructure/generate.ts +++ b/src/infrastructure/generate.ts @@ -493,14 +493,20 @@ function composeDefaultAutoSourceEvidence(args: { if (!projectedStart || projectedStart.start !== projectedStart.end) return spiNode const fileSymbols = symbolsByFile.get(spiFile) ?? [] - const compatibleSymbols = fileSymbols.filter((symbol) => - expectedSpiLabel(symbol) === spiNode.label && compatibleSpiNodeKind(symbol, spiNode), - ) + const compatibleSymbols = fileSymbols.filter((symbol) => { + const range = validSpiOwnerRange(symbol) + return range?.start === projectedStart.start + && expectedSpiLabel(symbol) === spiNode.label + && compatibleSpiNodeKind(symbol, spiNode) + }) if (compatibleSymbols.length !== 1) return spiNode const symbol = compatibleSymbols[0]! const ownerRange = validSpiOwnerRange(symbol) if (!ownerRange || ownerRange.start !== projectedStart.start) return spiNode - if (fileSymbols.some((other) => other !== symbol && validSpiOwnerRange(other)?.start === ownerRange.start)) return spiNode + if (fileSymbols.some((other) => other !== symbol && ( + validSpiOwnerRange(other)?.start === ownerRange.start + || (other.name === symbol.name && other.kind === symbol.kind) + ))) return spiNode const owner: AutoSourceOwner = { file: spiFile, start: ownerRange.start, end: ownerRange.end, symbol } const legacySourceLocation = legacyNode.source_location diff --git a/tests/unit/generate-auto-source-composition.test.ts b/tests/unit/generate-auto-source-composition.test.ts index b9ba8fab..d3e494f9 100644 --- a/tests/unit/generate-auto-source-composition.test.ts +++ b/tests/unit/generate-auto-source-composition.test.ts @@ -66,6 +66,12 @@ function nodeByLabel(nodes: GraphNode[], label: string): GraphNode { return matches[0]! } +function nodeById(nodes: GraphNode[], id: string): GraphNode { + const matches = nodes.filter((node) => node.id === id) + expect(matches, 'expected one generated node with ID ' + id).toHaveLength(1) + return matches[0]! +} + describe('default-auto source composition', () => { let sandbox: string @@ -109,6 +115,105 @@ describe('default-auto source composition', () => { ]) }) + it('preserves each qualified owner when distinct classes use the same method name', () => { + const source = [ + 'export class A {', + ' run() {', + " return 'alpha'", + ' }', + '}', + '', + 'export class B {', + ' run() {', + " return 'beta'", + ' }', + '}', + ].join('\n') + '\n' + writeFile(sandbox, 'src/methods.js', source) + + let rawSpi: SemanticProgramIndex | undefined + let rawLegacy: ExtractionData | undefined + compositionControls.spiTransform = (spi) => { + rawSpi = spi + return spi + } + compositionControls.legacyTransform = (extraction) => { + rawLegacy = extraction + return extraction + } + + const nodes = generatedNodes(sandbox) + const spiMethods = rawSpi!.symbols + .filter((symbol) => symbol.kind === 'method' && (symbol.name === 'A.run' || symbol.name === 'B.run')) + .sort((left, right) => left.range.start.line - right.range.start.line) + expect(spiMethods.map((symbol) => ({ + name: symbol.name, + kind: symbol.kind, + start: symbol.range.start.line, + end: symbol.range.end.line, + }))).toEqual([ + { name: 'A.run', kind: 'method', start: 2, end: 4 }, + { name: 'B.run', kind: 'method', start: 8, end: 10 }, + ]) + + const legacyMethods = rawLegacy!.nodes + .filter((node) => node.id === 'methods_a_run' || node.id === 'methods_b_run') + .sort((left, right) => left.id.localeCompare(right.id)) + expect(legacyMethods.map(({ id, source_location, snippet }) => ({ id, source_location, snippet }))).toEqual([ + { + id: 'methods_a_run', + source_location: 'L2-L4', + snippet: "run() {\n return 'alpha'\n }", + }, + { + id: 'methods_b_run', + source_location: 'L8-L10', + snippet: "run() {\n return 'beta'\n }", + }, + ]) + + expect(nodeById(nodes, 'methods_a_run')).toMatchObject({ + label: '.run()', + source_location: 'L2-L4', + snippet: "run() {\n return 'alpha'\n }", + }) + expect(nodeById(nodes, 'methods_b_run')).toMatchObject({ + label: '.run()', + source_location: 'L8-L10', + snippet: "run() {\n return 'beta'\n }", + }) + }) + + it('does not borrow same-named method evidence from another qualified destination', () => { + writeFile(sandbox, 'src/methods.js', [ + 'export class A {', + ' run() {', + " return 'alpha'", + ' }', + '}', + '', + 'export class B {', + ' run() {', + " return 'beta'", + ' }', + '}', + ].join('\n') + '\n') + compositionControls.legacyTransform = (extraction) => ({ + ...extraction, + nodes: extraction.nodes.map((node) => node.id === 'methods_a_run' + ? { + ...node, + source_location: 'L8-L10', + snippet: "run() {\n return 'beta'\n }", + } + : node), + }) + + const nodes = generatedNodes(sandbox) + expect(nodeById(nodes, 'methods_a_run')).toMatchObject({ source_location: 'L2' }) + expect(nodeById(nodes, 'methods_a_run').snippet).toBeUndefined() + }) + it('rejects wrong-path legacy source ownership', () => { writeFile(sandbox, 'src/owner.ts', 'export function owned(): number { return 1 }\n') compositionControls.legacyTransform = (extraction) => ({ From 3657f47a2c8cf3664ceb1d96663c1399e0a590a7 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 20:29:15 +0400 Subject: [PATCH 06/27] fix: visit lexical body before function environment --- src/runtime/query-evidence-dependencies.ts | 9 +-- .../unit/retrieve-owner-declarations.test.ts | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index cf2c9283..4e46c1cb 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -391,12 +391,7 @@ function lexicalEnvironmentChain( if (functionOwner !== owner && !crossNestedOwners) { return null } - const preceding = environments.length - 1 - if (preceding >= 0 && environments[preceding] === functionOwner.body) { - environments.splice(preceding, 0, functionOwner) - } else { - environments.push(functionOwner) - } + environments.push(functionOwner) if (functionOwner === owner) { foundOwner = true break @@ -434,7 +429,7 @@ function resolveLexicalBinding( const scopes = lexicalScopeChain(identifier, owner) environments = scopes ? scopes.flatMap((scope) => ( - scope === owner.body ? [owner, scope] : [scope] + scope === owner.body ? [scope, owner] : [scope] )) : null } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 1f0fe412..777d1843 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -591,6 +591,61 @@ describe('owner-local declaration completion', () => { }) describe('owner-local binding identity correction', () => { + it('resolves a function-declaration body const before the function name', () => { + const evidence = evidenceFor([ + 'export function assemble() {', + ' const assemble = 12.5', + ' return { displayedText: assemble }', + '}', + ], { question: 'What is the displayed text?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const assemble = 12.5', + 'L3: return { displayedText: assemble }', + ].join('\n')) + }) + + it('resolves a named-function-expression body const before the function name', () => { + const evidence = evidenceFor([ + 'export const assemble = function assemble() {', + ' const assemble = 12.5', + ' return { displayedText: assemble }', + '}', + ], { question: 'What is the displayed text?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const assemble = 12.5', + 'L3: return { displayedText: assemble }', + ].join('\n')) + }) + + it('does not let a nested named-owner shadow write poison the body const', () => { + const evidence = evidenceFor([ + 'export function assemble() {', + ' const assemble = 12.5', + ' const changeNested = function assemble() { assemble = 99 }', + ' return { displayedText: assemble }', + '}', + ], { question: 'What is the displayed text?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const assemble = 12.5', + 'L4: return { displayedText: assemble }', + ].join('\n')) + }) + + it('rejects the body const across a genuine nested-owner captured write', () => { + const evidence = evidenceFor([ + 'export function assemble() {', + ' const assemble = 12.5', + ' const changeNested = () => { assemble = 99 }', + ' return { displayedText: assemble }', + '}', + ], { question: 'What is the displayed text?', label: 'assemble' }) + + expect(evidence?.snippet).toBe('L4: return { displayedText: assemble }') + }) + it.each([ { barrier: 'function-scoped var', From 106d7e8ce2cb6828bfdb718ff312318e946da498 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 20:57:18 +0400 Subject: [PATCH 07/27] fix: match represented owner statements through physical source trivia --- src/runtime/query-evidence-dependencies.ts | 15 +++++- .../unit/retrieve-owner-declarations.test.ts | 46 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 4e46c1cb..d47ec961 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -171,6 +171,19 @@ function normalizedSource(value: string): string { return value.replace(/\s+/g, ' ').trim() } +function physicalSourceForLineRange( + range: { start: number; end: number }, + sourceFile: ts.SourceFile, +): string { + const lineStarts = sourceFile.getLineStarts() + const start = lineStarts[range.start - 1] + if (start === undefined) { + return '' + } + const end = lineStarts[range.end] ?? sourceFile.text.length + return sourceFile.text.slice(start, end) +} + function representedStatements( owner: FunctionOwner, representedSource: readonly RepresentedQueryEvidenceSource[], @@ -193,7 +206,7 @@ function representedStatements( if (ts.isStatement(node)) { const range = lineRangeOf(node, sourceFile) const texts = representedByRange.get(`${range.start}:${range.end}`) - if (texts?.has(normalizedSource(node.getText(sourceFile)))) { + if (texts?.has(normalizedSource(physicalSourceForLineRange(range, sourceFile)))) { statements.push(node) } } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 777d1843..e3a5607f 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -259,6 +259,52 @@ describe('owner-local declaration completion', () => { ].join('\n')) }) + it.each([ + { + name: 'ordinary trailing line comment', + statement: ' return dispatchOutcome(datum) // selected result', + }, + { + name: 'trailing block comment', + statement: ' return dispatchOutcome(datum) /* selected result */', + }, + { + name: 'leading and trailing block-comment trivia', + statement: ' /* selected result */ return dispatchOutcome(datum) /* trailing context */', + }, + { + name: 'comment-like string and template contents with trailing trivia', + statement: ' return dispatchOutcome(datum, "// literal", `/* template */`) // selected result', + }, + ])('attributes a fully represented statement with $name', ({ statement }) => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + statement, + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5', + `L3: ${statement.trim()}`, + ].join('\n')) + }) + + it('does not attribute a statement from a truncated partial physical line', () => { + const statement = ` return dispatchOutcome(datum) // ${'partial'.repeat(40)}` + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + statement, + '}', + ]) + + expect(evidence?.snippet).not.toContain('const datum = 12.5') + expect(evidence?.snippet).toContain('return dispatchOutcome(datum)') + expect(evidence?.snippet).toContain('...') + expect(evidence?.snippet).not.toContain(statement.trim()) + }) + it('uses a selected multiline statement only when its complete text is represented', () => { const evidence = evidenceFor([ 'export function executeSample() {', From 3ae84bf370452bc950195fecb8c9817da933b830 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 21:07:51 +0400 Subject: [PATCH 08/27] fix: reject ambiguous normalized source projection destinations --- src/infrastructure/generate.ts | 48 +++++++---- src/pipeline/spi/projector.ts | 6 +- .../generate-auto-source-composition.test.ts | 81 +++++++++++++++++++ 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/src/infrastructure/generate.ts b/src/infrastructure/generate.ts index 031dffb1..429bb2e1 100644 --- a/src/infrastructure/generate.ts +++ b/src/infrastructure/generate.ts @@ -1,5 +1,5 @@ import { existsSync, mkdirSync, readFileSync } from 'node:fs' -import { isAbsolute, join, relative, resolve, sep } from 'node:path' +import { basename, extname, isAbsolute, join, relative, resolve, sep } from 'node:path' import { KnowledgeGraph } from '../contracts/graph.js' import { rebindEvidenceOccurrence } from '../contracts/semantic-identity.js' @@ -39,8 +39,13 @@ import { import { createIndexingManifest, indexingStrictViolations, localIndexingPath } from '../pipeline/indexing-outcomes.js' import { buildSpiCached, type SpiCacheStats } from '../pipeline/spi/cache.js' import { isSpiSupportedSourceFile } from '../pipeline/spi/build.js' -import { projectSpiToExtraction } from '../pipeline/spi/projector.js' -import type { SemanticProgramIndex, SpiSymbol, SpiSymbolKind } from '../pipeline/spi/types.js' +import { + createProjectedFileStemById, + PROJECTABLE_SYMBOL_KINDS, + projectSpiToExtraction, + projectSymbol, +} from '../pipeline/spi/projector.js' +import type { SemanticProgramIndex, SpiSymbol } from '../pipeline/spi/types.js' import { generate as generateReport } from '../pipeline/report.js' import { toWiki } from '../pipeline/wiki.js' import { loadGraph } from '../runtime/serve.js' @@ -284,18 +289,6 @@ const AUTO_SOURCE_LOCATION = /^L([1-9]\d*)(?:-L([1-9]\d*))?$/ const AUTO_SOURCE_MAX_LINES = 25 const AUTO_SOURCE_MAX_CHARS = 2_000 const AUTO_SOURCE_MAX_TRUNCATED_CHARS = AUTO_SOURCE_MAX_CHARS + 3 -const AUTO_SOURCE_PROJECTABLE_KINDS: ReadonlySet = new Set([ - 'function', - 'class', - 'interface', - 'type-alias', - 'enum', - 'method', - 'constant', - 'variable', - 'namespace', -]) - type AutoSourceRange = { start: number end: number @@ -375,7 +368,7 @@ function validSpiOwnerRange(symbol: SpiSymbol): AutoSourceRange | null { } function expectedSpiLabel(symbol: SpiSymbol): string | null { - if (!AUTO_SOURCE_PROJECTABLE_KINDS.has(symbol.kind) || symbol.framework_metadata?.external_call === true) { + if (!PROJECTABLE_SYMBOL_KINDS.has(symbol.kind) || symbol.framework_metadata?.external_call === true) { return null } if (symbol.kind === 'method') { @@ -430,6 +423,7 @@ function composeDefaultAutoSourceEvidence(args: { spi: SemanticProgramIndex legacy: ExtractionData projectedSpi: ExtractionData + sharedFileStems?: ReadonlyMap }): { legacy: ExtractionData; projectedSpi: ExtractionData } { const rootPath = resolve(args.rootPath) const allowedFiles = new Set( @@ -439,6 +433,26 @@ function composeDefaultAutoSourceEvidence(args: { }), ) + const projectedFileById = new Map(args.spi.files.map((file) => [file.id, file])) + const projectedFileStemById = createProjectedFileStemById( + args.spi.files, + rootPath, + args.sharedFileStems, + ) + const rawProjectionCountByDestination = new Map() + for (const symbol of args.spi.symbols) { + if (!PROJECTABLE_SYMBOL_KINDS.has(symbol.kind)) continue + const file = projectedFileById.get(symbol.file_id) + if (!file) continue + const fileBaseStem = projectedFileStemById.get(file.id) ?? basename(file.path, extname(file.path)) + const projection = projectSymbol(symbol, fileBaseStem) + if (!projection) continue + rawProjectionCountByDestination.set( + projection.id, + (rawProjectionCountByDestination.get(projection.id) ?? 0) + 1, + ) + } + const spiFilesById = new Map() const spiFilesByPath = new Map() for (const file of args.spi.files) { @@ -479,6 +493,7 @@ function composeDefaultAutoSourceEvidence(args: { const composedNodeIds = new Set() const composedProjectedNodes = args.projectedSpi.nodes.map((spiNode) => { + if (rawProjectionCountByDestination.get(spiNode.id) !== 1) return spiNode const legacyNodes = legacyById.get(spiNode.id) if (!legacyNodes || legacyNodes.length !== 1 || projectedById.get(spiNode.id)?.length !== 1) return spiNode const legacyNode = legacyNodes[0]! @@ -1084,6 +1099,7 @@ export function generateGraph(rootPath = '.', options: GenerateGraphOptions = {} spi: built.spi, legacy: legacyAugmentationExtraction, projectedSpi: spiSupplementalExtraction, + ...(sharedFileStems ? { sharedFileStems } : {}), }) : null const codeExtraction = extractionMode === 'auto' diff --git a/src/pipeline/spi/projector.ts b/src/pipeline/spi/projector.ts index 8b4a9489..a0576994 100644 --- a/src/pipeline/spi/projector.ts +++ b/src/pipeline/spi/projector.ts @@ -84,7 +84,7 @@ export type ProjectSpiToExtractionOptions = { fileStemByAbsolutePath?: ReadonlyMap } -const PROJECTABLE_SYMBOL_KINDS: ReadonlySet = new Set([ +export const PROJECTABLE_SYMBOL_KINDS: ReadonlySet = new Set([ 'function', 'class', 'interface', @@ -356,7 +356,7 @@ function normalizeStemPath(filePath: string): string { return resolve(filePath).replaceAll('\\', '/') } -function createProjectedFileStemById( +export function createProjectedFileStemById( files: readonly SpiFile[], root: string, fileStemByAbsolutePath?: ReadonlyMap, @@ -393,7 +393,7 @@ function uniqueProjectedFileStem(filePath: string): string { return withoutExtension.split('/').filter(Boolean).join('_') } -function projectSymbol(symbol: SpiSymbol, fileBaseStem: string): SymbolProjection | null { +export function projectSymbol(symbol: SpiSymbol, fileBaseStem: string): SymbolProjection | null { const storageOperation = typeof symbol.framework_metadata?.storage_operation === 'string' ? symbol.framework_metadata.storage_operation : null diff --git a/tests/unit/generate-auto-source-composition.test.ts b/tests/unit/generate-auto-source-composition.test.ts index d3e494f9..4f584167 100644 --- a/tests/unit/generate-auto-source-composition.test.ts +++ b/tests/unit/generate-auto-source-composition.test.ts @@ -184,6 +184,87 @@ describe('default-auto source composition', () => { }) }) + it.each([ + ['case', 'A', 'a'], + ['leading underscore', 'A', '_A'], + ])('does not compose source for a %s-normalized destination collision', (_collision, firstClass, secondClass) => { + writeFile(sandbox, 'src/methods.js', [ + `export class ${firstClass} {`, + ' run() {', + " return 'first'", + ' }', + '}', + '', + `export class ${secondClass} {`, + ' run() {', + " return 'second'", + ' }', + '}', + '', + 'export class B {', + ' run() {', + " return 'distinct'", + ' }', + '}', + ].join('\n') + '\n') + + let rawSpi: SemanticProgramIndex | undefined + let rawLegacy: ExtractionData | undefined + compositionControls.spiTransform = (spi) => { + const distinct = spi.symbols.find((symbol) => symbol.kind === 'method' && symbol.name === 'B.run')! + rawSpi = { + ...spi, + symbols: [ + ...[...spi.symbols].sort((left, right) => left.range.start.line - right.range.start.line), + { + ...distinct, + id: `${distinct.id}:missing-file-peer`, + file_id: 'file:missing-file-peer', + }, + ], + } + return rawSpi + } + compositionControls.legacyTransform = (extraction) => { + rawLegacy = extraction + return extraction + } + + const nodes = generatedNodes(sandbox) + const collidingSymbols = rawSpi!.symbols.filter((symbol) => ( + symbol.kind === 'method' + && (symbol.name === `${firstClass}.run` || symbol.name === `${secondClass}.run`) + )).sort((left, right) => left.range.start.line - right.range.start.line) + expect(collidingSymbols.map((symbol) => ({ name: symbol.name, start: symbol.range.start.line }))).toEqual([ + { name: `${firstClass}.run`, start: 2 }, + { name: `${secondClass}.run`, start: 8 }, + ]) + + const collidedLegacy = rawLegacy!.nodes.find((node) => ( + node.label === '.run()' && node.source_location === 'L2-L4' + ))! + expect(collidedLegacy).toMatchObject({ + snippet: "run() {\n return 'first'\n }", + }) + const collidedDestination = nodeById(nodes, collidedLegacy.id) + expect(collidedDestination.source_location).toBe('L2') + expect(collidedDestination.snippet).toBeUndefined() + + const distinctLegacy = rawLegacy!.nodes.find((node) => ( + node.label === '.run()' && node.source_location === 'L14-L16' + ))! + const distinctDestination = nodeById(nodes, distinctLegacy.id) + expect(distinctDestination).toMatchObject({ + source_location: 'L14-L16', + snippet: "run() {\n return 'distinct'\n }", + }) + expect(rawSpi!.symbols).toContainEqual(expect.objectContaining({ + id: expect.stringContaining(':missing-file-peer'), + file_id: 'file:missing-file-peer', + name: 'B.run', + })) + }) + it('does not borrow same-named method evidence from another qualified destination', () => { writeFile(sandbox, 'src/methods.js', [ 'export class A {', From 875fd1e62f1a179b06ddf20e42ffd583f037fa11 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 21:32:57 +0400 Subject: [PATCH 09/27] fix: preserve complete comment trivia in owner declaration evidence --- src/runtime/query-evidence-dependencies.ts | 57 +++++- .../unit/retrieve-owner-declarations.test.ts | 192 ++++++++++++++++++ 2 files changed, 246 insertions(+), 3 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index d47ec961..90d81a0f 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -110,6 +110,49 @@ function lineRangeOf(node: ts.Node, sourceFile: ts.SourceFile): { start: number; return { start, end } } +function evidenceLineRangeOf( + statement: ts.Statement, + sourceFile: ts.SourceFile, +): { start: number; end: number } { + const tokenRange = lineRangeOf(statement, sourceFile) + let start = tokenRange.start + let end = tokenRange.end + + const leadingComments = ts.getLeadingCommentRanges( + sourceFile.text, + statement.getFullStart(), + ) ?? [] + for (let index = leadingComments.length - 1; index >= 0; index -= 1) { + const comment = leadingComments[index]! + const commentEnd = sourceFile.getLineAndCharacterOfPosition( + Math.max(comment.pos, comment.end - 1), + ).line + 1 + if (commentEnd !== start) { + break + } + start = sourceFile.getLineAndCharacterOfPosition(comment.pos).line + 1 + } + + const trailingComments = ts.getTrailingCommentRanges( + sourceFile.text, + statement.getEnd(), + ) ?? [] + for (const comment of trailingComments) { + const commentStart = sourceFile.getLineAndCharacterOfPosition(comment.pos).line + 1 + if (commentStart > end) { + break + } + end = Math.max( + end, + sourceFile.getLineAndCharacterOfPosition( + Math.max(comment.pos, comment.end - 1), + ).line + 1, + ) + } + + return { start, end } +} + function functionOwnerWithBody(node: ts.Node): FunctionOwner | null { if ( !ts.isFunctionDeclaration(node) @@ -189,6 +232,7 @@ function representedStatements( representedSource: readonly RepresentedQueryEvidenceSource[], sourceFile: ts.SourceFile, ): ts.Statement[] { + const ownerRange = lineRangeOf(owner, sourceFile) const representedByRange = new Map>() for (const represented of representedSource) { const key = `${represented.startLine}:${represented.endLine}` @@ -204,7 +248,10 @@ function representedStatements( return } if (ts.isStatement(node)) { - const range = lineRangeOf(node, sourceFile) + const range = evidenceLineRangeOf(node, sourceFile) + if (range.start < ownerRange.start || range.end > ownerRange.end) { + return + } const texts = representedByRange.get(`${range.start}:${range.end}`) if (texts?.has(normalizedSource(physicalSourceForLineRange(range, sourceFile)))) { statements.push(node) @@ -709,7 +756,7 @@ export function ownerLocalDeclarationEvidence( })) return closure .map((statement): OwnerDeclarationEvidence => { - const range = lineRangeOf(statement, sourceFile) + const range = evidenceLineRangeOf(statement, sourceFile) return { startLine: range.start, endLine: range.end, @@ -718,8 +765,12 @@ export function ownerLocalDeclarationEvidence( .map((text, offset) => ({ lineNumber: range.start + offset, text })), } }) + .filter((declaration) => ( + input.ownerRange.start <= declaration.startLine + && declaration.endLine <= input.ownerRange.end + )) .filter((declaration) => !representedRanges.some((range) => ( - range.start <= declaration.startLine && declaration.endLine <= range.end + range.start <= declaration.endLine && declaration.startLine <= range.end ))) .sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine) } catch { diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index e3a5607f..1d19179e 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -13,6 +13,7 @@ vi.mock('typescript', async (importOriginal) => { import * as ts from 'typescript' +import { ownerLocalDeclarationEvidence } from '../../src/runtime/query-evidence-dependencies.js' import { readQueryEvidenceSnippet, type QueryEvidenceSnippet } from '../../src/runtime/retrieve.js' const QUESTION = 'How does dispatch outcome return a validated result with retry handling?' @@ -259,6 +260,197 @@ describe('owner-local declaration completion', () => { ].join('\n')) }) + it('preserves every physical line of trailing multiline declaration trivia', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5 /* declaration context', + ' continued */', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5 /* declaration context', + 'L3: continued */', + 'L4: return dispatchOutcome(datum)', + ].join('\n')) + }) + + it('preserves a chain of adjacent multiline declaration trivia atomically', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5 /* first context', + ' second context */ /* third context', + ' fourth context */', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5 /* first context', + 'L3: second context */ /* third context', + 'L4: fourth context */', + 'L5: return dispatchOutcome(datum)', + ].join('\n')) + }) + + it('preserves leading multiline trivia only when it intersects an emitted line', () => { + const intersecting = evidenceFor([ + 'export function executeSample() {', + ' /* declaration context', + ' closes here */ const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ]) + const standalone = evidenceFor([ + 'export function executeSample() {', + ' /* standalone context', + ' remains separate */', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(intersecting?.snippet).toBe([ + 'L2: /* declaration context', + 'L3: closes here */ const datum = 12.5', + 'L4: return dispatchOutcome(datum)', + ].join('\n')) + expect(standalone?.snippet).toBe([ + 'L4: const datum = 12.5', + 'L5: return dispatchOutcome(datum)', + ].join('\n')) + }) + + it('uses syntax trivia without mistaking comment-like template content for comments', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = `/* template text', + ' // remains literal */` /* declaration context', + ' continued */', + ' return dispatchOutcome(datum)', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = `/* template text', + 'L3: // remains literal */` /* declaration context', + 'L4: continued */', + 'L5: return dispatchOutcome(datum)', + ].join('\n')) + }) + + it.each([ + { + boundary: 'four rendered lines', + declaration: [ + ' const datum = 12.5 /* declaration context', + ' continued', + ' across another physical line', + ' through the fifth rendered line */', + ], + }, + { + boundary: '220-character physical line', + declaration: [ + ' const datum = 12.5 /* declaration context', + ` ${'x'.repeat(215)} */`, + ], + }, + { + boundary: '300-character aggregate', + declaration: [ + ' const datum = 12.5 /* declaration context', + ` ${'x'.repeat(205)} */`, + ], + }, + ])('declines all declaration lines when closing trivia exceeds the $boundary limit', ({ declaration }) => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ...declaration, + " return dispatchOutcome(datum, 'selected result with enough context to cross the aggregate boundary')", + '}', + ]) + + expect(evidence?.snippet).toBe( + "L4: return dispatchOutcome(datum, 'selected result with enough context to cross the aggregate boundary')" + .replace('L4', `L${declaration.length + 2}`), + ) + }) + + it('attributes a statement whose multiline trailing comment is fully represented', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum) /* selected context ?', + ' complete */', + '}', + ]) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5', + 'L3: return dispatchOutcome(datum) /* selected context ? complete */', + ].join('\n')) + }) + + it('does not attribute a statement whose multiline trailing comment is only partially represented', () => { + const evidence = evidenceFor([ + 'export function executeSample() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum) /* selected context', + ' complete */', + '}', + ]) + + expect(evidence?.snippet).toBe('L3: return dispatchOutcome(datum) /* selected context') + }) + + it.each([ + { representation: 'fully', endLine: 3 }, + { representation: 'partially', endLine: 2 }, + ])('does not re-emit a $representation represented declaration comment', ({ endLine }) => { + const sourceLines = [ + 'export function executeSample() {', + ' const datum = 12.5 /* declaration context', + ' complete */', + ' return dispatchOutcome(datum)', + '}', + ] + + expect(ownerLocalDeclarationEvidence({ + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 5 }, + representedSource: [ + { + startLine: 2, + endLine, + text: sourceLines.slice(1, endLine).join('\n'), + }, + { + startLine: 4, + endLine: 4, + text: sourceLines[3]!, + }, + ], + })).toEqual([]) + }) + + it('keeps function-owner authority token-bounded before trailing multiline trivia', () => { + const evidence = evidenceFor([ + 'export const executeSample = () => {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + '} /* outside owner context', + ' continued */', + ], { sourceLocation: 'L1-L4' }) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5', + 'L3: return dispatchOutcome(datum)', + ].join('\n')) + }) + it.each([ { name: 'ordinary trailing line comment', From b9823699825655088a49218412c68c1465445e43 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 22:21:32 +0400 Subject: [PATCH 10/27] Recognize complete statements across exact contiguous source fragments --- src/runtime/query-evidence-dependencies.ts | 40 ++++-- .../unit/retrieve-owner-declarations.test.ts | 120 ++++++++++++++++++ 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 90d81a0f..61b80974 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -233,13 +233,23 @@ function representedStatements( sourceFile: ts.SourceFile, ): ts.Statement[] { const ownerRange = lineRangeOf(owner, sourceFile) - const representedByRange = new Map>() - for (const represented of representedSource) { - const key = `${represented.startLine}:${represented.endLine}` - const texts = representedByRange.get(key) ?? new Set() - texts.add(normalizedSource(represented.text)) - representedByRange.set(key, texts) - } + const lineCount = sourceFile.getLineStarts().length + const verifiedRepresentedRanges = representedSource.flatMap((represented) => { + if ( + !Number.isInteger(represented.startLine) + || !Number.isInteger(represented.endLine) + || represented.startLine < ownerRange.start + || represented.endLine > ownerRange.end + || represented.startLine > represented.endLine + || represented.endLine > lineCount + ) { + return [] + } + const range = { start: represented.startLine, end: represented.endLine } + return normalizedSource(represented.text) === normalizedSource(physicalSourceForLineRange(range, sourceFile)) + ? [range] + : [] + }) const statements: ts.Statement[] = [] const visit = (node: ts.Node): void => { @@ -252,8 +262,20 @@ function representedStatements( if (range.start < ownerRange.start || range.end > ownerRange.end) { return } - const texts = representedByRange.get(`${range.start}:${range.end}`) - if (texts?.has(normalizedSource(physicalSourceForLineRange(range, sourceFile)))) { + const coveredLines = new Set() + for (const represented of verifiedRepresentedRanges) { + if (represented.start < range.start || represented.end > range.end) { + continue + } + for (let line = represented.start; line <= represented.end; line += 1) { + coveredLines.add(line) + } + } + if ( + coveredLines.size === range.end - range.start + 1 + && coveredLines.has(range.start) + && coveredLines.has(range.end) + ) { statements.push(node) } } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 1d19179e..407fd836 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -393,6 +393,126 @@ describe('owner-local declaration completion', () => { ].join('\n')) }) + it('completes an ordinary multiline statement represented by exact physical fragments', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return { displayedText: datum,', + ' recordedValue: datum };', + '}', + ], { + label: 'assemble', + question: 'What are the displayed text and recorded value?', + }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + 'L3: return { displayedText: datum,', + 'L4: recordedValue: datum };', + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('completes a trailing-comment statement represented by exact physical fragments', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return { displayedText: datum }; /* detail', + ' displayed text */', + '}', + ], { + label: 'assemble', + question: 'What is the displayed text?', + }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + 'L3: return { displayedText: datum }; /* detail', + 'L4: displayed text */', + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it.each([ + { + coverage: 'split-incomplete', + representedSource: (sourceLines: readonly string[]) => [ + { startLine: 3, endLine: 3, text: sourceLines[2]! }, + { startLine: 4, endLine: 4, text: sourceLines[3]! }, + { startLine: 5, endLine: 5, text: sourceLines[4]! }, + ], + }, + { + coverage: 'gapped', + representedSource: (sourceLines: readonly string[]) => [ + { startLine: 3, endLine: 3, text: sourceLines[2]! }, + { startLine: 4, endLine: 4, text: sourceLines[3]! }, + { startLine: 6, endLine: 6, text: sourceLines[5]! }, + ], + }, + { + coverage: 'clipped', + representedSource: (sourceLines: readonly string[]) => [ + { startLine: 3, endLine: 3, text: sourceLines[2]! }, + { startLine: 4, endLine: 4, text: sourceLines[3]!.slice(0, -1) }, + { startLine: 5, endLine: 5, text: sourceLines[4]! }, + { startLine: 6, endLine: 6, text: sourceLines[5]! }, + ], + }, + { + coverage: 'wrong-byte', + representedSource: (sourceLines: readonly string[]) => [ + { startLine: 3, endLine: 3, text: sourceLines[2]! }, + { startLine: 4, endLine: 4, text: sourceLines[3]!.replace('datum', 'other') }, + { startLine: 5, endLine: 5, text: sourceLines[4]! }, + { startLine: 6, endLine: 6, text: sourceLines[5]! }, + ], + }, + { + coverage: 'foreign-owner', + representedSource: (sourceLines: readonly string[]) => [ + { startLine: 3, endLine: 3, text: sourceLines[2]! }, + { startLine: 4, endLine: 4, text: sourceLines[3]! }, + { startLine: 5, endLine: 8, text: sourceLines.slice(4, 8).join('\n') }, + ], + }, + { + coverage: 'invalid-overlap', + representedSource: (sourceLines: readonly string[]) => [ + { startLine: 3, endLine: 4, text: sourceLines.slice(2, 4).join('\n') }, + { + startLine: 4, + endLine: 6, + text: sourceLines.slice(3, 6).join('\n').replace('datum', 'other'), + }, + ], + }, + ])('rejects $coverage represented coverage for a multiline statement', ({ representedSource }) => { + const sourceLines = [ + 'function assemble() {', + ' const datum = 12.5', + ' return {', + ' displayedText: datum,', + ' recordedValue: datum,', + ' }', + '}', + 'function foreignOwner() { return null }', + ] + + expect(ownerLocalDeclarationEvidence({ + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 7 }, + representedSource: representedSource(sourceLines), + })).toEqual([]) + }) + it('does not attribute a statement whose multiline trailing comment is only partially represented', () => { const evidence = evidenceFor([ 'export function executeSample() {', From 0d08d3fba5440726a410ce79bb5d4e4996dbef04 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 23:14:55 +0400 Subject: [PATCH 11/27] fix: preserve literal bytes in query evidence --- src/runtime/query-evidence-dependencies.ts | 234 +++++++++++++++++- src/runtime/retrieve.ts | 98 +++++--- .../unit/retrieve-owner-declarations.test.ts | 223 ++++++++++++++++- 3 files changed, 510 insertions(+), 45 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 61b80974..da4a0371 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -26,6 +26,11 @@ export interface OwnerDeclarationCompletionInput { representedSource: readonly RepresentedQueryEvidenceSource[] } +export interface QueryEvidenceSourceProjection { + text: string + literalLineBreaks: Array<{ offset: number; lineNumber: number }> +} + interface ParsedSourceSnapshot { sourceFilePath: string sourceText: string @@ -214,17 +219,234 @@ function normalizedSource(value: string): string { return value.replace(/\s+/g, ' ').trim() } -function physicalSourceForLineRange( +interface ProtectedSourceToken { + start: number + end: number + text: string +} + +interface PhysicalSourceProjection extends QueryEvidenceSourceProjection { + identityText: string + tokens: ProtectedSourceToken[] +} + +const PROTECTED_LITERAL_KINDS = new Set([ + ts.SyntaxKind.StringLiteral, + ts.SyntaxKind.NoSubstitutionTemplateLiteral, + ts.SyntaxKind.RegularExpressionLiteral, + ts.SyntaxKind.TemplateHead, + ts.SyntaxKind.TemplateMiddle, + ts.SyntaxKind.TemplateTail, +]) + +function physicalSourceBounds( range: { start: number; end: number }, sourceFile: ts.SourceFile, -): string { +): { start: number; end: number } | null { const lineStarts = sourceFile.getLineStarts() const start = lineStarts[range.start - 1] if (start === undefined) { - return '' + return null + } + return { + start, + end: lineStarts[range.end] ?? sourceFile.text.length, + } +} + +function protectedTokensForRange( + range: { start: number; end: number }, + sourceFile: ts.SourceFile, +): ProtectedSourceToken[] | null { + const bounds = physicalSourceBounds(range, sourceFile) + if (!bounds) { + return null + } + const tokens: ProtectedSourceToken[] = [] + let clippedToken = false + const visit = (node: ts.Node): void => { + if (PROTECTED_LITERAL_KINDS.has(node.kind)) { + const start = node.getStart(sourceFile) + const end = node.getEnd() + if (start < bounds.end && bounds.start < end) { + if (start < bounds.start || end > bounds.end) { + clippedToken = true + return + } + tokens.push({ + start, + end, + text: sourceFile.text.slice(start, end), + }) + } + } + ts.forEachChild(node, visit) + } + visit(sourceFile) + if (clippedToken) { + return null + } + return tokens.sort((left, right) => left.start - right.start || left.end - right.end) +} + +function tokenMarker(index: number): string { + return `\u0000literal-${index}\u0000` +} + +function physicalProjectionForRange( + range: { start: number; end: number }, + sourceFile: ts.SourceFile, +): PhysicalSourceProjection | null { + const bounds = physicalSourceBounds(range, sourceFile) + const tokens = protectedTokensForRange(range, sourceFile) + if (!bounds || !tokens) { + return null + } + + let marked = '' + let cursor = bounds.start + for (const [index, token] of tokens.entries()) { + marked += sourceFile.text.slice(cursor, token.start) + marked += tokenMarker(index) + cursor = token.end + } + marked += sourceFile.text.slice(cursor, bounds.end) + const identityText = normalizedSource(marked) + + let text = identityText + const literalLineBreaks: Array<{ offset: number; lineNumber: number }> = [] + for (let index = tokens.length - 1; index >= 0; index -= 1) { + const token = tokens[index]! + const marker = tokenMarker(index) + const markerOffset = text.indexOf(marker) + if (markerOffset < 0) { + return null + } + text = `${text.slice(0, markerOffset)}${token.text}${text.slice(markerOffset + marker.length)}` + } + + let projectedOffset = 0 + let identityCursor = 0 + for (const [index, token] of tokens.entries()) { + const marker = tokenMarker(index) + const markerOffset = identityText.indexOf(marker, identityCursor) + if (markerOffset < 0) { + return null + } + projectedOffset += markerOffset - identityCursor + let lineNumber = sourceFile.getLineAndCharacterOfPosition(token.start).line + 1 + for (let tokenOffset = 0; tokenOffset < token.text.length; tokenOffset += 1) { + if (token.text[tokenOffset] !== '\n') { + continue + } + lineNumber += 1 + literalLineBreaks.push({ + offset: projectedOffset + tokenOffset, + lineNumber, + }) + } + projectedOffset += token.text.length + identityCursor = markerOffset + marker.length + } + + return { text, identityText, literalLineBreaks, tokens } +} + +function representedSourceIdentity( + representedText: string, + projection: PhysicalSourceProjection, +): string | null { + let marked = '' + let cursor = 0 + for (const [index, token] of projection.tokens.entries()) { + const tokenOffset = representedText.indexOf(token.text, cursor) + if (tokenOffset < 0) { + return null + } + marked += representedText.slice(cursor, tokenOffset) + marked += tokenMarker(index) + cursor = tokenOffset + token.text.length + } + marked += representedText.slice(cursor) + return normalizedSource(marked) +} + +function representedSourceMatchesPhysicalRange( + representedText: string, + range: { start: number; end: number }, + sourceFile: ts.SourceFile, +): boolean { + const projection = physicalProjectionForRange(range, sourceFile) + return projection !== null + && representedSourceIdentity(representedText, projection) === projection.identityText +} + +/** + * Projects already-shaped query evidence back onto its authenticated physical + * source while normalizing layout only outside string/template/regex tokens. + */ +export function queryEvidenceSourceProjection(input: { + sourceFilePath: string + sourceLines: readonly string[] + representedSource: readonly RepresentedQueryEvidenceSource[] + shapedText: string +}): QueryEvidenceSourceProjection | null { + try { + const sourceFile = parsedSourceForSnapshot(input.sourceFilePath, input.sourceLines) + if (!sourceFile) { + return null + } + const parseDiagnostics = (sourceFile as ts.SourceFile & { + parseDiagnostics?: readonly ts.Diagnostic[] + }).parseDiagnostics + if (parseDiagnostics && parseDiagnostics.length > 0) { + return null + } + + const lineCount = sourceFile.getLineStarts().length + const shapedText = normalizedSource(input.shapedText) + let cursor = 0 + let projectedText = '' + const literalLineBreaks: Array<{ offset: number; lineNumber: number }> = [] + for (const represented of input.representedSource) { + if ( + !Number.isInteger(represented.startLine) + || !Number.isInteger(represented.endLine) + || represented.startLine < 1 + || represented.endLine > lineCount + || represented.startLine > represented.endLine + ) { + return null + } + const range = { start: represented.startLine, end: represented.endLine } + const projection = physicalProjectionForRange(range, sourceFile) + if ( + !projection + || representedSourceIdentity(represented.text, projection) !== projection.identityText + ) { + return null + } + const normalizedComponent = normalizedSource(represented.text) + const componentOffset = shapedText.indexOf(normalizedComponent, cursor) + if (normalizedComponent.length === 0 || componentOffset < 0) { + return null + } + projectedText += shapedText.slice(cursor, componentOffset) + const projectionStart = projectedText.length + projectedText += projection.text + for (const lineBreak of projection.literalLineBreaks) { + literalLineBreaks.push({ + offset: projectionStart + lineBreak.offset, + lineNumber: lineBreak.lineNumber, + }) + } + cursor = componentOffset + normalizedComponent.length + } + projectedText += shapedText.slice(cursor) + return { text: projectedText, literalLineBreaks } + } catch { + return null } - const end = lineStarts[range.end] ?? sourceFile.text.length - return sourceFile.text.slice(start, end) } function representedStatements( @@ -246,7 +468,7 @@ function representedStatements( return [] } const range = { start: represented.startLine, end: represented.endLine } - return normalizedSource(represented.text) === normalizedSource(physicalSourceForLineRange(range, sourceFile)) + return representedSourceMatchesPhysicalRange(represented.text, range, sourceFile) ? [range] : [] }) diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 94306133..d3c369fc 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -87,6 +87,7 @@ import { import { communitiesFromGraph, estimateQueryTokens } from './serve.js' import { ownerLocalDeclarationEvidence, + queryEvidenceSourceProjection, type RepresentedQueryEvidenceSource, } from './query-evidence-dependencies.js' @@ -984,7 +985,7 @@ function queryEvidenceFragments( ?? [{ startLine: start, endLine: end, - text: lines.slice(start - 1, end).join(' '), + text: lines.slice(start - 1, end).join('\n'), }], }) } @@ -1272,34 +1273,76 @@ function selectQueryEvidenceLines(range: QueryEvidenceRange): QueryEvidenceLine[ interface RenderedQueryEvidenceLine { source: QueryEvidenceLine text: string + representedSource: readonly RepresentedQueryEvidenceSource[] } -function renderQueryEvidenceLineEntries(lines: readonly QueryEvidenceLine[]): RenderedQueryEvidenceLine[] { +function renderQueryEvidenceLineEntries( + lines: readonly QueryEvidenceLine[], + sourceFile: string, + sourceLines: readonly string[], +): RenderedQueryEvidenceLine[] { const rendered: RenderedQueryEvidenceLine[] = [] let usedChars = 0 for (const line of lines) { const normalized = line.text.replace(/\s+/g, ' ').trim() - const content = normalized.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP - ? `${normalized.slice(0, QUERY_EVIDENCE_SNIPPET_LINE_CAP - 3).trimEnd()}...` - : normalized + const projection = queryEvidenceSourceProjection({ + sourceFilePath: sourceFile, + sourceLines, + representedSource: line.representedSource, + shapedText: line.text, + }) + let projectedContent = projection?.text ?? normalized + if (projection) { + for (let index = projection.literalLineBreaks.length - 1; index >= 0; index -= 1) { + const lineBreak = projection.literalLineBreaks[index]! + projectedContent = [ + projectedContent.slice(0, lineBreak.offset + 1), + `L${lineBreak.lineNumber}: `, + projectedContent.slice(lineBreak.offset + 1), + ].join('') + } + } + const projectedRows = projectedContent.split('\n') + const lineCapExceeded = projectedRows.some((row, index) => { + const content = index === 0 ? row : row.replace(/^L\d+: /, '') + return content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + }) + const content = lineCapExceeded + ? projectedRows.map((row, index) => { + const match = index > 0 ? /^(L\d+: )(.*)$/.exec(row) : null + const rowPrefix = match?.[1] ?? '' + const rowContent = match?.[2] ?? row + return rowContent.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + ? `${rowPrefix}${rowContent.slice(0, QUERY_EVIDENCE_SNIPPET_LINE_CAP - 3)}...` + : row + }).join('\n') + : projectedContent const prefix = `L${line.index}: ` const separatorChars = rendered.length > 0 ? 1 : 0 const remaining = QUERY_EVIDENCE_SNIPPET_CHAR_CAP - usedChars - separatorChars if (remaining <= prefix.length + 8) { break } - const bounded = `${prefix}${content}`.slice(0, remaining).trimEnd() + const faithful = `${prefix}${content}` + const bounded = faithful.slice(0, remaining).trimEnd() rendered.push({ source: line, text: bounded, + representedSource: projection && !lineCapExceeded && bounded === faithful + ? line.representedSource + : [], }) usedChars += bounded.length + separatorChars } return rendered } -function renderQueryEvidenceLines(lines: readonly QueryEvidenceLine[]): string | null { - const rendered = renderQueryEvidenceLineEntries(lines) +function renderQueryEvidenceLines( + lines: readonly QueryEvidenceLine[], + sourceFile: string, + sourceLines: readonly string[], +): string | null { + const rendered = renderQueryEvidenceLineEntries(lines, sourceFile, sourceLines) return rendered.length > 0 ? rendered.map((line) => line.text).join('\n') : null } @@ -1341,28 +1384,7 @@ function completeOwnerDeclarationEvidence( baseline: readonly RenderedQueryEvidenceLine[], ): { snippet: string; lineNumber: number } | null { try { - const representedSource = baseline.flatMap((rendered) => { - const fullContent = rendered.source.text.replace(/\s+/g, ' ').trim() - const prefix = `L${rendered.source.index}: ` - const renderedContent = rendered.text.startsWith(prefix) - ? rendered.text.slice(prefix.length) - : '' - let searchFrom = 0 - return rendered.source.representedSource.flatMap((represented) => { - const representedText = represented.text.replace(/\s+/g, ' ').trim() - const index = representedText.length > 0 - ? fullContent.indexOf(representedText, searchFrom) - : -1 - if (index < 0) { - return [] - } - const end = index + representedText.length - searchFrom = end - return renderedContent.slice(index, end) === fullContent.slice(index, end) - ? [represented] - : [] - }) - }) + const representedSource = baseline.flatMap((rendered) => rendered.representedSource) if (representedSource.length === 0) { return null } @@ -1408,11 +1430,11 @@ function completeOwnerDeclarationEvidence( || (left.baselineIndex === null ? -1 : left.baselineIndex) - (right.baselineIndex === null ? -1 : right.baselineIndex) )) - if (merged.length > QUERY_EVIDENCE_SNIPPET_MAX_LINES) { - return null - } const snippet = merged.map((line) => line.text).join('\n') - if (snippet.length > QUERY_EVIDENCE_SNIPPET_CHAR_CAP) { + if ( + snippet.split('\n').length > QUERY_EVIDENCE_SNIPPET_MAX_LINES + || snippet.length > QUERY_EVIDENCE_SNIPPET_CHAR_CAP + ) { return null } const preservedBaseline = merged @@ -1522,7 +1544,9 @@ export function readQueryEvidenceSnippet( } } } - const symbolSnippet = symbolLine ? renderQueryEvidenceLines([symbolLine]) : null + const symbolSnippet = symbolLine + ? renderQueryEvidenceLines([symbolLine], sourceFile, lines) + : null if (symbolLine && symbolSnippet) { let mergedLines = [symbolLine] for (const fileLine of selectedLines) { @@ -1536,14 +1560,14 @@ export function readQueryEvidenceSnippet( } const candidateLines = [...mergedLines, fileLine] .sort((left, right) => left.index - right.index) - if (renderQueryEvidenceLines(candidateLines)?.split('\n').includes(symbolSnippet)) { + if (renderQueryEvidenceLines(candidateLines, sourceFile, lines)?.split('\n').includes(symbolSnippet)) { mergedLines = candidateLines } } selectedLines = mergedLines } } - const baseline = renderQueryEvidenceLineEntries(selectedLines) + const baseline = renderQueryEvidenceLineEntries(selectedLines, sourceFile, lines) if (baseline.length === 0) { return null } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 407fd836..1921eaa1 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -22,13 +22,14 @@ const roots: string[] = [] function sourceFixture( sourceLines: readonly string[], extension = '.ts', + lineEnding = '\n', ): { sourceFile: string; sourceLocation: string } { const fixtureParent = resolve('out', 'test-runtime') mkdirSync(fixtureParent, { recursive: true }) const root = mkdtempSync(join(fixtureParent, 'owner-declaration-')) roots.push(root) const sourceFile = join(root, `sample${extension}`) - writeFileSync(sourceFile, sourceLines.join('\n'), 'utf8') + writeFileSync(sourceFile, sourceLines.join(lineEnding), 'utf8') return { sourceFile, sourceLocation: `L1-L${sourceLines.length}`, @@ -45,9 +46,10 @@ function evidenceFor( sourceLocation?: string | null derived?: boolean fileCache?: Map + lineEnding?: '\n' | '\r\n' } = {}, ): QueryEvidenceSnippet | null { - const fixture = sourceFixture(sourceLines, options.extension) + const fixture = sourceFixture(sourceLines, options.extension, options.lineEnding) return readQueryEvidenceSnippet(fixture.sourceFile, options.lineNumber ?? 1, { question: options.question ?? QUESTION, label: options.label ?? 'executeSample', @@ -227,6 +229,160 @@ describe('owner-local declaration completion', () => { ].join('\n')) }) + it.each([ + { + literalKind: 'escaped string with comment-looking text', + statement: ' return dispatchOutcome(datum, "alpha beta \\"// literal")', + changed: (statement: string) => statement.replace('alpha beta', 'alpha beta'), + }, + { + literalKind: 'template with a tab and escaped delimiter', + statement: ' return dispatchOutcome(datum, `alpha\tbeta \\` /* literal */`)', + changed: (statement: string) => statement.replace('\t', ' '), + }, + { + literalKind: 'regex with repeated spaces and escaped slashes', + statement: ' return dispatchOutcome(datum, /alpha beta\\/\\/tail/)', + changed: (statement: string) => statement.replace('alpha beta', 'alpha beta'), + }, + ])('authenticates exact $literalKind bytes and rejects changed bytes directly', ({ statement, changed }) => { + const sourceLines = [ + 'function assemble() {', + ' const datum = 12.5', + statement, + '}', + ] + const input = { + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 4 }, + } + + expect(ownerLocalDeclarationEvidence({ + ...input, + representedSource: [{ startLine: 3, endLine: 3, text: statement }], + })).toEqual([{ + startLine: 2, + endLine: 2, + lines: [{ lineNumber: 2, text: ' const datum = 12.5' }], + }]) + expect(ownerLocalDeclarationEvidence({ + ...input, + representedSource: [{ startLine: 3, endLine: 3, text: changed(statement) }], + })).toEqual([]) + }) + + it('authenticates complete multiline template boundaries without normalizing literal edges', () => { + const sourceLines = [ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ] + const represented = sourceLines.slice(2, 4).join('\n') + const input = { + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 5 }, + } + + expect(ownerLocalDeclarationEvidence({ + ...input, + representedSource: [{ startLine: 3, endLine: 4, text: represented }], + })).toHaveLength(1) + expect(ownerLocalDeclarationEvidence({ + ...input, + representedSource: [{ + startLine: 3, + endLine: 4, + text: represented.replace(' alpha ?\n beta ', ' alpha ?\nbeta '), + }], + })).toEqual([]) + }) + + it('preserves selected string, template, and regex literal bytes while completing their const', () => { + const statement = ' return dispatchOutcome(datum, "alpha beta \\"// literal", `left\tright`, /x y\\/\\/z/)' + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + statement, + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble' }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: ${statement.trim()}`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('preserves the sealed two-space selected literal instead of authenticating altered display bytes', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + " return dispatchOutcome(datum, 'alpha beta')", + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5', + "L3: return dispatchOutcome(datum, 'alpha beta')", + ].join('\n')) + }) + + it('keeps a whitespace-sensitive literal in a complete joined multiline expression', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return datum ?', + " dispatchOutcome(datum, 'alpha beta')", + ' : retryOutcome()', + '}', + ], { question: 'What are the dispatch outcome and retry outcome?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5', + "L3: return datum ? dispatchOutcome(datum, 'alpha beta') : retryOutcome()", + ].join('\n')) + }) + + it('retains multiline template line breaks and line-edge spaces with physical prefixes', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble' }) + + expect(evidence?.snippet).toBe([ + 'L2: const datum = 12.5', + 'L3: return dispatchOutcome(datum, ` alpha ?', + 'L4: beta `)', + ].join('\n')) + }) + + it.each(['\n', '\r\n'] as const)( + 'preserves literal content and intended indentation from %j source', + (lineEnding) => { + const evidence = evidenceFor([ + 'function assemble() {', + '\tconst datum = 12.5', + '\treturn dispatchOutcome(datum, `left\tright`, /x y/)', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble', lineEnding }) + + expect(evidence?.snippet).toBe([ + 'L2: \tconst datum = 12.5', + 'L3: return dispatchOutcome(datum, `left\tright`, /x y/)', + ].join('\n')) + }, + ) + it('preserves a literal tab in a string and repeated spaces in a regex', () => { const declaration = " const input = ['alpha\tbeta', /alpha beta/];" const evidence = evidenceFor([ @@ -1146,6 +1302,48 @@ describe('owner-local binding identity correction', () => { }) describe('owner declaration completion budgets and atomic preservation', () => { + it('admits faithful literal whitespace when the completed snippet exactly fills the cap', () => { + const literal = `${'value '.repeat(22)}end` + const returnSource = ` return dispatchOutcome(datum, '${literal}')` + const renderedReturn = `L3: ${returnSource.trim()}` + const declarationShell = "L2: const datum = ''" + const fillLength = 300 - renderedReturn.length - declarationShell.length - 1 + const declarationSource = ` const datum = '${'d'.repeat(fillLength)}'` + const renderedDeclaration = `L2: ${declarationSource}` + + const evidence = evidenceFor([ + 'export function executeSample() {', + declarationSource, + returnSource, + '}', + ]) + + expect(fillLength).toBeGreaterThan(0) + expect(returnSource.length).toBeLessThanOrEqual(220) + expect(evidence?.snippet).toBe(`${renderedDeclaration}\n${renderedReturn}`) + expect(evidence?.snippet).toHaveLength(300) + }) + + it('counts literal whitespace for overflow without clipping selected source or adding its declaration', () => { + const literal = `${'value '.repeat(24)}end` + const returnSource = ` return dispatchOutcome(datum, '${literal}')` + const declarationSource = ` const datum = '${'d'.repeat(105)}'` + const baseline = `L3: ${returnSource.trim()}` + expect(returnSource.length).toBeLessThanOrEqual(220) + expect(`L2: ${declarationSource}\n${baseline}`.length).toBeGreaterThan(300) + + const evidence = evidenceFor([ + 'export function executeSample() {', + declarationSource, + returnSource, + '}', + ]) + + expect(evidence?.snippet).toBe(baseline) + expect(evidence?.snippet).not.toContain('const datum') + expect(evidence?.snippet).not.toContain('...') + }) + it('admits a declaration closure that exactly fills the 300-character cap', () => { const returnSource = ` return dispatchOutcome(datum, '${'r'.repeat(48)}')` const renderedReturn = `L3: ${returnSource.trim()}` @@ -1219,6 +1417,27 @@ describe('owner declaration completion budgets and atomic preservation', () => { }) describe('owner declaration parse snapshot cache', () => { + it('invalidates changed content or paths even when the source-line array identity is reused', () => { + const sourceLines = [ + 'export function executeSample() {', + " const datum = 'first'", + ' return dispatchOutcome(datum)', + '}', + ] + const evidence = (sourceFilePath: string) => ownerLocalDeclarationEvidence({ + sourceFilePath, + sourceLines, + ownerRange: { start: 1, end: 4 }, + representedSource: [{ startLine: 3, endLine: 3, text: sourceLines[2]! }], + }) + + expect(evidence('sample.ts')[0]?.lines[0]?.text).toContain("'first'") + sourceLines[1] = " const datum = 'second'" + expect(evidence('sample.ts')[0]?.lines[0]?.text).toContain("'second'") + expect(evidence('sample.mts')[0]?.lines[0]?.text).toContain("'second'") + expect(ts.createSourceFile).toHaveBeenCalledTimes(3) + }) + it('does not reuse a parsed source after content changes at the same path', () => { const firstSource = [ 'export function executeSample() {', From f3e4235999d177664c66678bfa5ec0c4671227c8 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Mon, 7 Sep 2026 23:46:36 +0400 Subject: [PATCH 12/27] fix: preserve raw literal line endings through source cache --- src/runtime/query-evidence-dependencies.ts | 77 +++++++- src/runtime/retrieve.ts | 5 +- .../unit/retrieve-owner-declarations.test.ts | 171 ++++++++++++++++++ 3 files changed, 245 insertions(+), 8 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index da4a0371..8f6a31bf 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -37,6 +37,12 @@ interface ParsedSourceSnapshot { sourceFile: ts.SourceFile } +interface SourceSnapshotProvenance { + sourceFilePath: string + normalizedSourceText: string + sourceText: string +} + interface ConstBinding { declaration: ts.VariableDeclaration statement: ts.VariableStatement @@ -53,8 +59,28 @@ type FunctionOwner = ( ) & { body: ts.ConciseBody } const parsedSourceSnapshots = new WeakMap() +const sourceSnapshotProvenance = new WeakMap() +const sourceFilesWithNormalizedLineCaches = new WeakSet() const JS_TS_EXTENSIONS = new Set(['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs']) +/** Retains bytes from the existing source read alongside its normalized line cache. */ +export function retainQueryEvidenceSourceSnapshot(input: { + sourceFilePath: string + sourceLines: readonly string[] + sourceText: string +}): void { + const normalizedSourceText = input.sourceLines.join('\n') + if (input.sourceText.split(/\r?\n/).join('\n') !== normalizedSourceText) { + sourceSnapshotProvenance.delete(input.sourceLines) + return + } + sourceSnapshotProvenance.set(input.sourceLines, { + sourceFilePath: input.sourceFilePath, + normalizedSourceText, + sourceText: input.sourceText, + }) +} + function scriptKindForPath(sourceFilePath: string): ts.ScriptKind | null { switch (extname(sourceFilePath).toLowerCase()) { case '.ts': @@ -87,7 +113,14 @@ function parsedSourceForSnapshot( return null } - const sourceText = sourceLines.join('\n') + const normalizedSourceText = sourceLines.join('\n') + const provenance = sourceSnapshotProvenance.get(sourceLines) + const sourceText = ( + provenance?.sourceFilePath === sourceFilePath + && provenance.normalizedSourceText === normalizedSourceText + ) + ? provenance.sourceText + : normalizedSourceText const cached = parsedSourceSnapshots.get(sourceLines) if ( cached @@ -104,6 +137,9 @@ function parsedSourceForSnapshot( true, scriptKind, ) + if (sourceText !== normalizedSourceText) { + sourceFilesWithNormalizedLineCaches.add(sourceFile) + } parsedSourceSnapshots.set(sourceLines, { sourceFilePath, sourceText, sourceFile }) return sourceFile } @@ -223,6 +259,7 @@ interface ProtectedSourceToken { start: number end: number text: string + representedText: string } interface PhysicalSourceProjection extends QueryEvidenceSourceProjection { @@ -273,10 +310,14 @@ function protectedTokensForRange( clippedToken = true return } + const text = sourceFile.text.slice(start, end) tokens.push({ start, end, - text: sourceFile.text.slice(start, end), + text, + representedText: sourceFilesWithNormalizedLineCaches.has(sourceFile) + ? text.replace(/\r\n/g, '\n') + : text, }) } } @@ -359,13 +400,13 @@ function representedSourceIdentity( let marked = '' let cursor = 0 for (const [index, token] of projection.tokens.entries()) { - const tokenOffset = representedText.indexOf(token.text, cursor) + const tokenOffset = representedText.indexOf(token.representedText, cursor) if (tokenOffset < 0) { return null } marked += representedText.slice(cursor, tokenOffset) marked += tokenMarker(index) - cursor = tokenOffset + token.text.length + cursor = tokenOffset + token.representedText.length } marked += representedText.slice(cursor) return normalizedSource(marked) @@ -381,6 +422,30 @@ function representedSourceMatchesPhysicalRange( && representedSourceIdentity(representedText, projection) === projection.identityText } +function literalDelimiterPreservingLines( + range: { start: number; end: number }, + sourceFile: ts.SourceFile, + sourceLines: readonly string[], +): OwnerDeclarationEvidenceLine[] { + const tokens = protectedTokensForRange(range, sourceFile) ?? [] + const lineStarts = sourceFile.getLineStarts() + return sourceLines + .slice(range.start - 1, range.end) + .map((text, offset) => { + const lineNumber = range.start + offset + const nextLineStart = lineStarts[lineNumber] + const hasProtectedCRLF = nextLineStart !== undefined + && sourceFile.text.slice(nextLineStart - 2, nextLineStart) === '\r\n' + && tokens.some((token) => ( + token.start <= nextLineStart - 2 && nextLineStart <= token.end + )) + return { + lineNumber, + text: hasProtectedCRLF && !text.endsWith('\r') ? `${text}\r` : text, + } + }) +} + /** * Projects already-shaped query evidence back onto its authenticated physical * source while normalizing layout only outside string/template/regex tokens. @@ -1004,9 +1069,7 @@ export function ownerLocalDeclarationEvidence( return { startLine: range.start, endLine: range.end, - lines: input.sourceLines - .slice(range.start - 1, range.end) - .map((text, offset) => ({ lineNumber: range.start + offset, text })), + lines: literalDelimiterPreservingLines(range, sourceFile, input.sourceLines), } }) .filter((declaration) => ( diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index d3c369fc..de48f2b4 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -88,6 +88,7 @@ import { communitiesFromGraph, estimateQueryTokens } from './serve.js' import { ownerLocalDeclarationEvidence, queryEvidenceSourceProjection, + retainQueryEvidenceSourceSnapshot, type RepresentedQueryEvidenceSource, } from './query-evidence-dependencies.js' @@ -703,7 +704,9 @@ function fileLinesForSnippet(sourceFile: string, fileCache?: Map { })).toEqual([]) }) + it('authenticates exact raw CRLF template delimiters and rejects altered delimiters directly', () => { + const rawSource = [ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ].join('\r\n') + const sourceLines = rawSource.split('\n') + const represented = sourceLines.slice(2, 4).join('\n') + const input = { + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 5 }, + } + + expect(ownerLocalDeclarationEvidence({ + ...input, + representedSource: [{ startLine: 3, endLine: 4, text: represented }], + })).toEqual([{ + startLine: 2, + endLine: 2, + lines: [{ lineNumber: 2, text: ' const datum = 12.5\r' }], + }]) + expect(ownerLocalDeclarationEvidence({ + ...input, + representedSource: [{ + startLine: 3, + endLine: 4, + text: represented.replace(/\r\n/g, '\n'), + }], + })).toEqual([]) + }) + + it.each([ + { name: 'CRLF', lineEnding: '\r\n' as const, literalDelimiter: '\r\n' }, + { name: 'LF', lineEnding: '\n' as const, literalDelimiter: '\n' }, + ])('preserves raw multiline template delimiters from a public $name read', ({ lineEnding, literalDelimiter }) => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble', lineEnding }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: return dispatchOutcome(datum, \` alpha ?${literalDelimiter}L4: beta \`)`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it.each(['\n', '\r\n'] as const)( + 'normalizes %j source layout outside literals while retaining ordinary indentation', + (lineEnding) => { + const evidence = evidenceFor([ + 'function assemble() {', + '\tconst datum = 12.5', + '\treturn datum ?', + '\t\tdispatchOutcome(datum, `single line`)', + '\t\t: retryOutcome()', + '}', + ], { + question: 'What are the dispatch outcome and retry outcome?', + label: 'assemble', + lineEnding, + }) + + expect(evidence?.snippet).toBe([ + 'L2: \tconst datum = 12.5', + 'L3: return datum ? dispatchOutcome(datum, `single line`) : retryOutcome()', + ].join('\n')) + expect(evidence?.snippet).not.toContain('\r') + }, + ) + it('preserves selected string, template, and regex literal bytes while completing their const', () => { const statement = ' return dispatchOutcome(datum, "alpha beta \\"// literal", `left\tright`, /x y\\/\\/z/)' const evidence = evidenceFor([ @@ -1417,6 +1497,97 @@ describe('owner declaration completion budgets and atomic preservation', () => { }) describe('owner declaration parse snapshot cache', () => { + it('preserves raw CRLF literal provenance across a warm public cache hit', () => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ] + const { sourceFile, sourceLocation } = sourceFixture(source, '.ts', '\r\n') + const fileCache = new Map() + const read = () => readQueryEvidenceSnippet(sourceFile, 1, { + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation, + fileCache, + }) + + const cold = read() + const warm = read() + + expect(cold?.snippet).toContain('` alpha ?\r\nL4: beta `') + expect(warm).toEqual(cold) + expect(fileCache.get(sourceFile)).toEqual(source) + expect(ts.createSourceFile).toHaveBeenCalledTimes(1) + }) + + it('does not invent CRLF bytes for a caller-supplied normalized cache', () => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ] + const { sourceFile, sourceLocation } = sourceFixture(source, '.ts', '\r\n') + const fileCache = new Map([[sourceFile, [...source]]]) + const evidence = readQueryEvidenceSnippet(sourceFile, 1, { + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation, + fileCache, + }) + + expect(evidence?.snippet).toContain('` alpha ?\nL4: beta `') + expect(evidence?.snippet).not.toContain('\r') + }) + + it('does not reuse raw provenance after the cache array content or path changes', () => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha ?', + ' beta `)', + '}', + ] + const firstFixture = sourceFixture(source, '.ts', '\r\n') + const secondFixture = sourceFixture(source, '.mts', '\r\n') + const fileCache = new Map() + const options = (sourceLocation: string) => ({ + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation, + fileCache, + }) + + const cold = readQueryEvidenceSnippet( + firstFixture.sourceFile, + 1, + options(firstFixture.sourceLocation), + ) + const cachedLines = fileCache.get(firstFixture.sourceFile)! + fileCache.set(secondFixture.sourceFile, cachedLines) + const changedPath = readQueryEvidenceSnippet( + secondFixture.sourceFile, + 1, + options(secondFixture.sourceLocation), + ) + cachedLines[2] = ' return dispatchOutcome(datum, ` changed ?' + const changedContent = readQueryEvidenceSnippet( + firstFixture.sourceFile, + 1, + options(firstFixture.sourceLocation), + ) + + expect(cold?.snippet).toContain('` alpha ?\r\nL4: beta `') + expect(changedPath?.snippet).toContain('` alpha ?\nL4: beta `') + expect(changedPath?.snippet).not.toContain('\r') + expect(changedContent?.snippet).toContain('` changed ?\nL4: beta `') + expect(changedContent?.snippet).not.toContain('\r') + }) + it('invalidates changed content or paths even when the source-line array identity is reused', () => { const sourceLines = [ 'export function executeSample() {', From ead4cbcd126cdbddc9afc3f21323e8dcac410b1f Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 00:27:33 +0400 Subject: [PATCH 13/27] fix(retrieval): preserve complete multiline literal evidence --- src/runtime/query-evidence-dependencies.ts | 151 +++++++++++++++++- src/runtime/retrieve.ts | 146 +++++++++++------ .../unit/retrieve-owner-declarations.test.ts | 131 +++++++++++++++ 3 files changed, 374 insertions(+), 54 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 8f6a31bf..fe88ecfd 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -29,6 +29,7 @@ export interface OwnerDeclarationCompletionInput { export interface QueryEvidenceSourceProjection { text: string literalLineBreaks: Array<{ offset: number; lineNumber: number }> + hasProtectedTokens: boolean } interface ParsedSourceSnapshot { @@ -294,6 +295,7 @@ function physicalSourceBounds( function protectedTokensForRange( range: { start: number; end: number }, sourceFile: ts.SourceFile, + allowClippedTokens = false, ): ProtectedSourceToken[] | null { const bounds = physicalSourceBounds(range, sourceFile) if (!bounds) { @@ -307,7 +309,32 @@ function protectedTokensForRange( const end = node.getEnd() if (start < bounds.end && bounds.start < end) { if (start < bounds.start || end > bounds.end) { - clippedToken = true + if (!allowClippedTokens) { + clippedToken = true + return + } + const segmentStart = Math.max(start, bounds.start) + let segmentEnd = Math.min(end, bounds.end) + if (end > bounds.end) { + if (sourceFile.text.slice(segmentEnd - 2, segmentEnd) === '\r\n') { + segmentEnd -= 2 + } else if (/^[\r\n]$/.test(sourceFile.text.slice(segmentEnd - 1, segmentEnd))) { + segmentEnd -= 1 + } + } + if (segmentStart >= segmentEnd) { + clippedToken = true + return + } + const text = sourceFile.text.slice(segmentStart, segmentEnd) + tokens.push({ + start: segmentStart, + end: segmentEnd, + text, + representedText: sourceFilesWithNormalizedLineCaches.has(sourceFile) + ? text.replace(/\r\n/g, '\n') + : text, + }) return } const text = sourceFile.text.slice(start, end) @@ -337,9 +364,10 @@ function tokenMarker(index: number): string { function physicalProjectionForRange( range: { start: number; end: number }, sourceFile: ts.SourceFile, + allowClippedTokens = false, ): PhysicalSourceProjection | null { const bounds = physicalSourceBounds(range, sourceFile) - const tokens = protectedTokensForRange(range, sourceFile) + const tokens = protectedTokensForRange(range, sourceFile, allowClippedTokens) if (!bounds || !tokens) { return null } @@ -390,7 +418,13 @@ function physicalProjectionForRange( identityCursor = markerOffset + marker.length } - return { text, identityText, literalLineBreaks, tokens } + return { + text, + identityText, + literalLineBreaks, + hasProtectedTokens: tokens.length > 0, + tokens, + } } function representedSourceIdentity( @@ -472,6 +506,7 @@ export function queryEvidenceSourceProjection(input: { const shapedText = normalizedSource(input.shapedText) let cursor = 0 let projectedText = '' + let hasProtectedTokens = false const literalLineBreaks: Array<{ offset: number; lineNumber: number }> = [] for (const represented of input.representedSource) { if ( @@ -484,13 +519,14 @@ export function queryEvidenceSourceProjection(input: { return null } const range = { start: represented.startLine, end: represented.endLine } - const projection = physicalProjectionForRange(range, sourceFile) + const projection = physicalProjectionForRange(range, sourceFile, true) if ( !projection || representedSourceIdentity(represented.text, projection) !== projection.identityText ) { return null } + hasProtectedTokens ||= projection.hasProtectedTokens const normalizedComponent = normalizedSource(represented.text) const componentOffset = shapedText.indexOf(normalizedComponent, cursor) if (normalizedComponent.length === 0 || componentOffset < 0) { @@ -508,7 +544,112 @@ export function queryEvidenceSourceProjection(input: { cursor = componentOffset + normalizedComponent.length } projectedText += shapedText.slice(cursor) - return { text: projectedText, literalLineBreaks } + return { + text: projectedText, + literalLineBreaks, + hasProtectedTokens, + } + } catch { + return null + } +} + +/** + * Expands one exact selected source fragment to its containing literal-bearing + * statement when both the statement and its unambiguous owner are supplied. + */ +export function completeQueryEvidenceLiteralStatement(input: { + sourceFilePath: string + sourceLines: readonly string[] + ownerRange: { start: number; end: number } + representedSource: readonly RepresentedQueryEvidenceSource[] +}): RepresentedQueryEvidenceSource | null { + try { + if (input.representedSource.length !== 1) { + return null + } + const sourceFile = parsedSourceForSnapshot(input.sourceFilePath, input.sourceLines) + if (!sourceFile) { + return null + } + const parseDiagnostics = (sourceFile as ts.SourceFile & { + parseDiagnostics?: readonly ts.Diagnostic[] + }).parseDiagnostics + if (parseDiagnostics && parseDiagnostics.length > 0) { + return null + } + + const owner = uniquelyIdentifiedOwner(sourceFile, input.ownerRange) + if (!owner || !ts.isBlock(owner.body)) { + return null + } + const represented = input.representedSource[0]! + if ( + !Number.isInteger(represented.startLine) + || !Number.isInteger(represented.endLine) + || represented.startLine < input.ownerRange.start + || represented.endLine > input.ownerRange.end + || represented.startLine > represented.endLine + || represented.text !== input.sourceLines + .slice(represented.startLine - 1, represented.endLine) + .join('\n') + ) { + return null + } + const representedRange = { start: represented.startLine, end: represented.endLine } + const candidates: Array<{ start: number; end: number }> = [] + const visit = (node: ts.Node): void => { + if (node !== owner && isNestedFunctionOrClass(node)) { + return + } + if (node !== owner && ts.isStatement(node)) { + const range = evidenceLineRangeOf(node, sourceFile) + if ( + input.ownerRange.start <= range.start + && range.end <= input.ownerRange.end + && range.start <= representedRange.start + && representedRange.end <= range.end + && (range.start !== representedRange.start || range.end !== representedRange.end) + ) { + let crossesRepresentedBoundary = false + const findCrossingLiteral = (descendant: ts.Node): void => { + if (crossesRepresentedBoundary || (descendant !== node && isNestedFunctionOrClass(descendant))) { + return + } + if (PROTECTED_LITERAL_KINDS.has(descendant.kind)) { + const literalRange = lineRangeOf(descendant, sourceFile) + if ( + literalRange.start <= representedRange.end + && representedRange.start <= literalRange.end + && ( + literalRange.start < representedRange.start + || literalRange.end > representedRange.end + ) + ) { + crossesRepresentedBoundary = true + return + } + } + ts.forEachChild(descendant, findCrossingLiteral) + } + findCrossingLiteral(node) + if (crossesRepresentedBoundary) { + candidates.push(range) + } + } + } + ts.forEachChild(node, visit) + } + visit(owner) + if (candidates.length !== 1) { + return null + } + const range = candidates[0]! + return { + startLine: range.start, + endLine: range.end, + text: input.sourceLines.slice(range.start - 1, range.end).join('\n'), + } } catch { return null } diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index de48f2b4..76b078dc 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -86,6 +86,7 @@ import { } from './retrieve/pipeline.js' import { communitiesFromGraph, estimateQueryTokens } from './serve.js' import { + completeQueryEvidenceLiteralStatement, ownerLocalDeclarationEvidence, queryEvidenceSourceProjection, retainQueryEvidenceSourceSnapshot, @@ -1283,59 +1284,106 @@ function renderQueryEvidenceLineEntries( lines: readonly QueryEvidenceLine[], sourceFile: string, sourceLines: readonly string[], + ownerRange?: { start: number; end: number } | null, ): RenderedQueryEvidenceLine[] { const rendered: RenderedQueryEvidenceLine[] = [] let usedChars = 0 + let usedRows = 0 for (const line of lines) { - const normalized = line.text.replace(/\s+/g, ' ').trim() - const projection = queryEvidenceSourceProjection({ - sourceFilePath: sourceFile, - sourceLines, - representedSource: line.representedSource, - shapedText: line.text, - }) - let projectedContent = projection?.text ?? normalized - if (projection) { - for (let index = projection.literalLineBreaks.length - 1; index >= 0; index -= 1) { - const lineBreak = projection.literalLineBreaks[index]! - projectedContent = [ - projectedContent.slice(0, lineBreak.offset + 1), - `L${lineBreak.lineNumber}: `, - projectedContent.slice(lineBreak.offset + 1), - ].join('') - } - } - const projectedRows = projectedContent.split('\n') - const lineCapExceeded = projectedRows.some((row, index) => { - const content = index === 0 ? row : row.replace(/^L\d+: /, '') - return content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP - }) - const content = lineCapExceeded - ? projectedRows.map((row, index) => { - const match = index > 0 ? /^(L\d+: )(.*)$/.exec(row) : null - const rowPrefix = match?.[1] ?? '' - const rowContent = match?.[2] ?? row - return rowContent.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP - ? `${rowPrefix}${rowContent.slice(0, QUERY_EVIDENCE_SNIPPET_LINE_CAP - 3)}...` - : row - }).join('\n') - : projectedContent - const prefix = `L${line.index}: ` const separatorChars = rendered.length > 0 ? 1 : 0 const remaining = QUERY_EVIDENCE_SNIPPET_CHAR_CAP - usedChars - separatorChars - if (remaining <= prefix.length + 8) { + const completedSource = ownerRange + ? completeQueryEvidenceLiteralStatement({ + sourceFilePath: sourceFile, + sourceLines, + ownerRange, + representedSource: line.representedSource, + }) + : null + const completedLine: QueryEvidenceLine | null = completedSource + ? { + ...line, + index: completedSource.startLine, + endIndex: completedSource.endLine, + text: completedSource.text, + representedSource: [completedSource], + } + : null + let selected: { + source: QueryEvidenceLine + projection: ReturnType + content: string + lineCapExceeded: boolean + faithful: string + } | null = null + + for (const candidate of completedLine ? [completedLine, line] : [line]) { + const normalized = candidate.text.replace(/\s+/g, ' ').trim() + const projection = queryEvidenceSourceProjection({ + sourceFilePath: sourceFile, + sourceLines, + representedSource: candidate.representedSource, + shapedText: candidate.text, + }) + let projectedContent = projection?.text ?? normalized + if (projection) { + for (let index = projection.literalLineBreaks.length - 1; index >= 0; index -= 1) { + const lineBreak = projection.literalLineBreaks[index]! + projectedContent = [ + projectedContent.slice(0, lineBreak.offset + 1), + `L${lineBreak.lineNumber}: `, + projectedContent.slice(lineBreak.offset + 1), + ].join('') + } + } + const projectedRows = projectedContent.split('\n') + const lineCapExceeded = projectedRows.some((row, index) => { + const content = index === 0 ? row : row.replace(/^L\d+: /, '') + return content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + }) + const completedSourceWouldBeAltered = candidate === completedLine && ( + projection === null + || projection.hasProtectedTokens === false + || lineCapExceeded + || usedRows + projectedRows.length > QUERY_EVIDENCE_SNIPPET_MAX_LINES + || `L${candidate.index}: ${projectedContent}`.length > remaining + ) + if (completedSourceWouldBeAltered) { + continue + } + const content = lineCapExceeded + ? projectedRows.map((row, index) => { + const match = index > 0 ? /^(L\d+: )(.*)$/.exec(row) : null + const rowPrefix = match?.[1] ?? '' + const rowContent = match?.[2] ?? row + return rowContent.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + ? `${rowPrefix}${rowContent.slice(0, QUERY_EVIDENCE_SNIPPET_LINE_CAP - 3)}...` + : row + }).join('\n') + : projectedContent + const prefix = `L${candidate.index}: ` + const faithful = `${prefix}${content}` + if ( + remaining <= prefix.length + 8 + ) { + continue + } + selected = { source: candidate, projection, content, lineCapExceeded, faithful } break } - const faithful = `${prefix}${content}` - const bounded = faithful.slice(0, remaining).trimEnd() + if (!selected) { + continue + } + const bounded = selected.faithful.slice(0, remaining).trimEnd() rendered.push({ - source: line, + source: selected.source, text: bounded, - representedSource: projection && !lineCapExceeded && bounded === faithful - ? line.representedSource + representedSource: selected.projection && !selected.lineCapExceeded && bounded === selected.faithful + ? selected.source.representedSource : [], }) usedChars += bounded.length + separatorChars + usedRows += bounded.split('\n').length } return rendered } @@ -1481,6 +1529,14 @@ export function readQueryEvidenceSnippet( .map((obligation) => obligation.index), ) const parsedRange = lineRangeFromSourceLocation(options.sourceLocation) + const ownerRange = explicitOwnerRange( + options.sourceLocation, + parsedRange, + lineNumber, + lines.length, + options.derived, + options.fileNodeLike, + ) const fallbackHalfWindow = options.derived ? DERIVED_SNIPPET_HALF_WINDOW : SNIPPET_HALF_WINDOW const symbolRange = boundedSourceRange(lines.length, parsedRange ?? { start: lineNumber - fallbackHalfWindow, @@ -1570,19 +1626,11 @@ export function readQueryEvidenceSnippet( selectedLines = mergedLines } } - const baseline = renderQueryEvidenceLineEntries(selectedLines, sourceFile, lines) + const baseline = renderQueryEvidenceLineEntries(selectedLines, sourceFile, lines, ownerRange) if (baseline.length === 0) { return null } const baselineSnippet = baseline.map((line) => line.text).join('\n') - const ownerRange = explicitOwnerRange( - options.sourceLocation, - parsedRange, - lineNumber, - lines.length, - options.derived, - options.fileNodeLike, - ) const completed = ownerRange ? completeOwnerDeclarationEvidence(sourceFile, lines, ownerRange, baseline) : null diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index cb1bb500..b3a13a4a 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -1381,6 +1381,137 @@ describe('owner-local binding identity correction', () => { }) }) +describe('bounded multiline literal statement completion', () => { + it.each([ + { name: 'LF', lineEnding: '\n' as const, literalDelimiter: '\n' }, + { name: 'CRLF', lineEnding: '\r\n' as const, literalDelimiter: '\r\n' }, + ])('completes an ordinary $name no-substitution template statement', ({ lineEnding, literalDelimiter }) => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha', + ' beta `)', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble', lineEnding }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: return dispatchOutcome(datum, \` alpha${literalDelimiter}L4: beta \`)`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('completes an ordinary interpolated template statement', () => { + const evidence = evidenceFor([ + 'function assemble(suffix: string) {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha', + ' ${suffix} beta `)', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble' }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + 'L3: return dispatchOutcome(datum, ` alpha', + 'L4: ${suffix} beta `)', + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('completes a supported physical-line continued string statement', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, " alpha\\', + ' beta ")', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble' }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + 'L3: return dispatchOutcome(datum, " alpha\\', + 'L4: beta ")', + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('keeps an owner-clipped literal fragment faithful without inferring its declaration', () => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return dispatchOutcome(datum, ` alpha', + ' beta `)', + '}', + ], { + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation: 'L1-L3', + }) + + expect(evidence).toEqual({ + snippet: 'L3: return dispatchOutcome(datum, ` alpha', + lineNumber: 3, + scope: 'symbol', + }) + }) + + it('does not complete a literal statement from a foreign owner', () => { + const evidence = evidenceFor([ + 'function assemble() {', + " const datum = 'owner'", + ' return computeWidget(datum)', + '}', + 'async function transmit() {', + " const remote = 'foreign'", + ' return retryDelivery(await dispatchRecord(validateRequest(remote), ` alpha', + ' beta `))', + '}', + ], { + question: 'How does compute widget validate and dispatch a record with retry delivery?', + label: 'assemble', + sourceLocation: 'L1-L4', + }) + + expect(evidence?.scope).toBe('source_file') + expect(evidence?.snippet).toContain("L2: const datum = 'owner'") + expect(evidence?.snippet).toContain('return computeWidget(datum)') + expect(evidence?.snippet).toContain('` alpha') + expect(evidence?.snippet).not.toContain('beta `') + expect(evidence?.snippet).not.toContain("const remote = 'foreign'") + }) + + it('declines an overflowing completion while retaining other selected evidence', () => { + const firstLiteralRow = ` return dispatchOutcome(datum, \` ${'a'.repeat(160)}` + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' await retryOutcome()', + firstLiteralRow, + ` ${'b'.repeat(160)} \`)`, + '}', + ], { + question: 'What are the dispatch outcome and retry outcome?', + label: 'assemble', + }) + + expect(evidence?.snippet).toBe([ + 'L3: await retryOutcome()', + `L4: ${firstLiteralRow.trimStart()}`, + ].join('\n')) + expect(evidence?.snippet).not.toContain('const datum') + expect(evidence?.snippet).not.toContain('...') + }) +}) + describe('owner declaration completion budgets and atomic preservation', () => { it('admits faithful literal whitespace when the completed snippet exactly fills the cap', () => { const literal = `${'value '.repeat(22)}end` From 2a85a1152ae9585bfc635eca536275368e201d23 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 00:45:08 +0400 Subject: [PATCH 14/27] fix(retrieval): select innermost literal statement by ancestry --- src/runtime/query-evidence-dependencies.ts | 28 +++++++++-- .../unit/retrieve-owner-declarations.test.ts | 50 +++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index fe88ecfd..23264292 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -480,6 +480,17 @@ function literalDelimiterPreservingLines( }) } +function isStrictAncestorOf(ancestor: ts.Node, descendant: ts.Node): boolean { + let current = descendant.parent + while (current) { + if (current === ancestor) { + return true + } + current = current.parent + } + return false +} + /** * Projects already-shaped query evidence back onto its authenticated physical * source while normalizing layout only outside string/template/regex tokens. @@ -597,7 +608,10 @@ export function completeQueryEvidenceLiteralStatement(input: { return null } const representedRange = { start: represented.startLine, end: represented.endLine } - const candidates: Array<{ start: number; end: number }> = [] + const candidates: Array<{ + statement: ts.Statement + range: { start: number; end: number } + }> = [] const visit = (node: ts.Node): void => { if (node !== owner && isNestedFunctionOrClass(node)) { return @@ -634,17 +648,23 @@ export function completeQueryEvidenceLiteralStatement(input: { } findCrossingLiteral(node) if (crossesRepresentedBoundary) { - candidates.push(range) + candidates.push({ statement: node, range }) } } } ts.forEachChild(node, visit) } visit(owner) - if (candidates.length !== 1) { + const innermostCandidates = candidates.filter((candidate) => ( + !candidates.some((other) => ( + other !== candidate + && isStrictAncestorOf(candidate.statement, other.statement) + )) + )) + if (innermostCandidates.length !== 1) { return null } - const range = candidates[0]! + const range = innermostCandidates[0]!.range return { startLine: range.start, endLine: range.end, diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index b3a13a4a..aaa26a38 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -1444,6 +1444,56 @@ describe('bounded multiline literal statement completion', () => { }) }) + it.each([ + { name: 'LF', lineEnding: '\n' as const, literalDelimiter: '\n' }, + { name: 'CRLF', lineEnding: '\r\n' as const, literalDelimiter: '\r\n' }, + ])('completes an inline-if $name multiline literal return and its declaration', ({ + lineEnding, + literalDelimiter, + }) => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' if (true) return dispatchOutcome(datum, ` alpha', + ' beta `)', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble', lineEnding }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: if (true) return dispatchOutcome(datum, \` alpha${literalDelimiter}L4: beta \`)`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it.each([ + { name: 'LF', lineEnding: '\n' as const, literalDelimiter: '\n' }, + { name: 'CRLF', lineEnding: '\r\n' as const, literalDelimiter: '\r\n' }, + ])('completes a block/loop-nested $name multiline literal return and its declaration', ({ + lineEnding, + literalDelimiter, + }) => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' for (;;) { if (true) { return dispatchOutcome(datum, ` alpha', + ' beta `) } }', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble', lineEnding }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: for (;;) { if (true) { return dispatchOutcome(datum, \` alpha${literalDelimiter}L4: beta \`) } }`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + it('keeps an owner-clipped literal fragment faithful without inferring its declaration', () => { const evidence = evidenceFor([ 'function assemble() {', From 14c4b130d14e0860aad636370b71222322f6eee1 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 01:13:19 +0400 Subject: [PATCH 15/27] fix(retrieval): preserve disjoint fragments during literal completion --- src/runtime/query-evidence-dependencies.ts | 95 +++++++++---- src/runtime/retrieve.ts | 56 ++++++-- .../unit/retrieve-owner-declarations.test.ts | 132 +++++++++++++++++- 3 files changed, 244 insertions(+), 39 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 23264292..d646c5bf 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -566,17 +566,17 @@ export function queryEvidenceSourceProjection(input: { } /** - * Expands one exact selected source fragment to its containing literal-bearing - * statement when both the statement and its unambiguous owner are supplied. + * Expands exact selected source fragments to their uniquely containing + * literal-bearing statement while preserving other authenticated fragments. */ export function completeQueryEvidenceLiteralStatement(input: { sourceFilePath: string sourceLines: readonly string[] ownerRange: { start: number; end: number } representedSource: readonly RepresentedQueryEvidenceSource[] -}): RepresentedQueryEvidenceSource | null { +}): RepresentedQueryEvidenceSource[] | null { try { - if (input.representedSource.length !== 1) { + if (input.representedSource.length === 0) { return null } const sourceFile = parsedSourceForSnapshot(input.sourceFilePath, input.sourceLines) @@ -594,20 +594,36 @@ export function completeQueryEvidenceLiteralStatement(input: { if (!owner || !ts.isBlock(owner.body)) { return null } - const represented = input.representedSource[0]! - if ( - !Number.isInteger(represented.startLine) - || !Number.isInteger(represented.endLine) - || represented.startLine < input.ownerRange.start - || represented.endLine > input.ownerRange.end - || represented.startLine > represented.endLine - || represented.text !== input.sourceLines - .slice(represented.startLine - 1, represented.endLine) - .join('\n') - ) { - return null + const authenticated = [...input.representedSource] + .sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine) + const representedSource: RepresentedQueryEvidenceSource[] = [] + for (const represented of authenticated) { + if ( + !Number.isInteger(represented.startLine) + || !Number.isInteger(represented.endLine) + || represented.startLine < input.ownerRange.start + || represented.endLine > input.ownerRange.end + || represented.startLine > represented.endLine + || represented.text !== input.sourceLines + .slice(represented.startLine - 1, represented.endLine) + .join('\n') + ) { + return null + } + const previous = representedSource.at(-1) + if ( + previous + && previous.startLine === represented.startLine + && previous.endLine === represented.endLine + && previous.text === represented.text + ) { + continue + } + if (previous && represented.startLine <= previous.endLine) { + return null + } + representedSource.push(represented) } - const representedRange = { start: represented.startLine, end: represented.endLine } const candidates: Array<{ statement: ts.Statement range: { start: number; end: number } @@ -618,13 +634,17 @@ export function completeQueryEvidenceLiteralStatement(input: { } if (node !== owner && ts.isStatement(node)) { const range = evidenceLineRangeOf(node, sourceFile) - if ( - input.ownerRange.start <= range.start - && range.end <= input.ownerRange.end - && range.start <= representedRange.start - && representedRange.end <= range.end - && (range.start !== representedRange.start || range.end !== representedRange.end) - ) { + const completesRepresentedFragment = representedSource.some((represented) => { + const representedRange = { start: represented.startLine, end: represented.endLine } + if ( + input.ownerRange.start > range.start + || range.end > input.ownerRange.end + || range.start > representedRange.start + || representedRange.end > range.end + || (range.start === representedRange.start && range.end === representedRange.end) + ) { + return false + } let crossesRepresentedBoundary = false const findCrossingLiteral = (descendant: ts.Node): void => { if (crossesRepresentedBoundary || (descendant !== node && isNestedFunctionOrClass(descendant))) { @@ -647,9 +667,10 @@ export function completeQueryEvidenceLiteralStatement(input: { ts.forEachChild(descendant, findCrossingLiteral) } findCrossingLiteral(node) - if (crossesRepresentedBoundary) { - candidates.push({ statement: node, range }) - } + return crossesRepresentedBoundary + }) + if (completesRepresentedFragment) { + candidates.push({ statement: node, range }) } } ts.forEachChild(node, visit) @@ -665,11 +686,29 @@ export function completeQueryEvidenceLiteralStatement(input: { return null } const range = innermostCandidates[0]!.range - return { + const completed: RepresentedQueryEvidenceSource = { startLine: range.start, endLine: range.end, text: input.sourceLines.slice(range.start - 1, range.end).join('\n'), } + const result: RepresentedQueryEvidenceSource[] = [] + let insertedCompletion = false + for (const represented of representedSource) { + const insideCompletion = range.start <= represented.startLine && represented.endLine <= range.end + const overlapsCompletion = represented.startLine <= range.end && range.start <= represented.endLine + if (overlapsCompletion && !insideCompletion) { + return null + } + if (insideCompletion) { + if (!insertedCompletion) { + result.push(completed) + insertedCompletion = true + } + } else { + result.push(represented) + } + } + return insertedCompletion ? result : null } catch { return null } diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 76b078dc..83be4783 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -1280,6 +1280,40 @@ interface RenderedQueryEvidenceLine { representedSource: readonly RepresentedQueryEvidenceSource[] } +function completedQueryEvidenceSourceProjection( + sourceFilePath: string, + sourceLines: readonly string[], + representedSource: readonly RepresentedQueryEvidenceSource[], +): ReturnType { + let text = '' + let hasProtectedTokens = false + const literalLineBreaks: Array<{ offset: number; lineNumber: number }> = [] + for (const [index, represented] of representedSource.entries()) { + const projection = queryEvidenceSourceProjection({ + sourceFilePath, + sourceLines, + representedSource: [represented], + shapedText: represented.text, + }) + if (!projection) { + return null + } + if (index > 0) { + text += `\nL${represented.startLine}: ` + } + const projectionStart = text.length + text += projection.text + hasProtectedTokens ||= projection.hasProtectedTokens + for (const lineBreak of projection.literalLineBreaks) { + literalLineBreaks.push({ + offset: projectionStart + lineBreak.offset, + lineNumber: lineBreak.lineNumber, + }) + } + } + return { text, literalLineBreaks, hasProtectedTokens } +} + function renderQueryEvidenceLineEntries( lines: readonly QueryEvidenceLine[], sourceFile: string, @@ -1303,10 +1337,10 @@ function renderQueryEvidenceLineEntries( const completedLine: QueryEvidenceLine | null = completedSource ? { ...line, - index: completedSource.startLine, - endIndex: completedSource.endLine, - text: completedSource.text, - representedSource: [completedSource], + index: completedSource[0]!.startLine, + endIndex: completedSource.at(-1)!.endLine, + text: completedSource.map((source) => source.text).join(' '), + representedSource: completedSource, } : null let selected: { @@ -1319,12 +1353,14 @@ function renderQueryEvidenceLineEntries( for (const candidate of completedLine ? [completedLine, line] : [line]) { const normalized = candidate.text.replace(/\s+/g, ' ').trim() - const projection = queryEvidenceSourceProjection({ - sourceFilePath: sourceFile, - sourceLines, - representedSource: candidate.representedSource, - shapedText: candidate.text, - }) + const projection = candidate === completedLine + ? completedQueryEvidenceSourceProjection(sourceFile, sourceLines, candidate.representedSource) + : queryEvidenceSourceProjection({ + sourceFilePath: sourceFile, + sourceLines, + representedSource: candidate.representedSource, + shapedText: candidate.text, + }) let projectedContent = projection?.text ?? normalized if (projection) { for (let index = projection.literalLineBreaks.length - 1; index >= 0; index -= 1) { diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index aaa26a38..640d0f07 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -13,7 +13,10 @@ vi.mock('typescript', async (importOriginal) => { import * as ts from 'typescript' -import { ownerLocalDeclarationEvidence } from '../../src/runtime/query-evidence-dependencies.js' +import { + completeQueryEvidenceLiteralStatement, + ownerLocalDeclarationEvidence, +} from '../../src/runtime/query-evidence-dependencies.js' import { readQueryEvidenceSnippet, type QueryEvidenceSnippet } from '../../src/runtime/retrieve.js' const QUESTION = 'How does dispatch outcome return a validated result with retry handling?' @@ -1494,6 +1497,133 @@ describe('bounded multiline literal statement completion', () => { }) }) + it.each([ + { name: 'LF', lineEnding: '\n' as const, literalDelimiter: '\n' }, + { name: 'CRLF', lineEnding: '\r\n' as const, literalDelimiter: '\r\n' }, + ])('completes a separate-line loop/if $name multiline literal return without dropping the header', ({ + lineEnding, + literalDelimiter, + }) => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' for (const ready of [true]) {', + ' if (ready) {', + ' return dispatchOutcome(datum, ` alpha', + ' beta `)', + ' }', + ' }', + '}', + ], { question: 'What is the dispatch outcome?', label: 'assemble', lineEnding }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + 'L3: for (const ready of [true]) {', + `L5: return dispatchOutcome(datum, \` alpha${literalDelimiter}L6: beta \`)`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('preserves authenticated disjoint fragments while completing only their unique literal statement', () => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ' for (const ready of [true]) {', + ' if (ready) {', + ' return dispatchOutcome(datum, ` alpha', + ' beta `)', + ' }', + ' }', + '}', + ] + const { sourceFile } = sourceFixture(source) + + expect(completeQueryEvidenceLiteralStatement({ + sourceFilePath: sourceFile, + sourceLines: source, + ownerRange: { start: 1, end: 9 }, + representedSource: [ + { startLine: 3, endLine: 3, text: source[2]! }, + { startLine: 5, endLine: 5, text: source[4]! }, + ], + })).toEqual([ + { startLine: 3, endLine: 3, text: source[2] }, + { startLine: 5, endLine: 6, text: `${source[4]}\n${source[5]}` }, + ]) + }) + + it.each([ + { + name: 'clipped source bytes', + represented: { startLine: 5, endLine: 5, text: 'return dispatchOutcome(datum, ` alpha' }, + }, + { + name: 'wrong source coordinates', + represented: { startLine: 4, endLine: 4, text: ' return dispatchOutcome(datum, ` alpha' }, + }, + { + name: 'a foreign owner fragment', + represented: { startLine: 11, endLine: 11, text: ' return dispatchOutcome(remote, ` gamma' }, + }, + ])('rejects $name in an otherwise authenticated multi-fragment selection', ({ represented }) => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ' for (const ready of [true]) {', + ' if (ready) {', + ' return dispatchOutcome(datum, ` alpha', + ' beta `)', + ' }', + ' }', + '}', + 'function transmit() {', + ' return dispatchOutcome(remote, ` gamma', + ' delta `)', + '}', + ] + const { sourceFile } = sourceFixture(source) + + expect(completeQueryEvidenceLiteralStatement({ + sourceFilePath: sourceFile, + sourceLines: source, + ownerRange: { start: 1, end: 9 }, + representedSource: [ + { startLine: 3, endLine: 3, text: source[2]! }, + represented, + ], + })).toBeNull() + }) + + it('does not coalesce clipped sibling literal statements across a source gap', () => { + const source = [ + 'function assemble() {', + " const first = 'first'", + ' if (first) {', + ' return dispatchOutcome(first, ` alpha', + ' beta `)', + ' }', + ' if (second) {', + ' return dispatchOutcome(second, ` gamma', + ' delta `)', + ' }', + '}', + ] + const { sourceFile } = sourceFixture(source) + + expect(completeQueryEvidenceLiteralStatement({ + sourceFilePath: sourceFile, + sourceLines: source, + ownerRange: { start: 1, end: 11 }, + representedSource: [ + { startLine: 4, endLine: 4, text: source[3]! }, + { startLine: 8, endLine: 8, text: source[7]! }, + ], + })).toBeNull() + }) + it('keeps an owner-clipped literal fragment faithful without inferring its declaration', () => { const evidence = evidenceFor([ 'function assemble() {', From 07f3782054c8be7894a4a7919741dcc2130c4d84 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 01:46:44 +0400 Subject: [PATCH 16/27] feat: add bounded stored-source discovery signal --- src/runtime/retrieve-source-terms.ts | 291 +++++++++++++++++++ src/runtime/retrieve.ts | 21 +- tests/unit/retrieve-source-discovery.test.ts | 270 +++++++++++++++++ tests/unit/retrieve-source-terms.test.ts | 167 +++++++++++ 4 files changed, 747 insertions(+), 2 deletions(-) create mode 100644 src/runtime/retrieve-source-terms.ts create mode 100644 tests/unit/retrieve-source-discovery.test.ts create mode 100644 tests/unit/retrieve-source-terms.test.ts diff --git a/src/runtime/retrieve-source-terms.ts b/src/runtime/retrieve-source-terms.ts new file mode 100644 index 00000000..97eaa1a8 --- /dev/null +++ b/src/runtime/retrieve-source-terms.ts @@ -0,0 +1,291 @@ +import { basename, extname, isAbsolute, relative, resolve, sep } from 'node:path' + +import type { KnowledgeGraph } from '../contracts/graph.js' +import { tokenizeLabel } from './retrieve/pipeline.js' + +const SOURCE_LOCATION = /^L([1-9]\d*)(?:-L([1-9]\d*))?$/ +const IDENTIFIER = /^[$_\p{ID_Start}][$\u200C\u200D\p{ID_Continue}]*$/u +const MAX_STORED_SNIPPET_LINES = 25 +const MAX_STORED_SNIPPET_CHARS = 2_000 +const MAX_TRUNCATED_SNIPPET_CHARS = MAX_STORED_SNIPPET_CHARS + 3 +const BUILTIN_EXTRACTOR_PREFIX = 'builtin:extract:' +const OWN = Object.prototype.hasOwnProperty +const EMPTY_TOKENS: readonly string[] = Object.freeze([]) + +const SOURCE_RELEVANT_FIELDS = [ + 'label', + 'file_type', + 'source_file', + 'source_location', + 'line_number', + 'node_kind', + 'snippet', + 'provenance', + 'framework_metadata', + 'external_call', + 'synthetic', + 'placeholder', + 'is_synthetic', + 'is_placeholder', +] as const + +type NodeEntry = [string, Record] + +export interface StoredSourceTermEntry { + readonly eligible: boolean + readonly tokens: readonly string[] + readonly normalizedSourceFile: string | null +} + +interface CachedStoredSourceTermEntry extends StoredSourceTermEntry { + readonly fingerprint: string +} + +const sourceTermCache = new WeakMap>() +const sourceTokenizationCount = new WeakMap() + +function hasOwn(value: object, key: PropertyKey): boolean { + return OWN.call(value, key) +} + +/** Type-tagged, length-delimited snapshot; unlike JSON it preserves absence, undefined, NaN, and -0. */ +function snapshotValue(value: unknown, seen = new Set()): string { + if (value === null) return 'null;' + switch (typeof value) { + case 'undefined': return 'undefined;' + case 'boolean': return value ? 'boolean:1;' : 'boolean:0;' + case 'number': + if (Number.isNaN(value)) return 'number:NaN;' + if (Object.is(value, -0)) return 'number:-0;' + return `number:${String(value)};` + case 'bigint': return `bigint:${String(value)};` + case 'string': return `string:${value.length}:${value}` + case 'symbol': return `symbol:${String(value.description ?? '')};` + case 'function': return `function:${String(value)};` + case 'object': { + if (seen.has(value)) return 'cycle;' + seen.add(value) + if (Array.isArray(value)) { + const parts = value.map((entry, index) => ( + hasOwn(value, index) ? `1:${snapshotValue(entry, seen)}` : '0:' + )) + seen.delete(value) + return `array:${value.length}:[${parts.join('|')}]` + } + const record = value as Record + const keys = Object.keys(record).sort() + const parts = keys.map((key) => `${key.length}:${key}=${snapshotValue(record[key], seen)}`) + seen.delete(value) + return `object:${keys.length}:{${parts.join('|')}}` + } + } + return 'unknown;' +} + +function sourceFingerprint(attributes: Record, normalizationRootContext: string | undefined): string { + const fields = SOURCE_RELEVANT_FIELDS.map((field) => ( + hasOwn(attributes, field) + ? `${field.length}:${field}=own:${snapshotValue(attributes[field])}` + : `${field.length}:${field}=absent;` + )) + return `root=${snapshotValue(normalizationRootContext)}|${fields.join('|')}` +} + +function canonicalSourcePath(rootContext: string | undefined, candidate: unknown): string | null { + if ( + typeof rootContext !== 'string' + || rootContext.length === 0 + || rootContext.includes('\0') + || typeof candidate !== 'string' + || candidate.length === 0 + || candidate.includes('\0') + ) { + return null + } + + const normalizedRoot = resolve(rootContext.replaceAll('\\', '/')) + const absolutePath = resolve(normalizedRoot, candidate.replaceAll('\\', '/')) + const fromRoot = relative(normalizedRoot, absolutePath) + if (fromRoot === '' || fromRoot === '..' || fromRoot.startsWith(`..${sep}`) || isAbsolute(fromRoot)) { + return null + } + return absolutePath.replaceAll('\\', '/') +} + +function parseOrderedSourceRange(value: unknown): { start: number; end: number } | null { + if (typeof value !== 'string') return null + const match = SOURCE_LOCATION.exec(value) + if (!match?.[1]) return null + const start = Number(match[1]) + const end = match[2] === undefined ? start : Number(match[2]) + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start <= 0 || end < start) return null + return { start, end } +} + +function validStoredSnippet(value: unknown, range: { start: number; end: number }): value is string { + if ( + typeof value !== 'string' + || value.length === 0 + || value.includes('\r') + || value !== value.trim() + || value.length > MAX_TRUNCATED_SNIPPET_CHARS + ) { + return false + } + const lines = value.split('\n') + if (lines.length > MAX_STORED_SNIPPET_LINES || lines.length > (range.end - range.start + 1)) return false + if (value.length <= MAX_STORED_SNIPPET_CHARS) return true + return value.endsWith('...') && !/\s/u.test(value.at(-4) ?? '') +} + +function hasFunctionIdentity(attributes: Record): boolean { + if (hasOwn(attributes, 'node_kind')) { + return attributes.node_kind === 'function' || attributes.node_kind === 'method' + } + if (!hasOwn(attributes, 'label') || typeof attributes.label !== 'string') return false + const match = /^(\.)?(.+)\(\)$/.exec(attributes.label) + return match?.[2] !== undefined && IDENTIFIER.test(match[2]) +} + +function hasValidExclusionState(attributes: Record): boolean { + for (const key of ['external_call', 'synthetic', 'placeholder', 'is_synthetic', 'is_placeholder'] as const) { + if (!hasOwn(attributes, key)) continue + if (typeof attributes[key] !== 'boolean' || attributes[key] === true) return false + } + + if (!hasOwn(attributes, 'framework_metadata')) return true + const metadata = attributes.framework_metadata + if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) return false + const record = metadata as Record + if (!hasOwn(record, 'external_call')) return true + return typeof record.external_call === 'boolean' && record.external_call === false +} + +function matchingProvenance( + value: unknown, + expectedCapability: string, + normalizedSourceFile: string, + normalizationRootContext: string | undefined, + rangeStart: number, +): boolean { + if (!Array.isArray(value)) return false + let matched = false + for (const item of value) { + if (!item || typeof item !== 'object' || Array.isArray(item)) continue + const entry = item as Record + const builtinExtractor = typeof entry.capability_id === 'string' && entry.capability_id.startsWith(BUILTIN_EXTRACTOR_PREFIX) + const applicable = entry.stage === 'extract' || builtinExtractor + if (!applicable) continue + const location = parseOrderedSourceRange(entry.source_location) + const sourceFile = canonicalSourcePath(normalizationRootContext, entry.source_file) + if ( + entry.stage !== 'extract' + || entry.capability_id !== expectedCapability + || sourceFile !== normalizedSourceFile + || location?.start !== rangeStart + ) { + return false + } + matched = true + } + return matched +} + +function evaluateStoredSourceTerms( + attributes: Record, + normalizationRootContext: string | undefined, +): Omit & { snippet: string | null } { + if (!hasOwn(attributes, 'file_type') || attributes.file_type !== 'code') return { eligible: false, normalizedSourceFile: null, snippet: null } + if (!hasFunctionIdentity(attributes) || !hasValidExclusionState(attributes)) return { eligible: false, normalizedSourceFile: null, snippet: null } + + const normalizedSourceFile = hasOwn(attributes, 'source_file') + ? canonicalSourcePath(normalizationRootContext, attributes.source_file) + : null + if (!normalizedSourceFile) return { eligible: false, normalizedSourceFile: null, snippet: null } + + if (typeof attributes.label === 'string' && attributes.label.trim().toLowerCase() === basename(normalizedSourceFile).trim().toLowerCase()) { + return { eligible: false, normalizedSourceFile, snippet: null } + } + + const extension = extname(normalizedSourceFile).toLowerCase() + const expectedCapability = extension === '.ts' || extension === '.tsx' + ? 'builtin:extract:typescript' + : extension === '.js' || extension === '.jsx' + ? 'builtin:extract:javascript' + : null + if (!expectedCapability) return { eligible: false, normalizedSourceFile, snippet: null } + + const range = hasOwn(attributes, 'source_location') ? parseOrderedSourceRange(attributes.source_location) : null + if (!range) return { eligible: false, normalizedSourceFile, snippet: null } + if (hasOwn(attributes, 'line_number')) { + const lineNumber = attributes.line_number + if (!Number.isSafeInteger(lineNumber) || typeof lineNumber !== 'number' || lineNumber <= 0 || lineNumber !== range.start) { + return { eligible: false, normalizedSourceFile, snippet: null } + } + } + + if (!hasOwn(attributes, 'snippet') || !validStoredSnippet(attributes.snippet, range)) { + return { eligible: false, normalizedSourceFile, snippet: null } + } + if (!hasOwn(attributes, 'provenance') || !matchingProvenance( + attributes.provenance, + expectedCapability, + normalizedSourceFile, + normalizationRootContext, + range.start, + )) { + return { eligible: false, normalizedSourceFile, snippet: null } + } + + return { eligible: true, normalizedSourceFile, snippet: attributes.snippet } +} + +/** Reconciles all supplied current nodes before retrieval filters and retains no omitted IDs. */ +export function reconcileStoredSourceTerms( + graph: KnowledgeGraph, + normalizationRootContext: string | undefined, + currentEntries: readonly NodeEntry[] = graph.nodeEntries(), +): ReadonlyMap { + const previous = sourceTermCache.get(graph) ?? new Map() + const next = new Map() + let tokenizations = sourceTokenizationCount.get(graph) ?? 0 + + for (const [id, attributes] of currentEntries) { + const fingerprint = sourceFingerprint(attributes, normalizationRootContext) + const cached = previous.get(id) + if (cached?.fingerprint === fingerprint) { + next.set(id, cached) + continue + } + + const evaluated = evaluateStoredSourceTerms(attributes, normalizationRootContext) + const tokens = evaluated.eligible && evaluated.snippet !== null + ? Object.freeze(tokenizeLabel(evaluated.snippet)) + : EMPTY_TOKENS + if (evaluated.eligible) tokenizations += 1 + next.set(id, Object.freeze({ + fingerprint, + eligible: evaluated.eligible, + tokens, + normalizedSourceFile: evaluated.normalizedSourceFile, + })) + } + + sourceTermCache.set(graph, next) + sourceTokenizationCount.set(graph, tokenizations) + return next +} + +/** Internal finite-control surface; intentionally not exported from a package barrel. */ +export function storedSourceTermCacheInspection(graph: KnowledgeGraph): { + entryCount: number + tokenizationCount: number + tokenArrays: ReadonlyMap +} { + const entries = sourceTermCache.get(graph) ?? new Map() + return { + entryCount: entries.size, + tokenizationCount: sourceTokenizationCount.get(graph) ?? 0, + tokenArrays: new Map([...entries].map(([id, entry]) => [id, entry.tokens])), + } +} diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index cf74b390..1c127601 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -59,6 +59,7 @@ import { relationIsPrimaryForPolicy, } from './retrieve/expansion.js' import { sliceCandidatesForRetrieve } from './retrieve/slicing.js' +import { reconcileStoredSourceTerms } from './retrieve-source-terms.js' import { CONCEPTUAL_WORKFLOW_RESERVATION_BOOST, finalizeConceptualFallbackPlan, @@ -1581,6 +1582,7 @@ interface SeedScoreBreakdown { labelExactScore: number labelPhraseScore: number labelTokenScore: number + sourceTokenScore: number sourcePathScore: number promptIdentifierScore: number communityScore: number @@ -2311,6 +2313,7 @@ function scoreSeedCandidate( labelExactScore, labelPhraseScore, labelTokenScore, + sourceTokenScore: 0, sourcePathScore, promptIdentifierScore, communityScore, @@ -4992,6 +4995,7 @@ function retrieveContextPass( ? graph.graph.root_path : undefined const classificationRootPath = inferredGraphRoot(graph) + const storedSourceTerms = reconcileStoredSourceTerms(graph, classificationRootPath) const retrievalGate = queryStage.retrieval_gate const effectiveRetrievalLevel = queryStage.effective_retrieval_level const underScopedDivergenceIds = underScopedDivergenceNodeIds(graph, question) @@ -5072,6 +5076,10 @@ function retrieveContextPass( const excludedDomains = retrievalGate.signals.excluded_domains ?? [] const excludedTerms = retrievalGate.signals.excluded_terms ?? [] const excludedPathHints = retrievalGate.signals.excluded_path_hints ?? [] + const excludedSourceTokens = new Set( + [...excludedTerms, ...excludedPathHints].flatMap((term) => tokenizeLabel(term)), + ) + const sourceQuestionTokens = [...new Set(questionTokens)].filter((token) => !excludedSourceTokens.has(token)) // Step 1+2: Score all nodes with explicit seed evidence weights. const tokenWeights = tokenWeightsForQuestion(graph, questionTokens) @@ -5164,11 +5172,18 @@ function retrieveContextPass( allowRuntimeBoundaryBoost: effectiveScore.total + anchorScore > 0 || explicitlyAnchored, }, ) + const sourceTokens = storedSourceTerms.get(id)?.tokens ?? [] + const sourceTokenScore = 0.5 * Math.min(2, scoreNode( + sourceQuestionTokens, + sourceTokens, + undefined, + Math.max(sourceTokens.length, 1), + )) const domainIntentPenalty = runtimeGenerationSourceDomainPenalty(retrievalGate, sourceDomain, explicitlyAnchored) const scriptMigrationPenalty = scriptMigrationPathPenalty(retrievalGate, sourceFile, label, question, explicitlyAnchored) const conceptualFallbackScore = conceptualNodeBoosts.get(id) ?? 0 - const totalSeedScore = effectiveScore.total + anchorScore + metadataBoost + domainAdjustment + conceptualFallbackScore + const totalSeedScore = effectiveScore.total + anchorScore + metadataBoost + domainAdjustment + conceptualFallbackScore + sourceTokenScore - sourceDomainPenalty - domainIntentPenalty - scriptMigrationPenalty const hasPositiveSeedEvidence = totalSeedScore > 0 || exactAnchorMatch || mentionedPathMatch if (hasPositiveSeedEvidence) { @@ -5194,6 +5209,7 @@ function retrieveContextPass( seedScore: { ...effectiveScore, labelExactScore: effectiveScore.labelExactScore + anchorScore, + sourceTokenScore, conceptualFallbackScore, total: totalSeedScore, }, @@ -5234,12 +5250,13 @@ function retrieveContextPass( rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.labelExactScore), rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.labelPhraseScore), rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.labelTokenScore), + rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.sourceTokenScore), rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.promptIdentifierScore), rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.sourcePathScore), rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.communityScore), rankedSeedCandidateIds(graph, filteredSeedCandidates, (candidate) => candidate.seedScore.conceptualFallbackScore), ], { - weights: [2, 1.5, 1.5, 0.5, 0.25, 0.25, 1.75], + weights: [2, 1.5, 1.5, 0.75, 0.5, 0.25, 0.25, 1.75], }) const scored: ScoredNode[] = filteredSeedCandidates.map((candidate) => ({ id: candidate.id, diff --git a/tests/unit/retrieve-source-discovery.test.ts b/tests/unit/retrieve-source-discovery.test.ts new file mode 100644 index 00000000..c52166c1 --- /dev/null +++ b/tests/unit/retrieve-source-discovery.test.ts @@ -0,0 +1,270 @@ +import { createHash } from 'node:crypto' + +import { describe, expect, it } from 'vitest' + +import { KnowledgeGraph } from '../../src/contracts/graph.js' +import { + compactRetrieveResult, + contextPackFromRetrieveResult, + retrieveContext, +} from '../../src/runtime/retrieve.js' + +type Attributes = Record +const ROOT = '/workspace' + +function graph(): KnowledgeGraph { + const value = new KnowledgeGraph({ directed: true }) + value.graph.root_path = ROOT + return value +} + +function provenance(sourceFile: string, sourceLocation = 'L7', capabilityId = 'builtin:extract:typescript'): Attributes[] { + return [{ capability_id: capabilityId, stage: 'extract', source_file: sourceFile, source_location: sourceLocation }] +} + +function sourceOwner(overrides: Attributes = {}): Attributes { + const sourceFile = typeof overrides.source_file === 'string' ? overrides.source_file : `${ROOT}/src/neutral.ts` + return { + label: '.operate()', + file_type: 'code', + source_file: sourceFile, + source_location: 'L7-L9', + node_kind: 'method', + snippet: 'export function neutral() {\n return ledger.parcel_checksum\n}', + provenance: provenance(sourceFile), + community: 0, + ...overrides, + } +} + +function addOwner(target: KnowledgeGraph, id: string, overrides: Attributes = {}): void { + target.addNode(id, sourceOwner(overrides)) +} + +function retrieveIds(target: KnowledgeGraph, question: string, options: Attributes = {}): string[] { + return retrieveContext(target, { + question, + budget: 1_200, + retrievalLevel: 2, + ...options, + }).matched_nodes.map((node) => node.node_id ?? '') +} + +const digest = (value: unknown): string => createHash('sha256').update(JSON.stringify(value)).digest('hex') + +describe('stored-source candidate discovery', () => { + it('admits a neutral body-only camelCase/snake_case member owner and expands its directed call', () => { + const target = graph() + addOwner(target, 'owner') + target.addNode('callee', { + label: '.forward()', file_type: 'code', source_file: `${ROOT}/src/downstream.ts`, source_location: 'L20', node_kind: 'method', community: 0, + }) + target.addEdge('owner', 'callee', { relation: 'calls' }) + const ids = retrieveIds(target, 'Explain parcelChecksum') + expect(ids).toContain('owner') + expect(ids).toContain('callee') + }) + + it('discovers the body owner in label/path competition instead of merely avoiding an empty result', () => { + const target = graph() + addOwner(target, 'body_owner', { label: '.transmit()', source_file: `${ROOT}/src/channel.ts`, provenance: provenance(`${ROOT}/src/channel.ts`) }) + target.addNode('label_distractor', { + label: 'ParcelChecksumCatalog', file_type: 'code', source_file: `${ROOT}/src/parcel-checksum-catalog.ts`, source_location: 'L30', node_kind: 'class', community: 0, + }) + const ids = retrieveIds(target, 'Explain parcelChecksum') + expect(ids).toContain('label_distractor') + expect(ids).toContain('body_owner') + }) + + it('uses an independent source ranking to order admitted body-only candidates', () => { + const target = graph() + addOwner(target, 'high_source', { + label: 'Zulu()', source_file: `${ROOT}/src/zulu.ts`, snippet: 'function zulu() {\n return orbit + detail\n}', provenance: provenance(`${ROOT}/src/zulu.ts`), + }) + addOwner(target, 'low_source', { + label: 'Alpha()', source_file: `${ROOT}/src/alpha.ts`, snippet: 'function alpha() {\n return orbit\n}', provenance: provenance(`${ROOT}/src/alpha.ts`), + }) + const ids = retrieveIds(target, 'Explain orbit detail') + expect(ids).toEqual(expect.arrayContaining(['high_source', 'low_source'])) + expect(ids.indexOf('high_source')).toBeLessThan(ids.indexOf('low_source')) + }) + + it('deduplicates repeated terms and keeps source scoring capped and deterministic', () => { + const target = graph() + addOwner(target, 'owner', { snippet: 'function neutral() {\n return parcel_checksum + parcelChecksum + parcel_checksum\n}' }) + const once = retrieveContext(target, { question: 'parcelChecksum', budget: 1_200, retrievalLevel: 1 }) + const repeated = retrieveContext(target, { question: 'parcelChecksum parcelChecksum parcelChecksum', budget: 1_200, retrievalLevel: 1 }) + expect(once.matched_nodes.find((node) => node.node_id === 'owner')?.match_score).toBe( + repeated.matched_nodes.find((node) => node.node_id === 'owner')?.match_score, + ) + expect(retrieveIds(target, 'parcelChecksum')).toEqual(retrieveIds(target, 'parcelChecksum')) + }) + + it('keeps ambiguous equally supported owners as related evidence without invented certainty', () => { + const target = graph() + addOwner(target, 'left', { label: 'Left()', source_file: `${ROOT}/src/left.ts`, provenance: provenance(`${ROOT}/src/left.ts`) }) + addOwner(target, 'right', { label: 'Right()', source_file: `${ROOT}/src/right.ts`, provenance: provenance(`${ROOT}/src/right.ts`) }) + const result = retrieveContext(target, { question: 'parcelChecksum', budget: 1_200, retrievalLevel: 1 }) + const owners = result.matched_nodes.filter((node) => node.node_id === 'left' || node.node_id === 'right') + expect(owners).toHaveLength(2) + expect(owners.map((node) => node.relevance_band)).toEqual(['related', 'related']) + expect(owners.every((node) => node.match_score > 0)).toBe(true) + }) + + it('keeps levels, exclusions, community and file-type filters effective', () => { + const target = graph() + addOwner(target, 'owner') + target.addNode('callee', { + label: '.forward()', file_type: 'code', source_file: `${ROOT}/src/downstream.ts`, source_location: 'L20', node_kind: 'method', community: 0, + }) + target.addEdge('owner', 'callee', { relation: 'calls' }) + expect(retrieveIds(target, 'parcelChecksum', { retrievalLevel: 0 })).toEqual([]) + expect(retrieveIds(target, 'parcelChecksum', { retrievalLevel: 1 })).toContain('owner') + expect(retrieveIds(target, 'parcelChecksum', { retrievalLevel: 1 })).not.toContain('callee') + expect(retrieveIds(target, 'parcelChecksum', { retrievalLevel: 2 })).toContain('callee') + expect(retrieveIds(target, 'parcelChecksum', { community: 9 })).toEqual([]) + expect(retrieveIds(target, 'parcelChecksum', { fileType: 'document' })).toEqual([]) + expect(retrieveIds(target, 'Explain parcelChecksum without parcelChecksum')).toEqual([]) + }) + + it('retains source-domain pollution penalties and only allows an intentionally requested domain', () => { + const target = graph() + addOwner(target, 'test_owner', { source_file: `${ROOT}/tests/neutral.test.ts`, provenance: provenance(`${ROOT}/tests/neutral.test.ts`) }) + expect(retrieveIds(target, 'Explain parcelChecksum production behavior')).not.toContain('test_owner') + expect(retrieveIds(target, 'Explain parcelChecksum tests')).toContain('test_owner') + }) + + it('does not let a body hit unlock a metadata-only runtime-boundary boost or anchor flags', () => { + const target = graph() + addOwner(target, 'owner', { snippet: 'function neutral() {\n return polar_cipher\n}', framework_metadata: { runtime_boundary: 'server' } }) + const result = retrieveContext(target, { question: 'Explain the next server polarCipher behavior', budget: 1_200, retrievalLevel: 1 }) + const owner = result.matched_nodes.find((node) => node.node_id === 'owner') + expect(owner).toEqual(expect.objectContaining({ framework_boost: 0, relevance_band: 'related' })) + expect(owner).not.toHaveProperty('source_token_score') + }) + + it('preserves exact raw, compact, and compiled bytes without eligible snippets', () => { + const target = graph() + target.addNode('first', { label: 'CopperRelay', file_type: 'code', source_file: `${ROOT}/src/copper-relay.ts`, source_location: 'L7-L9', node_kind: 'function', community: 0 }) + target.addNode('second', { label: 'RelayLedger', file_type: 'code', source_file: `${ROOT}/src/relay-ledger.ts`, source_location: 'L20-L21', node_kind: 'method', community: 0 }) + target.addEdge('first', 'second', { relation: 'calls' }) + const raw = retrieveContext(target, { question: 'Explain CopperRelay', budget: 800, retrievalLevel: 2 }) + expect(digest(raw)).toBe('6b8789709c070b669c156ba9cc25930abfecdb606dbe1369092b782f88b36d3d') + expect(digest(compactRetrieveResult(raw))).toBe('180d8692454d70748d34568fb6b32814b32a7113f3d3d17243b7e5a5258862ac') + expect(digest(contextPackFromRetrieveResult(raw))).toBe('69ed3fb7ee9faadd7ba507e1b8b136760cd5d6211d265d254719256edef91f39') + }) + + it.each([ + ['absent kind TypeScript', { node_kind: undefined }, true], + ['explicit function', { node_kind: 'function' }, true], + ['explicit method', { node_kind: 'method', label: '.operate()' }, true], + ['JavaScript extractor', { source_file: `${ROOT}/src/neutral.js`, provenance: provenance(`${ROOT}/src/neutral.js`, 'L7', 'builtin:extract:javascript') }, true], + ['JSX extractor', { source_file: `${ROOT}/src/neutral.JSX`, provenance: provenance(`${ROOT}/src/neutral.JSX`, 'L7', 'builtin:extract:javascript') }, true], + ['Unicode absent-kind function', { node_kind: undefined, label: 'προβολή()' }, true], + ['dollar absent-kind function', { node_kind: undefined, label: '$relay()' }, true], + ['underscore absent-kind method', { node_kind: undefined, label: '._relay()' }, true], + ['present undefined kind', { node_kind: undefined, label: 'operate()' }, false, true], + ['null kind', { node_kind: null, label: 'operate()' }, false], + ['empty kind', { node_kind: '', label: 'operate()' }, false], + ['unknown kind', { node_kind: 'unknown', label: 'operate()' }, false], + ['wrong typed kind', { node_kind: 1, label: 'operate()' }, false], + ['class kind', { node_kind: 'class' }, false], + ['type kind', { node_kind: 'type' }, false], + ['constant kind', { node_kind: 'constant' }, false], + ['qualified inferred label', { node_kind: undefined, label: 'Box.operate()' }, false], + ['escaped inferred label', { node_kind: undefined, label: '\\u006fperate()' }, false], + ['spaced inferred label', { node_kind: undefined, label: 'operate ()' }, false], + ['generic inferred label', { node_kind: undefined, label: 'operate()' }, false], + ['extra-dot inferred label', { node_kind: undefined, label: '..operate()' }, false], + ])('enforces declaration identity: %s', (_name, overrides, expected, forceOwnUndefined = false) => { + const target = graph() + const rawOverrides = overrides as Attributes + const attributes = sourceOwner(rawOverrides) + if (!forceOwnUndefined && Object.prototype.hasOwnProperty.call(rawOverrides, 'node_kind') && rawOverrides.node_kind === undefined) delete attributes.node_kind + target.addNode('candidate', attributes) + expect(retrieveIds(target, 'parcelChecksum').includes('candidate')).toBe(expected) + }) + + it.each([ + ['single L7', { source_location: 'L7', snippet: 'parcelChecksum' }, true], + ['single L7-L7', { source_location: 'L7-L7', snippet: 'parcelChecksum' }, true], + ['last safe line', { source_location: 'L9007199254740991', snippet: 'parcelChecksum', provenance: provenance(`${ROOT}/src/neutral.ts`, 'L9007199254740991') }, true], + ['partial', { source_location: 'L7-' }, false], + ['reversed', { source_location: 'L9-L7' }, false], + ['zero', { source_location: 'L0' }, false], + ['unsafe', { source_location: 'L9007199254740992', provenance: provenance(`${ROOT}/src/neutral.ts`, 'L9007199254740992') }, false], + ['contradictory line number', { line_number: 8 }, false], + ['multiline single range', { source_location: 'L7', snippet: 'parcelChecksum\nreturn true' }, false], + ['snippet exceeds span', { source_location: 'L7-L8', snippet: 'parcelChecksum\nreturn true\nend' }, false], + ['CRLF', { snippet: 'parcelChecksum\r\nreturn true' }, false], + ['outer whitespace', { snippet: ' parcelChecksum' }, false], + ['26 lines', { source_location: 'L7-L40', snippet: Array.from({ length: 26 }, () => 'parcelChecksum').join('\n') }, false], + ['2000 chars', { source_location: 'L7', snippet: `parcelChecksum${'x'.repeat(1_986)}` }, true], + ['2003 canonical chars', { source_location: 'L7', snippet: `parcelChecksum${'x'.repeat(1_986)}...` }, true], + ['2001 no suffix', { source_location: 'L7', snippet: `parcelChecksum${'x'.repeat(1_987)}` }, false], + ['whitespace before suffix', { source_location: 'L7', snippet: `parcelChecksum${'x'.repeat(1_983)} ...` }, false], + ])('enforces ranges and literal snippet shape: %s', (_name, overrides, expected) => { + const target = graph() + target.addNode('candidate', sourceOwner(overrides as Attributes)) + expect(retrieveIds(target, 'parcelChecksum').includes('candidate')).toBe(expected) + }) + + it.each([ + ['missing source path', { source_file: undefined }, false], + ['foreign path', { source_file: '/foreign/src/neutral.ts', provenance: provenance('/foreign/src/neutral.ts') }, false], + ['escaping path', { source_file: '../foreign.ts', provenance: provenance('../foreign.ts') }, false], + ['NUL path', { source_file: `${ROOT}/src/bad\0.ts`, provenance: provenance(`${ROOT}/src/bad\0.ts`) }, false], + ['wrong extractor', { provenance: provenance(`${ROOT}/src/neutral.ts`, 'L7', 'builtin:extract:javascript') }, false], + ['wrong stage', { provenance: [{ capability_id: 'builtin:extract:typescript', stage: 'normalize', source_file: `${ROOT}/src/neutral.ts`, source_location: 'L7' }] }, false], + ['wrong provenance path', { provenance: provenance(`${ROOT}/src/other.ts`) }, false], + ['wrong provenance start', { provenance: provenance(`${ROOT}/src/neutral.ts`, 'L8') }, false], + ['matching plus unrelated', { provenance: [...provenance(`${ROOT}/src/neutral.ts`), { capability_id: 'custom:index', stage: 'index' }] }, true], + ['matching plus conflicting applicable', { provenance: [...provenance(`${ROOT}/src/neutral.ts`), ...provenance(`${ROOT}/src/other.ts`)] }, false], + ['matching plus malformed applicable', { provenance: [...provenance(`${ROOT}/src/neutral.ts`), { stage: 'extract' }] }, false], + ['flat external call', { external_call: true }, false], + ['nested external call', { framework_metadata: { external_call: true } }, false], + ['malformed flat exclusion', { external_call: 'yes' }, false], + ['malformed nested exclusion', { framework_metadata: { external_call: 'yes' } }, false], + ['file-like owner', { label: 'neutral.ts' }, false], + ['placeholder owner', { placeholder: true }, false], + ['synthetic owner', { synthetic: true }, false], + ['explicit placeholder owner', { is_placeholder: true }, false], + ['explicit synthetic owner', { is_synthetic: true }, false], + ['non-code owner', { file_type: 'document' }, false], + ])('enforces source/provenance/exclusion identity: %s', (_name, overrides, expected) => { + const target = graph() + target.addNode('candidate', sourceOwner(overrides as Attributes)) + expect(retrieveIds(target, 'parcelChecksum').includes('candidate')).toBe(expected) + }) + + it('invalidates same-graph source, identity, nested metadata, filtered nodes, and root mutations', () => { + const target = graph() + const metadata: Attributes = { external_call: false } + const original = sourceOwner({ framework_metadata: metadata }) + target.addNode('candidate', original) + expect(retrieveIds(target, 'parcelChecksum')).toContain('candidate') + target.addNode('candidate', { ...original, snippet: 'return revisedToken' }) + expect(retrieveIds(target, 'parcelChecksum')).not.toContain('candidate') + expect(retrieveIds(target, 'revisedToken')).toContain('candidate') + const deletedSnippet = { ...original } + delete deletedSnippet.snippet + target.addNode('candidate', deletedSnippet) + expect(retrieveIds(target, 'parcelChecksum')).not.toContain('candidate') + target.addNode('candidate', original) + metadata.external_call = true + expect(retrieveIds(target, 'parcelChecksum')).not.toContain('candidate') + metadata.external_call = false + const absentKindWithBadLabel: Attributes = { ...original, label: 'not canonical' } + delete absentKindWithBadLabel.node_kind + target.addNode('candidate', absentKindWithBadLabel) + expect(retrieveIds(target, 'parcelChecksum')).not.toContain('candidate') + target.addNode('candidate', original) + expect(retrieveIds(target, 'parcelChecksum', { community: 9 })).toEqual([]) + target.addNode('candidate', { ...original, provenance: provenance(`${ROOT}/src/other.ts`) }) + expect(retrieveIds(target, 'parcelChecksum', { community: 9 })).toEqual([]) + expect(retrieveIds(target, 'parcelChecksum')).not.toContain('candidate') + target.addNode('candidate', original) + target.graph.root_path = '/different-root' + expect(retrieveIds(target, 'parcelChecksum')).not.toContain('candidate') + }) +}) diff --git a/tests/unit/retrieve-source-terms.test.ts b/tests/unit/retrieve-source-terms.test.ts new file mode 100644 index 00000000..bda6ef2f --- /dev/null +++ b/tests/unit/retrieve-source-terms.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from 'vitest' + +import { KnowledgeGraph } from '../../src/contracts/graph.js' +import { + reconcileStoredSourceTerms, + storedSourceTermCacheInspection, +} from '../../src/runtime/retrieve-source-terms.js' +import { retrieveContext, scoreNode, tokenizeLabel, tokenizeQuestion } from '../../src/runtime/retrieve.js' + +type Attributes = Record +const ROOT = '/workspace' + +function graph(): KnowledgeGraph { + const value = new KnowledgeGraph({ directed: true }) + value.graph.root_path = ROOT + return value +} + +function owner(sourceFile: string, overrides: Attributes = {}): Attributes { + return { + label: '.operate()', + file_type: 'code', + source_file: sourceFile, + source_location: 'L7-L9', + node_kind: 'method', + snippet: 'function neutral() {\n return parcel_checksum\n}', + provenance: [{ + capability_id: 'builtin:extract:typescript', + stage: 'extract', + source_file: sourceFile, + source_location: 'L7', + }], + community: 0, + ...overrides, + } +} + +describe('stored-source term cache controls', () => { + it('reuses unchanged token arrays and retokenizes only the changed tuple', () => { + const target = graph() + target.addNode('one', owner(`${ROOT}/src/one.ts`)) + target.addNode('two', owner(`${ROOT}/src/two.ts`, { snippet: 'function neutral() {\n return amber_signal\n}' })) + + const cold = reconcileStoredSourceTerms(target, ROOT) + const oneCold = cold.get('one')?.tokens + const twoCold = cold.get('two')?.tokens + const warm = reconcileStoredSourceTerms(target, ROOT) + expect(warm.get('one')?.tokens).toBe(oneCold) + expect(warm.get('two')?.tokens).toBe(twoCold) + expect(storedSourceTermCacheInspection(target).tokenizationCount).toBe(2) + + target.addNode('one', owner(`${ROOT}/src/one.ts`, { snippet: 'function neutral() {\n return revised_signal\n}' })) + const revised = reconcileStoredSourceTerms(target, ROOT) + expect(revised.get('one')?.tokens).not.toBe(oneCold) + expect(revised.get('one')?.tokens).toEqual(expect.arrayContaining(['revised', 'signal'])) + expect(revised.get('two')?.tokens).toBe(twoCold) + expect(storedSourceTermCacheInspection(target).tokenizationCount).toBe(3) + }) + + it('detects nested object edits even when the containing reference is reused', () => { + const target = graph() + const metadata: Attributes = { external_call: false } + target.addNode('one', owner(`${ROOT}/src/one.ts`, { framework_metadata: metadata })) + const before = reconcileStoredSourceTerms(target, ROOT).get('one') + expect(before?.eligible).toBe(true) + + metadata.external_call = true + const after = reconcileStoredSourceTerms(target, ROOT).get('one') + expect(after?.eligible).toBe(false) + expect(after).not.toBe(before) + }) + + it('takes add/replace/removal views and bounds retained entries to current cardinality', () => { + const target = graph() + target.addNode('one', owner(`${ROOT}/src/one.ts`)) + reconcileStoredSourceTerms(target, ROOT) + expect(storedSourceTermCacheInspection(target).entryCount).toBe(1) + + target.addNode('two', owner(`${ROOT}/src/two.ts`)) + reconcileStoredSourceTerms(target, ROOT) + expect(storedSourceTermCacheInspection(target).entryCount).toBe(2) + + const controlledCurrentView = target.nodeEntries().filter(([id]) => id !== 'one') + reconcileStoredSourceTerms(target, ROOT, controlledCurrentView) + expect(storedSourceTermCacheInspection(target).entryCount).toBe(controlledCurrentView.length) + expect(storedSourceTermCacheInspection(target).tokenArrays.has('one')).toBe(false) + + target.addNode('two', owner(`${ROOT}/src/two.ts`, { snippet: 'function neutral() {\n return replacement_value\n}' })) + reconcileStoredSourceTerms(target, ROOT, target.nodeEntries().filter(([id]) => id === 'two')) + expect(storedSourceTermCacheInspection(target).entryCount).toBe(1) + expect(storedSourceTermCacheInspection(target).tokenArrays.get('two')).toEqual(expect.arrayContaining(['replacement', 'value'])) + }) + + it('reconciles early-return and filtered-out nodes before later queries', () => { + const target = graph() + target.addNode('one', owner(`${ROOT}/src/one.ts`)) + retrieveContext(target, { question: 'parcelChecksum', budget: 500, retrievalLevel: 1 }) + const initialTokens = storedSourceTermCacheInspection(target).tokenArrays.get('one') + + target.addNode('one', owner(`${ROOT}/src/one.ts`, { snippet: 'function neutral() {\n return early_return_value\n}' })) + retrieveContext(target, { question: 'hello', budget: 500, retrievalLevel: 0 }) + const earlyTokens = storedSourceTermCacheInspection(target).tokenArrays.get('one') + expect(earlyTokens).not.toBe(initialTokens) + expect(earlyTokens).toEqual(expect.arrayContaining(['early', 'return', 'value'])) + + target.addNode('one', owner(`${ROOT}/src/one.ts`, { + provenance: [{ capability_id: 'builtin:extract:typescript', stage: 'extract', source_file: `${ROOT}/src/foreign.ts`, source_location: 'L7' }], + })) + retrieveContext(target, { question: 'parcelChecksum', budget: 500, retrievalLevel: 1, community: 99 }) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(false) + expect(retrieveContext(target, { question: 'parcelChecksum', budget: 500, retrievalLevel: 1 }).matched_nodes).toEqual([]) + }) + + it('invalidates path, range, kind presence, label, provenance, and root context independently', () => { + const target = graph() + const original = owner(`${ROOT}/src/one.ts`) + target.addNode('one', original) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(true) + + target.addNode('one', { ...original, source_location: 'L9-L7' }) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(false) + target.addNode('one', { ...original, source_file: `${ROOT}/src/other.ts` }) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(false) + target.addNode('one', { ...original, node_kind: 'class' }) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(false) + + const absentKind: Attributes = { ...original, label: 'operate()' } + delete absentKind.node_kind + target.addNode('one', absentKind) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(true) + target.addNode('one', { ...absentKind, label: 'Owner.operate()' }) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(false) + + target.addNode('one', original) + expect(reconcileStoredSourceTerms(target, '/different-root').get('one')?.eligible).toBe(false) + expect(reconcileStoredSourceTerms(target, ROOT).get('one')?.eligible).toBe(true) + }) + + it('owns independent state per graph and keeps cold/warm retrieval bytes equal', () => { + const first = graph() + const second = graph() + first.addNode('same', owner(`${ROOT}/src/same.ts`)) + second.addNode('same', owner(`${ROOT}/src/same.ts`, { snippet: 'function neutral() {\n return second_graph\n}' })) + reconcileStoredSourceTerms(first, ROOT) + reconcileStoredSourceTerms(second, ROOT) + expect(storedSourceTermCacheInspection(first).tokenArrays.get('same')).not.toEqual( + storedSourceTermCacheInspection(second).tokenArrays.get('same'), + ) + + const options = { question: 'parcelChecksum', budget: 800, retrievalLevel: 1 as const } + const cold = retrieveContext(first, options) + const warm = retrieveContext(first, options) + expect(JSON.stringify(warm)).toBe(JSON.stringify(cold)) + expect(Object.keys(warm.matched_nodes[0] ?? {})).not.toContain('sourceTokenScore') + }) + + it('fixes the unit-weight source formula with deduplicated normalized question terms and a cap of one', () => { + const sourceTokens = tokenizeLabel('orbit orbit detail detail member_token') + const questionTokens = [...new Set(tokenizeQuestion('orbit detail memberToken orbit detail'))] + const score = 0.5 * Math.min(2, scoreNode(questionTokens, sourceTokens, undefined, Math.max(sourceTokens.length, 1))) + expect(score).toBe(1) + + const single = 0.5 * Math.min(2, scoreNode(['orbit'], sourceTokens, undefined, sourceTokens.length)) + expect(single).toBeGreaterThan(0) + expect(single).toBeLessThanOrEqual(1) + }) +}) From 80e118f050cf5022a0b53a0f6a675bafa6b24a5d Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 01:52:23 +0400 Subject: [PATCH 17/27] fix: reserve complete selected evidence before literal expansion --- src/runtime/retrieve.ts | 206 ++++++++++-------- .../unit/retrieve-owner-declarations.test.ts | 130 +++++++++++ 2 files changed, 250 insertions(+), 86 deletions(-) diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 83be4783..cf78e1d0 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -1280,6 +1280,14 @@ interface RenderedQueryEvidenceLine { representedSource: readonly RepresentedQueryEvidenceSource[] } +interface QueryEvidenceLineRenderCandidate { + source: QueryEvidenceLine + projection: ReturnType + content: string + lineCapExceeded: boolean + faithful: string +} + function completedQueryEvidenceSourceProjection( sourceFilePath: string, sourceLines: readonly string[], @@ -1314,100 +1322,69 @@ function completedQueryEvidenceSourceProjection( return { text, literalLineBreaks, hasProtectedTokens } } -function renderQueryEvidenceLineEntries( +function queryEvidenceLineRenderCandidate( + candidate: QueryEvidenceLine, + sourceFile: string, + sourceLines: readonly string[], + completed: boolean, +): QueryEvidenceLineRenderCandidate { + const normalized = candidate.text.replace(/\s+/g, ' ').trim() + const projection = completed + ? completedQueryEvidenceSourceProjection(sourceFile, sourceLines, candidate.representedSource) + : queryEvidenceSourceProjection({ + sourceFilePath: sourceFile, + sourceLines, + representedSource: candidate.representedSource, + shapedText: candidate.text, + }) + let projectedContent = projection?.text ?? normalized + if (projection) { + for (let index = projection.literalLineBreaks.length - 1; index >= 0; index -= 1) { + const lineBreak = projection.literalLineBreaks[index]! + projectedContent = [ + projectedContent.slice(0, lineBreak.offset + 1), + `L${lineBreak.lineNumber}: `, + projectedContent.slice(lineBreak.offset + 1), + ].join('') + } + } + const projectedRows = projectedContent.split('\n') + const lineCapExceeded = projectedRows.some((row, index) => { + const content = index === 0 ? row : row.replace(/^L\d+: /, '') + return content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + }) + const content = lineCapExceeded + ? projectedRows.map((row, index) => { + const match = index > 0 ? /^(L\d+: )(.*)$/.exec(row) : null + const rowPrefix = match?.[1] ?? '' + const rowContent = match?.[2] ?? row + return rowContent.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP + ? `${rowPrefix}${rowContent.slice(0, QUERY_EVIDENCE_SNIPPET_LINE_CAP - 3)}...` + : row + }).join('\n') + : projectedContent + return { + source: candidate, + projection, + content, + lineCapExceeded, + faithful: `L${candidate.index}: ${content}`, + } +} + +function renderUnexpandedQueryEvidenceLineEntries( lines: readonly QueryEvidenceLine[], sourceFile: string, sourceLines: readonly string[], - ownerRange?: { start: number; end: number } | null, ): RenderedQueryEvidenceLine[] { const rendered: RenderedQueryEvidenceLine[] = [] let usedChars = 0 - let usedRows = 0 for (const line of lines) { const separatorChars = rendered.length > 0 ? 1 : 0 const remaining = QUERY_EVIDENCE_SNIPPET_CHAR_CAP - usedChars - separatorChars - const completedSource = ownerRange - ? completeQueryEvidenceLiteralStatement({ - sourceFilePath: sourceFile, - sourceLines, - ownerRange, - representedSource: line.representedSource, - }) - : null - const completedLine: QueryEvidenceLine | null = completedSource - ? { - ...line, - index: completedSource[0]!.startLine, - endIndex: completedSource.at(-1)!.endLine, - text: completedSource.map((source) => source.text).join(' '), - representedSource: completedSource, - } - : null - let selected: { - source: QueryEvidenceLine - projection: ReturnType - content: string - lineCapExceeded: boolean - faithful: string - } | null = null - - for (const candidate of completedLine ? [completedLine, line] : [line]) { - const normalized = candidate.text.replace(/\s+/g, ' ').trim() - const projection = candidate === completedLine - ? completedQueryEvidenceSourceProjection(sourceFile, sourceLines, candidate.representedSource) - : queryEvidenceSourceProjection({ - sourceFilePath: sourceFile, - sourceLines, - representedSource: candidate.representedSource, - shapedText: candidate.text, - }) - let projectedContent = projection?.text ?? normalized - if (projection) { - for (let index = projection.literalLineBreaks.length - 1; index >= 0; index -= 1) { - const lineBreak = projection.literalLineBreaks[index]! - projectedContent = [ - projectedContent.slice(0, lineBreak.offset + 1), - `L${lineBreak.lineNumber}: `, - projectedContent.slice(lineBreak.offset + 1), - ].join('') - } - } - const projectedRows = projectedContent.split('\n') - const lineCapExceeded = projectedRows.some((row, index) => { - const content = index === 0 ? row : row.replace(/^L\d+: /, '') - return content.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP - }) - const completedSourceWouldBeAltered = candidate === completedLine && ( - projection === null - || projection.hasProtectedTokens === false - || lineCapExceeded - || usedRows + projectedRows.length > QUERY_EVIDENCE_SNIPPET_MAX_LINES - || `L${candidate.index}: ${projectedContent}`.length > remaining - ) - if (completedSourceWouldBeAltered) { - continue - } - const content = lineCapExceeded - ? projectedRows.map((row, index) => { - const match = index > 0 ? /^(L\d+: )(.*)$/.exec(row) : null - const rowPrefix = match?.[1] ?? '' - const rowContent = match?.[2] ?? row - return rowContent.length > QUERY_EVIDENCE_SNIPPET_LINE_CAP - ? `${rowPrefix}${rowContent.slice(0, QUERY_EVIDENCE_SNIPPET_LINE_CAP - 3)}...` - : row - }).join('\n') - : projectedContent - const prefix = `L${candidate.index}: ` - const faithful = `${prefix}${content}` - if ( - remaining <= prefix.length + 8 - ) { - continue - } - selected = { source: candidate, projection, content, lineCapExceeded, faithful } - break - } - if (!selected) { + const selected = queryEvidenceLineRenderCandidate(line, sourceFile, sourceLines, false) + const prefix = `L${line.index}: ` + if (remaining <= prefix.length + 8) { continue } const bounded = selected.faithful.slice(0, remaining).trimEnd() @@ -1419,11 +1396,68 @@ function renderQueryEvidenceLineEntries( : [], }) usedChars += bounded.length + separatorChars - usedRows += bounded.split('\n').length } return rendered } +function renderQueryEvidenceLineEntries( + lines: readonly QueryEvidenceLine[], + sourceFile: string, + sourceLines: readonly string[], + ownerRange?: { start: number; end: number } | null, +): RenderedQueryEvidenceLine[] { + const baseline = renderUnexpandedQueryEvidenceLineEntries(lines, sourceFile, sourceLines) + if (!ownerRange || baseline.some((line) => line.representedSource.length === 0)) { + return baseline + } + + let reserved = baseline + for (const [baselineIndex, baselineLine] of baseline.entries()) { + const completedSource = completeQueryEvidenceLiteralStatement({ + sourceFilePath: sourceFile, + sourceLines, + ownerRange, + representedSource: baselineLine.representedSource, + }) + if (!completedSource) { + continue + } + const completedLine: QueryEvidenceLine = { + ...baselineLine.source, + index: completedSource[0]!.startLine, + endIndex: completedSource.at(-1)!.endLine, + text: completedSource.map((source) => source.text).join(' '), + representedSource: completedSource, + } + const candidate = queryEvidenceLineRenderCandidate(completedLine, sourceFile, sourceLines, true) + if ( + !candidate.projection + || !candidate.projection.hasProtectedTokens + || candidate.lineCapExceeded + ) { + continue + } + const proposed = reserved.map((line, index): RenderedQueryEvidenceLine => ( + index === baselineIndex + ? { + source: candidate.source, + text: candidate.faithful, + representedSource: candidate.source.representedSource, + } + : line + )) + const proposedSnippet = proposed.map((line) => line.text).join('\n') + if ( + proposedSnippet.split('\n').length > QUERY_EVIDENCE_SNIPPET_MAX_LINES + || proposedSnippet.length > QUERY_EVIDENCE_SNIPPET_CHAR_CAP + ) { + continue + } + reserved = proposed + } + return reserved +} + function renderQueryEvidenceLines( lines: readonly QueryEvidenceLine[], sourceFile: string, diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 640d0f07..396c401a 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -1690,6 +1690,136 @@ describe('bounded multiline literal statement completion', () => { expect(evidence?.snippet).not.toContain('const datum') expect(evidence?.snippet).not.toContain('...') }) + + it.each([ + { name: 'LF', lineEnding: '\n' as const }, + { name: 'CRLF', lineEnding: '\r\n' as const }, + ])('reserves the full $name selected baseline before declining an oversized expansion', ({ + lineEnding, + }) => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ` dispatchOutcome(datum, \` ${'a'.repeat(150)}`, + ` ${'b'.repeat(90)} \`)`, + ' return retryOutcome()', + '}', + ] + const evidence = evidenceFor(source, { + question: 'What are the dispatch outcome and retry outcome?', + label: 'assemble', + lineEnding, + }) + + expect(evidence).toEqual({ + snippet: [ + `L3: ${source[2]!.trimStart()}`, + 'L5: return retryOutcome()', + ].join('\n'), + lineNumber: 3, + scope: 'symbol', + }) + }) + + it('admits a fitting expansion while retaining later selected evidence', () => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ` dispatchOutcome(datum, \` ${'a'.repeat(60)}`, + ` ${'b'.repeat(30)} \`)`, + ' return retryOutcome()', + '}', + ] + const evidence = evidenceFor(source, { + question: 'What are the dispatch outcome and retry outcome?', + label: 'assemble', + }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: ${source[2]!.trimStart()}`, + `L4: ${source[3]}`, + 'L5: return retryOutcome()', + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('declines an expansion that would overflow the full selection by one character', () => { + const firstLiteralRow = ` dispatchOutcome(\` ${'a'.repeat(120)}` + const laterSource = ' return retryOutcome()' + const firstRendered = `L2: ${firstLiteralRow.trimStart()}` + const laterRendered = `L4: ${laterSource.trim()}` + const closingShell = ' \`)' + const fillLength = 301 + - firstRendered.length + - 1 + - `L3: ${closingShell}`.length + - 1 + - laterRendered.length + const closingLiteralRow = ` ${'b'.repeat(fillLength)} \`)` + const completed = [firstRendered, `L3: ${closingLiteralRow}`, laterRendered].join('\n') + const evidence = evidenceFor([ + 'function assemble() {', + firstLiteralRow, + closingLiteralRow, + laterSource, + '}', + ], { + question: 'What are the dispatch outcome and retry outcome?', + label: 'assemble', + }) + + expect(fillLength).toBeGreaterThan(0) + expect(completed).toHaveLength(301) + expect(evidence?.snippet).toBe([firstRendered, laterRendered].join('\n')) + }) + + it('declines an expansion that would exceed the four-row reservation', () => { + const source = [ + 'function assemble() {', + ' dispatchOutcome(` alpha', + ' beta `)', + ' retryOutcome()', + ' validateResult()', + ' return finalizeResponse()', + '}', + ] + const evidence = evidenceFor(source, { + question: 'What are the dispatch outcome, retry outcome, validate result, and finalize response?', + label: 'assemble', + }) + + expect(evidence?.snippet).toBe([ + `L2: ${source[1]!.trimStart()}`, + 'L4: retryOutcome()', + 'L5: validateResult()', + 'L6: return finalizeResponse()', + ].join('\n')) + }) + + it('reserves an accepted expansion while evaluating a second candidate expansion', () => { + const source = [ + 'function assemble() {', + ` dispatchOutcome(\` ${'a'.repeat(80)}`, + ` ${'b'.repeat(45)} \`)`, + ` return retryOutcome(\` ${'c'.repeat(80)}`, + ` ${'d'.repeat(45)} \`)`, + '}', + ] + const evidence = evidenceFor(source, { + question: 'What are the dispatch outcome and retry outcome?', + label: 'assemble', + }) + + expect(evidence?.snippet).toBe([ + `L2: ${source[1]!.trimStart()}`, + `L3: ${source[2]}`, + `L4: ${source[3]!.trimStart()}`, + ].join('\n')) + }) }) describe('owner declaration completion budgets and atomic preservation', () => { From 8a6a90721c0d07c41ba7539e5062b137a028b7f9 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 02:43:55 +0400 Subject: [PATCH 18/27] fix: coalesce authenticated literal statement selections --- src/runtime/retrieve.ts | 137 ++++++++++++++++-- .../unit/retrieve-owner-declarations.test.ts | 110 ++++++++++++++ 2 files changed, 233 insertions(+), 14 deletions(-) diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index cf78e1d0..7e3fb791 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -1400,6 +1400,47 @@ function renderUnexpandedQueryEvidenceLineEntries( return rendered } +function sameRepresentedQueryEvidenceSource( + left: RepresentedQueryEvidenceSource, + right: RepresentedQueryEvidenceSource, +): boolean { + return left.startLine === right.startLine + && left.endLine === right.endLine + && left.text === right.text +} + +function insertedLiteralStatementCompletion(input: { + sourceFile: string + sourceLines: readonly string[] + ownerRange: { start: number; end: number } + representedSource: readonly RepresentedQueryEvidenceSource[] +}): RepresentedQueryEvidenceSource | null { + const completedSource = completeQueryEvidenceLiteralStatement({ + sourceFilePath: input.sourceFile, + sourceLines: input.sourceLines, + ownerRange: input.ownerRange, + representedSource: input.representedSource, + }) + if (!completedSource) { + return null + } + const inserted = completedSource.filter((completed) => ( + !input.representedSource.some((represented) => ( + sameRepresentedQueryEvidenceSource(completed, represented) + )) + )) + if ( + inserted.length !== 1 + || !input.representedSource.some((represented) => ( + inserted[0]!.startLine <= represented.startLine + && represented.endLine <= inserted[0]!.endLine + )) + ) { + return null + } + return inserted[0]! +} + function renderQueryEvidenceLineEntries( lines: readonly QueryEvidenceLine[], sourceFile: string, @@ -1411,17 +1452,67 @@ function renderQueryEvidenceLineEntries( return baseline } - let reserved = baseline - for (const [baselineIndex, baselineLine] of baseline.entries()) { + const authenticatedStatements: RepresentedQueryEvidenceSource[] = [] + for (const baselineLine of baseline) { + const statement = insertedLiteralStatementCompletion({ + sourceFile, + sourceLines, + ownerRange, + representedSource: baselineLine.representedSource, + }) + if ( + statement + && !authenticatedStatements.some((candidate) => ( + sameRepresentedQueryEvidenceSource(candidate, statement) + )) + ) { + authenticatedStatements.push(statement) + } + } + + const plans = authenticatedStatements.map((statement) => ({ + statement, + baselineIndices: baseline.flatMap((line, baselineIndex) => ( + line.representedSource.some((represented) => ( + statement.startLine <= represented.startLine + && represented.endLine <= statement.endLine + )) + ? [baselineIndex] + : [] + )), + })).filter((plan, planIndex, allPlans) => ( + plan.baselineIndices.length > 0 + && !allPlans.some((other, otherIndex) => ( + otherIndex !== planIndex + && other.baselineIndices.some((baselineIndex) => plan.baselineIndices.includes(baselineIndex)) + )) + )).sort((left, right) => ( + left.baselineIndices[0]! - right.baselineIndices[0]! + )) + + let reserved = baseline.map((line, baselineIndex) => ({ + line, + baselineIndices: [baselineIndex], + })) + for (const plan of plans) { + const representedSource = plan.baselineIndices.flatMap((baselineIndex) => ( + baseline[baselineIndex]!.representedSource + )) const completedSource = completeQueryEvidenceLiteralStatement({ sourceFilePath: sourceFile, sourceLines, ownerRange, - representedSource: baselineLine.representedSource, + representedSource, }) - if (!completedSource) { + if ( + !completedSource + || completedSource.filter((completed) => ( + sameRepresentedQueryEvidenceSource(completed, plan.statement) + )).length !== 1 + ) { continue } + const baselineLine = baseline[plan.baselineIndices[0]!]! const completedLine: QueryEvidenceLine = { ...baselineLine.source, index: completedSource[0]!.startLine, @@ -1437,16 +1528,34 @@ function renderQueryEvidenceLineEntries( ) { continue } - const proposed = reserved.map((line, index): RenderedQueryEvidenceLine => ( - index === baselineIndex - ? { - source: candidate.source, - text: candidate.faithful, - representedSource: candidate.source.representedSource, - } - : line + const plannedIndices = new Set(plan.baselineIndices) + const removed = reserved.filter((entry) => ( + entry.baselineIndices.some((baselineIndex) => plannedIndices.has(baselineIndex)) + )) + const removedIndices = new Set(removed.flatMap((entry) => entry.baselineIndices)) + if ( + removedIndices.size !== plannedIndices.size + || [...removedIndices].some((baselineIndex) => !plannedIndices.has(baselineIndex)) + ) { + continue + } + const proposed = [ + ...reserved.filter((entry) => ( + !entry.baselineIndices.some((baselineIndex) => plannedIndices.has(baselineIndex)) + )), + { + line: { + source: candidate.source, + text: candidate.faithful, + representedSource: candidate.source.representedSource, + }, + baselineIndices: plan.baselineIndices, + }, + ].sort((left, right) => ( + left.line.source.index - right.line.source.index + || left.baselineIndices[0]! - right.baselineIndices[0]! )) - const proposedSnippet = proposed.map((line) => line.text).join('\n') + const proposedSnippet = proposed.map((entry) => entry.line.text).join('\n') if ( proposedSnippet.split('\n').length > QUERY_EVIDENCE_SNIPPET_MAX_LINES || proposedSnippet.length > QUERY_EVIDENCE_SNIPPET_CHAR_CAP @@ -1455,7 +1564,7 @@ function renderQueryEvidenceLineEntries( } reserved = proposed } - return reserved + return reserved.map((entry) => entry.line) } function renderQueryEvidenceLines( diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 396c401a..3b88f6a2 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -1527,6 +1527,116 @@ describe('bounded multiline literal statement completion', () => { }) }) + it.each([ + { name: 'LF', lineEnding: '\n' as const, literalDelimiter: '\n' }, + { name: 'CRLF', lineEnding: '\r\n' as const, literalDelimiter: '\r\n' }, + ])('coalesces overlapping selected fragments of one $name literal statement before declaration completion', ({ + lineEnding, + literalDelimiter, + }) => { + const evidence = evidenceFor([ + 'function assemble() {', + ' const datum = 12.5', + ' return { displayedText: datum, note: `alpha', + ' recordedValue: beta` }', + '}', + ], { + question: 'What are the displayed text and recorded value?', + label: 'assemble', + lineEnding, + }) + + expect(evidence).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: return { displayedText: datum, note: \`alpha${literalDelimiter}L4: recordedValue: beta\` }`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it('retains an unrelated selected header and later fragment while coalescing one statement', () => { + const source = [ + 'function assemble() {', + ' const datum = 12.5', + ' for (const readyState of [true]) {', + ' return { displayedText: datum, note: `alpha', + ' recordedValue: beta` }', + ' }', + ' return summarizeResult()', + '}', + ] + const evidence = evidenceFor(source, { + question: 'What are the ready state, displayed text, recorded value, and summarize result?', + label: 'assemble', + }) + + expect(evidence).toEqual({ + snippet: [ + 'L3: for (const readyState of [true]) {', + 'L4: return { displayedText: datum, note: `alpha', + 'L5: recordedValue: beta` }', + 'L7: return summarizeResult()', + ].join('\n'), + lineNumber: 3, + scope: 'symbol', + }) + }) + + it('keeps sibling literal statements independent within the four-row reservation', () => { + const evidence = evidenceFor([ + 'function assemble() {', + " const first = 'first'", + " const second = 'second'", + ' if (first) return { displayedText: first, note: `alpha', + ' recordedValue: first` }', + ' if (second) return { retryText: second, note: `gamma', + ' deliveryValue: second` }', + '}', + ], { + question: 'What are the displayed text, recorded value, retry text, and delivery value?', + label: 'assemble', + }) + + expect(evidence).toEqual({ + snippet: [ + 'L4: if (first) return { displayedText: first, note: `alpha', + 'L5: recordedValue: first` }', + 'L6: if (second) return { retryText: second, note: `gamma', + 'L7: deliveryValue: second` }', + ].join('\n'), + lineNumber: 4, + scope: 'symbol', + }) + }) + + it('retains a coalesced statement when its declaration would exceed the total reservation', () => { + const declaration = ` const datum = '${'d'.repeat(202)}'` + const completedSnippet = [ + 'L3: return { displayedText: datum, note: `alphaX', + 'L4: recordedValue: beta` }', + ].join('\n') + const evidence = evidenceFor([ + 'function assemble() {', + declaration, + ' return { displayedText: datum, note: `alphaX', + ' recordedValue: beta` }', + '}', + ], { + question: 'What are the displayed text and recorded value?', + label: 'assemble', + }) + + expect(declaration).toHaveLength(220) + expect(`L2: ${declaration}\n${completedSnippet}`).toHaveLength(301) + expect(evidence).toEqual({ + snippet: completedSnippet, + lineNumber: 3, + scope: 'symbol', + }) + }) + it('preserves authenticated disjoint fragments while completing only their unique literal statement', () => { const source = [ 'function assemble() {', From 4236995d6bf00599245cb0673b2e31285518d5b5 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 03:31:42 +0400 Subject: [PATCH 19/27] fix: project overlapping declaration source ranges once --- src/runtime/query-evidence-dependencies.ts | 37 +-- .../unit/retrieve-owner-declarations.test.ts | 235 ++++++++++++++++++ 2 files changed, 257 insertions(+), 15 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index d646c5bf..acac6a99 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -1263,23 +1263,30 @@ export function ownerLocalDeclarationEvidence( start: represented.startLine, end: represented.endLine, })) - return closure - .map((statement): OwnerDeclarationEvidence => { - const range = evidenceLineRangeOf(statement, sourceFile) - return { - startLine: range.start, - endLine: range.end, - lines: literalDelimiterPreservingLines(range, sourceFile, input.sourceLines), - } - }) - .filter((declaration) => ( - input.ownerRange.start <= declaration.startLine - && declaration.endLine <= input.ownerRange.end + const eligibleRanges = closure + .map((statement) => evidenceLineRangeOf(statement, sourceFile)) + .filter((range) => ( + input.ownerRange.start <= range.start + && range.end <= input.ownerRange.end )) - .filter((declaration) => !representedRanges.some((range) => ( - range.start <= declaration.endLine && declaration.startLine <= range.end + .filter((range) => !representedRanges.some((represented) => ( + represented.start <= range.end && range.start <= represented.end ))) - .sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine) + .sort((left, right) => left.start - right.start || left.end - right.end) + const physicalRanges: Array<{ start: number; end: number }> = [] + for (const range of eligibleRanges) { + const previous = physicalRanges.at(-1) + if (previous && range.start <= previous.end) { + previous.end = Math.max(previous.end, range.end) + } else { + physicalRanges.push({ ...range }) + } + } + return physicalRanges.map((range) => ({ + startLine: range.start, + endLine: range.end, + lines: literalDelimiterPreservingLines(range, sourceFile, input.sourceLines), + })) } catch { return [] } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 3b88f6a2..e514d062 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -16,6 +16,7 @@ import * as ts from 'typescript' import { completeQueryEvidenceLiteralStatement, ownerLocalDeclarationEvidence, + retainQueryEvidenceSourceSnapshot, } from '../../src/runtime/query-evidence-dependencies.js' import { readQueryEvidenceSnippet, type QueryEvidenceSnippet } from '../../src/runtime/retrieve.js' @@ -116,6 +117,205 @@ describe('owner-local declaration completion', () => { ].join('\n')) }) + it.each([ + { name: 'LF', lineEnding: '\n' as const }, + { name: 'CRLF', lineEnding: '\r\n' as const }, + ])('projects one direct physical declaration row for same-line $name bindings', ({ lineEnding }) => { + const sourceLines = [ + 'export function executeSample() {', + ' const origin = 12.5; const alias = origin', + ' return dispatchOutcome(origin, alias)', + '}', + ] + const sourceText = sourceLines.join(lineEnding) + retainQueryEvidenceSourceSnapshot({ sourceFilePath: 'sample.ts', sourceLines, sourceText }) + + expect(ownerLocalDeclarationEvidence({ + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 4 }, + representedSource: [{ startLine: 3, endLine: 3, text: sourceLines[2]! }], + })).toEqual([{ + startLine: 2, + endLine: 2, + lines: [{ lineNumber: 2, text: sourceLines[1]! }], + }]) + }) + + it.each([ + { name: 'LF', lineEnding: '\n' as const }, + { name: 'CRLF', lineEnding: '\r\n' as const }, + ])('projects one direct overlapping declaration range for multiline-boundary $name bindings', ({ lineEnding }) => { + const sourceLines = [ + 'export function executeSample() {', + ' const origin = 12.5; const alias =', + ' origin', + ' return dispatchOutcome(origin, alias)', + '}', + ] + const sourceText = sourceLines.join(lineEnding) + retainQueryEvidenceSourceSnapshot({ sourceFilePath: 'sample.ts', sourceLines, sourceText }) + + expect(ownerLocalDeclarationEvidence({ + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 5 }, + representedSource: [{ startLine: 4, endLine: 4, text: sourceLines[3]! }], + })).toEqual([{ + startLine: 2, + endLine: 3, + lines: [ + { lineNumber: 2, text: sourceLines[1]! }, + { lineNumber: 3, text: sourceLines[2]! }, + ], + }]) + }) + + it.each([ + { + name: 'same-line LF', + lineEnding: '\n' as const, + source: [ + 'export function executeSample() {', + ' const origin = 12.5; const alias = origin', + ' return dispatchOutcome(origin, alias)', + '}', + ], + expected: [ + 'L2: const origin = 12.5; const alias = origin', + 'L3: return dispatchOutcome(origin, alias)', + ].join('\n'), + }, + { + name: 'same-line CRLF', + lineEnding: '\r\n' as const, + source: [ + 'export function executeSample() {', + ' const origin = 12.5; const alias = origin', + ' return dispatchOutcome(origin, alias)', + '}', + ], + expected: [ + 'L2: const origin = 12.5; const alias = origin', + 'L3: return dispatchOutcome(origin, alias)', + ].join('\n'), + }, + { + name: 'multiline-boundary LF', + lineEnding: '\n' as const, + source: [ + 'export function executeSample() {', + ' const origin = 12.5; const alias =', + ' origin', + ' return dispatchOutcome(origin, alias)', + '}', + ], + expected: [ + 'L2: const origin = 12.5; const alias =', + 'L3: origin', + 'L4: return dispatchOutcome(origin, alias)', + ].join('\n'), + }, + { + name: 'multiline-boundary CRLF', + lineEnding: '\r\n' as const, + source: [ + 'export function executeSample() {', + ' const origin = 12.5; const alias =', + ' origin', + ' return dispatchOutcome(origin, alias)', + '}', + ], + expected: [ + 'L2: const origin = 12.5; const alias =', + 'L3: origin', + 'L4: return dispatchOutcome(origin, alias)', + ].join('\n'), + }, + ])('emits each physical declaration row once through the public $name layout', ({ + lineEnding, + source, + expected, + }) => { + const evidence = evidenceFor(source, { lineEnding }) + + expect(evidence).toEqual({ + snippet: expected, + lineNumber: 2, + scope: 'symbol', + }) + expect(evidence?.snippet.match(/const origin/g)).toHaveLength(1) + expect(evidence?.snippet.match(/return dispatchOutcome/g)).toHaveLength(1) + }) + + it('keeps equal declaration text at distinct physical locations separate', () => { + const sourceLines = [ + 'export function executeSample() {', + ' {', + ' const datum = 12.5', + ' dispatchOutcome(datum)', + ' }', + ' {', + ' const datum = 12.5', + ' return dispatchOutcome(datum)', + ' }', + '}', + ] + + expect(ownerLocalDeclarationEvidence({ + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 10 }, + representedSource: [ + { startLine: 4, endLine: 4, text: sourceLines[3]! }, + { startLine: 8, endLine: 8, text: sourceLines[7]! }, + ], + })).toEqual([ + { + startLine: 3, + endLine: 3, + lines: [{ lineNumber: 3, text: sourceLines[2]! }], + }, + { + startLine: 7, + endLine: 7, + lines: [{ lineNumber: 7, text: sourceLines[6]! }], + }, + ]) + }) + + it.each([ + { name: 'LF', lineEnding: '\n' as const, literalLineSuffix: '' }, + { name: 'CRLF', lineEnding: '\r\n' as const, literalLineSuffix: '\r' }, + ])('normalizes overlapping $name ranges before projecting multiline literal and trivia bytes', ({ + lineEnding, + literalLineSuffix, + }) => { + const sourceLines = [ + 'export function executeSample() {', + ' const origin = 12.5; /* declaration trivia */ const alias = ` alpha', + ' beta ` ? origin : origin', + ' return dispatchOutcome(origin, alias)', + '}', + ] + const sourceText = sourceLines.join(lineEnding) + retainQueryEvidenceSourceSnapshot({ sourceFilePath: 'sample.ts', sourceLines, sourceText }) + + expect(ownerLocalDeclarationEvidence({ + sourceFilePath: 'sample.ts', + sourceLines, + ownerRange: { start: 1, end: 5 }, + representedSource: [{ startLine: 4, endLine: 4, text: sourceLines[3]! }], + })).toEqual([{ + startLine: 2, + endLine: 3, + lines: [ + { lineNumber: 2, text: `${sourceLines[1]}${literalLineSuffix}` }, + { lineNumber: 3, text: sourceLines[2]! }, + ], + }]) + }) + it('deduplicates repeated uses and resolves multiple simple declarators by identity', () => { const evidence = evidenceFor([ 'export function executeSample() {', @@ -1933,6 +2133,41 @@ describe('bounded multiline literal statement completion', () => { }) describe('owner declaration completion budgets and atomic preservation', () => { + it('still rejects an overlapping declaration row above the 220-character cap', () => { + const declaration = ` const origin = '${'d'.repeat(190)}'; const alias = origin` + const baseline = 'L3: return dispatchOutcome(origin, alias)' + expect(declaration.length).toBeGreaterThan(220) + + const evidence = evidenceFor([ + 'export function executeSample() {', + declaration, + ' return dispatchOutcome(origin, alias)', + '}', + ]) + + expect(evidence?.snippet).toBe(baseline) + expect(evidence?.snippet).not.toContain('const origin') + }) + + it('still rejects an overlapping declaration closure above the total character cap', () => { + const declaration = ` const origin = '${'d'.repeat(160)}'; const alias = origin` + const returnSource = ` return dispatchOutcome(origin, alias, '${'r'.repeat(80)}')` + const baseline = `L3: ${returnSource.trim()}` + expect(declaration.length).toBeLessThanOrEqual(220) + expect(`L2: ${declaration}\n${baseline}`.length).toBeGreaterThan(300) + + const evidence = evidenceFor([ + 'export function executeSample() {', + declaration, + returnSource, + '}', + ]) + + expect(evidence?.snippet).toBe(baseline) + expect(evidence?.snippet).not.toContain('const origin') + expect(evidence?.snippet).not.toContain('...') + }) + it('admits faithful literal whitespace when the completed snippet exactly fills the cap', () => { const literal = `${'value '.repeat(22)}end` const returnSource = ` return dispatchOutcome(datum, '${literal}')` From 0ec585bf1a0a089e719ddf6c2a4f16c01bed0fa4 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 04:02:23 +0400 Subject: [PATCH 20/27] test: cover warm source cache property presence --- tests/unit/retrieve-source-terms.test.ts | 155 +++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/tests/unit/retrieve-source-terms.test.ts b/tests/unit/retrieve-source-terms.test.ts index bda6ef2f..173bed3b 100644 --- a/tests/unit/retrieve-source-terms.test.ts +++ b/tests/unit/retrieve-source-terms.test.ts @@ -57,6 +57,161 @@ describe('stored-source term cache controls', () => { expect(storedSourceTermCacheInspection(target).tokenizationCount).toBe(3) }) + it('distinguishes absent, own undefined, and own null kind across warm controlled views', () => { + const target = graph() + const original = owner(`${ROOT}/src/one.ts`, { label: 'operate()' }) + delete original.node_kind + target.addNode('one', original) + target.addNode('peer', owner(`${ROOT}/src/peer.ts`, { snippet: 'function neutral() {\n return peer_signal\n}' })) + + const entriesFor = (attributes: Attributes): Array<[string, Attributes]> => ( + target.nodeEntries().map<[string, Attributes]>(([id, current]) => [ + id, + id === 'one' ? attributes : current, + ]) + ) + const ownsKind = (entries: readonly [string, Attributes][]): boolean => Object.prototype.hasOwnProperty.call( + entries.find(([id]) => id === 'one')?.[1], + 'node_kind', + ) + const assertPeerAndCardinality = (peerTokens: readonly string[] | undefined): void => { + const inspection = storedSourceTermCacheInspection(target) + expect(inspection.entryCount).toBe(target.numberOfNodes()) + expect(inspection.tokenArrays.get('peer')).toBe(peerTokens) + } + + const absentEntries = entriesFor(original) + expect(ownsKind(absentEntries)).toBe(false) + const absent = reconcileStoredSourceTerms(target, ROOT, absentEntries) + const absentEntry = absent.get('one') + const peerTokens = absent.get('peer')?.tokens + expect(absentEntry?.eligible).toBe(true) + const absentWarm = reconcileStoredSourceTerms(target, ROOT, absentEntries) + expect(absentWarm.get('one')).toBe(absentEntry) + assertPeerAndCardinality(peerTokens) + + const ownUndefinedEntries = entriesFor({ ...original, node_kind: undefined }) + expect(ownsKind(ownUndefinedEntries)).toBe(true) + const ownUndefined = reconcileStoredSourceTerms(target, ROOT, ownUndefinedEntries) + const ownUndefinedEntry = ownUndefined.get('one') + expect(ownUndefinedEntry?.eligible).toBe(false) + expect(ownUndefinedEntry).not.toBe(absentEntry) + const ownUndefinedWarm = reconcileStoredSourceTerms(target, ROOT, ownUndefinedEntries) + expect(ownUndefinedWarm.get('one')).toBe(ownUndefinedEntry) + assertPeerAndCardinality(peerTokens) + + const ownNullEntries = entriesFor({ ...original, node_kind: null }) + expect(ownsKind(ownNullEntries)).toBe(true) + const ownNull = reconcileStoredSourceTerms(target, ROOT, ownNullEntries) + const ownNullEntry = ownNull.get('one') + expect(ownNullEntry?.eligible).toBe(false) + expect(ownNullEntry).not.toBe(ownUndefinedEntry) + const ownNullWarm = reconcileStoredSourceTerms(target, ROOT, ownNullEntries) + expect(ownNullWarm.get('one')).toBe(ownNullEntry) + assertPeerAndCardinality(peerTokens) + + const restoredEntries = entriesFor(original) + expect(ownsKind(restoredEntries)).toBe(false) + const restored = reconcileStoredSourceTerms(target, ROOT, restoredEntries) + const restoredEntry = restored.get('one') + expect(restoredEntry?.eligible).toBe(true) + expect(restoredEntry).not.toBe(ownNullEntry) + expect(restoredEntry?.tokens).toEqual(absentEntry?.tokens) + const restoredWarm = reconcileStoredSourceTerms(target, ROOT, restoredEntries) + expect(restoredWarm.get('one')).toBe(restoredEntry) + assertPeerAndCardinality(peerTokens) + + const options = { question: 'parcelChecksum', budget: 800, retrievalLevel: 1 as const } + const cold = retrieveContext(target, options) + const warm = retrieveContext(target, options) + expect(cold.matched_nodes.map((node) => node.node_id)).toContain('one') + expect(JSON.stringify(warm)).toBe(JSON.stringify(cold)) + assertPeerAndCardinality(peerTokens) + }) + + it('distinguishes valid, own undefined, absent, and own null snippet across warm controlled views', () => { + const target = graph() + const original = owner(`${ROOT}/src/one.ts`) + target.addNode('one', original) + target.addNode('peer', owner(`${ROOT}/src/peer.ts`, { snippet: 'function neutral() {\n return peer_signal\n}' })) + + const entriesFor = (attributes: Attributes): Array<[string, Attributes]> => ( + target.nodeEntries().map<[string, Attributes]>(([id, current]) => [ + id, + id === 'one' ? attributes : current, + ]) + ) + const ownsSnippet = (entries: readonly [string, Attributes][]): boolean => Object.prototype.hasOwnProperty.call( + entries.find(([id]) => id === 'one')?.[1], + 'snippet', + ) + const assertPeerAndCardinality = (peerTokens: readonly string[] | undefined): void => { + const inspection = storedSourceTermCacheInspection(target) + expect(inspection.entryCount).toBe(target.numberOfNodes()) + expect(inspection.tokenArrays.get('peer')).toBe(peerTokens) + } + + const validEntries = entriesFor(original) + expect(ownsSnippet(validEntries)).toBe(true) + const valid = reconcileStoredSourceTerms(target, ROOT, validEntries) + const validEntry = valid.get('one') + const peerTokens = valid.get('peer')?.tokens + expect(validEntry?.eligible).toBe(true) + const validWarm = reconcileStoredSourceTerms(target, ROOT, validEntries) + expect(validWarm.get('one')).toBe(validEntry) + assertPeerAndCardinality(peerTokens) + + const ownUndefinedEntries = entriesFor({ ...original, snippet: undefined }) + expect(ownsSnippet(ownUndefinedEntries)).toBe(true) + const ownUndefined = reconcileStoredSourceTerms(target, ROOT, ownUndefinedEntries) + const ownUndefinedEntry = ownUndefined.get('one') + expect(ownUndefinedEntry?.eligible).toBe(false) + expect(ownUndefinedEntry).not.toBe(validEntry) + const ownUndefinedWarm = reconcileStoredSourceTerms(target, ROOT, ownUndefinedEntries) + expect(ownUndefinedWarm.get('one')).toBe(ownUndefinedEntry) + assertPeerAndCardinality(peerTokens) + + const absentSnippet: Attributes = { ...original } + delete absentSnippet.snippet + const absentEntries = entriesFor(absentSnippet) + expect(ownsSnippet(absentEntries)).toBe(false) + const absent = reconcileStoredSourceTerms(target, ROOT, absentEntries) + const absentEntry = absent.get('one') + expect(absentEntry?.eligible).toBe(false) + expect(absentEntry).not.toBe(ownUndefinedEntry) + const absentWarm = reconcileStoredSourceTerms(target, ROOT, absentEntries) + expect(absentWarm.get('one')).toBe(absentEntry) + assertPeerAndCardinality(peerTokens) + + const ownNullEntries = entriesFor({ ...original, snippet: null }) + expect(ownsSnippet(ownNullEntries)).toBe(true) + const ownNull = reconcileStoredSourceTerms(target, ROOT, ownNullEntries) + const ownNullEntry = ownNull.get('one') + expect(ownNullEntry?.eligible).toBe(false) + expect(ownNullEntry).not.toBe(absentEntry) + const ownNullWarm = reconcileStoredSourceTerms(target, ROOT, ownNullEntries) + expect(ownNullWarm.get('one')).toBe(ownNullEntry) + assertPeerAndCardinality(peerTokens) + + const restoredEntries = entriesFor(original) + expect(ownsSnippet(restoredEntries)).toBe(true) + const restored = reconcileStoredSourceTerms(target, ROOT, restoredEntries) + const restoredEntry = restored.get('one') + expect(restoredEntry?.eligible).toBe(true) + expect(restoredEntry).not.toBe(ownNullEntry) + expect(restoredEntry?.tokens).toEqual(validEntry?.tokens) + const restoredWarm = reconcileStoredSourceTerms(target, ROOT, restoredEntries) + expect(restoredWarm.get('one')).toBe(restoredEntry) + assertPeerAndCardinality(peerTokens) + + const options = { question: 'parcelChecksum', budget: 800, retrievalLevel: 1 as const } + const cold = retrieveContext(target, options) + const warm = retrieveContext(target, options) + expect(cold.matched_nodes.map((node) => node.node_id)).toContain('one') + expect(JSON.stringify(warm)).toBe(JSON.stringify(cold)) + assertPeerAndCardinality(peerTokens) + }) + it('detects nested object edits even when the containing reference is reused', () => { const target = graph() const metadata: Attributes = { external_call: false } From 79924e681a47ccc8fb828fe55867f76ff82bb78b Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 04:49:21 +0400 Subject: [PATCH 21/27] fix: authenticate evidence literals at physical token positions --- src/runtime/query-evidence-dependencies.ts | 82 +++++++++-- .../unit/retrieve-owner-declarations.test.ts | 133 ++++++++++++++++++ 2 files changed, 207 insertions(+), 8 deletions(-) diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index acac6a99..1fbf8986 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -265,6 +265,8 @@ interface ProtectedSourceToken { interface PhysicalSourceProjection extends QueryEvidenceSourceProjection { identityText: string + sourceStart: number + sourceEnd: number tokens: ProtectedSourceToken[] } @@ -421,28 +423,92 @@ function physicalProjectionForRange( return { text, identityText, + sourceStart: bounds.start, + sourceEnd: bounds.end, literalLineBreaks, hasProtectedTokens: tokens.length > 0, tokens, } } +function alignedOutsideSourceEnd( + physicalText: string, + representedText: string, + representedStart: number, + trimStart: boolean, + trimEnd: boolean, +): number | null { + let physicalOffset = 0 + let representedOffset = representedStart + if (trimStart && physicalText.length > 0) { + while (/\s/.test(representedText[representedOffset] ?? '')) { + representedOffset += 1 + } + } + while (physicalOffset < physicalText.length) { + if (/\s/.test(physicalText[physicalOffset]!)) { + const whitespaceStart = physicalOffset + while (physicalOffset < physicalText.length && /\s/.test(physicalText[physicalOffset]!)) { + physicalOffset += 1 + } + const optional = (trimStart && whitespaceStart === 0) + || (trimEnd && physicalOffset === physicalText.length) + if (!optional && !/\s/.test(representedText[representedOffset] ?? '')) { + return null + } + while (/\s/.test(representedText[representedOffset] ?? '')) { + representedOffset += 1 + } + continue + } + if (representedText[representedOffset] !== physicalText[physicalOffset]) { + return null + } + physicalOffset += 1 + representedOffset += 1 + } + return representedOffset +} + function representedSourceIdentity( representedText: string, projection: PhysicalSourceProjection, + sourceFile: ts.SourceFile, ): string | null { let marked = '' - let cursor = 0 + let physicalCursor = projection.sourceStart + let representedCursor = 0 for (const [index, token] of projection.tokens.entries()) { - const tokenOffset = representedText.indexOf(token.representedText, cursor) - if (tokenOffset < 0) { + const tokenOffset = alignedOutsideSourceEnd( + sourceFile.text.slice(physicalCursor, token.start), + representedText, + representedCursor, + physicalCursor === projection.sourceStart, + false, + ) + if ( + tokenOffset === null + || representedText.slice(tokenOffset, tokenOffset + token.representedText.length) + !== token.representedText + ) { return null } - marked += representedText.slice(cursor, tokenOffset) + marked += representedText.slice(representedCursor, tokenOffset) marked += tokenMarker(index) - cursor = tokenOffset + token.representedText.length + physicalCursor = token.end + representedCursor = tokenOffset + token.representedText.length + } + const representedEnd = alignedOutsideSourceEnd( + sourceFile.text.slice(physicalCursor, projection.sourceEnd), + representedText, + representedCursor, + projection.tokens.length === 0, + true, + ) + if (representedEnd === null || representedText.slice(representedEnd).trim().length > 0) { + return null } - marked += representedText.slice(cursor) + marked += representedText.slice(representedCursor, representedEnd) return normalizedSource(marked) } @@ -453,7 +519,7 @@ function representedSourceMatchesPhysicalRange( ): boolean { const projection = physicalProjectionForRange(range, sourceFile) return projection !== null - && representedSourceIdentity(representedText, projection) === projection.identityText + && representedSourceIdentity(representedText, projection, sourceFile) === projection.identityText } function literalDelimiterPreservingLines( @@ -533,7 +599,7 @@ export function queryEvidenceSourceProjection(input: { const projection = physicalProjectionForRange(range, sourceFile, true) if ( !projection - || representedSourceIdentity(represented.text, projection) !== projection.identityText + || representedSourceIdentity(represented.text, projection, sourceFile) !== projection.identityText ) { return null } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index e514d062..814175de 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -16,6 +16,7 @@ import * as ts from 'typescript' import { completeQueryEvidenceLiteralStatement, ownerLocalDeclarationEvidence, + queryEvidenceSourceProjection, retainQueryEvidenceSourceSnapshot, } from '../../src/runtime/query-evidence-dependencies.js' import { readQueryEvidenceSnippet, type QueryEvidenceSnippet } from '../../src/runtime/retrieve.js' @@ -432,6 +433,138 @@ describe('owner-local declaration completion', () => { ].join('\n')) }) + it.each([ + { name: 'LF', lineEnding: '\n' as const }, + { name: 'CRLF', lineEnding: '\r\n' as const }, + ])('authenticates a matching-comment literal at its physical $name token location', ({ lineEnding }) => { + const statement = ' return dispatchOutcome(datum, /* "alpha beta" */ "alpha beta")' + const sourceLines = [ + 'function assemble() {', + ' const datum = 12.5', + statement, + '}', + ] + const fixture = sourceFixture(sourceLines, '.ts', lineEnding) + retainQueryEvidenceSourceSnapshot({ + sourceFilePath: fixture.sourceFile, + sourceLines, + sourceText: sourceLines.join(lineEnding), + }) + const input = { + sourceFilePath: fixture.sourceFile, + sourceLines, + representedSource: [{ startLine: 3, endLine: 3, text: statement }], + } + const expectedStatement = 'return dispatchOutcome(datum, /* "alpha beta" */ "alpha beta")' + + expect(ownerLocalDeclarationEvidence({ + ...input, + ownerRange: { start: 1, end: 4 }, + })).toEqual([{ + startLine: 2, + endLine: 2, + lines: [{ lineNumber: 2, text: ' const datum = 12.5' }], + }]) + expect(queryEvidenceSourceProjection({ ...input, shapedText: statement })).toEqual({ + text: expectedStatement, + literalLineBreaks: [], + hasProtectedTokens: true, + }) + expect(readQueryEvidenceSnippet(fixture.sourceFile, 1, { + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation: fixture.sourceLocation, + fileCache: new Map(), + })).toEqual({ + snippet: [ + 'L2: const datum = 12.5', + `L3: ${expectedStatement}`, + ].join('\n'), + lineNumber: 2, + scope: 'symbol', + }) + }) + + it.each([ + { + name: 'repeated string literals', + statement: ' return dispatchOutcome(datum, "alpha beta", /* "alpha beta" */ "alpha beta")', + expectedStatement: 'return dispatchOutcome(datum, "alpha beta", /* "alpha beta" */ "alpha beta")', + }, + { + name: 'a regular expression literal', + statement: ' return dispatchOutcome(datum, /* /alpha beta/ */ /alpha beta/)', + expectedStatement: 'return dispatchOutcome(datum, /* /alpha beta/ */ /alpha beta/)', + }, + ])('authenticates $name by physical token location when matching bytes precede a token', ({ + statement, + expectedStatement, + }) => { + const sourceLines = [ + 'function assemble() {', + ' const datum = 12.5', + statement, + '}', + ] + const fixture = sourceFixture(sourceLines) + const input = { + sourceFilePath: fixture.sourceFile, + sourceLines, + representedSource: [{ startLine: 3, endLine: 3, text: statement }], + } + + expect(ownerLocalDeclarationEvidence({ + ...input, + ownerRange: { start: 1, end: 4 }, + })).toHaveLength(1) + expect(queryEvidenceSourceProjection({ ...input, shapedText: statement })?.text).toBe(expectedStatement) + expect(readQueryEvidenceSnippet(fixture.sourceFile, 1, { + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation: fixture.sourceLocation, + fileCache: new Map(), + })?.snippet).toBe([ + 'L2: const datum = 12.5', + `L3: ${expectedStatement}`, + ].join('\n')) + }) + + it('rejects deceptive changed literal bytes despite an unchanged matching comment', () => { + const statement = ' return dispatchOutcome(datum, /* "alpha beta" */ "alpha beta")' + const deceptive = statement.replace('*/ "alpha beta"', '*/ "alpha beta"') + const sourceLines = [ + 'function assemble() {', + ' const datum = 12.5', + statement, + '}', + ] + const fixture = sourceFixture(sourceLines) + const input = { + sourceFilePath: fixture.sourceFile, + sourceLines, + } + + expect(ownerLocalDeclarationEvidence({ + ...input, + ownerRange: { start: 1, end: 4 }, + representedSource: [{ startLine: 3, endLine: 3, text: deceptive }], + })).toEqual([]) + expect(queryEvidenceSourceProjection({ + ...input, + representedSource: [{ startLine: 3, endLine: 3, text: deceptive }], + shapedText: deceptive, + })).toBeNull() + expect(readQueryEvidenceSnippet(fixture.sourceFile, 1, { + question: 'What is the dispatch outcome?', + label: 'assemble', + sourceLocation: fixture.sourceLocation, + fileCache: new Map(), + })?.snippet).toBe([ + 'L2: const datum = 12.5', + 'L3: return dispatchOutcome(datum, /* "alpha beta" */ "alpha beta")', + ].join('\n')) + }) + it.each([ { literalKind: 'escaped string with comment-looking text', From 0eb02676f5807724882f5b4e2e47193e14677543 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 05:01:39 +0400 Subject: [PATCH 22/27] test: align extraction cache diagnostic version assertion --- tests/unit/indexing-completeness.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/indexing-completeness.test.ts b/tests/unit/indexing-completeness.test.ts index 6f8dcaee..fccedb54 100644 --- a/tests/unit/indexing-completeness.test.ts +++ b/tests/unit/indexing-completeness.test.ts @@ -306,7 +306,7 @@ describe('indexing extraction cache', () => { }) expect(JSON.parse(readFileSync(cachePath, 'utf8'))).toMatchObject({ - __madarTsExtractorVersion: 68, + __madarTsExtractorVersion: 69, diagnostics: [{ code: 'tree_sitter_python_fallback', level: 'warning', From b3ff98c85812c750df39520b3b427bf04d42776b Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 07:42:05 +0400 Subject: [PATCH 23/27] fix: preserve generated owner evidence across synthetic call boundaries --- src/infrastructure/generate.ts | 2 +- src/runtime/retrieve.ts | 2 +- ...retrieve-generated-source-evidence.test.ts | 190 ++++++++++++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 tests/unit/retrieve-generated-source-evidence.test.ts diff --git a/src/infrastructure/generate.ts b/src/infrastructure/generate.ts index 429bb2e1..9317cc0d 100644 --- a/src/infrastructure/generate.ts +++ b/src/infrastructure/generate.ts @@ -518,7 +518,7 @@ function composeDefaultAutoSourceEvidence(args: { const symbol = compatibleSymbols[0]! const ownerRange = validSpiOwnerRange(symbol) if (!ownerRange || ownerRange.start !== projectedStart.start) return spiNode - if (fileSymbols.some((other) => other !== symbol && ( + if (fileSymbols.some((other) => other !== symbol && expectedSpiLabel(other) !== null && ( validSpiOwnerRange(other)?.start === ownerRange.start || (other.name === symbol.name && other.kind === symbol.kind) ))) return spiNode diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 05d898cb..3b443576 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -5264,7 +5264,7 @@ function buildRetrieveResultFromOrderedCandidates( label: node.label, sourceLocation: node.sourceLocation, fileNodeLike: node.fileNodeLike, - derived: node.lineNumberDerived, + derived: node.lineNumberDerived && lineRangeFromSourceLocation(node.sourceLocation) === null, fileCache: snippetFileCache, }) const snippet = queryEvidenceSnippet?.snippet ?? node.storedSnippet ?? readSnippet(node.sourceFile, node.lineNumber, { diff --git a/tests/unit/retrieve-generated-source-evidence.test.ts b/tests/unit/retrieve-generated-source-evidence.test.ts new file mode 100644 index 00000000..ea5f19b6 --- /dev/null +++ b/tests/unit/retrieve-generated-source-evidence.test.ts @@ -0,0 +1,190 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { generateGraph } from '../../src/infrastructure/generate.js' +import { + compactRetrieveResultForStdio, + retrieveContext, +} from '../../src/runtime/retrieve.js' +import { loadGraph } from '../../src/runtime/serve.js' + +const QUESTION = 'Why can the displayed estimate differ from the charged total?' +const roots: string[] = [] + +interface GeneratedNode extends Record { + id: string + label: string + source_file: string +} + +interface MatchedSourceNode { + node_id?: string + snippet: string | null +} + +function isolatedRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'retrieve-generated-source-')) + roots.push(root) + return root +} + +function writeSource(root: string, relativePath: string, source: string): void { + const sourceFile = join(root, relativePath) + mkdirSync(dirname(sourceFile), { recursive: true }) + writeFileSync(sourceFile, source, 'utf8') +} + +function nodesFromGraph(graph: ReturnType): GeneratedNode[] { + return graph.nodeEntries().map(([id, attributes]) => ({ id, ...attributes }) as GeneratedNode) +} + +function oneNode(nodes: readonly GeneratedNode[], label: string, relativePath: string): GeneratedNode { + const matches = nodes.filter((node) => node.label === label && node.source_file.endsWith(relativePath)) + expect(matches, `expected one ${label} node in ${relativePath}`).toHaveLength(1) + return matches[0]! +} + +function oneMatched(nodes: readonly MatchedSourceNode[], nodeId: string): MatchedSourceNode { + const matches = nodes.filter((node) => node.node_id === nodeId) + expect(matches, `expected retrieved node ${nodeId}`).toHaveLength(1) + return matches[0]! +} + +function expectSnippetLines(snippet: string | null, lines: readonly string[]): void { + const actualLines = snippet?.split('\n') ?? [] + for (const line of lines) { + expect(actualLines).toContain(line) + } +} + +function writeRoundingLayout(root: string, layout: 'same-file' | 'relative-import'): { + ownerFile: string + helperFile: string + ownerLocation: string + ownerConst: string + ownerReturn: string +} { + const firstHelper = 'export function readSavedValue(value) { return Math.floor(value); }' + const secondHelper = 'export function rebuildValue(value) { return Math.ceil(value); }' + if (layout === 'same-file') { + writeSource(root, 'src/engine.js', [ + firstHelper, + secondHelper, + 'export function assemble() {', + ' const input = 12.5;', + ' return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', + '}', + ].join('\n') + '\n') + return { + ownerFile: '/src/engine.js', + helperFile: '/src/engine.js', + ownerLocation: 'L3-L6', + ownerConst: 'L4: const input = 12.5;', + ownerReturn: 'L5: return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', + } + } + + writeSource(root, 'src/rounding.js', `${firstHelper}\n${secondHelper}\n`) + writeSource(root, 'src/engine.js', [ + "import { readSavedValue, rebuildValue } from './rounding.js';", + 'export function assemble() {', + ' const input = 12.5;', + ' return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', + '}', + ].join('\n') + '\n') + return { + ownerFile: '/src/engine.js', + helperFile: '/src/rounding.js', + ownerLocation: 'L2-L5', + ownerConst: 'L3: const input = 12.5;', + ownerReturn: 'L4: return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', + } +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('fresh generated source evidence', () => { + it('authenticates a complete generated owner range without fabricating line_number', () => { + const root = isolatedRoot() + const layout = writeRoundingLayout(root, 'same-file') + const generated = generateGraph(root, { noHtml: true }) + const graph = loadGraph(generated.graphPath) + const owner = oneNode(nodesFromGraph(graph), 'assemble()', layout.ownerFile) + + expect(owner.source_location).toBe(layout.ownerLocation) + expect(owner).not.toHaveProperty('line_number') + + const raw = retrieveContext(graph, { question: QUESTION, budget: 4_000 }) + const compact = compactRetrieveResultForStdio(raw) + expectSnippetLines(oneMatched(raw.matched_nodes, owner.id).snippet, [layout.ownerConst, layout.ownerReturn]) + expectSnippetLines(oneMatched(compact.matched_nodes, owner.id).snippet, [layout.ownerConst, layout.ownerReturn]) + }) + + it.each(['same-file', 'relative-import'] as const)( + 'preserves external-call owners through default generation, canonical reload, and retrieval: %s', + (layoutName) => { + const root = isolatedRoot() + const layout = writeRoundingLayout(root, layoutName) + const firstSource = 'export function readSavedValue(value) { return Math.floor(value); }' + const secondSource = 'export function rebuildValue(value) { return Math.ceil(value); }' + + const cold = generateGraph(root, { noHtml: true }) + const warm = generateGraph(root, { noHtml: true }) + expect(cold.cache?.hit).toBe(false) + expect(warm.cache?.hit).toBe(true) + + const graph = loadGraph(warm.graphPath) + const nodes = nodesFromGraph(graph) + const owner = oneNode(nodes, 'assemble()', layout.ownerFile) + const first = oneNode(nodes, 'readSavedValue()', layout.helperFile) + const second = oneNode(nodes, 'rebuildValue()', layout.helperFile) + const floor = oneNode(nodes, 'Math.floor', layout.helperFile) + const ceil = oneNode(nodes, 'Math.ceil', layout.helperFile) + + expect(owner).toMatchObject({ source_location: layout.ownerLocation }) + expect(first).toMatchObject({ source_location: 'L1', snippet: firstSource }) + expect(second).toMatchObject({ source_location: 'L2', snippet: secondSource }) + expect(first).not.toHaveProperty('line_number') + expect(second).not.toHaveProperty('line_number') + expect(floor).toMatchObject({ source_location: 'L1', external_call: true }) + expect(ceil).toMatchObject({ source_location: 'L2', external_call: true }) + expect(floor).not.toHaveProperty('snippet') + expect(ceil).not.toHaveProperty('snippet') + + const raw = retrieveContext(graph, { question: QUESTION, budget: 4_000 }) + const compact = compactRetrieveResultForStdio(raw) + const firstRetrievedSource = layoutName === 'same-file' ? `L1: ${firstSource}` : firstSource + const secondRetrievedSource = layoutName === 'same-file' ? `L2: ${secondSource}` : secondSource + for (const result of [raw, compact]) { + expectSnippetLines(oneMatched(result.matched_nodes, owner.id).snippet, [layout.ownerConst, layout.ownerReturn]) + expectSnippetLines(oneMatched(result.matched_nodes, first.id).snippet, [firstRetrievedSource]) + expectSnippetLines(oneMatched(result.matched_nodes, second.id).snippet, [secondRetrievedSource]) + } + }, + 60_000, + ) + + it('preserves an ordinary non-rounding declaration beside its synthetic external call', () => { + const root = isolatedRoot() + const source = 'export function serializeLedger(value) { return JSON.stringify(value); }' + writeSource(root, 'src/ledger.js', `${source}\n`) + + const generated = generateGraph(root, { noHtml: true }) + const graph = loadGraph(generated.graphPath) + const nodes = nodesFromGraph(graph) + const owner = oneNode(nodes, 'serializeLedger()', '/src/ledger.js') + const external = oneNode(nodes, 'JSON.stringify', '/src/ledger.js') + + expect(owner).toMatchObject({ source_location: 'L1', snippet: source }) + expect(owner).not.toHaveProperty('line_number') + expect(external).toMatchObject({ source_location: 'L1', external_call: true }) + expect(external).not.toHaveProperty('snippet') + }) +}) From 0da76dd7671ed5415069af29e97f5e4c0d6aaf15 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 08:30:46 +0400 Subject: [PATCH 24/27] fix: retain complete small owner source within retrieval budgets --- src/runtime/context-pack.ts | 15 +- src/runtime/query-evidence-dependencies.ts | 270 ++++++++ src/runtime/retrieve.ts | 199 +++++- .../context-pack-representation-modes.test.ts | 22 + ...retrieve-generated-source-evidence.test.ts | 4 +- .../unit/retrieve-owner-declarations.test.ts | 25 + ...etrieve-small-owner-representation.test.ts | 630 ++++++++++++++++++ 7 files changed, 1135 insertions(+), 30 deletions(-) create mode 100644 tests/unit/retrieve-small-owner-representation.test.ts diff --git a/src/runtime/context-pack.ts b/src/runtime/context-pack.ts index 8993f5bd..6f62d0ea 100644 --- a/src/runtime/context-pack.ts +++ b/src/runtime/context-pack.ts @@ -1219,14 +1219,15 @@ export function renderCompiledContextPackNodes< } } - const renderedNodes = nodes.some((node) => typeof node.representation_type === 'string') - ? [...nodes] - : applyContextPackResolution(nodes, { - resolution: resolutionForTaskBudget(taskContract, nodes), - relationships, - task_kind: taskContract.task_kind, - }).nodes.map((node, index) => { + const renderedNodes = applyContextPackResolution(nodes, { + resolution: resolutionForTaskBudget(taskContract, nodes), + relationships, + task_kind: taskContract.task_kind, + }).nodes.map((node, index) => { const originalNode = nodes[index]! + if (typeof originalNode.representation_type === 'string') { + return originalNode + } const originalCost = estimateContextPackEntryTokens( originalNode.label, originalNode.source_file, diff --git a/src/runtime/query-evidence-dependencies.ts b/src/runtime/query-evidence-dependencies.ts index 1fbf8986..f5111e9d 100644 --- a/src/runtime/query-evidence-dependencies.ts +++ b/src/runtime/query-evidence-dependencies.ts @@ -32,6 +32,20 @@ export interface QueryEvidenceSourceProjection { hasProtectedTokens: boolean } +export interface CompleteSmallOwnerSourceInput { + sourceFilePath: string + sourceLines: readonly string[] + ownerRange: { start: number; end: number } + label: string + nodeKind?: string + externalCall?: boolean +} + +export interface CompleteSmallOwnerSourceEvidence { + snippet: string + lineNumber: number +} + interface ParsedSourceSnapshot { sourceFilePath: string sourceText: string @@ -63,6 +77,8 @@ const parsedSourceSnapshots = new WeakMap() const sourceFilesWithNormalizedLineCaches = new WeakSet() const JS_TS_EXTENSIONS = new Set(['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs']) +const COMPLETE_SMALL_OWNER_MAX_LINES = 25 +const COMPLETE_SMALL_OWNER_MAX_CHARACTERS = 2000 /** Retains bytes from the existing source read alongside its normalized line cache. */ export function retainQueryEvidenceSourceSnapshot(input: { @@ -252,6 +268,260 @@ function uniquelyIdentifiedOwner( return owners.length === 1 ? owners[0]! : null } +interface IdentifiedCompleteOwner { + owner: FunctionOwner + sourceNode: ts.Node + ownerKind: 'function' | 'method' + label: string + alternateLabel?: string +} + +function unwrapOwnerExpression(owner: ts.FunctionExpression | ts.ArrowFunction): ts.Expression { + let expression: ts.Expression = owner + while ( + (ts.isParenthesizedExpression(expression.parent) + || ts.isAsExpression(expression.parent) + || ts.isSatisfiesExpression(expression.parent) + || ts.isTypeAssertionExpression(expression.parent) + || ts.isNonNullExpression(expression.parent)) + && expression.parent.expression === expression + ) { + expression = expression.parent + } + return expression +} + +function simplePropertyName(name: ts.PropertyName | undefined): string | null { + if (!name) { + return null + } + if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) { + return name.text + } + return null +} + +function identifiedCompleteOwner(owner: FunctionOwner): IdentifiedCompleteOwner | null { + if (ts.isFunctionDeclaration(owner)) { + if (owner.name) { + return { + owner, + sourceNode: owner, + ownerKind: 'function', + label: `${owner.name.text}()`, + alternateLabel: owner.name.text, + } + } + const modifiers = ts.canHaveModifiers(owner) ? ts.getModifiers(owner) : undefined + const defaultExport = modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.DefaultKeyword) + && modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword) + return defaultExport + ? { owner, sourceNode: owner, ownerKind: 'function', label: 'default()', alternateLabel: 'default' } + : null + } + + if (ts.isMethodDeclaration(owner)) { + const name = simplePropertyName(owner.name) + return name + ? { owner, sourceNode: owner, ownerKind: 'method', label: `.${name}()`, alternateLabel: name } + : null + } + + if (ts.isConstructorDeclaration(owner)) { + return { + owner, + sourceNode: owner, + ownerKind: 'method', + label: '.constructor()', + alternateLabel: 'constructor', + } + } + + if (!ts.isFunctionExpression(owner) && !ts.isArrowFunction(owner)) { + return null + } + const expression = unwrapOwnerExpression(owner) + const parent = expression.parent + if ( + ts.isVariableDeclaration(parent) + && parent.initializer === expression + && ts.isIdentifier(parent.name) + && ts.isVariableDeclarationList(parent.parent) + && parent.parent.declarations.length === 1 + && ts.isVariableStatement(parent.parent.parent) + ) { + return { + owner, + sourceNode: parent.parent.parent, + ownerKind: 'function', + label: `${parent.name.text}()`, + alternateLabel: parent.name.text, + } + } + if ( + ts.isPropertyDeclaration(parent) + && parent.initializer === expression + ) { + const name = simplePropertyName(parent.name) + return name + ? { owner, sourceNode: parent, ownerKind: 'method', label: `.${name}()`, alternateLabel: name } + : null + } + if (ts.isExportAssignment(parent) && parent.expression === expression && !parent.isExportEquals) { + return { + owner, + sourceNode: parent, + ownerKind: 'function', + label: 'default()', + alternateLabel: 'default', + } + } + return null +} + +function compatibleCompleteOwnerNodeKind( + ownerKind: IdentifiedCompleteOwner['ownerKind'], + nodeKind: string | undefined, +): boolean { + const kind = nodeKind?.trim().toLowerCase() ?? '' + if (kind.length === 0) { + return true + } + return ownerKind === 'function' + ? kind === 'function' || kind === 'component' + : kind === 'method' || kind === 'function' || kind === 'route' +} + +function sourceSpanForCompleteOwner( + identified: IdentifiedCompleteOwner, + sourceFile: ts.SourceFile, +): { text: string; startLine: number; endLine: number } | null { + let start = identified.sourceNode.getStart(sourceFile) + let end = identified.sourceNode.getEnd() + const lineStarts = sourceFile.getLineStarts() + const startLineIndex = sourceFile.getLineAndCharacterOfPosition(start).line + const endLineIndex = sourceFile.getLineAndCharacterOfPosition(Math.max(start, end - 1)).line + const lineStart = lineStarts[startLineIndex] + const nextLineStart = lineStarts[endLineIndex + 1] ?? sourceFile.text.length + if (lineStart === undefined) { + return null + } + + if (sourceFile.text.slice(lineStart, start).trim().length === 0) { + start = lineStart + } + let lineContentEnd = nextLineStart + if (sourceFile.text.slice(Math.max(0, lineContentEnd - 2), lineContentEnd) === '\r\n') { + lineContentEnd -= 2 + } else if (/^[\r\n]$/.test(sourceFile.text.slice(Math.max(0, lineContentEnd - 1), lineContentEnd))) { + lineContentEnd -= 1 + } + if (sourceFile.text.slice(end, lineContentEnd).trim().length === 0) { + end = lineContentEnd + } + if (start >= end) { + return null + } + + return { + text: sourceFile.text.slice(start, end), + startLine: startLineIndex + 1, + endLine: endLineIndex + 1, + } +} + +function numberPhysicalSourceRows(sourceText: string, startLine: number): { + snippet: string + endLine: number +} { + const lineBreakPattern = /\r\n|\r|\n/g + let snippet = '' + let cursor = 0 + let lineNumber = startLine + for (const match of sourceText.matchAll(lineBreakPattern)) { + const offset = match.index + snippet += `L${lineNumber}: ${sourceText.slice(cursor, offset)}${match[0]}` + cursor = offset + match[0].length + lineNumber += 1 + } + snippet += `L${lineNumber}: ${sourceText.slice(cursor)}` + return { snippet, endLine: lineNumber } +} + +/** + * Authenticates and serializes a complete, exact, small JS/TS function or + * method owner from the source snapshot already retained by retrieval. + */ +export function completeSmallOwnerSourceEvidence( + input: CompleteSmallOwnerSourceInput, +): CompleteSmallOwnerSourceEvidence | null { + try { + if ( + input.externalCall === true + || !Number.isInteger(input.ownerRange.start) + || !Number.isInteger(input.ownerRange.end) + || input.ownerRange.start < 1 + || input.ownerRange.end < input.ownerRange.start + || input.ownerRange.end - input.ownerRange.start + 1 > COMPLETE_SMALL_OWNER_MAX_LINES + ) { + return null + } + const sourceFile = parsedSourceForSnapshot(input.sourceFilePath, input.sourceLines) + if (!sourceFile || input.ownerRange.end > sourceFile.getLineStarts().length) { + return null + } + const parseDiagnostics = (sourceFile as ts.SourceFile & { + parseDiagnostics?: readonly ts.Diagnostic[] + }).parseDiagnostics + if (parseDiagnostics && parseDiagnostics.length > 0) { + return null + } + + const candidates: IdentifiedCompleteOwner[] = [] + const visit = (node: ts.Node): void => { + const owner = functionOwnerWithBody(node) + if (owner) { + const identified = identifiedCompleteOwner(owner) + if ( + identified + && (identified.label === input.label || identified.alternateLabel === input.label) + && compatibleCompleteOwnerNodeKind(identified.ownerKind, input.nodeKind) + ) { + const range = lineRangeOf(identified.sourceNode, sourceFile) + if (range.start === input.ownerRange.start && range.end === input.ownerRange.end) { + candidates.push(identified) + } + } + } + ts.forEachChild(node, visit) + } + visit(sourceFile) + if (candidates.length !== 1) { + return null + } + + const source = sourceSpanForCompleteOwner(candidates[0]!, sourceFile) + if ( + !source + || source.startLine !== input.ownerRange.start + || source.endLine !== input.ownerRange.end + || source.text.length > COMPLETE_SMALL_OWNER_MAX_CHARACTERS + ) { + return null + } + const numbered = numberPhysicalSourceRows(source.text, source.startLine) + if (numbered.endLine !== source.endLine) { + return null + } + return { + snippet: numbered.snippet, + lineNumber: source.startLine, + } + } catch { + return null + } +} + function normalizedSource(value: string): string { return value.replace(/\s+/g, ' ').trim() } diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 3b443576..37df297d 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -87,6 +87,7 @@ import { } from './retrieve/pipeline.js' import { communitiesFromGraph, estimateQueryTokens } from './serve.js' import { + completeSmallOwnerSourceEvidence, completeQueryEvidenceLiteralStatement, ownerLocalDeclarationEvidence, queryEvidenceSourceProjection, @@ -552,41 +553,76 @@ function applyRetrieveSnippetBudgetToNodes { + const completeOwner = completeOwnerMatchedNodes.get(node) + const priorTruncation = 'snippet_truncated' in node + && (node as { snippet_truncated?: boolean }).snippet_truncated === true const originalSnippet = typeof node.snippet === 'string' && node.snippet.trim().length > 0 ? node.snippet : null - if (originalSnippet === null) { - return { + if (originalSnippet === null && !completeOwner) { + return attachCompleteOwnerState({ ...node, snippet: null, - snippet_truncated: false, - } + snippet_truncated: priorTruncation, + }, completeOwner) } const snippetEligible = eligibleNodeIndexes === undefined ? index < topNWithSnippet : eligibleNodeIndexes.has(index) if (!snippetEligible) { - return { - ...node, + return attachCompleteOwnerState({ + ...(completeOwner ? withoutCompleteOwnerRepresentationClaim(node) : node), snippet: null, - snippet_truncated: false, - } + snippet_truncated: completeOwner ? true : priorTruncation, + }, completeOwner) } const remainingSnippetBudget = Math.max(0, snippetBudget - usedTokens) - const shapedSnippet = truncateSnippetToTokenBudget(originalSnippet, remainingSnippetBudget) + if (completeOwner) { + const fullTokens = snippetTokenCount(completeOwner.fullSnippet) + if (fullTokens <= remainingSnippetBudget) { + usedTokens += fullTokens + return attachCompleteOwnerState({ + ...node, + snippet: completeOwner.fullSnippet, + snippet_line_number: completeOwner.fullLineNumber, + snippet_scope: 'symbol' as const, + snippet_truncated: priorTruncation, + representation_reason: COMPLETE_OWNER_REPRESENTATION_REASON, + }, completeOwner) + } + + const fallbackTokens = snippetTokenCount(completeOwner.fallbackSnippet) + if (fallbackTokens <= remainingSnippetBudget) { + usedTokens += fallbackTokens + return attachCompleteOwnerState({ + ...withoutCompleteOwnerRepresentationClaim(node), + snippet: completeOwner.fallbackSnippet, + snippet_line_number: completeOwner.fallbackLineNumber, + snippet_scope: completeOwner.fallbackScope, + snippet_truncated: true, + }, completeOwner) + } + return attachCompleteOwnerState({ + ...withoutCompleteOwnerRepresentationClaim(node), + snippet: null, + snippet_truncated: true, + }, completeOwner) + } + + const shapedSnippet = truncateSnippetToTokenBudget(originalSnippet!, remainingSnippetBudget) const serializedSnippetTokens = snippetTokenCount(shapedSnippet.snippet) const boundedSnippet = serializedSnippetTokens <= remainingSnippetBudget ? shapedSnippet : truncateSnippetToTokenBudget(shapedSnippet.snippet ?? '', remainingSnippetBudget) usedTokens += snippetTokenCount(boundedSnippet.snippet) - return { + return attachCompleteOwnerState({ ...node, snippet: boundedSnippet.snippet, - snippet_truncated: shapedSnippet.truncated || boundedSnippet.truncated, - } + snippet_truncated: priorTruncation || shapedSnippet.truncated || boundedSnippet.truncated, + }, completeOwner) }) const serializedSnippetTokensUsed = shapedNodes.reduce( @@ -751,12 +787,49 @@ export interface QueryEvidenceSnippet { interface QueryEvidenceSnippetOptions { question: string label: string + nodeKind?: string + externalCall?: boolean + authenticatedOwner?: boolean sourceLocation?: string | null fileNodeLike?: boolean derived?: boolean fileCache?: Map } +interface CompleteOwnerSnippetState { + fullSnippet: string + fullLineNumber: number + fallbackSnippet: string + fallbackLineNumber: number + fallbackScope: QueryEvidenceSnippet['scope'] +} + +const completeOwnerQuerySnippets = new WeakMap() +const completeOwnerMatchedNodes = new WeakMap() +const COMPLETE_OWNER_REPRESENTATION_REASON = 'complete small owner source' + +function attachCompleteOwnerState( + target: T, + state: CompleteOwnerSnippetState | undefined, +): T { + if (state) { + completeOwnerMatchedNodes.set(target, state) + } + return target +} + +function copyCompleteOwnerState(source: object, target: T): T { + return attachCompleteOwnerState(target, completeOwnerMatchedNodes.get(source)) +} + +function withoutCompleteOwnerRepresentationClaim(source: T): T { + const output = { ...source } as T & { representation_reason?: string } + if (output.representation_reason === COMPLETE_OWNER_REPRESENTATION_REASON) { + delete output.representation_reason + } + return output +} + interface QueryEvidenceLine { index: number endIndex: number @@ -1814,11 +1887,38 @@ export function readQueryEvidenceSnippet( const completed = ownerRange ? completeOwnerDeclarationEvidence(sourceFile, lines, ownerRange, baseline) : null - return { + const fallback: QueryEvidenceSnippet = { snippet: completed?.snippet ?? baselineSnippet, lineNumber: completed?.lineNumber ?? baseline[0]!.source.index, scope, } + const completeOwner = ownerRange && options.authenticatedOwner === true + ? completeSmallOwnerSourceEvidence({ + sourceFilePath: sourceFile, + sourceLines: lines, + ownerRange, + label: options.label, + ...(options.nodeKind ? { nodeKind: options.nodeKind } : {}), + ...(options.externalCall !== undefined ? { externalCall: options.externalCall } : {}), + }) + : null + if (!completeOwner) { + return fallback + } + + const result: QueryEvidenceSnippet = { + snippet: completeOwner.snippet, + lineNumber: completeOwner.lineNumber, + scope: 'symbol', + } + completeOwnerQuerySnippets.set(result, { + fullSnippet: completeOwner.snippet, + fullLineNumber: completeOwner.lineNumber, + fallbackSnippet: fallback.snippet, + fallbackLineNumber: fallback.lineNumber, + fallbackScope: fallback.scope, + }) + return result } catch { return null } @@ -1964,6 +2064,7 @@ function scoredNodeFromGraphEntry( lineNumberDerived: resolvedLine.derived, storedSnippet: storedSnippetFromAttributes(attributes), nodeKind, + externalCall: attributes.external_call === true, framework: typeof attributes.framework === 'string' ? attributes.framework : undefined, frameworkRole: frameworkRole || undefined, sourceDomain: classifySourceDomain(String(attributes.source_file ?? ''), rootPath), @@ -2046,6 +2147,7 @@ interface SeedCandidate { lineNumberDerived: boolean storedSnippet: string | null nodeKind: string + externalCall: boolean framework?: string | undefined frameworkRole?: string | undefined sourceDomain: SourceDomain @@ -2071,6 +2173,7 @@ interface ScoredNode { lineNumberDerived: boolean storedSnippet: string | null nodeKind: string + externalCall: boolean framework?: string | undefined frameworkRole?: string | undefined sourceDomain: SourceDomain @@ -2101,6 +2204,7 @@ function scoredNodeFromGraph(graph: KnowledgeGraph, nodeId: string, score: numbe lineNumberDerived: resolvedLine.derived, storedSnippet: storedSnippetFromAttributes(attributes), nodeKind: String(attributes.node_kind ?? ''), + externalCall: attributes.external_call === true, framework: typeof attributes.framework === 'string' ? attributes.framework : undefined, frameworkRole: typeof attributes.framework_role === 'string' ? attributes.framework_role : undefined, sourceDomain: classifySourceDomain(String(attributes.source_file ?? ''), rootPath), @@ -5183,14 +5287,21 @@ export function contextPackFromRetrieveResult( budget: result.token_count, prompt: result.question, }) + const sourceNodes = result.matched_nodes.map((node) => copyCompleteOwnerState(node, { + ...node, + evidence_class: node.evidence_class ?? retrieveEvidenceClassForBand(node.relevance_band), + })) const renderedNodes = renderCompiledContextPackNodes( taskContract, - result.matched_nodes.map((node) => ({ - ...node, - evidence_class: node.evidence_class ?? retrieveEvidenceClassForBand(node.relevance_band), - })), + sourceNodes, result.relationships, ) + renderedNodes.nodes.forEach((node, index) => { + const sourceNode = sourceNodes[index] + if (sourceNode) { + copyCompleteOwnerState(sourceNode, node) + } + }) return { task_contract: taskContract, @@ -5241,6 +5352,7 @@ function buildRetrieveResultFromOrderedCandidates( sliceMetadata, rootPath, ) + const completeOwnerStatesByNodeId = new Map() const nodeCandidates: Array> = orderedCandidates.map((node) => { let builtEntry: RetrieveMatchedNode | undefined let tokenCost: number | undefined @@ -5262,17 +5374,23 @@ function buildRetrieveResultFromOrderedCandidates( const queryEvidenceSnippet = readQueryEvidenceSnippet(node.sourceFile, node.lineNumber, { question: options.question, label: node.label, + nodeKind: node.nodeKind, + externalCall: node.externalCall, + authenticatedOwner: true, sourceLocation: node.sourceLocation, fileNodeLike: node.fileNodeLike, derived: node.lineNumberDerived && lineRangeFromSourceLocation(node.sourceLocation) === null, fileCache: snippetFileCache, }) + const completeOwnerState = queryEvidenceSnippet + ? completeOwnerQuerySnippets.get(queryEvidenceSnippet) + : undefined const snippet = queryEvidenceSnippet?.snippet ?? node.storedSnippet ?? readSnippet(node.sourceFile, node.lineNumber, { derived: node.lineNumberDerived, fileCache: snippetFileCache, }) const serializedSourceFile = relativizeSourceFile(node.sourceFile, rootPath) - builtEntry = { + builtEntry = attachCompleteOwnerState({ node_id: node.id, label: node.label, source_file: serializedSourceFile, @@ -5295,6 +5413,9 @@ function buildRetrieveResultFromOrderedCandidates( ...(node.framework ? { framework: node.framework } : {}), ...(node.frameworkRole ? { framework_role: node.frameworkRole } : {}), ...(node.nodeKind.trim().length > 0 ? { node_kind: node.nodeKind } : {}), + }, completeOwnerState) + if (completeOwnerState) { + completeOwnerStatesByNodeId.set(node.id, completeOwnerState) } tokenCost = estimateRetrieveEntryTokens(node.label, serializedSourceFile, node.lineNumber, snippet) return builtEntry @@ -5358,7 +5479,25 @@ function buildRetrieveResultFromOrderedCandidates( selection_strategy: 'value-per-token', retrieval_gate: retrievalGate, }), options.onStageDiagnostic) - const matchedNodes = pack.nodes as RetrieveMatchedNode[] + const packedNodes = pack.nodes as RetrieveMatchedNode[] + const matchedNodes = packedNodes.map((node) => { + const state = typeof node.node_id === 'string' + ? completeOwnerStatesByNodeId.get(node.node_id) + : undefined + if (!state) { + return node + } + return attachCompleteOwnerState({ + ...node, + snippet: state.fullSnippet, + snippet_line_number: state.fullLineNumber, + snippet_scope: 'symbol' as const, + representation_type: 'detail' as const, + representation_reason: COMPLETE_OWNER_REPRESENTATION_REASON, + }, state) + }) + const packedNodeTokens = tokenCountForMatchedNodes(packedNodes) + const matchedNodeTokens = tokenCountForMatchedNodes(matchedNodes) const answerContract = buildRuntimeGenerationAnswerContract( taskContract, retrievalGate, @@ -5381,7 +5520,7 @@ function buildRetrieveResultFromOrderedCandidates( return { question: options.question, - token_count: pack.token_count, + token_count: Math.max(0, pack.token_count - packedNodeTokens + matchedNodeTokens), matched_nodes: matchedNodes, relationships: pack.relationships, community_context: pack.community_context, @@ -5646,6 +5785,7 @@ function retrieveContextPass( lineNumberDerived: resolvedLine.derived, storedSnippet: storedSnippetFromAttributes(attributes), nodeKind, + externalCall: attributes.external_call === true, framework, frameworkRole: frameworkRole || undefined, sourceDomain, @@ -5714,6 +5854,7 @@ function retrieveContextPass( lineNumberDerived: candidate.lineNumberDerived, storedSnippet: candidate.storedSnippet, nodeKind: candidate.nodeKind, + externalCall: candidate.externalCall, framework: candidate.framework, frameworkRole: candidate.frameworkRole, fileType: candidate.fileType, @@ -5995,6 +6136,7 @@ function retrieveContextPass( lineNumberDerived: resolvedLine.derived, storedSnippet: storedSnippetFromAttributes(attributes), nodeKind: String(attributes.node_kind ?? ''), + externalCall: attributes.external_call === true, framework: typeof attributes.framework === 'string' ? attributes.framework : undefined, frameworkRole: typeof attributes.framework_role === 'string' ? attributes.framework_role : undefined, sourceDomain, @@ -6551,6 +6693,18 @@ export function compactRetrieveResult(result: RetrieveResult, options: RetrieveS kind: 'retrieve', ...(Number.isFinite(compactFrameworkLimit) ? { max_nodes: compactFrameworkLimit } : {}), }) + for (const compactNode of compactPack.nodes) { + if (typeof compactNode.snippet !== 'string' || compactNode.snippet.length === 0) { + continue + } + const sourceNode = fullPack.nodes.find((node) => ( + typeof compactNode.node_id === 'string' + && compactNode.node_id === node.node_id + )) + if (sourceNode) { + copyCompleteOwnerState(sourceNode, compactNode) + } + } const useCallEndpointSnippetAllocation = !promotedSlice && options.topNWithSnippet === undefined @@ -6573,7 +6727,10 @@ export function compactRetrieveResult(result: RetrieveResult, options: RetrieveS (total, node) => total + estimateRetrieveEntryTokens(node.label, node.source_file, node.line_number, node.snippet ?? null), 0, ) - const matchedNodes = shapedNodes.nodes.map(({ evidence_class: _evidenceClass, ...node }) => node as CompactRetrieveMatchedNode) + const matchedNodes = shapedNodes.nodes.map((shapedNode) => { + const { evidence_class: _evidenceClass, ...node } = shapedNode + return copyCompleteOwnerState(shapedNode, node as CompactRetrieveMatchedNode) + }) const retrievalPlan = reconcileRetrievalPlanQueryEvidence( result.retrieval_plan, result.question, diff --git a/tests/unit/context-pack-representation-modes.test.ts b/tests/unit/context-pack-representation-modes.test.ts index bcca3123..c7444b99 100644 --- a/tests/unit/context-pack-representation-modes.test.ts +++ b/tests/unit/context-pack-representation-modes.test.ts @@ -5,6 +5,7 @@ import type { ContextPackRelationship, ContextPackTaskKind, } from '../../src/contracts/context-pack.js' +import { classifyTaskContract, renderCompiledContextPackNodes } from '../../src/runtime/context-pack.js' import { applyContextPackResolution } from '../../src/runtime/context-pack-resolution.js' type TaskAwareResolutionOptions = Parameters[1] @@ -162,6 +163,27 @@ function renderForTask(taskKind: ContextPackTaskKind) { } describe('adaptive context representation modes (#176)', () => { + it('preserves one explicit detail node without bypassing resolution for its peers', () => { + const { nodes, relationships } = makeFixture() + const mixed = nodes.slice(0, 2).map((entry, index) => ( + index === 0 + ? { + ...entry, + representation_type: 'detail' as const, + representation_reason: 'authenticated complete owner', + } + : entry + )) + const rendered = renderCompiledContextPackNodes( + classifyTaskContract('review', { budget: 256, prompt: 'Review authentication' }), + mixed, + relationships, + ) + + expect(rendered.nodes[0]).toEqual(mixed[0]) + expect(rendered.nodes[1]?.representation_type).toBe('signature') + }) + it('keeps the same selected nodes while task-aware rendering changes representations', () => { const { nodes } = makeFixture() const explain = renderForTask('explain') diff --git a/tests/unit/retrieve-generated-source-evidence.test.ts b/tests/unit/retrieve-generated-source-evidence.test.ts index ea5f19b6..4d95e7a8 100644 --- a/tests/unit/retrieve-generated-source-evidence.test.ts +++ b/tests/unit/retrieve-generated-source-evidence.test.ts @@ -83,7 +83,7 @@ function writeRoundingLayout(root: string, layout: 'same-file' | 'relative-impor helperFile: '/src/engine.js', ownerLocation: 'L3-L6', ownerConst: 'L4: const input = 12.5;', - ownerReturn: 'L5: return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', + ownerReturn: 'L5: return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', } } @@ -100,7 +100,7 @@ function writeRoundingLayout(root: string, layout: 'same-file' | 'relative-impor helperFile: '/src/rounding.js', ownerLocation: 'L2-L5', ownerConst: 'L3: const input = 12.5;', - ownerReturn: 'L4: return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', + ownerReturn: 'L4: return { displayedEstimate: readSavedValue(input), chargedTotal: rebuildValue(input) };', } } diff --git a/tests/unit/retrieve-owner-declarations.test.ts b/tests/unit/retrieve-owner-declarations.test.ts index 814175de..f67fea46 100644 --- a/tests/unit/retrieve-owner-declarations.test.ts +++ b/tests/unit/retrieve-owner-declarations.test.ts @@ -14,6 +14,7 @@ vi.mock('typescript', async (importOriginal) => { import * as ts from 'typescript' import { + completeSmallOwnerSourceEvidence, completeQueryEvidenceLiteralStatement, ownerLocalDeclarationEvidence, queryEvidenceSourceProjection, @@ -2581,4 +2582,28 @@ describe('owner declaration parse snapshot cache', () => { expect(ts.createSourceFile).toHaveBeenCalledTimes(1) }) + + it('reuses the cached AST when authenticating the same complete small owner', () => { + const sourceLines = [ + 'export function cachedOwner() {', + ' return 1', + '}', + ] + retainQueryEvidenceSourceSnapshot({ + sourceFilePath: 'cached-owner.ts', + sourceLines, + sourceText: sourceLines.join('\n'), + }) + const authenticate = () => completeSmallOwnerSourceEvidence({ + sourceFilePath: 'cached-owner.ts', + sourceLines, + ownerRange: { start: 1, end: 3 }, + label: 'cachedOwner()', + nodeKind: 'function', + }) + + expect(authenticate()?.snippet).toContain('L2: return 1') + expect(authenticate()?.snippet).toContain('L2: return 1') + expect(ts.createSourceFile).toHaveBeenCalledTimes(1) + }) }) diff --git a/tests/unit/retrieve-small-owner-representation.test.ts b/tests/unit/retrieve-small-owner-representation.test.ts new file mode 100644 index 00000000..a43f4fc0 --- /dev/null +++ b/tests/unit/retrieve-small-owner-representation.test.ts @@ -0,0 +1,630 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { generateGraph } from '../../src/infrastructure/generate.js' +import { + compactRetrieveResultForStdio, + retrieveContext, + withRetrieveSnippetBudget, + type RetrieveOptions, + type RetrieveResult, +} from '../../src/runtime/retrieve.js' +import { + completeSmallOwnerSourceEvidence, + retainQueryEvidenceSourceSnapshot, +} from '../../src/runtime/query-evidence-dependencies.js' +import { estimateQueryTokens, loadGraph } from '../../src/runtime/serve.js' +import { handleStdioRequest } from '../../src/runtime/stdio-server.js' + +const PRE_SOURCE = `export function parseDecimalRatio(input) { + if (typeof input !== 'string') return null; + const normalized = input.trim(); + if (normalized !== '' && !/^(?:[0-9]+(?:\\.[0-9]+)?|\\.[0-9]+)$/.test(normalized)) return null; + const value = Number(normalized); + return Number.isFinite(value) && value >= 0 && value <= 1 ? value : null; +} +` + +const POST_SOURCE = PRE_SOURCE.replace( + " if (normalized !== ''", + " if (normalized === '') return null;\n if (normalized !== ''", +) + +const PACKAGE_SOURCE = '{"type":"module"}\n' + +const PUBLIC_CALLS = [ + { + itemId: 'item_2', + state: 'pre', + question: 'Return the current source of parseDecimalRatio in parser.js, especially lines 1-7 and the defect span at lines 3-5. Do not include unrelated files.', + budget: 1_200, + snippetBudget: 800, + topNWithSnippet: 5, + }, + { + itemId: 'item_4', + state: 'pre', + question: 'Show the complete implementation text of export function parseDecimalRatio(input) from parser.js, including every source line from the opening brace through the closing brace. Exact source is required.', + budget: 3_000, + snippetBudget: 2_400, + topNWithSnippet: 12, + retrievalLevel: 5, + retrievalStrategy: 'slice-v1', + }, + { + itemId: 'item_5', + state: 'pre', + question: 'In parser.js parseDecimalRatio, return the exact source statements that reject non-string input, trim input, validate the ASCII decimal regular-expression grammar, and convert normalized input with Number(). Include source snippets verbatim.', + budget: 3_000, + snippetBudget: 2_400, + topNWithSnippet: 20, + retrievalLevel: 5, + }, + { + itemId: 'item_6', + state: 'pre', + question: 'What is the exact regular-expression validation source line in parser.js immediately after `const normalized = input.trim();` and before `const value = Number(normalized);`? Return that parser.js line verbatim.', + budget: 1_600, + snippetBudget: 1_200, + topNWithSnippet: 12, + retrievalLevel: 5, + }, + { + itemId: 'item_12', + state: 'post', + question: "Return the current post-edit source of parseDecimalRatio in parser.js. Specifically include the explicit trimmed-empty guard `if (normalized === '') return null;` and surrounding lines so its presence can be verified.", + budget: 2_400, + snippetBudget: 1_800, + topNWithSnippet: 12, + retrievalLevel: 5, + }, + { + itemId: 'item_14', + state: 'post', + question: "Return parser.js line 4 verbatim from the current source. It is the trimmed-empty conditional inside parseDecimalRatio between `const normalized = input.trim();` on line 3 and the regex validation on line 5.", + budget: 2_400, + snippetBudget: 1_800, + topNWithSnippet: 20, + retrievalLevel: 5, + }, +] as const + +type PublicState = (typeof PUBLIC_CALLS)[number]['state'] + +interface PublicFixture { + graph: ReturnType + graphPath: string + fullSnippet: string +} + +interface PublicResult { + itemId: string + state: PublicState + raw: RetrieveResult + compact: ReturnType +} + +const roots: string[] = [] +const fixtures = new Map() + +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +function numberedOwner(source: string): string { + return source.trimEnd().split('\n').map((line, index) => `L${index + 1}: ${line}`).join('\n') +} + +function ownerSnippet(result: { + matched_nodes: ReadonlyArray<{ label: string; snippet: string | null }> +}): string | null | undefined { + const matches = result.matched_nodes.filter((node) => node.label === 'parseDecimalRatio()') + expect(matches, 'generated parseDecimalRatio owner must be uniquely retrieved').toHaveLength(1) + return matches[0]?.snippet +} + +function matchedByLabel( + result: { matched_nodes: ReadonlyArray<{ + label: string + snippet: string | null + snippet_truncated?: boolean + }> }, + label: string, +) { + const matches = result.matched_nodes.filter((node) => node.label === label) + expect(matches, `expected one retrieved ${label}`).toHaveLength(1) + return matches[0]! +} + +function numberedLines(source: string, startLine = 1): string { + const lineEnding = source.includes('\r\n') ? '\r\n' : '\n' + const withoutTerminalDelimiter = source.endsWith(lineEnding) + ? source.slice(0, -lineEnding.length) + : source + return withoutTerminalDelimiter + .split(lineEnding) + .map((line, offset) => `L${startLine + offset}: ${line}`) + .join(lineEnding) +} + +function generatedFixture(source: string, relativePath = 'src/sample.ts') { + const root = mkdtempSync(join(tmpdir(), 'small-owner-generic-')) + roots.push(root) + const directory = join(root, 'src') + mkdirSync(directory, { recursive: true }) + writeFileSync(join(root, relativePath), source, 'utf8') + writeFileSync(join(root, 'package.json'), PACKAGE_SOURCE, 'utf8') + const generated = generateGraph(root, { noHtml: true }) + return { + root, + sourceFile: join(root, relativePath), + graph: loadGraph(generated.graphPath), + } +} + +function completeFromText(input: { + source: string + ownerRange: { start: number; end: number } + label: string + nodeKind?: string + externalCall?: boolean + sourceFilePath?: string +}) { + const sourceFilePath = input.sourceFilePath ?? 'sample.ts' + const sourceLines = input.source.split(/\r?\n/) + retainQueryEvidenceSourceSnapshot({ sourceFilePath, sourceLines, sourceText: input.source }) + return completeSmallOwnerSourceEvidence({ + sourceFilePath, + sourceLines, + ownerRange: input.ownerRange, + label: input.label, + ...(input.nodeKind ? { nodeKind: input.nodeKind } : {}), + ...(input.externalCall !== undefined ? { externalCall: input.externalCall } : {}), + }) +} + +function createPublicFixture(state: PublicState, source: string, expectedHash: string): PublicFixture { + expect(Buffer.byteLength(source)).toBe(state === 'pre' ? 334 : 372) + expect(sha256(source)).toBe(expectedHash) + expect(Buffer.byteLength(PACKAGE_SOURCE)).toBe(18) + expect(sha256(PACKAGE_SOURCE)).toBe('1239d4d885dcad42201a27ed9324f8f0f760b78700d8db9ced39a511cffe7eae') + + const root = mkdtempSync(join(tmpdir(), `small-owner-public-${state}-`)) + roots.push(root) + mkdirSync(root, { recursive: true }) + writeFileSync(join(root, 'parser.js'), source, 'utf8') + writeFileSync(join(root, 'package.json'), PACKAGE_SOURCE, 'utf8') + const generated = generateGraph(root, { noHtml: true }) + return { + graph: loadGraph(generated.graphPath), + graphPath: generated.graphPath, + fullSnippet: numberedOwner(source), + } +} + +beforeAll(() => { + fixtures.set('pre', createPublicFixture( + 'pre', + PRE_SOURCE, + 'e7c2028b74b2d7c1c094af7fa284c68acb76447202196de64b8255857215a676', + )) + fixtures.set('post', createPublicFixture( + 'post', + POST_SOURCE, + '6d003964273fecef9bf750995bea5a59c7cc8d29235834aa9e4a64d4e6418020', + )) +}) + +afterAll(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('complete small-owner source representation', () => { + it('returns every physical owner line for all six frozen public parser requests', () => { + const results: PublicResult[] = PUBLIC_CALLS.map((call) => { + const fixture = fixtures.get(call.state)! + const options: RetrieveOptions = { + question: call.question, + budget: call.budget, + ...('retrievalLevel' in call ? { retrievalLevel: call.retrievalLevel } : {}), + ...('retrievalStrategy' in call ? { retrievalStrategy: call.retrievalStrategy } : {}), + } + const retrieved = retrieveContext(fixture.graph, options) + const snippetOptions = { + snippetBudget: call.snippetBudget, + topNWithSnippet: call.topNWithSnippet, + } + return { + itemId: call.itemId, + state: call.state, + raw: withRetrieveSnippetBudget(retrieved, snippetOptions), + compact: compactRetrieveResultForStdio(retrieved, snippetOptions), + } + }) + + for (const result of results) { + const expected = fixtures.get(result.state)!.fullSnippet + expect.soft(ownerSnippet(result.raw), `${result.itemId} raw complete owner`).toBe(expected) + expect.soft(ownerSnippet(result.compact), `${result.itemId} compact complete owner`).toBe(expected) + } + }, 120_000) + + it('carries the complete owner through the actual verbose stdio retrieve handler', async () => { + const fixture = fixtures.get('post')! + const call = PUBLIC_CALLS[4] + const response = await Promise.resolve(handleStdioRequest(fixture.graphPath, { + id: 12, + method: 'tools/call', + params: { + name: 'retrieve', + arguments: { + question: call.question, + budget: call.budget, + snippet_budget: call.snippetBudget, + top_n_with_snippet: call.topNWithSnippet, + retrieval_level: call.retrievalLevel, + verbose: true, + }, + }, + })) + const text = (response as { result?: { content?: Array<{ text?: string }> } }) + .result?.content?.[0]?.text + const payload = JSON.parse(text ?? '{}') as RetrieveResult + + expect(ownerSnippet(payload)).toBe(fixture.fullSnippet) + expect(JSON.stringify(payload)).toContain("L4: if (normalized === '') return null;") + }, 120_000) + + it('represents renamed function and method owners from generated graphs', () => { + const source = [ + 'export function renamedFlow(input: string) {', + ' // retain this decision comment', + " if (input === '') return null;", + ' const normalized = input.trim();', + ' auditTrail.push(normalized);', + ' return normalized;', + '}', + 'export const arrowOwner = (value: number) => {', + ' if (value < 0) return 0;', + ' return value + 1;', + '};', + 'export const expressionOwner = function (value: number) {', + ' record(value);', + ' return value * 2;', + '};', + 'export class RenamedProcessor {', + ' calculate(value: number) {', + ' if (!value) return 0;', + ' this.seen = value;', + ' return value;', + ' }', + '}', + ].join('\n') + '\n' + const fixture = generatedFixture(source) + const cases = [ + { label: 'renamedFlow()', start: 1, end: 7 }, + { label: '.calculate()', start: 17, end: 21 }, + ] + const generatedLabels = fixture.graph.nodeEntries().map(([, attributes]) => attributes.label) + + for (const owner of cases) { + expect(generatedLabels, `generated labels for ${owner.label}`).toContain(owner.label) + const result = retrieveContext(fixture.graph, { + question: `How does ${owner.label} process and return the value? Return its exact complete source implementation in sample.ts`, + budget: 4_000, + retrievalLevel: 5, + }) + const expected = source + .trimEnd() + .split('\n') + .slice(owner.start - 1, owner.end) + .map((line, offset) => `L${owner.start + offset}: ${line}`) + .join('\n') + expect(matchedByLabel(result, owner.label).snippet, owner.label).toBe(expected) + } + }, 120_000) + + it.each([ + { + name: 'arrow binding', + source: 'export const renamedBinding = (value: number) => {\n return value + 1;\n};', + label: 'renamedBinding', + }, + { + name: 'function-expression binding', + source: 'export const renamedBinding = function (value: number) {\n return value + 1;\n};', + label: 'renamedBinding()', + }, + ])('resolves a supported $name by its AST binding identity', ({ source, label }) => { + expect(completeFromText({ + source, + ownerRange: { start: 1, end: 3 }, + label, + nodeKind: 'function', + })?.snippet).toBe(numberedLines(source)) + }) + + it('keeps nested, adjacent, and same-line owners bounded to their own declarations', () => { + const source = [ + 'export function outerOwner(value: number) {', + ' function nestedOwner() { return value + 1; }', + ' return nestedOwner();', + '}', + 'export function adjacentLeft() { return 1; } export function adjacentRight() { return 2; }', + 'export function siblingCall(value: number) { return Math.floor(value); } notifySibling();', + ].join('\n') + '\n' + const fixture = generatedFixture(source) + + const expected = new Map([ + ['outerOwner()', numberedLines(source.split('\n').slice(0, 4).join('\n'))], + ['nestedOwner()', 'L2: function nestedOwner() { return value + 1; }'], + ['adjacentLeft()', 'L5: export function adjacentLeft() { return 1; }'], + ['adjacentRight()', 'L5: export function adjacentRight() { return 2; }'], + ['siblingCall()', 'L6: export function siblingCall(value: number) { return Math.floor(value); }'], + ]) + for (const [label, snippet] of expected) { + const result = retrieveContext(fixture.graph, { + question: `Return exact complete source for ${label}`, + budget: 4_000, + retrievalLevel: 5, + }) + expect(matchedByLabel(result, label).snippet).toBe(snippet) + } + }, 120_000) + + it('rejects synthetic, mismatched, foreign, malformed, and unsupported owner authority', () => { + const source = 'export function trustedOwner(value: number) { return Math.floor(value); }\n' + const valid = { + source, + ownerRange: { start: 1, end: 1 }, + label: 'trustedOwner()', + nodeKind: 'function', + } + + expect(completeFromText(valid)?.snippet).toBe( + 'L1: export function trustedOwner(value: number) { return Math.floor(value); }', + ) + expect(completeFromText({ ...valid, label: 'Math.floor', externalCall: true })).toBeNull() + expect(completeFromText({ ...valid, label: 'misleadingOwner()' })).toBeNull() + expect(completeFromText({ ...valid, nodeKind: 'class' })).toBeNull() + expect(completeFromText({ ...valid, ownerRange: { start: 1, end: 2 } })).toBeNull() + expect(completeFromText({ ...valid, sourceFilePath: 'sample.go' })).toBeNull() + expect(completeFromText({ + source: 'export function unfinished(value: number): number;\n', + ownerRange: { start: 1, end: 1 }, + label: 'unfinished()', + nodeKind: 'function', + })).toBeNull() + expect(completeFromText({ + source: 'export function malformed( {\n', + ownerRange: { start: 1, end: 1 }, + label: 'malformed()', + nodeKind: 'function', + })).toBeNull() + }) + + it('admits exactly 25 physical lines and rejects 26', () => { + const owner = (lineCount: number) => [ + 'export function lineBoundary() {', + ...Array.from({ length: lineCount - 2 }, (_, index) => ` void ${index};`), + '}', + ].join('\n') + + expect(completeFromText({ + source: owner(25), + ownerRange: { start: 1, end: 25 }, + label: 'lineBoundary()', + nodeKind: 'function', + })?.snippet.split('\n')).toHaveLength(25) + expect(completeFromText({ + source: owner(26), + ownerRange: { start: 1, end: 26 }, + label: 'lineBoundary()', + nodeKind: 'function', + })).toBeNull() + }) + + it('admits exactly 2000 original owner characters and rejects 2001', () => { + const owner = (characterCount: number) => { + const prefix = 'export function characterBoundary() {\n /*' + const suffix = '*/\n}' + return `${prefix}${'x'.repeat(characterCount - prefix.length - suffix.length)}${suffix}` + } + const exact = owner(2_000) + const tooLarge = owner(2_001) + expect(exact).toHaveLength(2_000) + expect(tooLarge).toHaveLength(2_001) + expect(completeFromText({ + source: exact, + ownerRange: { start: 1, end: 3 }, + label: 'characterBoundary()', + nodeKind: 'function', + })).not.toBeNull() + expect(completeFromText({ + source: tooLarge, + ownerRange: { start: 1, end: 3 }, + label: 'characterBoundary()', + nodeKind: 'function', + })).toBeNull() + }) + + it('preserves LF and CRLF bytes through multiline literals, regex, escapes, and interpolation', () => { + for (const lineEnding of ['\n', '\r\n'] as const) { + const source = [ + 'export function literalOwner(name: string) {', + ' const escaped = "a\\\\b\\\"c";', + ' const pattern = /^(?:a\\/b|c\\s+)$/;', + ' const message = ` hello ${name}', + ' continued `;', + ' return pattern.test(escaped) ? message : null;', + '}', + ].join(lineEnding) + const evidence = completeFromText({ + source, + ownerRange: { start: 1, end: 7 }, + label: 'literalOwner()', + nodeKind: 'function', + }) + expect(evidence?.snippet).toBe(numberedLines(source)) + expect(evidence?.snippet).toContain('const escaped = "a\\\\b\\\"c";') + expect(evidence?.snippet).toContain('/^(?:a\\/b|c\\s+)$/') + expect(evidence?.snippet).toContain(`\${name}${lineEnding}L5: continued \`;`) + } + }) + + it('uses current cached source instead of a stale stored graph snippet', () => { + const oldSource = [ + 'export function currentOwner(value: string) {', + " if (value === 'old') return null;", + ' return value;', + '}', + ].join('\n') + '\n' + const fixture = generatedFixture(oldSource) + const currentSource = oldSource.replace("value === 'old'", "value === 'new'") + writeFileSync(fixture.sourceFile, currentSource, 'utf8') + + const result = retrieveContext(fixture.graph, { + question: 'Return exact complete currentOwner source', + budget: 4_000, + retrievalLevel: 5, + }) + const snippet = matchedByLabel(result, 'currentOwner()').snippet + expect(snippet).toBe(numberedLines(currentSource)) + expect(snippet).not.toContain("value === 'old'") + }, 120_000) + + it('keeps complete owners atomic at exact and one-token-less snippet budgets', () => { + const fixture = fixtures.get('pre')! + const call = PUBLIC_CALLS[1] + const fresh = () => retrieveContext(fixture.graph, { + question: call.question, + budget: call.budget, + retrievalLevel: 5, + retrievalStrategy: 'slice-v1', + }) + const full = fresh() + const fullSnippet = ownerSnippet(full)! + const exactCost = estimateQueryTokens(fullSnippet) + + const exact = withRetrieveSnippetBudget(full, { + snippetBudget: exactCost, + topNWithSnippet: 12, + }) + expect(ownerSnippet(exact)).toBe(fullSnippet) + expect(exact.snippet_budget_tokens_used).toBe(exactCost) + expect(matchedByLabel(exact, 'parseDecimalRatio()').snippet_truncated).toBe(false) + + const tight = withRetrieveSnippetBudget(fresh(), { + snippetBudget: exactCost - 1, + topNWithSnippet: 12, + }) + expect(ownerSnippet(tight)).not.toBe(fullSnippet) + expect(ownerSnippet(tight)).not.toBeNull() + expect(matchedByLabel(tight, 'parseDecimalRatio()').snippet_truncated).toBe(true) + expect((matchedByLabel(tight, 'parseDecimalRatio()') as { representation_reason?: string }) + .representation_reason).not.toBe('complete small owner source') + expect(tight.snippet_budget_tokens_used).toBe(estimateQueryTokens(ownerSnippet(tight)!)) + + const restored = withRetrieveSnippetBudget(tight, { + snippetBudget: exactCost, + topNWithSnippet: 12, + }) + expect(ownerSnippet(restored)).toBe(fullSnippet) + expect(matchedByLabel(restored, 'parseDecimalRatio()').snippet_truncated).toBe(true) + + const roundTrip = JSON.parse(JSON.stringify(restored)) as RetrieveResult + expect(Object.keys(matchedByLabel(roundTrip, 'parseDecimalRatio()')) + .some((key) => /complete|fallback|owner/i.test(key))).toBe(false) + const reshapedRoundTrip = withRetrieveSnippetBudget(roundTrip, { + snippetBudget: exactCost, + topNWithSnippet: 12, + }) + expect(ownerSnippet(reshapedRoundTrip)).toBe(fullSnippet) + expect(matchedByLabel(reshapedRoundTrip, 'parseDecimalRatio()').snippet_truncated).toBe(true) + }, 120_000) + + it('omits atomic owners safely for zero snippet budget and top-N', () => { + const fixture = fixtures.get('pre')! + const fresh = () => retrieveContext(fixture.graph, { + question: PUBLIC_CALLS[1].question, + budget: 3_000, + retrievalLevel: 5, + retrievalStrategy: 'slice-v1', + }) + for (const options of [ + { snippetBudget: 0, topNWithSnippet: 12 }, + { snippetBudget: 2_400, topNWithSnippet: 0 }, + ]) { + const shaped = withRetrieveSnippetBudget(fresh(), options) + expect(ownerSnippet(shaped)).toBeNull() + expect(matchedByLabel(shaped, 'parseDecimalRatio()').snippet_truncated).toBe(true) + expect(shaped.snippet_budget_tokens_used).toBe(0) + } + }, 120_000) + + it('reconciles two complete owners against one shared snippet budget', () => { + const source = [ + 'export function amberOwner(value: string) {', + " if (value === '') return null;", + ' return value.trim();', + '}', + 'export function cobaltOwner(value: string) {', + " if (value === '') return null;", + ' return value.toUpperCase();', + '}', + ].join('\n') + '\n' + const fixture = generatedFixture(source) + const fresh = retrieveContext(fixture.graph, { + question: 'Return complete exact source of amberOwner and cobaltOwner', + budget: 4_000, + retrievalLevel: 5, + }) + const owners = fresh.matched_nodes.filter((node) => ( + node.label === 'amberOwner()' || node.label === 'cobaltOwner()' + )) + expect(owners).toHaveLength(2) + const costs = owners.map((node) => estimateQueryTokens(node.snippet!)) + const budget = costs[0]! + costs[1]! - 1 + const shaped = withRetrieveSnippetBudget(fresh, { + snippetBudget: budget, + topNWithSnippet: fresh.matched_nodes.length, + }) + const shapedOwners = shaped.matched_nodes.filter((node) => ( + node.label === 'amberOwner()' || node.label === 'cobaltOwner()' + )) + expect(shapedOwners[0]?.snippet).toBe(owners[0]?.snippet) + expect(shapedOwners[1]?.snippet).not.toBe(owners[1]?.snippet) + expect(shapedOwners[1]?.snippet_truncated).toBe(true) + expect(shaped.snippet_budget_tokens_used).toBe( + shaped.matched_nodes.reduce((total, node) => total + estimateQueryTokens(node.snippet ?? ''), 0), + ) + expect(shaped.snippet_budget_tokens_used).toBeLessThanOrEqual(budget) + }, 120_000) + + it('preserves complete owners and truncation history through compact stdio shaping', () => { + const fixture = fixtures.get('post')! + const retrieved = retrieveContext(fixture.graph, { + question: PUBLIC_CALLS[4].question, + budget: 2_400, + retrievalLevel: 5, + }) + const fullSnippet = ownerSnippet(retrieved)! + const exactCost = estimateQueryTokens(fullSnippet) + const compact = compactRetrieveResultForStdio(retrieved, { + snippetBudget: exactCost - 1, + topNWithSnippet: 12, + }) + const compactOwner = matchedByLabel(compact, 'parseDecimalRatio()') + expect(compactOwner.snippet).not.toBe(fullSnippet) + expect(compactOwner.snippet_truncated).toBe(true) + expect(compact.snippet_budget_tokens_used).toBe( + compact.matched_nodes.reduce((total, node) => total + estimateQueryTokens(node.snippet ?? ''), 0), + ) + }, 120_000) +}) From c36dfdf1ed4f2de3faf20ae39ffe9045a8dd5e31 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 08:33:03 +0400 Subject: [PATCH 25/27] test: normalize generated source paths on Windows --- tests/unit/retrieve-generated-source-evidence.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/retrieve-generated-source-evidence.test.ts b/tests/unit/retrieve-generated-source-evidence.test.ts index 4d95e7a8..728b3968 100644 --- a/tests/unit/retrieve-generated-source-evidence.test.ts +++ b/tests/unit/retrieve-generated-source-evidence.test.ts @@ -42,7 +42,9 @@ function nodesFromGraph(graph: ReturnType): GeneratedNode[] { } function oneNode(nodes: readonly GeneratedNode[], label: string, relativePath: string): GeneratedNode { - const matches = nodes.filter((node) => node.label === label && node.source_file.endsWith(relativePath)) + const matches = nodes.filter( + (node) => node.label === label && node.source_file.replaceAll('\\', '/').endsWith(relativePath), + ) expect(matches, `expected one ${label} node in ${relativePath}`).toHaveLength(1) return matches[0]! } From 5e84791d8aba06b4653629af38ca32b8dec0d90c Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 08:54:26 +0400 Subject: [PATCH 26/27] fix: clear complete owner claims after serialized clipping --- src/runtime/retrieve.ts | 8 +- ...etrieve-small-owner-representation.test.ts | 85 +++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/runtime/retrieve.ts b/src/runtime/retrieve.ts index 37df297d..de39938a 100644 --- a/src/runtime/retrieve.ts +++ b/src/runtime/retrieve.ts @@ -562,7 +562,7 @@ function applyRetrieveSnippetBudgetToNodes }, label: string, ) { @@ -549,6 +551,89 @@ describe('complete small-owner source representation', () => { expect(matchedByLabel(reshapedRoundTrip, 'parseDecimalRatio()').snippet_truncated).toBe(true) }, 120_000) + it('clears complete-owner metadata after serialized generic shaping clips or omits public source', () => { + const fixture = fixtures.get('post')! + const retrieved = retrieveContext(fixture.graph, { + question: PUBLIC_CALLS[4].question, + budget: PUBLIC_CALLS[4].budget, + retrievalLevel: 5, + }) + const fullSnippet = fixture.fullSnippet + const exactCost = estimateQueryTokens(fullSnippet) + const serialized = JSON.parse(JSON.stringify(retrieved)) as RetrieveResult + const serializedOwner = matchedByLabel(serialized, 'parseDecimalRatio()') + + expect(exactCost).toBe(130) + expect(serializedOwner.snippet).toBe(fullSnippet) + expect(serializedOwner.representation_reason).toBe('complete small owner source') + expect(Object.keys(serializedOwner).some((key) => /complete|fallback|owner/i.test(key))).toBe(false) + + const exact = withRetrieveSnippetBudget(serialized, { + snippetBudget: exactCost, + topNWithSnippet: 12, + }) + const exactOwner = matchedByLabel(exact, 'parseDecimalRatio()') + expect(exactOwner.snippet).toBe(fullSnippet) + expect(exactOwner.snippet_truncated).toBe(false) + expect(exactOwner.representation_reason).toBe('complete small owner source') + expect(exact.snippet_budget_tokens_used).toBe(exactCost) + + const clipped = withRetrieveSnippetBudget(serialized, { + snippetBudget: exactCost - 1, + topNWithSnippet: 12, + }) + const clippedOwner = matchedByLabel(clipped, 'parseDecimalRatio()') + expect(clippedOwner.snippet).toBe(fullSnippet.split('\n').slice(1).join('\n')) + expect(clippedOwner.snippet_truncated).toBe(true) + expect(clippedOwner.representation_reason).toBeUndefined() + expect(clippedOwner.representation_type).toBe('detail') + expect(clipped.snippet_budget_tokens_used).toBe( + clipped.matched_nodes.reduce((total, node) => ( + total + (typeof node.snippet === 'string' && node.snippet.trim().length > 0 + ? estimateQueryTokens(node.snippet) + : 0) + ), 0), + ) + expect(clipped.snippet_budget_tokens_used).toBe(exactCost - 1) + + const clippedAgain = withRetrieveSnippetBudget(clipped, { + snippetBudget: exactCost - 1, + topNWithSnippet: 12, + }) + const clippedAgainOwner = matchedByLabel(clippedAgain, 'parseDecimalRatio()') + expect(clippedAgainOwner.snippet).toBe(clippedOwner.snippet) + expect(clippedAgainOwner.snippet_truncated).toBe(true) + expect(clippedAgainOwner.representation_reason).toBeUndefined() + + for (const { options, expectedTruncation } of [ + { options: { snippetBudget: 0, topNWithSnippet: 12 }, expectedTruncation: true }, + { options: { snippetBudget: exactCost, topNWithSnippet: 0 }, expectedTruncation: false }, + ]) { + const omitted = withRetrieveSnippetBudget(serialized, options) + const omittedOwner = matchedByLabel(omitted, 'parseDecimalRatio()') + expect(omittedOwner.snippet).toBeNull() + expect(omittedOwner.snippet_truncated).toBe(expectedTruncation) + expect(omittedOwner.representation_reason).toBeUndefined() + expect(omitted.snippet_budget_tokens_used).toBe(0) + + const omittedAgain = withRetrieveSnippetBudget(omitted, options) + const omittedAgainOwner = matchedByLabel(omittedAgain, 'parseDecimalRatio()') + expect(omittedAgainOwner.snippet).toBeNull() + expect(omittedAgainOwner.snippet_truncated).toBe(expectedTruncation) + expect(omittedAgainOwner.representation_reason).toBeUndefined() + expect(omittedAgain.snippet_budget_tokens_used).toBe(0) + } + + const ordinary = JSON.parse(JSON.stringify(serialized)) as RetrieveResult + matchedByLabel(ordinary, 'parseDecimalRatio()').representation_reason = 'signature compression' + const ordinaryClipped = withRetrieveSnippetBudget(ordinary, { + snippetBudget: exactCost - 1, + topNWithSnippet: 12, + }) + expect(matchedByLabel(ordinaryClipped, 'parseDecimalRatio()').representation_reason) + .toBe('signature compression') + }, 120_000) + it('omits atomic owners safely for zero snippet budget and top-N', () => { const fixture = fixtures.get('pre')! const fresh = () => retrieveContext(fixture.graph, { From b3219e0532e802d4df15536e0ccecf68901c14e7 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 8 Sep 2026 08:54:26 +0400 Subject: [PATCH 27/27] test: retain final membership mutation controls --- scripts/lib/semantic-independence-selftest.mjs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/lib/semantic-independence-selftest.mjs b/scripts/lib/semantic-independence-selftest.mjs index 6762a838..687870c0 100644 --- a/scripts/lib/semantic-independence-selftest.mjs +++ b/scripts/lib/semantic-independence-selftest.mjs @@ -92,7 +92,22 @@ const RETRIEVE_ANCHOR = ` let orderedCandidates = inclusionOrder * the pack's own budgeted selection, so an injection there can be discarded and * prove nothing -- which is exactly what the membership premise caught. */ -const MEMBERSHIP_ANCHOR = ` const matchedNodes = pack.nodes as RetrieveMatchedNode[] +const MEMBERSHIP_ANCHOR = ` const matchedNodes = packedNodes.map((node) => { + const state = typeof node.node_id === 'string' + ? completeOwnerStatesByNodeId.get(node.node_id) + : undefined + if (!state) { + return node + } + return attachCompleteOwnerState({ + ...node, + snippet: state.fullSnippet, + snippet_line_number: state.fullLineNumber, + snippet_scope: 'symbol' as const, + representation_type: 'detail' as const, + representation_reason: COMPLETE_OWNER_REPRESENTATION_REASON, + }, state) + }) ` function runOwningTest(root, testNameFilter, extraEnv = {}) {