diff --git a/README.md b/README.md index 37aa2bd..f97a1df 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ greplica config greplica doctor [--check-embeddings] greplica graph read greplica graph context "" [--debug] -greplica graph audit anchors +greplica graph audit anchors [--invalidate] greplica graph view [--out ] [--no-open] greplica graph export greplica transcript bundle --platform codex|claude|copilot --file [--file ...] --out @@ -133,6 +133,7 @@ greplica proposal apply - `greplica graph context ""` - returns Markdown for agent use. Add `--debug` for the full retrieval payload with ranking signals. - `greplica graph read` - prints the current graph view: all components, flows, claims, sources, and edges in scope. +- `greplica graph audit anchors` - reports `code_verified` claims whose code anchors no longer resolve (drift). Add `--invalidate` to auto-demote fully-drifted claims to `truth: unknown` (non-destructively, via supersession) and record why in the invalidation log. Without the flag it is report-only, so it doubles as a dry run. - `greplica graph view` to visualise the current memory in a local HTML, opens in your default browser. Use `--out` to choose where the file is written; by default it goes to a temp path. - `greplica transcript bundle` - converts one or more Codex, Claude Code, or GitHub Copilot CLI JSONL transcripts into a sanitized Markdown bundle for `greplica-fast-session-bootstrap`. - `greplica doctor` - verifies installation and diagnoses configuration failures. Not a required preflight before every command. diff --git a/apps/cli/main.ts b/apps/cli/main.ts index a74bdeb..67ce52c 100644 --- a/apps/cli/main.ts +++ b/apps/cli/main.ts @@ -5,7 +5,7 @@ import { isatty } from "node:tty"; import { basename, dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { createLocalKnowledgeGraphService, KnowledgeGraphService } from "../../libs/knowledge-graph/service.js"; -import type { ClaimAnchorAuditResult, RepoRef } from "../../libs/knowledge-graph/service.js"; +import type { AnchorInvalidationResult, ClaimAnchorAuditResult, RepoRef } from "../../libs/knowledge-graph/service.js"; import { envVarSource, loadRepoEnv, type LoadedRepoEnv } from "../../libs/env/load-local-env.js"; import { ensureGreplicaConfig, @@ -100,7 +100,7 @@ const cliCommands = [ { key: "graphAuditAnchors", path: ["graph", "audit", "anchors"], - usage: "graph audit anchors", + usage: "graph audit anchors [--invalidate]", handler: runGraphAuditAnchorsCommand, showInTopLevelHelp: true, }, @@ -271,11 +271,53 @@ async function runGraphContextCommand(args: string[]): Promise { } } -async function runGraphAuditAnchorsCommand(_args: string[]): Promise { +async function runGraphAuditAnchorsCommand(args: string[]): Promise { + const options = parseAuditAnchorsArgs(args); const { repo, service } = createCommandContext(); - const result = await service.auditCodeAnchors(repo); - printAnchorAudit(result); - if (anchorAuditIssueCount(result) > 0) process.exitCode = 1; + + // Always show the diagnostic report first; --invalidate then demotes the + // fully-drifted claims. Report-only runs keep the exit-1 CI signal. + const audit = await service.auditCodeAnchors(repo); + printAnchorAudit(audit); + + if (!options.invalidate) { + if (anchorAuditIssueCount(audit) > 0) process.exitCode = 1; + return; + } + + const result = await service.invalidateDriftedAnchors(repo); + printAnchorInvalidation(result); +} + +function parseAuditAnchorsArgs(args: string[]): { invalidate: boolean } { + let invalidate = false; + for (const arg of args) { + if (arg === "--invalidate") { + invalidate = true; + continue; + } + throw new Error(usage("graphAuditAnchors")); + } + return { invalidate }; +} + +function printAnchorInvalidation(result: AnchorInvalidationResult): void { + console.log(""); + if (result.invalidated.length === 0) { + console.log("No drifted claims to invalidate."); + } else { + console.log(`Invalidated ${result.invalidated.length} drifted claim(s):`); + for (const record of result.invalidated) { + console.log(`- ${record.claim_id} -> ${record.superseding_claim_id} (${record.broken_anchor}, ${record.resolver_status})`); + } + console.log(`Memory commit: ${result.memory_commit_id}`); + } + + if (result.errors.length > 0) { + console.log(""); + console.log(`Skipped ${result.errors.length} claim(s) due to resolver errors:`); + for (const error of result.errors) console.log(`- ${error.claim_id}: ${error.message}`); + } } function runGraphExportCommand(args: string[]): void { diff --git a/evals/ranking-optimizer/train.ts b/evals/ranking-optimizer/train.ts index 3737249..d43cd4c 100644 --- a/evals/ranking-optimizer/train.ts +++ b/evals/ranking-optimizer/train.ts @@ -328,6 +328,7 @@ function toClaimResults(ranked: RankedContextDocument[], config: GraphContextCon about: document.document.about, evidence: [], code_anchors: [], + freshness: { state: "fresh", reason: null, broken: [] }, }; }); } diff --git a/libs/config/greplica-config.ts b/libs/config/greplica-config.ts index e6c9374..5681000 100644 --- a/libs/config/greplica-config.ts +++ b/libs/config/greplica-config.ts @@ -22,6 +22,7 @@ export interface SessionConfig { timeThresholdMinutes: number; currentGraceMinutes: number; autoMemoryUpdates: boolean; + autoHealDrift: boolean; } export interface EmbeddingConfigInput { @@ -51,6 +52,7 @@ export const defaultSessionConfig: SessionConfig = { timeThresholdMinutes: 40, currentGraceMinutes: 5, autoMemoryUpdates: true, + autoHealDrift: true, }; export const defaultGreplicaConfig: GreplicaConfig = { @@ -158,6 +160,7 @@ function normalizeSessionConfig(value: unknown, path: string): SessionConfig { path, ), autoMemoryUpdates: parseBoolean(value.autoMemoryUpdates, defaultSessionConfig.autoMemoryUpdates, "session.autoMemoryUpdates", path), + autoHealDrift: parseBoolean(value.autoHealDrift, defaultSessionConfig.autoHealDrift, "session.autoHealDrift", path), }; } diff --git a/libs/hooks/worker.ts b/libs/hooks/worker.ts index 7b32f29..eb9fb4e 100644 --- a/libs/hooks/worker.ts +++ b/libs/hooks/worker.ts @@ -1,16 +1,21 @@ import { spawn } from "node:child_process"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import type { ClaimedMemoryUpdateAttempt } from "./session-state.js"; import { HookSessionStore } from "./session-state.js"; import { WorkerLease } from "../utils/worker-lease.js"; import { ensureGreplicaConfig, type GreplicaConfig } from "../config/greplica-config.js"; import { platformInstaller } from "../install/platforms/index.js"; +import type { PlatformInstaller } from "../install/platforms/types.js"; import { openDatabase } from "../storage/sqlite/db.js"; +import { SqliteRepository } from "../storage/sqlite/repository.js"; +import { KnowledgeGraphService, type HealResult, type RepoRef } from "../knowledge-graph/service.js"; +import type { Claim } from "../knowledge-graph/claim.js"; const hookWorkerLockName = "hook-memory-update-worker"; const hookWorkerHeartbeatMs = 60 * 1000; +const reverifyLimit = 3; // ponytail: fixed per-cycle cap; make it config only if repos need different limits. export function startHookWorker(): void { const script = process.argv[1]; @@ -54,6 +59,15 @@ export async function runHookWorker(): Promise { if (!leaseValid || !lease.renew()) return; await maybeUpdateWorkingMemory(attempt); } + + if (leaseValid) { + const service = new KnowledgeGraphService(new SqliteRepository(db)); + const renew = () => leaseValid && lease.renew(); + await runDriftHealPass(service, activeRepoRefs(attempts), config.session.autoHealDrift, renew); + if (config.session.autoHealDrift) { + await runReverifyForActiveRepos(service, attempts, reverifyLimit, renew); + } + } } finally { if (heartbeat !== undefined) clearInterval(heartbeat); if (acquired) lease.release(); @@ -61,6 +75,51 @@ export async function runHookWorker(): Promise { } } +interface DriftHealer { + healDriftedAnchorsFromCheckpoint(input: RepoRef): Promise; +} + +/** One minimal RepoRef per distinct active repo root; repo_name/default_branch are unused by the heal. */ +function activeRepoRefs(attempts: ClaimedMemoryUpdateAttempt[]): RepoRef[] { + const refs: RepoRef[] = []; + for (const attempt of attempts) { + const cwd = attempt.session.cwd; + if (cwd !== null) refs.push({ repo_root: cwd, repo_name: basename(cwd), default_branch: "main" }); + } + return refs; +} + +/** + * Heal drift for each distinct repo among the active sessions. Deduped by repo + * root, gated by `autoHealDrift`, and best-effort — a failure on one repo never + * breaks the worker. Stops early if the lease is lost. + */ +export async function runDriftHealPass( + service: DriftHealer, + refs: RepoRef[], + autoHealDrift: boolean, + renewLease: () => boolean = () => true, + log: (summary: Record) => void = (summary) => console.error(JSON.stringify(summary)), +): Promise { + if (!autoHealDrift) return; + const seen = new Set(); + for (const ref of refs) { + const key = ref.repo_root ?? ref.repo_name; + if (seen.has(key)) continue; + seen.add(key); + if (!renewLease()) return; + try { + const result = await service.healDriftedAnchorsFromCheckpoint(ref); + if (result.rechecked > 0 || result.demoted.length > 0) { + log({ event: "freshness_heal", repo: ref.repo_name, rechecked: result.rechecked, demoted: result.demoted.length }); + } + } catch (error) { + // Best-effort: one repo's failure must not abort the pass. Log, keep going. + log({ event: "freshness_heal_error", repo: ref.repo_name, error: errorMessage(error) }); + } + } +} + async function maybeUpdateWorkingMemory(attempt: ClaimedMemoryUpdateAttempt): Promise { const cwd = attempt.session.cwd; const transcriptPath = attempt.session.transcript_path; @@ -72,24 +131,86 @@ async function maybeUpdateWorkingMemory(attempt: ClaimedMemoryUpdateAttempt): Pr const transcriptMarkdown = runner.transcriptToMarkdown(transcript); if (transcriptMarkdown.trim().length === 0) return; - const runDir = mkdtempSync( - join(tmpdir(), `greplica-hook-${safePathSegment(attempt.session.platform)}-${safePathSegment(attempt.session.session_id)}-`), + await runMemoryAgent(runner, cwd, (proposalPath) => + updateWorkingMemoryPrompt(transcriptMarkdown, attempt, sessionRef, proposalPath), ); - const proposalPath = join(runDir, "working-memory.proposal.json"); +} + +/** Re-verify demoted claims for each distinct active repo, best-effort. */ +async function runReverifyForActiveRepos( + service: KnowledgeGraphService, + attempts: ClaimedMemoryUpdateAttempt[], + limit: number, + renewLease: () => boolean, +): Promise { + const seen = new Set(); + for (const attempt of attempts) { + const cwd = attempt.session.cwd; + if (cwd === null || seen.has(cwd)) continue; + seen.add(cwd); + if (!renewLease()) return; + const ref: RepoRef = { repo_root: cwd, repo_name: basename(cwd), default_branch: "main" }; + try { + await runReverifyPass(platformInstaller(attempt.session.platform), cwd, service.reverifyWorklist(ref, limit)); + } catch (error) { + // One repo's failure (e.g. greplica not installed there) must not abort the + // whole pass; log it and keep draining the rest. + console.error(JSON.stringify({ event: "freshness_reverify_error", repo: basename(cwd), error: errorMessage(error) })); + } + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** Hand the re-verify worklist to the agent runner. Empty worklist -> no agent spawn. */ +export async function runReverifyPass( + runner: Pick, + cwd: string, + claims: Claim[], +): Promise { + if (claims.length === 0) return; + await runMemoryAgent(runner, cwd, (proposalPath) => reverifyPrompt(claims, proposalPath)); +} + +/** Prompt the agent to re-verify drift-demoted claims against the current code. */ +export function reverifyPrompt(claims: Claim[], proposalPath: string): string { + const list = claims.map((claim) => `- ${claim.id}: ${claim.text}${anchorHint(claim)}`).join("\n"); + return `Some code_verified memory claims were auto-demoted to truth: unknown because their anchored code drifted. Re-verify each against the CURRENT repository code. + +For each claim: read the anchored code now. If the fact still holds, write a corrected code_verified claim that supersedes the unknown one; if it no longer holds, leave it demoted. Do not invent facts — verify against the files. + +Write the proposal JSON to ${proposalPath} and apply it with greplica, per the greplica-update-working-memory skill. +Claims to re-verify: +${list} +`; +} + +function anchorHint(claim: Claim): string { + const anchors = claim.code_anchors ?? []; + if (anchors.length === 0) return ""; + return ` (anchors: ${anchors.map((anchor) => (anchor.symbol ? `${anchor.file}#${anchor.symbol}` : anchor.file)).join(", ")})`; +} + +/** Run a platform agent with a fresh temp proposal path; best-effort, always cleans up. */ +async function runMemoryAgent( + runner: Pick, + cwd: string, + buildPrompt: (proposalPath: string) => string, +): Promise { + const runDir = mkdtempSync(join(tmpdir(), "greplica-hook-agent-")); try { await runner.runWorkingMemoryUpdate({ cwd, - env: { - ...process.env, - GREPLICA_HOOK_DISABLE: "1", - }, - prompt: updateWorkingMemoryPrompt(transcriptMarkdown, attempt, sessionRef, proposalPath), + env: { ...process.env, GREPLICA_HOOK_DISABLE: "1" }, + prompt: buildPrompt(join(runDir, "working-memory.proposal.json")), transcriptPath: join(runDir, "agent-events.jsonl"), finalMessagePath: join(runDir, "final-message.md"), }); } catch { - // Failed background updates should not affect foreground hook sessions. + // Failed background agent runs must not affect foreground hook sessions. } finally { rmSync(runDir, { recursive: true, force: true }); } @@ -125,7 +246,3 @@ ${transcriptMarkdown.trim()} `; } - -function safePathSegment(value: string): string { - return value.toLowerCase().replace(/[^a-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") || "unknown"; -} diff --git a/libs/knowledge-graph/anchor-fingerprints.ts b/libs/knowledge-graph/anchor-fingerprints.ts new file mode 100644 index 0000000..d463e99 --- /dev/null +++ b/libs/knowledge-graph/anchor-fingerprints.ts @@ -0,0 +1,68 @@ +import type { AnchorFingerprintRow } from "../storage/sqlite/repository.js"; +import type { AnchorCheck } from "./code-anchors/freshness.js"; +import { hashAnchorSpan, statAnchorFile } from "./code-anchors/span-hash.js"; +import type { ResolvedCodeAnchor } from "./code-anchors/types.js"; + +/** Fingerprints for one claim, keyed by anchor identity (see {@link anchorKey}). */ +type StoredByAnchor = Map; + +/** Stable map key for an anchor's identity; `''` symbol matches the storage sentinel. */ +function anchorKey(file: string, symbol: string | null | undefined): string { + return JSON.stringify([file, symbol ?? ""]); +} + +/** + * Group stored fingerprint rows into `claim_id -> (anchorKey -> row)` so the + * foreground can look up a claim's baseline hashes in O(1) after one batched read. + */ +export function indexFingerprintsByClaim(rows: AnchorFingerprintRow[]): Map { + const byClaim = new Map(); + for (const row of rows) { + let byAnchor = byClaim.get(row.claim_id); + if (byAnchor === undefined) { + byAnchor = new Map(); + byClaim.set(row.claim_id, byAnchor); + } + byAnchor.set(anchorKey(row.file, row.symbol), row); + } + return byClaim; +} + +/** + * Build the `AnchorCheck[]` for one claim's resolved anchors, applying the stat + * prefilter (cache-aside): when the file's mtime+size still match the stored + * fingerprint, reuse the stored hash instead of re-reading and re-hashing the span. + * `stored` is that claim's slice of {@link indexFingerprintsByClaim}. + */ +export function freshnessChecks( + resolved: ResolvedCodeAnchor[], + stored: StoredByAnchor | undefined, + repoRoot: string | undefined, +): AnchorCheck[] { + return resolved.map((anchor) => { + const row = stored?.get(anchorKey(anchor.file, anchor.symbol)); + return { + anchor, + storedHash: row?.content_hash, + currentHash: currentSpanHash(anchor, row, repoRoot), + }; + }); +} + +/** The span's hash right now — reused from the fingerprint when the file is untouched. */ +function currentSpanHash( + anchor: ResolvedCodeAnchor, + row: AnchorFingerprintRow | undefined, + repoRoot: string | undefined, +): string | undefined { + if (row !== undefined && fileUntouched(row, repoRoot, anchor.file)) { + return row.content_hash; // cache hit: skip the re-read + re-hash + } + return hashAnchorSpan(repoRoot, anchor); +} + +/** True when the file's mtime+size still match the stored fingerprint (the stat prefilter). */ +function fileUntouched(row: AnchorFingerprintRow, repoRoot: string | undefined, file: string): boolean { + const stat = statAnchorFile(repoRoot, file); + return stat.mtime_ms === row.file_mtime_ms && stat.size === row.file_size; +} diff --git a/libs/knowledge-graph/anchor-invalidation.ts b/libs/knowledge-graph/anchor-invalidation.ts new file mode 100644 index 0000000..678af5b --- /dev/null +++ b/libs/knowledge-graph/anchor-invalidation.ts @@ -0,0 +1,134 @@ +import type { Claim, ClaimCodeAnchor } from "./claim.js"; +import type { ResolvedCodeAnchor, ResolvedCodeAnchorStatus } from "./code-anchors/types.js"; +import type { Edge } from "./edge.js"; +import { isInvalidationResolverStatus, type InvalidationEventInput } from "./invalidation.js"; +import { + normalizeProposal, + type CompactEdge, + type CompactMemoryProposal, + type MemoryCommitProposal, + type ProposalSubjectLookup, +} from "./proposal.js"; +import type { GraphObjectType } from "./schema.js"; +import type { GraphReadResult } from "./service.js"; + +/** Suffix that turns an original claim id into its rebuilt (demoted) counterpart. */ +const driftSuffix = "__drift"; + +/** + * A claim to demote, and why. `reason: "structural"` means every anchor stopped + * resolving (`anchors` are the broken ones); `reason: "content"` means a still- + * resolving anchor's span changed (`anchors` are the drifted, resolving ones). + */ +export interface ClaimDemotion { + claim: Claim; + reason: "structural" | "content"; + anchors: ResolvedCodeAnchor[]; +} + +export interface AnchorInvalidationPlan { + proposal: MemoryCommitProposal; + events: InvalidationEventInput[]; +} + +/** + * Pure translation of demoted claims into the writes that supersede them: for + * each claim, a rebuilt `truth: unknown` copy (keeping its anchors as evidence), + * its `about`/`evidenced_by` edges re-pointed at the rebuild, a `supersedes` + * edge rebuild -> original, and one invalidation event recording the drift kind. + * + * No I/O: edge ids are minted by `normalizeProposal` using an in-memory lookup + * built from the graph, and outgoing edges are read from a `from_id` index built + * once (O(edges + claims), never O(edges * claims)). + */ +export function buildAnchorInvalidation(demotions: ClaimDemotion[], graph: GraphReadResult): AnchorInvalidationPlan { + const edgesByFrom = indexEdgesByFrom(graph.edges); + + const claims: Claim[] = []; + const edges: CompactEdge[] = []; + const events: InvalidationEventInput[] = []; + + for (const { claim, reason, anchors } of demotions) { + const supersedingId = `${claim.id}${driftSuffix}`; + + claims.push({ + id: supersedingId, + kind: claim.kind, + text: claim.text, + truth: "unknown", + intent: claim.intent, + code_anchors: claim.code_anchors, + }); + + // The original's edges go dead once it is superseded, so clone the ones that + // keep the claim meaningful onto the rebuild. + for (const edge of edgesByFrom.get(claim.id) ?? []) { + if (edge.kind === "about" || edge.kind === "evidenced_by") { + edges.push({ kind: edge.kind, from: supersedingId, to: edge.to_id, metadata: edge.metadata }); + } + } + edges.push({ kind: "supersedes", from: supersedingId, to: claim.id }); + + events.push(demotionEvent(claim.id, supersedingId, reason, anchors[0])); + } + + const proposal: CompactMemoryProposal = { + title: invalidationTitle(demotions.length), + creates: { claims, edges }, + }; + + return { proposal: normalizeProposal(proposal, graphSubjectLookup(graph)), events }; +} + +function demotionEvent( + originalId: string, + supersedingId: string, + reason: ClaimDemotion["reason"], + anchor: ResolvedCodeAnchor, +): InvalidationEventInput { + return { + original_claim_id: originalId, + superseding_claim_id: supersedingId, + reason: reason === "content" ? "content_drift" : "anchor_drift", + broken_anchor: formatAnchor(anchor), + // Structural drift must carry a drift status (guarded); content drift records + // the anchor's still-resolving status as-is. + resolver_status: reason === "content" ? anchor.status : assertDriftStatus(anchor), + }; +} + +function indexEdgesByFrom(edges: Edge[]): Map { + const index = new Map(); + for (const edge of edges) { + const existing = index.get(edge.from_id); + if (existing) existing.push(edge); + else index.set(edge.from_id, [edge]); + } + return index; +} + +function graphSubjectLookup(graph: GraphReadResult): ProposalSubjectLookup { + const types = new Map(); + for (const component of graph.components) types.set(component.id, "component"); + for (const flow of graph.flows) types.set(flow.id, "flow"); + for (const claim of graph.claims) types.set(claim.id, "claim"); + for (const source of graph.sources) types.set(source.id, "source"); + return { subjectType: (id) => types.get(id) }; +} + +function assertDriftStatus(anchor: ResolvedCodeAnchor): ResolvedCodeAnchorStatus { + // A structural demotion's anchors only ever carry drift statuses; this guard + // narrows the type and fails loud if that invariant is ever violated. + if (!isInvalidationResolverStatus(anchor.status)) { + throw new Error(`Anchor ${formatAnchor(anchor)} has non-drift status "${anchor.status}".`); + } + return anchor.status; +} + +function formatAnchor(anchor: ClaimCodeAnchor): string { + return anchor.symbol === undefined ? anchor.file : `${anchor.file}#${anchor.symbol}`; +} + +function invalidationTitle(count: number): string { + return `Anchor drift invalidation (${count} claim${count === 1 ? "" : "s"})`; +} diff --git a/libs/knowledge-graph/code-anchors/drift.ts b/libs/knowledge-graph/code-anchors/drift.ts new file mode 100644 index 0000000..c3964bf --- /dev/null +++ b/libs/knowledge-graph/code-anchors/drift.ts @@ -0,0 +1,63 @@ +import type { Claim } from "../claim.js"; +import { CodeAnchorResolver } from "./resolver.js"; +import { classifyFreshness } from "./freshness.js"; +import type { ResolvedCodeAnchor } from "./types.js"; + +/** A code_verified claim whose anchors have all stopped resolving. */ +export interface DriftedClaim { + claim: Claim; + broken: ResolvedCodeAnchor[]; +} + +/** A claim that could not be re-resolved (unexpected resolver failure), recorded but not fatal. */ +export interface DriftScanError { + claim_id: string; + message: string; +} + +export interface DriftScanResult { + drifted: DriftedClaim[]; + errors: DriftScanError[]; +} + +/** + * Re-resolves every `code_verified` claim's anchors against the current working + * tree and reports which claims have fully drifted. + * + * Policy (Option A): a claim drifts only when it has anchors and *every* anchor + * is broken. If any anchor still resolves — or is inconclusive, e.g. + * `unsupported_language` — the claim keeps its `code_verified` standing. + * + * Resilient: a resolver failure on one claim is recorded in `errors` and + * skipped, never fatal to the pass. Cache-friendly: a single shared + * `CodeAnchorResolver` parses each file once even when many claims anchor it. + */ +export async function scanDriftedClaims( + repoRoot: string | undefined, + claims: Claim[], +): Promise { + const resolver = new CodeAnchorResolver(); + const drifted: DriftedClaim[] = []; + const errors: DriftScanError[] = []; + + for (const claim of claims) { + const anchors = claim.code_anchors ?? []; + if (claim.truth !== "code_verified" || anchors.length === 0) continue; + + try { + const resolved = await resolver.resolveMany(repoRoot, anchors); + // Structural-only detection: no stored hashes, so only "every anchor broken" trips. + const checks = resolved.map((anchor) => ({ anchor, currentHash: undefined, storedHash: undefined })); + const verdict = classifyFreshness(checks); + if (verdict.state === "stale") drifted.push({ claim, broken: verdict.broken }); + } catch (error) { + errors.push({ claim_id: claim.id, message: errorMessage(error) }); + } + } + + return { drifted, errors }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : "Unexpected error resolving code anchors."; +} diff --git a/libs/knowledge-graph/code-anchors/freshness.ts b/libs/knowledge-graph/code-anchors/freshness.ts new file mode 100644 index 0000000..5c3e663 --- /dev/null +++ b/libs/knowledge-graph/code-anchors/freshness.ts @@ -0,0 +1,76 @@ +import { invalidationResolverStatuses } from "../invalidation.js"; +import type { ResolvedCodeAnchor, ResolvedCodeAnchorStatus } from "./types.js"; + +export type FreshnessState = "fresh" | "stale" | "unknown"; +export type FreshnessReason = "structural" | "content"; + +/** Why a claim is stale, and which anchors are structurally broken. */ +export interface FreshnessVerdict { + state: FreshnessState; + reason: FreshnessReason | null; + broken: ResolvedCodeAnchor[]; +} + +/** One anchor together with its span hash now and the last time we verified the claim. */ +export interface AnchorCheck { + anchor: ResolvedCodeAnchor; + currentHash: string | undefined; // hash of the span right now (undefined if unreadable) + storedHash: string | undefined; // hash recorded when the claim was last verified +} + +const brokenStatuses: ReadonlySet = new Set(invalidationResolverStatuses); + +/** + * The single fresh/stale rule, shared by the foreground signal and the background heal. + * + * A claim is stale when either kind of drift has happened: + * - **structural** — every anchor stopped resolving (symbol moved / renamed / deleted); + * - **content** — a still-resolving anchor's span changed since we last verified it. + * + * When no drift is proven but a resolving anchor's span can't be hashed right now + * (unreadable file / resolver error), the verdict is **unknown** rather than a false + * "fresh" — the caller distrusts it lightly instead of vouching for it. A missing + * baseline alone (span readable, no stored hash) stays fresh. + */ +export function classifyFreshness(checks: AnchorCheck[]): FreshnessVerdict { + if (checks.length === 0) return fresh(); + + const broken = checks.filter(isStructurallyBroken).map((check) => check.anchor); + if (broken.length === checks.length) { + return { state: "stale", reason: "structural", broken }; + } + + if (checks.some(hasContentDrift)) { + // Content drift wins the reason, but still surface any anchors that broke + // structurally in the same claim so the caller can act on them too. + return { state: "stale", reason: "content", broken }; + } + + if (checks.some(isUndeterminable)) { + // Freshness can't be proven, but still surface any anchors that broke + // structurally in the same claim (consistent with the content branch). + return { state: "unknown", reason: null, broken }; + } + + return fresh(); +} + +function isStructurallyBroken(check: AnchorCheck): boolean { + return brokenStatuses.has(check.anchor.status); +} + +/** A still-resolving anchor whose current span hash could not be computed. */ +function isUndeterminable(check: AnchorCheck): boolean { + return !isStructurallyBroken(check) && check.currentHash === undefined; +} + +/** A still-resolving anchor whose span hash changed since the claim was verified. */ +export function hasContentDrift(check: AnchorCheck): boolean { + if (isStructurallyBroken(check)) return false; // handled as structural drift + if (check.storedHash === undefined || check.currentHash === undefined) return false; // nothing to compare + return check.currentHash !== check.storedHash; +} + +function fresh(): FreshnessVerdict { + return { state: "fresh", reason: null, broken: [] }; +} diff --git a/libs/knowledge-graph/code-anchors/span-hash.ts b/libs/knowledge-graph/code-anchors/span-hash.ts new file mode 100644 index 0000000..ee2035e --- /dev/null +++ b/libs/knowledge-graph/code-anchors/span-hash.ts @@ -0,0 +1,74 @@ +import { createHash } from "node:crypto"; +import { readFileSync, realpathSync, statSync } from "node:fs"; +import { isAbsolute, join, sep } from "node:path"; +import type { ResolvedCodeAnchor } from "./types.js"; + +export interface FileStat { + mtime_ms: number; + size: number; +} + +/** + * SHA-256 of the anchored span's source text — the content fingerprint. + * + * Hashes just the `start_line..end_line` range when the resolver pinned a symbol, + * otherwise the whole file. Returns `undefined` when the file cannot be read + * (missing, an absolute path, or escaping the repo) so callers degrade gracefully + * rather than throwing into a query or heal pass. + */ +export function hashAnchorSpan(repoRoot: string | undefined, anchor: ResolvedCodeAnchor): string | undefined { + const text = readRepoFile(repoRoot, anchor.file); + if (text === undefined) return undefined; + return createHash("sha256").update(spanText(text, anchor.start_line, anchor.end_line)).digest("hex"); +} + +/** Cheap `stat` used as the freshness prefilter; zeros when the file is unavailable. */ +export function statAnchorFile(repoRoot: string | undefined, file: string): FileStat { + const abs = resolveWithinRepo(repoRoot, file); + if (abs === undefined) return { mtime_ms: 0, size: 0 }; + try { + const stat = statSync(abs); + return { mtime_ms: stat.mtimeMs, size: stat.size }; + } catch { + return { mtime_ms: 0, size: 0 }; + } +} + +function readRepoFile(repoRoot: string | undefined, file: string): string | undefined { + const abs = resolveWithinRepo(repoRoot, file); + if (abs === undefined) return undefined; + try { + return readFileSync(abs, "utf8"); + } catch { + return undefined; + } +} + +/** + * The canonical absolute path for a repo-relative `file`, or `undefined` if it + * escapes the repo — via a literal `..`/absolute path OR a symlink that resolves + * outside `repoRoot` (realpath containment, stronger than a pure-path check). + */ +function resolveWithinRepo(repoRoot: string | undefined, file: string): string | undefined { + if (repoRoot === undefined || isAbsolute(file) || file.split(/[\\/]/).includes("..")) return undefined; + try { + const root = realpathSync(repoRoot); + const abs = realpathSync(join(root, file)); + return abs === root || abs.startsWith(root + sep) ? abs : undefined; + } catch { + return undefined; // missing file or broken symlink + } +} + +function spanText(fileText: string, startLine: number | undefined, endLine: number | undefined): string { + // Normalize CRLF/CR to LF (matching the resolver's `split(/\r?\n/)`) so a + // cross-platform checkout doesn't hash a trailing \r into every line and + // report false content drift. + const lines = fileText.split(/\r?\n/); + if (startLine === undefined) return lines.join("\n"); + // Anchor lines are 1-based and inclusive; slice() wants a 0-based [start, end) range, + // so start-1 and an exclusive end of endLine keeps the [startLine..endLine] rows. + const start = Math.max(0, startLine - 1); + const end = endLine ?? startLine; + return lines.slice(start, end).join("\n"); +} diff --git a/libs/knowledge-graph/graph-context/claim-freshness.ts b/libs/knowledge-graph/graph-context/claim-freshness.ts new file mode 100644 index 0000000..9ef7c68 --- /dev/null +++ b/libs/knowledge-graph/graph-context/claim-freshness.ts @@ -0,0 +1,27 @@ +import type { SqliteRepository } from "../../storage/sqlite/repository.js"; +import { freshnessChecks, indexFingerprintsByClaim } from "../anchor-fingerprints.js"; +import { classifyFreshness } from "../code-anchors/freshness.js"; +import type { ClaimContextResult } from "./types.js"; + +/** + * Attach a freshness verdict to each selected claim (read-only, per query). + * + * One batched fingerprint read for all claims (no N+1), then per claim a + * stat-prefiltered set of anchor checks fed to the shared `classifyFreshness`. + * Returns new result objects — it never mutates its inputs and never writes to + * the graph; persistence is the background heal's job. + */ +export function attachFreshness( + claims: Omit[], + repository: Pick, + repoRoot: string | undefined, +): ClaimContextResult[] { + if (claims.length === 0) return []; + const storedByClaim = indexFingerprintsByClaim( + repository.fingerprintsForClaims(claims.map((claim) => claim.object.id)), + ); + return claims.map((claim) => ({ + ...claim, + freshness: classifyFreshness(freshnessChecks(claim.code_anchors, storedByClaim.get(claim.object.id), repoRoot)), + })); +} diff --git a/libs/knowledge-graph/graph-context/context-builder.ts b/libs/knowledge-graph/graph-context/context-builder.ts index a0093fe..019bc9d 100644 --- a/libs/knowledge-graph/graph-context/context-builder.ts +++ b/libs/knowledge-graph/graph-context/context-builder.ts @@ -17,6 +17,7 @@ import { applyGraphRanking } from "./graph-rank.js"; import { rankContextDocuments, roundScore, selectRankedDocuments, type RankedContextDocument, type SemanticScoreEntry } from "./rank.js"; import type { ClaimContextResult, ClaimEvidenceResult, ComponentContextResult, EmbeddingStatus, FlowContextResult, GraphContextResult, RankedContextDebugResult } from "./types.js"; import { rankPacketResults, roundRankedSignals, selectGraphObjects } from "./packet-rank.js"; +import { attachFreshness } from "./claim-freshness.js"; import { CodeAnchorResolver } from "../code-anchors/resolver.js"; import type { ResolvedCodeAnchor } from "../code-anchors/types.js"; @@ -56,13 +57,16 @@ export class GraphContextBuilder { flows: this.rankDocuments(repoId, query, queryEmbedding, flowDocuments, config), }; const ranked = applyGraphRanking(baseRanked, graph, config); - const selectedClaims = await selectClaims( + const claimResults = await selectClaims( ranked.claims, evidenceByClaim, config, this.codeAnchorResolver, options.repoRoot, ); + // Read-only freshness: label each claim fresh/stale/unknown against the working + // tree using one batched fingerprint read. No graph writes on the query path. + const selectedClaims = attachFreshness(claimResults, this.repository, options.repoRoot); const selectedComponents = selectGraphObjects( ranked.components, selectedClaims, @@ -227,7 +231,7 @@ function selectClaims( config: GraphContextConfig, resolver: CodeAnchorResolver, repoRoot: string | undefined, -): Promise { +): Promise[]> { return Promise.all(selectRankedDocuments(ranked, config, { minimumSelected: config.ranking.minimumSelectedClaims }) .sort((left, right) => right.score - left.score || left.document.key.localeCompare(right.document.key)) .map((document, index) => toClaimResult(document, index, evidenceByClaim, resolver, repoRoot))); @@ -239,7 +243,7 @@ async function toClaimResult( evidenceByClaim: Map, resolver: CodeAnchorResolver, repoRoot: string | undefined, -): Promise { +): Promise> { const claim = document.document.object as Claim; return { rank: index + 1, diff --git a/libs/knowledge-graph/graph-context/render.ts b/libs/knowledge-graph/graph-context/render.ts index 1c0fc3a..2254067 100644 --- a/libs/knowledge-graph/graph-context/render.ts +++ b/libs/knowledge-graph/graph-context/render.ts @@ -1,3 +1,4 @@ +import type { FreshnessVerdict } from "../code-anchors/freshness.js"; import type { GraphContextResult, RankedGraphContextResult, @@ -7,6 +8,8 @@ export function renderGraphContextMarkdown(result: GraphContextResult): string { const rankedComponents = result.ranked_results.filter((item) => item.type === "component"); const rankedFlows = result.ranked_results.filter((item) => item.type === "flow"); const rankedClaims = result.ranked_results.filter((item) => item.type === "claim"); + const staleClaims = rankedClaims.filter((claim) => claim.freshness.state === "stale"); + const liveClaims = rankedClaims.filter((claim) => claim.freshness.state !== "stale"); const componentsById = new Map(result.components.map((component) => [component.object.id, component.object.name])); const flowsById = new Map(result.flows.map((flow) => [flow.object.id, flow.object.name])); const content = [ @@ -14,7 +17,8 @@ export function renderGraphContextMarkdown(result: GraphContextResult): string { "", "## Best Claims", "", - ...renderRankedClaims(rankedClaims, componentsById, flowsById), + ...renderRankedClaims(liveClaims, componentsById, flowsById), + ...renderNeedsReverification(staleClaims, componentsById, flowsById), "", "## Related Components", "", @@ -51,6 +55,27 @@ function renderRankedFlows( }); } +/** + * Claims whose freshness verdict is `stale` are quarantined into their own + * section — a distrust signal, not a command. Omitted entirely when nothing + * drifted, so healthy packets stay noise-free. + */ +function renderNeedsReverification( + claims: Array>, + componentsById: Map, + flowsById: Map, +): string[] { + if (claims.length === 0) return []; + return [ + "", + "## Needs re-verification", + "", + "These facts were code-verified but their anchored code has since drifted. Re-verify against the current code before relying on them.", + "", + ...renderRankedClaims(claims, componentsById, flowsById), + ]; +} + function renderRankedClaims( claims: Array>, componentsById: Map, @@ -65,12 +90,22 @@ function renderRankedClaims( "", claim.object.text, "", - `${anchors}${about}`.trim(), + `Truth: \`${claim.object.truth}\`.${freshnessLabel(claim.freshness)}${anchors}${about}`.trim(), "", ]; }); } +function freshnessLabel(freshness: FreshnessVerdict): string { + if (freshness.state === "stale") { + return ` [STALE: ${freshness.reason} drift — re-verify against the current code].`; + } + if (freshness.state === "unknown") { + return " [UNVERIFIABLE: anchored code could not be read]."; + } + return ""; +} + function anchorLabel(anchor: Extract["code_anchors"][number]): string { const base = anchor.symbol === undefined ? anchor.file : `${anchor.file}#${anchor.symbol}`; if (anchor.status === "resolved" && anchor.start_line !== undefined) { diff --git a/libs/knowledge-graph/graph-context/types.ts b/libs/knowledge-graph/graph-context/types.ts index 0d7e7b3..0170c8e 100644 --- a/libs/knowledge-graph/graph-context/types.ts +++ b/libs/knowledge-graph/graph-context/types.ts @@ -1,5 +1,6 @@ import type { Claim } from "../claim.js"; import type { ResolvedCodeAnchor } from "../code-anchors/types.js"; +import type { FreshnessVerdict } from "../code-anchors/freshness.js"; import type { Component, Flow, Source } from "../schema.js"; export interface EmbeddingStatus { @@ -47,6 +48,7 @@ export interface ClaimContextResult { about: Array<{ type: "component" | "flow"; id: string }>; evidence: ClaimEvidenceResult[]; code_anchors: ResolvedCodeAnchor[]; + freshness: FreshnessVerdict; } export interface GraphObjectContextResult { diff --git a/libs/knowledge-graph/invalidation.ts b/libs/knowledge-graph/invalidation.ts new file mode 100644 index 0000000..97fb1b9 --- /dev/null +++ b/libs/knowledge-graph/invalidation.ts @@ -0,0 +1,51 @@ +/** + * Records why and when a `code_verified` claim was demoted to `truth: unknown` + * because its anchored code drifted — the anchor stopped resolving (structural) + * or its span content changed (content). The claim itself is never mutated — it + * is superseded by a rebuilt copy — so this table is the queryable audit trail + * of what went stale. + */ + +import type { ResolvedCodeAnchorStatus } from "./code-anchors/types.js"; + +/** Why a claim was invalidated: the anchor stopped resolving, or its span content changed. */ +export type InvalidationReason = "anchor_drift" | "content_drift"; + +/** + * The resolver statuses that count as drift (a subset of ResolvedCodeAnchorStatus). + * Single source of truth for both detection (drift.ts) and the audit trail. + */ +export const invalidationResolverStatuses = ["missing_file", "missing_symbol", "ambiguous_symbol"] as const; + +export type InvalidationResolverStatus = (typeof invalidationResolverStatuses)[number]; + +export function isInvalidationResolverStatus(status: string): status is InvalidationResolverStatus { + return (invalidationResolverStatuses as readonly string[]).includes(status); +} + +/** A persisted invalidation event, one row per demoted claim. */ +export interface InvalidationEvent { + id: string; + repo_id: string; + original_claim_id: string; + superseding_claim_id: string; + memory_commit_id: string; + reason: InvalidationReason; + broken_anchor: string; + resolver_status: ResolvedCodeAnchorStatus; + git_commit_sha?: string; + created_at: string; +} + +/** + * The caller-supplied portion of an invalidation event. The repository stamps + * `id`, `repo_id`, `memory_commit_id`, `git_commit_sha`, and `created_at` when + * it writes the batch, so callers only describe what drifted. + */ +export interface InvalidationEventInput { + original_claim_id: string; + superseding_claim_id: string; + reason: InvalidationReason; + broken_anchor: string; + resolver_status: ResolvedCodeAnchorStatus; +} diff --git a/libs/knowledge-graph/service.ts b/libs/knowledge-graph/service.ts index efb6e65..5ea0585 100644 --- a/libs/knowledge-graph/service.ts +++ b/libs/knowledge-graph/service.ts @@ -9,8 +9,16 @@ import type { EmbeddingStatus, GraphContextResult } from "./graph-context/types. import { buildGraphViewHtml } from "./graph-view/build-graph-view.js"; import { auditClaimCodeAnchors } from "./code-anchors/audit.js"; import type { ClaimAnchorAuditResult } from "./code-anchors/types.js"; +import { scanDriftedClaims, type DriftScanError } from "./code-anchors/drift.js"; +import { CodeAnchorResolver } from "./code-anchors/resolver.js"; +import { hashAnchorSpan, statAnchorFile } from "./code-anchors/span-hash.js"; +import { classifyFreshness, hasContentDrift, type AnchorCheck, type FreshnessVerdict } from "./code-anchors/freshness.js"; +import { freshnessChecks, indexFingerprintsByClaim } from "./anchor-fingerprints.js"; +import { buildAnchorInvalidation, type ClaimDemotion } from "./anchor-invalidation.js"; +import type { ResolvedCodeAnchorStatus } from "./code-anchors/types.js"; +import { changedFilesSince, gitHeadSha } from "../utils/git.js"; import { defaultDatabasePath, openDatabase } from "../storage/sqlite/db.js"; -import type { SqliteRepository } from "../storage/sqlite/repository.js"; +import type { AnchorFingerprintInput, SqliteRepository } from "../storage/sqlite/repository.js"; import { SqliteRepository as SqliteKnowledgeGraphRepository } from "../storage/sqlite/repository.js"; export type { GraphContextResult } from "./graph-context/types.js"; @@ -52,6 +60,25 @@ export interface ApplyProposalResult { }; } +export interface AnchorInvalidationRecord { + claim_id: string; + superseding_claim_id: string; + broken_anchor: string; + resolver_status: ResolvedCodeAnchorStatus; +} + +export interface AnchorInvalidationResult { + memory_commit_id?: string; + invalidated: AnchorInvalidationRecord[]; + errors: DriftScanError[]; +} + +export interface HealResult { + demoted: string[]; + rechecked: number; + headSha?: string; +} + export class KnowledgeGraphService { constructor( private readonly repository: SqliteRepository, @@ -164,6 +191,8 @@ export class KnowledgeGraphService { this.contextConfig, ); + await this.writeFingerprints(initialized.repo_id, input, normalizedProposal.creates.claims ?? []); + return { memory_commit_id: memoryCommit.id, scope_id: working.id, @@ -178,6 +207,196 @@ export class KnowledgeGraphService { }; } + /** + * Records a content fingerprint per anchor of every `code_verified` claim, so + * later reads/heals can tell whether the anchored code has since changed. + * + * Best effort, and deliberately non-throwing: the proposal is already durably + * persisted by the time this runs, so a fingerprinting failure (an unreadable + * span, a resolver hiccup) must not turn a successful apply into a failed one. + */ + private async writeFingerprints(repoId: string, input: RepoRef, claims: Claim[]): Promise { + try { + const resolver = new CodeAnchorResolver(); + const rows: AnchorFingerprintInput[] = []; + for (const claim of claims) { + const anchors = claim.code_anchors ?? []; + if (claim.truth !== "code_verified" || anchors.length === 0) continue; + const resolved = await resolver.resolveMany(input.repo_root, anchors); + for (const anchor of resolved) { + const contentHash = hashAnchorSpan(input.repo_root, anchor); + if (contentHash === undefined) continue; + const stat = statAnchorFile(input.repo_root, anchor.file); + rows.push({ + repo_id: repoId, + claim_id: claim.id, + file: anchor.file, + symbol: anchor.symbol ?? "", // "" sentinel for file-only anchors (see schema) + content_hash: contentHash, + file_mtime_ms: stat.mtime_ms, + file_size: stat.size, + resolver_status: anchor.status, + }); + } + } + this.repository.upsertAnchorFingerprints(rows); + } catch { + // Freshness metadata is an optimization; never fail apply over it. + } + } + + /** + * Re-verifies every code_verified claim's anchors and demotes the ones that + * have fully drifted to `truth: unknown`, non-destructively (via supersession) + * and atomically. Returns the demotions plus any per-claim detection errors. + * The report-only `auditCodeAnchors` is left untouched. + */ + async invalidateDriftedAnchors(input: RepoRef): Promise { + const initialized = this.requireRepo(input); + const graph = this.repository.readGraphView(initialized.repo_id); + const { drifted, errors } = await scanDriftedClaims(input.repo_root, graph.claims); + + if (drifted.length === 0) { + return { invalidated: [], errors }; + } + + // buildAnchorInvalidation already returns a normalized proposal (edge ids + // minted via an in-memory graph lookup), so no further normalization here. + // scanDriftedClaims only finds structural drift, so every demotion is structural. + const demotions = drifted.map((d) => ({ claim: d.claim, reason: "structural" as const, anchors: d.broken })); + const { proposal, events } = buildAnchorInvalidation(demotions, graph); + const working = this.repository.requireWorkingScope(initialized.repo_id); + const { memory_commit_id } = this.repository.applyAnchorInvalidation({ + repoId: initialized.repo_id, + scopeId: working.id, + proposal, + events, + commit: { title: proposal.title, git_commit_sha: gitHeadSha(input.repo_root) }, + }); + + // Embed the rebuilt claims so they stay retrievable via graph context. + await this.contextBuilder.ensureForGraph( + initialized.repo_id, + this.repository.readGraphView(initialized.repo_id), + this.contextConfig, + ); + + return { + memory_commit_id, + invalidated: events.map((event) => ({ + claim_id: event.original_claim_id, + superseding_claim_id: event.superseding_claim_id, + broken_anchor: event.broken_anchor, + resolver_status: event.resolver_status, + })), + errors, + }; + } + + /** + * Change-scoped background heal. Re-checks only claims whose anchored files + * changed since `sinceSha` (undefined = full sweep), demotes the genuinely + * stale ones — structural OR content — through the same supersession writer as + * #96, drops their fingerprints, and advances the freshness checkpoint. Never + * demotes on `unknown` (unreadable / undeterminable). Deterministic: no agent. + */ + async healDriftedAnchors(input: RepoRef, sinceSha?: string): Promise { + const initialized = this.requireRepo(input); + const headSha = gitHeadSha(input.repo_root); + const graph = this.repository.readGraphView(initialized.repo_id); + + const candidates = this.healCandidates(initialized.repo_id, graph.claims, input.repo_root, sinceSha); + if (candidates.length === 0) { + this.saveCheckpoint(initialized.repo_id, headSha); + return { demoted: [], rechecked: 0, headSha }; + } + + const { demotions, rechecked } = await this.classifyCandidates(candidates, input.repo_root); + if (demotions.length === 0) { + this.saveCheckpoint(initialized.repo_id, headSha); + return { demoted: [], rechecked, headSha }; + } + + const { proposal, events } = buildAnchorInvalidation(demotions, graph); + const working = this.repository.requireWorkingScope(initialized.repo_id); + const demoted = demotions.map((demotion) => demotion.claim.id); + this.repository.applyAnchorInvalidation({ + repoId: initialized.repo_id, + scopeId: working.id, + proposal, + events, + commit: { title: proposal.title, git_commit_sha: headSha }, + }); + + // Embed the rebuilt claims so they stay retrievable via graph context. + await this.contextBuilder.ensureForGraph( + initialized.repo_id, + this.repository.readGraphView(initialized.repo_id), + this.contextConfig, + ); + this.saveCheckpoint(initialized.repo_id, headSha); + return { demoted, rechecked, headSha }; + } + + /** Heal using this repo's stored checkpoint as `sinceSha` (undefined -> full sweep). */ + async healDriftedAnchorsFromCheckpoint(input: RepoRef): Promise { + const initialized = this.requireRepo(input); + return this.healDriftedAnchors(input, this.repository.getFreshnessCheckpoint(initialized.repo_id)); + } + + /** The re-verify worklist for this repo: drift-demoted claims still `truth: unknown`. */ + reverifyWorklist(input: RepoRef, limit: number): Claim[] { + return this.repository.claimsNeedingReverify(this.requireRepo(input).repo_id, limit); + } + + /** The code_verified claims to re-check: the full set on a sweep, else only those in changed files. */ + private healCandidates(repoId: string, claims: Claim[], repoRoot: string | undefined, sinceSha: string | undefined): Claim[] { + const codeVerified = claims.filter((claim) => claim.truth === "code_verified" && (claim.code_anchors?.length ?? 0) > 0); + if (sinceSha === undefined) return codeVerified; // first run / no checkpoint -> full sweep + const changed = changedFilesSince(repoRoot, sinceSha); + if (changed === undefined) return codeVerified; // git probe failed -> full sweep, don't silently skip + if (changed.length === 0) return []; + const affected = new Set(this.repository.claimIdsForFiles(repoId, changed)); // reverse index (repo-scoped) + return codeVerified.filter((claim) => affected.has(claim.id)); + } + + /** Re-resolve + classify each candidate; collect the ones that genuinely drifted. */ + private async classifyCandidates( + candidates: Claim[], + repoRoot: string | undefined, + ): Promise<{ demotions: ClaimDemotion[]; rechecked: number }> { + const resolver = new CodeAnchorResolver(); + const storedByClaim = indexFingerprintsByClaim(this.repository.fingerprintsForClaims(candidates.map((claim) => claim.id))); + const demotions: ClaimDemotion[] = []; + let rechecked = 0; + for (const claim of candidates) { + try { + const resolved = await resolver.resolveMany(repoRoot, claim.code_anchors ?? []); + rechecked += 1; + const checks = freshnessChecks(resolved, storedByClaim.get(claim.id), repoRoot); + const demotion = toDemotion(claim, classifyFreshness(checks), checks); + if (demotion !== undefined) demotions.push(demotion); + } catch { + // A resolver failure on one claim is skipped, never fatal to the pass. + } + } + return { demotions, rechecked }; + } + + private saveCheckpoint(repoId: string, headSha: string | undefined): void { + if (headSha !== undefined) this.repository.setFreshnessCheckpoint(repoId, headSha); + } + +} + +/** Map a freshness verdict to a demotion, or undefined when the claim must be left alone. */ +function toDemotion(claim: Claim, verdict: FreshnessVerdict, checks: AnchorCheck[]): ClaimDemotion | undefined { + if (verdict.state !== "stale") return undefined; // fresh or unknown -> never demote + if (verdict.reason === "content") { + // Only the anchors whose hash actually drifted, so the audit event names them accurately. + return { claim, reason: "content", anchors: checks.filter(hasContentDrift).map((check) => check.anchor) }; + } + return { claim, reason: "structural", anchors: verdict.broken }; } function anchorAuditErrors(result: ClaimAnchorAuditResult): string[] { diff --git a/libs/storage/sqlite/repository.ts b/libs/storage/sqlite/repository.ts index de8f4f4..d23a015 100644 --- a/libs/storage/sqlite/repository.ts +++ b/libs/storage/sqlite/repository.ts @@ -3,6 +3,7 @@ import { createHash, randomUUID } from "node:crypto"; import type { MemoryCommit } from "../../knowledge-graph/commit.js"; import type { Edge } from "../../knowledge-graph/edge.js"; import type { MemoryCommitProposal } from "../../knowledge-graph/proposal.js"; +import type { InvalidationEvent, InvalidationEventInput } from "../../knowledge-graph/invalidation.js"; import type { Component, Flow, GraphObjectType, Source } from "../../knowledge-graph/schema.js"; import type { Claim } from "../../knowledge-graph/claim.js"; import type { GraphScope, GraphScopeKind } from "../../knowledge-graph/scope.js"; @@ -45,8 +46,32 @@ type MembershipRow = { type ComponentRow = Omit & { code_anchor: string | null }; type ClaimRow = Omit & { code_anchors: string | null }; type EdgeRow = Omit & { metadata: string | null }; +type InvalidationEventRow = Omit & { git_commit_sha: string | null }; type RepoMatch = { repo: RepoRecord; matchedBy: "remote" | "root" }; +export interface ApplyAnchorInvalidationInput { + repoId: string; + scopeId: string; + proposal: MemoryCommitProposal; + events: InvalidationEventInput[]; + commit: { title: string; summary?: string; git_commit_sha?: string }; +} + +export interface AnchorFingerprintRow { + repo_id: string; + claim_id: string; + file: string; + symbol: string; // "" for file-only anchors (see schema: non-null keeps upserts idempotent) + content_hash: string; + file_mtime_ms: number; + file_size: number; + resolver_status: string; + checked_at: string; +} + +/** A fingerprint row without the server-stamped `checked_at`. */ +export type AnchorFingerprintInput = Omit; + export type EmbeddingObjectType = "claim" | "component" | "flow"; export interface GraphObjectEmbeddingRecord { @@ -286,49 +311,187 @@ export class SqliteRepository { createProposalRecords(scopeId: string, memoryCommitId: string, proposal: MemoryCommitProposal): void { const write = this.db.transaction(() => { - for (const component of proposal.creates.components ?? []) { - this.db - .prepare("INSERT INTO components (id, name, code_anchor) VALUES (@id, @name, @code_anchor)") - .run({ ...component, code_anchor: component.code_anchor ?? null }); - this.createMembership(scopeId, "component", component.id, memoryCommitId); - } + this.insertProposalRecords(scopeId, memoryCommitId, proposal); + }); + write(); + } - for (const flow of proposal.creates.flows ?? []) { - this.db.prepare("INSERT INTO flows (id, name) VALUES (@id, @name)").run(flow); - this.createMembership(scopeId, "flow", flow.id, memoryCommitId); - } + /** + * Demotes drifted claims by writing their rebuilt (superseding) claims and + * edges alongside the invalidation events, all under one memory commit and one + * transaction. Integrity relies on the claim primary key plus the transaction: + * a duplicate rebuilt-claim id or any constraint breach rolls back the whole + * batch, so nothing partial is ever committed. + */ + applyAnchorInvalidation(input: ApplyAnchorInvalidationInput): { memory_commit_id: string } { + const insertEvent = this.db.prepare( + `INSERT INTO invalidation_events + (id, repo_id, original_claim_id, superseding_claim_id, memory_commit_id, reason, broken_anchor, resolver_status, git_commit_sha, created_at) + VALUES + (@id, @repo_id, @original_claim_id, @superseding_claim_id, @memory_commit_id, @reason, @broken_anchor, @resolver_status, @git_commit_sha, @created_at)`, + ); - for (const claim of proposal.creates.claims ?? []) { - this.db - .prepare( - `INSERT INTO claims (id, kind, text, truth, intent, code_anchors) - VALUES (@id, @kind, @text, @truth, @intent, @code_anchors)`, - ) - .run({ - ...claim, - code_anchors: claim.code_anchors === undefined ? null : JSON.stringify(claim.code_anchors), - }); - this.createMembership(scopeId, "claim", claim.id, memoryCommitId); + const write = this.db.transaction((): string => { + const commit = this.createMemoryCommit({ + scope_id: input.scopeId, + title: input.commit.title, + summary: input.commit.summary, + git_commit_sha: input.commit.git_commit_sha, + }); + this.insertProposalRecords(input.scopeId, commit.id, input.proposal); + + const createdAt = now(); + for (const event of input.events) { + insertEvent.run({ + id: `ev_${randomUUID()}`, + repo_id: input.repoId, + original_claim_id: event.original_claim_id, + superseding_claim_id: event.superseding_claim_id, + memory_commit_id: commit.id, + reason: event.reason, + broken_anchor: event.broken_anchor, + resolver_status: event.resolver_status, + git_commit_sha: input.commit.git_commit_sha ?? null, + created_at: createdAt, + }); } - for (const source of proposal.creates.sources ?? []) { - this.db - .prepare("INSERT INTO sources (id, kind, ref, title) VALUES (@id, @kind, @ref, @title)") - .run({ ...source, title: source.title ?? null }); - } + // Demoted claims are no longer code_verified, so drop their fingerprints + // (in the same txn) to keep the reverse file->claims index tight. Derived + // from the events so every caller gets cleanup, not just the heal path. + this.deleteAnchorFingerprints(input.events.map((event) => event.original_claim_id)); - for (const edge of proposal.creates.edges ?? []) { - this.db - .prepare( - `INSERT INTO edges (id, from_id, from_type, to_id, to_type, kind, metadata) - VALUES (@id, @from_id, @from_type, @to_id, @to_type, @kind, @metadata)`, - ) - .run({ ...edge, metadata: edge.metadata === undefined ? null : JSON.stringify(edge.metadata) }); - this.createMembership(scopeId, "edge", edge.id, memoryCommitId); - } + return commit.id; }); - write(); + return { memory_commit_id: write() }; + } + + listInvalidationEvents(repoId: string): InvalidationEvent[] { + const rows = this.db + .prepare( + `SELECT id, repo_id, original_claim_id, superseding_claim_id, memory_commit_id, + reason, broken_anchor, resolver_status, git_commit_sha, created_at + FROM invalidation_events + WHERE repo_id = ? + ORDER BY created_at DESC`, + ) + .all(repoId) as InvalidationEventRow[]; + return rows.map((row) => ({ ...row, git_commit_sha: row.git_commit_sha ?? undefined })); + } + + /** Cache-aside write: upsert (INSERT OR REPLACE) the freshness fingerprints, one transaction. */ + upsertAnchorFingerprints(rows: AnchorFingerprintInput[]): void { + if (rows.length === 0) return; + const insert = this.db.prepare( + `INSERT OR REPLACE INTO anchor_fingerprints + (repo_id, claim_id, file, symbol, content_hash, file_mtime_ms, file_size, resolver_status, checked_at) + VALUES + (@repo_id, @claim_id, @file, @symbol, @content_hash, @file_mtime_ms, @file_size, @resolver_status, @checked_at)`, + ); + const write = this.db.transaction((records: AnchorFingerprintInput[]) => { + const checkedAt = now(); + for (const record of records) insert.run({ ...record, checked_at: checkedAt }); + }); + write(rows); + } + + /** Batch read (no N+1) of the fingerprints for the given claims. */ + fingerprintsForClaims(claimIds: string[]): AnchorFingerprintRow[] { + if (claimIds.length === 0) return []; + return this.db + .prepare(`SELECT * FROM anchor_fingerprints WHERE claim_id IN (${placeholders(claimIds)})`) + .all(...claimIds) as AnchorFingerprintRow[]; + } + + /** Reverse index: the distinct claims in this repo anchored in any of the given files. */ + claimIdsForFiles(repoId: string, files: string[]): string[] { + if (files.length === 0) return []; + const rows = this.db + .prepare(`SELECT DISTINCT claim_id FROM anchor_fingerprints WHERE repo_id = ? AND file IN (${placeholders(files)})`) + .all(repoId, ...files) as { claim_id: string }[]; + return rows.map((row) => row.claim_id); + } + + /** Drop every fingerprint row for the given claims (e.g. once they are demoted). */ + deleteAnchorFingerprints(claimIds: string[]): void { + if (claimIds.length === 0) return; + this.db.prepare(`DELETE FROM anchor_fingerprints WHERE claim_id IN (${placeholders(claimIds)})`).run(...claimIds); + } + + /** The HEAD sha at which this repo's claims were last checked for drift, if ever. */ + getFreshnessCheckpoint(repoId: string): string | undefined { + const row = this.db + .prepare("SELECT last_checked_sha FROM freshness_checkpoints WHERE repo_id = ?") + .get(repoId) as { last_checked_sha: string } | undefined; + return row?.last_checked_sha; + } + + /** Record the HEAD sha the heal just checked up to (cache-aside checkpoint). */ + setFreshnessCheckpoint(repoId: string, sha: string): void { + this.db + .prepare( + `INSERT OR REPLACE INTO freshness_checkpoints (repo_id, last_checked_sha, checked_at) + VALUES (?, ?, ?)`, + ) + .run(repoId, sha, now()); + } + + /** + * The re-verify worklist: current claims that were drift-demoted to `truth: + * unknown` and not yet rewritten. No queue table — derived from existing state, + * so a claim drops out for free once an agent supersedes it with a fresh one. + * ponytail: reuses readGraphView; swap to one SQL join only if the graph grows. + */ + claimsNeedingReverify(repoId: string, limit: number): Claim[] { + const demoted = new Set(this.listInvalidationEvents(repoId).map((event) => event.superseding_claim_id)); + if (demoted.size === 0) return []; + return this.readGraphView(repoId) + .claims.filter((claim) => claim.truth === "unknown" && demoted.has(claim.id)) + .slice(0, limit); + } + + private insertProposalRecords(scopeId: string, memoryCommitId: string, proposal: MemoryCommitProposal): void { + for (const component of proposal.creates.components ?? []) { + this.db + .prepare("INSERT INTO components (id, name, code_anchor) VALUES (@id, @name, @code_anchor)") + .run({ ...component, code_anchor: component.code_anchor ?? null }); + this.createMembership(scopeId, "component", component.id, memoryCommitId); + } + + for (const flow of proposal.creates.flows ?? []) { + this.db.prepare("INSERT INTO flows (id, name) VALUES (@id, @name)").run(flow); + this.createMembership(scopeId, "flow", flow.id, memoryCommitId); + } + + for (const claim of proposal.creates.claims ?? []) { + this.db + .prepare( + `INSERT INTO claims (id, kind, text, truth, intent, code_anchors) + VALUES (@id, @kind, @text, @truth, @intent, @code_anchors)`, + ) + .run({ + ...claim, + code_anchors: claim.code_anchors === undefined ? null : JSON.stringify(claim.code_anchors), + }); + this.createMembership(scopeId, "claim", claim.id, memoryCommitId); + } + + for (const source of proposal.creates.sources ?? []) { + this.db + .prepare("INSERT INTO sources (id, kind, ref, title) VALUES (@id, @kind, @ref, @title)") + .run({ ...source, title: source.title ?? null }); + } + + for (const edge of proposal.creates.edges ?? []) { + this.db + .prepare( + `INSERT INTO edges (id, from_id, from_type, to_id, to_type, kind, metadata) + VALUES (@id, @from_id, @from_type, @to_id, @to_type, @kind, @metadata)`, + ) + .run({ ...edge, metadata: edge.metadata === undefined ? null : JSON.stringify(edge.metadata) }); + this.createMembership(scopeId, "edge", edge.id, memoryCommitId); + } } subjectExists(type: GraphObjectType, id: string): boolean { diff --git a/libs/storage/sqlite/schema.ts b/libs/storage/sqlite/schema.ts index 6f94c0b..3e59dab 100644 --- a/libs/storage/sqlite/schema.ts +++ b/libs/storage/sqlite/schema.ts @@ -105,6 +105,44 @@ CREATE TABLE IF NOT EXISTS agent_worker_locks ( updated_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS invalidation_events ( + id TEXT PRIMARY KEY, + repo_id TEXT NOT NULL REFERENCES repos(id) ON DELETE CASCADE, + original_claim_id TEXT NOT NULL, + superseding_claim_id TEXT NOT NULL, + memory_commit_id TEXT NOT NULL REFERENCES memory_commits(id) ON DELETE CASCADE, + reason TEXT NOT NULL, + broken_anchor TEXT NOT NULL, + resolver_status TEXT NOT NULL, + git_commit_sha TEXT, + created_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS anchor_fingerprints ( + -- Scopes the reverse file->claims lookup to one repo (the DB is shared across + -- repos). ponytail: no FK to repos — matches the delete-free supersession model + -- (same as freshness_checkpoints); add ON DELETE CASCADE if repos ever get deleted. + repo_id TEXT NOT NULL, + claim_id TEXT NOT NULL, + file TEXT NOT NULL, + -- '' sentinel for file-only anchors: SQLite treats each NULL in a composite + -- PRIMARY KEY as distinct, which would let INSERT OR REPLACE accumulate + -- duplicate rows for the same (claim, file). A non-null key stays idempotent. + symbol TEXT NOT NULL DEFAULT '', + content_hash TEXT NOT NULL, + file_mtime_ms INTEGER NOT NULL, + file_size INTEGER NOT NULL, + resolver_status TEXT NOT NULL, + checked_at TEXT NOT NULL, + PRIMARY KEY (claim_id, file, symbol) +); + +CREATE TABLE IF NOT EXISTS freshness_checkpoints ( + repo_id TEXT PRIMARY KEY, + last_checked_sha TEXT NOT NULL, + checked_at TEXT NOT NULL +); + CREATE INDEX IF NOT EXISTS graph_scopes_repo_idx ON graph_scopes(repo_id); CREATE INDEX IF NOT EXISTS memory_commits_scope_idx ON memory_commits(scope_id); CREATE INDEX IF NOT EXISTS graph_memberships_scope_idx ON graph_memberships(scope_id); @@ -113,4 +151,7 @@ CREATE INDEX IF NOT EXISTS graph_object_embeddings_repo_idx ON graph_object_embe CREATE INDEX IF NOT EXISTS agent_sessions_repo_idx ON agent_sessions(repo_id); CREATE INDEX IF NOT EXISTS agent_sessions_seen_idx ON agent_sessions(last_seen_at); CREATE INDEX IF NOT EXISTS agent_worker_locks_until_idx ON agent_worker_locks(locked_until_at); +CREATE INDEX IF NOT EXISTS invalidation_events_repo_idx ON invalidation_events(repo_id); +CREATE INDEX IF NOT EXISTS invalidation_events_claim_idx ON invalidation_events(original_claim_id); +CREATE INDEX IF NOT EXISTS anchor_fingerprints_file_idx ON anchor_fingerprints(repo_id, file); `; diff --git a/libs/utils/git.ts b/libs/utils/git.ts new file mode 100644 index 0000000..8379dae --- /dev/null +++ b/libs/utils/git.ts @@ -0,0 +1,52 @@ +import { execFileSync } from "node:child_process"; + +/** Run a git command in `repoRoot`, returning stdout, or `undefined` on any failure. */ +function git(repoRoot: string, args: string[]): string | undefined { + try { + return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); + } catch { + return undefined; + } +} + +/** + * Current HEAD commit SHA for `repoRoot`, or `undefined` when unavailable (no root, + * not a git repo, git not installed). Best-effort: a provenance hint on memory + * writes, never a hard requirement. + */ +export function gitHeadSha(repoRoot: string | undefined): string | undefined { + if (repoRoot === undefined) return undefined; + const sha = git(repoRoot, ["rev-parse", "HEAD"])?.trim(); + return sha !== undefined && sha.length > 0 ? sha : undefined; +} + +/** + * Repo-relative paths changed since `sinceSha`: the union of committed changes + * (`git diff --name-only ..HEAD`) and uncommitted working-tree changes + * (`git status --porcelain`). The second half is what lets a caller catch edits + * that haven't been committed yet — the SHA-gate's blind spot. + * + * Returns `undefined` when the git probe fails (no repo, bad sha) — distinct from + * `[]` (nothing changed) — so the caller can full-sweep instead of silently + * skipping every claim. + */ +export function changedFilesSince(repoRoot: string | undefined, sinceSha: string | undefined): string[] | undefined { + if (repoRoot === undefined) return undefined; + const committed = sinceSha === undefined ? [] : gitLines(repoRoot, ["diff", "--name-only", `${sinceSha}..HEAD`]); + const uncommitted = gitLines(repoRoot, ["status", "--porcelain"]); + if (committed === undefined || uncommitted === undefined) return undefined; + return [...new Set([...committed, ...uncommitted.map(porcelainPath)])]; +} + +function gitLines(repoRoot: string, args: string[]): string[] | undefined { + const out = git(repoRoot, args); + if (out === undefined) return undefined; + return out.split("\n").map((line) => line.trimEnd()).filter((line) => line.length > 0); +} + +/** Extract the path from a `git status --porcelain` line (rename shows `old -> new`). */ +function porcelainPath(line: string): string { + const path = line.slice(3); + const arrow = path.indexOf(" -> "); + return arrow === -1 ? path : path.slice(arrow + 4); +} diff --git a/package.json b/package.json index a4d253e..944d349 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "typecheck": "tsc --noEmit", "smoke:openhands": "npm run build && node scripts/smoke-openhands-install.mjs", "smoke:copilot": "npm run build && node scripts/smoke-copilot-install.mjs", - "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js", + "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-anchor-drift.js && node scripts/check-span-hash.js && node scripts/check-freshness.js && node scripts/check-freshness-foreground.js && node scripts/check-freshness-background.js", + "test:anchor-drift": "npm run build && node scripts/check-anchor-drift.js", "test:transcript-bundle": "npm run build && node scripts/check-transcript-bundle.js", "test:repo-context": "npm run build && node scripts/check-repo-context.js", "eval:bootstrap-current": "npm run build && node dist/evals/cases/bootstrap-current-repo-at-8038fe8/run.js", diff --git a/scripts/check-anchor-drift.js b/scripts/check-anchor-drift.js new file mode 100644 index 0000000..478a66d --- /dev/null +++ b/scripts/check-anchor-drift.js @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); + +// Stub the embedding step so the suite never downloads a model; invalidation +// correctness does not depend on embeddings. +const stubContextBuilder = { ensureForGraph: async () => ({ checked_objects: 0, created: 0, reused: 0 }) }; + +function gitEnv() { + return { + ...process.env, + GIT_AUTHOR_NAME: "t", + GIT_AUTHOR_EMAIL: "t@t", + GIT_COMMITTER_NAME: "t", + GIT_COMMITTER_EMAIL: "t@t", + }; +} + +function setup() { + const home = mkdtempSync(join(tmpdir(), "greplica-drift-home-")); + const repoRoot = mkdtempSync(join(tmpdir(), "greplica-drift-repo-")); + execFileSync("git", ["init", "-q"], { cwd: repoRoot }); + execFileSync("git", ["commit", "--allow-empty", "-q", "-m", "init"], { cwd: repoRoot, env: gitEnv() }); + + const repository = new SqliteRepository(openDatabase(join(home, "graph.db"))); + const service = new KnowledgeGraphService(repository, undefined, stubContextBuilder); + const ref = { repo_root: repoRoot, repo_name: "drift-test", default_branch: "main" }; + service.initRepo(ref); + return { repoRoot, repository, service, ref }; +} + +function write(repoRoot, name, contents) { + writeFileSync(join(repoRoot, name), contents); +} + +// 1. Happy path: rename a symbol -> the claim is demoted, superseded, logged, +// and kept in history; edges (with reasons) are re-pointed at the rebuild. +async function happyPath() { + const { repoRoot, repository, service, ref } = setup(); + write(repoRoot, "auth.ts", "export function validateToken(t){return t.length>0;}\n"); + await service.applyProposal(ref, { + title: "seed", + creates: { + components: [{ id: "comp.auth", name: "auth" }], + sources: [{ id: "src.s1", kind: "session", ref: "codex:1", title: "s1" }], + claims: [ + { + id: "claim.tv", + kind: "fact", + text: "validated in validateToken", + truth: "code_verified", + intent: "intended", + code_anchors: [{ file: "auth.ts", symbol: "validateToken" }], + about: ["comp.auth"], + }, + ], + edges: [{ kind: "evidenced_by", from: "claim.tv", to: "src.s1", metadata: { reason: "stated in s1" } }], + }, + }); + + write(repoRoot, "auth.ts", "export function verifyToken(t){return t.length>0;}\n"); + const result = await service.invalidateDriftedAnchors(ref); + + assert.equal(result.invalidated.length, 1); + assert.deepEqual(result.invalidated[0], { + claim_id: "claim.tv", + superseding_claim_id: "claim.tv__drift", + broken_anchor: "auth.ts#validateToken", + resolver_status: "missing_symbol", + }); + assert.equal(typeof result.memory_commit_id, "string"); + assert.deepEqual(result.errors, []); + + const graph = service.readGraph(ref); + assert.equal(graph.claims.find((c) => c.id === "claim.tv"), undefined, "original is superseded"); + const rebuilt = graph.claims.find((c) => c.id === "claim.tv__drift"); + assert.ok(rebuilt, "rebuilt claim exists"); + assert.equal(rebuilt.truth, "unknown"); + assert.deepEqual(rebuilt.code_anchors, [{ file: "auth.ts", symbol: "validateToken" }], "broken anchor kept as evidence"); + + // The `supersedes` edge itself is intentionally hidden by readGraphView (it + // points at the now-inactive original); its effect — the original dropping out + // of the active view, asserted above — is the observable proof it was written. + const rebuiltEdges = graph.edges.filter((e) => e.from_id === "claim.tv__drift"); + assert.ok(rebuiltEdges.some((e) => e.kind === "about" && e.to_id === "comp.auth"), "about edge cloned"); + const evidence = rebuiltEdges.find((e) => e.kind === "evidenced_by"); + assert.equal(evidence?.metadata?.reason, "stated in s1", "evidenced_by reason preserved"); + + const repoId = service.requireRepo(ref).repo_id; + const events = repository.listInvalidationEvents(repoId); + assert.equal(events.length, 1); + assert.equal(events[0].original_claim_id, "claim.tv"); + assert.equal(events[0].resolver_status, "missing_symbol"); + assert.equal(repository.subjectExists("claim", "claim.tv"), true, "original retained in history"); + + const again = await service.invalidateDriftedAnchors(ref); + assert.equal(again.invalidated.length, 0, "second run is idempotent"); + assert.equal(again.memory_commit_id, undefined); +} + +// 2. Option A: a multi-anchor claim survives while any anchor resolves, and is +// demoted only once every anchor is broken. +async function optionA() { + const { repoRoot, service, ref } = setup(); + write(repoRoot, "auth.ts", "export function issueToken(){return 'x';}\nexport function validateToken(t){return t.length>0;}\n"); + await service.applyProposal(ref, { + title: "seed", + creates: { + claims: [ + { + id: "claim.multi", + kind: "fact", + text: "issue and validate", + truth: "code_verified", + intent: "intended", + code_anchors: [ + { file: "auth.ts", symbol: "issueToken" }, + { file: "auth.ts", symbol: "validateToken" }, + ], + }, + ], + }, + }); + + write(repoRoot, "auth.ts", "export function issueToken(){return 'x';}\nexport function verifyToken(t){return t.length>0;}\n"); + const partial = await service.invalidateDriftedAnchors(ref); + assert.equal(partial.invalidated.length, 0, "not demoted while one anchor resolves"); + + write(repoRoot, "auth.ts", "export const nothing = 1;\n"); + const full = await service.invalidateDriftedAnchors(ref); + assert.equal(full.invalidated.length, 1, "demoted once all anchors are broken"); + assert.equal(full.invalidated[0].claim_id, "claim.multi"); +} + +// 3. Unchanged and unsupported-language anchors are never demoted. +async function neverDemoted() { + const { repoRoot, repository, service, ref } = setup(); + write(repoRoot, "keep.ts", "export function stable(){return 1;}\n"); + write(repoRoot, "data.xyz", "blob foo blob\n"); + + await service.applyProposal(ref, { + title: "seed", + creates: { + claims: [ + { id: "claim.stable", kind: "fact", text: "stable", truth: "code_verified", intent: "intended", code_anchors: [{ file: "keep.ts", symbol: "stable" }] }, + ], + }, + }); + + // An unsupported-language anchor cannot pass proposal validation, so seed it + // straight through the repository to exercise the invalidation policy. + const repoId = service.requireRepo(ref).repo_id; + const working = repository.requireWorkingScope(repoId); + const commit = repository.createMemoryCommit({ scope_id: working.id, title: "direct" }); + repository.createProposalRecords(working.id, commit.id, { + title: "direct", + creates: { + claims: [ + { id: "claim.unsup", kind: "fact", text: "unsupported", truth: "code_verified", intent: "intended", code_anchors: [{ file: "data.xyz", symbol: "foo" }] }, + ], + }, + }); + + const result = await service.invalidateDriftedAnchors(ref); + assert.equal(result.invalidated.length, 0, "unchanged and unsupported-language claims are left alone"); +} + +await happyPath(); +await optionA(); +await neverDemoted(); + +console.log("Anchor drift checks passed."); diff --git a/scripts/check-freshness-background.js b/scripts/check-freshness-background.js new file mode 100644 index 0000000..a7e4f7e --- /dev/null +++ b/scripts/check-freshness-background.js @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); + +function git(cwd, ...args) { + return execFileSync("git", ["-c", "user.email=t@t.io", "-c", "user.name=t", ...args], { + cwd, + stdio: ["ignore", "pipe", "ignore"], + }).toString().trim(); +} + +// --------------------------------------------------------------------------- +// changedFilesSince: git diff (committed) ∪ git status --porcelain (uncommitted) +// --------------------------------------------------------------------------- +const { changedFilesSince } = await import(new URL("dist/libs/utils/git.js", root)); + +const gitRepo = mkdtempSync(join(tmpdir(), "greplica-cf-")); +git(gitRepo, "init", "-q"); +writeFileSync(join(gitRepo, "a.ts"), "export const a = 1;\n"); +writeFileSync(join(gitRepo, "b.ts"), "export const b = 1;\n"); +git(gitRepo, "add", "-A"); +git(gitRepo, "commit", "-qm", "seed"); +const headSha = git(gitRepo, "rev-parse", "HEAD"); + +// Uncommitted edit to a.ts is caught; b.ts (untouched) is not. +writeFileSync(join(gitRepo, "a.ts"), "export const a = 2;\n"); +let changed = changedFilesSince(gitRepo, headSha); +assert.ok(changed.includes("a.ts"), "uncommitted edit is caught"); +assert.ok(!changed.includes("b.ts"), "untouched file is not reported"); + +// Commit the edit -> still reported as changed since the old headSha (committed diff). +git(gitRepo, "add", "-A"); +git(gitRepo, "commit", "-qm", "edit a"); +changed = changedFilesSince(gitRepo, headSha); +assert.ok(changed.includes("a.ts"), "committed change since sinceSha is caught"); + +// From the new HEAD with a clean tree -> nothing changed. +const headSha2 = git(gitRepo, "rev-parse", "HEAD"); +assert.deepEqual(changedFilesSince(gitRepo, headSha2), [], "clean tree at HEAD -> no changes"); + +// Git probe failure -> undefined (distinct from [] clean), so the caller full-sweeps. +assert.equal(changedFilesSince(mkdtempSync(join(tmpdir(), "greplica-nogit-")), headSha), undefined, "no git -> undefined"); +assert.equal(changedFilesSince(undefined, undefined), undefined, "no repo root -> undefined"); + +// --------------------------------------------------------------------------- +// freshness checkpoint + fingerprint deletion (repository) +// --------------------------------------------------------------------------- +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); + +const cpRepo = new SqliteRepository(openDatabase(join(mkdtempSync(join(tmpdir(), "greplica-cp-")), "graph.db"))); +assert.equal(cpRepo.getFreshnessCheckpoint("repo1"), undefined, "no checkpoint -> undefined"); +cpRepo.setFreshnessCheckpoint("repo1", "sha-abc"); +assert.equal(cpRepo.getFreshnessCheckpoint("repo1"), "sha-abc", "checkpoint round-trips"); +cpRepo.setFreshnessCheckpoint("repo1", "sha-def"); +assert.equal(cpRepo.getFreshnessCheckpoint("repo1"), "sha-def", "checkpoint updates in place"); +assert.equal(cpRepo.getFreshnessCheckpoint("repo2"), undefined, "checkpoints are per-repo"); + +cpRepo.upsertAnchorFingerprints([ + { repo_id: "r1", claim_id: "cx", file: "x.ts", symbol: "s", content_hash: "h", file_mtime_ms: 1, file_size: 2, resolver_status: "resolved" }, +]); +assert.equal(cpRepo.fingerprintsForClaims(["cx"]).length, 1, "fingerprint written"); +cpRepo.deleteAnchorFingerprints(["cx"]); +assert.equal(cpRepo.fingerprintsForClaims(["cx"]).length, 0, "fingerprints deleted for demoted claim"); + +// --------------------------------------------------------------------------- +// buildAnchorInvalidation generalizes to content drift (not just structural) +// --------------------------------------------------------------------------- +const { buildAnchorInvalidation } = await import(new URL("dist/libs/knowledge-graph/anchor-invalidation.js", root)); + +const demotedClaim = { id: "claim.c", kind: "fact", text: "t", truth: "code_verified", intent: "intended", code_anchors: [{ file: "a.ts", symbol: "f" }] }; +const miniGraph = { components: [], flows: [], claims: [demotedClaim], sources: [], edges: [] }; + +// content drift: the anchor still resolves, but its span changed. +const contentPlan = buildAnchorInvalidation( + [{ claim: demotedClaim, reason: "content", anchors: [{ file: "a.ts", symbol: "f", status: "resolved", start_line: 1, end_line: 1 }] }], + miniGraph, +); +assert.equal(contentPlan.events[0].reason, "content_drift", "content demotion -> content_drift event"); +assert.equal(contentPlan.events[0].resolver_status, "resolved", "content event records the resolving status"); +assert.ok(contentPlan.proposal.creates.claims.some((c) => c.truth === "unknown"), "content demotion rebuilds a truth:unknown claim"); + +// structural drift: unchanged behavior (broken anchor + anchor_drift reason). +const structuralPlan = buildAnchorInvalidation( + [{ claim: demotedClaim, reason: "structural", anchors: [{ file: "a.ts", symbol: "f", status: "missing_symbol" }] }], + miniGraph, +); +assert.equal(structuralPlan.events[0].reason, "anchor_drift", "structural demotion -> anchor_drift event"); +assert.equal(structuralPlan.events[0].resolver_status, "missing_symbol", "structural event records the drift status"); + +// --------------------------------------------------------------------------- +// service.healDriftedAnchors: change-scoped structural + content heal +// --------------------------------------------------------------------------- +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); +const stubBuilder = { ensureForGraph: async () => ({ checked_objects: 0, created: 0, reused: 0 }) }; + +const healRoot = mkdtempSync(join(tmpdir(), "greplica-heal-")); +git(healRoot, "init", "-q"); +writeFileSync(join(healRoot, "a.ts"), "export function fa() { return 1; }\n"); +writeFileSync(join(healRoot, "b.ts"), "export function fb() { return 1; }\n"); +git(healRoot, "add", "-A"); +git(healRoot, "commit", "-qm", "seed"); + +const healRepo = new SqliteRepository(openDatabase(join(mkdtempSync(join(tmpdir(), "greplica-heal-home-")), "graph.db"))); +const healSvc = new KnowledgeGraphService(healRepo, undefined, stubBuilder); +const healRef = { repo_root: healRoot, repo_name: "heal", default_branch: "main" }; +healSvc.initRepo(healRef); +await healSvc.applyProposal(healRef, { title: "seed", creates: { claims: [ + { id: "claim.a", kind: "fact", text: "a", truth: "code_verified", intent: "intended", code_anchors: [{ file: "a.ts", symbol: "fa" }] }, + { id: "claim.b", kind: "fact", text: "b", truth: "code_verified", intent: "intended", code_anchors: [{ file: "b.ts", symbol: "fb" }] }, +]}}); +const head = git(healRoot, "rev-parse", "HEAD"); + +// Content-edit a.ts only (uncommitted). Heal must demote claim.a, not claim.b, +// and only recheck the changed file's claim (reverse index). +writeFileSync(join(healRoot, "a.ts"), "export function fa() { return 99999; }\n"); +const healed = await healSvc.healDriftedAnchors(healRef, head); +assert.deepEqual(healed.demoted, ["claim.a"], "content-drifted claim demoted"); +assert.equal(healed.rechecked, 1, "only the changed file's claim was rechecked (reverse index)"); +assert.equal(typeof healed.headSha, "string", "heal returns the current HEAD sha"); + +const g = healSvc.readGraph(healRef); +assert.ok(g.claims.some((c) => c.id === "claim.b" && c.truth === "code_verified"), "unchanged claim untouched"); +assert.ok(g.claims.some((c) => c.truth === "unknown"), "drifted claim rebuilt as unknown"); +assert.ok(!g.claims.some((c) => c.id === "claim.a" && c.truth === "code_verified"), "original demoted"); +assert.equal(healRepo.fingerprintsForClaims(["claim.a"]).length, 0, "demoted claim fingerprints removed"); + +// Second heal with no new edits -> idempotent no-op (early cutoff). +const again = await healSvc.healDriftedAnchors(healRef, head); +assert.deepEqual(again.demoted, [], "second heal demotes nothing (idempotent)"); + +// Unknown never demotes: make b.ts unreadable, run a full sweep (no sinceSha). +chmodSync(join(healRoot, "b.ts"), 0o000); +const sweep = await healSvc.healDriftedAnchors(healRef); +chmodSync(join(healRoot, "b.ts"), 0o644); +assert.ok(!sweep.demoted.includes("claim.b"), "unreadable (unknown) claim is not demoted"); + +// Bad checkpoint sha -> git probe fails -> full sweep (never a silent skip). +writeFileSync(join(healRoot, "b.ts"), "export function fb() { return 12345; }\n"); +const badSha = await healSvc.healDriftedAnchors(healRef, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); +assert.deepEqual(badSha.demoted, ["claim.b"], "bad checkpoint sha -> full sweep demotes the drifted claim"); + +// Checkpoint advanced to the current HEAD. +assert.equal(healRepo.getFreshnessCheckpoint(healSvc.requireRepo(healRef).repo_id), head, "checkpoint set to HEAD"); + +// --------------------------------------------------------------------------- +// config default + worker drift-heal pass (dedupe + gate) +// --------------------------------------------------------------------------- +const { defaultSessionConfig } = await import(new URL("dist/libs/config/greplica-config.js", root)); +assert.equal(defaultSessionConfig.autoHealDrift, true, "autoHealDrift defaults on"); + +const { runDriftHealPass } = await import(new URL("dist/libs/hooks/worker.js", root)); +const healCalls = []; +const fakeService = { + healDriftedAnchorsFromCheckpoint: async (ref) => { + healCalls.push(ref.repo_root); + return { demoted: [], rechecked: 1 }; + }, +}; +const r = (path) => ({ repo_root: path, repo_name: path, default_branch: "main" }); + +// Gate off -> no heal. +await runDriftHealPass(fakeService, [r("/r1")], false, () => true, () => {}); +assert.equal(healCalls.length, 0, "autoHealDrift off -> no heal"); + +// Gate on -> each distinct repo healed once (deduped). +await runDriftHealPass(fakeService, [r("/r1"), r("/r1"), r("/r2")], true, () => true, () => {}); +assert.deepEqual(healCalls, ["/r1", "/r2"], "heals each distinct repo once"); + +// --------------------------------------------------------------------------- +// Phase 4: demoted (truth:unknown) claims form the re-verify worklist +// --------------------------------------------------------------------------- +// (healSvc has demoted claim.a and claim.b to `unknown` in the steps above.) +const worklist = healSvc.reverifyWorklist(healRef, 10); +assert.ok(worklist.length > 0, "demoted claims are queued for re-verify"); +assert.ok(worklist.every((c) => c.truth === "unknown"), "worklist holds only unknown claims"); +assert.ok(worklist.some((c) => c.id === "claim.a__drift"), "the drift-demoted claim is queued"); +assert.equal(healSvc.reverifyWorklist(healRef, 1).length, 1, "worklist respects the limit"); + +// A manually-created unknown claim (not drift-demoted) is NOT queued. +await healSvc.applyProposal(healRef, { title: "manual", creates: { claims: [ + { id: "claim.manual", kind: "question", text: "q", truth: "unknown", intent: "unknown" }, +]}}); +assert.ok(!healSvc.reverifyWorklist(healRef, 10).some((c) => c.id === "claim.manual"), "non-drift unknown claim is not queued"); + +// reverifyPrompt names the claims; runReverifyPass hands them to the agent runner. +const { reverifyPrompt, runReverifyPass } = await import(new URL("dist/libs/hooks/worker.js", root)); +const driftedClaim = { id: "claim.x__drift", kind: "fact", text: "x does y", truth: "unknown", intent: "intended", code_anchors: [{ file: "a.ts", symbol: "fa" }] }; + +const prompt = reverifyPrompt([driftedClaim], "/tmp/p.json"); +assert.ok(prompt.includes("claim.x__drift"), "prompt names the claim"); +assert.ok(prompt.includes("a.ts#fa"), "prompt names the anchor"); +assert.ok(/re-verify/i.test(prompt), "prompt asks to re-verify"); + +let captured; +await runReverifyPass({ runWorkingMemoryUpdate: async (input) => { captured = input.prompt; } }, healRoot, [driftedClaim]); +assert.ok(captured.includes("claim.x__drift"), "runReverifyPass hands the worklist to the agent"); + +let called = false; +await runReverifyPass({ runWorkingMemoryUpdate: async () => { called = true; } }, healRoot, []); +assert.equal(called, false, "empty worklist -> no agent spawn"); + +console.log("Freshness background checks passed."); diff --git a/scripts/check-freshness-foreground.js b/scripts/check-freshness-foreground.js new file mode 100644 index 0000000..8bad47e --- /dev/null +++ b/scripts/check-freshness-foreground.js @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); +const { attachFreshness } = await import(new URL("dist/libs/knowledge-graph/graph-context/claim-freshness.js", root)); +const { hashAnchorSpan, statAnchorFile } = await import(new URL("dist/libs/knowledge-graph/code-anchors/span-hash.js", root)); + +const repoRoot = mkdtempSync(join(tmpdir(), "greplica-fg-ctx-")); +writeFileSync(join(repoRoot, "svc.ts"), "line1\nexport function handle() { return 1; }\nline3\n"); +const anchor = { file: "svc.ts", symbol: "handle", status: "resolved", start_line: 2, end_line: 2 }; + +const db = openDatabase(join(mkdtempSync(join(tmpdir(), "greplica-fg-db-")), "graph.db")); +const repo = new SqliteRepository(db); + +// Minimal ClaimContextResult (sans the freshness attachFreshness computes). +const claimResult = (id, anchors) => ({ + rank: 1, score: 1, signals: {}, about: [], evidence: [], code_anchors: anchors, + object: { id, kind: "fact", text: "t", truth: "code_verified", intent: "intended" }, +}); + +// No fingerprint stored yet, span readable -> fresh (no false stale/unknown). +let out = attachFreshness([claimResult("c1", [anchor])], repo, repoRoot); +assert.equal(out[0].freshness.state, "fresh", "no baseline, readable -> fresh"); + +// Seed a fingerprint matching the current file -> fresh. +const stat = statAnchorFile(repoRoot, "svc.ts"); +repo.upsertAnchorFingerprints([ + { repo_id: "r1", claim_id: "c1", file: "svc.ts", symbol: "handle", content_hash: hashAnchorSpan(repoRoot, anchor), file_mtime_ms: stat.mtime_ms, file_size: stat.size, resolver_status: "resolved" }, +]); +out = attachFreshness([claimResult("c1", [anchor])], repo, repoRoot); +assert.equal(out[0].freshness.state, "fresh", "baseline matches current -> fresh"); + +// Change the body -> content drift (foreground detects it live). +writeFileSync(join(repoRoot, "svc.ts"), "line1\nexport function handle() { return 99999; }\nline3\n"); +out = attachFreshness([claimResult("c1", [anchor])], repo, repoRoot); +assert.equal(out[0].freshness.state, "stale", "changed body -> stale"); +assert.equal(out[0].freshness.reason, "content", "changed body -> content drift"); + +// Unreadable anchor -> unknown. +out = attachFreshness([claimResult("c2", [{ file: "gone.ts", symbol: "x", status: "resolved", start_line: 1, end_line: 1 }])], repo, repoRoot); +assert.equal(out[0].freshness.state, "unknown", "unreadable anchor -> unknown"); + +// All anchors broken -> structural. +out = attachFreshness([claimResult("c3", [{ file: "svc.ts", symbol: "handle", status: "missing_symbol" }])], repo, repoRoot); +assert.equal(out[0].freshness.state, "stale", "all broken -> stale"); +assert.equal(out[0].freshness.reason, "structural", "all broken -> structural"); + +// Read-only: attachFreshness returns new objects, never mutates its input. +const input = claimResult("c1", [anchor]); +attachFreshness([input], repo, repoRoot); +assert.equal(input.freshness, undefined, "attachFreshness does not mutate its input"); + +// --- render: truth + `## Needs re-verification` section --- +const { renderGraphContextMarkdown } = await import(new URL("dist/libs/knowledge-graph/graph-context/render.js", root)); + +const claimItem = (id, truth, freshness, anchors = []) => ({ + type: "claim", rank: 1, score: 1, signals: {}, about: [], evidence: [], code_anchors: anchors, + object: { id, kind: "fact", text: `text of ${id}`, truth, intent: "intended" }, freshness, +}); +const freshV = { state: "fresh", reason: null, broken: [] }; +const staleV = { state: "stale", reason: "content", broken: [] }; +const unknownV = { state: "unknown", reason: null, broken: [] }; +const packet = (items) => ({ + query: "q", search_config_version: "v", embedding_status: { checked_objects: 0, created: 0, reused: 0 }, + claims: [], components: [], flows: [], sources: [], ranked_results: items, +}); + +const md = renderGraphContextMarkdown(packet([ + claimItem("claim.fresh", "code_verified", freshV), + claimItem("claim.stale", "code_verified", staleV), + claimItem("claim.unknown", "code_verified", unknownV), +])); + +const bestIdx = md.indexOf("## Best Claims"); +const reverifyIdx = md.indexOf("## Needs re-verification"); +assert.ok(reverifyIdx !== -1, "stale claims get a Needs re-verification section"); +assert.ok(md.indexOf("claim.stale") > reverifyIdx, "stale claim rendered under Needs re-verification"); +assert.ok(md.indexOf("claim.fresh") > bestIdx && md.indexOf("claim.fresh") < reverifyIdx, "fresh claim under Best Claims"); +assert.ok(md.indexOf("claim.unknown") > bestIdx && md.indexOf("claim.unknown") < reverifyIdx, "unknown claim stays under Best Claims"); +assert.ok(md.includes("code_verified"), "truth surfaced on claim lines"); +assert.ok(/unverifiable/i.test(md), "unknown claim carries an unverifiable caveat"); +assert.ok(/re-verify/i.test(md), "stale claim carries a re-verify instruction"); + +// No stale claims -> the section is omitted (no noise on healthy packets). +const mdClean = renderGraphContextMarkdown(packet([claimItem("claim.ok", "code_verified", freshV)])); +assert.ok(!mdClean.includes("## Needs re-verification"), "no stale section when nothing drifted"); + +console.log("Freshness foreground checks passed."); diff --git a/scripts/check-freshness.js b/scripts/check-freshness.js new file mode 100644 index 0000000..df578bd --- /dev/null +++ b/scripts/check-freshness.js @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; + +const root = new URL("..", import.meta.url); +const { classifyFreshness } = await import(new URL("dist/libs/knowledge-graph/code-anchors/freshness.js", root)); + +const resolvesAnchor = { file: "a.ts", symbol: "f", status: "resolved" }; +const brokenAnchor = { file: "a.ts", symbol: "f", status: "missing_symbol" }; +// An AnchorCheck bundles one anchor with its current + last-known span hash. +const check = (anchor, currentHash, storedHash) => ({ anchor, currentHash, storedHash }); + +// No anchors -> fresh. +assert.equal(classifyFreshness([]).state, "fresh", "no anchors -> fresh"); + +// Every anchor broken -> structural drift. +const structural = classifyFreshness([check(brokenAnchor, undefined, "h1")]); +assert.equal(structural.state, "stale"); +assert.equal(structural.reason, "structural"); +assert.equal(structural.broken.length, 1, "structural verdict carries the broken anchor"); + +// Resolves and the span hash matches -> fresh. +assert.equal(classifyFreshness([check(resolvesAnchor, "h1", "h1")]).state, "fresh", "unchanged span -> fresh"); + +// Resolves but the span hash changed -> content drift. +const content = classifyFreshness([check(resolvesAnchor, "h2", "h1")]); +assert.equal(content.state, "stale"); +assert.equal(content.reason, "content"); + +// No stored fingerprint yet -> cannot compare -> fresh (never a false stale). +assert.equal(classifyFreshness([check(resolvesAnchor, "h1", undefined)]).state, "fresh", "no baseline -> fresh"); + +// One anchor broken, another resolves with a changed hash -> content (not all broken). +const mixed = classifyFreshness([check(brokenAnchor, undefined, "h0"), check(resolvesAnchor, "h2", "h1")]); +assert.equal(mixed.reason, "content", "partial break + changed hash -> content"); +assert.equal(mixed.broken.length, 1, "content verdict still surfaces the structurally-broken anchor"); +assert.equal(mixed.broken[0].status, "missing_symbol", "the broken anchor is carried through"); + +// Resolving anchor whose span can't be hashed now (unreadable file / resolver error) -> unknown. +const unknown = classifyFreshness([check(resolvesAnchor, undefined, "h1")]); +assert.equal(unknown.state, "unknown", "unreadable span -> unknown"); +assert.equal(unknown.reason, null, "unknown carries no drift reason"); +assert.equal(unknown.broken.length, 0, "unknown carries no broken anchors"); + +// Undeterminable with no baseline either -> still unknown (we couldn't read the code). +assert.equal(classifyFreshness([check(resolvesAnchor, undefined, undefined)]).state, "unknown", "no current hash -> unknown"); + +// Real drift always beats unknown: a changed hash on any anchor still wins as content. +assert.equal( + classifyFreshness([check(resolvesAnchor, undefined, "h1"), check(resolvesAnchor, "h2", "h1")]).state, + "stale", + "content drift beats unknown", +); + +// One anchor structurally broken, another undeterminable -> unknown, but the broken +// anchor is still surfaced (Phase 3's healer needs to know which anchors broke). +const brokenPlusUnknown = classifyFreshness([check(brokenAnchor, undefined, "h1"), check(resolvesAnchor, undefined, "h1")]); +assert.equal(brokenPlusUnknown.state, "unknown", "partial break + undeterminable -> unknown"); +assert.equal(brokenPlusUnknown.broken.length, 1, "unknown verdict still surfaces the structurally-broken anchor"); +assert.equal(brokenPlusUnknown.broken[0].status, "missing_symbol", "the broken anchor is carried through"); + +// --- storage layer: anchor_fingerprints table + repository --- +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); + +const db = openDatabase(join(mkdtempSync(join(tmpdir(), "greplica-fp-")), "graph.db")); +const table = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='anchor_fingerprints'").get(); +assert.equal(table?.name, "anchor_fingerprints", "table exists"); +const idx = db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name='anchor_fingerprints_file_idx'").get(); +assert.equal(idx?.name, "anchor_fingerprints_file_idx", "file index exists"); + +const repo = new SqliteRepository(db); +assert.deepEqual(repo.claimIdsForFiles("r1", []), [], "empty input -> empty"); +assert.deepEqual(repo.fingerprintsForClaims([]), [], "empty input -> empty"); + +repo.upsertAnchorFingerprints([ + { repo_id: "r1", claim_id: "c1", file: "a.ts", symbol: "f", content_hash: "h1", file_mtime_ms: 10, file_size: 20, resolver_status: "resolved" }, + { repo_id: "r1", claim_id: "c2", file: "b.ts", symbol: "g", content_hash: "h9", file_mtime_ms: 5, file_size: 6, resolver_status: "resolved" }, + { repo_id: "r2", claim_id: "cother", file: "a.ts", symbol: "f", content_hash: "hx", file_mtime_ms: 1, file_size: 1, resolver_status: "resolved" }, +]); +assert.deepEqual(repo.claimIdsForFiles("r1", ["a.ts"]), ["c1"], "reverse index is repo-scoped (excludes other repos' same-path claims)"); +assert.equal(repo.fingerprintsForClaims(["c1"])[0].content_hash, "h1", "read fingerprint back"); +assert.equal(repo.fingerprintsForClaims(["c1"])[0].checked_at !== undefined, true, "checked_at stamped"); + +repo.upsertAnchorFingerprints([ + { repo_id: "r1", claim_id: "c1", file: "a.ts", symbol: "f", content_hash: "h2", file_mtime_ms: 11, file_size: 21, resolver_status: "resolved" }, +]); +assert.equal(repo.fingerprintsForClaims(["c1"])[0].content_hash, "h2", "upsert replaces existing row"); +assert.equal(repo.fingerprintsForClaims(["c1"]).length, 1, "no duplicate row on upsert"); + +// File-only anchors use the "" symbol sentinel and must upsert idempotently +// (a nullable PK column would let SQLite treat each NULL as a distinct row). +repo.upsertAnchorFingerprints([ + { repo_id: "r1", claim_id: "c3", file: "d.ts", symbol: "", content_hash: "h1", file_mtime_ms: 1, file_size: 2, resolver_status: "resolved" }, +]); +repo.upsertAnchorFingerprints([ + { repo_id: "r1", claim_id: "c3", file: "d.ts", symbol: "", content_hash: "h2", file_mtime_ms: 3, file_size: 4, resolver_status: "resolved" }, +]); +assert.equal(repo.fingerprintsForClaims(["c3"]).length, 1, "file-only anchor upserts, no duplicate"); +assert.equal(repo.fingerprintsForClaims(["c3"])[0].content_hash, "h2", "file-only anchor row replaced"); + +// --- service: fingerprints are written when a proposal is applied --- +import { writeFileSync } from "node:fs"; +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); +const stubBuilder = { ensureForGraph: async () => ({ checked_objects: 0, created: 0, reused: 0 }) }; + +const repoRoot = mkdtempSync(join(tmpdir(), "greplica-fp-repo-")); +writeFileSync(join(repoRoot, "auth.ts"), "export function validateToken(t) { return t.length > 0; }\n"); +const svcRepo = new SqliteRepository(openDatabase(join(mkdtempSync(join(tmpdir(), "greplica-fp-home-")), "graph.db"))); +const svc = new KnowledgeGraphService(svcRepo, undefined, stubBuilder); +const ref = { repo_root: repoRoot, repo_name: "fp", default_branch: "main" }; +svc.initRepo(ref); + +await svc.applyProposal(ref, { title: "seed", creates: { claims: [ + { id: "claim.tv", kind: "fact", text: "t", truth: "code_verified", intent: "intended", code_anchors: [{ file: "auth.ts", symbol: "validateToken" }] }, +]}}); +const fps = svcRepo.fingerprintsForClaims(["claim.tv"]); +assert.equal(fps.length, 1, "fingerprint written on apply for code_verified claim"); +assert.equal(fps[0].file, "auth.ts"); +assert.equal(fps[0].resolver_status, "resolved"); +assert.equal(fps[0].content_hash.length, 64, "sha256 span hash stored"); + +await svc.applyProposal(ref, { title: "seed2", creates: { claims: [ + { id: "claim.sv", kind: "decision", text: "d", truth: "source_verified", intent: "intended" }, +]}}); +assert.equal(svcRepo.fingerprintsForClaims(["claim.sv"]).length, 0, "source_verified claim -> no fingerprint"); + +// --- foreground: fingerprint index + stat-prefiltered freshness checks --- +const { indexFingerprintsByClaim, freshnessChecks } = await import(new URL("dist/libs/knowledge-graph/anchor-fingerprints.js", root)); +const { hashAnchorSpan, statAnchorFile } = await import(new URL("dist/libs/knowledge-graph/code-anchors/span-hash.js", root)); + +const fgRoot = mkdtempSync(join(tmpdir(), "greplica-fg-")); +writeFileSync(join(fgRoot, "svc.ts"), "line1\nexport function handle() { return 1; }\nline3\n"); +const fgAnchor = { file: "svc.ts", symbol: "handle", status: "resolved", start_line: 2, end_line: 2 }; + +// Readable span, no stored fingerprint yet -> fresh (no false stale, no false unknown). +assert.equal(classifyFreshness(freshnessChecks([fgAnchor], undefined, fgRoot)).state, "fresh", "readable span, no baseline -> fresh"); + +// Build a stored row that matches the current file (hash + real stat). +const fgStat = statAnchorFile(fgRoot, "svc.ts"); +const fgHash = hashAnchorSpan(fgRoot, fgAnchor); +const fgRows = [ + { repo_id: "r1", claim_id: "cf", file: "svc.ts", symbol: "handle", content_hash: fgHash, file_mtime_ms: fgStat.mtime_ms, file_size: fgStat.size, resolver_status: "resolved", checked_at: "t" }, +]; +const fgIndex = indexFingerprintsByClaim(fgRows); +assert.equal(fgIndex.get("cf").size, 1, "fingerprints grouped by claim id"); + +// Stat prefilter hit: file untouched -> reuse stored hash (no re-hash) -> fresh. +const hit = freshnessChecks([fgAnchor], fgIndex.get("cf"), fgRoot); +assert.equal(hit[0].currentHash, fgHash, "prefilter hit reuses the stored hash"); +assert.equal(classifyFreshness(hit).state, "fresh", "unchanged file -> fresh"); + +// Body changed (size differs -> prefilter miss) -> re-hash -> content drift. +writeFileSync(join(fgRoot, "svc.ts"), "line1\nexport function handle() { return 99999; }\nline3\n"); +const miss = freshnessChecks([fgAnchor], fgIndex.get("cf"), fgRoot); +assert.notEqual(miss[0].currentHash, fgHash, "prefilter miss re-hashes the span"); +assert.equal(classifyFreshness(miss).reason, "content", "changed body -> content drift"); + +// Unreadable anchor -> current hash undefined -> unknown. +const goneChecks = freshnessChecks([{ file: "gone.ts", symbol: "x", status: "resolved", start_line: 1, end_line: 1 }], undefined, fgRoot); +assert.equal(classifyFreshness(goneChecks).state, "unknown", "unreadable anchor -> unknown"); + +console.log("Freshness checks passed."); diff --git a/scripts/check-span-hash.js b/scripts/check-span-hash.js new file mode 100644 index 0000000..deeab38 --- /dev/null +++ b/scripts/check-span-hash.js @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); +const { hashAnchorSpan, statAnchorFile } = await import(new URL("dist/libs/knowledge-graph/code-anchors/span-hash.js", root)); + +const repo = mkdtempSync(join(tmpdir(), "greplica-span-hash-")); +writeFileSync(join(repo, "auth.ts"), "line 1\nexport function validateToken(t) { return t.length > 0; }\nline 3\n"); + +const anchor = { file: "auth.ts", symbol: "validateToken", status: "resolved", start_line: 2, end_line: 2 }; + +// Deterministic: same content -> same hash. +const first = hashAnchorSpan(repo, anchor); +assert.equal(typeof first, "string"); +assert.equal(first.length, 64, "sha256 hex"); +assert.equal(hashAnchorSpan(repo, anchor), first, "same span content -> same hash"); + +// A change to the span body changes the hash, even at the same line. +writeFileSync(join(repo, "auth.ts"), "line 1\nexport function validateToken(t) { return t.length > 99; }\nline 3\n"); +assert.notEqual(hashAnchorSpan(repo, anchor), first, "changed span body -> different hash"); + +// A change outside the span does NOT change the hash (span-level granularity). +writeFileSync(join(repo, "auth.ts"), "CHANGED\nexport function validateToken(t) { return t.length > 0; }\nline 3\n"); +assert.equal(hashAnchorSpan(repo, anchor), first, "change outside span -> same hash"); + +// Line endings are normalized: the same logical source hashes identically +// whether checked out with LF or CRLF (no false content drift on Windows). +writeFileSync(join(repo, "auth.ts"), "line 1\nexport function validateToken(t) { return t.length > 0; }\nline 3\n"); +const lfHash = hashAnchorSpan(repo, anchor); +writeFileSync(join(repo, "auth.ts"), "line 1\r\nexport function validateToken(t) { return t.length > 0; }\r\nline 3\r\n"); +assert.equal(hashAnchorSpan(repo, anchor), lfHash, "CRLF vs LF -> same hash"); + +// File-only anchor (no line range) hashes the whole file. +const fileOnly = { file: "auth.ts", status: "file_only" }; +assert.equal(typeof hashAnchorSpan(repo, fileOnly), "string", "file-only anchor hashes whole file"); + +// Missing / unreadable file -> undefined (graceful degradation, never throws). +assert.equal(hashAnchorSpan(repo, { file: "gone.ts", status: "missing_file" }), undefined, "missing file -> undefined"); +assert.equal(hashAnchorSpan(repo, { file: "../escape.ts", status: "resolved" }), undefined, "path escape -> undefined"); +assert.equal(hashAnchorSpan(undefined, anchor), undefined, "no repo root -> undefined"); + +// statAnchorFile returns real metadata, zeros when unavailable. +const stat = statAnchorFile(repo, "auth.ts"); +assert.ok(stat.mtime_ms > 0 && stat.size > 0, "stat returns real metadata"); +assert.deepEqual(statAnchorFile(repo, "gone.ts"), { mtime_ms: 0, size: 0 }, "missing file -> zero stat"); + +console.log("Span hash checks passed.");