From c1997dc9ae6028998eeb46635ed39dd7c633f050 Mon Sep 17 00:00:00 2001 From: burrows99 Date: Mon, 6 Jul 2026 17:38:20 +0100 Subject: [PATCH] feat: add `greplica graph gc` to prune stale knowledge-graph memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `graph gc [--dry-run]` command that removes memory a repo no longer supports: - stale anchors — components/claims whose `code_anchor` file was deleted (reuses the existing code-anchor audit for claims; adds the same check for component anchors) - orphaned claims/flows — no edge references them - dangling edges — an endpoint object no longer exists Pruning a subject cascades to every edge that touches it and runs in a single transaction. Object rows are global across repos (#103), so gc only unlinks this repo's `graph_memberships` + repo-scoped `graph_object_embeddings` and deletes the shared row only when no membership anywhere still references it. `--dry-run` reports the plan without touching the database. Adds `scripts/check-graph-gc.js` (wired into `npm test`) covering all three defect classes, cascade cleanup, healthy-object safety, and idempotency. Closes #24 Co-Authored-By: Claude Opus 4.8 --- README.md | 2 + apps/cli/main.ts | 33 +++++++- libs/knowledge-graph/graph-gc.ts | 133 ++++++++++++++++++++++++++++++ libs/knowledge-graph/service.ts | 51 ++++++++++++ libs/storage/sqlite/repository.ts | 75 +++++++++++++++++ package.json | 2 +- scripts/check-graph-gc.js | 86 +++++++++++++++++++ 7 files changed, 380 insertions(+), 2 deletions(-) create mode 100644 libs/knowledge-graph/graph-gc.ts create mode 100644 scripts/check-graph-gc.js diff --git a/README.md b/README.md index 8fe8e5b..12e9cde 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ greplica doctor [--check-embeddings] greplica graph read greplica graph context "" [--debug] greplica graph audit anchors +greplica graph gc [--dry-run] greplica graph view [--out ] [--no-open] greplica graph export greplica transcript bundle --platform codex|claude|copilot --file [--file ...] --out @@ -133,6 +134,7 @@ greplica proposal apply - `greplica graph context ""` - returns Markdown for agent use. Add `--debug` for the full retrieval payload with ranking signals. - `greplica graph read` - prints the current graph view: all components, flows, claims, sources, and edges in scope. +- `greplica graph gc [--dry-run]` - prunes stale memory from this repo's graph: components/claims whose code anchor file no longer exists, orphaned claims/flows, and dangling edges. `--dry-run` reports what would be pruned without touching the database. - `greplica graph view` to visualise the current memory in a local HTML, opens in your default browser. Use `--out` to choose where the file is written; by default it goes to a temp path. - `greplica transcript bundle` - converts one or more Codex, Claude Code, or GitHub Copilot CLI JSONL transcripts into a sanitized Markdown bundle for `greplica-fast-session-bootstrap`. - `greplica doctor` - verifies installation and diagnoses configuration failures. Not a required preflight before every command. diff --git a/apps/cli/main.ts b/apps/cli/main.ts index a74bdeb..d31ad6b 100644 --- a/apps/cli/main.ts +++ b/apps/cli/main.ts @@ -5,7 +5,7 @@ import { isatty } from "node:tty"; import { basename, dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { createLocalKnowledgeGraphService, KnowledgeGraphService } from "../../libs/knowledge-graph/service.js"; -import type { ClaimAnchorAuditResult, RepoRef } from "../../libs/knowledge-graph/service.js"; +import type { ClaimAnchorAuditResult, GcReport, RepoRef } from "../../libs/knowledge-graph/service.js"; import { envVarSource, loadRepoEnv, type LoadedRepoEnv } from "../../libs/env/load-local-env.js"; import { ensureGreplicaConfig, @@ -104,6 +104,13 @@ const cliCommands = [ handler: runGraphAuditAnchorsCommand, showInTopLevelHelp: true, }, + { + key: "graphGc", + path: ["graph", "gc"], + usage: "graph gc [--dry-run]", + handler: runGraphGcCommand, + showInTopLevelHelp: true, + }, { key: "graphExport", path: ["graph", "export"], @@ -278,6 +285,30 @@ async function runGraphAuditAnchorsCommand(_args: string[]): Promise { if (anchorAuditIssueCount(result) > 0) process.exitCode = 1; } +async function runGraphGcCommand(args: string[]): Promise { + const dryRun = args.includes("--dry-run"); + if (args.some((arg) => arg !== "--dry-run")) throw new Error(usage("graphGc")); + const { repo, service } = createCommandContext(); + const report = await service.gcGraph(repo, { dryRun }); + printGcReport(report); +} + +function printGcReport(report: GcReport): void { + console.log(report.dry_run ? "Graph gc (dry run)" : "Graph gc"); + console.log(""); + printAuditSection("Stale components", report.stale_components, (issue) => `${issue.id} -> ${issue.anchor}`); + printAuditSection("Stale claims", report.stale_claims, (issue) => `${issue.id} -> ${issue.anchor}`); + printAuditSection("Orphaned claims", report.orphaned_claims, (id) => id); + printAuditSection("Orphaned flows", report.orphaned_flows, (id) => id); + printAuditSection("Dangling edges", report.dangling_edges, (edge) => `${edge.id} (${edge.from} -> ${edge.to})`); + + const { components, flows, claims, edges } = report.pruned; + const total = components + flows + claims + edges; + const verb = report.dry_run ? "Would prune" : "Pruned"; + console.log(`${verb}: ${components} components, ${flows} flows, ${claims} claims, ${edges} edges.`); + if (report.dry_run && total > 0) console.log("Run `greplica graph gc` to prune."); +} + function runGraphExportCommand(args: string[]): void { const outputDir = requireFile(args[0], usage("graphExport")); const { repo, service } = createCommandContext(); diff --git a/libs/knowledge-graph/graph-gc.ts b/libs/knowledge-graph/graph-gc.ts new file mode 100644 index 0000000..a10144c --- /dev/null +++ b/libs/knowledge-graph/graph-gc.ts @@ -0,0 +1,133 @@ +import type { Edge } from "./edge.js"; + +export type GcSubjectType = "component" | "flow" | "claim" | "edge"; + +export interface GcSubjectRef { + type: GcSubjectType; + id: string; +} + +export interface GcAnchorIssue { + id: string; + anchor: string; +} + +export interface GcDanglingEdge { + id: string; + from: string; + to: string; +} + +export interface GcPrunedCounts { + components: number; + flows: number; + claims: number; + edges: number; +} + +export interface GcReport { + dry_run: boolean; + stale_components: GcAnchorIssue[]; + stale_claims: GcAnchorIssue[]; + orphaned_claims: string[]; + orphaned_flows: string[]; + dangling_edges: GcDanglingEdge[]; + pruned: GcPrunedCounts; +} + +export interface GcPlanInput { + components: { id: string }[]; + flows: { id: string }[]; + claims: { id: string }[]; + edges: Edge[]; + /** "type:id" keys for edge endpoints whose object row still exists. */ + existingKeys: ReadonlySet; + staleComponentIds: readonly string[]; + staleClaimIds: readonly string[]; +} + +export interface GcPlan { + /** components/flows/claims to prune. */ + subjects: GcSubjectRef[]; + /** edge ids to prune (dangling + edges attached to a pruned subject). */ + edges: string[]; + orphaned_claims: string[]; + orphaned_flows: string[]; + dangling_edges: GcDanglingEdge[]; + pruned: GcPrunedCounts; +} + +function subjectKey(type: string, id: string): string { + return `${type}:${id}`; +} + +/** + * Compute what `graph gc` should remove from a single repo's active graph. + * + * Detection passes: + * - stale anchors: components/claims whose code anchor file no longer exists + * (ids supplied by the caller, which owns filesystem access); + * - orphaned claims/flows: no edge references them; + * - dangling edges: an endpoint object row no longer exists. + * + * Pruning a subject cascades to every edge that touches it, so a stale claim's + * `about`/`evidenced_by` edges are removed in the same pass. + */ +export function planGc(input: GcPlanInput): GcPlan { + const referenced = new Set(); + for (const edge of input.edges) { + referenced.add(subjectKey(edge.from_type, edge.from_id)); + referenced.add(subjectKey(edge.to_type, edge.to_id)); + } + + const orphanedClaims = input.claims.filter((claim) => !referenced.has(subjectKey("claim", claim.id))).map((claim) => claim.id); + const orphanedFlows = input.flows.filter((flow) => !referenced.has(subjectKey("flow", flow.id))).map((flow) => flow.id); + + const danglingEdges: GcDanglingEdge[] = input.edges + .filter( + (edge) => + !input.existingKeys.has(subjectKey(edge.from_type, edge.from_id)) || + !input.existingKeys.has(subjectKey(edge.to_type, edge.to_id)), + ) + .map((edge) => ({ + id: edge.id, + from: subjectKey(edge.from_type, edge.from_id), + to: subjectKey(edge.to_type, edge.to_id), + })); + + const subjectKeys = new Set(); + const subjects: GcSubjectRef[] = []; + const addSubject = (type: GcSubjectType, id: string): void => { + const key = subjectKey(type, id); + if (subjectKeys.has(key)) return; + subjectKeys.add(key); + subjects.push({ type, id }); + }; + for (const id of input.staleComponentIds) addSubject("component", id); + for (const id of input.staleClaimIds) addSubject("claim", id); + for (const id of orphanedClaims) addSubject("claim", id); + for (const id of orphanedFlows) addSubject("flow", id); + + const edgeIds = new Set(danglingEdges.map((edge) => edge.id)); + for (const edge of input.edges) { + if (subjectKeys.has(subjectKey(edge.from_type, edge.from_id)) || subjectKeys.has(subjectKey(edge.to_type, edge.to_id))) { + edgeIds.add(edge.id); + } + } + + const pruned: GcPrunedCounts = { + components: subjects.filter((subject) => subject.type === "component").length, + flows: subjects.filter((subject) => subject.type === "flow").length, + claims: subjects.filter((subject) => subject.type === "claim").length, + edges: edgeIds.size, + }; + + return { + subjects, + edges: [...edgeIds], + orphaned_claims: orphanedClaims, + orphaned_flows: orphanedFlows, + dangling_edges: danglingEdges, + pruned, + }; +} diff --git a/libs/knowledge-graph/service.ts b/libs/knowledge-graph/service.ts index efb6e65..5daa87f 100644 --- a/libs/knowledge-graph/service.ts +++ b/libs/knowledge-graph/service.ts @@ -1,5 +1,8 @@ +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; import { normalizeProposal } from "./proposal.js"; import { validateProposal, type ProposalValidationResult } from "./validate-proposal.js"; +import { planGc, type GcReport } from "./graph-gc.js"; import type { Claim } from "./claim.js"; import type { Edge } from "./edge.js"; import type { Component, Flow, Source } from "./schema.js"; @@ -15,6 +18,7 @@ import { SqliteRepository as SqliteKnowledgeGraphRepository } from "../storage/s export type { GraphContextResult } from "./graph-context/types.js"; export type { ClaimAnchorAuditResult } from "./code-anchors/types.js"; +export type { GcReport } from "./graph-gc.js"; export interface RepoRef { repo_root?: string; @@ -125,6 +129,39 @@ export class KnowledgeGraphService { return auditClaimCodeAnchors(input.repo_root, this.repository.readGraphView(initialized.repo_id).claims); } + async gcGraph(input: RepoRef, options: { dryRun: boolean }): Promise { + const initialized = this.requireRepo(input); + const graph = this.repository.readGcGraph(initialized.repo_id); + + const audit = await auditClaimCodeAnchors(input.repo_root, graph.claims); + const staleClaims = audit.missing_files.map((issue) => ({ id: issue.claim_id, anchor: formatAnchor(issue.anchor) })); + const staleComponents = staleComponentAnchors(input.repo_root, graph.components); + + const plan = planGc({ + components: graph.components, + flows: graph.flows, + claims: graph.claims, + edges: graph.edges, + existingKeys: graph.existingKeys, + staleComponentIds: staleComponents.map((component) => component.id), + staleClaimIds: staleClaims.map((claim) => claim.id), + }); + + if (!options.dryRun) { + this.repository.applyGc(initialized.repo_id, { subjects: plan.subjects, edges: plan.edges }); + } + + return { + dry_run: options.dryRun, + stale_components: staleComponents, + stale_claims: staleClaims, + orphaned_claims: plan.orphaned_claims, + orphaned_flows: plan.orphaned_flows, + dangling_edges: plan.dangling_edges, + pruned: plan.pruned, + }; + } + async validateProposal(input: RepoRef, proposal: unknown): Promise { this.requireRepo(input); const normalizedProposal = normalizeProposal(proposal, this.repository); @@ -195,6 +232,20 @@ function formatAnchor(anchor: { file: string; symbol?: string } | undefined): st return anchor.symbol === undefined ? anchor.file : `${anchor.file}#${anchor.symbol}`; } +function staleComponentAnchors(repoRoot: string | undefined, components: Component[]): { id: string; anchor: string }[] { + if (repoRoot === undefined) return []; + const root = resolve(repoRoot); + const stale: { id: string; anchor: string }[] = []; + for (const component of components) { + if (component.code_anchor === undefined || component.code_anchor.length === 0) continue; + const file = component.code_anchor.split("#")[0]; + const absolute = resolve(root, file); + if (absolute !== root && !absolute.startsWith(`${root}/`)) continue; + if (!existsSync(absolute)) stale.push({ id: component.id, anchor: component.code_anchor }); + } + return stale; +} + export function createLocalKnowledgeGraphService( config: GraphContextConfig = graphContextConfig, ): KnowledgeGraphService { diff --git a/libs/storage/sqlite/repository.ts b/libs/storage/sqlite/repository.ts index de8f4f4..82ee295 100644 --- a/libs/storage/sqlite/repository.ts +++ b/libs/storage/sqlite/repository.ts @@ -6,6 +6,7 @@ import type { MemoryCommitProposal } from "../../knowledge-graph/proposal.js"; import type { Component, Flow, GraphObjectType, Source } from "../../knowledge-graph/schema.js"; import type { Claim } from "../../knowledge-graph/claim.js"; import type { GraphScope, GraphScopeKind } from "../../knowledge-graph/scope.js"; +import type { GcSubjectRef } from "../../knowledge-graph/graph-gc.js"; export interface RepoRecord { id: string; @@ -257,6 +258,80 @@ export class SqliteRepository { }; } + /** + * Raw view for `graph gc`: the repo's active components/flows/claims plus ALL + * of its edges (unfiltered, so dangling ones are visible) and the set of edge + * endpoints whose object row still exists. + */ + readGcGraph(repoId: string): { + components: Component[]; + flows: Flow[]; + claims: Claim[]; + edges: Edge[]; + existingKeys: Set; + } { + const scopeIds = this.currentScopeIds(repoId); + const memberships = this.membershipsForScopes(scopeIds); + const edges = this.loadEdges(selectIds(memberships, "edge")); + const active = activeSubjectKeys(memberships, edges); + + const existingKeys = new Set(); + for (const edge of edges) { + for (const endpoint of [ + { type: edge.from_type, id: edge.from_id }, + { type: edge.to_type, id: edge.to_id }, + ]) { + if (this.subjectExists(endpoint.type, endpoint.id)) existingKeys.add(subjectKey(endpoint.type, endpoint.id)); + } + } + + return { + components: this.loadComponents(selectActiveIds(memberships, active, "component")), + flows: this.loadFlows(selectActiveIds(memberships, active, "flow")), + claims: this.loadClaims(selectActiveIds(memberships, active, "claim")), + edges, + existingKeys, + }; + } + + /** + * Prune a `graph gc` plan for one repo in a single transaction. Object rows + * are global across repos (issue #103), so we only unlink this repo's + * memberships and repo-scoped embeddings, then delete the shared row when no + * membership anywhere still references it. + */ + applyGc(repoId: string, plan: { subjects: GcSubjectRef[]; edges: string[] }): void { + const scopeIds = this.currentScopeIds(repoId); + if (scopeIds.length === 0) return; + const refs: GcSubjectRef[] = [...plan.subjects, ...plan.edges.map((id) => ({ type: "edge" as const, id }))]; + if (refs.length === 0) return; + + const deleteMembership = this.db.prepare( + `DELETE FROM graph_memberships WHERE scope_id IN (${placeholders(scopeIds)}) AND subject_type = ? AND subject_id = ?`, + ); + const deleteEmbedding = this.db.prepare( + "DELETE FROM graph_object_embeddings WHERE repo_id = ? AND object_type = ? AND object_id = ?", + ); + const membershipRemains = this.db.prepare( + "SELECT 1 FROM graph_memberships WHERE subject_type = ? AND subject_id = ? LIMIT 1", + ); + const deleteRow: Record = { + component: this.db.prepare("DELETE FROM components WHERE id = ?"), + flow: this.db.prepare("DELETE FROM flows WHERE id = ?"), + claim: this.db.prepare("DELETE FROM claims WHERE id = ?"), + edge: this.db.prepare("DELETE FROM edges WHERE id = ?"), + }; + + const write = this.db.transaction((items: GcSubjectRef[]) => { + for (const ref of items) { + deleteMembership.run(...scopeIds, ref.type, ref.id); + if (ref.type !== "edge") deleteEmbedding.run(repoId, ref.type, ref.id); + if (membershipRemains.get(ref.type, ref.id) === undefined) deleteRow[ref.type].run(ref.id); + } + }); + write(refs); + } + createMemoryCommit(input: CreateMemoryCommitInput): MemoryCommit { const parent = this.db .prepare("SELECT id FROM memory_commits WHERE scope_id = ? ORDER BY created_at DESC LIMIT 1") diff --git a/package.json b/package.json index 43833c3..ba0225b 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "typecheck": "tsc --noEmit", "smoke:openhands": "npm run build && node scripts/smoke-openhands-install.mjs", "smoke:copilot": "npm run build && node scripts/smoke-copilot-install.mjs", - "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-proposal-validate.js", + "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-graph-gc.js && node scripts/check-proposal-validate.js", "test:transcript-bundle": "npm run build && node scripts/check-transcript-bundle.js", "test:repo-context": "npm run build && node scripts/check-repo-context.js", "eval:bootstrap-current": "npm run build && node dist/evals/cases/bootstrap-current-repo-at-8038fe8/run.js", diff --git a/scripts/check-graph-gc.js b/scripts/check-graph-gc.js new file mode 100644 index 0000000..feec04a --- /dev/null +++ b/scripts/check-graph-gc.js @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); + +const tmp = mkdtempSync(join(tmpdir(), "greplica-graph-gc-test-")); +const repoRoot = join(tmp, "repo"); +mkdirSync(join(repoRoot, "src"), { recursive: true }); +writeFileSync(join(repoRoot, "src", "keep.ts"), "export const keep = 1;\n"); +writeFileSync(join(repoRoot, "src", "drop.ts"), "export const drop = 1;\n"); + +const db = openDatabase(join(tmp, "graph.db")); + +try { + const repository = new SqliteRepository(db); + const service = new KnowledgeGraphService(repository); + const repo = { repo_root: repoRoot, repo_name: "graph-gc", default_branch: "main" }; + + const initialized = service.initRepo(repo); + const memoryCommit = repository.createMemoryCommit({ scope_id: initialized.working_scope_id, title: "Seed" }); + + repository.createProposalRecords(initialized.working_scope_id, memoryCommit.id, { + title: "Seed", + creates: { + components: [ + { id: "component.keep", name: "Keep", code_anchor: "src/keep.ts" }, + { id: "component.drop", name: "Drop", code_anchor: "src/drop.ts" }, + ], + claims: [ + { id: "claim.keep", kind: "fact", text: "keep", truth: "code_verified", intent: "intended", code_anchors: [{ file: "src/keep.ts" }] }, + { id: "claim.drop", kind: "fact", text: "drop", truth: "code_verified", intent: "intended", code_anchors: [{ file: "src/drop.ts" }] }, + ], + edges: [ + { id: "edge.about_keep", from_id: "claim.keep", from_type: "claim", to_id: "component.keep", to_type: "component", kind: "about" }, + { id: "edge.about_drop", from_id: "claim.drop", from_type: "claim", to_id: "component.drop", to_type: "component", kind: "about" }, + { id: "edge.dangling", from_id: "claim.keep", from_type: "claim", to_id: "component.ghost", to_type: "component", kind: "about" }, + ], + }, + }); + + // A refactor deletes drop.ts -> its component + claim anchors go stale. + rmSync(join(repoRoot, "src", "drop.ts")); + + // --- dry run: reports defects, changes nothing --- + const dry = await service.gcGraph(repo, { dryRun: true }); + assert.equal(dry.dry_run, true); + assert.deepEqual(dry.stale_components.map((c) => c.id), ["component.drop"], "stale component detected"); + assert.deepEqual(dry.stale_claims.map((c) => c.id), ["claim.drop"], "stale claim detected"); + assert.deepEqual(dry.dangling_edges.map((e) => e.id), ["edge.dangling"], "dangling edge detected"); + assert.deepEqual(dry.pruned, { components: 1, flows: 0, claims: 1, edges: 2 }, "dry-run plan counts"); + + let graph = service.readGraph(repo); + assert.equal(graph.claims.length, 2, "dry run must not delete anything"); + assert.equal(graph.components.length, 2, "dry run must not delete anything"); + + // --- real run: prunes the defects, keeps healthy objects --- + const run = await service.gcGraph(repo, { dryRun: false }); + assert.equal(run.dry_run, false); + assert.deepEqual(run.pruned, { components: 1, flows: 0, claims: 1, edges: 2 }, "prune counts"); + + graph = service.readGraph(repo); + assert.deepEqual(graph.components.map((c) => c.id), ["component.keep"], "only healthy component remains"); + assert.deepEqual(graph.claims.map((c) => c.id), ["claim.keep"], "only healthy claim remains"); + + assert.equal(repository.subjectExists("claim", "claim.drop"), false, "stale claim row deleted"); + assert.equal(repository.subjectExists("component", "component.drop"), false, "stale component row deleted"); + assert.equal(repository.subjectExists("edge", "edge.about_drop"), false, "cascaded edge deleted"); + assert.equal(repository.subjectExists("edge", "edge.dangling"), false, "dangling edge deleted"); + + assert.equal(repository.subjectExists("claim", "claim.keep"), true, "healthy claim kept"); + assert.equal(repository.subjectExists("component", "component.keep"), true, "healthy component kept"); + assert.equal(repository.subjectExists("edge", "edge.about_keep"), true, "healthy edge kept"); + + // --- idempotent: nothing left to prune --- + const again = await service.gcGraph(repo, { dryRun: true }); + assert.deepEqual(again.pruned, { components: 0, flows: 0, claims: 0, edges: 0 }, "second pass is a no-op"); +} finally { + db.close(); +} + +console.log("Graph gc checks passed.");