diff --git a/README.md b/README.md index 1d432f1..4f920a2 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,8 @@ greplica graph context "" [--debug] greplica graph audit anchors greplica graph view [--out ] [--no-open] greplica graph export +greplica git ingest [--max-commits ] [--max-age ] [--prs] [--github-token ] [--dry-run] +greplica git watch [--daemon] [--once] [--interval ] [--anchor-threshold ] greplica proposal validate greplica proposal apply greplica session mark-memory-current --session-ref @@ -136,6 +138,8 @@ greplica transcript bundle --platform codex|claude|copilot|opencode --file "` - returns Markdown for agent use. Add `--debug` for the full retrieval payload with ranking signals. - `greplica graph read` - prints the current graph view: all components, flows, claims, sources, and edges in scope. - `greplica graph view` to visualise the current memory in a local HTML, opens in your default browser. Use `--out` to choose where the file is written; by default it goes to a temp path. +- `greplica git ingest` deterministically bootstraps memory from conventional commits, changed modules, and repeated co-change patterns. Add `--prs` to enrich commit provenance from GitHub or `--dry-run` to inspect the proposal. +- `greplica git watch` checks changed code anchors after new commits and marks stale claims for review. Use `--daemon` for continuous polling or `--once` for automation. - `greplica transcript bundle` - converts one or more Codex, Claude Code, GitHub Copilot CLI, or OpenCode transcripts into a sanitized Markdown bundle for `greplica-fast-session-bootstrap`. - `greplica embeddings prewarm` - downloads and initializes the local embedding model ahead of the first query when local embeddings are configured. - `greplica session mark-memory-current` - marks a tracked agent session as already reflected in working memory. diff --git a/apps/cli/main.ts b/apps/cli/main.ts index 1f390a7..f235e10 100644 --- a/apps/cli/main.ts +++ b/apps/cli/main.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node import { spawn } from "node:child_process"; +import { setTimeout as sleep } from "node:timers/promises"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { isatty } from "node:tty"; import { basename, dirname, join, resolve } from "node:path"; @@ -17,6 +18,7 @@ import { graphContextConfigFromGreplicaConfig } from "../../libs/knowledge-graph import { createEmbedder } from "../../libs/knowledge-graph/graph-context/embedder.js"; import { renderGraphContextMarkdown } from "../../libs/knowledge-graph/graph-context/render.js"; import { buildGraphFolderExport } from "../../libs/knowledge-graph/folder-export.js"; +import type { MemoryCommitProposal } from "../../libs/knowledge-graph/proposal.js"; import { buildTranscriptBundle } from "../../libs/session-transcript/bundle.js"; import { installGreplica, platformDisplayName } from "../../libs/install/install.js"; import { allPlatformInstallers, platformInstaller } from "../../libs/install/platforms/index.js"; @@ -29,6 +31,8 @@ import { runHookWorker, shouldRunAutoMemoryUpdates, startHookWorker } from "../. import { withLocalModelLock } from "../../libs/knowledge-graph/graph-context/local-model-lock.js"; import { openDatabase } from "../../libs/storage/sqlite/db.js"; import { SqliteRepository as SqliteKnowledgeGraphRepository } from "../../libs/storage/sqlite/repository.js"; +import { attachPullRequestMetadata, collectGitHistory, generateGitHistoryProposal } from "../../libs/git-memory/history.js"; +import { runGitWatchCheck } from "../../libs/git-memory/watch.js"; import { detectRepoContext } from "./repo-context.js"; interface CommandContext { @@ -122,6 +126,20 @@ const cliCommands = [ handler: withCommandContext(runGraphViewCommand), showInTopLevelHelp: true, }, + { + key: "gitIngest", + path: ["git", "ingest"], + usage: "git ingest [--max-commits ] [--max-age ] [--prs] [--github-token ] [--dry-run]", + handler: runGitIngestCommand, + showInTopLevelHelp: true, + }, + { + key: "gitWatch", + path: ["git", "watch"], + usage: "git watch [--daemon] [--once] [--interval ] [--anchor-threshold ]", + handler: runGitWatchCommand, + showInTopLevelHelp: true, + }, { key: "proposalValidate", path: ["proposal", "validate"], @@ -312,6 +330,72 @@ function runGraphViewCommand(args: string[], getContext: CommandContextProvider) } } +async function runGitIngestCommand(args: string[]): Promise { + const options = parseGitIngestArgs(args); + const context = createGitMemoryContext(); + try { + context.service.requireRepo(context.repo); + const repoRoot = context.repo.repo_root ?? process.cwd(); + let commits = collectGitHistory(repoRoot, { + maxCommits: options.maxCommits, + maxAgeDays: options.maxAgeDays, + }); + if (options.prs) { + commits = await attachPullRequestMetadata( + commits, + context.repo.remote_url, + options.githubToken ?? process.env.GITHUB_TOKEN, + ); + } + const plan = generateGitHistoryProposal(repoRoot, commits, context.service.readGraph(context.repo)); + printGitIngestStats(plan.stats); + if (options.dryRun) { + console.log(""); + console.log(JSON.stringify(plan.proposal, null, 2)); + return; + } + if (!proposalHasCreates(plan.proposal)) { + console.log("No new git-derived memory to apply."); + return; + } + const result = await context.service.applyProposal(context.repo, plan.proposal); + console.log(`Applied git history proposal as memory commit ${result.memory_commit_id}.`); + } finally { + context.service.close(); + } +} + +async function runGitWatchCommand(args: string[]): Promise { + const options = parseGitWatchArgs(args); + if (options.daemon) { + startGitWatchDaemon(options); + console.log("Started Greplica git watch daemon."); + return; + } + + const context = createGitMemoryContext(); + try { + do { + const result = await runGitWatchCheck( + context.repo.repo_root ?? process.cwd(), + context.repo, + context.service, + context.repository, + { anchorThresholdDays: options.anchorThresholdDays }, + ); + console.log( + result.skipped + ? `Git watch: ${result.head.slice(0, 12)} unchanged; no audit due.` + : `Git watch: audited ${result.audited_claims} claim(s), marked ${result.marked_claims} for review.`, + ); + if (options.once) return; + await sleep(options.intervalSeconds * 1000); + } while (true); + } finally { + context.service.close(); + } +} + async function runProposalValidateCommand(args: string[], getContext: CommandContextProvider): Promise { const file = requireFile(args[0], usage("proposalValidate")); const { repo, service } = getContext(); @@ -359,6 +443,7 @@ function printAnchorAudit(result: ClaimAnchorAuditResult): void { printAuditSection("Missing anchors", result.missing_anchors, (issue) => issue.claim_id); printAuditSection("Invalid files", result.missing_files, (issue) => `${issue.claim_id} -> ${formatAuditAnchor(issue.anchor)}`); printAuditSection("Missing symbols", result.missing_symbols, (issue) => `${issue.claim_id} -> ${formatAuditAnchor(issue.anchor)}`); + printAuditSection("Drifted anchors", result.drifted, (issue) => `${issue.claim_id} -> ${formatAuditAnchor(issue.anchor)}`); printAuditSection("Ambiguous symbols", result.ambiguous_symbols, (issue) => `${issue.claim_id} -> ${formatAuditAnchor(issue.anchor)}`); printAuditSection("Unsupported languages", result.unsupported_languages, (issue) => `${issue.claim_id} -> ${formatAuditAnchor(issue.anchor)}`); } @@ -367,6 +452,7 @@ function anchorAuditIssueCount(result: ClaimAnchorAuditResult): number { return result.missing_anchors.length + result.missing_files.length + result.missing_symbols.length + + result.drifted.length + result.ambiguous_symbols.length + result.unsupported_languages.length; } @@ -410,6 +496,18 @@ function withCommandContext(handler: CommandContextHandler): CliCommand["handler }; } +function createGitMemoryContext(): CommandContext & { + repository: SqliteKnowledgeGraphRepository; +} { + const repo = detectRepoContext(); + const env = loadRepoEnv(repo.repo_root ?? process.cwd()); + const config = ensureGreplicaConfig(); + const db = openDatabase(); + const repository = new SqliteKnowledgeGraphRepository(db); + const service = new KnowledgeGraphService(repository, graphContextConfigFromGreplicaConfig(config)); + return { repo, env, config, service, repository }; +} + function runHookIngest(args: string[]): void { if (process.env.GREPLICA_HOOK_DISABLE === "1") return; @@ -736,6 +834,96 @@ interface TranscriptBundleOptions { outputPath: string; } +interface GitIngestOptions { + maxCommits: number; + maxAgeDays?: number; + prs: boolean; + githubToken?: string; + dryRun: boolean; +} + +interface GitWatchCliOptions { + daemon: boolean; + once: boolean; + intervalSeconds: number; + anchorThresholdDays: number; +} + +function parseGitIngestArgs(args: string[]): GitIngestOptions { + const options: GitIngestOptions = { maxCommits: 200, prs: false, dryRun: false }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--prs") { + options.prs = true; + continue; + } + if (arg === "--dry-run") { + options.dryRun = true; + continue; + } + if (arg === "--max-commits" || arg === "--max-age" || arg === "--github-token") { + const value = requireFlagValue(args, index, arg, usage("gitIngest")); + if (arg === "--max-commits") options.maxCommits = positiveInteger(value, arg, usage("gitIngest")); + if (arg === "--max-age") options.maxAgeDays = positiveInteger(value, arg, usage("gitIngest")); + if (arg === "--github-token") options.githubToken = value; + index += 1; + continue; + } + if (arg.startsWith("--max-commits=")) { + options.maxCommits = positiveInteger(arg.slice("--max-commits=".length), "--max-commits", usage("gitIngest")); + continue; + } + if (arg.startsWith("--max-age=")) { + options.maxAgeDays = positiveInteger(arg.slice("--max-age=".length), "--max-age", usage("gitIngest")); + continue; + } + if (arg.startsWith("--github-token=")) { + options.githubToken = requireInlineFlagValue(arg.slice("--github-token=".length), "--github-token", usage("gitIngest")); + continue; + } + throw new Error(usage("gitIngest")); + } + return options; +} + +function parseGitWatchArgs(args: string[]): GitWatchCliOptions { + const options: GitWatchCliOptions = { + daemon: false, + once: false, + intervalSeconds: 30, + anchorThresholdDays: 7, + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--daemon") { + options.daemon = true; + continue; + } + if (arg === "--once") { + options.once = true; + continue; + } + if (arg === "--interval" || arg === "--anchor-threshold") { + const value = requireFlagValue(args, index, arg, usage("gitWatch")); + if (arg === "--interval") options.intervalSeconds = positiveInteger(value, arg, usage("gitWatch")); + if (arg === "--anchor-threshold") options.anchorThresholdDays = positiveInteger(value, arg, usage("gitWatch")); + index += 1; + continue; + } + if (arg.startsWith("--interval=")) { + options.intervalSeconds = positiveInteger(arg.slice("--interval=".length), "--interval", usage("gitWatch")); + continue; + } + if (arg.startsWith("--anchor-threshold=")) { + options.anchorThresholdDays = positiveInteger(arg.slice("--anchor-threshold=".length), "--anchor-threshold", usage("gitWatch")); + continue; + } + throw new Error(usage("gitWatch")); + } + if (options.daemon && options.once) throw new Error(`--daemon and --once cannot be combined.\n${usage("gitWatch")}`); + return options; +} + function parseTranscriptBundleArgs(args: string[]): TranscriptBundleOptions { let platform: InstallPlatform | undefined; let outputPath: string | undefined; @@ -792,6 +980,17 @@ function requireFlagValue(args: string[], index: number, flag: string, usageText return value; } +function requireInlineFlagValue(value: string, flag: string, usageText: string): string { + if (value.trim().length === 0) throw new Error(`Missing value for ${flag}.\n${usageText}`); + return value; +} + +function positiveInteger(value: string, flag: string, usageText: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive integer.\n${usageText}`); + return parsed; +} + function parseInstallPlatform(value: string): InstallPlatform { if ((installPlatforms as readonly string[]).includes(value)) return value as InstallPlatform; throw new Error(`Invalid --platform ${value}.\n${usage("install")}`); @@ -847,6 +1046,45 @@ function printInstallResult(result: Awaited>) for (const note of result.notes) console.log(`- ${note}`); } +function printGitIngestStats(stats: { + commits_analyzed: number; + claims: number; + components: number; + flows: number; + sources: number; + edges: number; +}): void { + console.log(`Commits analyzed: ${stats.commits_analyzed}`); + console.log(`Claims to create: ${stats.claims}`); + console.log(`Components to create: ${stats.components}`); + console.log(`Flows to create: ${stats.flows}`); + console.log(`Sources to create: ${stats.sources}`); + console.log(`Edges to create: ${stats.edges}`); +} + +function proposalHasCreates(proposal: MemoryCommitProposal): boolean { + return Object.values(proposal.creates).some((items) => Array.isArray(items) && items.length > 0); +} + +function startGitWatchDaemon(options: GitWatchCliOptions): void { + const script = process.argv[1]; + if (script === undefined) throw new Error("Cannot start git watch daemon without a CLI script path."); + const child = spawn( + process.execPath, + [ + script, + "git", + "watch", + "--interval", + String(options.intervalSeconds), + "--anchor-threshold", + String(options.anchorThresholdDays), + ], + { detached: true, stdio: "ignore", env: process.env }, + ); + child.unref(); +} + function cliName(): string { return basename(process.argv[1] ?? "greplica"); } diff --git a/libs/git-memory/history.ts b/libs/git-memory/history.ts new file mode 100644 index 0000000..b420fd0 --- /dev/null +++ b/libs/git-memory/history.ts @@ -0,0 +1,380 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import type { ClaimKind } from "../knowledge-graph/claim.js"; +import type { Edge } from "../knowledge-graph/edge.js"; +import type { MemoryCommitProposal } from "../knowledge-graph/proposal.js"; +import type { GraphReadResult } from "../knowledge-graph/service.js"; + +export interface GitHistoryOptions { + maxCommits: number; + maxAgeDays?: number; +} + +export interface GitChangedFile { + path: string; + additions: number | null; + deletions: number | null; +} + +export interface GitPullRequestMetadata { + number: number; + title: string; + body?: string; + html_url?: string; +} + +export interface GitCommitRecord { + sha: string; + authored_at: string; + subject: string; + body: string; + files: GitChangedFile[]; + pull_request?: GitPullRequestMetadata; +} + +export interface GitIngestStats { + commits_analyzed: number; + claims: number; + components: number; + flows: number; + sources: number; + edges: number; +} + +export interface GitIngestPlan { + proposal: MemoryCommitProposal; + stats: GitIngestStats; +} + +const conventionalCommit = /^(feat|fix|refactor|perf|build|ci)(?:\(([^)]+)\))?!?:\s*(.+)$/i; +const decisionPattern = /\b(decided?|chose|chosen|adopt(?:ed)?|migrat(?:e|ed)|replac(?:e|ed)|standardiz(?:e|ed))\b/i; +const requirementPattern = /\b(must|should|required?|ensure[sd]?)\b/i; + +export function collectGitHistory(repoRoot: string, options: GitHistoryOptions): GitCommitRecord[] { + const args = [ + "log", + `--max-count=${options.maxCommits}`, + "--date=iso-strict", + "--no-renames", + "--format=%x1e%H%x1f%aI%x1f%s%x1f%b%x1d", + "--numstat", + ]; + if (options.maxAgeDays !== undefined) args.splice(2, 0, `--since=${options.maxAgeDays} days ago`); + const output = execFileSync("git", args, { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + return parseGitHistory(output); +} + +export function parseGitHistory(output: string): GitCommitRecord[] { + const commits: GitCommitRecord[] = []; + for (const rawChunk of output.split("\x1e")) { + const chunk = rawChunk.trim(); + if (chunk.length === 0) continue; + const separator = chunk.indexOf("\x1d"); + if (separator === -1) continue; + const metadata = chunk.slice(0, separator); + const numstat = chunk.slice(separator + 1); + const [sha, authoredAt, subject, ...bodyParts] = metadata.split("\x1f"); + if (!sha || !authoredAt || !subject) continue; + const files = numstat + .split(/\r?\n/) + .map((line) => /^(\d+|-)\t(\d+|-)\t(.+)$/.exec(line.trim())) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => ({ + path: match[3] ?? "", + additions: match[1] === "-" ? null : Number(match[1]), + deletions: match[2] === "-" ? null : Number(match[2]), + })) + .filter((file) => file.path.length > 0); + commits.push({ + sha, + authored_at: authoredAt, + subject: subject.trim(), + body: bodyParts.join("\x1f").trim(), + files, + }); + } + return commits; +} + +export async function attachPullRequestMetadata( + commits: GitCommitRecord[], + remoteUrl: string | undefined, + token?: string, +): Promise { + const repository = githubRepository(remoteUrl); + if (repository === undefined) throw new Error("--prs requires a GitHub origin remote."); + const headers: Record = { + Accept: "application/vnd.github+json", + "User-Agent": "greplica-git-ingest", + "X-GitHub-Api-Version": "2022-11-28", + }; + if (token?.trim()) headers.Authorization = `Bearer ${token.trim()}`; + + const enriched: GitCommitRecord[] = []; + for (const commit of commits) { + const response = await fetch(`https://api.github.com/repos/${repository}/commits/${commit.sha}/pulls`, { headers }); + if (!response.ok) throw new Error(`GitHub PR lookup failed for ${commit.sha.slice(0, 8)}: ${response.status} ${response.statusText}`); + const payload = await response.json() as unknown; + const first = Array.isArray(payload) ? payload[0] : undefined; + const pullRequest = pullRequestMetadata(first); + enriched.push(pullRequest === undefined ? commit : { ...commit, pull_request: pullRequest }); + } + return enriched; +} + +export function generateGitHistoryProposal( + repoRoot: string, + commits: GitCommitRecord[], + graph: GraphReadResult, +): GitIngestPlan { + const existingIds = new Set([ + ...graph.components.map((item) => item.id), + ...graph.flows.map((item) => item.id), + ...graph.claims.map((item) => item.id), + ...graph.sources.map((item) => item.id), + ...graph.edges.map((item) => item.id), + ]); + const existingClaimTexts = new Set(graph.claims.map((claim) => normalizeText(claim.text))); + const components = new Map(); + const claims: NonNullable = []; + const sources: NonNullable = []; + const edges: Edge[] = []; + const modulesByCommit = new Map(); + + for (const commit of commits) { + const modules = [...new Set(commit.files.map((file) => moduleForPath(file.path)).filter((value): value is string => value !== undefined))].sort(); + modulesByCommit.set(commit.sha, modules); + for (const module of modules) { + const id = stableId("component.git", module); + if (existingIds.has(id) || components.has(id)) continue; + components.set(id, { id, name: `${module} module`, code_anchor: `${module}/` }); + } + + const extracted = extractedClaims(commit); + const sourceId = stableId("source.git", commit.sha); + if (!existingIds.has(sourceId)) { + const pr = commit.pull_request; + sources.push({ + id: sourceId, + kind: "git_history", + ref: `git:${commit.sha}`, + title: pr === undefined + ? `${commit.sha.slice(0, 8)} ${commit.subject}` + : `#${pr.number} ${pr.title} (${commit.sha.slice(0, 8)})`, + }); + } + + for (const extractedClaim of extracted) { + const claimId = stableId("claim.git", `${commit.sha}:${extractedClaim.kind}:${extractedClaim.text}`); + const normalizedText = normalizeText(extractedClaim.text); + if (existingIds.has(claimId) || existingClaimTexts.has(normalizedText)) continue; + const codeAnchors = commit.files + .map((file) => file.path) + .filter((file) => existsSync(join(repoRoot, file))) + .slice(0, 3) + .map((file) => ({ file })); + claims.push({ + id: claimId, + kind: extractedClaim.kind, + text: extractedClaim.text, + truth: "source_verified", + intent: "intended", + code_anchors: codeAnchors.length === 0 ? undefined : codeAnchors, + }); + existingClaimTexts.add(normalizedText); + edges.push(makeEdge("evidenced_by", "claim", claimId, "source", sourceId, { + reason: `Extracted deterministically from git commit ${commit.sha.slice(0, 12)}.`, + })); + for (const module of modules) { + const componentId = stableId("component.git", module); + if (!existingIds.has(componentId) && !components.has(componentId)) continue; + edges.push(makeEdge("about", "claim", claimId, "component", componentId)); + } + } + + } + + const flows = coChangeFlows(commits, modulesByCommit, existingIds, components, edges); + const dedupedEdges = dedupeEdges(edges).filter((edge) => !existingIds.has(edge.id)); + const proposal: MemoryCommitProposal = { + title: `Ingest ${commits.length} git commit${commits.length === 1 ? "" : "s"}`, + summary: "Deterministic components, claims, flows, and provenance extracted from git history.", + creates: { + components: [...components.values()].sort((left, right) => left.id.localeCompare(right.id)), + flows, + claims, + sources, + edges: dedupedEdges, + }, + }; + return { + proposal, + stats: { + commits_analyzed: commits.length, + claims: claims.length, + components: components.size, + flows: flows.length, + sources: sources.length, + edges: dedupedEdges.length, + }, + }; +} + +export function githubRepository(remoteUrl: string | undefined): string | undefined { + if (!remoteUrl) return undefined; + const match = /github\.com[/:]([^/]+)\/([^/]+?)(?:\.git)?$/.exec(remoteUrl.trim()); + if (!match) return undefined; + return `${match[1]}/${match[2]}`; +} + +function extractedClaims(commit: GitCommitRecord): Array<{ kind: ClaimKind; text: string }> { + const match = conventionalCommit.exec(commit.subject); + const context = [commit.subject, commit.body, commit.pull_request?.title, commit.pull_request?.body] + .filter((value): value is string => typeof value === "string" && value.trim().length > 0) + .join("\n"); + const extracted: Array<{ kind: ClaimKind; text: string }> = []; + if (match) { + const type = match[1]?.toLowerCase() ?? ""; + const scope = match[2]?.trim(); + const description = match[3]?.trim() ?? commit.subject; + const kind: ClaimKind = type === "fix" || type === "perf" ? "fact" : type === "build" || type === "ci" ? "requirement" : "insight"; + extracted.push({ + kind, + text: scope ? `Git history records ${scope}: ${description}.` : `Git history records: ${description}.`, + }); + } + for (const sentence of sentences(context)) { + if (decisionPattern.test(sentence)) extracted.push({ kind: "decision", text: sentence }); + else if (requirementPattern.test(sentence)) extracted.push({ kind: "requirement", text: sentence }); + } + return dedupeClaims(extracted).slice(0, 3); +} + +function sentences(value: string): string[] { + return value + .split(/(?:\r?\n)+|(?<=[.!?])\s+/) + .map((sentence) => sentence.replace(/^[-*]\s*/, "").trim()) + .filter((sentence) => sentence.length >= 12 && sentence.length <= 320); +} + +function dedupeClaims(claims: Array<{ kind: ClaimKind; text: string }>): Array<{ kind: ClaimKind; text: string }> { + const seen = new Set(); + return claims.filter((claim) => { + const key = `${claim.kind}:${normalizeText(claim.text)}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function coChangeFlows( + commits: GitCommitRecord[], + modulesByCommit: Map, + existingIds: Set, + components: Map, + edges: Edge[], +): NonNullable { + const moduleCounts = new Map(); + const pairCounts = new Map(); + for (const commit of commits) { + const modules = modulesByCommit.get(commit.sha) ?? []; + for (const module of modules) moduleCounts.set(module, (moduleCounts.get(module) ?? 0) + 1); + for (let leftIndex = 0; leftIndex < modules.length; leftIndex += 1) { + for (let rightIndex = leftIndex + 1; rightIndex < modules.length; rightIndex += 1) { + const left = modules[leftIndex]; + const right = modules[rightIndex]; + if (!left || !right) continue; + const key = `${left}\0${right}`; + const pair = pairCounts.get(key) ?? { left, right, count: 0 }; + pair.count += 1; + pairCounts.set(key, pair); + } + } + } + + const flows: NonNullable = []; + for (const pair of [...pairCounts.values()].sort((left, right) => left.left.localeCompare(right.left) || left.right.localeCompare(right.right))) { + const leftCount = moduleCounts.get(pair.left) ?? 0; + const rightCount = moduleCounts.get(pair.right) ?? 0; + const lift = commits.length === 0 ? 0 : (pair.count * commits.length) / Math.max(1, leftCount * rightCount); + if (pair.count < 2 || lift < 1.2) continue; + const id = stableId("flow.git", `${pair.left}:${pair.right}`); + if (existingIds.has(id)) continue; + const leftId = stableId("component.git", pair.left); + const rightId = stableId("component.git", pair.right); + if ((!existingIds.has(leftId) && !components.has(leftId)) || (!existingIds.has(rightId) && !components.has(rightId))) continue; + flows.push({ id, name: `${pair.left} and ${pair.right} co-change flow` }); + edges.push(makeEdge("touches", "flow", id, "component", leftId, { commits: pair.count, lift: round(lift) })); + edges.push(makeEdge("touches", "flow", id, "component", rightId, { commits: pair.count, lift: round(lift) })); + } + return flows; +} + +function moduleForPath(path: string): string | undefined { + const parts = path.replace(/\\/g, "/").split("/").filter(Boolean); + if (parts.length < 2) return undefined; + if (["apps", "libs", "packages", "services", "src"].includes(parts[0] ?? "") && parts.length > 2) { + return `${parts[0]}/${parts[1]}`; + } + return parts[0]; +} + +function makeEdge( + kind: Edge["kind"], + fromType: Edge["from_type"], + fromId: string, + toType: Edge["to_type"], + toId: string, + metadata?: Record, +): Edge { + return { + id: stableId("edge.git", `${kind}:${fromType}:${fromId}:${toType}:${toId}`), + from_type: fromType, + from_id: fromId, + to_type: toType, + to_id: toId, + kind, + metadata, + }; +} + +function stableId(prefix: string, value: string): string { + const slug = value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 48) || "item"; + const hash = createHash("sha1").update(value).digest("hex").slice(0, 8); + return `${prefix}.${slug}.${hash}`; +} + +function normalizeText(value: string): string { + return value.toLowerCase().replace(/\s+/g, " ").trim(); +} + +function dedupeEdges(edges: Edge[]): Edge[] { + const seen = new Set(); + return edges.filter((edge) => { + if (seen.has(edge.id)) return false; + seen.add(edge.id); + return true; + }); +} + +function pullRequestMetadata(value: unknown): GitPullRequestMetadata | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + const record = value as Record; + if (typeof record.number !== "number" || typeof record.title !== "string") return undefined; + return { + number: record.number, + title: record.title, + body: typeof record.body === "string" ? record.body : undefined, + html_url: typeof record.html_url === "string" ? record.html_url : undefined, + }; +} + +function round(value: number): number { + return Math.round(value * 1000) / 1000; +} diff --git a/libs/git-memory/watch.ts b/libs/git-memory/watch.ts new file mode 100644 index 0000000..65857c0 --- /dev/null +++ b/libs/git-memory/watch.ts @@ -0,0 +1,85 @@ +import { execFileSync } from "node:child_process"; +import type { KnowledgeGraphService, RepoRef, StaleAnchorReviewResult } from "../knowledge-graph/service.js"; +import type { SqliteRepository } from "../storage/sqlite/repository.js"; + +export interface GitWatchOptions { + anchorThresholdDays: number; + now?: Date; +} + +export interface GitWatchCheckResult extends StaleAnchorReviewResult { + head: string; + changed_files: string[]; + full_audit: boolean; + skipped: boolean; +} + +export async function runGitWatchCheck( + repoRoot: string, + repoRef: RepoRef, + service: KnowledgeGraphService, + repository: SqliteRepository, + options: GitWatchOptions, +): Promise { + const initialized = service.requireRepo(repoRef); + const state = repository.getGitWatchState(initialized.repo_id); + const head = git(repoRoot, ["rev-parse", "HEAD"]); + const now = options.now ?? new Date(); + let fullAudit = state === undefined || fullAuditDue(state.last_full_audit_at, now, options.anchorThresholdDays); + const changedFilesResult = state === undefined || state.last_head === head + ? [] + : changedFilesSince(repoRoot, state.last_head, head); + if (changedFilesResult === undefined) fullAudit = true; + const changedFiles = changedFilesResult ?? []; + if (!fullAudit && changedFiles.length === 0) { + return { head, changed_files: [], full_audit: false, skipped: true, audited_claims: 0, marked_claims: 0 }; + } + + const reviewed = await service.markStaleAnchors(repoRef, { + changedFiles: fullAudit ? undefined : new Set(changedFiles), + head, + }); + repository.upsertGitWatchState({ + repo_id: initialized.repo_id, + last_head: head, + last_full_audit_at: fullAudit ? now.toISOString() : (state?.last_full_audit_at ?? null), + updated_at: now.toISOString(), + }); + return { + ...reviewed, + head, + changed_files: changedFiles, + full_audit: fullAudit, + skipped: false, + }; +} + +export function gitHead(repoRoot: string): string { + return git(repoRoot, ["rev-parse", "HEAD"]); +} + +function changedFilesSince(repoRoot: string, previousHead: string, head: string): string[] | undefined { + try { + return git(repoRoot, ["diff", "--name-only", `${previousHead}..${head}`]) + .split(/\r?\n/) + .map((file) => file.trim()) + .filter((file) => file.length > 0); + } catch { + return undefined; + } +} + +function fullAuditDue(lastAuditAt: string | null, now: Date, thresholdDays: number): boolean { + if (lastAuditAt === null) return true; + const parsed = new Date(lastAuditAt); + if (Number.isNaN(parsed.getTime())) return true; + return now.getTime() - parsed.getTime() >= thresholdDays * 24 * 60 * 60 * 1000; +} + +function git(repoRoot: string, args: string[]): string { + return execFileSync("git", args, { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +} diff --git a/libs/knowledge-graph/claim.ts b/libs/knowledge-graph/claim.ts index a4aff6c..198e207 100644 --- a/libs/knowledge-graph/claim.ts +++ b/libs/knowledge-graph/claim.ts @@ -4,6 +4,7 @@ export type ClaimKind = | "fact" | "requirement" | "decision" + | "insight" | "task" | "question" | "risk"; diff --git a/libs/knowledge-graph/edge.ts b/libs/knowledge-graph/edge.ts index d3c432b..9856375 100644 --- a/libs/knowledge-graph/edge.ts +++ b/libs/knowledge-graph/edge.ts @@ -5,7 +5,8 @@ export type EdgeKind = | "contains" | "touches" | "supersedes" - | "evidenced_by"; + | "evidenced_by" + | "needs_review"; export type EdgeMetadata = Record; @@ -19,7 +20,9 @@ export interface Edge { metadata?: EdgeMetadata; } -export function isAllowedEdge(edge: Pick): boolean { +export function isAllowedEdge( + edge: Pick, +): boolean { switch (edge.kind) { case "about": return edge.from_type === "claim" && (edge.to_type === "component" || edge.to_type === "flow"); @@ -41,5 +44,8 @@ export function isAllowedEdge(edge: Pick case "evidenced_by": return edge.from_type === "claim" && edge.to_type === "source"; + + case "needs_review": + return edge.from_type === "claim" && edge.to_type === "claim" && edge.from_id === edge.to_id; } } diff --git a/libs/knowledge-graph/folder-export.ts b/libs/knowledge-graph/folder-export.ts index e660ab7..4ed3f60 100644 --- a/libs/knowledge-graph/folder-export.ts +++ b/libs/knowledge-graph/folder-export.ts @@ -333,6 +333,8 @@ function claimKindTitle(kind: ClaimKind): string { return "Decisions"; case "fact": return "Facts"; + case "insight": + return "Insights"; case "question": return "Questions"; case "requirement": diff --git a/libs/knowledge-graph/graph-view/build-graph-view.ts b/libs/knowledge-graph/graph-view/build-graph-view.ts index 91d928b..449f046 100644 --- a/libs/knowledge-graph/graph-view/build-graph-view.ts +++ b/libs/knowledge-graph/graph-view/build-graph-view.ts @@ -49,8 +49,10 @@ export interface GraphViewClaimRow { text: string; kind: string; session: string; - source: "code" | "session"; + source: "code" | "session" | "git_history"; freshness: "active" | "superseded"; + review: "current" | "needs_review"; + reviewReasons: string[]; componentIds: string[]; flowIds: string[]; createdAt: string | null; @@ -72,6 +74,7 @@ export interface GraphViewData { flows: number; claims: number; superseded: number; + needsReview: number; }; components: GraphViewComponentRow[]; flows: GraphViewFlowRow[]; @@ -87,11 +90,12 @@ export interface BuildGraphViewOptions { repoName?: string; } -const CLAIM_KIND_ORDER = ["fact", "decision", "requirement", "task", "risk", "question"]; +const CLAIM_KIND_ORDER = ["fact", "decision", "insight", "requirement", "task", "risk", "question"]; const CLAIM_KIND_COLORS: Record = { fact: "#4e79a7", decision: "#59a14f", + insight: "#af7aa1", requirement: "#f28e2b", task: "#b07aa1", risk: "#e15759", @@ -105,6 +109,7 @@ export function buildGraphViewData( ): GraphViewData { const provenanceByClaimId = new Map(provenance.map((row) => [row.claim_id, row])); const sourceById = new Map(graph.sources.map((source) => [source.id, source])); + const reviewReasonsByClaimId = needsReviewReasonsByClaimId(graph.edges); const topLevelComponents = selectTopLevelComponents(graph.components, graph.edges); const components = topLevelComponents.map((component) => ({ id: component.id, @@ -138,8 +143,10 @@ export function buildGraphViewData( text: claim.text, kind: claim.kind, session, - source: isFromSession(session) ? "session" : "code", + source: sourceKindForClaim(claim.id, graph.edges, sourceById), freshness, + review: reviewReasonsByClaimId.has(claim.id) ? "needs_review" : "current", + reviewReasons: reviewReasonsByClaimId.get(claim.id) ?? [], componentIds: componentIdsForClaim(claim.id, graph.edges), flowIds: flowIdsForClaim(claim.id, graph.edges), createdAt: record?.created_at ?? null, @@ -163,6 +170,7 @@ export function buildGraphViewData( flows: flows.length, claims: claims.length, superseded: superseded.length, + needsReview: claims.filter((claim) => claim.review === "needs_review").length, }, components, flows, @@ -289,6 +297,30 @@ function sessionLabelForClaim(claimId: string, edges: Edge[], sourceById: Map 0 ? labels.join("; ") : "from code"; } +function sourceKindForClaim( + claimId: string, + edges: Edge[], + sourceById: Map, +): GraphViewClaimRow["source"] { + const sourceKinds = edges + .filter((edge) => edge.kind === "evidenced_by" && edge.from_id === claimId) + .map((edge) => sourceById.get(edge.to_id)?.kind); + if (sourceKinds.includes("git_history")) return "git_history"; + return sourceKinds.includes("session") ? "session" : "code"; +} + +function needsReviewReasonsByClaimId(edges: Edge[]): Map { + const reasons = new Map(); + for (const edge of edges) { + if (edge.kind !== "needs_review" || edge.from_type !== "claim") continue; + const reason = typeof edge.metadata?.reason === "string" ? edge.metadata.reason : "anchor_stale"; + const existing = reasons.get(edge.from_id) ?? []; + if (!existing.includes(reason)) existing.push(reason); + reasons.set(edge.from_id, existing); + } + return reasons; +} + function isFromSession(session: string): boolean { return session !== "from code"; } @@ -385,7 +417,10 @@ function kindColor(kind: string): string { function renderClaimRow(claim: GraphViewClaimRow): string { const badge = `${escapeHtml(claim.kind)}`; - return ` ${escapeHtml(claim.text)}
${escapeHtml(claim.id)}
${escapeHtml(claim.session)}${badge}${escapeHtml(formatDateTime(claim.createdAt))}`; + const warning = claim.review === "needs_review" + ? `⚠` + : ""; + return ` ${warning}${escapeHtml(claim.text)}
${escapeHtml(claim.id)}
${escapeHtml(claim.session)}${badge}${escapeHtml(formatDateTime(claim.createdAt))}`; } function renderHtml(data: GraphViewData, title: string): string { @@ -434,7 +469,7 @@ function renderHtml(data: GraphViewData, title: string): string { }) .join("\n"); - const defaultClaimsMeta = `${data.claims.length} active claims · session from evidenced_by source, otherwise from code`; + const defaultClaimsMeta = `${data.claims.length} active claims · ${data.counts.needsReview} need review`; const graphDataJson = jsonForScriptTag(data); const chartJsSource = readVendorScript("chart.js", "chart.umd.js"); const chartDataLabelsSource = readVendorScript("chartjs-plugin-datalabels", "chartjs-plugin-datalabels.min.js"); @@ -613,6 +648,11 @@ function renderHtml(data: GraphViewData, title: string): string { color: #fff; text-transform: capitalize; } + .review-warning { + color: #b45309; + cursor: help; + margin-right: 0.45rem; + } td.created { white-space: nowrap; font-variant-numeric: tabular-nums; @@ -806,6 +846,7 @@ function renderHtml(data: GraphViewData, title: string): string { Components Flows Claims + Claims - Needs review Claims - Timeline Claims - Overview @@ -902,7 +943,7 @@ ${timelineEvents} const CLAIM_KIND_ORDER = ${JSON.stringify(CLAIM_KIND_ORDER)}; const CLAIM_KIND_COLORS = ${JSON.stringify(CLAIM_KIND_COLORS)}; - const SOURCE_COLORS = { code: "#4e79a7", session: "#f28e2b" }; + const SOURCE_COLORS = { code: "#4e79a7", session: "#f28e2b", git_history: "#59a14f" }; const FRESHNESS_COLORS = { active: "#59a14f", superseded: "#bab0ac" }; const allClaims = graphData.claims.concat(graphData.supersededClaims); @@ -1083,13 +1124,16 @@ ${timelineEvents} let codeCount = 0; let sessionCount = 0; + let gitHistoryCount = 0; for (const claim of graphData.claims) { if (claim.source === "session") sessionCount += 1; + else if (claim.source === "git_history") gitHistoryCount += 1; else codeCount += 1; } renderOverviewChart("chart-source", "legend-source", [ { label: "from code", count: codeCount, color: SOURCE_COLORS.code, href: "#claims?source=code" }, { label: "from session", count: sessionCount, color: SOURCE_COLORS.session, href: "#claims?source=session" }, + { label: "from git history", count: gitHistoryCount, color: SOURCE_COLORS.git_history, href: "#claims?source=git_history" }, ]); renderOverviewChart("chart-freshness", "legend-freshness", [ @@ -1107,7 +1151,7 @@ ${timelineEvents} } function filterFromParams(params) { - for (const type of ["component", "flow", "kind", "source", "freshness", "commit"]) { + for (const type of ["component", "flow", "kind", "source", "freshness", "review", "commit"]) { const value = params.get(type); if (value) return { type, value }; } @@ -1136,9 +1180,12 @@ ${timelineEvents} case "kind": return "of type " + filter.value; case "source": - return filter.value === "session" ? "from session" : "from code"; + if (filter.value === "session") return "from session"; + return filter.value === "git_history" ? "from git history" : "from code"; case "freshness": return filter.value === "superseded" ? "superseded" : "active"; + case "review": + return filter.value === "needs_review" ? "needing review" : "current"; case "commit": { const event = graphData.claimsTimeline.events.find((item) => item.memoryCommitId === filter.value); if (event && event.createdAt) return "from commit on " + formatDateTimeClient(event.createdAt); @@ -1156,6 +1203,7 @@ ${timelineEvents} if (freshness !== "active") return false; if (filter.type === "kind") return row.dataset.kind === filter.value; if (filter.type === "source") return row.dataset.source === filter.value; + if (filter.type === "review") return row.dataset.review === filter.value; if (filter.type === "commit") return row.dataset.memoryCommitId === filter.value; if (filter.type === "component") { return (componentIdsByClaim.get(row.dataset.id) || []).includes(filter.value); diff --git a/libs/knowledge-graph/proposal.ts b/libs/knowledge-graph/proposal.ts index c04ff83..2c63edd 100644 --- a/libs/knowledge-graph/proposal.ts +++ b/libs/knowledge-graph/proposal.ts @@ -241,6 +241,8 @@ function defaultToType(kind: EdgeKind): GraphObjectType { return "component"; case "evidenced_by": return "source"; + case "needs_review": + return "claim"; } } diff --git a/libs/knowledge-graph/schema.ts b/libs/knowledge-graph/schema.ts index 178d89a..44dd126 100644 --- a/libs/knowledge-graph/schema.ts +++ b/libs/knowledge-graph/schema.ts @@ -30,7 +30,7 @@ export interface Flow { name: string; } -export type SourceKind = "session"; +export type SourceKind = "session" | "git_history"; export interface Source { id: SourceId; diff --git a/libs/knowledge-graph/service.ts b/libs/knowledge-graph/service.ts index 87314f4..36ea399 100644 --- a/libs/knowledge-graph/service.ts +++ b/libs/knowledge-graph/service.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { normalizeProposal, type MemoryCommitProposal } from "./proposal.js"; import { validateProposal, type ProposalValidationResult } from "./validate-proposal.js"; import type { Claim } from "./claim.js"; @@ -11,6 +12,7 @@ import { auditClaimCodeAnchors } from "./code-anchors/audit.js"; import { CodeAnchorResolver } from "./code-anchors/resolver.js"; import { fingerprintClaimAnchors } from "./code-anchors/fingerprint.js"; import type { ClaimAnchorAuditResult } from "./code-anchors/types.js"; +import type { ClaimAnchorAuditIssue } from "./code-anchors/types.js"; import { defaultDatabasePath, openDatabase } from "../storage/sqlite/db.js"; import type { SqliteRepository } from "../storage/sqlite/repository.js"; import { SqliteRepository as SqliteKnowledgeGraphRepository } from "../storage/sqlite/repository.js"; @@ -62,6 +64,12 @@ export interface ProposalReviewResult extends ProposalValidationResult { duplicate_warnings: Record; } +export interface StaleAnchorReviewResult { + audited_claims: number; + marked_claims: number; + memory_commit_id?: string; +} + export class KnowledgeGraphService { constructor( private readonly repository: SqliteRepository, @@ -144,6 +152,39 @@ export class KnowledgeGraphService { return auditClaimCodeAnchors(input.repo_root, claims, new CodeAnchorResolver(), baselineFingerprints); } + async markStaleAnchors( + input: RepoRef, + options: { changedFiles?: ReadonlySet; head?: string } = {}, + ): Promise { + const initialized = this.requireRepo(input); + const graph = this.repository.readGraphView(initialized.repo_id); + const result = await this.auditCodeAnchors(input); + const issues = staleAnchorIssues(result) + .filter((issue) => options.changedFiles === undefined || issue.anchor === undefined || options.changedFiles.has(issue.anchor.file)); + const byClaim = new Map(); + for (const issue of issues) if (!byClaim.has(issue.claim_id)) byClaim.set(issue.claim_id, issue); + const existingReviewClaims = new Set( + graph.edges + .filter((edge) => edge.kind === "needs_review" && edge.from_type === "claim") + .map((edge) => edge.from_id), + ); + const edges = [...byClaim.values()] + .filter((issue) => !existingReviewClaims.has(issue.claim_id)) + .map((issue) => needsReviewEdge(issue, options.head)); + if (edges.length === 0) return { audited_claims: graph.claims.length, marked_claims: 0 }; + + const applied = await this.applyProposal(input, { + title: `Mark ${edges.length} stale anchor${edges.length === 1 ? "" : "s"} for review`, + summary: "Automatically generated by git-aware anchor validation.", + creates: { edges }, + }); + return { + audited_claims: graph.claims.length, + marked_claims: edges.length, + memory_commit_id: applied.memory_commit_id, + }; + } + async validateProposal(input: RepoRef, proposal: unknown): Promise { const initialized = this.requireRepo(input); const normalizedProposal = normalizeProposal(proposal, this.subjectLookup(initialized.repo_id)); @@ -288,14 +329,43 @@ export class KnowledgeGraphService { private subjectLookup(repoId: string): { subjectExists: (type: GraphObjectType, id: string) => boolean; subjectType: (id: string) => GraphObjectType | undefined; + claimNeedsReview: (id: string) => boolean; } { return { subjectExists: (type, id) => this.repository.subjectExists(repoId, type, id), subjectType: (id) => this.repository.subjectType(repoId, id), + claimNeedsReview: (id) => this.repository.claimNeedsReview(repoId, id), }; } } +function staleAnchorIssues(result: ClaimAnchorAuditResult): ClaimAnchorAuditIssue[] { + return [...result.missing_files, ...result.missing_symbols, ...result.drifted]; +} + +function needsReviewEdge(issue: ClaimAnchorAuditIssue, head: string | undefined): Edge { + const reason = issue.status === "missing_file" + ? "file_deleted" + : issue.status === "missing_symbol" + ? "symbol_removed" + : "file_content_changed"; + const hash = createHash("sha1").update(issue.claim_id).digest("hex").slice(0, 12); + return { + id: `edge.needs_review.${hash}`, + from_type: "claim", + from_id: issue.claim_id, + to_type: "claim", + to_id: issue.claim_id, + kind: "needs_review", + metadata: { + reason, + anchor: issue.anchor, + head, + detected_at: new Date().toISOString(), + }, + }; +} + function anchorAuditErrors(result: ClaimAnchorAuditResult): string[] { return [ ...result.missing_anchors.map((issue) => `${issue.claim_id} is code_verified but has no code anchors`), diff --git a/libs/knowledge-graph/validate-proposal.ts b/libs/knowledge-graph/validate-proposal.ts index 7cdad63..db0e9d1 100644 --- a/libs/knowledge-graph/validate-proposal.ts +++ b/libs/knowledge-graph/validate-proposal.ts @@ -3,16 +3,17 @@ import type { EdgeKind } from "./edge.js"; import type { MemoryCommitProposal, ProposalSubject } from "./proposal.js"; import type { GraphObjectType } from "./schema.js"; -const claimKinds = new Set(["fact", "requirement", "decision", "task", "question", "risk"]); +const claimKinds = new Set(["fact", "requirement", "decision", "insight", "task", "question", "risk"]); const claimTruths = new Set(["code_verified", "source_verified", "unknown"]); const claimIntents = new Set(["intended", "accidental", "unknown"]); -const sourceKinds = new Set(["session"]); -const edgeKinds = new Set(["about", "contains", "touches", "supersedes", "evidenced_by"]); +const sourceKinds = new Set(["session", "git_history"]); +const edgeKinds = new Set(["about", "contains", "touches", "supersedes", "evidenced_by", "needs_review"]); const graphObjectTypes = new Set(["component", "flow", "claim", "edge", "source"]); const maxCodeAnchorsPerClaim = 3; export interface ExistingSubjectLookup { subjectExists(type: GraphObjectType, id: string): boolean; + claimNeedsReview?(id: string): boolean; } export interface ProposalValidationResult { @@ -134,7 +135,13 @@ export function validateProposal( graphObjectTypes.has(fromType) && graphObjectTypes.has(toType) && edgeKinds.has(kind) && - !isAllowedEdge({ from_type: fromType, to_type: toType, kind }) + !isAllowedEdge({ + from_id: String(edge.from_id), + from_type: fromType, + to_id: String(edge.to_id), + to_type: toType, + kind, + }) ) { errors.push(`Edge ${stringId(edge.id)} has invalid direction for ${kind}.`); } @@ -143,23 +150,45 @@ export function validateProposal( errors.push(`Edge ${stringId(edge.id)} metadata must be an object when present.`); } - if (kind === "evidenced_by") { + if (kind === "evidenced_by" || kind === "needs_review") { validateEvidenceMetadata(edge, errors); } + + if (kind !== "needs_review" && kind !== "supersedes") { + validateReviewedClaimReference(edge, existingSubjects, errors); + } } return { valid: errors.length === 0, errors }; } +function validateReviewedClaimReference( + edge: Record, + existingSubjects: ExistingSubjectLookup | undefined, + errors: string[], +): void { + if (existingSubjects?.claimNeedsReview === undefined) return; + const references = [ + edge.from_type === "claim" && typeof edge.from_id === "string" ? edge.from_id : undefined, + edge.to_type === "claim" && typeof edge.to_id === "string" ? edge.to_id : undefined, + ].filter((id): id is string => id !== undefined); + for (const claimId of new Set(references)) { + if (existingSubjects.claimNeedsReview(claimId)) { + errors.push(`Claim ${claimId} needs review before it can be referenced by new graph relationships.`); + } + } +} + function validateEvidenceMetadata(edge: Record, errors: string[]): void { + const kind = String(edge.kind); if (!isRecord(edge.metadata)) { - errors.push(`Edge ${stringId(edge.id)} evidenced_by edges require metadata.reason.`); + errors.push(`Edge ${stringId(edge.id)} ${kind} edges require metadata.reason.`); return; } const reason = edge.metadata.reason; if (!isNonEmptyString(reason)) { - errors.push(`Edge ${stringId(edge.id)} evidenced_by metadata.reason must be a non-empty string.`); + errors.push(`Edge ${stringId(edge.id)} ${kind} metadata.reason must be a non-empty string.`); } } diff --git a/libs/storage/sqlite/repository.ts b/libs/storage/sqlite/repository.ts index 774c67b..7e04551 100644 --- a/libs/storage/sqlite/repository.ts +++ b/libs/storage/sqlite/repository.ts @@ -79,6 +79,13 @@ export interface ClaimProvenanceRecord { memory_commit_id: string; } +export interface GitWatchStateRecord { + repo_id: string; + last_head: string; + last_full_audit_at: string | null; + updated_at: string; +} + export class SqliteRepository { constructor(private readonly db: Database.Database) {} @@ -236,6 +243,35 @@ export class SqliteRepository { .all(repoId) as ClaimProvenanceRecord[]; } + getGitWatchState(repoId: string): GitWatchStateRecord | undefined { + return this.db.prepare("SELECT * FROM git_watch_state WHERE repo_id = ?").get(repoId) as GitWatchStateRecord | undefined; + } + + upsertGitWatchState(input: GitWatchStateRecord): void { + this.db + .prepare( + `INSERT INTO git_watch_state (repo_id, last_head, last_full_audit_at, updated_at) + VALUES (@repo_id, @last_head, @last_full_audit_at, @updated_at) + ON CONFLICT(repo_id) DO UPDATE SET + last_head = excluded.last_head, + last_full_audit_at = excluded.last_full_audit_at, + updated_at = excluded.updated_at`, + ) + .run(input); + } + + claimNeedsReview(repoId: string, claimId: string): boolean { + const row = this.db + .prepare( + `SELECT id FROM edges + WHERE repo_id = ? AND kind = 'needs_review' + AND from_type = 'claim' AND from_id = ? + LIMIT 1`, + ) + .get(repoId, claimId); + return row !== undefined; + } + // Baseline anchor fingerprints stored when each claim was written, keyed by // claim id then by anchor key. Used by the anchor audit to detect drift. readClaimAnchorFingerprints(repoId: string, ids: string[]): Map> { diff --git a/libs/storage/sqlite/schema.ts b/libs/storage/sqlite/schema.ts index 490c556..1892f4b 100644 --- a/libs/storage/sqlite/schema.ts +++ b/libs/storage/sqlite/schema.ts @@ -116,6 +116,13 @@ CREATE TABLE IF NOT EXISTS agent_worker_locks ( updated_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS git_watch_state ( + repo_id TEXT PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE, + last_head TEXT NOT NULL, + last_full_audit_at TEXT, + updated_at TEXT NOT NULL +); + CREATE INDEX IF NOT EXISTS graph_scopes_repo_idx ON graph_scopes(repo_id); CREATE INDEX IF NOT EXISTS memory_commits_scope_idx ON memory_commits(scope_id); CREATE INDEX IF NOT EXISTS graph_memberships_scope_idx ON graph_memberships(scope_id); diff --git a/package.json b/package.json index 6544d30..b54e21d 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "smoke:copilot": "npm run build && node scripts/smoke-copilot-install.mjs", "smoke:opencode": "npm run build && node scripts/smoke-opencode-install.mjs", "smoke:cursor": "npm run build && node scripts/smoke-cursor-install.mjs", - "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-graph-view-offline-browser.js && node scripts/check-source-memberships.js && node scripts/check-proposal-validate.js && node scripts/check-bm25-tokenizer.js && node scripts/check-anchor-drift.js && node scripts/check-find-similar-claims.js && node scripts/check-apply-proposal-dedupe.js && node scripts/check-opencode-sqlite-transcript.js", + "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-graph-view-offline-browser.js && node scripts/check-source-memberships.js && node scripts/check-proposal-validate.js && node scripts/check-bm25-tokenizer.js && node scripts/check-anchor-drift.js && node scripts/check-find-similar-claims.js && node scripts/check-apply-proposal-dedupe.js && node scripts/check-opencode-sqlite-transcript.js && node scripts/check-git-memory.js", "test:transcript-bundle": "npm run build && node scripts/check-transcript-bundle.js", "test:repo-context": "npm run build && node scripts/check-repo-context.js", "test:source-memberships": "npm run build && node scripts/check-source-memberships.js", diff --git a/scripts/check-git-memory.js b/scripts/check-git-memory.js new file mode 100644 index 0000000..79d9c82 --- /dev/null +++ b/scripts/check-git-memory.js @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); +const { collectGitHistory, generateGitHistoryProposal, githubRepository } = await import( + new URL("dist/libs/git-memory/history.js", root) +); +const { runGitWatchCheck } = await import(new URL("dist/libs/git-memory/watch.js", root)); +const { validateProposal } = await import(new URL("dist/libs/knowledge-graph/validate-proposal.js", root)); +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); + +const repoRoot = mkdtempSync(join(tmpdir(), "greplica-git-memory-test-")); +git(["init", "-b", "main"]); +git(["config", "user.email", "test@example.com"]); +git(["config", "user.name", "Greplica Test"]); + +mkdirSync(join(repoRoot, "src", "auth"), { recursive: true }); +mkdirSync(join(repoRoot, "src", "api"), { recursive: true }); +writeFileSync(join(repoRoot, "src", "auth", "token.ts"), "export const token = 1;\n"); +writeFileSync(join(repoRoot, "src", "api", "client.ts"), "export const client = 1;\n"); +commitAll("feat(auth): add token integration", "We decided to keep token handling in auth."); + +writeFileSync(join(repoRoot, "src", "auth", "token.ts"), "export const token = 2;\n"); +writeFileSync(join(repoRoot, "src", "api", "client.ts"), "export const client = 2;\n"); +commitAll("refactor(api): align authentication client"); + +mkdirSync(join(repoRoot, "docs")); +writeFileSync(join(repoRoot, "docs", "notes.md"), "History notes.\n"); +commitAll("docs: add history notes"); + +const commits = collectGitHistory(repoRoot, { maxCommits: 10 }); +assert.equal(commits.length, 3); +assert.deepEqual(commits[0].files.map((file) => file.path), ["docs/notes.md"]); + +const emptyGraph = { components: [], flows: [], claims: [], sources: [], edges: [] }; +const plan = generateGitHistoryProposal(repoRoot, commits, emptyGraph); +assert.equal(plan.stats.commits_analyzed, 3); +assert.equal(plan.stats.components, 3); +assert.equal(plan.stats.flows, 1, "repeated auth/api co-changes should produce a flow"); +assert.equal(plan.stats.sources, 3, "every analyzed commit should retain provenance"); +assert.ok(plan.proposal.creates.claims.some((claim) => claim.kind === "insight")); +assert.ok(plan.proposal.creates.claims.some((claim) => claim.kind === "decision")); +assert.ok(plan.proposal.creates.sources.every((source) => source.kind === "git_history")); +assert.deepEqual(validateProposal(plan.proposal), { valid: true, errors: [] }); +assert.equal(githubRepository("git@github.com:Autoloops/greplica.git"), "Autoloops/greplica"); + +const database = openDatabase(join(mkdtempSync(join(tmpdir(), "greplica-git-memory-db-")), "graph.db")); +try { + const repository = new SqliteRepository(database); + const embeddingBuilder = { + async ensureForGraph() { + return { checked_objects: 0, created: 0, reused: 0 }; + }, + }; + const service = new KnowledgeGraphService(repository, undefined, embeddingBuilder); + const repo = { repo_root: repoRoot, repo_name: "git-memory-test", default_branch: "main" }; + service.initRepo(repo); + await service.applyProposal(repo, { + title: "Seed anchored claim", + creates: { + components: [{ id: "component.auth", name: "Authentication" }], + claims: [{ + id: "claim.token_value", + kind: "fact", + text: "The token constant is 2.", + truth: "code_verified", + intent: "intended", + code_anchors: [{ file: "src/auth/token.ts" }], + }], + edges: [{ + id: "edge.claim_token_auth", + from_id: "claim.token_value", + from_type: "claim", + to_id: "component.auth", + to_type: "component", + kind: "about", + }], + }, + }); + + const first = await runGitWatchCheck(repoRoot, repo, service, repository, { + anchorThresholdDays: 7, + now: new Date("2026-01-01T00:00:00Z"), + }); + assert.equal(first.full_audit, true); + assert.equal(first.marked_claims, 0); + + writeFileSync(join(repoRoot, "src", "auth", "token.ts"), "export const token = 3;\n"); + commitAll("fix(auth): rotate token constant"); + const second = await runGitWatchCheck(repoRoot, repo, service, repository, { + anchorThresholdDays: 7, + now: new Date("2026-01-02T00:00:00Z"), + }); + assert.equal(second.full_audit, false); + assert.deepEqual(second.changed_files, ["src/auth/token.ts"]); + assert.equal(second.marked_claims, 1); + + const reviewEdge = service.readGraph(repo).edges.find((edge) => edge.kind === "needs_review"); + assert.equal(reviewEdge?.from_id, "claim.token_value"); + assert.equal(reviewEdge?.to_id, "claim.token_value"); + assert.equal(reviewEdge?.metadata?.reason, "file_content_changed"); + + const staleReference = await service.validateProposal(repo, { + title: "Reference stale claim", + creates: { edges: [{ + id: "edge.stale_claim_auth", + from_id: "claim.token_value", + from_type: "claim", + to_id: "component.auth", + to_type: "component", + kind: "about", + }] }, + }); + assert.equal(staleReference.valid, false); + assert.ok(staleReference.errors.some((error) => error.includes("needs review"))); + + const unchanged = await runGitWatchCheck(repoRoot, repo, service, repository, { + anchorThresholdDays: 7, + now: new Date("2026-01-02T01:00:00Z"), + }); + assert.equal(unchanged.skipped, true); +} finally { + database.close(); +} + +console.log("check-git-memory: ok"); + +function git(args) { + return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" }).trim(); +} + +function commitAll(subject, body) { + git(["add", "."]); + const args = ["commit", "-m", subject]; + if (body) args.push("-m", body); + git(args); +} diff --git a/scripts/check-graph-view.js b/scripts/check-graph-view.js index f68d48f..0e082ce 100644 --- a/scripts/check-graph-view.js +++ b/scripts/check-graph-view.js @@ -113,7 +113,10 @@ async function checkRichGraphIsSelfContained() { normalizeProposal({ title: "Seed rich graph — batch 1", creates: { - sources: [{ id: "source.pairing", kind: "session", ref: "session-abc", title: "Pairing session with Jane" }], + sources: [ + { id: "source.pairing", kind: "session", ref: "session-abc", title: "Pairing session with Jane" }, + { id: "source.git", kind: "git_history", ref: "git:abc123", title: "abc123 add auth" }, + ], components: [ { id: "component.auth", name: "Auth Service", code_anchor: "libs/auth/service.ts:1-40", contains: "component.auth.token" }, { id: "component.auth.token", name: "Token Store", code_anchor: "libs/auth/token-store.ts:1-20" }, @@ -170,7 +173,25 @@ async function checkRichGraphIsSelfContained() { intent: "intended", about: "component.auth", }, + { + id: "claim.git_insight", + kind: "insight", + text: "Auth and token storage change together", + truth: "source_verified", + intent: "intended", + evidenced_by: "source.git", + about: "component.auth", + }, ], + edges: [{ + id: "edge.git_insight_review", + from_id: "claim.git_insight", + from_type: "claim", + to_id: "claim.git_insight", + to_type: "claim", + kind: "needs_review", + metadata: { reason: "file_content_changed" }, + }], }, }), ); @@ -217,7 +238,11 @@ async function checkRichGraphIsSelfContained() { assert.match(html, /Pairing session with Jane/); assert.match(html, /Add token rotation with configurable interval/); assert.match(html, /data-freshness="superseded"/, "expected the superseded claim to be marked as such"); - assert.match(html, /"total":6/, "expected claims timeline summary to count 6 active claims"); + assert.match(html, /data-review="needs_review"/, "expected the stale claim to be marked for review"); + assert.match(html, /Claims - Needs review/); + assert.match(html, /from git history/); + assert.match(html, /"needsReview":1/); + assert.match(html, /"total":7/, "expected claims timeline summary to count 7 active claims"); // ...and, regardless of how much data or how many kinds/sources/superseded // claims it contains, it remains fully self-contained (this is the actual