diff --git a/.changeset/file-history-turn-snapshots.md b/.changeset/file-history-turn-snapshots.md new file mode 100644 index 00000000000..2d7dfa6e258 --- /dev/null +++ b/.changeset/file-history-turn-snapshots.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": minor +--- + +Add experimental turn-level file snapshots: the daemon backs up the original content of every file it edits and versions all tracked files at each turn boundary. Enable with KIMI_CODE_EXPERIMENTAL_FILE_HISTORY=1. diff --git a/packages/agent-core-v2/docs/state-manifest.d.ts b/packages/agent-core-v2/docs/state-manifest.d.ts index ee469d519b4..8f8bdb0002d 100644 --- a/packages/agent-core-v2/docs/state-manifest.d.ts +++ b/packages/agent-core-v2/docs/state-manifest.d.ts @@ -27,7 +27,7 @@ // references become '(circular)', and class instances collapse to a '(ClassName)' // marker — the wire shape of an entry is the JSON projection of the type here. // -// Index (App: 0 keys · Workspace: 6 keys · Session: 9 keys · Agent: 79 keys) +// Index (App: 0 keys · Workspace: 6 keys · Session: 9 keys · Agent: 80 keys) // App // Workspace // workspaceDirs.ephemeralDirs src/workspace/workspaceDirs/workspaceDirsService.ts @@ -59,6 +59,7 @@ // contextMemory src/agent/contextMemory/contextOps.ts // contextProjector.lastRepairSignature src/agent/contextProjector/contextProjectorService.ts // externalHooks.stopHookContinuationUsed src/features/externalHooks/agent/agentExternalHooksService.ts +// fileHistory src/features/fileHistory/fileHistoryOps.ts // fullCompaction src/agent/fullCompaction/compactionOps.ts // fullCompaction.activeTurnId src/agent/fullCompaction/fullCompactionService.ts // fullCompaction.compactionCountInTurn src/agent/fullCompaction/fullCompactionService.ts @@ -1493,6 +1494,23 @@ export interface AgentStateSnapshot { }>; // src/features/externalHooks/agent/agentExternalHooksService.ts 'externalHooks.stopHookContinuationUsed': boolean; + // src/features/fileHistory/fileHistoryOps.ts + // replayable · durable — folds: FileHistoryCheckpointed, FileHistoryTracked + 'fileHistory': /* FileHistoryState — packages/agent-core-v2/src/features/fileHistory/fileHistory.ts */ { + readonly checkpoints: readonly /* FileHistoryCheckpointRecord — packages/agent-core-v2/src/features/fileHistory/fileHistory.ts */ { + readonly turnId: number; + readonly phase?: 'start' | 'end'; + readonly entries: Readonly>; + }[]; + readonly tracked: readonly string[]; + }; // src/features/plan/injection/planModeInjection.ts 'plan.wasActive': boolean; // src/features/plan/planOps.ts diff --git a/packages/agent-core-v2/docs/wire-manifest.d.ts b/packages/agent-core-v2/docs/wire-manifest.d.ts index 2654e8c9be5..145d0d58c70 100644 --- a/packages/agent-core-v2/docs/wire-manifest.d.ts +++ b/packages/agent-core-v2/docs/wire-manifest.d.ts @@ -24,7 +24,7 @@ // cross-reducers), blobs (the folding states whose blob codec offloads inline // media to blob storage), owner (the source file declaring the class). -// Index (55 record types) +// Index (57 record types) // config.update profile src/agent/profile/profileOps.ts // context.append_loop_event contextMemory, turn src/agent/contextMemory/contextEvents.ts // context.append_message contextMemory, plan, task.notificationDelivery src/agent/contextMemory/contextEvents.ts @@ -34,6 +34,8 @@ // cron.add (none) src/features/cron/cronOps.ts // cron.cursor (none) src/features/cron/cronOps.ts // cron.delete (none) src/features/cron/cronOps.ts +// file_history.checkpoint fileHistory src/features/fileHistory/fileHistoryOps.ts +// file_history.tracked fileHistory src/features/fileHistory/fileHistoryOps.ts // forked (none) src/features/goal/goalOps.ts // full_compaction.begin fullCompaction src/agent/fullCompaction/compactionOps.ts // full_compaction.cancel fullCompaction src/agent/fullCompaction/compactionOps.ts @@ -214,6 +216,37 @@ interface CronDeletePayload { ids: string[]; } +/** + * states: fileHistory + * owner: src/features/fileHistory/fileHistoryOps.ts + */ +interface FileHistoryCheckpointPayload { + _name: 'file_history.checkpoint'; + agentId: string; + turnId: number; + phase?: 'start' | 'end'; + entries: Record; +} + +/** + * states: fileHistory + * owner: src/features/fileHistory/fileHistoryOps.ts + */ +interface FileHistoryTrackedPayload { + _name: 'file_history.tracked'; + agentId: string; + turnId: number; + path: string; + entry: { + key: string | null; + version: number; + contentHash?: string; + size?: number; + oversize?: boolean; + mtimeMs?: number; + }; +} + /** * states: (none) * owner: src/features/goal/goalOps.ts @@ -837,6 +870,8 @@ interface WirePayloadMap { "cron.add": CronAddPayload; "cron.cursor": CronCursorPayload; "cron.delete": CronDeletePayload; + "file_history.checkpoint": FileHistoryCheckpointPayload; + "file_history.tracked": FileHistoryTrackedPayload; "forked": ForkedPayload; "full_compaction.begin": FullCompactionBeginPayload; "full_compaction.cancel": FullCompactionCancelPayload; diff --git a/packages/agent-core-v2/src/features/fileHistory/fileHistory.ts b/packages/agent-core-v2/src/features/fileHistory/fileHistory.ts new file mode 100644 index 00000000000..959f92e6c3b --- /dev/null +++ b/packages/agent-core-v2/src/features/fileHistory/fileHistory.ts @@ -0,0 +1,57 @@ +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; + +export interface FileBackupEntry { + readonly key: string | null; + readonly version: number; + readonly contentHash?: string; + readonly size?: number; + readonly oversize?: boolean; + readonly mtimeMs?: number; +} + +export type FileHistoryCheckpointPhase = 'start' | 'end'; + +export interface FileHistoryCheckpointRecord { + readonly turnId: number; + readonly phase?: FileHistoryCheckpointPhase; + readonly entries: Readonly>; +} + +export interface FileHistoryState { + readonly checkpoints: readonly FileHistoryCheckpointRecord[]; + readonly tracked: readonly string[]; +} + +export type FileHistoryChangeStatus = 'added' | 'modified' | 'deleted'; + +export interface FileHistoryChange { + readonly path: string; + readonly status: FileHistoryChangeStatus; + readonly additions: number; + readonly deletions: number; + readonly binary?: boolean; + readonly oversize?: boolean; +} + +export interface FileHistoryContent { + readonly version: number; + readonly content?: string; + readonly binary?: boolean; +} + +export interface IAgentFileHistoryService { + readonly _serviceBrand: undefined; + + enabled(): boolean; + history(): FileHistoryState; + settled(): Promise; + changes(turnId: number): Promise; + contentAt( + turnId: number, + path: string, + phase?: FileHistoryCheckpointPhase, + ): Promise; +} + +export const IAgentFileHistoryService: ServiceIdentifier = + createDecorator('agentFileHistoryService'); diff --git a/packages/agent-core-v2/src/features/fileHistory/fileHistoryFeature.ts b/packages/agent-core-v2/src/features/fileHistory/fileHistoryFeature.ts new file mode 100644 index 00000000000..deb2e2b101d --- /dev/null +++ b/packages/agent-core-v2/src/features/fileHistory/fileHistoryFeature.ts @@ -0,0 +1,17 @@ +import { Feature } from '#/features/feature'; +import { registerFeature } from '#/features/featureRegistry'; + +import './flag'; +import { IAgentFileHistoryService } from './fileHistory'; +import { AgentFileHistoryService } from './fileHistoryService'; + +export class FileHistoryFeature extends Feature { + static override readonly name = 'fileHistory'; + + constructor() { + super(); + this.contributeAgentService(IAgentFileHistoryService, AgentFileHistoryService); + } +} + +registerFeature(FileHistoryFeature); diff --git a/packages/agent-core-v2/src/features/fileHistory/fileHistoryOps.ts b/packages/agent-core-v2/src/features/fileHistory/fileHistoryOps.ts new file mode 100644 index 00000000000..8032f618642 --- /dev/null +++ b/packages/agent-core-v2/src/features/fileHistory/fileHistoryOps.ts @@ -0,0 +1,115 @@ +/* oxlint-disable typescript-eslint/no-unsafe-declaration-merging, eslint-plugin-import/namespace -- Event2 class+payload-interface declaration merging is the sanctioned event-declaration idiom. */ +import { z } from 'zod'; + +import { AgentEvent2 } from '#/app/event/event2'; +import { defineState } from '#/state/state'; + +import type { + FileBackupEntry, + FileHistoryCheckpointPhase, + FileHistoryState, +} from './fileHistory'; + +export const FILE_HISTORY_CHECKPOINT_CAP = 400; + +const backupEntrySchema = z.object({ + key: z.string().nullable(), + version: z.number(), + contentHash: z.string().optional(), + size: z.number().optional(), + oversize: z.boolean().optional(), + mtimeMs: z.number().optional(), +}); + +const fileHistoryTrackedSchema = z.object({ + agentId: z.string(), + turnId: z.number(), + path: z.string(), + entry: backupEntrySchema, +}); + +export class FileHistoryTracked extends AgentEvent2> { + static override readonly type = 'file_history.tracked'; + static override readonly durable = true; + static override readonly schema = fileHistoryTrackedSchema; +} +export interface FileHistoryTracked { + readonly agentId: string; + readonly turnId: number; + readonly path: string; + readonly entry: FileBackupEntry; +} + +const fileHistoryCheckpointedSchema = z.object({ + agentId: z.string(), + turnId: z.number(), + phase: z.enum(['start', 'end']).optional(), + entries: z.record(z.string(), backupEntrySchema), +}); + +export class FileHistoryCheckpointed extends AgentEvent2< + z.infer +> { + static override readonly type = 'file_history.checkpoint'; + static override readonly durable = true; + static override readonly schema = fileHistoryCheckpointedSchema; +} +export interface FileHistoryCheckpointed { + readonly agentId: string; + readonly turnId: number; + readonly phase?: FileHistoryCheckpointPhase; + readonly entries: Readonly>; +} + +export function checkpointPhaseOf(record: { + readonly phase?: FileHistoryCheckpointPhase; +}): FileHistoryCheckpointPhase { + return record.phase ?? 'start'; +} + +function cloneEntries( + entries: Readonly>, +): Record { + const clone: Record = Object.create(null) as Record< + string, + FileBackupEntry + >; + for (const [path, entry] of Object.entries(entries)) clone[path] = entry; + return clone; +} + +export const fileHistoryKey = defineState( + 'fileHistory', + (): FileHistoryState => ({ checkpoints: [], tracked: [] }), +) + .replayable({ schema: z.custom() }) + .on(FileHistoryCheckpointed, (s, e) => { + const phase = checkpointPhaseOf(e); + const base = s.checkpoints.at(-1)?.entries; + const merged = cloneEntries(base ?? {}); + for (const [path, entry] of Object.entries(e.entries)) merged[path] = { ...entry }; + const existing = s.checkpoints.find( + (c) => c.turnId === e.turnId && checkpointPhaseOf(c) === phase, + ); + if (existing !== undefined) { + existing.entries = merged; + return; + } + s.checkpoints.push({ turnId: e.turnId, phase, entries: merged }); + if (s.checkpoints.length > FILE_HISTORY_CHECKPOINT_CAP) { + s.checkpoints.splice(0, s.checkpoints.length - FILE_HISTORY_CHECKPOINT_CAP); + } + }) + .on(FileHistoryTracked, (s, e) => { + if (!s.tracked.includes(e.path)) s.tracked.push(e.path); + let checkpoint = s.checkpoints.find( + (c) => c.turnId === e.turnId && checkpointPhaseOf(c) === 'start', + ); + if (checkpoint === undefined) { + s.checkpoints.push({ turnId: e.turnId, phase: 'start', entries: {} }); + checkpoint = s.checkpoints.at(-1); + } + if (checkpoint !== undefined && !Object.hasOwn(checkpoint.entries, e.path)) { + checkpoint.entries[e.path] = { ...e.entry }; + } + }); diff --git a/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts b/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts new file mode 100644 index 00000000000..bc5ad978dd4 --- /dev/null +++ b/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts @@ -0,0 +1,559 @@ +import { createHash } from 'node:crypto'; +import { isAbsolute, relative, resolve } from 'pathe'; + +import { Service } from '#/_base/di/service'; +import { unwrapErrorCause } from '#/_base/errors/errors'; +import { onUnexpectedError } from '#/_base/errors/unexpectedError'; +import { IAgentRuntimeService } from '#/agent/runtimeBinding/agentRuntime'; +import { IAgentScopeContext } from '#/agent/scopeContext/scopeContext'; +import { IAgentStateService } from '#/agent/state/agentState'; +import { IAgentToolExecutorService } from '#/agent/toolExecutor/toolExecutor'; +import type { WillExecuteToolEvent } from '#/agent/toolExecutor/toolHooks'; +import { TurnStarted } from '#/agent/loop/turnEvents'; +import { TurnEnded } from '#/agent/loop/turnOps'; +import { IEventBus } from '#/app/event/eventBus'; +import { IFlagService } from '#/app/flag/flag'; +import { IBlobStore } from '#/persistence/interface/blobStore'; +import { MAIN_AGENT_ID } from '#/session/agentLifecycle/agentLifecycle'; +import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; +import { IEventDispatcher } from '#/state/eventDispatcher'; +import type { ToolInputDisplay } from '#/tool/toolInputDisplay'; + +import { + IAgentFileHistoryService, + type FileBackupEntry, + type FileHistoryChange, + type FileHistoryCheckpointPhase, + type FileHistoryCheckpointRecord, + type FileHistoryContent, + type FileHistoryState, +} from './fileHistory'; +import { + FILE_HISTORY_CHECKPOINT_CAP, + FileHistoryCheckpointed, + FileHistoryTracked, + checkpointPhaseOf, + fileHistoryKey, +} from './fileHistoryOps'; +import { FILE_HISTORY_FLAG_ID } from './flag'; + +export const FILE_HISTORY_MAX_FILE_BYTES = 4 * 1024 * 1024; +export const FILE_HISTORY_BLOB_PREFIX = 'file-history'; + +export class AgentFileHistoryService extends Service implements IAgentFileHistoryService { + declare readonly _serviceBrand: undefined; + + private queue: Promise = Promise.resolve(); + + constructor( + @IAgentScopeContext private readonly agentCtx: IAgentScopeContext, + @IAgentStateService private readonly agentState: IAgentStateService, + @IAgentToolExecutorService toolExecutor: IAgentToolExecutorService, + @IEventBus eventBus: IEventBus, + @IEventDispatcher private readonly dispatcher: IEventDispatcher, + @IFlagService private readonly flags: IFlagService, + @IAgentRuntimeService private readonly runtime: IAgentRuntimeService, + @IBlobStore private readonly blobs: IBlobStore, + @ISessionWorkspaceContext private readonly workspaceCtx: ISessionWorkspaceContext, + ) { + super(); + this.agentState.contributeState(fileHistoryKey); + if (this.agentCtx.agentId !== MAIN_AGENT_ID) return; + + this._register( + toolExecutor.onWillExecuteTool((event) => this.onWillExecuteTool(event)), + ); + this._register( + eventBus.subscribe(TurnStarted, (event) => { + if (event.agentId !== this.agentCtx.agentId || !this.enabled()) return; + void this.enqueue(() => this.checkpoint(event.turnId, 'start')); + }), + ); + this._register( + eventBus.subscribe(TurnEnded, (event) => { + if (event.agentId !== this.agentCtx.agentId || !this.enabled()) return; + void this.enqueue(() => this.checkpoint(event.turnId, 'end')); + }), + ); + } + + enabled(): boolean { + return this.flags.enabled(FILE_HISTORY_FLAG_ID); + } + + history(): FileHistoryState { + return this.agentState.get(fileHistoryKey); + } + + settled(): Promise { + return this.queue; + } + + changes(turnId: number): Promise { + if (!this.enabled()) return Promise.resolve([]); + return this.enqueueValue(() => this.readChanges(turnId)); + } + + private async readChanges(turnId: number): Promise { + const state = this.history(); + const index = state.checkpoints.findIndex( + (c) => c.turnId === turnId && checkpointPhaseOf(c) === 'start', + ); + if (index < 0) return []; + const end = state.checkpoints.find( + (c) => c.turnId === turnId && checkpointPhaseOf(c) === 'end', + ); + const live = end === undefined && index === state.checkpoints.length - 1; + if (end === undefined && !live) return []; + + const paths = new Set(); + for (const path of Object.keys(state.checkpoints[index]!.entries)) paths.add(path); + if (end !== undefined) for (const path of Object.keys(end.entries)) paths.add(path); + else for (const path of state.tracked) paths.add(path); + const endIndex = end === undefined ? -1 : state.checkpoints.indexOf(end); + + const changes: FileHistoryChange[] = []; + for (const path of [...paths].toSorted()) { + const before = entryAt(state.checkpoints, index, path); + const after = end !== undefined ? entryAt(state.checkpoints, endIndex, path) : undefined; + if (before?.oversize === true && after?.oversize === true) { + if (before.version !== after.version) { + changes.push({ path, status: 'modified', additions: 0, deletions: 0, oversize: true }); + } + continue; + } + const beforeMissing = before === undefined || (before.key === null && before.oversize !== true); + let liveOversize: { size: number; mtimeMs?: number } | undefined; + let liveMissing = false; + let afterBytes: Uint8Array | undefined; + if (end !== undefined) { + if (before?.oversize !== true && after?.oversize !== true) { + afterBytes = await this.entryBytes(after); + } + } else { + const current = await this.readCurrent(path); + if (current === 'unreadable') continue; + if (current instanceof Uint8Array) afterBytes = current; + else if (current === 'missing') liveMissing = true; + else liveOversize = { size: current.oversizeBytes, mtimeMs: current.mtimeMs }; + } + const afterMissing = + end !== undefined + ? after === undefined || (after.key === null && after.oversize !== true) + : liveMissing; + if (before?.oversize === true || after?.oversize === true || liveOversize !== undefined) { + if ( + before?.oversize === true && + liveOversize !== undefined && + before.size === liveOversize.size && + before.mtimeMs === liveOversize.mtimeMs + ) { + continue; + } + const status = beforeMissing ? 'added' : afterMissing ? 'deleted' : 'modified'; + changes.push({ path, status, additions: 0, deletions: 0, oversize: true }); + continue; + } + const beforeBytes = await this.entryBytes(before); + const change = diffChange(path, beforeBytes, afterBytes); + if (change !== undefined) changes.push(change); + } + return changes; + } + + contentAt( + turnId: number, + path: string, + phase: FileHistoryCheckpointPhase = 'start', + ): Promise { + if (!this.enabled()) return Promise.resolve(undefined); + return this.enqueueValue(() => this.readContentAt(turnId, path, phase)); + } + + private async readContentAt( + turnId: number, + path: string, + phase: FileHistoryCheckpointPhase, + ): Promise { + const state = this.history(); + const index = state.checkpoints.findIndex( + (c) => c.turnId === turnId && checkpointPhaseOf(c) === phase, + ); + if (index < 0) return undefined; + const entry = entryAt(state.checkpoints, index, this.pathKey(path)); + if (entry === undefined || entry.oversize === true) return undefined; + if (entry.key === null) return { version: entry.version }; + const bytes = await this.blobs.get(this.agentCtx.scope(), entry.key); + if (bytes === undefined) return undefined; + const content = decodeText(bytes); + if (content === undefined) return { version: entry.version, binary: true }; + return { version: entry.version, content }; + } + + private onWillExecuteTool(event: WillExecuteToolEvent): void { + if (!this.enabled()) return; + const path = editTargetPath(event.execution.display); + if (path === undefined) return; + event.waitUntil(this.enqueue(() => this.capture(path, event.turnId))); + } + + private enqueue(op: () => Promise): Promise { + return this.enqueueValue(op); + } + + private enqueueValue(op: () => Promise): Promise { + const run = this.queue.then(op); + this.queue = run.then( + () => undefined, + (error) => { + onUnexpectedError(error); + }, + ); + return run; + } + + private async capture(path: string, turnId: number): Promise { + const pathKey = this.pathKey(path); + const state = this.history(); + if (state.tracked.includes(pathKey)) return; + + const current = await this.readCurrent(pathKey); + if (current === 'unreadable') return; + let entry: FileBackupEntry; + if (current === 'missing') entry = { key: null, version: 1 }; + else if (current instanceof Uint8Array) entry = await this.backup(pathKey, 1, current); + else { + entry = { + key: null, + version: 1, + oversize: true, + size: current.oversizeBytes, + mtimeMs: current.mtimeMs, + }; + } + await this.dispatcher.dispatch( + new FileHistoryTracked({ agentId: this.agentCtx.agentId, turnId, path: pathKey, entry }), + ); + } + + private async checkpoint(turnId: number, phase: FileHistoryCheckpointPhase): Promise { + const state = this.history(); + if (state.checkpoints.some((c) => c.turnId === turnId && checkpointPhaseOf(c) === phase)) { + return; + } + + const entries: Record = Object.create(null) as Record< + string, + FileBackupEntry + >; + for (const pathKey of state.tracked) { + const latest = latestEntry(state.checkpoints, pathKey); + const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; + const current = await this.readCurrent(pathKey); + if (current === 'unreadable') continue; + if (current === 'missing') { + if (latest === undefined || latest.key !== null || latest.oversize === true) { + entries[pathKey] = { key: null, version: nextVersion }; + } + continue; + } + if (!(current instanceof Uint8Array)) { + if ( + latest?.oversize !== true || + latest.size !== current.oversizeBytes || + latest.mtimeMs !== current.mtimeMs + ) { + entries[pathKey] = { + key: null, + version: nextVersion, + oversize: true, + size: current.oversizeBytes, + mtimeMs: current.mtimeMs, + }; + } + continue; + } + const contentHash = sha256(current); + if (latest !== undefined && latest.contentHash === contentHash) continue; + entries[pathKey] = await this.backup(pathKey, nextVersion, current, contentHash); + } + + const evictable = evictableCheckpoints(state.checkpoints); + await this.dispatcher.dispatch( + new FileHistoryCheckpointed({ agentId: this.agentCtx.agentId, turnId, phase, entries }), + ); + await this.evictBlobs(evictable, this.history().checkpoints); + } + + private async backup( + pathKey: string, + version: number, + content: Uint8Array, + contentHash?: string, + ): Promise { + const hash = contentHash ?? sha256(content); + const key = blobKey(pathKey, version); + await this.blobs.put(this.agentCtx.scope(), key, content); + return { key, version, contentHash: hash, size: content.byteLength }; + } + + private async evictBlobs( + evicted: readonly FileHistoryCheckpointRecord[], + retained: readonly FileHistoryCheckpointRecord[], + ): Promise { + if (evicted.length === 0) return; + const retainedKeys = new Set(); + for (const checkpoint of retained) { + for (const entry of Object.values(checkpoint.entries)) { + if (entry.key !== null) retainedKeys.add(entry.key); + } + } + for (const checkpoint of evicted) { + for (const entry of Object.values(checkpoint.entries)) { + if (entry.key === null || entry.version === 1 || retainedKeys.has(entry.key)) continue; + try { + await this.blobs.delete(this.agentCtx.scope(), entry.key); + } catch (error) { + onUnexpectedError(error); + } + } + } + } + + private async entryBytes(entry: FileBackupEntry | undefined): Promise { + if (entry === undefined || entry.key === null) return undefined; + return this.blobs.get(this.agentCtx.scope(), entry.key); + } + + private async readCurrent( + pathKey: string, + ): Promise< + Uint8Array | 'missing' | 'unreadable' | { oversizeBytes: number; mtimeMs?: number } + > { + const absolute = isAbsolute(pathKey) ? pathKey : resolve(this.workspaceCtx.workDir, pathKey); + const lease = this.runtime.acquire(['fs']); + try { + const fs = lease.runtime.fs; + if (fs === undefined) return 'unreadable'; + let info; + try { + info = await fs.stat(absolute); + } catch (error) { + const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; + return code === 'ENOENT' ? 'missing' : 'unreadable'; + } + if (!info.isFile) return 'unreadable'; + if (info.size > FILE_HISTORY_MAX_FILE_BYTES) { + return { oversizeBytes: info.size, mtimeMs: info.mtimeMs }; + } + try { + const bytes = await fs.readBytes(absolute, FILE_HISTORY_MAX_FILE_BYTES + 1); + if (bytes.byteLength > FILE_HISTORY_MAX_FILE_BYTES) { + const grown = await fs.stat(absolute).catch(() => undefined); + return { + oversizeBytes: grown?.size ?? bytes.byteLength, + mtimeMs: grown?.mtimeMs, + }; + } + if (bytes.byteLength !== info.size) return 'unreadable'; + return bytes; + } catch { + return 'unreadable'; + } + } finally { + lease.dispose(); + } + } + + private pathKey(path: string): string { + let raw = path; + if (isAbsolute(path)) { + const relativePath = relative(this.workspaceCtx.workDir, path); + if (relativePath !== '' && relativePath !== '..' && !relativePath.startsWith('../')) { + raw = relativePath; + } + } + const key = this.comparisonKey(raw); + const existing = this.history().tracked.find( + (tracked) => this.comparisonKey(tracked) === key, + ); + return existing ?? raw; + } + + private comparisonKey(pathKey: string): string { + return isWindowsPath(this.workspaceCtx.workDir) ? pathKey.toLowerCase() : pathKey; + } +} + +function isWindowsPath(value: string): boolean { + return /^[a-zA-Z]:[\\/]/.test(value) || /^[\\/]{2}[^\\/]+[\\/][^\\/]+/.test(value); +} + +function editTargetPath(display: ToolInputDisplay | undefined): string | undefined { + if (display === undefined || display.kind !== 'file_io') return undefined; + if (display.operation !== 'edit' && display.operation !== 'write') return undefined; + return display.path; +} + +function entryAt( + checkpoints: readonly FileHistoryCheckpointRecord[], + index: number, + path: string, +): FileBackupEntry | undefined { + for (let i = index; i >= 0; i -= 1) { + const record = checkpoints[i]!.entries; + if (Object.hasOwn(record, path)) return record[path]; + } + return undefined; +} + +function latestEntry( + checkpoints: readonly FileHistoryCheckpointRecord[], + path: string, +): FileBackupEntry | undefined { + return entryAt(checkpoints, checkpoints.length - 1, path); +} + +function maxVersion( + checkpoints: readonly FileHistoryCheckpointRecord[], + path: string, +): number { + let max = 0; + for (const checkpoint of checkpoints) { + const entry = Object.hasOwn(checkpoint.entries, path) ? checkpoint.entries[path] : undefined; + if (entry !== undefined && entry.version > max) max = entry.version; + } + return max; +} + +function evictableCheckpoints( + checkpoints: readonly FileHistoryCheckpointRecord[], +): readonly FileHistoryCheckpointRecord[] { + const overflow = checkpoints.length + 1 - FILE_HISTORY_CHECKPOINT_CAP; + return overflow > 0 ? checkpoints.slice(0, overflow) : []; +} + +function blobKey(pathKey: string, version: number): string { + const hash = createHash('sha256').update(pathKey, 'utf8').digest('hex').slice(0, 16); + return `${FILE_HISTORY_BLOB_PREFIX}/${hash}@v${String(version)}`; +} + +function sha256(bytes: Uint8Array): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function decodeText(bytes: Uint8Array): string | undefined { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + return undefined; + } +} + +function diffChange( + path: string, + beforeBytes: Uint8Array | undefined, + afterBytes: Uint8Array | undefined, +): FileHistoryChange | undefined { + if (beforeBytes === undefined && afterBytes === undefined) return undefined; + const before = beforeBytes === undefined ? undefined : decodeText(beforeBytes); + const after = afterBytes === undefined ? undefined : decodeText(afterBytes); + const binary = + (beforeBytes !== undefined && before === undefined) || + (afterBytes !== undefined && after === undefined); + + if (beforeBytes === undefined) { + return binary + ? { path, status: 'added', additions: 0, deletions: 0, binary } + : { path, status: 'added', additions: countLines(after ?? ''), deletions: 0 }; + } + if (afterBytes === undefined) { + return binary + ? { path, status: 'deleted', additions: 0, deletions: 0, binary } + : { path, status: 'deleted', additions: 0, deletions: countLines(before ?? '') }; + } + if (binary) { + return bytesEqual(beforeBytes, afterBytes) + ? undefined + : { path, status: 'modified', additions: 0, deletions: 0, binary }; + } + if (before === after) return undefined; + const counted = countLineDiff(before ?? '', after ?? ''); + if (counted === undefined) { + return { path, status: 'modified', additions: 0, deletions: 0, oversize: true }; + } + return { path, status: 'modified', additions: counted.additions, deletions: counted.deletions }; +} + +function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i += 1) if (a[i] !== b[i]) return false; + return true; +} + +function splitLines(content: string): string[] { + if (content === '') return []; + const lines = content.split('\n'); + if (lines.at(-1) === '') lines.pop(); + return lines; +} + +function countLines(content: string): number { + return splitLines(content).length; +} + +export function countLineDiff( + before: string, + after: string, +): { additions: number; deletions: number } | undefined { + const beforeLines = splitLines(before); + const afterLines = splitLines(after); + + let start = 0; + while ( + start < beforeLines.length && + start < afterLines.length && + beforeLines[start] === afterLines[start] + ) { + start += 1; + } + let beforeEnd = beforeLines.length; + let afterEnd = afterLines.length; + while ( + beforeEnd > start && + afterEnd > start && + beforeLines[beforeEnd - 1] === afterLines[afterEnd - 1] + ) { + beforeEnd -= 1; + afterEnd -= 1; + } + + const oldSlice = beforeLines.slice(start, beforeEnd); + const newSlice = afterLines.slice(start, afterEnd); + if (oldSlice.length * newSlice.length > LCS_CELL_BUDGET) return undefined; + const common = lcsLength(oldSlice, newSlice); + return { + additions: newSlice.length - common, + deletions: oldSlice.length - common, + }; +} + +const LCS_CELL_BUDGET = 4_000_000; + + + +function lcsLength(a: readonly string[], b: readonly string[]): number { + if (a.length === 0 || b.length === 0) return 0; + let previous = new Uint32Array(b.length + 1); + let current = new Uint32Array(b.length + 1); + for (let i = 1; i <= a.length; i += 1) { + for (let j = 1; j <= b.length; j += 1) { + current[j] = + a[i - 1] === b[j - 1] + ? previous[j - 1]! + 1 + : Math.max(previous[j]!, current[j - 1]!); + } + [previous, current] = [current, previous]; + } + return previous[b.length]!; +} diff --git a/packages/agent-core-v2/src/features/fileHistory/flag.ts b/packages/agent-core-v2/src/features/fileHistory/flag.ts new file mode 100644 index 00000000000..a9c18e7a61d --- /dev/null +++ b/packages/agent-core-v2/src/features/fileHistory/flag.ts @@ -0,0 +1,16 @@ +import { type FlagDefinitionInput, registerFlagDefinition } from '#/app/flag/flagRegistry'; + +export const FILE_HISTORY_FLAG_ID = 'file_history'; +export const FILE_HISTORY_FLAG_ENV = 'KIMI_CODE_EXPERIMENTAL_FILE_HISTORY'; + +export const fileHistoryFlag: FlagDefinitionInput = { + id: FILE_HISTORY_FLAG_ID, + title: 'Turn-level file history', + description: + 'Back up the original content of every file the session edits and version all tracked files at each turn boundary, so per-turn file diffs come from real whole-file snapshots instead of tool-argument reconstruction.', + env: FILE_HISTORY_FLAG_ENV, + default: false, + surface: 'core', +}; + +registerFlagDefinition(fileHistoryFlag); diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index d5a646d23cd..12ad075a370 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -341,6 +341,11 @@ export * from '#/features/plan/planOps'; export * from '#/features/plan/planService'; import '#/features/dateChange/dateChangeFeature'; import '#/features/plan/planFeature'; +export * from '#/features/fileHistory/fileHistory'; +export * from '#/features/fileHistory/fileHistoryOps'; +export * from '#/features/fileHistory/fileHistoryService'; +export * from '#/features/fileHistory/flag'; +import '#/features/fileHistory/fileHistoryFeature'; export * from '#/features/externalHooks/configSection'; export * from '#/features/externalHooks/app/externalHooksRunner'; export * from '#/features/externalHooks/app/externalHooksRunnerService'; diff --git a/packages/agent-core-v2/test/index.test.ts b/packages/agent-core-v2/test/index.test.ts index e904dffa1c5..6c71179771b 100644 --- a/packages/agent-core-v2/test/index.test.ts +++ b/packages/agent-core-v2/test/index.test.ts @@ -87,6 +87,8 @@ const V2_RECORD_TYPES: ReadonlySet = new Set([ 'interaction.request', 'interaction.resolved', 'plan.revision', + 'file_history.tracked', + 'file_history.checkpoint', 'interruptionReminder.recorded', 'plugin.session_start', 'runtime.set_binding', diff --git a/packages/agent-core-v2/test/state/builtinReplayableKeys.ts b/packages/agent-core-v2/test/state/builtinReplayableKeys.ts index 68675c29f03..cbdaa8c7458 100644 --- a/packages/agent-core-v2/test/state/builtinReplayableKeys.ts +++ b/packages/agent-core-v2/test/state/builtinReplayableKeys.ts @@ -19,6 +19,7 @@ import { runtimeBindingKey } from '#/agent/runtimeBinding/runtimeBindingOps'; import { taskKey } from '#/agent/task/taskOps'; import { taskNotificationDeliveryKey } from '#/agent/task/taskService'; import { userToolKey } from '#/agent/userTool/userToolOps'; +import { fileHistoryKey } from '#/features/fileHistory/fileHistoryOps'; import { planKey } from '#/features/plan/planOps'; import { swarmKey } from '#/features/swarm/swarmOps'; import { towerKey, towerOwnerKey } from '#/features/tower/towerOps'; @@ -42,6 +43,7 @@ export const BUILTIN_REPLAYABLE_STATE_KEYS: readonly ReplayableStateKey[] = taskKey, taskNotificationDeliveryKey, userToolKey, + fileHistoryKey, planKey, swarmKey, towerKey, diff --git a/packages/kap-server/src/protocol/rest-file-history.ts b/packages/kap-server/src/protocol/rest-file-history.ts new file mode 100644 index 00000000000..9cc03265a27 --- /dev/null +++ b/packages/kap-server/src/protocol/rest-file-history.ts @@ -0,0 +1,39 @@ +import { z } from 'zod'; + +export const fileHistoryChangesQuerySchema = z.object({ + turn_id: z.coerce.number().int().nonnegative(), +}); +export type FileHistoryChangesQuery = z.infer; + +export const fileHistoryContentQuerySchema = z.object({ + turn_id: z.coerce.number().int().nonnegative(), + path: z.string().min(1), + phase: z.enum(['start', 'end']).optional(), +}); +export type FileHistoryContentQuery = z.infer; + +export const fileHistoryChangeSchema = z.object({ + path: z.string(), + status: z.enum(['added', 'modified', 'deleted']), + additions: z.number(), + deletions: z.number(), + binary: z.boolean().optional(), + oversize: z.boolean().optional(), +}); +export type WireFileHistoryChange = z.infer; + +export const fileHistoryChangesResponseSchema = z.object({ + changes: z.array(fileHistoryChangeSchema), +}); +export type FileHistoryChangesResponse = z.infer; + +export const fileHistoryContentEntrySchema = z.object({ + version: z.number(), + content: z.string().optional(), + binary: z.boolean().optional(), +}); + +export const fileHistoryContentResponseSchema = z.object({ + content: fileHistoryContentEntrySchema.nullable(), +}); +export type FileHistoryContentResponse = z.infer; diff --git a/packages/kap-server/src/routes/fileHistory.ts b/packages/kap-server/src/routes/fileHistory.ts new file mode 100644 index 00000000000..8b9245bf0eb --- /dev/null +++ b/packages/kap-server/src/routes/fileHistory.ts @@ -0,0 +1,101 @@ +import { + IAgentFileHistoryService, + resumeSessionById, + type Scope, +} from '@moonshot-ai/agent-core-v2'; +import { z } from 'zod'; + +import { errEnvelope, okEnvelope } from '../envelope'; +import { defineRoute } from '../middleware/defineRoute'; +import { ErrorCode } from '../protocol/error-codes'; +import { + fileHistoryChangesQuerySchema, + fileHistoryChangesResponseSchema, + fileHistoryContentQuerySchema, + fileHistoryContentResponseSchema, +} from '../protocol/rest-file-history'; +import { ensureMainAgent } from '../transport/mainAgent'; + +const sessionIdParamSchema = z.object({ + session_id: z.string().min(1), +}); + +interface FileHistoryRouteHost { + get( + path: string, + options: { preHandler: unknown[]; schema?: Record } | undefined, + handler: ( + req: { id: string; query: unknown; params: unknown }, + reply: { send(payload: unknown): unknown }, + ) => Promise | void, + ): unknown; +} + +export function registerFileHistoryRoutes(app: FileHistoryRouteHost, core: Scope): void { + const changesRoute = defineRoute( + { + method: 'GET', + path: '/sessions/{session_id}/file-history/changes', + params: sessionIdParamSchema, + querystring: fileHistoryChangesQuerySchema, + success: { data: fileHistoryChangesResponseSchema }, + errors: { + [ErrorCode.SESSION_NOT_FOUND]: {}, + }, + description: "List one turn's file changes from the turn-level file history", + tags: ['sessions'], + }, + async (req, reply) => { + const { session_id } = req.params; + const session = await resumeSessionById(core.accessor, session_id); + if (session === undefined) { + reply.send( + errEnvelope(ErrorCode.SESSION_NOT_FOUND, `session ${session_id} does not exist`, req.id), + ); + return; + } + const agent = await ensureMainAgent(session); + const history = agent.accessor.get(IAgentFileHistoryService); + reply.send(okEnvelope({ changes: await history.changes(req.query.turn_id) }, req.id)); + }, + ); + app.get( + changesRoute.path, + changesRoute.options, + changesRoute.handler as Parameters[2], + ); + + const contentRoute = defineRoute( + { + method: 'GET', + path: '/sessions/{session_id}/file-history/content', + params: sessionIdParamSchema, + querystring: fileHistoryContentQuerySchema, + success: { data: fileHistoryContentResponseSchema }, + errors: { + [ErrorCode.SESSION_NOT_FOUND]: {}, + }, + description: "A file's content as captured at a turn's file-history checkpoint", + tags: ['sessions'], + }, + async (req, reply) => { + const { session_id } = req.params; + const session = await resumeSessionById(core.accessor, session_id); + if (session === undefined) { + reply.send( + errEnvelope(ErrorCode.SESSION_NOT_FOUND, `session ${session_id} does not exist`, req.id), + ); + return; + } + const agent = await ensureMainAgent(session); + const history = agent.accessor.get(IAgentFileHistoryService); + const content = await history.contentAt(req.query.turn_id, req.query.path, req.query.phase); + reply.send(okEnvelope({ content: content ?? null }, req.id)); + }, + ); + app.get( + contentRoute.path, + contentRoute.options, + contentRoute.handler as Parameters[2], + ); +} diff --git a/packages/kap-server/src/routes/registerApiV1Routes.ts b/packages/kap-server/src/routes/registerApiV1Routes.ts index c187f748640..ec4e72d03c8 100644 --- a/packages/kap-server/src/routes/registerApiV1Routes.ts +++ b/packages/kap-server/src/routes/registerApiV1Routes.ts @@ -15,6 +15,7 @@ import { registerAuthRoute } from './auth'; import { registerCapabilitiesRoutes } from './capabilities'; import { registerConfigRoutes } from './config'; import { registerConnectionsRoutes } from './connections'; +import { registerFileHistoryRoutes } from './fileHistory'; import { registerFilesRoutes } from './files'; import { registerFsRoutes } from './fs'; import { registerGuiStoreRoutes } from './guiStore'; @@ -168,6 +169,10 @@ export async function registerApiV1Routes( registerFsRoutes(apiV1 as unknown as Parameters[0], core); registerGuiStoreRoutes(apiV1 as unknown as Parameters[0], opts.guiStore); registerToolsRoutes(apiV1 as unknown as Parameters[0], core); + registerFileHistoryRoutes( + apiV1 as unknown as Parameters[0], + core, + ); if (opts.enableTerminals !== false) { registerTerminalsRoutes( apiV1 as unknown as Parameters[0], diff --git a/packages/kap-server/test/__snapshots__/apiSurface.snapshot.test.ts.snap b/packages/kap-server/test/__snapshots__/apiSurface.snapshot.test.ts.snap index a2a5f4b78c6..14a3890db96 100644 --- a/packages/kap-server/test/__snapshots__/apiSurface.snapshot.test.ts.snap +++ b/packages/kap-server/test/__snapshots__/apiSurface.snapshot.test.ts.snap @@ -188,6 +188,14 @@ exports[`API surface snapshot > matches the documented v2 route table and meta e "GET", "/api/v1/sessions/{session_id}/children", ], + [ + "GET", + "/api/v1/sessions/{session_id}/file-history/changes", + ], + [ + "GET", + "/api/v1/sessions/{session_id}/file-history/content", + ], [ "GET", "/api/v1/sessions/{session_id}/fs/{*}",