diff --git a/apps/server/src/modules/canvas/persistence-validation.ts b/apps/server/src/modules/canvas/persistence-validation.ts new file mode 100644 index 000000000..20b3613f4 --- /dev/null +++ b/apps/server/src/modules/canvas/persistence-validation.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Runtime validation shared by structured storage adapters. */ + +function finiteNumber(value: unknown): boolean { + return typeof value === 'number' && Number.isFinite(value); +} + +/** Return the first minimal CanvasFile shape violation, if any. */ +export function canvasFileShapeError( + value: unknown, + expectedCanvasId: string, +): string | null { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return 'must be an object'; + } + + const record = value as Record; + if (record['canvasId'] !== expectedCanvasId) { + return `canvasId must equal ${JSON.stringify(expectedCanvasId)}`; + } + if (record['title'] !== null && typeof record['title'] !== 'string') { + return 'title must be a string or null'; + } + if (!finiteNumber(record['version'])) + return 'version must be a finite number'; + if (!finiteNumber(record['createdAt'])) { + return 'createdAt must be a finite number'; + } + if (!finiteNumber(record['updatedAt'])) { + return 'updatedAt must be a finite number'; + } + + const state = record['state']; + if (typeof state !== 'object' || state === null || Array.isArray(state)) { + return 'state must be an object'; + } + const stateRecord = state as Record; + if (!Array.isArray(stateRecord['nodes'])) + return 'state.nodes must be an array'; + if (!Array.isArray(stateRecord['edges'])) + return 'state.edges must be an array'; + return null; +} diff --git a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts index b688e8fbb..4c65c7acb 100644 --- a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts +++ b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts @@ -53,8 +53,10 @@ describeSpaceNodesContract('Disk', async () => { if (!created.ok) throw new Error('Node contract Space already exists'); const store = new DiskStructuredStore(); + const space = store.space('node-space'); return { - repository: store.space('node-space').nodes, + repository: space.nodes, + space, missingRepository: store.space('missing-node-space').nodes, expectedCanvasId: 'node-space', cleanup: () => { diff --git a/apps/server/src/modules/storage/backends/disk/space-record-validation.ts b/apps/server/src/modules/storage/backends/disk/space-record-validation.ts index 1792458c4..e31bb697c 100644 --- a/apps/server/src/modules/storage/backends/disk/space-record-validation.ts +++ b/apps/server/src/modules/storage/backends/disk/space-record-validation.ts @@ -1,55 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -/** Runtime validation shared by strict Disk Space-record boundaries. */ +/** Runtime validation and strict reads for Disk Space-record boundaries. */ import { readJsonStrict } from '../../../../utils/fs.js'; +import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; import type { CanvasFile } from '../../../canvas/persistence-types.js'; -function finiteNumber(value: unknown): boolean { - return typeof value === 'number' && Number.isFinite(value); -} - -/** Return the first minimal {@link CanvasFile} shape violation, if any. */ -export function canvasFileShapeError( - value: unknown, - expectedCanvasId: string, -): string | null { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return 'must be an object'; - } - - const record = value as Record; - if (record['canvasId'] !== expectedCanvasId) { - return `canvasId must equal ${JSON.stringify(expectedCanvasId)}`; - } - if (record['title'] !== null && typeof record['title'] !== 'string') { - return 'title must be a string or null'; - } - if (!finiteNumber(record['version'])) { - return 'version must be a finite number'; - } - if (!finiteNumber(record['createdAt'])) { - return 'createdAt must be a finite number'; - } - if (!finiteNumber(record['updatedAt'])) { - return 'updatedAt must be a finite number'; - } - - const state = record['state']; - if (typeof state !== 'object' || state === null || Array.isArray(state)) { - return 'state must be an object'; - } - const stateRecord = state as Record; - if (!Array.isArray(stateRecord['nodes'])) { - return 'state.nodes must be an array'; - } - if (!Array.isArray(stateRecord['edges'])) { - return 'state.edges must be an array'; - } - return null; -} +export { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; /** * Strictly read and validate one indexed `space.json` path. diff --git a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts index 72278d483..aa5079588 100644 --- a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts @@ -22,6 +22,7 @@ import { import { DiskStructuredStore } from './structured-store.js'; import { toSafeFilename } from '../../../../utils/naming.js'; import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; +import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; import type { CanvasFile } from '../../../canvas/persistence-types.js'; @@ -90,6 +91,33 @@ describeSpaceLogsContract('Disk Space logs', () => { }; }); +describeSpaceTasksContract('Disk', () => { + const root = freshWorkspace('huabu-task-contract-'); + seedSpace(root, 'canvas-task', 'Canvas Task'); + const store = new DiskStructuredStore(); + const tasks = store.space('canvas-task').tasks; + const concurrent = store.space('canvas-task').tasks; + + return { + tasks, + concurrent, + canvasId: 'canvas-task', + missing: store.space('missing-canvas').tasks, + missingCanvasId: 'missing-canvas', + beginDelete: async () => { + const result = await store.spaces().beginDelete({ + canvasId: 'canvas-task', + }); + if (!result.ok) throw new Error('Ordinary Space must be deletable'); + return result.session; + }, + cleanup: () => { + resetStorageCache(); + rmSync(root, { recursive: true, force: true }); + }, + }; +}); + describe('Disk Space Tasks', () => { let root = ''; let store: DiskStructuredStore; @@ -106,132 +134,8 @@ describe('Disk Space Tasks', () => { rmSync(root, { recursive: true, force: true }); }); - it('serializes Task and Run mutations across independent handles', async () => { - const first = store.space('canvas-task').tasks; - const second = store.space('canvas-task').tasks; - await Promise.all([ - first.create({ - taskId: 'task-a', - canvasId: 'canvas-task', - goal: 'Goal A', - defaultRootProfileId: 'profile-a', - anchorNodeId: 'node-a', - createdAt: 1, - }), - second.create({ - taskId: 'task-b', - canvasId: 'canvas-task', - goal: 'Goal B', - defaultRootProfileId: 'profile-b', - anchorNodeId: 'node-b', - createdAt: 2, - }), - ]); - await first.runs.create({ - runId: 'run-a', - taskId: 'task-a', - canvasIdSnapshot: 'canvas-task', - goalSnapshot: 'Goal A', - rootProfileIdSnapshot: 'profile-a', - status: 'pending', - createdAt: 3, - }); - const updated = await second.runs.update('run-a', { - rootNodeId: 'node-root', - rootThreadId: 'thread-root', - status: 'running', - startedAt: 4, - }); - - expect(updated.status).toBe('running'); - await expect(first.read()).resolves.toMatchObject({ - version: 1, - tasks: [ - expect.objectContaining({ taskId: 'task-a' }), - expect.objectContaining({ taskId: 'task-b' }), - ], - runs: [ - expect.objectContaining({ - runId: 'run-a', - rootNodeId: 'node-root', - rootThreadId: 'thread-root', - }), - ], - }); - }); - - it('returns an empty versioned snapshot when no Task store exists', async () => { - await expect(store.space('canvas-empty').tasks.read()).resolves.toEqual({ - version: 1, - tasks: [], - runs: [], - }); - }); - - it('completes a running Run atomically and keeps its message immutable', async () => { - const runs = store.space('canvas-task').tasks.runs; - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 5, - message: 'PR merged', - }), - ).resolves.toMatchObject({ - outcome: 'completed', - run: { - status: 'completed', - completion: { completedAt: 5, message: 'PR merged' }, - }, - }); - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 6, - message: 'PR merged', - }), - ).resolves.toMatchObject({ - outcome: 'unchanged', - run: { completion: { completedAt: 5, message: 'PR merged' } }, - }); - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 7, - message: 'Different result', - }), - ).resolves.toMatchObject({ outcome: 'completion_conflict' }); - - await runs.create({ - runId: 'run-pending', - taskId: 'task-b', - canvasIdSnapshot: 'canvas-task', - goalSnapshot: 'Goal B', - rootProfileIdSnapshot: 'profile-b', - status: 'pending', - createdAt: 8, - }); - await expect( - runs.complete('task-b', 'run-pending', { completedAt: 9 }), - ).resolves.toMatchObject({ outcome: 'run_not_running' }); - await expect( - runs.complete('missing-task', 'run-a', { completedAt: 9 }), - ).resolves.toEqual({ outcome: 'task_not_found' }); - await expect( - runs.complete('task-a', 'missing-run', { completedAt: 9 }), - ).resolves.toEqual({ outcome: 'run_not_found' }); - }); - - it('rejects mutations for a missing Space', async () => { - await expect( - store.space('missing-canvas').tasks.create({ - taskId: 'task-missing', - canvasId: 'missing-canvas', - goal: 'Missing', - defaultRootProfileId: 'profile-a', - anchorNodeId: 'node-missing', - createdAt: 1, - }), - ).rejects.toThrow(/cannot write a missing Space/); - }); - it('fails fast on malformed and internally inconsistent Task stores', async () => { + mkdirSync(path.dirname(tasksPath('canvas-task')), { recursive: true }); writeFileSync(tasksPath('canvas-task'), '{"version":1,"tasks":{}}'); await expect(store.space('canvas-task').tasks.read()).rejects.toThrow( /Invalid Task store/, diff --git a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts new file mode 100644 index 000000000..b30803d4e --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SqliteStructuredStore } from './structured-store.js'; +import { + createSqliteTestFile, + installDeltaAbortTrigger, + openSqliteTestStore, + readSqliteDeltaLog, +} from './test-support.js'; +import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; +import { describeSpaceNodesContract } from '../../ports/contracts/space-nodes.contract.js'; +import { describeSpaceRepositoryContract } from '../../ports/contracts/space-repository.contract.js'; +import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; +import { describeSpaceWriteContract } from '../../ports/contracts/space-write.contract.js'; +import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; + +import type { NodeContent } from '../../../canvas/persistence-types.js'; + +function note(nodeId: string, label: string, content: string): NodeContent { + return { nodeId, type: 'note', label, content }; +} + +async function createOrdinarySpace( + store: SqliteStructuredStore, + canvasId: string, + title: string, +): Promise { + const created = await store.spaces().create({ canvasId, title }); + if (!created.ok) throw new Error(`Could not create test Space ${canvasId}`); +} + +describeStructuredStoreContract('SQLite', () => { + const file = createSqliteTestFile('huabu-sqlite-structured-contract-'); + return { + store: new SqliteStructuredStore(file.filename), + cleanup: file.remove, + }; +}); + +describeSpaceRepositoryContract('SQLite', async () => { + const harness = await openSqliteTestStore( + 'huabu-sqlite-space-repository-contract-', + ); + return { + repository: harness.store.spaces(), + read: (canvasId: string) => harness.store.space(canvasId).read(), + worldCanvasId: harness.world.canvasId, + attemptMutation: (canvasId: string) => + harness.store.space(canvasId).nodes.put({ + nodeId: 'contract-delete-fence-node', + record: note( + 'contract-delete-fence-node', + 'Deletion fence node', + 'body', + ), + }), + cleanup: harness.cleanup, + }; +}); + +describeSpaceNodesContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-nodes-contract-'); + const canvasId = 'sqlite-nodes-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Nodes Contract'); + const space = harness.store.space(canvasId); + return { + repository: space.nodes, + space, + missingRepository: harness.store.space('sqlite-nodes-missing').nodes, + expectedCanvasId: canvasId, + cleanup: harness.cleanup, + }; +}); + +describeSpaceWriteContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-write-contract-'); + const canvasId = 'sqlite-write-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Write Contract'); + const existingNode = note( + 'contract-existing-node', + 'Existing contract node', + 'before', + ); + const space = harness.store.space(canvasId); + const put = await space.nodes.put({ + nodeId: existingNode.nodeId, + record: existingNode, + }); + if (!put.ok) { + throw new Error(`Could not seed SQLite write contract: ${put.reason}`); + } + + return { + space, + concurrent: harness.store.space(canvasId), + missing: harness.store.space('sqlite-write-missing'), + existingNode, + newNode: note('contract-new-node', 'New contract node', 'after'), + readJournal: async () => readSqliteDeltaLog(harness.filename, canvasId), + failNextDeltaAppend: (error: Error) => + installDeltaAbortTrigger(harness.filename, error.message), + cleanup: harness.cleanup, + }; +}); + +describeSpaceLogsContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-logs-contract-'); + const canvasId = 'sqlite-logs-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Logs Contract'); + const first = harness.store.space(canvasId); + const second = harness.store.space(canvasId); + return { + events: first.events, + changes: first.changes, + concurrent: { + events: second.events, + changes: second.changes, + }, + cleanup: harness.cleanup, + }; +}); + +describeSpaceTasksContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-tasks-contract-'); + const canvasId = 'sqlite-tasks-contract'; + const missingCanvasId = 'sqlite-tasks-missing'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Tasks Contract'); + return { + tasks: harness.store.space(canvasId).tasks, + concurrent: harness.store.space(canvasId).tasks, + canvasId, + missing: harness.store.space(missingCanvasId).tasks, + missingCanvasId, + beginDelete: async () => { + const result = await harness.store.spaces().beginDelete({ canvasId }); + if (!result.ok) throw new Error('Ordinary Space must be deletable'); + return result.session; + }, + cleanup: harness.cleanup, + }; +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/database.ts b/apps/server/src/modules/storage/backends/sqlite/database.ts new file mode 100644 index 000000000..d4459bbe4 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/database.ts @@ -0,0 +1,381 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { DatabaseSync } from 'node:sqlite'; + +import type { StorageHealth } from '../../ports/common.js'; + +export const SQLITE_SCHEMA_VERSION = 1; +export const SQLITE_WORLD_COLLISION_KEY = '.world'; + +const SCHEMA_V1 = ` + CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + title TEXT, + collision_key TEXT NOT NULL UNIQUE, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)) + ) STRICT; + + CREATE UNIQUE INDEX spaces_single_world + ON spaces(is_world) + WHERE is_world = 1; + + CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision INTEGER NOT NULL CHECK (revision > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + + CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; +`; + +export interface SqliteMigration { + readonly version: number; + readonly sql: string; +} + +export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = Object.freeze([ + Object.freeze({ version: 1, sql: SCHEMA_V1 }), +]); + +function readUserVersion(database: DatabaseSync): number { + const row = database.prepare('PRAGMA user_version').get(); + const version = row?.['user_version']; + if (typeof version !== 'number' || !Number.isSafeInteger(version)) { + throw new Error('SQLite returned an invalid PRAGMA user_version'); + } + return version; +} + +export function applySqliteMigrations( + database: DatabaseSync, + migrations: readonly SqliteMigration[] = SQLITE_MIGRATIONS, +): void { + for (let index = 0; index < migrations.length; index += 1) { + const expectedVersion = index + 1; + if (migrations[index]?.version !== expectedVersion) { + throw new Error( + `SQLite migrations must be contiguous from version 1; expected ${expectedVersion}`, + ); + } + } + const targetVersion = migrations.at(-1)?.version ?? 0; + const current = readUserVersion(database); + if (current > targetVersion) { + throw new Error( + `SQLite schema version ${current} is newer than supported version ${targetVersion}`, + ); + } + if (current === targetVersion) return; + + database.exec('BEGIN IMMEDIATE'); + try { + let version = readUserVersion(database); + for (const migration of migrations) { + if (migration.version <= version) continue; + if (migration.version !== version + 1) { + throw new Error( + `No SQLite migration path from schema version ${version} to ${targetVersion}`, + ); + } + database.exec(migration.sql); + database.exec(`PRAGMA user_version = ${migration.version}`); + version = migration.version; + } + if (version !== targetVersion) { + throw new Error( + `No SQLite migration path from schema version ${version} to ${targetVersion}`, + ); + } + database.exec('COMMIT'); + } catch (error) { + if (database.isTransaction) database.exec('ROLLBACK'); + throw error; + } +} + +function reserveWorldCollisionKey(database: DatabaseSync): void { + withImmediateTransaction(database, () => { + const world = database + .prepare('SELECT canvas_id, collision_key FROM spaces WHERE is_world = 1') + .get(); + if (world === undefined) return; + + const canvasId = world['canvas_id']; + const collisionKey = world['collision_key']; + if (typeof canvasId !== 'string' || typeof collisionKey !== 'string') { + throw new SyntaxError('SQLite World Space has malformed identity fields'); + } + if (collisionKey === SQLITE_WORLD_COLLISION_KEY) return; + + const conflict = database + .prepare( + `SELECT canvas_id + FROM spaces + WHERE collision_key = ? AND canvas_id <> ?`, + ) + .get(SQLITE_WORLD_COLLISION_KEY, canvasId); + if (conflict !== undefined) { + throw new Error( + `Cannot reserve SQLite World collision slot ${JSON.stringify( + SQLITE_WORLD_COLLISION_KEY, + )}: it is already occupied`, + ); + } + + const result = database + .prepare( + `UPDATE spaces + SET collision_key = ? + WHERE canvas_id = ? AND is_world = 1`, + ) + .run(SQLITE_WORLD_COLLISION_KEY, canvasId); + if (Number(result.changes) !== 1) { + throw new Error('Could not reserve the SQLite World collision slot'); + } + }); +} + +type DeleteAdmission = { + readonly resolve: (release: () => void) => void; + readonly reject: (error: Error) => void; +}; + +class SpaceDeleteGate { + #active = false; + #closed = false; + readonly #waiting: DeleteAdmission[] = []; + + get pending(): boolean { + return this.#active || this.#waiting.length > 0; + } + + get idle(): boolean { + return !this.#active && this.#waiting.length === 0; + } + + acquire(): Promise<() => void> { + if (this.#closed) { + return Promise.reject(new Error('SQLite store is closed')); + } + if (!this.#active) { + this.#active = true; + return Promise.resolve(this.#releaseFunction()); + } + return new Promise((resolve, reject) => { + this.#waiting.push({ resolve, reject }); + }); + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + const error = new Error('SQLite store is closed'); + for (const admission of this.#waiting.splice(0)) { + admission.reject(error); + } + } + + #releaseFunction(): () => void { + let released = false; + return () => { + if (released) return; + released = true; + this.#active = false; + if (this.#closed) return; + const next = this.#waiting.shift(); + if (!next) return; + this.#active = true; + next.resolve(this.#releaseFunction()); + }; + } +} + +/** One connection and all adapter-lifetime process-local state. */ +export class SqliteStoreContext { + readonly now: () => number; + + readonly #database: DatabaseSync; + readonly #deleteGates = new Map(); + readonly #nodeTombstones = new Set(); + #state: 'new' | 'open' | 'closed' = 'new'; + + constructor(filename: string, now: () => number) { + this.now = now; + this.#database = new DatabaseSync(filename, { open: false }); + } + + init(): void { + if (this.#state === 'open') return; + if (this.#state === 'closed') { + throw new Error('SQLite store is closed'); + } + + try { + this.#database.open(); + this.#database.exec('PRAGMA foreign_keys = ON'); + const foreignKeys = this.#database.prepare('PRAGMA foreign_keys').get()?.[ + 'foreign_keys' + ]; + if (foreignKeys !== 1) { + throw new Error('Could not enable SQLite foreign key enforcement'); + } + applySqliteMigrations(this.#database); + reserveWorldCollisionKey(this.#database); + this.#state = 'open'; + } catch (error) { + if (this.#database.isOpen) this.#database.close(); + this.#state = 'closed'; + throw error; + } + } + + health(kind: string): StorageHealth { + this.assertOpen(); + try { + const value = this.#database.prepare('SELECT 1 AS ok').get()?.['ok']; + return value === 1 + ? { ok: true, kind } + : { ok: false, kind, detail: 'SQLite liveness query returned no row' }; + } catch (error) { + return { + ok: false, + kind, + detail: error instanceof Error ? error.message : String(error), + }; + } + } + + close(): void { + if (this.#state === 'closed') return; + this.#state = 'closed'; + for (const gate of this.#deleteGates.values()) gate.close(); + this.#deleteGates.clear(); + if (this.#database.isOpen) this.#database.close(); + } + + database(): DatabaseSync { + this.assertOpen(); + return this.#database; + } + + assertOpen(): void { + if (this.#state !== 'open') { + throw new Error( + this.#state === 'closed' + ? 'SQLite store is closed' + : 'SQLite store is not initialized', + ); + } + } + + assertMutationAllowed(canvasId: string): void { + this.assertOpen(); + if (this.#deleteGates.get(canvasId)?.pending) { + throw new Error( + `Cannot mutate Space "${canvasId}" while deletion is pending`, + ); + } + } + + async acquireDelete(canvasId: string): Promise<() => void> { + this.assertOpen(); + let gate = this.#deleteGates.get(canvasId); + if (!gate) { + gate = new SpaceDeleteGate(); + this.#deleteGates.set(canvasId, gate); + } + const releaseGate = await gate.acquire(); + try { + this.assertOpen(); + } catch (error) { + releaseGate(); + throw error; + } + + let released = false; + return () => { + if (released) return; + released = true; + releaseGate(); + if (gate?.idle && this.#deleteGates.get(canvasId) === gate) { + this.#deleteGates.delete(canvasId); + } + }; + } + + isNodeTombstoned(canvasId: string, nodeId: string): boolean { + return this.#nodeTombstones.has(this.#nodeKey(canvasId, nodeId)); + } + + setNodeTombstone(canvasId: string, nodeId: string, present: boolean): void { + const key = this.#nodeKey(canvasId, nodeId); + if (present) this.#nodeTombstones.add(key); + else this.#nodeTombstones.delete(key); + } + + clearCanvasTombstones(canvasId: string): void { + const prefix = `${canvasId}\0`; + for (const key of this.#nodeTombstones) { + if (key.startsWith(prefix)) this.#nodeTombstones.delete(key); + } + } + + #nodeKey(canvasId: string, nodeId: string): string { + return `${canvasId}\0${nodeId}`; + } +} + +export function withImmediateTransaction( + database: DatabaseSync, + operation: () => T, +): T { + database.exec('BEGIN IMMEDIATE'); + try { + const result = operation(); + database.exec('COMMIT'); + return result; + } catch (error) { + if (database.isTransaction) database.exec('ROLLBACK'); + throw error; + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql new file mode 100644 index 000000000..c50d52a0e --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql @@ -0,0 +1,109 @@ +-- Immutable SQLite structured-store schema v1 fixture. +-- Add a new fixture for later schema versions; do not rewrite this history. + +PRAGMA foreign_keys = ON; +BEGIN IMMEDIATE; + +CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + title TEXT, + collision_key TEXT NOT NULL UNIQUE, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)) +) STRICT; + +CREATE UNIQUE INDEX spaces_single_world + ON spaces(is_world) + WHERE is_world = 1; + +CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision INTEGER NOT NULL CHECK (revision > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + +CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +INSERT INTO spaces ( + canvas_id, title, collision_key, version, state_json, + created_at, updated_at, is_world +) VALUES ( + 'fixture-world', 'World', 'world', 0, + '{"nodes":[],"edges":[]}', 1, 1, 1 +); + +INSERT INTO spaces ( + canvas_id, title, collision_key, version, state_json, + created_at, updated_at, is_world +) VALUES ( + 'fixture-space', 'Fixture Space', 'fixture space', 3, + '{"nodes":[{"id":"fixture-node","type":"note"}],"edges":[]}', + 10, 13, 0 +); + +INSERT INTO nodes ( + canvas_id, node_id, record_json, revision, label_collision_key +) VALUES ( + 'fixture-space', 'fixture-node', + '{"nodeId":"fixture-node","type":"note","label":"Fixture Node","content":"fixture body"}', + 7, 'fixture node' +); + +INSERT INTO events (canvas_id, event_json) VALUES ( + 'fixture-space', + '{"payload":{"action":"node_selected","node":{"id":"fixture-node","type":"note","label":"Fixture Node"}},"ts":12}' +); + +INSERT INTO changes (canvas_id, thread_id, snapshot_json) VALUES ( + 'fixture-space', 'fixture-thread', '[]' +); + +INSERT INTO tasks (canvas_id, snapshot_json) VALUES ( + 'fixture-space', '{"version":1,"tasks":[],"runs":[]}' +); + +INSERT INTO delta_log (canvas_id, version, entry_json) VALUES ( + 'fixture-space', 3, + '{"version":3,"ts":13,"commands":[],"deltas":[],"originator":{"source":"system"}}' +); + +PRAGMA user_version = 1; +COMMIT; diff --git a/apps/server/src/modules/storage/backends/sqlite/identity.ts b/apps/server/src/modules/storage/backends/sqlite/identity.ts new file mode 100644 index 000000000..3a84a7497 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/identity.ts @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Allocation of the names a Space or Node is filed under. + * + * The `collision_key` columns carry a UNIQUE constraint, so a title or label + * has to be de-duplicated before it reaches the database rather than after a + * failed insert. These rules are pure and share `utils/naming` with Disk, so + * both backends hand out the same ` (2)` suffixes for the same inputs — see + * `backends/disk/space-title.ts` for the directory-locator half. + */ + +import { + dedupeName, + normalizeForCompare, + toSafeFilename, +} from '../../../../utils/naming.js'; + +import type { NodeContent } from '../../../canvas/persistence-types.js'; + +function allocatedSpaceTitle( + requested: string | null, + canvasId: string, + allocatedName: string, +): string | null { + if (requested === null) return null; + const base = toSafeFilename(requested, canvasId); + if (allocatedName === base) return requested; + const candidate = `${requested}${allocatedName.slice(base.length)}`; + return toSafeFilename(candidate, canvasId) === allocatedName + ? candidate + : allocatedName; +} + +export function allocateSpaceIdentity( + requestedTitle: string | null, + canvasId: string, + occupiedCollisionKeys: Iterable, +): { readonly title: string | null; readonly collisionKey: string } { + const base = toSafeFilename(requestedTitle, canvasId); + const allocated = dedupeName(base, occupiedCollisionKeys); + return { + title: allocatedSpaceTitle(requestedTitle, canvasId, allocated), + collisionKey: normalizeForCompare(allocated), + }; +} + +export function collisionKeyForTitle( + title: string | null, + canvasId: string, +): string { + return normalizeForCompare(toSafeFilename(title, canvasId)); +} + +export function allocateNodeIdentity( + record: NodeContent, + nodeId: string, + existingCollisionKey: string | null, + occupiedCollisionKeys: Iterable, +): { + readonly record: NodeContent; + readonly collisionKey: string; + readonly desiredCollisionKey: string; +} { + const trimmedLabel = + typeof record.label === 'string' && record.label.trim().length > 0 + ? record.label + : null; + if (trimmedLabel === null && existingCollisionKey !== null) { + return { + record, + collisionKey: existingCollisionKey, + desiredCollisionKey: existingCollisionKey, + }; + } + + const desired = toSafeFilename(trimmedLabel, nodeId); + const allocated = dedupeName(desired, occupiedCollisionKeys); + const suffix = + allocated.length > desired.length && allocated.startsWith(desired) + ? allocated.slice(desired.length) + : ''; + return { + record: + suffix && trimmedLabel + ? { ...record, label: `${trimmedLabel}${suffix}` } + : record, + collisionKey: normalizeForCompare(allocated), + desiredCollisionKey: normalizeForCompare(desired), + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts new file mode 100644 index 000000000..43311516c --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts @@ -0,0 +1,578 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { readFileSync } from 'node:fs'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { extractCanvasChanges } from '@huabu/shared/canvas-engine'; + +import { applySqliteMigrations, SQLITE_SCHEMA_VERSION } from './database.js'; +import { SqliteStructuredStore } from './structured-store.js'; +import { + createSqliteTestFile, + installDeltaAbortTrigger, + openSqliteTestStore, + readSqliteDeltaLog, + withTestDatabase, +} from './test-support.js'; + +import type { + CanvasFile, + DeltaLogEntry, + NodeContent, +} from '../../../canvas/persistence-types.js'; +import type { TaskRecord } from '@huabu/shared'; +import type { CanvasNode } from '@huabu/shared/canvas-engine'; + +const cleanups: Array<() => Promise | void> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) { + await cleanup(); + } +}); + +function trackedFile(prefix: string) { + const file = createSqliteTestFile(prefix); + cleanups.push(file.remove); + return file; +} + +function trackedStore(filename: string): SqliteStructuredStore { + const store = new SqliteStructuredStore(filename); + cleanups.push(() => store.close()); + return store; +} + +async function trackedOpenStore(prefix: string) { + const harness = await openSqliteTestStore(prefix); + cleanups.push(harness.cleanup); + return harness; +} + +function note(nodeId: string, label: string, content: string): NodeContent { + return { nodeId, type: 'note', label, content }; +} + +function nextRecord(current: CanvasFile): CanvasFile { + return { + ...current, + version: current.version + 1, + updatedAt: current.updatedAt + 1, + }; +} + +function delta(version: number, marker: string): DeltaLogEntry { + return { + version, + ts: version + 100, + commands: [{ marker }], + deltas: [{ marker }], + originator: { source: 'system' }, + }; +} + +async function createSpace( + store: SqliteStructuredStore, + canvasId: string, + title: string, +): Promise { + const result = await store.spaces().create({ canvasId, title }); + if (!result.ok) throw new Error(`Could not create test Space ${canvasId}`); + return result.record; +} + +describe('SqliteStructuredStore lifecycle and schema', () => { + it('rejects an empty database filename', () => { + expect(() => new SqliteStructuredStore('')).toThrow(/filename.*empty/i); + }); + + it('rejects before init and after close while lifecycle operations stay idempotent', async () => { + const file = trackedFile('huabu-sqlite-lifecycle-'); + const store = trackedStore(file.filename); + + await expect(store.health()).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => store.space('lifecycle-space').read()), + ).rejects.toThrow(/not initialized/); + + await expect(store.init()).resolves.toBeUndefined(); + await expect(store.init()).resolves.toBeUndefined(); + await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + + await expect(store.close()).resolves.toBeUndefined(); + await expect(store.close()).resolves.toBeUndefined(); + await expect(store.health()).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => store.space('lifecycle-space').read()), + ).rejects.toThrow(/closed/); + await expect(store.init()).rejects.toThrow(/closed/); + }); + + it('creates the complete STRICT v1 schema in a fresh database', async () => { + const file = trackedFile('huabu-sqlite-fresh-schema-'); + const store = trackedStore(file.filename); + await store.init(); + + withTestDatabase(file.filename, (database) => { + expect(database.prepare('PRAGMA user_version').get()).toEqual({ + user_version: SQLITE_SCHEMA_VERSION, + }); + const expectedTables = [ + 'changes', + 'delta_log', + 'events', + 'nodes', + 'spaces', + 'tasks', + ]; + const tableRows = database.prepare('PRAGMA table_list').all(); + const productionTables = tableRows.filter((row) => + expectedTables.includes(String(row['name'])), + ); + expect(productionTables.map((row) => row['name']).sort()).toEqual( + expectedTables, + ); + expect(productionTables.every((row) => row['strict'] === 1)).toBe(true); + expect( + database + .prepare('PRAGMA foreign_key_list(nodes)') + .all() + .map((row) => ({ + table: row['table'], + from: row['from'], + to: row['to'], + onDelete: row['on_delete'], + })), + ).toContainEqual({ + table: 'spaces', + from: 'canvas_id', + to: 'canvas_id', + onDelete: 'CASCADE', + }); + }); + }); + + it('opens the immutable v1 SQL fixture without rewriting its records', async () => { + const file = trackedFile('huabu-sqlite-v1-fixture-'); + const fixtureSql = readFileSync( + new URL('./fixtures/v1.sql', import.meta.url), + 'utf8', + ); + withTestDatabase(file.filename, (database) => database.exec(fixtureSql)); + + const store = trackedStore(file.filename); + await store.init(); + await expect(store.spaces().worldId()).resolves.toBe('fixture-world'); + await expect(store.spaces().list()).resolves.toEqual([ + { + canvasId: 'fixture-space', + title: 'Fixture Space', + nodeCount: 1, + createdAt: 10, + updatedAt: 13, + }, + ]); + const space = store.space('fixture-space'); + await expect(space.read()).resolves.toEqual({ + canvasId: 'fixture-space', + title: 'Fixture Space', + version: 3, + state: { + nodes: [{ id: 'fixture-node', type: 'note' }], + edges: [], + }, + createdAt: 10, + updatedAt: 13, + }); + await expect(space.nodes.read('fixture-node')).resolves.toEqual({ + record: note('fixture-node', 'Fixture Node', 'fixture body'), + revision: '7', + }); + await expect(space.events.read()).resolves.toEqual([ + { + payload: { + action: 'node_selected', + node: { id: 'fixture-node', type: 'note', label: 'Fixture Node' }, + }, + ts: 12, + }, + ]); + await expect(space.changes.read('fixture-thread')).resolves.toEqual([]); + await expect(space.tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + expect(readSqliteDeltaLog(file.filename, 'fixture-space')).toEqual([ + { + version: 3, + ts: 13, + commands: [], + deltas: [], + originator: { source: 'system' }, + }, + ]); + }); + + it('rejects a database whose user_version is from the future', async () => { + const file = trackedFile('huabu-sqlite-future-schema-'); + withTestDatabase(file.filename, (database) => { + database.exec(`PRAGMA user_version = ${SQLITE_SCHEMA_VERSION + 1}`); + }); + const store = trackedStore(file.filename); + + await expect(store.init()).rejects.toThrow(/newer than supported/); + await expect(store.health()).rejects.toThrow(/closed/); + }); + + it('rolls every migration step and user_version back when a later step fails', () => { + const file = trackedFile('huabu-sqlite-migration-rollback-'); + withTestDatabase(file.filename, (database) => { + expect(() => + applySqliteMigrations(database, [ + { + version: 1, + sql: 'CREATE TABLE migration_v1 (id INTEGER PRIMARY KEY) STRICT;', + }, + { + version: 2, + sql: ` + CREATE TABLE migration_v2 (id INTEGER PRIMARY KEY) STRICT; + INSERT INTO missing_migration_table (id) VALUES (1); + `, + }, + ]), + ).toThrow(/missing_migration_table|no such table/); + + expect(database.prepare('PRAGMA user_version').get()).toEqual({ + user_version: 0, + }); + expect( + database + .prepare( + `SELECT name + FROM sqlite_schema + WHERE type = 'table' AND name LIKE 'migration_%'`, + ) + .all(), + ).toEqual([]); + }); + }); +}); + +describe('SqliteStructuredStore persistence and transactions', () => { + it('persists Space and Node records across close and reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-reopen-'); + const canvasId = 'reopen-space'; + const created = await createSpace(harness.store, canvasId, 'Reopen Space'); + const record = note('reopen-node', 'Reopen Node', 'persisted body'); + const put = await harness.store.space(canvasId).nodes.put({ + nodeId: record.nodeId, + record, + }); + expect(put).toMatchObject({ ok: true, record }); + + await harness.store.close(); + const reopened = trackedStore(harness.filename); + await reopened.init(); + + await expect(reopened.spaces().worldId()).resolves.toBe( + harness.world.canvasId, + ); + await expect(reopened.space(canvasId).read()).resolves.toEqual(created); + await expect( + reopened.space(canvasId).nodes.read(record.nodeId), + ).resolves.toEqual(put.ok ? { record, revision: put.revision } : null); + }); + + it('rolls node, record, delta, and tombstone state back on a real trigger abort', async () => { + const harness = await trackedOpenStore('huabu-sqlite-trigger-rollback-'); + const canvasId = 'trigger-rollback-space'; + const baseline = await createSpace( + harness.store, + canvasId, + 'Trigger Rollback Space', + ); + const oldNode = note('old-node', 'Old Node', 'before'); + const newNode = note('new-node', 'New Node', 'after'); + const oldPut = await harness.store.space(canvasId).nodes.put({ + nodeId: oldNode.nodeId, + record: oldNode, + }); + if (!oldPut.ok) throw new Error('Could not seed rollback node'); + + const next: CanvasFile = { + ...nextRecord(baseline), + state: { + nodes: [{ id: newNode.nodeId, type: newNode.type }], + edges: [], + }, + }; + const restore = installDeltaAbortTrigger( + harness.filename, + 'forced delta abort', + ); + try { + await expect( + harness.store.space(canvasId).write({ + expectedVersion: baseline.version, + nextRecord: next, + nodeMutations: [ + { kind: 'delete', nodeId: oldNode.nodeId }, + { + kind: 'put', + nodeId: newNode.nodeId, + record: newNode, + authoritativeInsert: true, + }, + ], + delta: delta(next.version, 'trigger-abort'), + }), + ).rejects.toThrow('forced delta abort'); + } finally { + restore(); + } + + const space = harness.store.space(canvasId); + await expect(space.read()).resolves.toEqual(baseline); + await expect(space.nodes.read(oldNode.nodeId)).resolves.toEqual({ + record: oldPut.record, + revision: oldPut.revision, + }); + await expect(space.nodes.read(newNode.nodeId)).resolves.toBeNull(); + expect(readSqliteDeltaLog(harness.filename, canvasId)).toEqual([]); + await expect( + space.nodes.put({ + nodeId: oldNode.nodeId, + expectedRevision: oldPut.revision, + record: { ...oldNode, content: 'still writable' }, + }), + ).resolves.toMatchObject({ ok: true }); + }); + + it('rejects sparse JSON arrays without changing the exact persisted Node', async () => { + const harness = await trackedOpenStore('huabu-sqlite-sparse-json-'); + const canvasId = 'sparse-json-space'; + await createSpace(harness.store, canvasId, 'Sparse JSON Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('sparse-json-node', 'Sparse JSON Node', 'before'); + const baseline = await nodes.put({ nodeId: record.nodeId, record }); + if (!baseline.ok) throw new Error('Could not seed sparse JSON node'); + const sparse: unknown[] = []; + sparse[1] = 'present'; + expect(0 in sparse).toBe(false); + + await expect( + nodes.put({ + nodeId: record.nodeId, + expectedRevision: baseline.revision, + record: { ...record, metadata: sparse }, + }), + ).rejects.toThrow(/sparse array/i); + await expect(nodes.read(record.nodeId)).resolves.toEqual({ + record, + revision: baseline.revision, + }); + }); + + it('releases deletion admission when post-acquire Space setup throws', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-setup-'); + const canvasId = 'delete-setup-space'; + const record = await createSpace( + harness.store, + canvasId, + 'Delete Setup Space', + ); + const repository = harness.store.spaces(); + + const malformedAttempt = repository.beginDelete({ canvasId }); + withTestDatabase(harness.filename, (database) => { + database + .prepare('UPDATE spaces SET state_json = ? WHERE canvas_id = ?') + .run('[]', canvasId); + }); + await expect(malformedAttempt).rejects.toThrow(/Invalid Space/); + withTestDatabase(harness.filename, (database) => { + database + .prepare('UPDATE spaces SET state_json = ? WHERE canvas_id = ?') + .run(JSON.stringify(record.state), canvasId); + }); + + let secondResult: + | Awaited> + | undefined; + let secondError: unknown; + const secondSettled = repository.beginDelete({ canvasId }).then( + (result) => { + secondResult = result; + }, + (error: unknown) => { + secondError = error; + }, + ); + await new Promise((resolve) => setImmediate(resolve)); + + expect(secondError).toBeUndefined(); + expect(secondResult).toMatchObject({ ok: true }); + if (!secondResult?.ok) { + throw new Error('Deletion gate remained occupied after setup failure'); + } + await secondResult.session.abort(); + await secondSettled; + }); + + it('cascades every child record when a deletion session finishes', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-session-'); + const canvasId = 'delete-session-space'; + const baseline = await createSpace( + harness.store, + canvasId, + 'Delete Session Space', + ); + const record = note('deleted-node', 'Deleted Node', 'stale body'); + const handle = harness.store.space(canvasId); + await handle.nodes.put({ nodeId: record.nodeId, record }); + await handle.events.append([ + { + payload: { + action: 'node_selected', + node: { + id: record.nodeId, + type: 'note', + label: record.label ?? undefined, + }, + }, + ts: 2, + }, + ]); + const changeNode: CanvasNode = { + id: 'change-node', + type: 'note', + position: { x: 0, y: 0 }, + data: { label: 'Change Node', content: 'change body' }, + } as CanvasNode; + await handle.changes.append( + 'delete-thread', + extractCanvasChanges([{ type: 'INSERT_NODE', node: changeNode }]), + ); + const task: TaskRecord = { + taskId: 'delete-task', + canvasId, + goal: 'Delete this fixture', + defaultRootProfileId: 'profile-delete', + anchorNodeId: record.nodeId, + createdAt: 3, + }; + await handle.tasks.create(task); + const next = nextRecord(baseline); + await expect( + handle.write({ + expectedVersion: baseline.version, + nextRecord: next, + nodeMutations: [], + delta: delta(next.version, 'delete-session'), + }), + ).resolves.toEqual({ ok: true }); + + withTestDatabase(harness.filename, (database) => { + for (const table of [ + 'nodes', + 'events', + 'changes', + 'tasks', + 'delta_log', + ]) { + expect( + database + .prepare( + `SELECT count(*) AS count FROM ${table} WHERE canvas_id = ?`, + ) + .get(canvasId)?.['count'], + ).toBe(1); + } + }); + + const started = await harness.store.spaces().beginDelete({ canvasId }); + if (!started.ok) throw new Error('Ordinary Space must be deletable'); + await expect(handle.read()).resolves.toEqual(next); + await expect(handle.nodes.read(record.nodeId)).resolves.toMatchObject({ + record, + }); + await expect(started.session.finish()).resolves.toEqual({ + ok: true, + reason: 'deleted', + }); + + withTestDatabase(harness.filename, (database) => { + for (const table of [ + 'nodes', + 'events', + 'changes', + 'tasks', + 'delta_log', + ]) { + expect( + database + .prepare( + `SELECT count(*) AS count FROM ${table} WHERE canvas_id = ?`, + ) + .get(canvasId)?.['count'], + ).toBe(0); + } + }); + + await expect(handle.read()).resolves.toBeNull(); + }); + + it('does not create a tombstone when deleting an already absent node', async () => { + const harness = await trackedOpenStore('huabu-sqlite-absent-delete-'); + const canvasId = 'absent-delete-space'; + await createSpace(harness.store, canvasId, 'Absent Delete Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('not-yet-created', 'Not Yet Created', 'body'); + + await expect(nodes.delete(record.nodeId)).resolves.toBe('absent'); + await expect( + nodes.put({ nodeId: record.nodeId, record }), + ).resolves.toMatchObject({ ok: true, record }); + }); + + it('forgets a successful node deletion tombstone after close and reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-tombstone-reopen-'); + const canvasId = 'tombstone-reopen-space'; + await createSpace(harness.store, canvasId, 'Tombstone Reopen Space'); + const record = note('tombstoned-node', 'Tombstoned Node', 'before'); + const nodes = harness.store.space(canvasId).nodes; + await nodes.put({ nodeId: record.nodeId, record }); + + await expect(nodes.delete(record.nodeId)).resolves.toBe('deleted'); + await expect( + nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'late stale write' }, + }), + ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); + + await harness.store.close(); + const reopened = trackedStore(harness.filename); + await reopened.init(); + await expect( + reopened.space(canvasId).nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'after reopen' }, + }), + ).resolves.toMatchObject({ + ok: true, + record: { ...record, content: 'after reopen' }, + }); + }); +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/rows.ts b/apps/server/src/modules/storage/backends/sqlite/rows.ts new file mode 100644 index 000000000..e949a350d --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/rows.ts @@ -0,0 +1,292 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Movement of persisted values between domain records and SQLite rows. + * + * Every column this backend stores is either JSON text or a scalar, so the + * codecs here are the single place that decides what a well-formed stored + * value looks like. Reads validate on the way out: a row that no longer + * matches the domain shape is a corruption report, not a silent default. + */ + +import { SQLITE_WORLD_COLLISION_KEY } from './database.js'; +import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; + +import type { + CanvasFile, + NodeContent, +} from '../../../canvas/persistence-types.js'; +import type { DatabaseSync } from 'node:sqlite'; + +type JsonPrimitive = null | boolean | number | string; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +function assertJsonValue( + value: unknown, + context: string, + seen: Set, +): asserts value is JsonValue { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' + ) { + return; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new TypeError(`${context} contains a non-finite number`); + } + return; + } + if (typeof value !== 'object') { + throw new TypeError(`${context} contains a non-JSON value`); + } + if (seen.has(value)) throw new TypeError(`${context} contains a cycle`); + seen.add(value); + try { + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + if (!Object.prototype.hasOwnProperty.call(value, index)) { + throw new TypeError(`${context} contains a sparse array`); + } + assertJsonValue(value[index], `${context}[${index}]`, seen); + } + return; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${context} contains a non-plain object`); + } + for (const [key, entry] of Object.entries(value)) { + assertJsonValue(entry, `${context}.${key}`, seen); + } + } finally { + seen.delete(value); + } +} + +export function stringifyJson(value: unknown, context: string): string { + assertJsonValue(value, context, new Set()); + const encoded = JSON.stringify(value); + if (encoded === undefined) { + throw new TypeError(`${context} is not representable as JSON`); + } + return encoded; +} + +export function parseJson(value: unknown, context: string): unknown { + if (typeof value !== 'string') { + throw new SyntaxError(`${context} is not stored as JSON text`); + } + try { + return JSON.parse(value) as unknown; + } catch (error) { + throw new SyntaxError( + `Invalid JSON in ${context}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function rowObject(value: unknown, context: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError(`Missing or malformed SQLite row for ${context}`); + } + return value as Record; +} + +function stringColumn( + row: Record, + column: string, + context: string, +): string { + const value = row[column]; + if (typeof value !== 'string') { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +function nullableStringColumn( + row: Record, + column: string, + context: string, +): string | null { + const value = row[column]; + if (value !== null && typeof value !== 'string') { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +function numberColumn( + row: Record, + column: string, + context: string, +): number { + const value = row[column]; + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +export interface PersistedSpace { + readonly record: CanvasFile; + readonly collisionKey: string; + readonly isWorld: boolean; +} + +export function decodeSpaceRow(value: unknown): PersistedSpace { + const row = rowObject(value, 'Space'); + const canvasId = stringColumn(row, 'canvas_id', 'Space'); + const context = `Space ${JSON.stringify(canvasId)}`; + const record: CanvasFile = { + canvasId, + title: nullableStringColumn(row, 'title', context), + version: numberColumn(row, 'version', context), + state: parseJson( + row['state_json'], + `${context} state`, + ) as CanvasFile['state'], + createdAt: numberColumn(row, 'created_at', context), + updatedAt: numberColumn(row, 'updated_at', context), + }; + const shapeError = canvasFileShapeError(record, canvasId); + if (shapeError) throw new SyntaxError(`Invalid ${context}: ${shapeError}`); + const world = numberColumn(row, 'is_world', context); + if (world !== 0 && world !== 1) { + throw new SyntaxError(`Invalid is_world in ${context}`); + } + return { + record, + collisionKey: stringColumn(row, 'collision_key', context), + isWorld: world === 1, + }; +} + +export const SPACE_COLUMNS = + 'canvas_id, title, collision_key, version, state_json, created_at, updated_at, is_world'; + +export function readSpaceRow( + database: DatabaseSync, + canvasId: string, +): PersistedSpace | null { + const row = database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE canvas_id = ?`) + .get(canvasId); + return row === undefined ? null : decodeSpaceRow(row); +} + +export function validateCanvasFile(record: CanvasFile, canvasId: string): void { + const shapeError = canvasFileShapeError(record, canvasId); + if (shapeError) { + throw new TypeError(`Invalid Space record: ${shapeError}`); + } + stringifyJson(record.state, `Space ${JSON.stringify(canvasId)} state`); +} + +export function insertSpaceRow( + database: DatabaseSync, + record: CanvasFile, + collisionKey: string, + isWorld = false, +): void { + validateCanvasFile(record, record.canvasId); + database + .prepare( + `INSERT INTO spaces ( + canvas_id, title, collision_key, version, state_json, + created_at, updated_at, is_world + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + record.canvasId, + record.title, + isWorld ? SQLITE_WORLD_COLLISION_KEY : collisionKey, + record.version, + stringifyJson(record.state, `Space ${record.canvasId} state`), + record.createdAt, + record.updatedAt, + isWorld ? 1 : 0, + ); +} + +export function updateSpaceRow( + database: DatabaseSync, + record: CanvasFile, + expectedVersion: number, +): number { + validateCanvasFile(record, record.canvasId); + const result = database + .prepare( + `UPDATE spaces + SET version = ?, state_json = ?, updated_at = ? + WHERE canvas_id = ? AND version = ?`, + ) + .run( + record.version, + stringifyJson(record.state, `Space ${record.canvasId} state`), + record.updatedAt, + record.canvasId, + expectedVersion, + ); + return Number(result.changes); +} + +export function validateNodeContent( + record: NodeContent, + expectedNodeId: string, +): void { + if (typeof record !== 'object' || record === null || Array.isArray(record)) { + throw new TypeError('Node record must be an object'); + } + if (record.nodeId !== expectedNodeId) { + throw new Error( + `Node id mismatch: argument=${JSON.stringify(expectedNodeId)} ` + + `record=${JSON.stringify(record.nodeId)}`, + ); + } + if (typeof record.type !== 'string') { + throw new TypeError('Node record type must be a string'); + } + if (record.label !== null && typeof record.label !== 'string') { + throw new TypeError('Node record label must be a string or null'); + } + if (typeof record.content !== 'string') { + throw new TypeError('Node record content must be a string'); + } + stringifyJson(record, `Node ${JSON.stringify(expectedNodeId)} record`); +} + +export function decodeNodeRecord( + value: unknown, + expectedNodeId: string, +): NodeContent { + const parsed = parseJson(value, `Node ${JSON.stringify(expectedNodeId)}`); + try { + validateNodeContent(parsed as NodeContent, expectedNodeId); + } catch (error) { + throw new SyntaxError( + `Invalid persisted Node ${JSON.stringify(expectedNodeId)}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + return parsed as NodeContent; +} + +export function requirePositiveRevision( + value: unknown, + nodeId: string, +): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + throw new SyntaxError( + `Invalid persisted revision for Node ${JSON.stringify(nodeId)}`, + ); + } + return value; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts new file mode 100644 index 000000000..5fe88a840 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { canvasEventInputSchema, canvasEventRecordSchema } from '@huabu/shared'; +import { + coalesceChanges, + type CanvasChangeRecord, +} from '@huabu/shared/canvas-engine'; + +import { withImmediateTransaction } from './database.js'; +import { parseJson, stringifyJson } from './rows.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { CanvasEvent } from '../../../canvas/persistence-types.js'; +import type { + NewCanvasEvent, + SpaceChanges, + SpaceEvents, +} from '../../ports/structured.js'; +import type { z } from 'zod'; + +function firstIssue(error: z.ZodError): string { + const issue = error.issues[0]; + if (!issue) return 'unknown schema violation'; + const location = issue.path.length > 0 ? issue.path.join('.') : ''; + return `${location}: ${issue.message}`; +} + +function requireSpace(context: SqliteStoreContext, canvasId: string): void { + context.assertMutationAllowed(canvasId); + if ( + context + .database() + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] !== 1 + ) { + throw new Error( + `SQLite Space logs(${canvasId}) cannot mutate a missing Space`, + ); + } +} + +function decodeEvents(rows: readonly Record[]): CanvasEvent[] { + return rows.map((row, index) => { + const parsedJson = parseJson( + row['event_json'], + `Canvas event ${index + 1}`, + ); + const parsed = canvasEventRecordSchema.safeParse(parsedJson); + if (!parsed.success) { + throw new SyntaxError( + `Invalid persisted Canvas event ${index + 1}: ${firstIssue(parsed.error)}`, + ); + } + return parsedJson as CanvasEvent; + }); +} + +function decodeChanges( + value: unknown, + canvasId: string, + threadId: string, +): CanvasChangeRecord[] { + const parsed = parseJson( + value, + `changes for Space ${JSON.stringify(canvasId)} thread ${JSON.stringify(threadId)}`, + ); + if (!Array.isArray(parsed)) { + throw new SyntaxError( + `Persisted changes for Space ${canvasId} thread ${threadId} must be an array`, + ); + } + return coalesceChanges(parsed as CanvasChangeRecord[]); +} + +export interface SqliteSpaceLogs { + readonly events: SpaceEvents; + readonly changes: SpaceChanges; +} + +class SqliteSpaceLogCoordinator { + readonly #context: SqliteStoreContext; + readonly #canvasId: string; + + constructor(context: SqliteStoreContext, canvasId: string) { + this.#context = context; + this.#canvasId = canvasId; + } + + async readEvents(limit?: number): Promise { + const database = this.#context.database(); + if (limit !== undefined && !(limit > 0)) return []; + if (limit === undefined || !Number.isFinite(limit)) { + return decodeEvents( + database + .prepare( + `SELECT event_json + FROM events + WHERE canvas_id = ? + ORDER BY event_id ASC`, + ) + .all(this.#canvasId), + ); + } + const rows = database + .prepare( + `SELECT event_json + FROM events + WHERE canvas_id = ? + ORDER BY event_id DESC + LIMIT ?`, + ) + .all(this.#canvasId, Math.ceil(limit)) + .reverse(); + return decodeEvents(rows); + } + + async appendEvents(events: readonly NewCanvasEvent[]): Promise { + this.#context.assertOpen(); + if (events.length === 0) return; + const records: CanvasEvent[] = events.map((event, index) => { + const input = canvasEventInputSchema.safeParse(event); + if (!input.success) { + throw new TypeError( + `Invalid Canvas event append input at index ${index}: ${firstIssue(input.error)}`, + ); + } + const record = { + payload: event.payload, + ts: event.ts ?? this.#context.now(), + }; + const parsed = canvasEventRecordSchema.safeParse(record); + if (!parsed.success) { + throw new TypeError( + `Invalid Canvas event append record at index ${index}: ${firstIssue(parsed.error)}`, + ); + } + stringifyJson(record, `Canvas event append input ${index}`); + return record; + }); + + requireSpace(this.#context, this.#canvasId); + const database = this.#context.database(); + withImmediateTransaction(database, () => { + const insert = database.prepare( + 'INSERT INTO events (canvas_id, event_json) VALUES (?, ?)', + ); + for (const record of records) { + insert.run( + this.#canvasId, + stringifyJson(record, `Canvas event for ${this.#canvasId}`), + ); + } + }); + } + + async readChanges(threadIdInput: string): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + const row = this.#context + .database() + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + return row === undefined + ? [] + : decodeChanges(row['snapshot_json'], this.#canvasId, threadId); + } + + async appendChanges( + threadIdInput: string, + records: readonly CanvasChangeRecord[], + ): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + stringifyJson(records, `Changes for thread ${JSON.stringify(threadId)}`); + requireSpace(this.#context, this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const current = database + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + const existing = + current === undefined + ? [] + : decodeChanges(current['snapshot_json'], this.#canvasId, threadId); + const merged = coalesceChanges([...existing, ...records]); + database + .prepare( + `INSERT INTO changes (canvas_id, thread_id, snapshot_json) + VALUES (?, ?, ?) + ON CONFLICT(canvas_id, thread_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json`, + ) + .run( + this.#canvasId, + threadId, + stringifyJson(merged, `Changes for thread ${threadId}`), + ); + return merged; + }); + } + + async deleteChange( + threadIdInput: string, + changeId: string, + ): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + requireSpace(this.#context, this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const current = database + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + if (current === undefined) return null; + const existing = decodeChanges( + current['snapshot_json'], + this.#canvasId, + threadId, + ); + const index = existing.findIndex((record) => record.id === changeId); + if (index < 0) return null; + const [removed] = existing.splice(index, 1); + database + .prepare( + `UPDATE changes + SET snapshot_json = ? + WHERE canvas_id = ? AND thread_id = ?`, + ) + .run( + stringifyJson(existing, `Changes for thread ${threadId}`), + this.#canvasId, + threadId, + ); + return removed ?? null; + }); + } +} + +export function createSqliteSpaceLogs( + context: SqliteStoreContext, + canvasId: string, +): SqliteSpaceLogs { + const coordinator = new SqliteSpaceLogCoordinator(context, canvasId); + return Object.freeze({ + events: Object.freeze({ + read: (limit?: number) => coordinator.readEvents(limit), + append: (events: readonly NewCanvasEvent[]) => + coordinator.appendEvents(events), + }), + changes: Object.freeze({ + read: (threadId: string) => coordinator.readChanges(threadId), + append: (threadId: string, records: readonly CanvasChangeRecord[]) => + coordinator.appendChanges(threadId, records), + delete: (threadId: string, changeId: string) => + coordinator.deleteChange(threadId, changeId), + }), + }); +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts new file mode 100644 index 000000000..e12afd180 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { allocateNodeIdentity } from './identity.js'; +import { + decodeNodeRecord, + requirePositiveRevision, + stringifyJson, + validateNodeContent, +} from './rows.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + NodeDeleteResult, + NodePutInput, + NodePutResult, + NodeSnapshot, + SpaceNodes, +} from '../../ports/structured.js'; +import type { DatabaseSync } from 'node:sqlite'; + +interface NodeRow { + readonly record: NodeSnapshot['record']; + readonly revision: number; + readonly collisionKey: string; +} + +function decodeNodeRow(value: unknown, nodeId: string): NodeRow { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError(`Malformed persisted Node ${JSON.stringify(nodeId)}`); + } + const row = value as Record; + const collisionKey = row['label_collision_key']; + if (typeof collisionKey !== 'string') { + throw new SyntaxError( + `Invalid collision key for Node ${JSON.stringify(nodeId)}`, + ); + } + return { + record: decodeNodeRecord(row['record_json'], nodeId), + revision: requirePositiveRevision(row['revision'], nodeId), + collisionKey, + }; +} + +function readNodeRow( + database: DatabaseSync, + canvasId: string, + nodeId: string, +): NodeRow | null { + const row = database + .prepare( + `SELECT record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id = ?`, + ) + .get(canvasId, nodeId); + return row === undefined ? null : decodeNodeRow(row, nodeId); +} + +function spaceExists(database: DatabaseSync, canvasId: string): boolean { + return ( + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] === 1 + ); +} + +function validatePut(input: NodePutInput): string { + const nodeId = sanitizeId(input.nodeId, 'nodeId'); + validateNodeContent(input.record, nodeId); + if ( + input.expectedRevision !== undefined && + input.expectedRevision !== null && + typeof input.expectedRevision !== 'string' + ) { + throw new TypeError('expectedRevision must be a string, null, or omitted'); + } + return nodeId; +} + +export interface SqliteNodePutOptions { + readonly tombstoned: boolean; + readonly bypassTombstone?: boolean; +} + +/** Apply one node put inside the caller's active transaction. */ +export function putSqliteNodeInTransaction( + database: DatabaseSync, + canvasId: string, + input: NodePutInput, + options: SqliteNodePutOptions, +): NodePutResult { + const nodeId = validatePut(input); + if (options.tombstoned && options.bypassTombstone !== true) { + return { ok: false, reason: 'write-suppressed' }; + } + if (!spaceExists(database, canvasId)) { + return { ok: false, reason: 'not-found' }; + } + + const current = readNodeRow(database, canvasId, nodeId); + const currentRevision = current === null ? null : String(current.revision); + if ( + input.expectedRevision !== undefined && + input.expectedRevision !== currentRevision + ) { + return { + ok: false, + reason: 'revision-conflict', + currentRevision, + }; + } + + const occupied = database + .prepare( + `SELECT label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id <> ?`, + ) + .all(canvasId, nodeId) + .map((row) => row['label_collision_key']) + .filter((value): value is string => typeof value === 'string'); + const allocation = allocateNodeIdentity( + input.record, + nodeId, + current?.collisionKey ?? null, + input.strictLabel === true ? [] : occupied, + ); + + if (input.strictLabel === true) { + const conflict = database + .prepare( + `SELECT node_id, record_json, label_collision_key + FROM nodes + WHERE canvas_id = ? + AND label_collision_key = ? + AND node_id <> ?`, + ) + .get(canvasId, allocation.desiredCollisionKey, nodeId); + if (conflict !== undefined) { + const conflictingNodeId = conflict['node_id']; + const collisionKey = conflict['label_collision_key']; + if (typeof conflictingNodeId !== 'string') { + throw new SyntaxError('Invalid conflicting SQLite Node id'); + } + const conflicting = decodeNodeRecord( + conflict['record_json'], + conflictingNodeId, + ); + return { + ok: false, + reason: 'label-conflict', + conflictingNodeId, + conflictingLabel: + typeof conflicting.label === 'string' + ? conflicting.label + : typeof collisionKey === 'string' + ? collisionKey + : conflictingNodeId, + }; + } + } + + const revision = (current?.revision ?? 0) + 1; + if (!Number.isSafeInteger(revision)) { + throw new Error(`Node ${JSON.stringify(nodeId)} revision overflow`); + } + database + .prepare( + `INSERT INTO nodes ( + canvas_id, node_id, record_json, revision, label_collision_key + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(canvas_id, node_id) DO UPDATE SET + record_json = excluded.record_json, + revision = excluded.revision, + label_collision_key = excluded.label_collision_key`, + ) + .run( + canvasId, + nodeId, + stringifyJson(allocation.record, `Node ${JSON.stringify(nodeId)} record`), + revision, + allocation.collisionKey, + ); + return { + ok: true, + record: allocation.record, + revision: String(revision), + }; +} + +export class SqliteSpaceNodes implements SpaceNodes { + readonly canvasId: string; + + readonly #context: SqliteStoreContext; + + constructor(context: SqliteStoreContext, canvasId: string) { + this.#context = context; + this.canvasId = canvasId; + } + + async read(nodeIdInput: string): Promise { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + const current = readNodeRow( + this.#context.database(), + this.canvasId, + nodeId, + ); + return current === null + ? null + : { record: current.record, revision: String(current.revision) }; + } + + async put(input: NodePutInput): Promise { + const nodeId = validatePut(input); + this.#context.assertMutationAllowed(this.canvasId); + if (this.#context.isNodeTombstoned(this.canvasId, nodeId)) { + return { ok: false, reason: 'write-suppressed' }; + } + const database = this.#context.database(); + return withImmediateTransaction(database, () => + putSqliteNodeInTransaction(database, this.canvasId, input, { + tombstoned: false, + }), + ); + } + + async delete(nodeIdInput: string): Promise { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + this.#context.assertMutationAllowed(this.canvasId); + const database = this.#context.database(); + const result = withImmediateTransaction(database, () => { + if (!spaceExists(database, this.canvasId)) + return 'missing-space' as const; + const deleted = Number( + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(this.canvasId, nodeId).changes, + ); + return deleted === 1 ? ('deleted' as const) : ('absent' as const); + }); + if (result === 'missing-space') return 'absent'; + if (result === 'deleted') { + this.#context.setNodeTombstone(this.canvasId, nodeId, true); + } + return result; + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts new file mode 100644 index 000000000..e12705cb4 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts @@ -0,0 +1,235 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { allocateSpaceIdentity, collisionKeyForTitle } from './identity.js'; +import { + decodeSpaceRow, + insertSpaceRow, + readSpaceRow, + SPACE_COLUMNS, +} from './rows.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { CanvasFile } from '../../../canvas/persistence-types.js'; +import type { + SpaceBeginDeleteResult, + SpaceCreateInput, + SpaceCreateResult, + SpaceDeleteInput, + SpaceDeleteSession, + SpaceRenameInput, + SpaceRenameResult, + SpaceRepository, +} from '../../ports/structured.js'; +import type { CanvasSummary } from '@huabu/shared'; + +function validateTitle(title: unknown): asserts title is string | null { + if (title !== null && typeof title !== 'string') { + throw new TypeError('Space title must be a string or null'); + } +} + +export class SqliteSpaceRepository implements SpaceRepository { + readonly #context: SqliteStoreContext; + + constructor(context: SqliteStoreContext) { + this.#context = context; + } + + async list(): Promise { + const database = this.#context.database(); + return database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 0`) + .all() + .map((row) => { + const { record } = decodeSpaceRow(row); + return { + canvasId: record.canvasId, + title: record.title, + nodeCount: record.state.nodes.length, + createdAt: record.createdAt, + updatedAt: record.updatedAt, + }; + }); + } + + async worldId(): Promise { + const database = this.#context.database(); + const rows = database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 1`) + .all(); + if (rows.length !== 1) { + throw new Error( + rows.length === 0 + ? 'SQLite namespace has no World Space' + : 'SQLite namespace has multiple World Spaces', + ); + } + const world = decodeSpaceRow(rows[0]); + if (!world.isWorld) throw new Error('SQLite World Space is malformed'); + return sanitizeId(world.record.canvasId, 'world canvasId'); + } + + async create(input: SpaceCreateInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + validateTitle(input.title); + this.#context.assertMutationAllowed(canvasId); + const database = this.#context.database(); + + return withImmediateTransaction(database, () => { + if (readSpaceRow(database, canvasId) !== null) { + return { ok: false as const, reason: 'already-exists' as const }; + } + const occupied = database + .prepare('SELECT collision_key FROM spaces') + .all() + .map((row) => row['collision_key']) + .filter((value): value is string => typeof value === 'string'); + const identity = allocateSpaceIdentity(input.title, canvasId, occupied); + const timestamp = this.#context.now(); + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Space clock returned a non-finite value'); + } + const record: CanvasFile = { + canvasId, + title: identity.title, + version: 0, + state: { nodes: [], edges: [] }, + createdAt: timestamp, + updatedAt: timestamp, + }; + insertSpaceRow(database, record, identity.collisionKey); + return { ok: true as const, record }; + }); + } + + async beginDelete(input: SpaceDeleteInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + const beforeAdmission = readSpaceRow(this.#context.database(), canvasId); + if (beforeAdmission?.isWorld) { + return { ok: false, reason: 'world-forbidden' }; + } + + const release = await this.#context.acquireDelete(canvasId); + let sessionOwnsGate = false; + try { + const afterAdmission = readSpaceRow(this.#context.database(), canvasId); + if (afterAdmission?.isWorld) { + return { ok: false, reason: 'world-forbidden' }; + } + + let state: 'open' | 'finishing' | 'closed' = 'open'; + const close = (): void => { + if (state === 'closed') return; + state = 'closed'; + release(); + }; + const session: SpaceDeleteSession = Object.freeze({ + finish: async () => { + if (state !== 'open') { + throw new Error(`Space deletion session for ${canvasId} is closed`); + } + state = 'finishing'; + try { + this.#context.assertOpen(); + const database = this.#context.database(); + const result = withImmediateTransaction(database, () => { + const current = readSpaceRow(database, canvasId); + if (current?.isWorld) { + throw new Error(`Refusing to delete World Space ${canvasId}`); + } + if (current === null) { + return { + deleted: false, + }; + } + const deleted = Number( + database + .prepare('DELETE FROM spaces WHERE canvas_id = ?') + .run(canvasId).changes, + ); + return { deleted: deleted === 1 }; + }); + if (result.deleted) { + this.#context.clearCanvasTombstones(canvasId); + return { ok: true as const, reason: 'deleted' as const }; + } + return { ok: false as const, reason: 'not-found' as const }; + } finally { + close(); + } + }, + abort: async () => { + if (state === 'finishing') { + throw new Error( + `Space deletion session for ${canvasId} is already finishing`, + ); + } + if (state === 'closed') return; + try { + this.#context.assertOpen(); + } finally { + close(); + } + }, + }); + sessionOwnsGate = true; + return { ok: true, session }; + } finally { + if (!sessionOwnsGate) release(); + } + } + + async rename(input: SpaceRenameInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + validateTitle(input.title); + this.#context.assertMutationAllowed(canvasId); + const database = this.#context.database(); + + return withImmediateTransaction(database, () => { + const current = readSpaceRow(database, canvasId); + if (current === null) return { ok: false, reason: 'not-found' } as const; + if (current.isWorld) { + return { ok: false, reason: 'world-forbidden' } as const; + } + if (current.record.title === input.title) { + return { ok: true, record: current.record } as const; + } + + const collisionKey = collisionKeyForTitle(input.title, canvasId); + if (collisionKey !== current.collisionKey) { + const conflict = database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE collision_key = ? AND canvas_id <> ?`, + ) + .get(collisionKey, canvasId); + if (conflict !== undefined) { + return { + ok: false, + reason: 'title-conflict', + conflictingTitle: decodeSpaceRow(conflict).record.title, + } as const; + } + } + + const result = database + .prepare( + `UPDATE spaces + SET title = ?, collision_key = ? + WHERE canvas_id = ?`, + ) + .run(input.title, collisionKey, canvasId); + if (Number(result.changes) !== 1) { + throw new Error(`Could not rename SQLite Space ${canvasId}`); + } + return { + ok: true, + record: { ...current.record, title: input.title }, + } as const; + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts new file mode 100644 index 000000000..4d27e2d17 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + taskRecordSchema, + taskRunCompletionSchema, + taskRunRecordSchema, + taskStoreSnapshotSchema, + type TaskRecord, + type TaskRunCompletion, + type TaskRunRecord, + type TaskStoreSnapshot, +} from '@huabu/shared'; + +import { withImmediateTransaction } from './database.js'; +import { parseJson, stringifyJson } from './rows.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + SpaceTaskRuns, + SpaceTasks, + TaskRunCompletionResult, + TaskRunUpdate, +} from '../../ports/structured.js'; + +const EMPTY_TASKS: TaskStoreSnapshot = { + version: 1, + tasks: [], + runs: [], +}; + +function validateSnapshot(value: unknown, canvasId: string): TaskStoreSnapshot { + const parsed = taskStoreSnapshotSchema.safeParse(value); + if (!parsed.success) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: ${parsed.error.issues[0]?.message ?? 'schema violation'}`, + ); + } + const taskIds = new Set(); + for (const task of parsed.data.tasks) { + if (task.canvasId !== canvasId) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Task ${task.taskId} belongs to Canvas ${task.canvasId}`, + ); + } + if (taskIds.has(task.taskId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: duplicate Task ${task.taskId}`, + ); + } + taskIds.add(task.taskId); + } + const runIds = new Set(); + for (const run of parsed.data.runs) { + if (run.canvasIdSnapshot !== canvasId) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Run ${run.runId} belongs to Canvas ${run.canvasIdSnapshot}`, + ); + } + if (runIds.has(run.runId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: duplicate Run ${run.runId}`, + ); + } + if (!taskIds.has(run.taskId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Run ${run.runId} references missing Task ${run.taskId}`, + ); + } + runIds.add(run.runId); + } + return parsed.data; +} + +function readSnapshot( + context: SqliteStoreContext, + canvasId: string, +): TaskStoreSnapshot { + const row = context + .database() + .prepare('SELECT snapshot_json FROM tasks WHERE canvas_id = ?') + .get(canvasId); + if (row === undefined) { + return { ...EMPTY_TASKS, tasks: [], runs: [] }; + } + return validateSnapshot( + parseJson(row['snapshot_json'], `Task store for Canvas ${canvasId}`), + canvasId, + ); +} + +export class SqliteSpaceTasks implements SpaceTasks { + readonly runs: SpaceTaskRuns; + + readonly #context: SqliteStoreContext; + readonly #canvasId: string; + + constructor(context: SqliteStoreContext, canvasId: string) { + this.#context = context; + this.#canvasId = canvasId; + this.runs = Object.freeze({ + create: (run: TaskRunRecord) => this.#createRun(run), + update: (runId: string, update: TaskRunUpdate) => + this.#updateRun(runId, update), + complete: ( + taskId: string, + runId: string, + completion: TaskRunCompletion, + ) => this.#completeRun(taskId, runId, completion), + }); + } + + async read(): Promise { + this.#context.assertOpen(); + return readSnapshot(this.#context, this.#canvasId); + } + + async create(task: TaskRecord): Promise { + const parsed = taskRecordSchema.safeParse(task); + if (!parsed.success || parsed.data.canvasId !== this.#canvasId) { + throw new TypeError(`Invalid Task record for Canvas ${this.#canvasId}`); + } + this.#mutate((snapshot) => { + if ( + snapshot.tasks.some( + (candidate) => candidate.taskId === parsed.data.taskId, + ) + ) { + throw new Error(`Task ${parsed.data.taskId} already exists`); + } + snapshot.tasks.push(parsed.data); + }); + } + + async #createRun(run: TaskRunRecord): Promise { + const parsed = taskRunRecordSchema.safeParse(run); + if (!parsed.success || parsed.data.canvasIdSnapshot !== this.#canvasId) { + throw new TypeError(`Invalid Run record for Canvas ${this.#canvasId}`); + } + this.#mutate((snapshot) => { + if ( + snapshot.runs.some((candidate) => candidate.runId === parsed.data.runId) + ) { + throw new Error(`Run ${parsed.data.runId} already exists`); + } + if ( + !snapshot.tasks.some( + (candidate) => candidate.taskId === parsed.data.taskId, + ) + ) { + throw new Error(`Task ${parsed.data.taskId} does not exist`); + } + snapshot.runs.push(parsed.data); + }); + } + + async #updateRun( + runId: string, + update: TaskRunUpdate, + ): Promise { + return this.#mutate((snapshot) => { + const index = snapshot.runs.findIndex((run) => run.runId === runId); + if (index < 0) throw new Error(`Run ${runId} does not exist`); + const parsed = taskRunRecordSchema.safeParse({ + ...snapshot.runs[index], + ...update, + }); + if (!parsed.success) { + throw new TypeError(`Invalid update for Run ${runId}`); + } + snapshot.runs[index] = parsed.data; + return parsed.data; + }); + } + + async #completeRun( + taskId: string, + runId: string, + completion: TaskRunCompletion, + ): Promise { + const parsedCompletion = taskRunCompletionSchema.safeParse(completion); + if (!parsedCompletion.success) { + throw new TypeError(`Invalid completion for Run ${runId}`); + } + return this.#mutate((snapshot) => { + if (!snapshot.tasks.some((task) => task.taskId === taskId)) { + return { outcome: 'task_not_found' }; + } + const index = snapshot.runs.findIndex((run) => run.runId === runId); + if (index < 0 || snapshot.runs[index]?.taskId !== taskId) { + return { outcome: 'run_not_found' }; + } + const current = snapshot.runs[index]; + if (!current) return { outcome: 'run_not_found' }; + if (current.status === 'completed') { + return current.completion?.message === parsedCompletion.data.message + ? { outcome: 'unchanged', run: current } + : { outcome: 'completion_conflict', run: current }; + } + if (current.status !== 'running') { + return { outcome: 'run_not_running', run: current }; + } + const parsedRun = taskRunRecordSchema.safeParse({ + ...current, + status: 'completed', + completion: parsedCompletion.data, + }); + if (!parsedRun.success) { + throw new TypeError(`Invalid completion update for Run ${runId}`); + } + snapshot.runs[index] = parsedRun.data; + return { outcome: 'completed', run: parsedRun.data }; + }); + } + + #mutate(apply: (snapshot: TaskStoreSnapshot) => T): T { + this.#context.assertMutationAllowed(this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if ( + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(this.#canvasId)?.['present'] !== 1 + ) { + throw new Error( + `Space Tasks(${this.#canvasId}) cannot write a missing Space`, + ); + } + const current = readSnapshot(this.#context, this.#canvasId); + const next: TaskStoreSnapshot = { + version: 1, + tasks: [...current.tasks], + runs: [...current.runs], + }; + const result = apply(next); + database + .prepare( + `INSERT INTO tasks (canvas_id, snapshot_json) + VALUES (?, ?) + ON CONFLICT(canvas_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json`, + ) + .run( + this.#canvasId, + stringifyJson(next, `Task store for Canvas ${this.#canvasId}`), + ); + return result; + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-write.ts b/apps/server/src/modules/storage/backends/sqlite/space-write.ts new file mode 100644 index 000000000..ad914619b --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-write.ts @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { allocateSpaceIdentity } from './identity.js'; +import { + insertSpaceRow, + readSpaceRow, + stringifyJson, + updateSpaceRow, + validateCanvasFile, + validateNodeContent, +} from './rows.js'; +import { putSqliteNodeInTransaction } from './space-nodes.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + NodePutResult, + SpaceHandle, + SpaceNodeMutation, + SpaceWriteInput, + SpaceWriteResult, +} from '../../ports/structured.js'; + +function mutationError( + mutation: SpaceNodeMutation, + result: NodePutResult, +): Error { + const prefix = `Space write failed for node ${JSON.stringify(mutation.nodeId)}`; + if (result.ok) return new Error(`${prefix}: unexpected success result`); + switch (result.reason) { + case 'not-found': + return new Error(`${prefix}: Space does not exist`); + case 'revision-conflict': + return new Error(`${prefix}: unexpected revision conflict`); + case 'label-conflict': + return new Error( + `${prefix}: label conflicts with node ${JSON.stringify(result.conflictingNodeId)}`, + ); + case 'duplicate-node': + return new Error(`${prefix}: duplicate persisted node`); + case 'write-suppressed': + return new Error(`${prefix}: write is suppressed after deletion`); + } +} + +function validateInput(canvasId: string, input: SpaceWriteInput): void { + if (!Number.isFinite(input.expectedVersion)) { + throw new TypeError('expectedVersion must be a finite number'); + } + validateCanvasFile(input.nextRecord, canvasId); + if (input.nextRecord.version !== input.expectedVersion + 1) { + throw new Error( + `SpaceWrite(${canvasId}) expected nextRecord.version ` + + `${input.expectedVersion + 1}, received ${input.nextRecord.version}`, + ); + } + if ( + input.allowCreate === true && + (input.nodeMutations.length > 0 || input.delta !== undefined) + ) { + throw new Error( + 'allowCreate is valid only for a record-only structural write', + ); + } + if ( + input.delta !== undefined && + input.delta.version !== input.nextRecord.version + ) { + throw new Error( + 'delta.version must equal the committed Space record version', + ); + } + if (input.delta !== undefined) { + stringifyJson(input.delta, `Space ${JSON.stringify(canvasId)} delta`); + } + for (const mutation of input.nodeMutations) { + sanitizeId(mutation.nodeId, 'nodeId'); + if (mutation.kind === 'put') { + validateNodeContent(mutation.record, mutation.nodeId); + } + } +} + +/** Bind the atomic SQLite record/node/delta write to one Space. */ +export function createSqliteSpaceWrite( + context: SqliteStoreContext, + canvasId: string, +): SpaceHandle['write'] { + return async function write( + input: SpaceWriteInput, + ): Promise { + context.assertMutationAllowed(canvasId); + validateInput(canvasId, input); + const database = context.database(); + + const completed = withImmediateTransaction(database, () => { + const current = readSpaceRow(database, canvasId); + if (current === null) { + if (!input.allowCreate) { + return { + result: { ok: false, reason: 'not-found' } as const, + tombstones: new Map(), + }; + } + if (input.expectedVersion !== 0) { + throw new Error( + `SpaceWrite(${canvasId}) can create only from version 0`, + ); + } + const occupied = database + .prepare('SELECT collision_key FROM spaces') + .all() + .map((row) => row['collision_key']) + .filter((value): value is string => typeof value === 'string'); + const identity = allocateSpaceIdentity( + input.nextRecord.title, + canvasId, + occupied, + ); + insertSpaceRow( + database, + { ...input.nextRecord, title: identity.title }, + identity.collisionKey, + ); + return { + result: { ok: true } as const, + tombstones: new Map(), + }; + } + + if (current.record.version !== input.expectedVersion) { + return { + result: { + ok: false, + reason: 'version-conflict', + actualVersion: current.record.version, + } as const, + tombstones: new Map(), + }; + } + if (input.nextRecord.createdAt !== current.record.createdAt) { + throw new Error(`SpaceWrite(${canvasId}) refusing to change createdAt`); + } + if (input.nextRecord.title !== current.record.title) { + throw new Error( + `SpaceWrite(${canvasId}) cannot change title; ` + + 'use SpaceRepository.rename first', + ); + } + + const tombstones = new Map(); + const tombstoned = (nodeId: string): boolean => + tombstones.get(nodeId) ?? context.isNodeTombstoned(canvasId, nodeId); + + for (const mutation of input.nodeMutations) { + if (mutation.kind === 'delete') { + const deleted = Number( + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(canvasId, mutation.nodeId).changes, + ); + if (deleted === 1) tombstones.set(mutation.nodeId, true); + continue; + } + + const result = putSqliteNodeInTransaction( + database, + canvasId, + { + nodeId: mutation.nodeId, + record: mutation.record, + strictLabel: mutation.strictLabel, + }, + { + tombstoned: tombstoned(mutation.nodeId), + bypassTombstone: mutation.authoritativeInsert === true, + }, + ); + if (!result.ok) throw mutationError(mutation, result); + if (mutation.authoritativeInsert === true) { + tombstones.set(mutation.nodeId, false); + } + } + + if ( + updateSpaceRow(database, input.nextRecord, input.expectedVersion) !== 1 + ) { + throw new Error(`SpaceWrite(${canvasId}) lost its version race`); + } + if (input.delta !== undefined) { + database + .prepare( + `INSERT INTO delta_log (canvas_id, version, entry_json) + VALUES (?, ?, ?)`, + ) + .run( + canvasId, + input.delta.version, + stringifyJson(input.delta, `Space ${canvasId} delta`), + ); + } + return { result: { ok: true } as const, tombstones }; + }); + + if (completed.result.ok) { + for (const [nodeId, present] of completed.tombstones) { + context.setNodeTombstone(canvasId, nodeId, present); + } + } + return completed.result; + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts new file mode 100644 index 000000000..f1d372b11 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SqliteStoreContext } from './database.js'; +import { readSpaceRow } from './rows.js'; +import { createSqliteSpaceLogs } from './space-logs.js'; +import { SqliteSpaceNodes } from './space-nodes.js'; +import { SqliteSpaceRepository } from './space-repository.js'; +import { SqliteSpaceTasks } from './space-tasks.js'; +import { createSqliteSpaceWrite } from './space-write.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { StorageHealth } from '../../ports/common.js'; +import type { + SpaceHandle, + SpaceRepository, + StructuredStore, +} from '../../ports/structured.js'; + +/** Production structured-store adapter backed by one node:sqlite connection. */ +export class SqliteStructuredStore implements StructuredStore { + readonly kind = 'sqlite' as const; + + readonly #context: SqliteStoreContext; + + constructor(filename: string, now: () => number = Date.now) { + if (typeof filename !== 'string') { + throw new TypeError('SQLite filename must be a string'); + } + if (filename.length === 0) { + throw new TypeError('SQLite filename must not be empty'); + } + this.#context = new SqliteStoreContext(filename, now); + } + + async init(): Promise { + this.#context.init(); + } + + async health(): Promise { + return this.#context.health(this.kind); + } + + async close(): Promise { + this.#context.close(); + } + + spaces(): SpaceRepository { + return Object.freeze(new SqliteSpaceRepository(this.#context)); + } + + space(canvasIdInput: string): SpaceHandle { + const canvasId = sanitizeId(canvasIdInput, 'canvasId'); + const { events, changes } = createSqliteSpaceLogs(this.#context, canvasId); + const nodes = Object.freeze(new SqliteSpaceNodes(this.#context, canvasId)); + const tasks = Object.freeze(new SqliteSpaceTasks(this.#context, canvasId)); + return Object.freeze({ + canvasId, + read: async () => + readSpaceRow(this.#context.database(), canvasId)?.record ?? null, + write: createSqliteSpaceWrite(this.#context, canvasId), + nodes, + changes, + tasks, + events, + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/test-support.ts b/apps/server/src/modules/storage/backends/sqlite/test-support.ts new file mode 100644 index 000000000..bddebc8a8 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/test-support.ts @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { SQLITE_SCHEMA_VERSION } from './database.js'; +import { collisionKeyForTitle } from './identity.js'; +import { insertSpaceRow, parseJson } from './rows.js'; +import { SqliteStructuredStore } from './structured-store.js'; + +import type { + CanvasFile, + DeltaLogEntry, +} from '../../../canvas/persistence-types.js'; + +export const SQLITE_TEST_WORLD_ID = 'sqlite-test-world'; + +export interface SqliteTestFile { + readonly directory: string; + readonly filename: string; + readonly remove: () => void; +} + +export interface OpenSqliteTestStore extends SqliteTestFile { + readonly store: SqliteStructuredStore; + readonly world: CanvasFile; + readonly cleanup: () => Promise; +} + +export function createSqliteTestFile(prefix = 'huabu-sqlite-'): SqliteTestFile { + const directory = mkdtempSync(path.join(tmpdir(), prefix)); + const filename = path.join(directory, 'structured.sqlite'); + let removed = false; + return { + directory, + filename, + remove: () => { + if (removed) return; + removed = true; + rmSync(directory, { recursive: true, force: true }); + }, + }; +} + +/** Run a short test-only query through a connection independent of the store. */ +export function withTestDatabase( + filename: string, + operation: (database: DatabaseSync) => T, +): T { + const database = new DatabaseSync(filename); + try { + database.exec('PRAGMA foreign_keys = ON'); + return operation(database); + } finally { + database.close(); + } +} + +/** + * Seed World without reaching through the adapter under test. + * + * The store first creates the production schema. This helper then opens a + * separate node:sqlite connection and uses the production row encoder, so a + * contract cannot pass because World creation accidentally shares private + * adapter state with the operation being exercised. + */ +export function seedSqliteWorld( + filename: string, + canvasId = SQLITE_TEST_WORLD_ID, +): CanvasFile { + const record: CanvasFile = { + canvasId, + title: 'World', + version: 0, + state: { nodes: [], edges: [] }, + createdAt: 1, + updatedAt: 1, + }; + withTestDatabase(filename, (database) => { + const version = database.prepare('PRAGMA user_version').get()?.[ + 'user_version' + ]; + if (version !== SQLITE_SCHEMA_VERSION) { + throw new Error( + `Expected production SQLite schema v${SQLITE_SCHEMA_VERSION}, got ${String(version)}`, + ); + } + insertSpaceRow( + database, + record, + collisionKeyForTitle(record.title, record.canvasId), + true, + ); + }); + return record; +} + +export async function openSqliteTestStore( + prefix = 'huabu-sqlite-', + now?: () => number, +): Promise { + const file = createSqliteTestFile(prefix); + const store = new SqliteStructuredStore(file.filename, now); + try { + await store.init(); + const world = seedSqliteWorld(file.filename); + return { + ...file, + store, + world, + cleanup: async () => { + await store.close(); + file.remove(); + }, + }; + } catch (error) { + await store.close(); + file.remove(); + throw error; + } +} + +export function readSqliteDeltaLog( + filename: string, + canvasId: string, +): DeltaLogEntry[] { + return withTestDatabase(filename, (database) => + database + .prepare( + `SELECT entry_json + FROM delta_log + WHERE canvas_id = ? + ORDER BY version`, + ) + .all(canvasId) + .map( + (row, index) => + parseJson( + row['entry_json'], + `test delta row ${index} for ${canvasId}`, + ) as DeltaLogEntry, + ), + ); +} + +/** Install a real SQLite failure immediately before a delta row is inserted. */ +export function installDeltaAbortTrigger( + filename: string, + message: string, +): () => void { + const quotedMessage = message.split("'").join("''"); + withTestDatabase(filename, (database) => { + database.exec('DROP TRIGGER IF EXISTS test_abort_delta_insert'); + database.exec(` + CREATE TRIGGER test_abort_delta_insert + BEFORE INSERT ON delta_log + BEGIN + SELECT RAISE(ABORT, '${quotedMessage}'); + END + `); + }); + let restored = false; + return () => { + if (restored) return; + restored = true; + withTestDatabase(filename, (database) => { + database.exec('DROP TRIGGER IF EXISTS test_abort_delta_insert'); + }); + }; +} diff --git a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts index f6499cf9b..58bbe19e0 100644 --- a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts @@ -6,11 +6,18 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { NodeContent } from '../../../canvas/persistence-types.js'; -import type { NodePutInput, SpaceNodes, NodeSnapshot } from '../structured.js'; +import type { + NodePutInput, + SpaceHandle, + SpaceNodes, + NodeSnapshot, +} from '../structured.js'; export interface SpaceNodesContractHarness { /** Repository for an existing Space, initially empty at contract-owned ids. */ readonly repository: SpaceNodes; + /** Handle that owns `repository`, used to exercise ordered reinsertion. */ + readonly space: SpaceHandle; /** Repository scoped to a Space whose structural record is absent. */ readonly missingRepository: SpaceNodes; readonly expectedCanvasId: string; @@ -238,8 +245,8 @@ export function describeSpaceNodesContract( await expect(repository.delete(nodeId)).resolves.toBe('absent'); }); - it('suppresses a late standalone put after deletion', async () => { - const { repository } = await open(); + it('suppresses standalone resurrection until an authoritative ordered insert succeeds', async () => { + const { repository, space } = await open(); const nodeId = 'contract-late-put'; const record = note(nodeId, 'Contract late put', 'before'); await putSuccessfully(repository, { nodeId, record }); @@ -252,6 +259,56 @@ export function describeSpaceNodesContract( }), ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); await expect(repository.read(nodeId)).resolves.toBeNull(); + + const current = await space.read(); + if (current === null) + throw new Error('Contract fixture Space is missing'); + const authoritative = { + ...record, + content: 'authoritative resurrection', + }; + await expect( + space.write({ + expectedVersion: current.version, + nextRecord: { + ...current, + version: current.version + 1, + state: { + ...current.state, + nodes: [ + ...current.state.nodes, + { id: nodeId, type: authoritative.type }, + ], + }, + updatedAt: current.updatedAt + 1, + }, + nodeMutations: [ + { + kind: 'put', + nodeId, + record: authoritative, + authoritativeInsert: true, + }, + ], + }), + ).resolves.toEqual({ ok: true }); + + const restored = await repository.read(nodeId); + expect(restored).toMatchObject({ record: authoritative }); + if (restored === null) { + throw new Error('Authoritatively reinserted node is missing'); + } + + await expect( + repository.put({ + nodeId, + expectedRevision: restored.revision, + record: { ...authoritative, content: 'later standalone update' }, + }), + ).resolves.toMatchObject({ + ok: true, + record: { content: 'later standalone update' }, + }); }); }); } diff --git a/apps/server/src/modules/storage/ports/contracts/space-repository.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-repository.contract.ts index 32cdf2909..81cef4c2e 100644 --- a/apps/server/src/modules/storage/ports/contracts/space-repository.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/space-repository.contract.ts @@ -94,6 +94,40 @@ export function describeSpaceRepositoryContract( ).not.toContain(worldCanvasId); }); + it('does not reserve the World title from ordinary Spaces', async () => { + const { repository, read } = await open(); + const created = await repository.create({ + canvasId: 'contract-world-title', + title: 'World', + }); + expect(created).toMatchObject({ + ok: true, + record: { canvasId: 'contract-world-title', title: 'World' }, + }); + if (!created.ok) throw new Error('Expected create to succeed'); + await expect(read('contract-world-title')).resolves.toEqual( + created.record, + ); + + const renamedAway = await repository.rename({ + canvasId: 'contract-world-title', + title: 'Temporarily not World', + }); + if (!renamedAway.ok) throw new Error('Expected rename to succeed'); + const renamedBack = await repository.rename({ + canvasId: 'contract-world-title', + title: 'World', + }); + expect(renamedBack).toMatchObject({ + ok: true, + record: { canvasId: 'contract-world-title', title: 'World' }, + }); + if (!renamedBack.ok) throw new Error('Expected rename to succeed'); + await expect(read('contract-world-title')).resolves.toEqual( + renamedBack.record, + ); + }); + it('creates and returns the authoritative empty version-0 record', async () => { const { repository, read } = await open(); const result = await repository.create({ diff --git a/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts new file mode 100644 index 000000000..a02aba86f --- /dev/null +++ b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts @@ -0,0 +1,441 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Reusable behavioral contract for {@link SpaceTasks} and its Runs. */ + +import { afterEach, describe, expect, it } from 'vitest'; + +import type { + SpaceDeleteSession, + SpaceTasks, + TaskRunUpdate, +} from '../structured.js'; +import type { TaskRecord, TaskRunRecord } from '@huabu/shared'; + +export interface SpaceTasksContractHarness { + /** Task ledger for an existing Space, initially empty. */ + readonly tasks: SpaceTasks; + /** A second retained handle for the same existing Space. */ + readonly concurrent: SpaceTasks; + readonly canvasId: string; + /** Task ledger scoped to a Space whose structural record is absent. */ + readonly missing: SpaceTasks; + readonly missingCanvasId: string; + /** Open a structured-deletion fence for `canvasId`. */ + readonly beginDelete: () => Promise; + readonly cleanup?: () => Promise | void; +} + +function task(canvasId: string, taskId: string, createdAt: number): TaskRecord { + return { + taskId, + canvasId, + goal: `Goal for ${taskId}`, + defaultRootProfileId: `profile-${taskId}`, + anchorNodeId: `anchor-${taskId}`, + createdAt, + }; +} + +function run( + canvasId: string, + taskId: string, + runId: string, + createdAt: number, +): TaskRunRecord { + return { + runId, + taskId, + canvasIdSnapshot: canvasId, + goalSnapshot: `Goal snapshot for ${taskId}`, + rootProfileIdSnapshot: `profile-${taskId}`, + status: 'pending', + createdAt, + }; +} + +export function describeSpaceTasksContract( + name: string, + createHarness: () => + | Promise + | SpaceTasksContractHarness, +): void { + describe(`SpaceTasks contract: ${name}`, () => { + let harness: SpaceTasksContractHarness | null = null; + + async function open(): Promise { + harness = await createHarness(); + return harness; + } + + afterEach(async () => { + await harness?.cleanup?.(); + harness = null; + }); + + it('reads an empty versioned snapshot', async () => { + const { tasks } = await open(); + + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + }); + + it('creates a Task and rejects a duplicate id without replacing it', async () => { + const { tasks, canvasId } = await open(); + const original = task(canvasId, 'task-duplicate', 1); + await tasks.create(original); + + await expect( + tasks.create({ ...original, goal: 'Replacement goal', createdAt: 2 }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [original], + runs: [], + }); + }); + + it('requires an existing Task before creating its Run', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-owner', 1); + const ownedRun = run(canvasId, owner.taskId, 'run-owned', 2); + + await expect(tasks.runs.create(ownedRun)).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + + await tasks.create(owner); + await tasks.runs.create(ownedRun); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [ownedRun], + }); + }); + + it('rejects a duplicate Run id without replacing it', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-run-duplicate', 1); + const original = run(canvasId, owner.taskId, 'run-duplicate', 2); + await tasks.create(owner); + await tasks.runs.create(original); + + await expect( + tasks.runs.create({ + ...original, + status: 'running', + startedAt: 3, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [original], + }); + }); + + it('updates an existing Run and rejects a missing Run id', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-update', 1); + const original = run(canvasId, owner.taskId, 'run-update', 2); + await tasks.create(owner); + await tasks.runs.create(original); + const update: TaskRunUpdate = { + rootNodeId: 'root-node', + rootThreadId: 'root-thread', + status: 'running', + startedAt: 3, + }; + + await expect(tasks.runs.update(original.runId, update)).resolves.toEqual({ + ...original, + ...update, + }); + await expect( + tasks.runs.update('run-missing', { status: 'running' }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [{ ...original, ...update }], + }); + }); + + it('completes only a running Run and keeps the first completion immutable', async () => { + const { tasks, concurrent, canvasId } = await open(); + const owner = task(canvasId, 'task-complete', 1); + const other = task(canvasId, 'task-complete-other', 2); + const original = run(canvasId, owner.taskId, 'run-complete', 3); + await tasks.create(owner); + await tasks.create(other); + await tasks.runs.create(original); + + await expect( + tasks.runs.complete(owner.taskId, original.runId, { completedAt: 4 }), + ).resolves.toMatchObject({ outcome: 'run_not_running', run: original }); + + await tasks.runs.update(original.runId, { + status: 'running', + startedAt: 5, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 6, + message: 'Done', + }), + ).resolves.toMatchObject({ + outcome: 'completed', + run: { + status: 'completed', + completion: { completedAt: 6, message: 'Done' }, + }, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 7, + message: 'Done', + }), + ).resolves.toMatchObject({ + outcome: 'unchanged', + run: { completion: { completedAt: 6, message: 'Done' } }, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 8, + message: 'Different', + }), + ).resolves.toMatchObject({ outcome: 'completion_conflict' }); + await expect( + tasks.runs.complete('task-missing', original.runId, { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'task_not_found' }); + await expect( + tasks.runs.complete(other.taskId, original.runId, { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'run_not_found' }); + await expect( + tasks.runs.complete(owner.taskId, 'run-missing', { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'run_not_found' }); + await expect(concurrent.read()).resolves.toEqual({ + version: 1, + tasks: [owner, other], + runs: [ + { + ...original, + status: 'completed', + startedAt: 5, + completion: { completedAt: 6, message: 'Done' }, + }, + ], + }); + }); + + it('serializes competing completions and persists exactly one winner', async () => { + const { tasks, concurrent, canvasId } = await open(); + const owner = task(canvasId, 'task-competing-completion', 1); + const original = run( + canvasId, + owner.taskId, + 'run-competing-completion', + 2, + ); + await tasks.create(owner); + await tasks.runs.create(original); + await tasks.runs.update(original.runId, { + status: 'running', + startedAt: 3, + }); + + const results = await Promise.all([ + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 4, + message: 'First candidate', + }), + concurrent.runs.complete(owner.taskId, original.runId, { + completedAt: 5, + message: 'Second candidate', + }), + ]); + expect(results.map((result) => result.outcome).sort()).toEqual([ + 'completed', + 'completion_conflict', + ]); + const completed = results.find( + (result) => result.outcome === 'completed', + ); + const conflict = results.find( + (result) => result.outcome === 'completion_conflict', + ); + if (completed?.outcome !== 'completed') { + throw new Error('Expected one completion winner'); + } + if (conflict?.outcome !== 'completion_conflict') { + throw new Error('Expected one completion conflict'); + } + expect(conflict.run).toEqual(completed.run); + await expect(concurrent.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [completed.run], + }); + }); + + it('rejects Task and Run records scoped to another Space', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-scope', 1); + + await expect( + tasks.create({ ...owner, canvasId: 'another-space' }), + ).rejects.toThrow(); + await tasks.create(owner); + await expect( + tasks.runs.create({ + ...run(canvasId, owner.taskId, 'run-scope', 2), + canvasIdSnapshot: 'another-space', + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [], + }); + }); + + it('rejects malformed Task, Run, and Run-update input', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-validation', 1); + const ownedRun = run(canvasId, owner.taskId, 'run-validation', 2); + + await expect(tasks.create({ ...owner, goal: '' })).rejects.toThrow(); + await tasks.create(owner); + await expect( + tasks.runs.create({ ...ownedRun, goalSnapshot: '' }), + ).rejects.toThrow(); + await tasks.runs.create(ownedRun); + await expect( + tasks.runs.update(ownedRun.runId, { startedAt: -1 }), + ).rejects.toThrow(); + await expect( + tasks.runs.complete(owner.taskId, ownedRun.runId, { + completedAt: -1, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [ownedRun], + }); + }); + + it('preserves concurrent mutations through two retained handles', async () => { + const { tasks, concurrent, canvasId } = await open(); + const taskA = task(canvasId, 'task-concurrent-a', 1); + const taskB = task(canvasId, 'task-concurrent-b', 2); + await Promise.all([tasks.create(taskA), concurrent.create(taskB)]); + + const runA = run(canvasId, taskA.taskId, 'run-concurrent-a', 3); + const runB = run(canvasId, taskB.taskId, 'run-concurrent-b', 4); + await Promise.all([ + tasks.runs.create(runA), + concurrent.runs.create(runB), + ]); + await Promise.all([ + concurrent.runs.update(runA.runId, { + status: 'running', + startedAt: 5, + }), + tasks.runs.update(runB.runId, { + status: 'running', + startedAt: 6, + }), + ]); + + const snapshot = await tasks.read(); + expect(snapshot.tasks.map((record) => record.taskId).sort()).toEqual([ + taskA.taskId, + taskB.taskId, + ]); + expect(snapshot.runs.map((record) => record.runId).sort()).toEqual([ + runA.runId, + runB.runId, + ]); + expect(snapshot.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId: runA.runId, + status: 'running', + startedAt: 5, + }), + expect.objectContaining({ + runId: runB.runId, + status: 'running', + startedAt: 6, + }), + ]), + ); + }); + + it('rejects every mutation for a missing Space', async () => { + const { missing, missingCanvasId } = await open(); + const owner = task(missingCanvasId, 'task-missing-space', 1); + const ownedRun = run( + missingCanvasId, + owner.taskId, + 'run-missing-space', + 2, + ); + + await expect(missing.create(owner)).rejects.toThrow(); + await expect(missing.runs.create(ownedRun)).rejects.toThrow(); + await expect( + missing.runs.update(ownedRun.runId, { status: 'running' }), + ).rejects.toThrow(); + await expect( + missing.runs.complete(owner.taskId, ownedRun.runId, { + completedAt: 3, + }), + ).rejects.toThrow(); + }); + + it('rejects mutations while structured deletion is fenced', async () => { + const { tasks, canvasId, beginDelete } = await open(); + const owner = task(canvasId, 'task-delete-fence', 1); + const original = run(canvasId, owner.taskId, 'run-delete-fence', 2); + await tasks.create(owner); + await tasks.runs.create(original); + const before = await tasks.read(); + const session = await beginDelete(); + + try { + await expect( + tasks.create(task(canvasId, 'task-too-late', 3)), + ).rejects.toThrow(); + await expect( + tasks.runs.create(run(canvasId, owner.taskId, 'run-too-late', 4)), + ).rejects.toThrow(); + await expect( + tasks.runs.update(original.runId, { status: 'running' }), + ).rejects.toThrow(); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 5, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual(before); + } finally { + await session.abort(); + } + + await expect( + tasks.runs.update(original.runId, { + status: 'running', + startedAt: 5, + }), + ).resolves.toMatchObject({ status: 'running', startedAt: 5 }); + }); + }); +} diff --git a/apps/server/src/modules/storage/ports/structured.ts b/apps/server/src/modules/storage/ports/structured.ts index 274074304..67c1cd6c1 100644 --- a/apps/server/src/modules/storage/ports/structured.ts +++ b/apps/server/src/modules/storage/ports/structured.ts @@ -61,7 +61,7 @@ import type { CanvasChangeRecord } from '@huabu/shared/canvas-engine'; * that are configurable but unimplemented — belongs to `profile.ts`, which * owns rejecting them with an actionable message. */ -export type StructuredBackendKind = 'disk'; +export type StructuredBackendKind = 'disk' | 'sqlite'; /** A connection to a structured backend. Process-wide; handles are derived. */ export interface StructuredStore { @@ -254,12 +254,17 @@ export type SpaceNodeMutation = /** * Marks an executor-authoritative INSERT. * - * **Adapter-shaped**, like {@link NodePutResult}'s `write-suppressed`. - * It exists for a backend that suppresses writes to a recently deleted - * id, and lets such an adapter distinguish a real re-insertion from a - * late direct write that should stay suppressed. It is intentionally - * batch-only. An adapter whose deletes are immediately final — a SQL - * table with a unique key — can ignore it. + * After {@link SpaceNodes.delete} removes an existing id, standalone + * puts for that id must return `write-suppressed` within the same running + * {@link StructuredStore}. A successful ordered put carrying this flag + * is the portable signal that the id is intentionally being reinserted; + * it admits the write and clears that suppression for later standalone + * puts. It is intentionally batch-only so a late direct write cannot + * claim authority for itself. + * + * This is an in-memory connection-lifetime guarantee, not restart + * durability. Closing or recreating the StructuredStore may discard the + * deletion fence. */ readonly authoritativeInsert?: boolean; } @@ -448,19 +453,19 @@ export type NodeDeleteResult = 'deleted' | 'absent'; * `label-conflict`. The contract intentionally says nothing about filenames * or physical layout. Environmental and malformed-record failures reject. * - * Two mutation outcomes are **adapter-shaped** and optional: + * One mutation outcome is **adapter-shaped** and optional: * * - `duplicate-node`, for adapters that can observe conflicting physical * representations of one stable id. Such an adapter may return one readable * representative from `read` so a caller can construct the attempted * update, but it must refuse the `put` rather than overwrite an arbitrary * representation. - * - `write-suppressed`, for adapters that keep a deleted id fenced against - * late in-flight writes. See {@link SpaceNodeMutation}'s - * `authoritativeInsert`, which is how a batch re-insertion is distinguished - * from such a late write. * - * A SQL adapter with a unique key produces neither. + * `write-suppressed` is portable anti-resurrection behavior. After a + * successful delete of an existing node, standalone puts for that id are + * suppressed for the lifetime of the running {@link StructuredStore} until a + * successful ordered put marks the id as an `authoritativeInsert`. The fence + * need not survive closing or recreating the store. */ export interface SpaceNodes { /** diff --git a/apps/server/src/modules/storage/profile.test.ts b/apps/server/src/modules/storage/profile.test.ts index 18be15c91..c51255000 100644 --- a/apps/server/src/modules/storage/profile.test.ts +++ b/apps/server/src/modules/storage/profile.test.ts @@ -64,7 +64,16 @@ describe('validateStorageProfile', () => { structured: { kind: 'postgres' }, blobs: { kind: 'disk' }, }), - ).toThrow(/not implemented yet.*disk/s); + ).toThrow(/not implemented yet.*disk, sqlite/s); + }); + + it('rejects an available preview adapter that is not selectable', () => { + expect(() => + validateStorageProfile({ + structured: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, + }), + ).toThrow(/preview adapter.*not selectable yet.*Selectable: disk/s); }); it('rejects a known but unimplemented blob backend', () => { diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index f548af174..633f8545d 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -28,11 +28,20 @@ export interface StorageProfile { blobs: { kind: BlobBackendKind }; } +/** Backends with an adapter implementation, selectable or otherwise. */ +const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ + 'disk', + 'sqlite', +]; + /** - * Backends that exist today. Naming one that is not written yet must fail - * loudly rather than half-work. + * Backends whose complete capability matrix is safe for production use. + * + * SQLite deliberately stays out while physical Disk reads, World bootstrap, + * Blob placement, import/export, and Workspace remounting still have one + * authority only in the Disk profile. */ -const IMPLEMENTED_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; +const SELECTABLE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; const IMPLEMENTED_BLOBS: readonly BlobBackendKind[] = ['disk']; const STRUCTURED_KINDS: readonly RequestedStructuredKind[] = [ @@ -85,16 +94,24 @@ export function parseStorageProfile( /** * Reject profiles that cannot serve correctly, before any connection opens. * - * Today that means "named but not implemented". This is also where - * cross-axis rules belong as backends land — for example, Postgres paired - * with a node-local disk blob root is unsafe across replicas unless the - * path is a deliberately shared filesystem. + * A recognized kind may still lack an adapter, or it may have an isolated + * contract-proof adapter while application capabilities remain tied to Disk. + * This is also where cross-axis rules belong as backends land — for example, + * Postgres paired with a node-local disk blob root is unsafe across replicas + * unless the path is a deliberately shared filesystem. */ export function validateStorageProfile(profile: StorageProfile): void { - if (!IMPLEMENTED_STRUCTURED.includes(profile.structured.kind)) { + if (!AVAILABLE_STRUCTURED.includes(profile.structured.kind)) { throw new StorageProfileError( `Structured backend "${profile.structured.kind}" is not implemented yet. ` + - `Available: ${IMPLEMENTED_STRUCTURED.join(', ')}.`, + `Adapters available: ${AVAILABLE_STRUCTURED.join(', ')}.`, + ); + } + if (!SELECTABLE_STRUCTURED.includes(profile.structured.kind)) { + throw new StorageProfileError( + `Structured backend "${profile.structured.kind}" has a preview adapter ` + + `but is not selectable yet. Required application capabilities still ` + + `depend on Disk. Selectable: ${SELECTABLE_STRUCTURED.join(', ')}.`, ); } if (!IMPLEMENTED_BLOBS.includes(profile.blobs.kind)) { diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 39a9cabb9..f7bd57c01 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -1,10 +1,10 @@ # Canvas Storage Architecture -> Last updated: 2026-08-11 +> Last updated: 2026-08-18 ## 1. Overview -Every Space remains fully self-contained on Disk by default, but storage no longer presents one all-purpose `CanvasStore` as its backend contract. `apps/server/src/modules/storage/` separates backend-neutral blob and structured ports, Disk adapters, process-wide composition, and a shrinking Disk compatibility facade. Opaque artifact bytes flow through `BlobStore`; `StructuredStore` exposes one `spaces()` repository for the Space collection — membership, World identity, and create/delete/rename — while `SpaceHandle` exposes the Space's own async record read and ordered write plus the parts it holds: `nodes`, `changes`, `tasks`, and `events`. Space creation and deletion, standalone node writes, executor and revert batches, preprocessing persistence, and event/change mutations enter these ports. The remaining compatibility consumers are explicit Disk capabilities and paths such as ZIP import/export, RFS upload/delete, external-note observation/claim, bootstrap/migration, and hydration helpers; some read and some mutate physical files, so they keep non-Disk profiles unselectable until their own contracts are designed. +Every Space remains fully self-contained on Disk by default, but storage no longer presents one all-purpose `CanvasStore` as its backend contract. `apps/server/src/modules/storage/` separates backend-neutral blob and structured ports, the selectable Disk adapters, an isolated SQLite structured contract-preview adapter, process-wide composition, and a shrinking Disk compatibility facade. Opaque artifact bytes flow through `BlobStore`; `StructuredStore` exposes one `spaces()` repository for the Space collection — membership, World identity, and create/delete/rename — while `SpaceHandle` exposes the Space's own async record read and ordered write plus the parts it holds: `nodes`, `changes`, `tasks`, and `events`. Space creation and deletion, standalone node writes, executor and revert batches, preprocessing persistence, and event/change mutations enter these ports. The remaining compatibility consumers are explicit Disk capabilities and paths such as ZIP import/export, RFS upload/delete, external-note observation/claim, bootstrap/migration, and hydration helpers; some read and some mutate physical files, so they keep non-Disk profiles unselectable until their own contracts are designed. Runtime Home-folder activation prepares and migrates the selected directory in a disposable child process before committing it as the active workspace. This isolation is required because synchronous filesystem calls against cloud, network, or virtual drives can block indefinitely; a stuck preparation is terminated after 70 seconds with `WORKSPACE_ACTIVATION_TIMEOUT`, while the Server event loop and previously active workspace remain available. Concurrent activation attempts return `WORKSPACE_ACTIVATION_IN_PROGRESS`. Managed-mode startup still prepares synchronously before the Server accepts requests. @@ -52,13 +52,13 @@ Key points: - Persistent `frameRef` and `nodeRef` nodes have no markdown sidecars and store only their respective type plus `{ target: { canvasId, nodeId } }` and World-owned React Flow state. A `frameRef` is a Container snapshot of a source Frame, may recursively own matching `frameRef` / `nodeRef` descendants, and never reconciles later source hierarchy changes; direct references remain children of the matching `canvasRef`. `SET_PORTAL_NODE_PINS` is their sole create/remove path. - `GET /api/canvas/:worldCanvasId/references` batch-resolves Portal titles and pinned source-node display data for both reference types without writing it into World topology. Results distinguish `ok`, `canvas-missing`, and `node-missing`; storage or parse failures remain request errors. - Node filenames are `safe(label).md`; the node's stable id lives in the `id:` frontmatter field. -- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Ordinary callers resolve the scope through `canvasBlobs(canvasId)`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Only the Disk blob and structured backends are implemented and selectable today. +- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Ordinary callers resolve the scope through `canvasBlobs(canvasId)`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Disk remains the only selectable blob and structured backend; SQLite has an isolated structured adapter for contract and integration tests but is not constructed by runtime composition. - Remote PDF preprocessing writes the already-fetched source bytes into the Space BlobStore as `artifact-.pdf` before structured persistence and replaces the node's remote `src` with that key. As with other artifact imports, this blob write precedes the node write operation; a later structured persistence failure may therefore leave an unreferenced blob until Space deletion, while a blob-write failure degrades to retaining the remote URL. - Events are append-only JSONL (`events.jsonl`); each line is `{ ts: number, payload: RecentAction }`. - The memory analyzer reads Space existence and at most 100 recent action events through one `SpaceHandle`. A missing Space skips the pass before reading memory files or calling the model; corrupt part data still fails the pass. Memory body/state files remain materialized workspace paths, while Agenetes-owned chat history is not part of the curator bundle. - **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** The canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. - Durable Agenetes workload records live in `.history/threads.json` (`agenetes-v2` schema, one record per thread; written by `FileThreadStore`). The host-local `namespace.storage.root` is never persisted: reads bind each record to the current Space namespace, so a Home synchronized across computers cannot redirect storage back to another machine's absolute path. -- Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. +- Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`/`runs.complete`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. - Legacy chat files are one-way migrated into `chat_v2/` at workspace activation and retired to `.bak`: the oldest pi-ai `Context` `chat/.json` via `migrate-chat-threads.ts` (hop 1), then the M5.6 `chat/.turns.jsonl` / `.active.json` via `migrate-chat-turns.ts` (hop 2). If hop 1 finds both formats after an interrupted launch, it completes a strict converted prefix atomically or preserves an existing tail when the full conversion is its prefix. Divergent logs are retained rather than guessed or overwritten; hop 2 skips the paired turn log while a valid same-thread legacy Context remains or its JSON cannot be read safely, so a later activation can retry both copies without blocking unrelated migrations. The obsolete `CanvasStore` chat methods and `chatPath()` helper were removed in Phase 2; `chatDir()` remains because change-review and agent-owned files still use that directory. ## 3. Storage composition and ownership @@ -69,8 +69,9 @@ Key points: | ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ports/blob.ts` | Backend-neutral `BlobStore` connection/scope contract for opaque bytes and bounded materialization leases. | | `ports/structured.ts` | Backend-neutral `StructuredStore`, the `SpaceRepository` collection, and the `SpaceHandle` composite: record read/ordered write, nodes, changes, Tasks, and events. | -| `ports/contracts/` | Reusable Space-collection, node, Space-write, log, blob, and store suites; guarantees are the minimum every adapter implements. | +| `ports/contracts/` | Reusable Space-collection, node, Space-write, Task/Run, log, blob, and store suites; guarantees are the minimum every adapter implements. | | `backends/disk/` | Disk implementations plus before-image restoration for rejected in-process ordered batches; no journal or startup recovery. | +| `backends/sqlite/` | Isolated `node:sqlite` structured adapter, strict schema/migrations, transaction-backed writes, and real-file contract/integration tests; not runtime-selectable. | | `backends/disk/legacy/` | The legacy `CanvasStore` and its synchronous adapter primitives, bounded Workspace-qualified cache, and process-local node tombstones. | | `compatibility/canvas.ts` | Residual Disk reads plus direct-module create/delete test fixtures; lifecycle writers are not exported from the public storage barrel. | | `space-lifecycle-admission.ts` | Backend-neutral, writer-preferring single-process coordinator shared by structured mutations and blob puts during a delete session. | @@ -78,9 +79,9 @@ Key points: | `index.ts` | Public exports only; application code imports here rather than reaching into an adapter. | | `canvas-store.ts`, `paths.ts`, `canvas-dirs.ts` | Deprecated forwarding shims with no logic, retained only for high-fanout compatibility imports. | -The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; a SQL adapter may use a native transaction. +The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. The SQLite adapter instead owns one explicit database filename and connection; retained handles stay bound to that connection, and its `init`, `health`, and `close` lifecycle is exercised only by direct tests. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; SQLite uses a native transaction. -Canvas persistence DTOs and the write coordinator live under `modules/canvas/`; physical Workspace paths, name indexes, directory-handle arbitration, and boot migrations live under `modules/workspace/`; generic filesystem and Markdown codecs live under `utils/`. `canvasRoot()` validates the identifier and then verifies that the resolved Space directory remains a strict descendant of the active Workspace before any downstream Disk operation receives it. `module-boundaries.test.ts` enforces the storage dependency direction and prevents new consumers of the forwarding shims. +Canvas persistence DTOs and the write coordinator live under `modules/canvas/`; active Workspace selection and boot migrations live under `modules/workspace/`; Disk record and blob layouts, name indexes, directory-handle arbitration, and World bootstrap live under `storage/backends/disk/`; generic filesystem and Markdown codecs live under `utils/`. `canvasRoot()` validates the identifier and then verifies that the resolved Space directory remains a strict descendant of the active Workspace before any downstream Disk operation receives it. `module-boundaries.test.ts` enforces the storage dependency direction and prevents new consumers of the forwarding shims. Space deletion is serialized against composed blob puts by a writer-preferring admission coordinator and holds an active-Workspace lease across blob cleanup and structured destruction. `beginDelete()` acquires the exclusive session before blob I/O; `finish()` removes structured state, while `abort()` releases the fence without doing so. Blobs are swept before structure so a failed sweep can be retried while the Space record still names them. Puts already admitted may finish; a put queued behind a successful deletion rechecks existence and fails without recreating blobs, while a failed blob sweep leaves the record available for retry. Mutations through existing Space handles and repositories reject while deletion is active or queued; reads remain available for cleanup. Residual direct-filesystem capabilities such as ZIP import, RFS upload/delete, and external-note claim are outside this repository fence and remain blockers for a non-Disk profile. @@ -104,7 +105,7 @@ The launch path deliberately has no compensation transaction. A launch failure l ### 3.3 Task Run completion -`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. The Disk adapter performs lookup, `running → completed`, and persistence under the existing per-Canvas Task mutation mutex, so HTTP and built-in-tool callers share one atomic transition rather than performing a read-then-update race. +`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. Both structured adapters perform lookup, `running → completed`, and persistence inside one Task-snapshot mutation boundary: Disk uses the per-Canvas Task mutex and atomic file replacement, while SQLite uses an immediate transaction. HTTP and built-in-tool callers therefore share one atomic transition rather than performing a read-then-update race. A completed Run stores immutable `completion.completedAt` and an optional trimmed caller-owned `completion.message`. The platform treats the message as untrusted text and does not interpret issue, pull-request, or URL semantics. A retry with the same normalized message is idempotent and preserves the original timestamp; a different message conflicts. A `pending` Run cannot complete, and Agent turn termination never implies Run completion. diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index b787dd494..37371c799 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -1,7 +1,7 @@ # Multi-Backend Storage -Status: Phases 1–4 implemented -Last updated: 2026-08-11 +Status: Phases 1–4.5 implemented; Phase 5 SQLite contract preview +Last updated: 2026-08-18 > **Scope and decision confidence.** This proposal records the two-port > `StructuredStore` / `BlobStore` split and their target backend families as @@ -48,8 +48,11 @@ Last updated: 2026-08-11 > review are recorded in place, including the CAS race ordering (§12.2.5), > log-family interface segregation (§12.2.6), and retained-handle Workspace > guards (§12.2.4). Remaining Disk-only read and physical capabilities still -> keep non-Disk profiles unselectable. No SQLite, Postgres, or Azure adapter -> exists. §12 is the +> keep non-Disk profiles unselectable. Phase 4.5 is specified in §12.5 and is +> **implemented and merged** in PR #93: storage-owned Disk layout and naming +> now live inside the storage boundary. Phase 5 adds an isolated SQLite +> structured adapter as a contract proof, but profile validation deliberately +> keeps it unselectable; no Postgres or Azure adapter exists. §12 is the > authoritative phase plan; the decision table in §2 marks what each phase > has actually settled. @@ -81,7 +84,7 @@ built above these ports, but its form is intentionally unresolved here. | Topic | Status | Current position | | ------------------------------------------------------ | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | -| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Only Disk exists. | +| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk is selectable; SQLite has an isolated contract-preview adapter but is not selectable; Postgres has no adapter. | | Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations. Only Disk exists. | | Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | | Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | @@ -135,8 +138,9 @@ external-note discovery watches `nodes/`, and export archives the entire Space directory. Therefore wrapping `CanvasStore` in a database adapter would not by itself make the application backend-neutral. -Canvas/Space persistence is currently Disk-only. SQLite, Postgres, and Azure -Blob adapters for this data do not yet exist. +Runtime Canvas/Space persistence remains Disk-only. An isolated SQLite +structured adapter exists for contract and integration tests, while Postgres +and Azure Blob adapters do not yet exist. ## 4. Goals @@ -157,7 +161,9 @@ Blob adapters for this data do not yet exist. ## 5. Non-goals -- Selecting an ORM, SQL query builder, Postgres driver, or SQLite driver. +- Selecting a production ORM, SQL query builder, Postgres driver, or final + SQLite driver. The isolated Phase 5 preview uses built-in `node:sqlite` + without making that production choice. - Defining the final relational schema or migration framework. - Choosing a VFS, FUSE, materialization, cache, or write-back design. - Replacing RFS or the canonical `SpaceQuery` / `CanvasCommand` contracts in @@ -166,9 +172,9 @@ Blob adapters for this data do not yet exist. their product semantics are defined. - Implementing online backend migration, replication, backup, or disaster recovery. -- Shipping any non-Disk adapter. The phases in §12 remove reasons why SQLite, - Postgres, and Azure _cannot_ be implemented; that is not the same as - implementing them. +- Making a non-Disk adapter runtime-selectable. Phase 5 proves an isolated + adapter against the contracts without registering it in composition or + changing product capabilities. ## 6. Settled backend split and implemented minimum contracts @@ -292,8 +298,9 @@ into place makes the failed write invisible instead of unremovable. ### 6.3 Composition -Configuration has two axes. The current shape carries only a backend kind per -axis, because no adapter yet needs more: +Configuration has two axes. The runtime-selectable profile carries only a +backend kind per axis. The isolated SQLite preview receives its explicit +database filename directly and is not constructed from this profile: ```ts interface StorageProfile { @@ -312,7 +319,8 @@ node-local DiskBlob implementation is unsafe in a multi-replica deployment unless the path is a deliberately shared and supported filesystem. SQLite on a network filesystem has different correctness and availability constraints from local SQLite. `validateStorageProfile()` is where such rules live; today it -rejects kinds that are named but not implemented, so an unsupported profile +rejects recognized kinds that are unavailable or deliberately unselectable, +including SQLite's preview-specific diagnostic, so an unsupported profile fails at startup with an actionable message rather than nondeterministically while serving data. @@ -396,7 +404,7 @@ exceptions: one names what it returns, the other opens a session. ```ts interface StructuredStore { - readonly kind: StructuredBackendKind; // 'disk' — implemented adapters only + readonly kind: StructuredBackendKind; // 'disk' | 'sqlite'; only Disk is selectable init(): Promise; health(): Promise; @@ -458,7 +466,7 @@ interface SpaceChanges { interface SpaceTasks { read(): Promise; // Tasks and Runs in one snapshot create(task: TaskRecord): Promise; - readonly runs: SpaceTaskRuns; // create(run), update(runId, patch) + readonly runs: SpaceTaskRuns; // create, update, and atomic complete } ``` @@ -655,9 +663,10 @@ explicitly: ## 12. Migration plan -Phases 1–4 are implemented and specified below. Phase 5 onward keeps the -provisional character of the original outline: those entries record intended -order, not approved designs. +Phases 1–4.5 are implemented and merged. Phase 5 is implemented by this +isolated contract preview. Phase 6 onward keeps the provisional character of +the original outline: those entries record intended order, not approved +designs. The current on-disk format remains readable throughout port extraction. A database adapter must not require Disk consumers to simulate tables, and the @@ -1519,12 +1528,15 @@ justify. footing and was left alone as Phase-1 surface. Not changed, deliberately: `authoritativeInsert` and the `write-suppressed` -put outcome remain in the portable shapes. Both exist for Disk's in-memory -deletion fence, and neither has a portable meaning a SQL adapter would -produce. They are now documented as adapter-shaped, the way `duplicate-node` -already was, rather than renamed or pushed behind the adapter — the honest -resolution needs a second adapter to say what the shared abstraction is, and -inventing one now would be the same speculative move this trim is undoing. +put outcome remain in the portable shapes. At this phase boundary, both +existed for Disk's in-memory deletion fence and a second adapter was still +needed to establish their shared meaning. + +**Superseded by Phase 5:** the SQLite contract preview supplies that second +adapter and confirms the portable rule as a connection-lifetime +anti-resurrection fence: after deletion, standalone puts are suppressed until +an ordered authoritative insert commits (§12.6.2). The outcome is therefore +no longer merely adapter-shaped. The review also asked composition to move default-title allocation ("Untitled", "Untitled (1)", …) into `create`, which would have removed the @@ -1673,10 +1685,11 @@ bearing: change-review records and Tasks are not history, whatever Disk's arrives, the group comes back — and `events` is where it was before, so nothing else has to move. -### 12.5 Phase 4.5 — storage-owned layout moves inside the boundary — **implemented** +### 12.5 Phase 4.5 — storage-owned layout moves inside the boundary — **merged** -Phase 5 adds a second structured backend. Before it does, the layout knowledge -that belongs to the _Disk_ backend has to stop living outside `storage/`. +Phase 5 would introduce a second structured backend. Before that work, the +layout knowledge that belongs to the _Disk_ backend had to stop living outside +`storage/`. Otherwise every later backend inherits a module named `disk` as the ambient description of where Spaces are, and each one pays to migrate the same callers again. @@ -1759,11 +1772,11 @@ substrate-specific but fails the test for the same reason — it exists so Windows can rename a Space _directory_ safely, and under SQLite there is no such rename. -`naming.ts` is misfiled in a different way: pure string logic with no I/O, -already re-exported rather than owned. It passes the test trivially (a second -backend needs the identical rules) but has no business behind a `disk` -segment. Phase 5 extracts it to `utils/naming.ts` as a side effect of needing -it twice; that extraction belongs here, where it is the point. +`naming.ts` was misfiled in a different way: pure string logic with no I/O, +already re-exported rather than owned. It passed the test trivially (a second +backend needs the identical rules) but had no business behind a `disk` +segment. Phase 4.5 extracted it to `utils/naming.ts`, where the shared rule has +a backend-neutral owner. Because the residue that survives the test is three setting helpers and `getWorkspacePath()` itself — none of it filesystem-specific — the target is a @@ -1840,8 +1853,9 @@ boundary test; behavior parity is asserted by the existing Disk suites, which must pass unchanged — a diff that alters a Disk test's expectations is out of scope by definition. -Phase 5 rebases onto this and drops its `utils/naming.ts` extraction, its -`workspace/disk/naming.ts` shim, and the corresponding roadmap edits. +Phase 5 builds on this merged result and carries none of its former +`utils/naming.ts` extraction, `workspace/disk/naming.ts` shim, or parallel +roadmap edits. **Landed for the Workspace-to-storage substrate move.** `modules/workspace/` is flat and holds `paths.ts` plus `migrations/`; the Disk record layout, blob @@ -1916,13 +1930,102 @@ no working behavior moves. The `latestChatTs` field and its `lastSeenThreadCursor` plumbing survive the removal because they are the resume point such a digest would need. -### 12.6 Later phases — provisional +### 12.6 Phase 5 — SQLite contract proof + +Phase 5 adds exactly one non-Disk structured adapter: SQLite. Its purpose is to +exercise the Phase 4 boundary with a backend whose atomicity and concurrency +come from a database transaction rather than from synchronous filesystem +operations. The adapter is an isolated implementation and test target; it is +not a product profile yet. This preview builds on the merged Phase 4.5 +storage-boundary work, so it uses the canonical backend-neutral naming owner +and relocated Disk adapter layout. + +#### 12.6.1 Scope and capability state + +- `StructuredBackendKind` includes `sqlite`, and profile bookkeeping + distinguishes a recognized kind, an available adapter, and a selectable + backend. Disk is all three; SQLite is recognized and available but not + selectable; Postgres is recognized only. +- `HUABU_STRUCTURED_BACKEND=sqlite` continues to fail during profile + validation with a preview-specific diagnostic. The composition root does + not construct or export the adapter. +- The adapter owns one explicit database filename and one connection. Handles + remain bound to that connection; they never consult the mutable Workspace + path. `init`, `health`, and `close` are real lifecycle operations, but host + factory registration and Workspace remounting remain a selectability gate. +- Postgres, Azure Blob, Disk-to-SQLite data movement, Agenetes persistence, + RFS/file tools, external note watching, import/export, client/API changes, + and product UI are outside this phase. + +The isolated adapter does not make issue #74 complete. It settles adapter +lifecycle and immutable connection binding, while backend-family composition +and Workspace remounting remain later integration work. + +#### 12.6.2 Schema and behavior + +The implementation uses the built-in `node:sqlite` `DatabaseSync` API, so it +adds no package or native-addon dependency. Schema versioning uses +`PRAGMA user_version`; migrations run transactionally, reject a database from +the future, and create `STRICT` tables with foreign keys enabled. + +Version 1 stores: + +- Space identity, title collision key, version, state JSON, timestamps, and + World membership; +- complete node JSON, opaque revision, and label collision key, keyed by + `(canvas_id, node_id)`; +- ordered event rows, one coalesced change snapshot per Space/thread, and one + validated Task/Run snapshot per Space; +- private delta-journal rows keyed by Space and committed version. + +Opaque domain records remain JSON. Only fields needed for current identity, +ordering, version, and collision contracts become columns. There is no search +or full-text schema in this phase. + +Every ordered Space write runs node mutations, record replacement, and the +optional delta insert in one SQLite transaction. Same-baseline writes have one +winner. A deletion session uses an adapter-local admission gate: reads remain +available, mutations reject, concurrent sessions queue, and `finish` deletes +through foreign-key cascades. No SQL transaction stays open across blob +cleanup, and no multi-process deletion fence is promised. + +Node anti-resurrection is a portable running-store guarantee. After a delete, +a standalone put is `write-suppressed` until an ordered put marked +`authoritativeInsert` commits. SQLite keeps this fence in memory; it is not a +durable tombstone and does not survive `close`/reopen. + +Title and node-label collision keys use the same pure normalization and +allocation rules as Disk without importing a Disk adapter. Canvas record +shape validation likewise has one backend-neutral owner. + +#### 12.6.3 Proof and change budget + +The reusable structured contracts run against Disk and real temporary SQLite +files. Phase 5 adds the missing reusable Tasks/Runs contract. Because the +current merged-main baseline includes atomic Run completion in the port, that +shared contract covers atomic and competing completion, idempotent repeats, +completion conflicts, and deletion fencing on both adapters. It also covers +fresh schema creation, close/reopen persistence, immutable migration fixtures, +future-version rejection, transactional migration rollback, real SQL fault +injection between record and delta writes, and same-baseline concurrency. +Mocks may not replace SQLite where serialization, transactions, or reopening +are the behavior under test. + +Expected SQLite-preview churn is 22–32 files and roughly 2,200–3,800 added +lines, primarily under server storage. More than 32 files or 4,000 added lines, +or changes to web/shared protocols or application capability owners, stops the +phase for rescoping: it means production selectability has leaked into the +contract proof. A selectable SQLite profile is a separate Phase-4-scale effort +estimated at 50–80 files and 5,000–9,000 lines. + +The completed preview is 25 files and 3,879 additions / 247 deletions against +current `main`, which contains the merged Phase 4.5 baseline. It stays inside +the budget without +touching web/shared protocols, runtime composition, or product capability +owners. + +### 12.7 Later phases — provisional -5. Add one new adapter at a time — SQLite, then Postgres, then Azure Blob — - running the same contract suites, migration fixtures, failure injection, - and concurrency tests against each. An adapter may exist for isolated - testing before its backend profile is selectable; profile validation keeps - rejecting it until the required capability matrix is satisfied. 6. Migrate the currently synchronous Agenetes persistence ports without changing their persist-before-notify, sequence, and fencing semantics. 7. Refactor RFS and built-in file tools only after a logical file-view contract @@ -2140,18 +2243,19 @@ Before a new backend is production-ready: persistence ownership, namespace, sequence, and replay invariants. - [Agenetes-Agentlet Gateway Consolidation](./agenetes-agentlet-gateway-consolidation.md) — records removal of the old Agentlet SQLite session store; it must not be - confused with the proposed SQLite structured backend. + confused with the SQLite structured contract-preview backend. ## 17. Code entry points | File/dir | Responsibility | | -------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–4 tree (§§12.1–12.4), guarded by `module-boundaries.test.ts`. | +| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–5 tree (§§12.1–12.6), guarded by `module-boundaries.test.ts`. | | [`apps/server/src/modules/storage/ports/`](../../apps/server/src/modules/storage/ports/) | The two ports; reusable suites live in `ports/contracts/`. `blob.ts` is normative (§7.1); `structured.ts` owns the Space collection and the per-Space handle: record read/write, nodes, changes, Tasks, and history. | | [`apps/server/src/modules/storage/storage.ts`](../../apps/server/src/modules/storage/storage.ts) | Composition root: maps profiles to adapters, guards blob puts, and holds a lifecycle deletion session across the blob-first cleanup saga. | | [`.../storage/backends/disk/legacy/canvas-store-cache.ts`](../../apps/server/src/modules/storage/backends/disk/legacy/canvas-store-cache.ts) | Bounded LRU of legacy Disk Space objects. The single owner both the adapter and the facade resolve through, and the real limit of `space(id)` identity (§12.2.4). | | [`apps/server/src/modules/storage/profile.ts`](../../apps/server/src/modules/storage/profile.ts) | Two-axis backend selection from env, and the fail-fast validation hook for unsupported combinations. | | [`apps/server/src/modules/storage/backends/disk/`](../../apps/server/src/modules/storage/backends/disk/) | Every Disk implementation: blob/structured stores, the Space collection, and the per-Space record, node, log, and Task adapters, in-process batch restoration, and the legacy class under `legacy/`. | +| [`apps/server/src/modules/storage/backends/sqlite/`](../../apps/server/src/modules/storage/backends/sqlite/) | Isolated `node:sqlite` structured adapter, strict schema and migrations, transaction-backed writes, and real-file contract/integration tests; available for proof but not runtime-selectable. | | [`.../storage/compatibility/canvas.ts`](../../apps/server/src/modules/storage/compatibility/canvas.ts) | Residual Disk read surface plus direct-module lifecycle test fixtures; production structured mutations enumerated in §12.4 use the portable ports. | | [`apps/server/src/modules/agent/memory/analyzer.ts`](../../apps/server/src/modules/agent/memory/analyzer.ts) | P3 repository consumer for strict Space existence, bounded action events, and intent episodes; physical chat and memory files remain Disk-specific. | | [`apps/server/src/modules/canvas/write-coordinator.ts`](../../apps/server/src/modules/canvas/write-coordinator.ts) | Canvas mutation coordinator and per-Space write lock, held across asynchronous node read, revision CAS, and put. |