refactor(native-chat): keep agent-session records in the chat journal database

.github/workflows/pr.yml

@@ -862,7 +862,6 @@ jobs:
           tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts
           tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts
           tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts
-          tests/e2e/cross-version-wire/agent-session-unproven-release-downgrade.unit.test.ts
           tests/e2e/cross-version-wire/cross-version-worktree-ps-verdict.unit.test.ts
 
   managed_hook_node18:

src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts

@@ -0,0 +1,227 @@
+// Version 4: the chat records join the journal database, copied in by the migration's own
+// transaction, so "copied" is exactly `user_version >= 4`.
+
+import { existsSync } from 'node:fs'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import Database from '../../sqlite/sync-database'
+import {
+  journalPragmaNumber,
+  NO_LEGACY_JOURNAL_RECORDS,
+  openJournalDatabase,
+  readJournalDatabaseVersion,
+  type JournalLegacyRecordImport
+} from './journal-database'
+import { createJournalTablesSql } from './journal-database-schema'
+import { journalDatabasePath } from './journal-host-database'
+
+let root: string
+let dbPath: string
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), 'orca-journal-records-migration-'))
+  dbPath = journalDatabasePath(root)
+})
+
+afterEach(async () => {
+  vi.restoreAllMocks()
+  await rm(root, { recursive: true, force: true })
+})
+
+/** What a build from before the records moved left: the released version-3 shape. */
+function seedVersion3(): void {
+  const db = new Database(dbPath)
+  db.pragma('journal_mode = WAL')
+  db.exec(createJournalTablesSql())
+  db.prepare(
+    "INSERT INTO journal_sessions (session_id, workspace_id, epoch) VALUES ('s1', 'ws', 'e1')"
+  ).run()
+  db.pragma('user_version = 3')
+  db.close()
+}
+
+/** A copy of one record row, counting how many times the migration ran it. */
+function importOf(sessionId: string): JournalLegacyRecordImport & { runs: number } {
+  const copy = {
+    owed: false as const,
+    runs: 0,
+    write: (db: Database.Database) => {
+      copy.runs += 1
+      db.prepare(
+        'INSERT OR IGNORE INTO agent_session_records (session_id, record_json) VALUES (?, ?)'
+      ).run(sessionId, '{}')
+    }
+  }
+  return copy
+}
+
+function inspect<T>(read: (db: Database.Database) => T): T {
+  const db = new Database(dbPath)
+  try {
+    return read(db)
+  } finally {
+    db.close()
+  }
+}
+
+const recordIds = (db: Database.Database): string[] =>
+  db
+    .prepare('SELECT session_id FROM agent_session_records ORDER BY session_id')
+    .all()
+    .map((row) => String(row.session_id))
+
+const hasTable = (db: Database.Database, name: string): boolean =>
+  db
+    .prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = ?")
+    .get(name) !== undefined
+
+describe('the version-4 migration', () => {
+  it('moves a version-3 database to 4 with its rows and the copied records', () => {
+    seedVersion3()
+    const copy = importOf('session-a')
+
+    const opened = openJournalDatabase(dbPath, copy)
+    opened.db.close()
+
+    expect(opened).toMatchObject({ readOnly: false, legacyRecordImportOwed: false })
+    expect(copy.runs).toBe(1)
+    inspect((db) => {
+      expect(journalPragmaNumber(db, 'user_version')).toBe(4)
+      expect(recordIds(db)).toEqual(['session-a'])
+      expect(db.prepare('SELECT epoch FROM journal_sessions').get()).toEqual({ epoch: 'e1' })
+    })
+  })
+
+  it('never runs the copy again once the database is at 4', () => {
+    seedVersion3()
+    openJournalDatabase(dbPath, importOf('session-a')).db.close()
+    const again = importOf('session-b')
+
+    openJournalDatabase(dbPath, again).db.close()
+
+    expect(again.runs).toBe(0)
+    inspect((db) => expect(recordIds(db)).toEqual(['session-a']))
+  })
+
+  it('leaves version 3 and no record rows when it dies inside the migration, then copies once', () => {
+    seedVersion3()
+    const original = Database.prototype.pragma
+    const pragma = vi.spyOn(Database.prototype, 'pragma').mockImplementation(function (
+      this: Database.Database,
+      sql: string,
+      options?: { simple?: boolean }
+    ) {
+      if (sql === 'user_version = 4') {
+        throw new Error('crash before the version is published')
+      }
+      return original.call(this, sql, options)
+    })
+
+    expect(() => openJournalDatabase(dbPath, importOf('session-a'))).toThrow(
+      'crash before the version is published'
+    )
+    pragma.mockRestore()
+    inspect((db) => {
+      expect(journalPragmaNumber(db, 'user_version')).toBe(3)
+      expect(hasTable(db, 'agent_session_records')).toBe(false)
+    })
+
+    const retry = importOf('session-a')
+    openJournalDatabase(dbPath, retry).db.close()
+    expect(retry.runs).toBe(1)
+    inspect((db) => {
+      expect(journalPragmaNumber(db, 'user_version')).toBe(4)
+      expect(recordIds(db)).toEqual(['session-a'])
+    })
+  })
+
+  it('makes the tables but keeps the copy owed when the records file could not be read', () => {
+    seedVersion3()
+
+    const opened = openJournalDatabase(dbPath, { owed: true })
+    // A chat created while the copy is owed keeps its row through the copy that follows.
+    opened.db
+      .prepare("INSERT INTO agent_session_records (session_id, record_json) VALUES ('new', '{}')")
+      .run()
+    opened.db.close()
+
+    expect(opened.legacyRecordImportOwed).toBe(true)
+    inspect((db) => expect(journalPragmaNumber(db, 'user_version')).toBe(3))
+    openJournalDatabase(dbPath, importOf('session-a')).db.close()
+    inspect((db) => {
+      expect(journalPragmaNumber(db, 'user_version')).toBe(4)
+      expect(recordIds(db)).toEqual(['new', 'session-a'])
+    })
+  })
+
+  it('stamps a fresh file with the released version while the copy is owed', () => {
+    openJournalDatabase(dbPath, { owed: true }).db.close()
+
+    inspect((db) => {
+      expect(journalPragmaNumber(db, 'user_version')).toBe(3)
+      expect(hasTable(db, 'journal_rows')).toBe(true)
+      expect(hasTable(db, 'agent_session_records')).toBe(true)
+    })
+  })
+
+  // An install's first probe must not leave an empty database that reads as a profile with chats.
+  it('reads a missing database as version 0 without creating it', () => {
+    expect(readJournalDatabaseVersion(dbPath)).toBe(0)
+    expect(existsSync(dbPath)).toBe(false)
+  })
+
+  it('creates every table on a fresh file at version 4', () => {
+    openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db.close()
+
+    inspect((db) => {
+      expect(journalPragmaNumber(db, 'user_version')).toBe(4)
+      for (const table of [
+        'journal_rows',
+        'agent_session_records',
+        'agent_session_operations',
+        'agent_session_retired_claim_keys',
+        'agent_session_tabs',
+        'agent_session_store_meta'
+      ]) {
+        expect(hasTable(db, table)).toBe(true)
+      }
+    })
+  })
+})
+
+/**
+ * The open of a build released before the records moved (version 3), pinned here: it latches any
+ * higher version read-only. Its records file stays its own and writable, so nothing it does can
+ * reach the records this database holds.
+ */
+function openAsVersion3Build(path: string): { db: Database.Database; readOnly: boolean } {
+  const probe = new Database(path)
+  const stored = journalPragmaNumber(probe, 'user_version')
+  if (stored > 3) {
+    probe.close()
+    return { db: new Database(path, { readonly: true, fileMustExist: true }), readOnly: true }
+  }
+  return { db: probe, readOnly: false }
+}
+
+describe('a build from before the move', () => {
+  it('opens a version-4 database read-only, so it never writes beside the records', () => {
+    openJournalDatabase(dbPath, importOf('session-a')).db.close()
+
+    const older = openAsVersion3Build(dbPath)
+    try {
+      expect(older.readOnly).toBe(true)
+      expect(() =>
+        older.db
+          .prepare(
+            "INSERT INTO journal_sessions (session_id, workspace_id, epoch) VALUES ('s2', 'ws', 'e')"
+          )
+          .run()
+      ).toThrow(/readonly/i)
+    } finally {
+      older.db.close()
+    }
+  })
+})

src/main/native-chat/agent-session-journal/journal-database-schema.ts

@@ -6,12 +6,16 @@
 // for a rebuild a partial repair leaves behind (see journal-repair-marker.ts). `journal_imports`
 // records which per-chat file each chat was copied from (journal-per-session-reimport.ts), and
 // `journal_set_aside` each chat whose per-chat file is not this build's history and is never read
-// again.
+// again. The `agent_session_*` tables hold each chat's ownership record, its operation ledger, the
+// retired claim keys and the chat tab index (agent-session-record-rows.ts).
 
 /** DB shape version, carried in `PRAGMA user_version`. Independent of the row body version
  *  (`JournalRow.v`): a newer build can change either alone. A newer version opens read-only here,
- *  so every change stays additive. Versions below 3 were never released (journal-database.ts). */
-export const JOURNAL_DB_SCHEMA_VERSION = 3
+ *  so every change stays additive. */
+export const JOURNAL_DB_SCHEMA_VERSION = 4
+
+/** The first version a release wrote; 1 and 2 only ever came from development builds. */
+export const JOURNAL_DB_OLDEST_RELEASED_VERSION = 3
 
 export function createJournalTablesSql(): string {
   return `
@@ -46,3 +50,31 @@ CREATE TABLE IF NOT EXISTS journal_set_aside (
 );
 `
 }
+
+/** Version 4: the chat records, until then a JSON file beside the database. Each row is one JSON
+ *  value as the file held it, so a row this build cannot read stays byte-identical. */
+export function createAgentSessionRecordTablesSql(): string {
+  return `
+CREATE TABLE IF NOT EXISTS agent_session_records (
+  session_id  TEXT PRIMARY KEY,
+  record_json TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS agent_session_operations (
+  operation_key TEXT PRIMARY KEY,
+  row_json      TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS agent_session_retired_claim_keys (
+  key_id     TEXT PRIMARY KEY,
+  retired_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS agent_session_tabs (
+  tab_id     TEXT PRIMARY KEY,
+  session_id TEXT    NOT NULL,
+  position   INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS agent_session_store_meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+`
+}

src/main/native-chat/agent-session-journal/journal-database.test.ts

@@ -8,6 +8,7 @@ import {
   JOURNAL_BUSY_TIMEOUT_MS,
   JOURNAL_SIZE_LIMIT_BYTES,
   journalPragmaNumber,
+  NO_LEGACY_JOURNAL_RECORDS,
   openJournalDatabase
 } from './journal-database'
 import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema'
@@ -66,7 +67,7 @@ afterEach(async () => {
 
 describe('the host journal database open', () => {
   it('creates every table and reads back every load-bearing pragma', () => {
-    const db = openJournalDatabase(dbPath).db
+    const db = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db
     try {
       const tables = db
         .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")
@@ -97,14 +98,14 @@ describe('the host journal database open', () => {
 
   // T5: a newer build's database opens read-only, its rows readable, and is left byte-identical.
   it('opens a newer user_version read-only without touching the file', async () => {
-    const seeded = openJournalDatabase(dbPath).db
+    const seeded = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db
     publishJournalSessionEpoch(seeded, SESSION, 'epoch-1')
     insertJournalRow(seeded, 'session-1', epochRow(1))
     seeded.pragma(`user_version = ${JOURNAL_DB_SCHEMA_VERSION + 5}`)
     seeded.close()
     const before = await digest(dbPath)
 
-    const opened = openJournalDatabase(dbPath)
+    const opened = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS)
     try {
       expect(opened.readOnly).toBe(true)
       expect(readJournalSessionEpoch(opened.db, 'session-1')).toBe('epoch-1')
@@ -121,7 +122,7 @@ describe('the host journal database open', () => {
 
   it('closes the raw connection when schema setup throws', async () => {
     const failing = join(root, 'nested', 'agent-session-journal.db')
-    expect(() => openJournalDatabase(failing)).toThrow()
+    expect(() => openJournalDatabase(failing, NO_LEGACY_JOURNAL_RECORDS)).toThrow()
     await expect(stat(`${failing}-wal`)).rejects.toThrow()
     await expect(rm(root, { recursive: true, force: true })).resolves.toBeUndefined()
     root = await mkdtemp(join(tmpdir(), 'orca-journal-db-'))
@@ -130,7 +131,7 @@ describe('the host journal database open', () => {
 
 describe('journal row statements', () => {
   it('serves replay, resume, suffix truncation and an epoch discard', () => {
-    const db = openJournalDatabase(dbPath).db
+    const db = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db
     try {
       db.exec('BEGIN IMMEDIATE')
       for (let seq = 1; seq <= 5; seq += 1) {
@@ -161,7 +162,7 @@ describe('journal row statements', () => {
   })
 
   it('deletes only the rows no pointer names', () => {
-    const db = openJournalDatabase(dbPath).db
+    const db = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db
     try {
       insertJournalRow(db, 'session-1', epochRow(1, 'epoch-copying'))
       insertJournalRow(db, 'session-2', epochRow(1, 'epoch-live'))
@@ -180,7 +181,7 @@ describe('journal row statements', () => {
   })
 
   it('refuses a duplicate sequence inside one epoch of one chat', () => {
-    const db = openJournalDatabase(dbPath).db
+    const db = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db
     try {
       insertJournalRow(db, 'session-1', epochRow(1))
       expect(() => insertJournalRow(db, 'session-1', epochRow(1))).toThrow()
@@ -192,7 +193,7 @@ describe('journal row statements', () => {
   })
 
   it('moves the epoch pointer in place', () => {
-    const db = openJournalDatabase(dbPath).db
+    const db = openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db
     try {
       publishJournalSessionEpoch(db, SESSION, 'epoch-1')
       publishJournalSessionEpoch(db, SESSION, 'epoch-2')
@@ -220,7 +221,9 @@ describe('schema creation', () => {
       return original.call(this, sql, options)
     })
 
-    expect(() => openJournalDatabase(dbPath)).toThrow('crash before the version is published')
+    expect(() => openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS)).toThrow(
+      'crash before the version is published'
+    )
     pragma.mockRestore()
 
     const inspected = new Database(dbPath)
@@ -250,8 +253,12 @@ INSERT INTO journal_sessions VALUES ('s1', 'ws', 'e1', 0, NULL, NULL);`)
     earlier.close()
     const before = await digest(dbPath)
 
-    expect(() => openJournalDatabase(dbPath)).toThrow(`unreleased schema ${version}`)
-    expect(() => openJournalDatabase(dbPath)).toThrow(JournalUnreleasedSchemaError)
+    expect(() => openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS)).toThrow(
+      `unreleased schema ${version}`
+    )
+    expect(() => openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS)).toThrow(
+      JournalUnreleasedSchemaError
+    )
 
     expect(await digest(dbPath)).toBe(before)
     await expect(stat(`${dbPath}-wal`)).rejects.toThrow()