diff --git a/apps/app/src/app/lib/openwork-server.ts b/apps/app/src/app/lib/openwork-server.ts index ab38697656..46081d2ab8 100644 --- a/apps/app/src/app/lib/openwork-server.ts +++ b/apps/app/src/app/lib/openwork-server.ts @@ -204,7 +204,16 @@ export type OpenworkRuntimeConfigStatus = { sources?: { projectOpencode: { path: string; exists: boolean; keys: string[]; config: Record }; globalOpencode: { path: string; exists: boolean; keys: string[]; config: Record }; - runtimeDatabase: { keys: string[]; config: Record }; + runtimeDatabase: { + keys: string[]; + config: Record; + database?: { + path: string; + schemaVersion: number; + migrations: { version: number; name: string; appliedAt: number }[]; + tables: Record; + }; + }; injected: { keys: string[]; config: Record }; }; legacyOpenwork: { diff --git a/apps/app/src/react-app/domains/settings/pages/advanced-view-sections.tsx b/apps/app/src/react-app/domains/settings/pages/advanced-view-sections.tsx index 7c47387fae..e180b93561 100644 --- a/apps/app/src/react-app/domains/settings/pages/advanced-view-sections.tsx +++ b/apps/app/src/react-app/domains/settings/pages/advanced-view-sections.tsx @@ -186,6 +186,7 @@ function RuntimeConfigSourceBlock(props: { exists?: boolean; keys: string[]; config: Record; + details?: ReactNode; }) { return (
@@ -195,6 +196,7 @@ function RuntimeConfigSourceBlock(props: { {props.path ?
{props.path}
: null} {props.exists !== undefined ?
{props.exists ? "Found" : "Not found"}
: null}
Keys: {formatKeys(props.keys)}
+ {props.details}
@@ -292,6 +294,22 @@ export function AdvancedRuntimeMigrationSection(props: AdvancedRuntimeMigrationS description="OpenWork-managed runtime values stored outside workspace files." keys={props.configStatus.sources.runtimeDatabase.keys} config={props.configStatus.sources.runtimeDatabase.config} + details={props.configStatus.sources.runtimeDatabase.database ? ( +
+
{props.configStatus.sources.runtimeDatabase.database.path}
+
Schema version: {props.configStatus.sources.runtimeDatabase.database.schemaVersion}
+
+ Tables: {Object.entries(props.configStatus.sources.runtimeDatabase.database.tables) + .map(([name, count]) => `${name}: ${count}`) + .join(", ")} +
+
+ Migrations: {props.configStatus.sources.runtimeDatabase.database.migrations + .map((migration) => `${migration.version} ${migration.name}`) + .join(", ") || "none"} +
+
+ ) : null} /> { configJson: string } | undefined; - upsert: (value: { workspaceId: string; configJson: string; updatedAt: number }) => void; -}; function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); @@ -24,71 +9,18 @@ function normalizeOpenworkWorkspaceConfig(value: unknown): Record { - await ensureDir(dirname(path)); - if (typeof process.versions.bun === "string") { - const { Database } = await import("bun:sqlite"); - const { drizzle } = await import("drizzle-orm/bun-sqlite"); - const sqlite = new Database(path, { create: true }); - sqlite.run("CREATE TABLE IF NOT EXISTS openwork_workspace_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); - const db = drizzle(sqlite); - return { - get: (workspaceId) => db - .select() - .from(openworkWorkspaceConfigs) - .where(eq(openworkWorkspaceConfigs.workspaceId, workspaceId)) - .get(), - upsert: ({ workspaceId, configJson, updatedAt }) => { - db - .insert(openworkWorkspaceConfigs) - .values({ workspaceId, configJson, updatedAt }) - .onConflictDoUpdate({ - target: openworkWorkspaceConfigs.workspaceId, - set: { configJson, updatedAt }, - }) - .run(); - }, - }; +function parseOpenworkWorkspaceConfigJson(value: string | undefined): Record { + if (!value) return {}; + try { + return normalizeOpenworkWorkspaceConfig(JSON.parse(value)); + } catch { + return {}; } - const { DatabaseSync } = await import("node:sqlite"); - const sqlite = new DatabaseSync(path); - sqlite.exec("CREATE TABLE IF NOT EXISTS openwork_workspace_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); - const get = sqlite.prepare("SELECT config_json AS configJson FROM openwork_workspace_configs WHERE workspace_id = ?"); - const upsert = sqlite.prepare("INSERT INTO openwork_workspace_configs (workspace_id, config_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(workspace_id) DO UPDATE SET config_json = excluded.config_json, updated_at = excluded.updated_at"); - return { - get: (workspaceId) => { - const row = get.get(workspaceId); - if (!isRecord(row) || typeof row.configJson !== "string") return undefined; - return { configJson: row.configJson }; - }, - upsert: ({ workspaceId, configJson, updatedAt }) => { - upsert.run(workspaceId, configJson, updatedAt); - }, - }; -} - -const dbByPath = new Map>(); - -async function workspaceConfigDb(config: ServerConfig): Promise { - const path = runtimeDbPath(config); - const existing = dbByPath.get(path); - if (existing) return existing; - const db = openDb(path); - dbByPath.set(path, db); - return db; } export async function readOpenworkWorkspaceConfig(config: ServerConfig, workspaceId: string): Promise> { - const db = await workspaceConfigDb(config); - const row = db.get(workspaceId); + const db = await openRuntimeDb(config); + const row = db.getJsonRow("openwork_workspace_configs", workspaceId); if (!row) return {}; try { return normalizeOpenworkWorkspaceConfig(JSON.parse(row.configJson)); @@ -102,10 +34,11 @@ export async function writeOpenworkWorkspaceConfig( workspaceId: string, updater: (current: Record) => Record, ): Promise> { - const db = await workspaceConfigDb(config); - const next = normalizeOpenworkWorkspaceConfig(updater(await readOpenworkWorkspaceConfig(config, workspaceId))); - db.upsert({ workspaceId, configJson: JSON.stringify(next), updatedAt: Date.now() }); - return next; + const configJson = await updateRuntimeJsonRow(config, "openwork_workspace_configs", workspaceId, (currentJson) => { + const current = parseOpenworkWorkspaceConfigJson(currentJson); + return JSON.stringify(normalizeOpenworkWorkspaceConfig(updater(current))); + }); + return parseOpenworkWorkspaceConfigJson(configJson); } export function mergeOpenworkWorkspaceConfigs( diff --git a/apps/server/src/runtime-db-schema.ts b/apps/server/src/runtime-db-schema.ts new file mode 100644 index 0000000000..d89c86704e --- /dev/null +++ b/apps/server/src/runtime-db-schema.ts @@ -0,0 +1,35 @@ +import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; + +export const schemaMigrations = sqliteTable("schema_migrations", { + version: integer("version").primaryKey(), + name: text("name").notNull(), + appliedAt: integer("applied_at").notNull(), +}); + +export const migrationState = sqliteTable("migration_state", { + source: text("source").primaryKey(), + status: text("status").notNull(), + path: text("path").notNull().default(""), + hash: text("hash").notNull().default(""), + rowCount: integer("row_count").notNull().default(0), + importedAt: integer("imported_at").notNull(), +}); + +export const runtimeOpencodeConfigs = sqliteTable("runtime_opencode_configs", { + workspaceId: text("workspace_id").primaryKey(), + configJson: text("config_json").notNull(), + updatedAt: integer("updated_at").notNull(), +}); + +export const openworkWorkspaceConfigs = sqliteTable("openwork_workspace_configs", { + workspaceId: text("workspace_id").primaryKey(), + configJson: text("config_json").notNull(), + updatedAt: integer("updated_at").notNull(), +}); + +export const runtimeDbSchema = { + schemaMigrations, + migrationState, + runtimeOpencodeConfigs, + openworkWorkspaceConfigs, +}; diff --git a/apps/server/src/runtime-db.test.ts b/apps/server/src/runtime-db.test.ts new file mode 100644 index 0000000000..d9a0faa49b --- /dev/null +++ b/apps/server/src/runtime-db.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { addMcp } from "./mcp.js"; +import { readOpenworkWorkspaceConfig, writeOpenworkWorkspaceConfig } from "./openwork-workspace-config-store.js"; +import { addPlugin } from "./plugins.js"; +import { getRuntimeDbDiagnostics } from "./runtime-db.js"; +import { readRuntimeOpencodeConfig, writeRuntimeOpencodeConfig } from "./runtime-opencode-config-store.js"; +import type { ServerConfig } from "./types.js"; + +const WORKSPACE_ID = "ws_runtime_db_test"; + +function serverConfig(root: string): ServerConfig { + return { + host: "127.0.0.1", + port: 0, + token: "token", + hostToken: "host-token", + configPath: join(root, "server.json"), + approval: { mode: "auto", timeoutMs: 0 }, + corsOrigins: [], + workspaces: [{ id: WORKSPACE_ID, name: "Test", path: root, preset: "starter", workspaceType: "local" }], + authorizedRoots: [root], + readOnly: false, + startedAt: Date.now(), + tokenSource: "generated", + hostTokenSource: "generated", + logFormat: "pretty", + logRequests: false, + } satisfies ServerConfig; +} + +async function withRuntimeDb(fn: (input: { root: string; dbPath: string; config: ServerConfig }) => Promise) { + const root = await mkdtemp(join(tmpdir(), "openwork-runtime-db-")); + const previousDb = process.env.OPENWORK_RUNTIME_DB; + const dbPath = join(root, "runtime.sqlite"); + process.env.OPENWORK_RUNTIME_DB = dbPath; + try { + await fn({ root, dbPath, config: serverConfig(root) }); + } finally { + if (previousDb === undefined) delete process.env.OPENWORK_RUNTIME_DB; + else process.env.OPENWORK_RUNTIME_DB = previousDb; + await rm(root, { recursive: true, force: true }); + } +} + +function openSqlite(path: string): Database { + return new Database(path, { create: true }); +} + +describe("runtime DB", () => { + test("creates shared schema and records the runtime schema migration", async () => { + await withRuntimeDb(async ({ config }) => { + await addPlugin(config, WORKSPACE_ID, "runtime-plugin"); + await writeOpenworkWorkspaceConfig(config, WORKSPACE_ID, () => ({ cloudImports: { plugins: {} } })); + + const diagnostics = await getRuntimeDbDiagnostics(config); + + expect(diagnostics.schemaVersion).toBe(1); + expect(diagnostics.migrations).toMatchObject([{ version: 1, name: "initial_runtime_store" }]); + expect(diagnostics.tables.runtime_opencode_configs).toBe(1); + expect(diagnostics.tables.openwork_workspace_configs).toBe(1); + expect(diagnostics.tables.migration_state).toBe(0); + }); + }); + + test("opens an existing runtime DB with the old tables without dropping data", async () => { + await withRuntimeDb(async ({ dbPath, config }) => { + const sqlite = openSqlite(dbPath); + sqlite.run("CREATE TABLE runtime_opencode_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); + sqlite.run("CREATE TABLE openwork_workspace_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); + sqlite.run( + "INSERT INTO runtime_opencode_configs (workspace_id, config_json, updated_at) VALUES (?, ?, ?)", + [WORKSPACE_ID, JSON.stringify({ plugin: ["legacy-runtime-plugin"] }), Date.now()], + ); + sqlite.close(); + + expect((await readRuntimeOpencodeConfig(config, WORKSPACE_ID)).plugin).toEqual(["legacy-runtime-plugin"]); + const diagnostics = await getRuntimeDbDiagnostics(config); + + expect(diagnostics.schemaVersion).toBe(1); + expect(diagnostics.tables.runtime_opencode_configs).toBe(1); + expect(diagnostics.tables.openwork_workspace_configs).toBe(0); + }); + }); + + test("serializes runtime config updates so unrelated keys are preserved", async () => { + await withRuntimeDb(async ({ config }) => { + await Promise.all([ + addPlugin(config, WORKSPACE_ID, "runtime-plugin"), + addMcp(config, WORKSPACE_ID, "runtime-mcp", { type: "remote", url: "https://runtime.example/mcp" }), + writeRuntimeOpencodeConfig(config, WORKSPACE_ID, (current) => ({ + ...current, + provider: { runtimeProvider: { npm: "@openwork/runtime-provider" } }, + })), + ]); + + const runtime = await readRuntimeOpencodeConfig(config, WORKSPACE_ID); + + expect(runtime.plugin).toEqual(["runtime-plugin"]); + expect(runtime.mcp?.["runtime-mcp"]?.url).toBe("https://runtime.example/mcp"); + expect(runtime.provider?.runtimeProvider).toEqual({ npm: "@openwork/runtime-provider" }); + }); + }); + + test("malformed stored JSON still reads as empty config", async () => { + await withRuntimeDb(async ({ dbPath, config }) => { + const sqlite = openSqlite(dbPath); + sqlite.run("CREATE TABLE runtime_opencode_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); + sqlite.run("INSERT INTO runtime_opencode_configs (workspace_id, config_json, updated_at) VALUES (?, ?, ?)", [WORKSPACE_ID, "{ invalid", Date.now()]); + sqlite.close(); + + expect(await readRuntimeOpencodeConfig(config, WORKSPACE_ID)).toEqual({}); + await addPlugin(config, WORKSPACE_ID, "runtime-plugin"); + expect((await readRuntimeOpencodeConfig(config, WORKSPACE_ID)).plugin).toEqual(["runtime-plugin"]); + }); + }); + + test("workspace config updates share the same DB and preserve existing JSON keys", async () => { + await withRuntimeDb(async ({ config }) => { + await writeOpenworkWorkspaceConfig(config, WORKSPACE_ID, () => ({ cloudImports: { plugins: { plugin_1: { name: "one" } } } })); + await writeOpenworkWorkspaceConfig(config, WORKSPACE_ID, (current) => ({ ...current, desktopCloudSync: { fetchedAt: 123 } })); + + expect(await readOpenworkWorkspaceConfig(config, WORKSPACE_ID)).toEqual({ + cloudImports: { plugins: { plugin_1: { name: "one" } } }, + desktopCloudSync: { fetchedAt: 123 }, + }); + }); + }); +}); diff --git a/apps/server/src/runtime-db.ts b/apps/server/src/runtime-db.ts new file mode 100644 index 0000000000..c7128f5a75 --- /dev/null +++ b/apps/server/src/runtime-db.ts @@ -0,0 +1,240 @@ +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { eq } from "drizzle-orm"; +import type { BunSQLiteDatabase } from "drizzle-orm/bun-sqlite"; +import type { ServerConfig } from "./types.js"; +import { ensureDir } from "./utils.js"; +import { + openworkWorkspaceConfigs, + runtimeDbSchema, + runtimeOpencodeConfigs, + schemaMigrations, +} from "./runtime-db-schema.js"; + +const CURRENT_SCHEMA_VERSION = 1; +const INITIAL_SCHEMA_NAME = "initial_runtime_store"; + +type RuntimeJsonTable = "runtime_opencode_configs" | "openwork_workspace_configs"; +type RuntimeJsonRow = { configJson: string }; +type RuntimeMigrationRow = { version: number; name: string; appliedAt: number }; + +type RuntimeDb = { + path: string; + getJsonRow: (table: RuntimeJsonTable, workspaceId: string) => RuntimeJsonRow | undefined; + updateJsonRow: ( + table: RuntimeJsonTable, + workspaceId: string, + updater: (currentJson: string | undefined) => string, + ) => string; + diagnostics: () => RuntimeDbDiagnostics; +}; + +export type RuntimeDbDiagnostics = { + path: string; + schemaVersion: number; + migrations: RuntimeMigrationRow[]; + tables: Record; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function runtimeDbPath(config: ServerConfig): string { + const override = process.env.OPENWORK_RUNTIME_DB?.trim(); + if (override) return resolve(override); + const configPath = config.configPath?.trim(); + const configDir = configPath ? dirname(configPath) : join(homedir(), ".config", "openwork"); + return join(configDir, "runtime.sqlite"); +} + +function createSchemaSql(): string { + return [ + "CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL, applied_at INTEGER NOT NULL)", + "CREATE TABLE IF NOT EXISTS migration_state (source TEXT PRIMARY KEY NOT NULL, status TEXT NOT NULL, path TEXT NOT NULL DEFAULT '', hash TEXT NOT NULL DEFAULT '', row_count INTEGER NOT NULL DEFAULT 0, imported_at INTEGER NOT NULL)", + "CREATE TABLE IF NOT EXISTS runtime_opencode_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)", + "CREATE TABLE IF NOT EXISTS openwork_workspace_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)", + ].join("; "); +} + +function rowCountSql(table: RuntimeJsonTable | "migration_state"): string { + return `SELECT COUNT(1) AS count FROM ${table}`; +} + +function getSchemaVersion(migrations: RuntimeMigrationRow[]): number { + return migrations.reduce((max, row) => Math.max(max, row.version), 0); +} + +function normalizeMigrationRow(value: unknown): RuntimeMigrationRow | undefined { + if (!isRecord(value)) return undefined; + if (typeof value.version !== "number" || typeof value.name !== "string" || typeof value.appliedAt !== "number") { + return undefined; + } + return { version: value.version, name: value.name, appliedAt: value.appliedAt }; +} + +async function openBunRuntimeDb(path: string): Promise { + const { Database } = await import("bun:sqlite"); + const { drizzle } = await import("drizzle-orm/bun-sqlite"); + const sqlite = new Database(path, { create: true }); + sqlite.run("PRAGMA foreign_keys = ON"); + sqlite.run("PRAGMA journal_mode = WAL"); + sqlite.exec(createSchemaSql()); + const db: BunSQLiteDatabase = drizzle(sqlite, { schema: runtimeDbSchema }); + db.insert(schemaMigrations) + .values({ version: CURRENT_SCHEMA_VERSION, name: INITIAL_SCHEMA_NAME, appliedAt: Date.now() }) + .onConflictDoNothing() + .run(); + + const tableSchema = { + runtime_opencode_configs: runtimeOpencodeConfigs, + openwork_workspace_configs: openworkWorkspaceConfigs, + }; + + return { + path, + getJsonRow: (table, workspaceId) => db + .select({ configJson: tableSchema[table].configJson }) + .from(tableSchema[table]) + .where(eq(tableSchema[table].workspaceId, workspaceId)) + .get(), + updateJsonRow: (table, workspaceId, updater) => sqlite.transaction(() => { + const current = db + .select({ configJson: tableSchema[table].configJson }) + .from(tableSchema[table]) + .where(eq(tableSchema[table].workspaceId, workspaceId)) + .get(); + const configJson = updater(current?.configJson); + db.insert(tableSchema[table]) + .values({ workspaceId, configJson, updatedAt: Date.now() }) + .onConflictDoUpdate({ + target: tableSchema[table].workspaceId, + set: { configJson, updatedAt: Date.now() }, + }) + .run(); + return configJson; + })(), + diagnostics: () => { + const migrations = db.select().from(schemaMigrations).all(); + const count = (table: RuntimeJsonTable | "migration_state") => { + const row = sqlite.query(rowCountSql(table)).get(); + return isRecord(row) && typeof row.count === "number" ? row.count : 0; + }; + return { + path, + schemaVersion: getSchemaVersion(migrations), + migrations, + tables: { + runtime_opencode_configs: count("runtime_opencode_configs"), + openwork_workspace_configs: count("openwork_workspace_configs"), + migration_state: count("migration_state"), + }, + }; + }, + }; +} + +async function openNodeRuntimeDb(path: string): Promise { + const { DatabaseSync } = await import("node:sqlite"); + const sqlite = new DatabaseSync(path); + sqlite.exec("PRAGMA foreign_keys = ON"); + sqlite.exec("PRAGMA journal_mode = WAL"); + sqlite.exec(createSchemaSql()); + sqlite + .prepare("INSERT OR IGNORE INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)") + .run(CURRENT_SCHEMA_VERSION, INITIAL_SCHEMA_NAME, Date.now()); + + const selectStatements = { + runtime_opencode_configs: sqlite.prepare("SELECT config_json AS configJson FROM runtime_opencode_configs WHERE workspace_id = ?"), + openwork_workspace_configs: sqlite.prepare("SELECT config_json AS configJson FROM openwork_workspace_configs WHERE workspace_id = ?"), + }; + const upsertStatements = { + runtime_opencode_configs: sqlite.prepare("INSERT INTO runtime_opencode_configs (workspace_id, config_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(workspace_id) DO UPDATE SET config_json = excluded.config_json, updated_at = excluded.updated_at"), + openwork_workspace_configs: sqlite.prepare("INSERT INTO openwork_workspace_configs (workspace_id, config_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(workspace_id) DO UPDATE SET config_json = excluded.config_json, updated_at = excluded.updated_at"), + }; + + function getJsonRow(table: RuntimeJsonTable, workspaceId: string): RuntimeJsonRow | undefined { + const row = selectStatements[table].get(workspaceId); + if (!isRecord(row) || typeof row.configJson !== "string") return undefined; + return { configJson: row.configJson }; + } + + return { + path, + getJsonRow, + updateJsonRow: (table, workspaceId, updater) => { + sqlite.exec("BEGIN IMMEDIATE"); + try { + const configJson = updater(getJsonRow(table, workspaceId)?.configJson); + upsertStatements[table].run(workspaceId, configJson, Date.now()); + sqlite.exec("COMMIT"); + return configJson; + } catch (error) { + sqlite.exec("ROLLBACK"); + throw error; + } + }, + diagnostics: () => { + const migrations = sqlite + .prepare("SELECT version, name, applied_at AS appliedAt FROM schema_migrations ORDER BY version") + .all() + .map(normalizeMigrationRow) + .filter((row): row is RuntimeMigrationRow => row !== undefined); + const count = (table: RuntimeJsonTable | "migration_state") => { + const row = sqlite.prepare(rowCountSql(table)).get(); + return isRecord(row) && typeof row.count === "number" ? row.count : 0; + }; + return { + path, + schemaVersion: getSchemaVersion(migrations), + migrations, + tables: { + runtime_opencode_configs: count("runtime_opencode_configs"), + openwork_workspace_configs: count("openwork_workspace_configs"), + migration_state: count("migration_state"), + }, + }; + }, + }; +} + +const dbByPath = new Map>(); +const updateQueueByPath = new Map>(); + +export async function openRuntimeDb(config: ServerConfig): Promise { + const path = runtimeDbPath(config); + const existing = dbByPath.get(path); + if (existing) return existing; + await ensureDir(dirname(path)); + const db = typeof process.versions.bun === "string" ? openBunRuntimeDb(path) : openNodeRuntimeDb(path); + dbByPath.set(path, db); + return db; +} + +export async function updateRuntimeJsonRow( + config: ServerConfig, + table: RuntimeJsonTable, + workspaceId: string, + updater: (currentJson: string | undefined) => string, +): Promise { + const db = await openRuntimeDb(config); + const previous = updateQueueByPath.get(db.path) ?? Promise.resolve(); + const waitForPrevious = previous.catch(() => {}); + let release: () => void = () => {}; + const nextQueue = new Promise((resolveQueue) => { + release = resolveQueue; + }); + const queued = waitForPrevious.then(() => nextQueue); + updateQueueByPath.set(db.path, queued); + await waitForPrevious; + try { + return db.updateJsonRow(table, workspaceId, updater); + } finally { + release(); + if (updateQueueByPath.get(db.path) === queued) updateQueueByPath.delete(db.path); + } +} + +export async function getRuntimeDbDiagnostics(config: ServerConfig): Promise { + return (await openRuntimeDb(config)).diagnostics(); +} diff --git a/apps/server/src/runtime-opencode-config-store.ts b/apps/server/src/runtime-opencode-config-store.ts index 3bdd3671f8..e63b3282f8 100644 --- a/apps/server/src/runtime-opencode-config-store.ts +++ b/apps/server/src/runtime-opencode-config-store.ts @@ -1,9 +1,5 @@ -import { homedir } from "node:os"; -import { dirname, join, resolve } from "node:path"; -import { eq } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { openRuntimeDb, updateRuntimeJsonRow } from "./runtime-db.js"; import type { ServerConfig } from "./types.js"; -import { ensureDir } from "./utils.js"; export type RuntimeOpencodeConfig = { default_agent?: string; @@ -16,17 +12,6 @@ export type RuntimeOpencodeConfig = { provider?: Record; }; -const runtimeOpencodeConfigs = sqliteTable("runtime_opencode_configs", { - workspaceId: text("workspace_id").primaryKey(), - configJson: text("config_json").notNull(), - updatedAt: integer("updated_at").notNull(), -}); - -type RuntimeOpencodeDb = { - get: (workspaceId: string) => { configJson: string } | undefined; - upsert: (value: { workspaceId: string; configJson: string; updatedAt: number }) => void; -}; - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -52,66 +37,13 @@ function normalizeRuntimeOpencodeConfig(value: unknown): RuntimeOpencodeConfig { }; } -function runtimeDbPath(config: ServerConfig): string { - const override = process.env.OPENWORK_RUNTIME_DB?.trim(); - if (override) return resolve(override); - const configPath = config.configPath?.trim(); - const configDir = configPath ? dirname(configPath) : join(homedir(), ".config", "openwork"); - return join(configDir, "runtime.sqlite"); -} - -async function openRuntimeDb(path: string): Promise { - await ensureDir(dirname(path)); - if (typeof process.versions.bun === "string") { - const { Database } = await import("bun:sqlite"); - const { drizzle } = await import("drizzle-orm/bun-sqlite"); - const sqlite = new Database(path, { create: true }); - sqlite.run("CREATE TABLE IF NOT EXISTS runtime_opencode_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); - const db = drizzle(sqlite); - return { - get: (workspaceId) => db - .select() - .from(runtimeOpencodeConfigs) - .where(eq(runtimeOpencodeConfigs.workspaceId, workspaceId)) - .get(), - upsert: ({ workspaceId, configJson, updatedAt }) => { - db - .insert(runtimeOpencodeConfigs) - .values({ workspaceId, configJson, updatedAt }) - .onConflictDoUpdate({ - target: runtimeOpencodeConfigs.workspaceId, - set: { configJson, updatedAt }, - }) - .run(); - }, - }; +function parseRuntimeOpencodeConfigJson(value: string | undefined): RuntimeOpencodeConfig { + if (!value) return {}; + try { + return normalizeRuntimeOpencodeConfig(JSON.parse(value)); + } catch { + return {}; } - const { DatabaseSync } = await import("node:sqlite"); - const sqlite = new DatabaseSync(path); - sqlite.exec("CREATE TABLE IF NOT EXISTS runtime_opencode_configs (workspace_id TEXT PRIMARY KEY NOT NULL, config_json TEXT NOT NULL, updated_at INTEGER NOT NULL)"); - const get = sqlite.prepare("SELECT config_json AS configJson FROM runtime_opencode_configs WHERE workspace_id = ?"); - const upsert = sqlite.prepare("INSERT INTO runtime_opencode_configs (workspace_id, config_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(workspace_id) DO UPDATE SET config_json = excluded.config_json, updated_at = excluded.updated_at"); - return { - get: (workspaceId) => { - const row = get.get(workspaceId); - if (!isRecord(row) || typeof row.configJson !== "string") return undefined; - return { configJson: row.configJson }; - }, - upsert: ({ workspaceId, configJson, updatedAt }) => { - upsert.run(workspaceId, configJson, updatedAt); - }, - }; -} - -const dbByPath = new Map>(); - -async function runtimeDb(config: ServerConfig): Promise { - const path = runtimeDbPath(config); - const existing = dbByPath.get(path); - if (existing) return existing; - const db = openRuntimeDb(path); - dbByPath.set(path, db); - return db; } export function runtimePluginList(config: RuntimeOpencodeConfig): string[] { @@ -135,8 +67,8 @@ export function runtimeExternalDirectory(config: RuntimeOpencodeConfig): Record< } export async function readRuntimeOpencodeConfig(config: ServerConfig, workspaceId: string): Promise { - const db = await runtimeDb(config); - const row = db.get(workspaceId); + const db = await openRuntimeDb(config); + const row = db.getJsonRow("runtime_opencode_configs", workspaceId); if (!row) return {}; try { return normalizeRuntimeOpencodeConfig(JSON.parse(row.configJson)); @@ -150,12 +82,11 @@ export async function writeRuntimeOpencodeConfig( workspaceId: string, updater: (current: RuntimeOpencodeConfig) => RuntimeOpencodeConfig, ): Promise { - const db = await runtimeDb(config); - const next = normalizeRuntimeOpencodeConfig(updater(await readRuntimeOpencodeConfig(config, workspaceId))); - const now = Date.now(); - const configJson = JSON.stringify(next); - db.upsert({ workspaceId, configJson, updatedAt: now }); - return next; + const configJson = await updateRuntimeJsonRow(config, "runtime_opencode_configs", workspaceId, (currentJson) => { + const current = parseRuntimeOpencodeConfigJson(currentJson); + return JSON.stringify(normalizeRuntimeOpencodeConfig(updater(current))); + }); + return parseRuntimeOpencodeConfigJson(configJson); } export function mergeOpencodeConfigs( diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index a424f0019d..f5970922d7 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -72,6 +72,7 @@ import { type RuntimeOpencodeConfig, writeRuntimeOpencodeConfig, } from "./runtime-opencode-config-store.js"; +import { getRuntimeDbDiagnostics } from "./runtime-db.js"; import { mergeOpenworkWorkspaceConfigs, readOpenworkWorkspaceConfig, @@ -2673,6 +2674,7 @@ function createRoutes( const globalOpencode = (await readJsoncFile(globalOpencodePath, {} as Record, { allowInvalid: true })).data; const effectiveRuntime = await buildOpenworkRuntimeConfigObject(config, workspace.id); const user = userRuntimeConfigFromOpencodeConfig(persistedOpencode); + const runtimeDb = await getRuntimeDbDiagnostics(config); return jsonResponse({ runtime, @@ -2694,6 +2696,7 @@ function createRoutes( runtimeDatabase: { keys: runtimeConfigKeys(runtime), config: runtime, + database: runtimeDb, }, injected: { keys: runtimeConfigKeys(effectiveRuntime), diff --git a/evals/README.md b/evals/README.md index 741467c1b7..5b6bb78e0b 100644 --- a/evals/README.md +++ b/evals/README.md @@ -99,3 +99,6 @@ takes `browser_url` as the first argument. - [`environment-variable-flows.md`](./environment-variable-flows.md) — local environment variable CRUD, masking, validation, apply/restart behavior, and remote-workspace secret boundaries. +- [`runtime-db-flows.md`](./runtime-db-flows.md) — runtime SQLite DB behavior for + MCP/plugin/provider injection, OpenWork workspace metadata, diagnostics, and + upgrade safety. diff --git a/evals/runtime-db-flows.md b/evals/runtime-db-flows.md new file mode 100644 index 0000000000..573efa5763 --- /dev/null +++ b/evals/runtime-db-flows.md @@ -0,0 +1,170 @@ +# Runtime DB flows + +End-to-end scenarios for the OpenWork-owned runtime SQLite DB. These flows verify +that user-visible behavior is unchanged while runtime MCP/plugin/provider and +OpenWork workspace metadata are stored in OpenWork runtime storage instead of +rewriting user-owned workspace files. + +## Preflight + +1. Start OpenWork locally or on Daytona. +2. Create or select a local workspace. +3. Record the workspace path and workspace id from the URL. +4. If using Daytona, start with `DAYTONA_SECRETS_ENV=/tmp/no-daytona-secrets` when validating Den-provisioned providers so local secret env does not mask runtime DB behavior. + +## Flow 1: Runtime DB initializes without changing visible app behavior + +**Goal:** Opening a workspace creates the shared runtime DB schema and leaves the +workspace usable. + +### Steps + +1. Open the desktop app. +2. Create a local workspace. +3. Open Settings -> Advanced. +4. Expand the runtime/OpenCode config diagnostics. +5. Inspect the runtime DB path on disk if available. + +### Expected outcome + +- The app remains on the workspace/session screen. +- Runtime diagnostics load without an error. +- The runtime DB exists at the configured `OPENWORK_RUNTIME_DB` path or the app config `runtime.sqlite` path. +- The runtime DB contains `schema_migrations`, `migration_state`, `runtime_opencode_configs`, and `openwork_workspace_configs`. +- No new default `opencode.jsonc` is created just from opening the workspace. + +## Flow 2: MCP add/remove persists in runtime DB, not user config + +**Goal:** A user can add an MCP server and see it in the app without OpenWork +rewriting the user-owned `opencode.jsonc`. + +### Steps + +1. In the workspace, create an `opencode.jsonc` with a user-owned MCP entry. +2. Open Settings -> MCP. +3. Add a remote MCP server named `runtime-eval` with URL `https://runtime.example/mcp`. +4. Disable and re-enable it. +5. Reload the workspace if prompted. +6. Open Settings -> Advanced and inspect the effective injected config. +7. Inspect the workspace `opencode.jsonc`. + +### Expected outcome + +- The MCP server appears in the MCP list. +- The effective injected config contains `mcp.runtime-eval`. +- The original user-owned `opencode.jsonc` content is unchanged except for edits the user made manually. +- `.opencode/openwork.json` is not created for this MCP change. +- The MCP server still appears after app/workspace reload. + +## Flow 3: Plugin add/remove persists in runtime DB, not user config + +**Goal:** Plugin management remains user-visible in Settings while storage stays +OpenWork-owned. + +### Steps + +1. Open Settings -> Extensions or plugin management. +2. Add a plugin spec such as `runtime-eval-plugin`. +3. Remove it. +4. Add it again. +5. Reload the workspace if prompted. +6. Open Settings -> Advanced and inspect runtime diagnostics. +7. Inspect the workspace `opencode.jsonc`. + +### Expected outcome + +- The plugin appears after adding and disappears after removing. +- The plugin appears again after the final add and reload. +- Runtime diagnostics show the plugin under OpenWork runtime DB / injected config. +- User-owned `opencode.jsonc` is not rewritten for the plugin change. +- `.opencode/openwork.json` is not created for this plugin change. + +## Flow 4: Cloud import metadata persists without legacy workspace file writes + +**Goal:** OpenWork-owned workspace metadata, such as cloud import state, survives +reload without writing normal state to `.opencode/openwork.json`. + +### Steps + +1. Sign into Cloud Account. +2. Import any available cloud-managed plugin/provider/skill into the active workspace. +3. Reload the workspace if prompted. +4. Open the relevant Settings page and confirm the imported item is still shown as imported/available. +5. Inspect `/.opencode/openwork.json`. + +### Expected outcome + +- Imported item state survives reload. +- The Settings UI reflects the imported item state after reload. +- Normal cloud import metadata is not written into `.opencode/openwork.json`. +- Existing legacy `.opencode/openwork.json` metadata is still readable if present. + +## Flow 5: Advanced diagnostics tolerate malformed user files + +**Goal:** A malformed user-owned file does not prevent users from seeing runtime +diagnostics or using OpenWork-managed runtime config. + +### Steps + +1. Write malformed JSONC to `/opencode.jsonc`. +2. Add a plugin or MCP server through OpenWork Settings. +3. Open Settings -> Advanced. +4. Expand source diagnostics and effective injected config. + +### Expected outcome + +- Settings -> Advanced loads successfully. +- Diagnostics show a parse error for the malformed user file. +- OpenWork runtime DB values still appear in the injected config. +- The app does not crash or block runtime MCP/plugin reads. + +## Flow 6: Existing runtime DB upgrades in place + +**Goal:** Users upgrading from the first runtime DB implementation keep existing +runtime MCP/plugin/provider and cloud import state. + +### Steps + +1. Start from a build that has `runtime_opencode_configs` and `openwork_workspace_configs` but not `schema_migrations`. +2. Add a runtime plugin and a cloud import state entry. +3. Upgrade to this build. +4. Open the same workspace. +5. Open Settings -> Advanced and inspect diagnostics. +6. Verify plugin/cloud import state in the UI. + +### Expected outcome + +- Existing runtime data remains present. +- `schema_migrations` is created with version `1`. +- `migration_state` is created empty. +- Runtime plugin/provider/MCP and OpenWork workspace config still appear after upgrade. + +## Flow 7: Den-provisioned provider still runs an actual task + +**Goal:** Cloud/Den provider provisioning remains end-user functional after runtime +DB consolidation. + +### Steps + +1. Start a Den server sandbox and a desktop Electron sandbox pointed at it. +2. Sign into Cloud Account in the desktop app. +3. Create or assign an LLM provider in Den. +4. Wait for desktop sync or click refresh in Cloud Providers. +5. Import/select the provider in the desktop app. +6. Create a new session. +7. Send: `Reply with exactly: Runtime DB provider OK`. + +### Expected outcome + +- The provider appears in desktop Cloud Providers. +- The model is selectable in the composer/model selector. +- The task completes successfully. +- The assistant response is exactly `Runtime DB provider OK`. +- Session metadata shows the cloud provider id/model, not a local fallback provider. + +## Evidence to capture + +- Screenshot or video of the Settings page and successful task response. +- Runtime diagnostics JSON from Settings -> Advanced. +- The runtime DB path and schema version. +- Confirmation that user-owned `opencode.jsonc` and normal `.opencode/openwork.json` state were not rewritten for runtime MCP/plugin/cloud-import changes.