From 38d387be183448f452a9f39bf657520c3b66d2fa Mon Sep 17 00:00:00 2001 From: xa-x Date: Mon, 16 Nov 2026 01:16:16 +0300 Subject: [PATCH] Stop a stuck server, and check documents for a deploy without holding writes Watchdog. A query or mutation that loops without awaiting stops the server's one thread, and with it the timer that is its time limit. A second thread now watches whether the first is still turning. After 30 seconds stuck it prints which function started last and kills the process, so a supervisor starts a fresh one. `onebase serve` turns it on; $ONEBASE_WATCHDOG sets the wait or turns it off. The thread's program is a blob, so it is in compiled binaries without build changes. Deploy check. A deploy that changes the schema validated every document inside its write transaction, so every write waited for the whole scan. On SQLite the scan now runs first, 500 documents per read with the event loop turning in between, while writes record the ids they touch; the transaction re-checks only those. A dry run is checked the same way instead of in one long read. Shared databases keep the old check: other instances write too, and their writes can't be collected in time. Measured on 100,000 documents with queries and writes running: the longest stall during the deploy went from 98 ms to 3 ms. Co-Authored-By: Claude Fable 5.2 --- docs/cli.md | 1 + docs/deployment.md | 2 +- docs/functions.md | 2 + src/app.ts | 9 +++ src/cli.ts | 2 + src/engine/runtime.ts | 161 +++++++++++++++++++++++++++++-------- src/engine/watchdog.ts | 106 ++++++++++++++++++++++++ tests/deploy-check.test.ts | 134 ++++++++++++++++++++++++++++++ tests/watchdog.test.ts | 84 +++++++++++++++++++ 9 files changed, 466 insertions(+), 35 deletions(-) create mode 100644 src/engine/watchdog.ts create mode 100644 tests/deploy-check.test.ts create mode 100644 tests/watchdog.test.ts diff --git a/docs/cli.md b/docs/cli.md index 76ced56..ebfc56b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -70,6 +70,7 @@ Each flag can also be set with an environment variable, which can come from a [. | `ONEBASE_ADMIN_KEY` | Dashboard and deploy key (default: generated into `/admin.key`) | | `ONEBASE_DEPLOY_URL`, `ONEBASE_DEPLOY_KEY` | `deploy` and `env`: the server, and its admin key. Keep them in `.env.local`. | | `ONEBASE_QUERY_TIMEOUT`, `ONEBASE_MUTATION_TIMEOUT`, `ONEBASE_ACTION_TIMEOUT` | How long a call may run (default `10s`, `10s`, `10m`; `off` for no limit). See [Time limits](./functions.md#time-limits). | +| `ONEBASE_WATCHDOG` | How long the server's main thread may be stuck before the process stops itself for its supervisor to restart (default `30s`; `off` to disable). See [Time limits](./functions.md#time-limits) | | `ONEBASE_MAX_READS`, `ONEBASE_MAX_WRITES`, `ONEBASE_MAX_WRITE_MB`, `ONEBASE_MAX_DOCUMENT_MB` | How much one query or mutation may read and write (default `256000`, `128000`, `128`, `16`; `off` for no limit). See [Read and write limits](./functions.md#read-and-write-limits). | | `ONEBASE_QUERY_CACHE_MB` | Memory for repeated query results on SQLite (default `64`; `0` to run every read). See [Repeated reads](./functions.md#repeated-reads). | | `ONEBASE_VECTOR_MEMORY_MB` | Memory for SQLite's vector indexes (default `2048`; `0` to search them in SQL). See [Vector search](./vector-search.md#memory-and-threads-on-sqlite). | diff --git a/docs/deployment.md b/docs/deployment.md index 2b81c17..0a0e8a1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -52,7 +52,7 @@ Deploying 5 source file(s) (6.0 KB) and 1 static file(s) from public/ (7.7 KB) t What happens: 1. **On your machine:** `codegen` and `doctor` run, and doctor errors stop the deploy. Your functions and the npm packages they import are bundled into one module. The `public/` folder is included (pick another folder with `--public ./dist`). -2. **On the server:** the new code is loaded, and auth config and crons are checked. When `schema.ts` changed, every document is checked against the new schema, in the same transaction that updates the indexes, like a Convex push. If any check fails, nothing changes and the old code keeps running. +2. **On the server:** the new code is loaded, and auth config and crons are checked. When `schema.ts` changed, every document is checked against the new schema, in the same transaction that updates the indexes, like a Convex push. If any check fails, nothing changes and the old code keeps running. On SQLite that check doesn't hold up the server: documents are read 500 at a time before the transaction, with requests served in between, and the transaction only looks again at documents written meanwhile. 3. **The swap:** new calls use the new code, and calls already running finish with the old code. The replaced deployment's module is dropped from the module cache, so its memory is freed once those calls finish. Connected clients stay connected, and their live queries re-run on the new code. Two more forms: diff --git a/docs/functions.md b/docs/functions.md index f982285..7edfd63 100644 --- a/docs/functions.md +++ b/docs/functions.md @@ -166,6 +166,8 @@ An action's worker is stopped when its time is up, even in the middle of `while Queries and mutations run on the main thread, and JavaScript can't stop a function in the middle of an `await`. So a query or mutation keeps running until it next yields. Anything it then tries fails with the same error, so it can't read or write any more, call other functions, or schedule them. Code that never yields, such as `while (true) {}`, can't be stopped this way and blocks the whole server. (A background re-run on a [query worker](#live-queries-on-worker-threads) is the exception: its worker is stopped.) The same goes for actions with `ONEBASE_ACTION_WORKERS=0`. +**The watchdog.** For that last case, `onebase serve` runs a second thread that only watches whether the main one is still turning. When it has been stuck for 30 seconds, the process prints which function started last and stops itself, so whatever runs it (systemd, Docker, Fly) starts a fresh one: a server that would have hung until someone noticed is back in seconds. It stops with `SIGKILL`, since a stuck server can't shut down politely; nothing committed is lost. `ONEBASE_WATCHDOG` sets the wait (`2m`) or turns it off (`off`). Under `onebase dev` the server stops and comes back the next time you save a file. + The limit counts from when the call gets its database snapshot (a query), the write lock (a mutation) or a worker (an action), not from when it was queued. A mutation called from a mutation shares its caller's limit; one called from an action gets its own. Change the limits with `ONEBASE_QUERY_TIMEOUT`, `ONEBASE_MUTATION_TIMEOUT` and `ONEBASE_ACTION_TIMEOUT`, e.g. `30s`, `2m`, or `off`. Keep mutations short: while one runs, every other write waits for it. diff --git a/src/app.ts b/src/app.ts index d5d2d8e..97286aa 100644 --- a/src/app.ts +++ b/src/app.ts @@ -5,6 +5,7 @@ import { extname, join, resolve, sep } from "node:path"; import { RateLimiter, serve, type AssetResolver, type RateLimit } from "./engine/http"; import { startBackups, type BackupOptions } from "./backup"; import { createDriver } from "./engine/driver"; +import { startWatchdog } from "./engine/watchdog"; import { LocalBlobs, parseS3Url, S3Blobs } from "./engine/files"; import type { AuthConfig } from "./engine/jwt"; import { consoleLogger, DEFAULT_TIME_LIMITS, formatMs, jsonLogger, Runtime, type Authenticate, type Modules, type TimeLimits } from "./engine/runtime"; @@ -28,6 +29,11 @@ export type AppOptions = { dashboard?: boolean; /** Serve the dashboard and admin API only when the request's Host is this (with or without a port). */ adminHost?: string; + /** + * Stop the process when its main thread has been stuck this long (a function looping without + * awaiting), for a supervisor to restart it. Default 0: off. `onebase serve` turns it on at 30s. + */ + watchdogMs?: number; /** Accept `onebase deploy` and run what it pushed, instead of the code given here (default true; never in dev). */ deploys?: boolean; schema: SchemaDefinition | null; @@ -148,6 +154,8 @@ export async function startApp(opts: AppOptions) { callLimits: opts.callLimits, }); const { created, dropped } = await runtime.init(); + const watchdog = opts.watchdogMs ? startWatchdog(opts.watchdogMs) : null; + runtime.watchdog = watchdog; const server = serve({ runtime, port: opts.port, @@ -227,6 +235,7 @@ export async function startApp(opts: AppOptions) { runtime.log.warn(`Shutting down with ${runtime.calls.active} call(s) still running after ${formatMs(graceMs)}.`); } runtime.stop(); + watchdog?.stop(); await server.stop(true); await driver.close(); }; diff --git a/src/cli.ts b/src/cli.ts index fffebd7..78a54a3 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -258,6 +258,8 @@ function serverOptions(opts: ReturnType) { mutationMs: timeLimit("ONEBASE_MUTATION_TIMEOUT"), actionMs: timeLimit("ONEBASE_ACTION_TIMEOUT"), }, + // Stops a server whose one thread is stuck, so its supervisor restarts it. + watchdogMs: timeLimit("ONEBASE_WATCHDOG") ?? 30_000, callLimits: { documentsRead: callLimit(CALL_LIMIT_ENV.documentsRead), documentsWritten: callLimit(CALL_LIMIT_ENV.documentsWritten), diff --git a/src/engine/runtime.ts b/src/engine/runtime.ts index 90c0525..e944802 100644 --- a/src/engine/runtime.ts +++ b/src/engine/runtime.ts @@ -15,10 +15,11 @@ import { checkArgs, checkReturn, indexCode, lookupFunction, lookupRoute, type Mo import { buildCatalog, TableScope, tableMount, validatesTable, type Catalog } from "./catalog"; import { CronRunner } from "./crons"; import { Scheduler } from "./scheduler"; +import type { Watchdog } from "./watchdog"; import { BACKFILL_BATCH, backfilled, coalesce, hookState, readProgress, writeProgress, type HookProgress, type ResolvedHook } from "./hooks"; import { applySchema, DatabaseImpl, OnebaseError, readerView, writerView, type ReadSet, type VectorSearchQuery, type Write } from "./database"; import { isIdForTable, newId } from "./dialect"; -import { envNumber, savepoint, Semaphore, SerialQueue, SqliteDriver, type Driver, type Exec } from "./driver"; +import { envNumber, savepoint, Semaphore, SerialQueue, SqliteDriver, type Driver, type Exec, type Row } from "./driver"; import { CONFIRMED, DIRECT_UPLOAD_DEFAULT_SECONDS, @@ -270,6 +271,13 @@ export type CallOptions = { source?: CallSource; }; +/** Documents that don't match the schema: how many, and the first few. */ +type InvalidDocuments = { invalid: number; examples: string[] }; +/** A document check done ahead of a deploy's transaction: the schema it was for, and what was written during and after it. */ +type PrecheckedDocuments = { fingerprint: string; touched: Set; overflowed: boolean }; +/** Past this many documents written during a precheck, the deploy checks everything again in its transaction. */ +const MAX_PRECHECK_WRITES = 100_000; + /** What changed in a commit: exact writes, or (when too large to broadcast) just the tables. */ export type CommitEvent = ({ writes: Write[]; tables?: undefined } | { tables: string[]; writes?: undefined }) & { settle?: Settle }; @@ -396,6 +404,10 @@ export class Runtime { /** Signs function handles; loaded from the database by init(). */ private handleKey: Buffer | null = null; private stopped = false; + /** A deploy's document check in progress: every write adds the ids it touched (see precheckDocuments). */ + private precheck: PrecheckedDocuments | null = null; + /** Set by the server: told which function starts, to name it if the thread gets stuck (watchdog.ts). */ + watchdog: Watchdog | null = null; private draining = false; /** True once a shutdown has begun. */ get shuttingDown() { @@ -528,52 +540,124 @@ export class Runtime { return applied; } + /** + * The tables whose documents are validated, and a fingerprint of their validators. With + * schemaValidation off only component tables are checked; turning it back on changes the + * fingerprint, so everything is checked again then. + */ + private checkedTables(catalog: Catalog) { + const tables = Object.entries(catalog.tables).filter(([t]) => validatesTable(catalog, t)) as [string, { validator: AnyValidator }][]; + const fingerprint = new Bun.CryptoHasher("sha256").update(JSON.stringify(tables.map(([t, d]) => [t, d.validator]))).digest("hex"); + return { tables, fingerprint }; + } + + private checkRows(table: string, def: { validator: AnyValidator }, rows: Row[], found: InvalidDocuments) { + for (const row of rows) { + const errors = validate(def.validator, this.driver.dialect.parseDoc(row.doc), "document"); + if (!errors.length) continue; + found.invalid++; + if (found.examples.length < 5) found.examples.push(`${table} ${row.id}: ${errors.slice(0, 3).join("; ")}`); + } + } + + /** Checks a table 500 documents at a time, read with `all`; `between` runs after each batch. */ + private async checkTable( + table: string, + def: { validator: AnyValidator }, + found: InvalidDocuments, + all: (sql: string, params: unknown[]) => Promise, + between?: () => Promise, + ) { + const d = this.driver.dialect; + let after: [number, string] = [-1, ""]; + for (;;) { + const rows = await all( + `SELECT id, ct, doc FROM documents WHERE tbl = ? AND (ct, id) > (${d.placeholder("ct")}, ${d.placeholder("id")}) ORDER BY ct, id LIMIT 500`, + [table, ...after], + ); + this.checkRows(table, def, rows, found); + if (rows.length < 500) break; + after = [Number(rows[rows.length - 1]!.ct), rows[rows.length - 1]!.id as string]; + await between?.(); + } + } + + private schemaMismatch({ invalid, examples }: InvalidDocuments) { + return new OnebaseError( + `schema.ts doesn't match ${invalid} existing document${invalid === 1 ? "" : "s"}:\n - ${examples.join("\n - ")}` + + (invalid > examples.length ? `\n - … and ${invalid - examples.length} more` : "") + + `\nMake the schema accept the data (v.optional(...), v.union(...)), or fix the data: start with ` + + `defineSchema({ ... }, { schemaValidation: false }), update the documents in a mutation (or the dashboard), then turn validation back on.`, + ); + } + + /** + * A deploy's document check, without the write lock. Checking every document inside the + * deploy's transaction would hold up every write (and, with one thread, every request) for as + * long as the scan takes. So the scan happens first, a batch at a time with requests served in + * between, while the ids of documents written meanwhile are collected; the transaction then + * only re-checks those (checkDocuments). + * + * Returns null when there is nothing to gain: no schema, an unchanged one, or a shared database, + * where other instances write too and their writes can't be collected in time. The caller must + * call endPrecheck() once its transaction is over. + */ + private async precheckDocuments(catalog: Catalog | null): Promise { + if (!catalog || this.driver.shared) return null; + const { tables, fingerprint } = this.checkedTables(catalog); + const stored = await this.driver.read((tx) => tx.get("SELECT value FROM _meta WHERE \"key\" = 'schema'")); + if (stored?.value === fingerprint) return null; + const prechecked: PrecheckedDocuments = { fingerprint, touched: new Set(), overflowed: false }; + this.precheck = prechecked; + const found: InvalidDocuments = { invalid: 0, examples: [] }; + const read = (sql: string, params: unknown[]) => this.driver.read((tx) => tx.all(sql, params)); + const breathe = () => new Promise((resolve) => setImmediate(resolve)); + try { + for (const [table, def] of tables) await this.checkTable(table, def, found, read, breathe); + } catch (err) { + this.endPrecheck(); + throw err; + } + if (found.invalid) { + this.endPrecheck(); + throw this.schemaMismatch(found); + } + return prechecked; + } + + private endPrecheck() { + this.precheck = null; + } + /** * When schema.ts (or a component's schema) changes, every stored document is checked against * it, like a Convex deploy. The schema's fingerprint is saved once the data matches (unless * `record` is false, for a dry run), so unchanged schemas cost nothing. Returns warnings about * data the schema ignores. */ - private async checkDocuments(tx: Exec, catalog: Catalog | null, record = true): Promise { + private async checkDocuments(tx: Exec, catalog: Catalog | null, record = true, prechecked: PrecheckedDocuments | null = null): Promise { if (!catalog) { if (record) await tx.run("DELETE FROM _meta WHERE \"key\" = 'schema'"); return []; } - // With schemaValidation off only component tables are checked; turning it back on changes - // the fingerprint, so everything is checked again then. - const tables = Object.entries(catalog.tables).filter(([t]) => validatesTable(catalog, t)) as [string, { validator: AnyValidator }][]; - const fingerprint = new Bun.CryptoHasher("sha256").update(JSON.stringify(tables.map(([t, d]) => [t, d.validator]))).digest("hex"); + const { tables, fingerprint } = this.checkedTables(catalog); const stored = await tx.get("SELECT value FROM _meta WHERE \"key\" = 'schema'"); if (stored?.value === fingerprint) return []; - const d = this.driver.dialect; - const examples: string[] = []; - let invalid = 0; - for (const [table, def] of tables) { - let after: [number, string] = [-1, ""]; - for (;;) { - const rows = await tx.all( - `SELECT id, ct, doc FROM documents WHERE tbl = ? AND (ct, id) > (${d.placeholder("ct")}, ${d.placeholder("id")}) ORDER BY ct, id LIMIT 500`, - [table, ...after], - ); - for (const row of rows) { - const errors = validate(def.validator, d.parseDoc(row.doc), "document"); - if (!errors.length) continue; - invalid++; - if (examples.length < 5) examples.push(`${table} ${row.id}: ${errors.slice(0, 3).join("; ")}`); - } - if (rows.length < 500) break; - after = [Number(rows[rows.length - 1]!.ct), rows[rows.length - 1]!.id]; + const found: InvalidDocuments = { invalid: 0, examples: [] }; + if (prechecked?.fingerprint === fingerprint && !prechecked.overflowed) { + // Every document was checked a moment ago without the write lock (precheckDocuments). Only + // the ones written since then need another look, which is what keeps this transaction short. + const byTable = new Map(tables); + for (const id of prechecked.touched) { + const row = await tx.get("SELECT id, tbl, doc FROM documents WHERE id = ?", [id]); + const def = row ? byTable.get(row.tbl as string) : undefined; + if (def) this.checkRows(row!.tbl as string, def, [row!], found); } + } else { + for (const [table, def] of tables) await this.checkTable(table, def, found, (sql, params) => tx.all(sql, params)); } - if (invalid) { - throw new OnebaseError( - `schema.ts doesn't match ${invalid} existing document${invalid === 1 ? "" : "s"}:\n - ${examples.join("\n - ")}` + - (invalid > examples.length ? `\n - … and ${invalid - examples.length} more` : "") + - `\nMake the schema accept the data (v.optional(...), v.union(...)), or fix the data: start with ` + - `defineSchema({ ... }, { schemaValidation: false }), update the documents in a mutation (or the dashboard), then turn validation back on.`, - ); - } + if (found.invalid) throw this.schemaMismatch(found); const known = new Set(Object.keys(catalog.tables)); const orphans = (await tx.all("SELECT DISTINCT tbl FROM documents")) .map((r) => r.tbl as string) @@ -895,8 +979,10 @@ export class Runtime { ? `The onebase CLI that pushed this didn't send its version, so it's older than this server (${VERSION}). Update it, so a version mismatch can be caught.` : versionWarning(builtWith); const schema = opts.dryRun ? this.dryRunSchema(opts.schema) : prepared!.catalog; + // The long part of a schema change, done before the transaction (and, for a dry run, instead of a long read). + const prechecked = await this.precheckDocuments(schema); const apply = async (tx: Exec) => { - const warnings = await this.checkDocuments(tx, schema, !opts.dryRun); + const warnings = await this.checkDocuments(tx, schema, !opts.dryRun, prechecked); if (opts.dryRun) return { indexes: { created: [], dropped: [] }, warnings }; const indexes = await applySchema(tx, this.driver.dialect, schema); if (opts.save) await saveDeployment(tx, stored); @@ -905,7 +991,7 @@ export class Runtime { return { indexes, warnings }; }; // Queued with this process's mutations, so none of them runs between the check and the swap. - const { indexes, warnings } = opts.dryRun ? await this.driver.read(apply) : await this.withWriter(apply); + const { indexes, warnings } = await (opts.dryRun ? this.driver.read(apply) : this.withWriter(apply)).finally(() => this.endPrecheck()); const deployment = toDeployment(stored); if (!opts.dryRun) { await this.install(prepared!, deployment); @@ -1705,6 +1791,7 @@ export class Runtime { const checked = this.checkArgs(name, fn, args); const caller = this.componentFunctions.get(name)?.mount ?? null; const handle = (ref: unknown) => this.createHandle(ref, caller); + this.watchdog?.enter(name); const result = await currentFunction.run({ type: fn.type, name, handle, reads }, () => fn.handler(ctx, checked)); return this.checkReturn(name, fn, result); } @@ -1867,6 +1954,12 @@ export class Runtime { // Nothing commits before every table hook has seen every write. await this.flushHooks(state); budget?.check(); // a limit error the function caught still rolls the mutation back + // Before the commit, in the writer: a deploy's transaction, queued behind this one, sees every id. + const precheck = this.precheck; + if (precheck && state.writes.length) { + if (precheck.touched.size + state.writes.length > MAX_PRECHECK_WRITES) precheck.overflowed = true; + else for (const w of state.writes) precheck.touched.add(w.id); + } if (this.driver.shared && this.driver.publish && (state.writes.length || state.scheduled || state.revoked.length)) { for (const message of this.broadcastMessages(state)) await this.driver.publish(tx, message); } diff --git a/src/engine/watchdog.ts b/src/engine/watchdog.ts new file mode 100644 index 0000000..3f924d5 --- /dev/null +++ b/src/engine/watchdog.ts @@ -0,0 +1,106 @@ +// Queries and mutations run on the server's one thread, and a time limit is a timer on that same +// thread: a function that loops without awaiting (`while (true) {}`) stops the timer along with +// everything else, and the server sits there, accepting connections and answering none. +// +// The watchdog is a second thread that only watches whether the first is still turning. When it +// hasn't for `limitMs`, it says which function started last and kills the process, so whatever +// supervises it (systemd, Docker, Fly) starts a fresh one. A stuck server can't shut down +// politely, so this is SIGKILL; SQLite and the other databases are safe against that. + +/** How often the main thread reports in. */ +const BEAT_MS = 250; + +// Shared between the threads: [0] a counter the main thread keeps raising, [1] the id of the +// function that started last, [2] set when the watchdog should stop. +const BEAT = 0; +const LAST = 1; +const STOP = 2; + +export type Watchdog = { + /** Call as a function starts running: it is the one named if the thread never comes back. */ + enter(name: string): void; + stop(): void; +}; + +/** + * The watchdog thread's program. A blob, not a file, so that it's in the compiled binary without + * further ado. It can't use anything from this module: it is turned into text and run on its own. + */ +function watchdogMain() { + const scope = globalThis as any; + const names = new Map(); + let view: Int32Array; + let limitMs = 0; + let checkMs = 0; + let seen = -1; + let stalledChecks = 0; + + const check = () => { + if (Atomics.load(view, 2)) return scope.close(); + const beat = Atomics.load(view, 0); + if (beat !== seen) { + seen = beat; + stalledChecks = 0; + return; + } + // Counted in this thread's own checks, not by the clock: after the machine sleeps, the clock + // has jumped but only one check has gone by, and the main thread gets its turn to report in. + stalledChecks++; + if (stalledChecks * checkMs < limitMs) return; + const name = names.get(Atomics.load(view, 1)); + const seconds = limitMs / 1000; + const message = + `\nonebase: the server's main thread has been stuck for ${seconds}s, so this process is stopping for its supervisor to restart it.\n` + + (name ? ` The last function to start was ${name}.\n` : "") + + ` A query or mutation that loops without awaiting does this: time limits can't interrupt it. Find the loop, or move heavy work to an action.\n` + + ` $ONEBASE_WATCHDOG sets how long to wait (e.g. 2m), and "off" turns this off.\n`; + try { + // Straight to the file descriptor: nothing that needs the main thread. Loaded this way and + // not with require() or import, which a bundler rewrites into helpers this text doesn't have. + scope.process.getBuiltinModule("node:fs").writeSync(2, message); + } catch {} + scope.process.kill(scope.process.pid, "SIGKILL"); + }; + + scope.onmessage = ({ data }: { data: any }) => { + if (data.name !== undefined) return void names.set(data.id, data.name); + view = new Int32Array(data.buffer); + limitMs = data.limitMs; + checkMs = data.checkMs; + setInterval(check, checkMs); + }; +} + +let workerUrl: string | undefined; + +/** Starts watching the calling thread. It must stay unblocked for less than `limitMs` at a time. */ +export function startWatchdog(limitMs: number): Watchdog { + const buffer = new SharedArrayBuffer(3 * Int32Array.BYTES_PER_ELEMENT); + const view = new Int32Array(buffer); + workerUrl ??= URL.createObjectURL(new Blob([`(${watchdogMain})()`], { type: "text/javascript" })); + const worker = new Worker(workerUrl); + (worker as any).unref(); // watching mustn't keep the process alive + // Several checks per limit, so the stop comes within a quarter of it past the limit. + worker.postMessage({ buffer, limitMs, checkMs: Math.max(10, Math.min(1000, Math.floor(limitMs / 4))) }); + + const beat = setInterval(() => void Atomics.add(view, BEAT, 1), Math.min(BEAT_MS, Math.max(5, Math.floor(limitMs / 8)))); + beat.unref?.(); + + const ids = new Map(); + return { + enter(name) { + let id = ids.get(name); + if (id === undefined) { + id = ids.size + 1; + ids.set(name, id); + // Sent before the function runs: the message is on its way even if the thread never returns. + worker.postMessage({ id, name }); + } + Atomics.store(view, LAST, id); + }, + stop() { + clearInterval(beat); + Atomics.store(view, STOP, 1); + }, + }; +} diff --git a/tests/deploy-check.test.ts b/tests/deploy-check.test.ts new file mode 100644 index 0000000..4f7e8b0 --- /dev/null +++ b/tests/deploy-check.test.ts @@ -0,0 +1,134 @@ +// A deploy that changes the schema checks every document. That check must not hold the server +// up while it runs, and must still catch a document written while it was running. + +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { startApp } from "../src/app"; +import { deploy } from "../src/deploy"; +import { quiet } from "./helpers"; + +const root = mkdtempSync(join(tmpdir(), "onebase-deploy-check-")); +let app: Awaited>; +let url: string; +let key: string; + +const SCHEMA = (text: string, extra = "") => ` +import { defineSchema, defineTable } from "onebase/server"; +import { v } from "onebase/values"; +export default defineSchema({ notes: defineTable({ text: ${text}, n: v.number() })${extra} }); +`; +const NOTES = ` +import { mutation, query } from "onebase/server"; +import { v } from "onebase/values"; +export const addMany = mutation({ + args: { from: v.number(), count: v.number() }, + handler: async (ctx, { from, count }) => { + for (let i = from; i < from + count; i++) await ctx.db.insert("notes", { text: "note " + i, n: i }); + }, +}); +export const first = query({ args: {}, handler: async (ctx) => (await ctx.db.query("notes").first())?._id ?? null }); +export const setText = mutation({ args: { id: v.id("notes"), text: v.any() }, handler: (ctx, { id, text }) => ctx.db.patch(id, { text }) }); +export const count = query({ args: {}, handler: async (ctx) => (await ctx.db.query("notes").collect()).length }); +`; + +function project(name: string, schema: string): string { + const dir = join(root, name, "onebase"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "schema.ts"), schema); + writeFileSync(join(dir, "notes.ts"), NOTES); + return dir; +} +const loose = project("loose", SCHEMA("v.union(v.string(), v.number())")); +const strict = project("strict", SCHEMA("v.string()")); +const wider = project("wider", SCHEMA("v.union(v.string(), v.number(), v.null())")); +const push = (dir: string, extra: Partial[0]> = {}) => deploy({ dir, url, key, public: null, log: () => {}, ...extra }); + +const DOCS = 10_000; + +beforeAll(async () => { + const dataDir = join(root, "data"); + app = await startApp({ schema: null, modules: {}, dataDir, port: 0, hostname: "127.0.0.1", dev: false, quiet: true }); + Object.assign(app.runtime, { log: quiet }); + url = `http://127.0.0.1:${app.server.port}`; + key = readFileSync(join(dataDir, "admin.key"), "utf8").trim(); + await push(loose); + for (let from = 0; from < DOCS; from += 2000) await app.runtime.runMutation("notes:addMany", { from, count: 2000 }, { settle: "none" }); +}, 60_000); + +afterAll(async () => { + await app.close(); + rmSync(root, { recursive: true, force: true }); +}); + +/** + * Runs `work` and reports how the document check was done: how many separate short reads the + * database got (the check reads 500 documents at a time), and whether any of those reads happened + * while a write transaction was open. + */ +async function watching(work: () => Promise) { + const driver = app.driver as any; + const { read, write } = driver; + let reads = 0; + let scansInsideAWrite = 0; + driver.read = (fn: any) => (reads++, read.call(driver, fn)); + driver.write = (fn: any) => + write.call(driver, (tx: any) => { + const all = tx.all.bind(tx); + const counted = { ...tx, all: (sql: string, params: unknown[]) => (sql.includes("FROM documents WHERE tbl = ?") && sql.includes("LIMIT 500") && scansInsideAWrite++, all(sql, params)) }; + return fn(counted); + }); + try { + return { result: await work(), reads: () => reads, scansInsideAWrite: () => scansInsideAWrite }; + } finally { + driver.read = read; + driver.write = write; + } +} + +test("a deploy checks 10,000 documents in short reads, outside its write transaction", async () => { + const run = await watching(() => push(wider)); + expect(run.result).toMatchObject({ dryRun: false }); + // 20 batches of 500, each its own read, with the event loop coming round in between. Checked + // inside the deploy's transaction, as it used to be, every write waits for the whole scan. + expect(run.reads()).toBeGreaterThanOrEqual(20); + expect(run.scansInsideAWrite()).toBe(0); + expect((await app.runtime.runQuery("notes:count", {})).value).toBe(DOCS); +}, 60_000); + +test("a dry run checks the same way and changes nothing", async () => { + const run = await watching(() => push(strict, { dryRun: true })); + expect(run.result).toMatchObject({ dryRun: true }); + expect(run.reads()).toBeGreaterThanOrEqual(20); + // Still the wider schema: a number is accepted. + const id = (await app.runtime.runQuery("notes:first", {})).value; + await app.runtime.runMutation("notes:setText", { id, text: 7 }); + await expect(push(strict, { dryRun: true })).rejects.toThrow("doesn't match 1 existing document"); + await app.runtime.runMutation("notes:setText", { id, text: "note 0" }); +}, 60_000); + +test("a document that stops matching while the check runs still stops the deploy", async () => { + const id = (await app.runtime.runQuery("notes:first", {})).value as string; + const runtime = app.runtime as any; + // Once the check is under way (and past the first batch, which holds this document), make it a + // number: fine by the schema that is live, not by the one being deployed. + let wrote: Promise | null = null; + const watch = setInterval(() => { + if (!runtime.precheck || wrote) return; + wrote = app.runtime.runMutation("notes:setText", { id, text: 42 }); + }, 0); + try { + await expect(push(strict)).rejects.toThrow(`notes ${id}: document.text`); + } finally { + clearInterval(watch); + } + expect(wrote).not.toBeNull(); + await wrote; + // The old code is still live, the write is there, and nothing is left watching writes. + expect(runtime.precheck).toBeNull(); + expect((await app.runtime.runQuery("notes:count", {})).value).toBe(DOCS); + await app.runtime.runMutation("notes:setText", { id, text: "note 0" }); + expect(await push(strict)).toMatchObject({ dryRun: false }); + await expect(app.runtime.runMutation("notes:setText", { id, text: 42 })).rejects.toThrow(); +}, 60_000); diff --git a/tests/watchdog.test.ts b/tests/watchdog.test.ts new file mode 100644 index 0000000..bf30941 --- /dev/null +++ b/tests/watchdog.test.ts @@ -0,0 +1,84 @@ +// The watchdog: a server whose one thread is stuck in a function that never awaits stops itself, +// saying which function it was, so that a supervisor can start a fresh one. + +import { afterAll, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { startWatchdog } from "../src/engine/watchdog"; + +const root = mkdtempSync(join(tmpdir(), "onebase-watchdog-")); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +const SERVER = ` +import { startApp } from ${JSON.stringify(join(import.meta.dir, "../src/app"))}; +import { mutation, query } from ${JSON.stringify(join(import.meta.dir, "../src/server"))}; +const modules = { notes: { + ok: query({ args: {}, handler: async () => "fine" }), + spin: mutation({ args: {}, handler: async () => { for (;;) {} } }), +} }; +const app = await startApp({ schema: null, modules, dataDir: process.argv[2], port: 0, hostname: "127.0.0.1", dev: false, quiet: true, watchdogMs: Number(process.argv[3]) }); +console.log("READY " + app.server.port); +`; + +async function startServer(watchdogMs: number) { + const script = join(root, "server.ts"); + writeFileSync(script, SERVER); + const proc = Bun.spawn(["bun", script, mkdtempSync(join(root, "data-")), String(watchdogMs)], { stdout: "pipe", stderr: "pipe" }); + const reader = proc.stdout.getReader(); + let out = ""; + while (!out.includes("READY")) { + const { value, done } = await reader.read(); + if (done) throw new Error(`The server didn't start: ${await new Response(proc.stderr).text()}`); + out += new TextDecoder().decode(value); + } + const base = `http://127.0.0.1:${/READY (\d+)/.exec(out)![1]}`; + const call = (kind: string, path: string) => + fetch(`${base}/api/${kind}`, { method: "POST", body: JSON.stringify({ path, args: {} }), signal: AbortSignal.timeout(10_000) }).then( + (r) => r.json() as Promise, + () => "no answer", + ); + return { proc, call }; +} + +test("a function that never comes back gets the process stopped, and named", async () => { + const { proc, call } = await startServer(1000); + expect(await call("query", "notes:ok")).toEqual({ status: "success", value: "fine" }); + // Idle for longer than the limit: nothing is stuck, so nothing happens. + await Bun.sleep(1500); + expect(proc.exitCode).toBeNull(); + expect(await call("query", "notes:ok")).toEqual({ status: "success", value: "fine" }); + + const began = performance.now(); + const stuck = call("mutation", "notes:spin"); + await proc.exited; + const took = performance.now() - began; + expect(await stuck).toBe("no answer"); + expect(proc.signalCode).toBe("SIGKILL"); + expect(took).toBeGreaterThanOrEqual(900); + expect(took).toBeLessThan(5000); + const said = await new Response(proc.stderr).text(); + expect(said).toContain("main thread has been stuck for 1s"); + expect(said).toContain("The last function to start was notes:spin."); + expect(said).toContain("$ONEBASE_WATCHDOG"); +}, 30_000); + +test("with the watchdog off, a stuck server just stays stuck", async () => { + const { proc, call } = await startServer(0); + void call("mutation", "notes:spin"); + await Bun.sleep(1500); + expect(proc.exitCode).toBeNull(); + proc.kill("SIGKILL"); + await proc.exited; +}, 30_000); + +test("a stopped watchdog lets go of its thread", async () => { + const watchdog = startWatchdog(200); + watchdog.enter("notes:one"); + watchdog.enter("notes:two"); + watchdog.stop(); + // Blocking this thread now is nobody's business any more. + const until = performance.now() + 600; + while (performance.now() < until) {} + expect(true).toBe(true); +});