diff --git a/docs/cli.md b/docs/cli.md index cc3c59c..39d6c4c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -76,7 +76,7 @@ Each flag can also be set with an environment variable, which can come from a [. | `ONEBASE_QUERY_CACHE_MB` | Memory for repeated query results on SQLite (default `64`; `0` to run every read). See [Repeated reads](./functions.md#repeated-reads). | | `ONEBASE_VECTOR_MEMORY_MB` | Memory for SQLite's vector indexes (default `2048`; `0` to search them in SQL). See [Vector search](./vector-search.md#memory-and-threads-on-sqlite). | | `ONEBASE_ACTION_WORKERS` | Worker threads for actions (default: one per CPU core, up to 8; `0` runs actions on the main thread). See [Worker threads](./functions.md#worker-threads). | -| `ONEBASE_QUERY_WORKERS` | Worker threads that re-run live queries after a write, on SQLite (default `0`: on the main thread). See [Live queries on worker threads](./functions.md#live-queries-on-worker-threads). | +| `ONEBASE_QUERY_WORKERS` | Worker threads that re-run live queries after a write, on SQLite, once a write affects enough of them to keep the main thread busy (default: up to `2`, by the machine's cores; `0` keeps every query on the main thread). See [Live queries on worker threads](./functions.md#live-queries-on-worker-threads). | | `ONEBASE_VECTOR_WORKERS` | Threads for searching SQLite's vector indexes (default: up to 8, one less than the CPU cores; `0` for the main thread only) | ## env diff --git a/docs/functions.md b/docs/functions.md index 7edfd63..e5e3894 100644 --- a/docs/functions.md +++ b/docs/functions.md @@ -43,28 +43,33 @@ A query that reads the clock or a random source (`Date.now()`, `new Date()`, `Ma After a write, every live query it affects runs again. The write only waits for its own client's queries. The rest re-run in the background, but on the main thread. With many per-user subscriptions to one query, say 2,000 users each watching their own feed, that can be tens of milliseconds of main-thread work per write. -On SQLite, `ONEBASE_QUERY_WORKERS=2` moves those background re-runs to worker threads. Each worker loads its own copy of your code and opens its own read-only connection to the database file. A re-run is sent to a worker only after the write that caused it has committed, so it always sees that write. Everything else still runs on the main thread: +On SQLite, a large fan-out moves to worker threads. A round of background re-runs starts on the main thread; if it is still going after 4 milliseconds, the rest of it is sent to query workers. So a write that a handful of queries depend on is handled exactly as before, with no thread in between and no worker started, while one that thousands depend on stops taking the main thread's time. The first worker starts the first time a round runs that long, and the main thread carries on until it has loaded your code. + +Each worker loads its own copy of your code and opens its own read-only connection to the database file. A re-run is sent to a worker only after the write that caused it has committed, so it always sees that write. Everything else still runs on the main thread: - a subscription's first result - the queries a write waits for - one-off reads - re-runs after a deploy -The default is `0`, which keeps every query on the main thread. +`ONEBASE_QUERY_WORKERS` sets how many workers there may be. The default is 2, or 1 on a machine with two cores and none with one. `0` keeps every query on the main thread. More than 2 gets a very large fan-out to its subscribers sooner, but takes little more off the main thread. A query on a worker can do everything a query can, with the same results and errors. Like action workers, each has its own copy of your modules, so a module-level variable a query reads isn't shared with the main thread. It also gets the same read limit and the same time limit. One that runs past its time limit is stopped with its worker, even in the middle of `while (true) {}`. A new worker takes its place. Two cases run on the main thread from then on: a query that uses a function handle, and every query if the workers can't load your code (with a warning). After a deploy, or a change to environment variables, no result from the old code is delivered: queries running on the old code's workers run again on the new code. -Each worker costs about 15–30 MB of memory. Measured with `bun scripts/bench-query-workers.ts`: 2,000 per-user subscribers to one query, on a 20-core machine. +Each worker costs about 15–30 MB of memory once it has started; workers beyond the first stop again after 30 idle seconds. Measured with `bun scripts/bench-query-workers.ts`: 2,000 per-user subscribers to one query, on a 20-core machine. | Query | Workers | Main thread busy per write | Longest stall | Until all are updated | | --- | --- | --- | --- | --- | -| Reads ~50 documents, filters in JS | 0 | 102 ms | 8 ms | 88 ms | -| | 2 | 21 ms | 1 ms | 50 ms | -| | 4 | 23 ms | 2 ms | 36 ms | -| Two indexed reads | 0 | 38 ms | 7 ms | 38 ms | -| | 2 | 28 ms | 2 ms | 32 ms | +| Reads ~50 documents, filters in JS | 0 | 100 ms | 8 ms | 86 ms | +| | 1 | 20 ms | 4 ms | 84 ms | +| | 2 (default) | 26 ms | 4 ms | 48 ms | +| | 4 | 25 ms | 4 ms | 36 ms | +| Two indexed reads | 0 | 36 ms | 7 ms | 37 ms | +| | 2 (default) | 27 ms | 4 ms | 33 ms | + +With 10 subscribers (`SUBSCRIBERS=10`) the numbers are the same with and without workers, because none is started: every subscriber is updated in under a millisecond. The write itself returns in 1–3 ms either way. Finding the affected queries and sending each subscriber its result stay on the main thread, so the cheaper the query, the less workers save. diff --git a/scripts/bench-query-workers.ts b/scripts/bench-query-workers.ts index debffc1..17d84c0 100644 --- a/scripts/bench-query-workers.ts +++ b/scripts/bench-query-workers.ts @@ -3,6 +3,7 @@ * * bun scripts/bench-query-workers.ts # every configuration, each in its own process * bun scripts/bench-query-workers.ts 2 heavy # one: 2 workers, the heavier query + * SUBSCRIBERS=10 bun scripts/bench-query-workers.ts # a small app, where workers have less to give * * 2,000 subscribers, one per user, to one query that reads the caller's identity, so nothing is * shared between them. Each write changes a document every one of them reads. Reported per write @@ -47,7 +48,7 @@ function pieceMs(): number { return (performance.now() - t0) / 2000; } -const SUBSCRIBERS = 2000; +const SUBSCRIBERS = Number(process.env.SUBSCRIBERS ?? 2000); const WRITES = 12; const WARMUP = 2; @@ -84,7 +85,7 @@ async function one(workers: number, kind: keyof typeof QUERIES) { export const feed = query({ args: {}, handler: ${QUERIES[kind]} });`, ); const driver = new SqliteDriver(join(root, "db.sqlite")); - const runtime = new Runtime({ driver, ...(await loadProjectCode(dir)), queryWorkers: workers, log: { info() {}, warn() {}, error() {} } }); + const runtime = new Runtime({ driver, ...(await loadProjectCode(dir)), queryWorkers: workers, queryWorkersAfterMs: process.env.AFTER_MS ? Number(process.env.AFTER_MS) : undefined, log: { info() {}, warn() {}, error() {} } }); await runtime.init(); for (let i = 0; i < 50; i++) await runtime.runMutation("notes:add", { text: `n${i}`, owner: `user${i}` }, { settle: "none" }); const subs = new SubscriptionManager(runtime); @@ -112,6 +113,8 @@ async function one(workers: number, kind: keyof typeof QUERIES) { const writer = {}; // a connection with no subscriptions of its own: its write waits for nothing const rows: { write: number; stall: number; all: number }[] = []; + /** The first write after start: with workers, it also waits for one to load the code. */ + let first = 0; for (let k = 0; k < WARMUP + WRITES; k++) { marker = `w${k}-${Math.random().toString(36).slice(2, 8)}`; updated = 0; @@ -124,6 +127,7 @@ async function one(workers: number, kind: keyof typeof QUERIES) { const write = performance.now() - t0; await done; const all = performance.now() - t0; + if (k === 0) first = all; if (k >= WARMUP) rows.push({ write, stall: lag, all }); } clearInterval(ticker); @@ -163,6 +167,7 @@ async function one(workers: number, kind: keyof typeof QUERIES) { stallMs: median(rows.map((r) => r.stall)), mainBusyMs: median(busy), allUpdatedMs: median(rows.map((r) => r.all)), + firstUpdatedMs: first, rssMb: rss, workerRuns: runtime.queryWorkerRuns, }; diff --git a/src/app.ts b/src/app.ts index 97286aa..3ca27c7 100644 --- a/src/app.ts +++ b/src/app.ts @@ -84,7 +84,7 @@ export type AppOptions = { actionWorkers?: number; /** The action workers' program, for a binary from `onebase build`. */ actionWorkerEntry?: string | URL; - /** Worker threads that re-run live queries on SQLite (default: $ONEBASE_QUERY_WORKERS, else 0: the main thread). */ + /** Worker threads that re-run live queries on SQLite (default: $ONEBASE_QUERY_WORKERS, else up to 2; 0: the main thread). */ queryWorkers?: number; /** How much one query or mutation may read and write (default: DEFAULT_CALL_LIMITS). */ callLimits?: Partial; diff --git a/src/engine/query-pool.ts b/src/engine/query-pool.ts index bb8f093..83655e3 100644 --- a/src/engine/query-pool.ts +++ b/src/engine/query-pool.ts @@ -4,7 +4,8 @@ // the writer). Shared databases (Postgres, MySQL, libSQL) always run queries on the main thread. // // Which runs come here is sync.ts's choice: background re-runs only, the ones nobody is waiting -// for. Everything else (a subscription's first run, the queries a write waits for, one-off reads) +// for, and only once a round of them has kept the main thread busy for a few milliseconds, so a +// small fan-out never pays for a thread. Everything else (a subscription's first run, the queries a write waits for, one-off reads) // stays on the main thread. // // Snapshots: a re-run is sent only after the commit that caused it has finished, and the worker @@ -34,6 +35,16 @@ export const QUERIES_PER_WORKER = 4; /** At most this many query workers: one core stays for the main thread. */ export const maxQueryWorkers = () => Math.max(1, availableParallelism() - 1); +/** + * Query workers when nothing says how many: two, or fewer on a small machine, none on a single + * core. One worker already takes most of the work off the main thread; the second mostly gets a + * large fan-out to its subscribers sooner. They start only when a fan-out is large enough to need + * them (see SubscriptionManager.pump), so a small app never starts one. + */ +export const defaultQueryWorkers = () => Math.min(2, availableParallelism() - 1); +/** A round of background re-runs works on the main thread this long before the rest goes to the workers. */ +export const QUERY_WORKERS_AFTER_MS = 4; + export type QueryCall = { name: string; args: unknown; identity: UserIdentity | null }; /** How a query run on a worker ended. */ @@ -65,6 +76,8 @@ class Thread { idleTimer: ReturnType | null = null; failure: string | null = null; functions: string[] = []; + /** It has loaded the code and takes queries. */ + loaded = false; constructor(entry: string | URL, init: ToQueryThread, onMessage: (thread: Thread, m: FromQueryThread) => void, onExit: (thread: Thread, reason: string) => void) { // SHARE_ENV: the code sees the environment variables set from the dashboard, as on the main thread. @@ -75,6 +88,7 @@ class Thread { this.worker.onmessage = ({ data }: MessageEvent) => { if (data.t === "ready") { this.functions = data.functions; + this.loaded = true; return settle(true); } if (data.t === "failed") { @@ -129,6 +143,16 @@ export class QueryPool { return !this.broken && !this.mainOnly.has(name); } + /** Whether a worker has loaded the code, so a query sent now doesn't wait for that. */ + get warm(): boolean { + return this.threads.some((t) => t.loaded && !t.dead); + } + + /** Starts the first worker, if there is none, so it is ready by the time it is needed. */ + warmUp() { + if (!this.broken && !this.threads.some((t) => !t.dead)) this.idle(this.spawn()); + } + /** Runs a query on a worker. Never rejects: how it ended is in the outcome. */ run(call: QueryCall, limitMs: number): Promise { if (this.broken) return Promise.resolve({ ok: false, mainThread: this.broken, lines: [] }); diff --git a/src/engine/runtime.ts b/src/engine/runtime.ts index e944802..e5127d0 100644 --- a/src/engine/runtime.ts +++ b/src/engine/runtime.ts @@ -9,7 +9,7 @@ import type { SchemaDefinition } from "../schema"; import { isRegisteredFunction, type CronJob, type HttpAction, type HttpRouter, type RegisteredFunction, type UserIdentity } from "../server"; import { OneBaseError, describeValue, isPlainObject, validate, type AnyValidator } from "../values"; import { ActionPool, defaultActionWorkers, defaultWorkerEntry } from "./action-pool"; -import { QueryPool } from "./query-pool"; +import { defaultQueryWorkers, QUERY_WORKERS_AFTER_MS, QueryPool } from "./query-pool"; import type { CodeSource } from "./action-wire"; import { checkArgs, checkReturn, indexCode, lookupFunction, lookupRoute, type MountCode } from "./code"; import { buildCatalog, TableScope, tableMount, validatesTable, type Catalog } from "./catalog"; @@ -164,10 +164,17 @@ export type RuntimeOptions = { actionWorkerEntry?: string | URL; /** * Worker threads that re-run live queries in the background, each with its own read-only - * connection (default: $ONEBASE_QUERY_WORKERS, else 0: they re-run on the main thread). Only - * for SQLite on a file, outside dev mode, and code with a `source`. See query-pool.ts. + * connection (default: $ONEBASE_QUERY_WORKERS, else up to 2, by the machine's cores). 0 keeps + * every query on the main thread. Only for SQLite on a file, outside dev mode, and code with a + * `source`. See query-pool.ts. */ queryWorkers?: number; + /** + * How long a round of background re-runs works on the main thread before the rest of it goes to + * the query workers (default: QUERY_WORKERS_AFTER_MS). 0 sends every background re-run to a + * worker, even while the first one is still loading the code. + */ + queryWorkersAfterMs?: number; /** How much one query or mutation may read and write (default: DEFAULT_CALL_LIMITS). */ callLimits?: Partial; /** Applies stored environment variables to process.env (default: the process's one). */ @@ -382,6 +389,8 @@ export class Runtime { /** Re-runs the current code's live queries on worker threads; null: on the main thread. */ queryPool: QueryPool | null = null; private readonly queryWorkers: number; + /** See RuntimeOptions.queryWorkersAfterMs. */ + readonly queryWorkersAfterMs: number; /** Live-query re-runs answered by a query worker, for /api/metrics. */ queryWorkerRuns = 0; /** @@ -434,7 +443,8 @@ export class Runtime { this.loadTimeoutMs = opts.loadTimeoutMs ?? 30_000; this.actionWorkers = opts.actionWorkers ?? envNumber("ONEBASE_ACTION_WORKERS") ?? defaultActionWorkers(); this.actionWorkerEntry = opts.actionWorkerEntry ?? defaultWorkerEntry(); - this.queryWorkers = Math.floor(opts.queryWorkers ?? envNumber("ONEBASE_QUERY_WORKERS") ?? 0); + this.queryWorkers = Math.floor(opts.queryWorkers ?? envNumber("ONEBASE_QUERY_WORKERS") ?? defaultQueryWorkers()); + this.queryWorkersAfterMs = opts.queryWorkersAfterMs ?? QUERY_WORKERS_AFTER_MS; this.timeLimits = { ...DEFAULT_TIME_LIMITS }; for (const [key, ms] of Object.entries(opts.timeLimits ?? {}) as [keyof TimeLimits, number | undefined][]) { if (ms === undefined) continue; // unset: keep the default @@ -1875,6 +1885,12 @@ export class Runtime { runQueryOnWorker(name: string, args: unknown, opts: CallOptions & { readSet: ReadSet }): Promise<{ json: string; readSet: ReadSet }> | null { const pool = this.queryPool; if (!pool || !pool.accepts(name)) return null; + // Loading the code takes a worker tens of milliseconds. Unless told to always use workers, + // the main thread carries on meanwhile instead of making subscribers wait for that. + if (this.queryWorkersAfterMs > 0 && !pool.warm) { + pool.warmUp(); + return null; + } const generation = this.codeGeneration; const { fn, mount } = this.resolve(name, "query", opts.allowInternal ?? false, this.callerOf(opts)); return this.calls.track(name, "query", opts.source ?? "function", async () => { diff --git a/src/engine/sync.ts b/src/engine/sync.ts index 50d274f..26a3225 100644 --- a/src/engine/sync.ts +++ b/src/engine/sync.ts @@ -195,6 +195,8 @@ export class SubscriptionManager { /** Entries waiting for a background re-run. */ private backlog = new Set(); private pumping = false; + /** The round of background re-runs in progress has run long enough to use the query workers. */ + private offloadingRound = false; /** Counters for /api/metrics. */ readonly counts = { runs: 0, background: 0, shared: 0, split: 0 }; @@ -376,7 +378,7 @@ export class SubscriptionManager { private async run(entry: Entry, reads: ReadSet, background: boolean): Promise { const generation = this.runtime.codeGeneration; const opts = { identity: entry.identity, source: entry.source, readSet: reads }; - const onWorker = background ? this.runtime.runQueryOnWorker(entry.name, entry.args, opts) : null; + const onWorker = background && this.offloading ? this.runtime.runQueryOnWorker(entry.name, entry.args, opts) : null; if (onWorker) { const { json, readSet } = await onWorker; return { update: workerUpdate(json), readSet, json, generation }; @@ -526,10 +528,14 @@ export class SubscriptionManager { private async pump() { this.pumping = true; try { - let since = performance.now(); + const started = performance.now(); + let since = started; const inFlight = new Set>(); while (this.backlog.size || inFlight.size) { - const limit = this.runtime.queryWorkerSlots || BACKGROUND_BATCH; + // A round that is still going after a few milliseconds is a large fan-out: the rest of it + // goes to the query workers. A small one is done before that, with no thread in between. + if (!this.offloadingRound && this.backlog.size && performance.now() - started >= this.runtime.queryWorkersAfterMs) this.offloadingRound = true; + const limit = (this.offloading && this.runtime.queryWorkerSlots) || BACKGROUND_BATCH; for (const e of this.backlog) { if (inFlight.size >= limit) break; this.backlog.delete(e); @@ -547,9 +553,15 @@ export class SubscriptionManager { } } finally { this.pumping = false; + this.offloadingRound = false; } } + /** Whether background re-runs go to the query workers now (when there are some). */ + private get offloading(): boolean { + return this.offloadingRound || this.runtime.queryWorkersAfterMs === 0; + } + /** Subscriptions waiting for a background re-run. */ get waiting() { return this.backlog.size; diff --git a/tests/query-workers.test.ts b/tests/query-workers.test.ts index f9e0123..2eb6086 100644 --- a/tests/query-workers.test.ts +++ b/tests/query-workers.test.ts @@ -102,6 +102,7 @@ async function start(name: string, files: Record, opts: Partial< const started = (await openRuntime(() => new SqliteDriver(join(root, name, "db.sqlite")), { ...code, queryWorkers: 2, + queryWorkersAfterMs: 0, // every background re-run goes to a worker, however small the fan-out timeLimits: { queryMs: LIMIT }, ...opts, })) as { runtime: Runtime; driver: SqliteDriver }; @@ -300,6 +301,44 @@ test("a deploy while a worker runs the query: the old code's result is never del expect(got.at(-1)).toMatchObject({ v: "v2", where: "worker" }); }); +test("by default a small fan-out stays on the main thread and starts no worker; a large one moves to the workers", async () => { + // As `onebase serve` runs it: workers allowed, but only used once a round of re-runs has gone on for a few milliseconds. + const app = await start("adaptive", FILES, { queryWorkersAfterMs: undefined }); + const pool = app.runtime.queryPool!; + expect(pool).not.toBeNull(); + const watchAs = (name: string, args: Record, who: string) => { + const got: any[] = []; + app.subs.subscribe(name, args, user(who), (u) => got.push(u.ok ? u.value : u)); + return got; + }; + const idle = () => until(() => app.subs.waiting === 0, "the background re-runs"); + + const few = Array.from({ length: 5 }, (_, i) => watchAs("notes:mine", {}, `u${i}`)); + await until(() => few.every((w) => w.length > 0), "the first results"); + for (let i = 0; i < 5; i++) { + await app.runtime.runMutation("notes:add", { text: `small${i}`, owner: "u1" }, { settle: "none" }); + await until(() => few[1]!.at(-1).texts.includes(`small${i}`), `small${i}`); + await idle(); + } + expect(few.every((w) => w.at(-1).where === "main")).toBe(true); + expect(app.runtime.queryWorkerRuns).toBe(0); + expect(pool.size).toBe(0); + + // 80 queries of a few milliseconds each: far more than the main thread does before handing over. + const many = Array.from({ length: 80 }, (_, i) => watchAs("notes:slow", { n: 2_000_000 + i }, "u0")); + await until(() => many.every((w) => w.length > 0), "the first results", 20_000); + for (const text of ["large1", "large2"]) { + await app.runtime.runMutation("notes:add", { text }, { settle: "none" }); + await until(() => many.every((w) => w.at(-1).texts.includes(text)), text, 20_000); + await idle(); + } + // The first write's round starts a worker and carries on meanwhile; by the second, it is ready. + expect(app.runtime.queryWorkerRuns).toBeGreaterThan(0); + expect(many.some((w) => w.at(-1).where === "worker")).toBe(true); + // The start of every round still runs on the main thread. + expect(many.some((w) => w.at(-1).where === "main")).toBe(true); +}, 60_000); + test("queryWorkers: 0, dev mode and in-memory code keep every query on the main thread", async () => { const off = await start("off", FILES, { queryWorkers: 0 }); expect(off.runtime.queryPool).toBeNull();