From bd7456d50b831837b7ca81823bd97db2d044dc7e Mon Sep 17 00:00:00 2001 From: Jarek Krochmalski Date: Wed, 23 Sep 2026 05:56:15 +0200 Subject: [PATCH 1/4] don't crash when a quiet backup log stream trips undici's body timeout --- src/hooks.server.ts | 1 + src/lib/server/crash-guard.ts | 39 ++++++++++++++++ src/lib/server/dns-dispatcher.ts | 28 +++++++++++- src/lib/server/docker.ts | 16 ++++++- tests/crash-guard.test.ts | 73 ++++++++++++++++++++++++++++++ tests/streaming-dispatcher.test.ts | 28 ++++++++++++ vite.config.ts | 6 ++- 7 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 src/lib/server/crash-guard.ts create mode 100644 tests/crash-guard.test.ts create mode 100644 tests/streaming-dispatcher.test.ts diff --git a/src/hooks.server.ts b/src/hooks.server.ts index d0ea8aa09..5317f2160 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -1,4 +1,5 @@ // v1.0.12 +import '$lib/server/crash-guard.js'; import '$lib/server/dns-dispatcher.js'; import { initDatabase, hasAnyUser } from '$lib/server/db'; import { startSubprocesses, stopSubprocesses } from '$lib/server/subprocess-manager'; diff --git a/src/lib/server/crash-guard.ts b/src/lib/server/crash-guard.ts new file mode 100644 index 000000000..b60e95395 --- /dev/null +++ b/src/lib/server/crash-guard.ts @@ -0,0 +1,39 @@ +// Process-level safety net: a rejected promise that nothing awaits (e.g. an aborted +// streaming Docker request surfacing UND_ERR_BODY_TIMEOUT from undici's internals) would +// otherwise reach Node's default handler and terminate the whole server. Dockhand is a +// long-lived process serving many independent operations, so one background failure must +// not take everything down. Log it loudly (so it is never silent) and keep running. +// +// Scope is deliberately narrow: ONLY unhandledRejection. uncaughtException is left to +// Node's default (crash) - after a genuine uncaught throw the process state is undefined +// and continuing is unsafe; a rejected promise carries no such guarantee-of-corruption. +// +// Side-effect module: import once, as early as possible. + +/** + * Turn a rejection reason into the lines to log. Pure so it can be unit-tested without + * registering a real process listener. Never throws. + */ +export function formatUnhandledRejection(reason: unknown): string[] { + const err = reason instanceof Error ? reason : new Error(String(reason)); + const code = (err as NodeJS.ErrnoException).code; + const lines = [ + `[crash-guard] Unhandled promise rejection (process kept alive)${code ? ` [${code}]` : ''}: ${err.message}` + ]; + if (err.stack) lines.push(err.stack); + const cause = (err as Error & { cause?: unknown }).cause; + if (cause) lines.push(`[crash-guard] caused by: ${cause instanceof Error ? cause.stack || cause.message : String(cause)}`); + return lines; +} + +let installed = false; + +export function installCrashGuard(): void { + if (installed) return; + installed = true; + process.on('unhandledRejection', (reason: unknown) => { + for (const line of formatUnhandledRejection(reason)) console.error(line); + }); +} + +installCrashGuard(); diff --git a/src/lib/server/dns-dispatcher.ts b/src/lib/server/dns-dispatcher.ts index c906d5bb7..c4291c59d 100644 --- a/src/lib/server/dns-dispatcher.ts +++ b/src/lib/server/dns-dispatcher.ts @@ -1,4 +1,4 @@ -import { setGlobalDispatcher, Agent, EnvHttpProxyAgent } from 'undici'; +import { setGlobalDispatcher, Agent, EnvHttpProxyAgent, type Dispatcher } from 'undici'; import dns from 'node:dns'; import net from 'node:net'; @@ -136,3 +136,29 @@ if (ALLOW_IPV6) { } else { setGlobalDispatcher(new Agent({ connect: connectOptions })); } + +// Timeouts for long-lived streaming Docker requests (container logs follow, events, +// the backup helper's restic progress). undici's default bodyTimeout (300s) aborts a +// response body that goes quiet - which a backup legitimately does while restic uploads +// to a remote repo - and the abort surfaces as an unhandled UND_ERR_BODY_TIMEOUT that +// crashes the process. Only bodyTimeout is disabled: the body is the quiet phase, and +// the caller's AbortController (fired on container exit) cancels the read. headersTimeout +// keeps undici's default so a daemon that connects but never sends a response line is +// still bounded (nothing else caps that pre-header wait). +export const STREAMING_DISPATCHER_TIMEOUTS = { bodyTimeout: 0 }; + +// Dispatcher for streaming requests: the SAME proxy/DNS choice the global block makes +// (proxy honored when set; IPv4-pinning connect only when ALLOW_IPV6 is off, so the +// IPv6 opt-in keeps the native resolver), only with bodyTimeout added. Lazily built and +// reused (one dispatcher, not one per request). +let streamingDispatcher: Dispatcher | undefined; +export function getStreamingDispatcher(): Dispatcher { + if (!streamingDispatcher) { + // connectOptions carries the IPv4 pinning; ALLOW_IPV6 mode drops it for the native resolver. + const connect = ALLOW_IPV6 ? {} : { connect: connectOptions }; + streamingDispatcher = hasProxy + ? new EnvHttpProxyAgent({ ...connect, ...STREAMING_DISPATCHER_TIMEOUTS }) + : new Agent({ ...connect, ...STREAMING_DISPATCHER_TIMEOUTS }); + } + return streamingDispatcher; +} diff --git a/src/lib/server/docker.ts b/src/lib/server/docker.ts index 0e459dbe4..c3f0735e4 100644 --- a/src/lib/server/docker.ts +++ b/src/lib/server/docker.ts @@ -943,6 +943,17 @@ export async function dockerFetch( (finalOptions as RequestInit & { duplex?: string }).duplex = 'half'; } + // A streaming response (logs follow, events, backup helper progress) can stay + // quiet longer than undici's 300s bodyTimeout, which would abort it as an + // unhandled UND_ERR_BODY_TIMEOUT. Route these through a dispatcher whose bodyTimeout + // is disabled; they are bounded by their own AbortController (fired on container + // exit) instead. headersTimeout stays at undici's default so a pre-header stall + // is still capped. + if (streaming) { + const { getStreamingDispatcher } = await import('./dns-dispatcher'); + (finalOptions as RequestInit & { dispatcher?: unknown }).dispatcher = getStreamingDispatcher(); + } + try { const response = await fetch(url, finalOptions); const elapsed = Date.now() - startTime; @@ -5486,9 +5497,12 @@ async function streamLocalStderr( onStdout?: (data: string) => void ): Promise { const wantStdout = onStdout ? 'true' : 'false'; + // Pass the caller's abort signal to the fetch itself, not only to the reader below: + // with no body timeout on the streaming dispatcher, a fetch that stalls BEFORE the + // response arrives has no other bound, so container-exit must be able to cancel it. const response = await dockerFetch( `/containers/${containerId}/logs?stdout=${wantStdout}&stderr=true&follow=true`, - { streaming: true }, + { streaming: true, signal }, envId ); diff --git a/tests/crash-guard.test.ts b/tests/crash-guard.test.ts new file mode 100644 index 000000000..5a92c0475 --- /dev/null +++ b/tests/crash-guard.test.ts @@ -0,0 +1,73 @@ +import { describe, test, expect } from 'bun:test'; +import { formatUnhandledRejection, installCrashGuard } from '../src/lib/server/crash-guard'; + +// The crash guard keeps Dockhand alive on an unhandled promise rejection and logs a full +// dump. formatUnhandledRejection is the pure part - it must never throw and must surface +// message, code, stack, and cause so the leak is traceable. +describe('crash guard formatting', () => { + test('includes message and stack for an Error', () => { + const err = new Error('boom'); + const lines = formatUnhandledRejection(err); + expect(lines[0]).toContain('boom'); + expect(lines[0]).toContain('process kept alive'); + expect(lines.some((l) => l.includes('crash-guard.test'))).toBe(true); // the stack + }); + + test('includes an error code when present (e.g. undici body timeout)', () => { + const err = Object.assign(new Error('Body Timeout Error'), { code: 'UND_ERR_BODY_TIMEOUT' }); + const lines = formatUnhandledRejection(err); + expect(lines[0]).toContain('[UND_ERR_BODY_TIMEOUT]'); + }); + + test('surfaces a cause when the rejection wraps one', () => { + const cause = new Error('BodyTimeoutError'); + const err = new Error('terminated', { cause }); + const lines = formatUnhandledRejection(err); + expect(lines.some((l) => l.includes('caused by') && l.includes('BodyTimeoutError'))).toBe(true); + }); + + test('handles a non-Error reason without throwing', () => { + const lines = formatUnhandledRejection('just a string'); + expect(lines[0]).toContain('just a string'); + }); + + test('stringifies a null/undefined reason into the output', () => { + expect(() => formatUnhandledRejection(null)).not.toThrow(); + expect(formatUnhandledRejection(null)[0]).toContain('null'); + expect(formatUnhandledRejection(undefined)[0]).toContain('undefined'); + }); +}); + +// The installed handler is the safety-critical part: it must log-and-return (never exit +// or rethrow) so a background rejection cannot crash the process, and installCrashGuard +// must be idempotent (import already installed one listener at module load). +describe('crash guard installation', () => { + test('installCrashGuard is idempotent - a second call adds no listener', () => { + const before = process.listenerCount('unhandledRejection'); + installCrashGuard(); + installCrashGuard(); + expect(process.listenerCount('unhandledRejection')).toBe(before); + }); + + test('the installed handler logs and does not throw or exit', () => { + const listeners = process.listeners('unhandledRejection'); + expect(listeners.length).toBeGreaterThan(0); + const handler = listeners[listeners.length - 1] as (reason: unknown) => void; + + const origError = console.error; + const origExit = process.exit; + let logged = 0; + let exited = false; + console.error = () => { logged++; }; + // @ts-expect-error - stub exit to catch an accidental process.exit + process.exit = () => { exited = true; }; + try { + expect(() => handler(Object.assign(new Error('boom'), { code: 'UND_ERR_BODY_TIMEOUT' }))).not.toThrow(); + } finally { + console.error = origError; + process.exit = origExit; + } + expect(logged).toBeGreaterThan(0); + expect(exited).toBe(false); + }); +}); diff --git a/tests/streaming-dispatcher.test.ts b/tests/streaming-dispatcher.test.ts new file mode 100644 index 000000000..d6af3fe5d --- /dev/null +++ b/tests/streaming-dispatcher.test.ts @@ -0,0 +1,28 @@ +import { describe, test, expect } from 'bun:test'; +import { Agent } from 'undici'; +import { getStreamingDispatcher, STREAMING_DISPATCHER_TIMEOUTS } from '../src/lib/server/dns-dispatcher'; + +// The streaming dispatcher exists so a long, quiet Docker log/backup stream is not +// aborted by undici's default 300s bodyTimeout. Assert its timeouts are off and that +// it is a reused singleton (one Agent, not one per streaming request). +describe('streaming dispatcher', () => { + test('body timeout is disabled', () => { + // 0 = no timeout in undici. A non-zero bodyTimeout would let a quiet backup stream + // be aborted mid-run, the crash this dispatcher prevents. + expect(STREAMING_DISPATCHER_TIMEOUTS.bodyTimeout).toBe(0); + }); + + test('headers timeout is left at undici default', () => { + // Only the body goes quiet; the pre-header wait must stay bounded, so headersTimeout + // is deliberately NOT overridden (a wedged daemon that never replies is still capped). + expect('headersTimeout' in STREAMING_DISPATCHER_TIMEOUTS).toBe(false); + }); + + test('is an undici Agent', () => { + expect(getStreamingDispatcher()).toBeInstanceOf(Agent); + }); + + test('is a lazily-built singleton', () => { + expect(getStreamingDispatcher()).toBe(getStreamingDispatcher()); + }); +}); diff --git a/vite.config.ts b/vite.config.ts index 247cdac22..08e03b627 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -527,7 +527,11 @@ const wsToEnvId = new Map(); function webSocketPlugin(): Plugin { return { name: 'websocket', - configureServer() { + async configureServer() { + // Install the process crash guard before the dev WS server accepts anything, + // so dev matches prod (where it installs before listen via hooks.server.ts). + await import('./src/lib/server/crash-guard.js'); + // Start cleanup interval for dev mode only startCleanupInterval(); From ab56312b5ffc1fe72ea5e4b4d690692434dd8619 Mon Sep 17 00:00:00 2001 From: Jarek Krochmalski Date: Wed, 23 Sep 2026 05:56:15 +0200 Subject: [PATCH 2/4] select-all checkbox shows a partial state, not a full tick, for a partial selection --- src/lib/components/data-grid/DataGrid.svelte | 4 ++-- src/routes/images/+page.svelte | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/components/data-grid/DataGrid.svelte b/src/lib/components/data-grid/DataGrid.svelte index 67891ee83..9413759c0 100644 --- a/src/lib/components/data-grid/DataGrid.svelte +++ b/src/lib/components/data-grid/DataGrid.svelte @@ -1,7 +1,7 @@