Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/hooks.server.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// v1.0.12
import '$lib/server/crash-guard.js';
import '$lib/server/dns-dispatcher.js';
import { initDatabase, hasAnyUser } from '$lib/server/db';
import { startSubprocesses, stopSubprocesses } from '$lib/server/subprocess-manager';
Expand Down
4 changes: 2 additions & 2 deletions src/lib/components/data-grid/DataGrid.svelte
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<script lang="ts" generics="T">
import { onMount, onDestroy } from 'svelte';
import type { Snippet } from 'svelte';
import { CheckSquare, Square as SquareIcon, ArrowUp, ArrowDown, ArrowUpDown, ChevronDown, ChevronRight } from 'lucide-svelte';
import { CheckSquare, SquareMinus, Square as SquareIcon, ArrowUp, ArrowDown, ArrowUpDown, ChevronDown, ChevronRight } from 'lucide-svelte';
import { columnResize } from '$lib/actions/column-resize';
import { gridPreferencesStore } from '$lib/stores/grid-preferences';
import { getAllColumnConfigs } from '$lib/config/grid-columns';
Expand Down Expand Up @@ -810,7 +810,7 @@
{#if allSelected}
<CheckSquare class="w-3.5 h-3.5 text-muted-foreground" />
{:else if someSelected}
<CheckSquare class="w-3.5 h-3.5 text-muted-foreground" />
<SquareMinus class="w-3.5 h-3.5 text-muted-foreground" />
{:else}
<SquareIcon class="w-3.5 h-3.5 text-muted-foreground" />
{/if}
Expand Down
2 changes: 2 additions & 0 deletions src/lib/data/changelog.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
"date": "2026-09-22",
"comingSoon": true,
"changes": [
{ "type": "fix", "text": "select-all checkbox shows a partial state, not a full tick, for a partial selection (#1605)" },
{ "type": "fix", "text": "a long backup no longer crashes the app when its log stream goes quiet (#1610)" },
{ "type": "feature", "text": "tag containers and stacks, then filter and group the grid by tag (#781, #870)" },
{ "type": "fix", "text": "removing all of a stack's environment variables no longer leaves stale entries (#1598)" },
{ "type": "fix", "text": "templates with duplicate names no longer stop the templates page from rendering" },
Expand Down
39 changes: 39 additions & 0 deletions src/lib/server/crash-guard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Process-level safety net: a rejected promise that nothing awaits (e.g. an aborted
// streaming Docker request surfacing UND_ERR_BODY_TIMEOUT from undici's internals) would
// otherwise reach Node's default handler and terminate the whole server. Dockhand is a
// long-lived process serving many independent operations, so one background failure must
// not take everything down. Log it loudly (so it is never silent) and keep running.
//
// Scope is deliberately narrow: ONLY unhandledRejection. uncaughtException is left to
// Node's default (crash) - after a genuine uncaught throw the process state is undefined
// and continuing is unsafe; a rejected promise carries no such guarantee-of-corruption.
//
// Side-effect module: import once, as early as possible.

/**
* Turn a rejection reason into the lines to log. Pure so it can be unit-tested without
* registering a real process listener. Never throws.
*/
export function formatUnhandledRejection(reason: unknown): string[] {
const err = reason instanceof Error ? reason : new Error(String(reason));
const code = (err as NodeJS.ErrnoException).code;
const lines = [
`[crash-guard] Unhandled promise rejection (process kept alive)${code ? ` [${code}]` : ''}: ${err.message}`
];
if (err.stack) lines.push(err.stack);
const cause = (err as Error & { cause?: unknown }).cause;
if (cause) lines.push(`[crash-guard] caused by: ${cause instanceof Error ? cause.stack || cause.message : String(cause)}`);
return lines;
}

let installed = false;

export function installCrashGuard(): void {
if (installed) return;
installed = true;
process.on('unhandledRejection', (reason: unknown) => {
for (const line of formatUnhandledRejection(reason)) console.error(line);
});
}

installCrashGuard();
5 changes: 4 additions & 1 deletion src/lib/server/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4120,7 +4120,10 @@ export async function getContainerEvents(filters: ContainerEventFilters = {}): P
.from(containerEvents)
.leftJoin(environments, eq(containerEvents.environmentId, environments.id))
.where(whereClause)
.orderBy(desc(containerEvents.timestamp))
// id is the tie-breaker: timestamp alone is not unique (a burst of events shares
// a timestamp), and offset pagination over a non-unique sort is non-deterministic,
// so a row can repeat or vanish across pages.
.orderBy(desc(containerEvents.timestamp), desc(containerEvents.id))
.limit(limit)
.offset(offset);

Expand Down
28 changes: 27 additions & 1 deletion src/lib/server/dns-dispatcher.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { setGlobalDispatcher, Agent, EnvHttpProxyAgent } from 'undici';
import { setGlobalDispatcher, Agent, EnvHttpProxyAgent, type Dispatcher } from 'undici';
import dns from 'node:dns';
import net from 'node:net';

Expand Down Expand Up @@ -136,3 +136,29 @@ if (ALLOW_IPV6) {
} else {
setGlobalDispatcher(new Agent({ connect: connectOptions }));
}

// Timeouts for long-lived streaming Docker requests (container logs follow, events,
// the backup helper's restic progress). undici's default bodyTimeout (300s) aborts a
// response body that goes quiet - which a backup legitimately does while restic uploads
// to a remote repo - and the abort surfaces as an unhandled UND_ERR_BODY_TIMEOUT that
// crashes the process. Only bodyTimeout is disabled: the body is the quiet phase, and
// the caller's AbortController (fired on container exit) cancels the read. headersTimeout
// keeps undici's default so a daemon that connects but never sends a response line is
// still bounded (nothing else caps that pre-header wait).
export const STREAMING_DISPATCHER_TIMEOUTS = { bodyTimeout: 0 };

// Dispatcher for streaming requests: the SAME proxy/DNS choice the global block makes
// (proxy honored when set; IPv4-pinning connect only when ALLOW_IPV6 is off, so the
// IPv6 opt-in keeps the native resolver), only with bodyTimeout added. Lazily built and
// reused (one dispatcher, not one per request).
let streamingDispatcher: Dispatcher | undefined;
export function getStreamingDispatcher(): Dispatcher {
if (!streamingDispatcher) {
// connectOptions carries the IPv4 pinning; ALLOW_IPV6 mode drops it for the native resolver.
const connect = ALLOW_IPV6 ? {} : { connect: connectOptions };
streamingDispatcher = hasProxy
? new EnvHttpProxyAgent({ ...connect, ...STREAMING_DISPATCHER_TIMEOUTS })
: new Agent({ ...connect, ...STREAMING_DISPATCHER_TIMEOUTS });
}
return streamingDispatcher;
}
16 changes: 15 additions & 1 deletion src/lib/server/docker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,17 @@ export async function dockerFetch(
(finalOptions as RequestInit & { duplex?: string }).duplex = 'half';
}

// A streaming response (logs follow, events, backup helper progress) can stay
// quiet longer than undici's 300s bodyTimeout, which would abort it as an
// unhandled UND_ERR_BODY_TIMEOUT. Route these through a dispatcher whose bodyTimeout
// is disabled; they are bounded by their own AbortController (fired on container
// exit) instead. headersTimeout stays at undici's default so a pre-header stall
// is still capped.
if (streaming) {
const { getStreamingDispatcher } = await import('./dns-dispatcher');
(finalOptions as RequestInit & { dispatcher?: unknown }).dispatcher = getStreamingDispatcher();
}

try {
const response = await fetch(url, finalOptions);
const elapsed = Date.now() - startTime;
Expand Down Expand Up @@ -5486,9 +5497,12 @@ async function streamLocalStderr(
onStdout?: (data: string) => void
): Promise<void> {
const wantStdout = onStdout ? 'true' : 'false';
// Pass the caller's abort signal to the fetch itself, not only to the reader below:
// with no body timeout on the streaming dispatcher, a fetch that stalls BEFORE the
// response arrives has no other bound, so container-exit must be able to cancel it.
const response = await dockerFetch(
`/containers/${containerId}/logs?stdout=${wantStdout}&stderr=true&follow=true`,
{ streaming: true },
{ streaming: true, signal },
envId
);

Expand Down
4 changes: 2 additions & 2 deletions src/routes/images/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import * as Dialog from '$lib/components/ui/dialog';
import * as Select from '$lib/components/ui/select';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import { Trash2, Upload, RefreshCw, Play, Layers, Server, ShieldCheck, ShieldAlert, Shield, ShieldQuestion, CheckSquare, Square, Tag, Check, XCircle, Icon, AlertTriangle, X, Images, Copy, Download, ChevronRight, ChevronDown, Loader2, ArrowUp, ArrowDown, ArrowUpDown, CircleDashed, CircleDot, Circle, Filter, FileJson, FileSpreadsheet, ShieldPlus } from 'lucide-svelte';
import { Trash2, Upload, RefreshCw, Play, Layers, Server, ShieldCheck, ShieldAlert, Shield, ShieldQuestion, CheckSquare, SquareMinus, Square, Tag, Check, XCircle, Icon, AlertTriangle, X, Images, Copy, Download, ChevronRight, ChevronDown, Loader2, ArrowUp, ArrowDown, ArrowUpDown, CircleDashed, CircleDot, Circle, Filter, FileJson, FileSpreadsheet, ShieldPlus } from 'lucide-svelte';
import { broom, whale } from '@lucide/lab';
import { formatBytes } from '$lib/utils/format';
import * as Tooltip from '$lib/components/ui/tooltip';
Expand Down Expand Up @@ -1245,7 +1245,7 @@
{#if allSelected}
<CheckSquare class="w-3.5 h-3.5 text-muted-foreground" />
{:else if someSelected}
<CheckSquare class="w-3.5 h-3.5 text-muted-foreground" />
<SquareMinus class="w-3.5 h-3.5 text-muted-foreground" />
{:else}
<Square class="w-3.5 h-3.5 text-muted-foreground" />
{/if}
Expand Down
73 changes: 73 additions & 0 deletions tests/crash-guard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { describe, test, expect } from 'bun:test';
import { formatUnhandledRejection, installCrashGuard } from '../src/lib/server/crash-guard';

// The crash guard keeps Dockhand alive on an unhandled promise rejection and logs a full
// dump. formatUnhandledRejection is the pure part - it must never throw and must surface
// message, code, stack, and cause so the leak is traceable.
describe('crash guard formatting', () => {
test('includes message and stack for an Error', () => {
const err = new Error('boom');
const lines = formatUnhandledRejection(err);
expect(lines[0]).toContain('boom');
expect(lines[0]).toContain('process kept alive');
expect(lines.some((l) => l.includes('crash-guard.test'))).toBe(true); // the stack
});

test('includes an error code when present (e.g. undici body timeout)', () => {
const err = Object.assign(new Error('Body Timeout Error'), { code: 'UND_ERR_BODY_TIMEOUT' });
const lines = formatUnhandledRejection(err);
expect(lines[0]).toContain('[UND_ERR_BODY_TIMEOUT]');
});

test('surfaces a cause when the rejection wraps one', () => {
const cause = new Error('BodyTimeoutError');
const err = new Error('terminated', { cause });
const lines = formatUnhandledRejection(err);
expect(lines.some((l) => l.includes('caused by') && l.includes('BodyTimeoutError'))).toBe(true);
});

test('handles a non-Error reason without throwing', () => {
const lines = formatUnhandledRejection('just a string');
expect(lines[0]).toContain('just a string');
});

test('stringifies a null/undefined reason into the output', () => {
expect(() => formatUnhandledRejection(null)).not.toThrow();
expect(formatUnhandledRejection(null)[0]).toContain('null');
expect(formatUnhandledRejection(undefined)[0]).toContain('undefined');
});
});

// The installed handler is the safety-critical part: it must log-and-return (never exit
// or rethrow) so a background rejection cannot crash the process, and installCrashGuard
// must be idempotent (import already installed one listener at module load).
describe('crash guard installation', () => {
test('installCrashGuard is idempotent - a second call adds no listener', () => {
const before = process.listenerCount('unhandledRejection');
installCrashGuard();
installCrashGuard();
expect(process.listenerCount('unhandledRejection')).toBe(before);
});

test('the installed handler logs and does not throw or exit', () => {
const listeners = process.listeners('unhandledRejection');
expect(listeners.length).toBeGreaterThan(0);
const handler = listeners[listeners.length - 1] as (reason: unknown) => void;

const origError = console.error;
const origExit = process.exit;
let logged = 0;
let exited = false;
console.error = () => { logged++; };
// @ts-expect-error - stub exit to catch an accidental process.exit
process.exit = () => { exited = true; };
try {
expect(() => handler(Object.assign(new Error('boom'), { code: 'UND_ERR_BODY_TIMEOUT' }))).not.toThrow();
} finally {
console.error = origError;
process.exit = origExit;
}
expect(logged).toBeGreaterThan(0);
expect(exited).toBe(false);
});
});
28 changes: 28 additions & 0 deletions tests/streaming-dispatcher.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { describe, test, expect } from 'bun:test';
import { Agent } from 'undici';
import { getStreamingDispatcher, STREAMING_DISPATCHER_TIMEOUTS } from '../src/lib/server/dns-dispatcher';

// The streaming dispatcher exists so a long, quiet Docker log/backup stream is not
// aborted by undici's default 300s bodyTimeout. Assert its timeouts are off and that
// it is a reused singleton (one Agent, not one per streaming request).
describe('streaming dispatcher', () => {
test('body timeout is disabled', () => {
// 0 = no timeout in undici. A non-zero bodyTimeout would let a quiet backup stream
// be aborted mid-run, the crash this dispatcher prevents.
expect(STREAMING_DISPATCHER_TIMEOUTS.bodyTimeout).toBe(0);
});

test('headers timeout is left at undici default', () => {
// Only the body goes quiet; the pre-header wait must stay bounded, so headersTimeout
// is deliberately NOT overridden (a wedged daemon that never replies is still capped).
expect('headersTimeout' in STREAMING_DISPATCHER_TIMEOUTS).toBe(false);
});

test('is an undici Agent', () => {
expect(getStreamingDispatcher()).toBeInstanceOf(Agent);
});

test('is a lazily-built singleton', () => {
expect(getStreamingDispatcher()).toBe(getStreamingDispatcher());
});
});
6 changes: 5 additions & 1 deletion vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,11 @@ const wsToEnvId = new Map<any, number>();
function webSocketPlugin(): Plugin {
return {
name: 'websocket',
configureServer() {
async configureServer() {
// Install the process crash guard before the dev WS server accepts anything,
// so dev matches prod (where it installs before listen via hooks.server.ts).
await import('./src/lib/server/crash-guard.js');

// Start cleanup interval for dev mode only
startCleanupInterval();

Expand Down
Loading