diff --git a/apps/worker/src/sandbox.test.ts b/apps/worker/src/sandbox.test.ts index d7d2cd06..cb7c5848 100644 --- a/apps/worker/src/sandbox.test.ts +++ b/apps/worker/src/sandbox.test.ts @@ -7,6 +7,7 @@ import { closeDaytonaSandbox, configureDaytonaSandboxLifecycle, createDaytonaSandboxSession, + DaytonaCapacityError, deleteDaytonaSandboxByName, type DaytonaCleanupDependencies, pauseDaytonaSandbox, @@ -17,6 +18,7 @@ import { resumedSameDaytonaSandbox, sandboxDeletedAfterFailedCreation, sandboxesLeftAfterFailedCreation, + sweepDaytonaSandboxes, } from "./sandbox.js"; function cleanupHarness(options?: { @@ -477,7 +479,8 @@ describe("Daytona sandbox cleanup", () => { initialDeleteError, ); expect(sandboxDeletedAfterFailedCreation(failedBeforeCreation)).toBe(false); - expect(failedCleanup).toBeInstanceOf(AggregateError); + // A sandbox that never appeared does not hide why creation failed. + expect(failedCleanup).toHaveProperty("message", "sandbox creation timed out"); expect(sandboxDeletedAfterFailedCreation(failedCleanup)).toBe(false); expect(creator.create).toHaveBeenCalledOnce(); }); @@ -639,9 +642,11 @@ describe("Daytona sandbox cleanup", () => { it("keeps a thread sandbox after it pauses", async () => { const harness = cleanupHarness(); + const setAutoArchiveInterval = vi.fn().mockResolvedValue(undefined); const setAutoDeleteInterval = vi.fn().mockResolvedValue(undefined); harness.get.mockResolvedValue({ id: "sandbox-1", + setAutoArchiveInterval, setAutoDeleteInterval, }); @@ -656,6 +661,36 @@ describe("Daytona sandbox cleanup", () => { expect(setAutoDeleteInterval).toHaveBeenCalledWith(-1); }); + it("archives a paused sandbox so it stops counting toward Daytona's disk limit", async () => { + const harness = cleanupHarness(); + const setAutoArchiveInterval = vi.fn().mockResolvedValue(undefined); + const setAutoDeleteInterval = vi.fn().mockResolvedValue(undefined); + harness.get.mockResolvedValue({ + id: "sandbox-1", + setAutoArchiveInterval, + setAutoDeleteInterval, + }); + + await configureDaytonaSandboxLifecycle( + harness.session, + { daytonaApiKey: "daytona-test" }, + [], + 24 * 60, + harness.dependencies, + ); + await configureDaytonaSandboxLifecycle( + harness.session, + { daytonaApiKey: "daytona-test" }, + [], + 0, + harness.dependencies, + ); + + expect(setAutoDeleteInterval.mock.calls).toEqual([[24 * 60], [0]]); + // A sandbox deleted when it stops is never archived. + expect(setAutoArchiveInterval.mock.calls).toEqual([[30]]); + }); + it("retries transient lifecycle failures", async () => { const harness = cleanupHarness(); const gatewayError = Object.assign(new Error("bad gateway"), { @@ -781,6 +816,185 @@ describe("Daytona sandbox cleanup", () => { }); }); +describe("Daytona sandbox capacity", () => { + const now = Date.parse("2026-10-16T12:00:00.000Z"); + + function sandboxStopped(name: string, minutesAgo: number, state = "stopped") { + return { + archive: vi.fn().mockResolvedValue(undefined), + cpu: 2, + disk: 10, + memory: 4, + name, + state, + updatedAt: new Date(now - minutesAgo * 60_000).toISOString(), + }; + } + + function capacityHarness(sandboxes: Array>) { + const harness = cleanupHarness({ missing: true }); + const list = vi.fn((query?: { states?: string[] }) => + (async function* () { + for (const sandbox of sandboxes) { + if (!query?.states || query.states.includes(sandbox.state)) yield sandbox; + } + })()); + const dependencies = { + ...harness.dependencies, + createClient: vi.fn(() => ({ + delete: harness.deleteSandbox, + get: harness.get, + list, + [Symbol.asyncDispose]: harness.dispose, + })), + now: () => now, + } as unknown as DaytonaCleanupDependencies; + return { ...harness, dependencies, list }; + } + + it("archives the longest-stopped sandboxes and retries when Daytona's disk is full", async () => { + const longStopped = sandboxStopped("responder-automation-old", 120); + const justStopped = sandboxStopped("responder-automation-new", 2); + const harness = capacityHarness([longStopped, justStopped]); + const createdSession = { state: { sandboxId: "sandbox-2" } }; + const diskFull = new Error( + "DaytonaSandboxClient failed to create sandbox: Total disk limit exceeded. Maximum allowed: 2000GiB.", + ); + const creator = { + create: vi.fn() + .mockRejectedValueOnce(diskFull) + .mockResolvedValueOnce(createdSession), + }; + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect( + createDaytonaSandboxSession( + creator, + { daytonaApiKey: "daytona-test" }, + "responder-automation-run-1", + harness.dependencies, + ), + ).resolves.toBe(createdSession); + + expect(longStopped.archive).toHaveBeenCalledOnce(); + expect(justStopped.archive).not.toHaveBeenCalled(); + // Daytona created nothing, so the name is reused and nothing waits for + // the rejected sandbox to appear. + expect(creator.create.mock.calls).toEqual([ + [{ options: { createTimeoutSec: 120, name: "responder-automation-run-1" } }], + [{ options: { createTimeoutSec: 120, name: "responder-automation-run-1" } }], + ]); + expect(harness.get).toHaveBeenCalledTimes(2); + expect(harness.sleep).toHaveBeenCalledWith(5_000); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining("daytona_sandbox_quota_retry"), + ); + consoleError.mockRestore(); + }); + + it("waits for CPU to free up, then fails with a capacity error", async () => { + const harness = capacityHarness([sandboxStopped("responder-automation-old", 120)]); + const cpuFull = new Error( + "DaytonaSandboxClient failed to create sandbox: Total CPU limit exceeded. Maximum allowed: 500.", + ); + const creator = { create: vi.fn().mockRejectedValue(cpuFull) }; + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + const error = await createDaytonaSandboxSession( + creator, + { daytonaApiKey: "daytona-test" }, + "responder-automation-run-1", + harness.dependencies, + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(DaytonaCapacityError); + expect((error as Error).cause).toBe(cpuFull); + expect(sandboxDeletedAfterFailedCreation(error)).toBe(true); + expect(creator.create).toHaveBeenCalledTimes(6); + expect(harness.sleep.mock.calls).toEqual([[5_000], [15_000], [30_000], [60_000], [120_000]]); + // Archiving frees disk, not CPU. + expect(harness.list).not.toHaveBeenCalled(); + consoleError.mockRestore(); + }); + + it("cleans up after a rate limit instead of treating it as full capacity", async () => { + const harness = capacityHarness([]); + harness.get.mockImplementation(async (name: string) => ({ id: name })); + const createdSession = { state: { sandboxId: "sandbox-2" } }; + const rateLimited = Object.assign( + new Error("DaytonaSandboxClient failed to create sandbox: Rate limit exceeded"), + { statusCode: 429 }, + ); + const creator = { + create: vi.fn() + .mockRejectedValueOnce(rateLimited) + .mockResolvedValueOnce(createdSession), + }; + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect( + createDaytonaSandboxSession( + creator, + { daytonaApiKey: "daytona-test" }, + "responder-automation-run-1", + harness.dependencies, + ), + ).resolves.toBe(createdSession); + + // The rate-limited sandbox may exist, so it is deleted and replaced. + expect(creator.create).toHaveBeenLastCalledWith({ + options: { createTimeoutSec: 120, name: "responder-automation-run-1-2" }, + }); + expect(harness.deleteSandbox).toHaveBeenCalledWith( + { id: "responder-automation-run-1" }, + 60, + true, + ); + expect(harness.list).not.toHaveBeenCalled(); + consoleError.mockRestore(); + }); + + it("archives paused sandboxes, deletes expired ones and logs usage", async () => { + const paused = sandboxStopped("responder-automation-paused", 45); + const recent = sandboxStopped("responder-automation-recent", 10); + const expiredArchive = sandboxStopped("responder-automation-expired", 25 * 60, "archived"); + const freshArchive = sandboxStopped("responder-automation-archived", 2 * 60, "archived"); + const failed = sandboxStopped("responder-automation-failed", 30 * 60, "error"); + const buildFailed = sandboxStopped("responder-automation-build-failed", 30 * 60, "build_failed"); + const running = sandboxStopped("responder-automation-running", 0, "started"); + const harness = capacityHarness([ + paused, + recent, + expiredArchive, + freshArchive, + failed, + buildFailed, + running, + ]); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + await sweepDaytonaSandboxes({ daytonaApiKey: "daytona-test" }, harness.dependencies); + + expect(paused.archive).toHaveBeenCalledOnce(); + expect(recent.archive).not.toHaveBeenCalled(); + expect(harness.deleteSandbox.mock.calls).toEqual([ + [expiredArchive, 60, false], + [failed, 60, false], + [buildFailed, 60, false], + ]); + expect(JSON.parse(consoleLog.mock.calls[0]![0] as string)).toEqual({ + archived: 1, + cpu: 2, + deleted: 3, + diskGiB: 50, + event: "daytona_sandbox_sweep", + memoryGiB: 4, + sandboxes: 7, + }); + consoleLog.mockRestore(); + }); +}); + describe("Daytona sandbox resume", () => { it("gives the resumed session the API key its saved state left out", async () => { const session = { state: { sandboxId: "sandbox-1" } }; diff --git a/apps/worker/src/sandbox.ts b/apps/worker/src/sandbox.ts index 46228ea7..78f80b3b 100644 --- a/apps/worker/src/sandbox.ts +++ b/apps/worker/src/sandbox.ts @@ -1,6 +1,7 @@ import { Daytona, type Sandbox, + SandboxState, } from "@daytona/sdk"; import type { DaytonaSandboxClient, @@ -23,6 +24,7 @@ export interface DaytonaSandboxSecretMount { interface DaytonaCleanupClient { delete(sandbox: Sandbox, timeout?: number, wait?: boolean): Promise; get(sandboxIdOrName: string): Promise; + list(query?: Parameters[0]): AsyncIterableIterator; [Symbol.asyncDispose](): Promise; } @@ -30,12 +32,14 @@ type DaytonaSandboxCreator = Pick; export interface DaytonaCleanupDependencies { createClient(config: DaytonaCleanupConfig): DaytonaCleanupClient; + now(): number; reportException: typeof reportWorkerException; sleep(delayMs: number): Promise; } const defaultCleanupDependencies: DaytonaCleanupDependencies = { createClient: (config) => new Daytona(daytonaClientOptions(config)), + now: Date.now, reportException: reportWorkerException, sleep: (delayMs) => new Promise((resolve) => { @@ -64,6 +68,17 @@ const daytonaSettleRetryDelaysMs = [ ] as const; const daytonaCreateAttempts = 3; const daytonaCreateRetryDelayMs = 5_000; +// Waits between creations Daytona rejects at the organization's limits, about +// four minutes in all, while running sandboxes finish and free CPU and memory. +const daytonaQuotaRetryDelaysMs = [5_000, 15_000, 30_000, 60_000, 120_000] as const; +// Stopped sandboxes count toward Daytona's disk limit and archived ones do +// not, so a paused sandbox is archived once a quick follow-up is unlikely. +export const pausedSandboxArchiveMinutes = 30; +// Stopped sandboxes archived at once when Daytona's disk limit is reached. +const quotaEvictionBatchSize = 20; +// A sandbox stopped more recently may be about to restart, for example while +// its secrets are mounted. +const quotaEvictionMinStoppedMs = 5 * 60_000; // The Daytona SDK gives up on a sandbox that has not started within 60 // seconds by default, while Daytona keeps starting it. const daytonaCreateTimeoutSec = 120; @@ -77,6 +92,145 @@ function isDaytonaStartFailure(error: unknown): boolean { error.message.includes("Sandbox failed to start"); } +// Daytona rejects a new sandbox outright while the organization is at its +// disk, CPU or memory limit, for example "Total disk limit exceeded. Maximum +// allowed: 2000GiB." Nothing is created. A rate limit is not a quota: it can +// arrive after the sandbox exists, so it keeps the normal cleanup. +function isDaytonaQuotaError(error: unknown): boolean { + return error instanceof Error && + /\btotal (?:cpu|memory|disk) limit exceeded\b/i.test(error.message); +} + +function isDaytonaDiskQuotaError(error: unknown): boolean { + return error instanceof Error && /\bdisk limit exceeded\b/i.test(error.message); +} + +// Thrown when Daytona still refuses new sandboxes after every quota retry. +// The message is shown to the organization in place of Daytona's, which +// asks the reader to upgrade Responder's Daytona plan. +export class DaytonaCapacityError extends Error { + constructor(options: { cause: unknown }) { + super( + "Responder's sandbox capacity is temporarily full. Try again in a few minutes.", + options, + ); + this.name = "DaytonaCapacityError"; + } +} + +// Archives the stopped Responder sandboxes that were stopped longest ago, so +// their disk no longer counts toward Daytona's limit. An archived sandbox +// still resumes; it takes longer to start. Returns how many were archived. +export async function archiveStoppedSandboxes( + config: DaytonaCleanupConfig, + options: { limit: number; minStoppedMs: number }, + dependencies: DaytonaCleanupDependencies = defaultCleanupDependencies, +): Promise { + const client = dependencies.createClient(config); + try { + const stoppedBefore = dependencies.now() - options.minStoppedMs; + const candidates: Sandbox[] = []; + for await (const sandbox of client.list({ name: "responder-", states: [SandboxState.STOPPED] })) { + const updatedAt = Date.parse(sandbox.updatedAt ?? ""); + if (Number.isFinite(updatedAt) && updatedAt < stoppedBefore) { + candidates.push(sandbox); + } + } + candidates.sort((left, right) => + Date.parse(left.updatedAt ?? "") - Date.parse(right.updatedAt ?? "")); + let archived = 0; + for (const sandbox of candidates.slice(0, options.limit)) { + try { + await sandbox.archive(); + archived += 1; + } catch (error) { + // A sandbox that restarted or was deleted meanwhile is skipped. + console.error(JSON.stringify({ + errorName: error instanceof Error ? error.name : typeof error, + event: "daytona_sandbox_archive_failed", + sandboxId: sandbox.name, + })); + } + } + return archived; + } finally { + await client[Symbol.asyncDispose]().catch(() => undefined); + } +} + +// Paused sandboxes are kept for a day. Archived and failed sandboxes older +// than that are deleted in case Daytona's own deletion interval missed them. +const sandboxRetentionMs = 24 * 60 * 60_000; +const expiringSandboxStates: ReadonlySet = new Set([ + SandboxState.ARCHIVED, + SandboxState.BUILD_FAILED, + SandboxState.ERROR, +]); +// Stopped sandboxes archived in one sweep. +const sweepArchiveLimit = 100; + +// Keeps Responder's sandboxes within Daytona's limits: archives sandboxes +// paused longer than the archive interval, including ones set up before it +// existed, deletes expired ones, and logs what is left. +export async function sweepDaytonaSandboxes( + config: DaytonaCleanupConfig, + dependencies: DaytonaCleanupDependencies = defaultCleanupDependencies, +): Promise { + const archived = await archiveStoppedSandboxes( + config, + { limit: sweepArchiveLimit, minStoppedMs: pausedSandboxArchiveMinutes * 60_000 }, + dependencies, + ); + const client = dependencies.createClient(config); + try { + const expiredBefore = dependencies.now() - sandboxRetentionMs; + const expired: Sandbox[] = []; + let sandboxes = 0; + let cpu = 0; + let diskGiB = 0; + let memoryGiB = 0; + for await (const sandbox of client.list({ name: "responder-" })) { + sandboxes += 1; + if ( + expiringSandboxStates.has(sandbox.state ?? "") && + Date.parse(sandbox.updatedAt ?? "") < expiredBefore + ) { + expired.push(sandbox); + } + if (sandbox.state === SandboxState.ARCHIVED) continue; + diskGiB += sandbox.disk; + if (sandbox.state === SandboxState.STARTED) { + cpu += sandbox.cpu; + memoryGiB += sandbox.memory; + } + } + let deleted = 0; + for (const sandbox of expired) { + try { + await client.delete(sandbox, 60, false); + deleted += 1; + } catch (error) { + console.error(JSON.stringify({ + errorName: error instanceof Error ? error.name : typeof error, + event: "daytona_expired_sandbox_delete_failed", + sandboxId: sandbox.name, + })); + } + } + console.log(JSON.stringify({ + archived, + cpu, + deleted, + diskGiB, + event: "daytona_sandbox_sweep", + memoryGiB, + sandboxes, + })); + } finally { + await client[Symbol.asyncDispose]().catch(() => undefined); + } +} + // Cleanup errors for a sandbox that never appeared, already reported where // they were raised. const missingSandboxErrors = new WeakSet(); @@ -315,11 +469,17 @@ async function failCreation( const left = abandoned .filter((_sandbox, index) => deletionErrors[index] !== undefined) .map((sandbox) => sandbox.name); - const cleanupErrors = deletionErrors.filter((error) => error !== undefined); + // A sandbox that never appeared was already reported and must not hide + // why the creation failed. + const cleanupErrors = deletionErrors.filter((error) => + error !== undefined && + !(typeof error === "object" && error !== null && missingSandboxErrors.has(error))); const error = cleanupErrors.length > 0 ? new AggregateError( [createError, ...cleanupErrors], - `Unable to create or clean up Daytona sandbox ${sandboxName}`, + `Unable to create or clean up Daytona sandbox ${sandboxName}: ${ + createError instanceof Error ? createError.message : String(createError) + }`, ) : createError; if (typeof error === "object" && error !== null) { @@ -344,7 +504,9 @@ export async function createDaytonaSandboxSession( throw new Error(`Sandbox name ${sandboxName} is too long`); } const abandoned: AbandonedSandbox[] = []; - for (let attempt = 1; ; attempt += 1) { + let attempt = 1; + let quotaRetries = 0; + for (;;) { const attemptName = sandboxNameForAttempt(sandboxName, attempt); let session: DaytonaSandboxSession | undefined; try { @@ -362,6 +524,34 @@ export async function createDaytonaSandboxSession( }); } } catch (createError) { + // Daytona created nothing, so the same name is tried again once + // capacity frees up. + if (isDaytonaQuotaError(createError)) { + if (quotaRetries >= daytonaQuotaRetryDelaysMs.length || signal?.aborted) { + return await failCreation( + new DaytonaCapacityError({ cause: createError }), + sandboxName, + abandoned, + ); + } + const archived = isDaytonaDiskQuotaError(createError) + ? await archiveStoppedSandboxes( + config, + { limit: quotaEvictionBatchSize, minStoppedMs: quotaEvictionMinStoppedMs }, + dependencies, + ).catch(() => 0) + : 0; + console.error(JSON.stringify({ + archived, + error: createError instanceof Error ? createError.message.slice(0, 300) : String(createError), + event: "daytona_sandbox_quota_retry", + retry: quotaRetries + 1, + sandboxId: attemptName, + })); + await dependencies.sleep(daytonaQuotaRetryDelaysMs[quotaRetries]!); + quotaRetries += 1; + continue; + } abandoned.push(deleteAbandonedSandbox(attemptName, config, dependencies)); if ( attempt >= daytonaCreateAttempts || @@ -382,6 +572,7 @@ export async function createDaytonaSandboxSession( if (signal?.aborted) { return await failCreation(createError, sandboxName, abandoned); } + attempt += 1; continue; } // No session means the caller aborted before this attempt's creation. @@ -465,6 +656,12 @@ export async function configureDaytonaSandboxLifecycle( () => sandbox.setAutoDeleteInterval(autoDeleteInterval), dependencies, ); + if (autoDeleteInterval !== 0) { + await retryTransientDaytonaOperation( + () => sandbox.setAutoArchiveInterval(pausedSandboxArchiveMinutes), + dependencies, + ); + } } finally { await client[Symbol.asyncDispose]().catch(() => undefined); } diff --git a/apps/worker/src/worker.ts b/apps/worker/src/worker.ts index a1f4ed74..b0a12559 100644 --- a/apps/worker/src/worker.ts +++ b/apps/worker/src/worker.ts @@ -98,6 +98,7 @@ import { import { processGcpProjectSetupJob } from "@responder/core/integrations/gcp-setup-job"; import { purgeAutomationModelBrokerGrants } from "@responder/core/db/automation-model-broker"; import { settleUnbilledAutomationModelUsage } from "@responder/core/automations/model-usage-billing"; +import { requireDaytonaClientConfig } from "@responder/core/daytona-config"; import { settleUnbilledUsage } from "@responder/core/billing/usage-billing"; import { BILLING_NOTICE_RETRY_WINDOW_MS } from "@responder/core/billing/notifications"; import { defaultAlertAutomationDependencies, startAlertAutomations } from "./telemetry-alert-automations.js"; @@ -113,6 +114,7 @@ import { purgeSyntheticCheckRuns, } from "./synthetic-checks.js"; import { pullGcpTelemetrySources } from "./telemetry-sources.js"; +import { sweepDaytonaSandboxes } from "./sandbox.js"; import { sendUsageNotices } from "@responder/core/billing/usage-notices"; import { creditWaivedUsage, @@ -143,6 +145,7 @@ const pollers: { automationSuggestions?: NodeJS.Timeout; automationUsageBilling?: NodeJS.Timeout; brokerGrantCleanup?: NodeJS.Timeout; + daytonaSweep?: NodeJS.Timeout; gcpTelemetry?: NodeJS.Timeout; linearTicket?: NodeJS.Timeout; remediationRecovery?: NodeJS.Timeout; @@ -215,6 +218,21 @@ function pullGcpTelemetry(): Promise { return gcpTelemetryPass; } +let daytonaSweep: Promise | undefined; +// Runs at most one sandbox sweep at a time, when Daytona is configured. +function sweepSandboxes(): Promise { + if (!process.env.DAYTONA_API_KEY) return Promise.resolve(); + daytonaSweep ??= Promise.resolve() + .then(() => sweepDaytonaSandboxes(requireDaytonaClientConfig())) + .catch((error: unknown) => { + void reportWorkerException(error, { operation: "sandbox_cleanup" }); + }) + .finally(() => { + daytonaSweep = undefined; + }); + return daytonaSweep; +} + async function purgeExpiredAutomationBrokerGrants(): Promise { try { const deleted = await purgeAutomationModelBrokerGrants(); @@ -558,6 +576,7 @@ async function shutdown(signal: string): Promise { stopping = true; if (pollers.replayRequest) clearInterval(pollers.replayRequest); if (pollers.brokerGrantCleanup) clearInterval(pollers.brokerGrantCleanup); + if (pollers.daytonaSweep) clearInterval(pollers.daytonaSweep); if (pollers.automationUsageBilling) { clearInterval(pollers.automationUsageBilling); } @@ -1136,6 +1155,9 @@ pollers.brokerGrantCleanup = setInterval( 60 * 60 * 1_000, ); pollers.brokerGrantCleanup.unref(); +void sweepSandboxes(); +pollers.daytonaSweep = setInterval(() => void sweepSandboxes(), 10 * 60 * 1_000); +pollers.daytonaSweep.unref(); pollers.automationUsageBilling = setInterval( () => void settleAutomationUsageBilling(), 60 * 1_000,