From 04e7c5ec29d5df694ded2d7099174a0f80ccea3e Mon Sep 17 00:00:00 2001 From: Arseniy Shishaev <35579920+arseniycodes@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:32:50 +0200 Subject: [PATCH 1/3] Keep automations starting when Daytona's limits are reached Daytona rejects new sandboxes once the organization reaches its disk, CPU or memory limit. Stopped sandboxes count toward the 2000 GiB disk limit. Each automation run keeps its 10 GiB sandbox stopped for a day, so about 200 runs a day filled the limit and every new run failed on 2026-10-16. - Paused sandboxes are archived after 30 minutes. Archived sandboxes do not count toward the disk limit and still resume. - When Daytona rejects a creation at a limit, the worker archives the longest-stopped sandboxes (disk) and retries the same name for about four minutes (CPU and memory) before failing with a clear message. - A worker sweep every 10 minutes archives paused sandboxes, deletes archived and failed ones older than a day, and logs usage. - A sandbox that never appeared during cleanup no longer replaces the creation error, and the remaining aggregate message includes it. --- apps/worker/src/sandbox.test.ts | 169 ++++++++++++++++++++++++++- apps/worker/src/sandbox.ts | 196 +++++++++++++++++++++++++++++++- apps/worker/src/worker.ts | 22 ++++ 3 files changed, 383 insertions(+), 4 deletions(-) diff --git a/apps/worker/src/sandbox.test.ts b/apps/worker/src/sandbox.test.ts index d7d2cd06..ead04d33 100644 --- a/apps/worker/src/sandbox.test.ts +++ b/apps/worker/src/sandbox.test.ts @@ -7,6 +7,7 @@ import { closeDaytonaSandbox, configureDaytonaSandboxLifecycle, createDaytonaSandboxSession, + DaytonaCapacityError, deleteDaytonaSandboxByName, type DaytonaCleanupDependencies, pauseDaytonaSandbox, @@ -17,6 +18,7 @@ import { resumedSameDaytonaSandbox, sandboxDeletedAfterFailedCreation, sandboxesLeftAfterFailedCreation, + sweepDaytonaSandboxes, } from "./sandbox.js"; function cleanupHarness(options?: { @@ -477,7 +479,8 @@ describe("Daytona sandbox cleanup", () => { initialDeleteError, ); expect(sandboxDeletedAfterFailedCreation(failedBeforeCreation)).toBe(false); - expect(failedCleanup).toBeInstanceOf(AggregateError); + // A sandbox that never appeared does not hide why creation failed. + expect(failedCleanup).toHaveProperty("message", "sandbox creation timed out"); expect(sandboxDeletedAfterFailedCreation(failedCleanup)).toBe(false); expect(creator.create).toHaveBeenCalledOnce(); }); @@ -639,9 +642,11 @@ describe("Daytona sandbox cleanup", () => { it("keeps a thread sandbox after it pauses", async () => { const harness = cleanupHarness(); + const setAutoArchiveInterval = vi.fn().mockResolvedValue(undefined); const setAutoDeleteInterval = vi.fn().mockResolvedValue(undefined); harness.get.mockResolvedValue({ id: "sandbox-1", + setAutoArchiveInterval, setAutoDeleteInterval, }); @@ -656,6 +661,36 @@ describe("Daytona sandbox cleanup", () => { expect(setAutoDeleteInterval).toHaveBeenCalledWith(-1); }); + it("archives a paused sandbox so it stops counting toward Daytona's disk limit", async () => { + const harness = cleanupHarness(); + const setAutoArchiveInterval = vi.fn().mockResolvedValue(undefined); + const setAutoDeleteInterval = vi.fn().mockResolvedValue(undefined); + harness.get.mockResolvedValue({ + id: "sandbox-1", + setAutoArchiveInterval, + setAutoDeleteInterval, + }); + + await configureDaytonaSandboxLifecycle( + harness.session, + { daytonaApiKey: "daytona-test" }, + [], + 24 * 60, + harness.dependencies, + ); + await configureDaytonaSandboxLifecycle( + harness.session, + { daytonaApiKey: "daytona-test" }, + [], + 0, + harness.dependencies, + ); + + expect(setAutoDeleteInterval.mock.calls).toEqual([[24 * 60], [0]]); + // A sandbox deleted when it stops is never archived. + expect(setAutoArchiveInterval.mock.calls).toEqual([[30]]); + }); + it("retries transient lifecycle failures", async () => { const harness = cleanupHarness(); const gatewayError = Object.assign(new Error("bad gateway"), { @@ -781,6 +816,138 @@ describe("Daytona sandbox cleanup", () => { }); }); +describe("Daytona sandbox capacity", () => { + const now = Date.parse("2026-10-16T12:00:00.000Z"); + + function sandboxStopped(name: string, minutesAgo: number, state = "stopped") { + return { + archive: vi.fn().mockResolvedValue(undefined), + cpu: 2, + disk: 10, + memory: 4, + name, + state, + updatedAt: new Date(now - minutesAgo * 60_000).toISOString(), + }; + } + + function capacityHarness(sandboxes: Array>) { + const harness = cleanupHarness({ missing: true }); + const list = vi.fn((query?: { states?: string[] }) => + (async function* () { + for (const sandbox of sandboxes) { + if (!query?.states || query.states.includes(sandbox.state)) yield sandbox; + } + })()); + const dependencies = { + ...harness.dependencies, + createClient: vi.fn(() => ({ + delete: harness.deleteSandbox, + get: harness.get, + list, + [Symbol.asyncDispose]: harness.dispose, + })), + now: () => now, + } as unknown as DaytonaCleanupDependencies; + return { ...harness, dependencies, list }; + } + + it("archives the longest-stopped sandboxes and retries when Daytona's disk is full", async () => { + const longStopped = sandboxStopped("responder-automation-old", 120); + const justStopped = sandboxStopped("responder-automation-new", 2); + const harness = capacityHarness([longStopped, justStopped]); + const createdSession = { state: { sandboxId: "sandbox-2" } }; + const diskFull = new Error( + "DaytonaSandboxClient failed to create sandbox: Total disk limit exceeded. Maximum allowed: 2000GiB.", + ); + const creator = { + create: vi.fn() + .mockRejectedValueOnce(diskFull) + .mockResolvedValueOnce(createdSession), + }; + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect( + createDaytonaSandboxSession( + creator, + { daytonaApiKey: "daytona-test" }, + "responder-automation-run-1", + harness.dependencies, + ), + ).resolves.toBe(createdSession); + + expect(longStopped.archive).toHaveBeenCalledOnce(); + expect(justStopped.archive).not.toHaveBeenCalled(); + // Daytona created nothing, so the name is reused and nothing waits for + // the rejected sandbox to appear. + expect(creator.create.mock.calls).toEqual([ + [{ options: { createTimeoutSec: 120, name: "responder-automation-run-1" } }], + [{ options: { createTimeoutSec: 120, name: "responder-automation-run-1" } }], + ]); + expect(harness.get).toHaveBeenCalledTimes(2); + expect(harness.sleep).toHaveBeenCalledWith(5_000); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining("daytona_sandbox_quota_retry"), + ); + consoleError.mockRestore(); + }); + + it("waits for CPU to free up, then fails with a capacity error", async () => { + const harness = capacityHarness([sandboxStopped("responder-automation-old", 120)]); + const cpuFull = new Error( + "DaytonaSandboxClient failed to create sandbox: Total CPU limit exceeded. Maximum allowed: 500.", + ); + const creator = { create: vi.fn().mockRejectedValue(cpuFull) }; + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + const error = await createDaytonaSandboxSession( + creator, + { daytonaApiKey: "daytona-test" }, + "responder-automation-run-1", + harness.dependencies, + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(DaytonaCapacityError); + expect((error as Error).cause).toBe(cpuFull); + expect(sandboxDeletedAfterFailedCreation(error)).toBe(true); + expect(creator.create).toHaveBeenCalledTimes(6); + expect(harness.sleep.mock.calls).toEqual([[5_000], [15_000], [30_000], [60_000], [120_000]]); + // Archiving frees disk, not CPU. + expect(harness.list).not.toHaveBeenCalled(); + consoleError.mockRestore(); + }); + + it("archives paused sandboxes, deletes expired ones and logs usage", async () => { + const paused = sandboxStopped("responder-automation-paused", 45); + const recent = sandboxStopped("responder-automation-recent", 10); + const expiredArchive = sandboxStopped("responder-automation-expired", 25 * 60, "archived"); + const freshArchive = sandboxStopped("responder-automation-archived", 2 * 60, "archived"); + const failed = sandboxStopped("responder-automation-failed", 30 * 60, "error"); + const running = sandboxStopped("responder-automation-running", 0, "started"); + const harness = capacityHarness([paused, recent, expiredArchive, freshArchive, failed, running]); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + await sweepDaytonaSandboxes({ daytonaApiKey: "daytona-test" }, harness.dependencies); + + expect(paused.archive).toHaveBeenCalledOnce(); + expect(recent.archive).not.toHaveBeenCalled(); + expect(harness.deleteSandbox.mock.calls).toEqual([ + [expiredArchive, 60, false], + [failed, 60, false], + ]); + expect(JSON.parse(consoleLog.mock.calls[0]![0] as string)).toEqual({ + archived: 1, + cpu: 2, + deleted: 2, + diskGiB: 40, + event: "daytona_sandbox_sweep", + memoryGiB: 4, + sandboxes: 6, + }); + consoleLog.mockRestore(); + }); +}); + describe("Daytona sandbox resume", () => { it("gives the resumed session the API key its saved state left out", async () => { const session = { state: { sandboxId: "sandbox-1" } }; diff --git a/apps/worker/src/sandbox.ts b/apps/worker/src/sandbox.ts index 46228ea7..72a8b86d 100644 --- a/apps/worker/src/sandbox.ts +++ b/apps/worker/src/sandbox.ts @@ -1,6 +1,7 @@ import { Daytona, type Sandbox, + SandboxState, } from "@daytona/sdk"; import type { DaytonaSandboxClient, @@ -23,6 +24,7 @@ export interface DaytonaSandboxSecretMount { interface DaytonaCleanupClient { delete(sandbox: Sandbox, timeout?: number, wait?: boolean): Promise; get(sandboxIdOrName: string): Promise; + list(query?: Parameters[0]): AsyncIterableIterator; [Symbol.asyncDispose](): Promise; } @@ -30,12 +32,14 @@ type DaytonaSandboxCreator = Pick; export interface DaytonaCleanupDependencies { createClient(config: DaytonaCleanupConfig): DaytonaCleanupClient; + now(): number; reportException: typeof reportWorkerException; sleep(delayMs: number): Promise; } const defaultCleanupDependencies: DaytonaCleanupDependencies = { createClient: (config) => new Daytona(daytonaClientOptions(config)), + now: Date.now, reportException: reportWorkerException, sleep: (delayMs) => new Promise((resolve) => { @@ -64,6 +68,17 @@ const daytonaSettleRetryDelaysMs = [ ] as const; const daytonaCreateAttempts = 3; const daytonaCreateRetryDelayMs = 5_000; +// Waits between creations Daytona rejects at the organization's limits, about +// four minutes in all, while running sandboxes finish and free CPU and memory. +const daytonaQuotaRetryDelaysMs = [5_000, 15_000, 30_000, 60_000, 120_000] as const; +// Stopped sandboxes count toward Daytona's disk limit and archived ones do +// not, so a paused sandbox is archived once a quick follow-up is unlikely. +export const pausedSandboxArchiveMinutes = 30; +// Stopped sandboxes archived at once when Daytona's disk limit is reached. +const quotaEvictionBatchSize = 20; +// A sandbox stopped more recently may be about to restart, for example while +// its secrets are mounted. +const quotaEvictionMinStoppedMs = 5 * 60_000; // The Daytona SDK gives up on a sandbox that has not started within 60 // seconds by default, while Daytona keeps starting it. const daytonaCreateTimeoutSec = 120; @@ -77,6 +92,138 @@ function isDaytonaStartFailure(error: unknown): boolean { error.message.includes("Sandbox failed to start"); } +// Daytona rejects a new sandbox outright while the organization is at its +// disk, CPU or memory limit, for example "Total disk limit exceeded. Maximum +// allowed: 2000GiB." Nothing is created. +function isDaytonaQuotaError(error: unknown): boolean { + return error instanceof Error && /\blimit exceeded\b/i.test(error.message); +} + +function isDaytonaDiskQuotaError(error: unknown): boolean { + return error instanceof Error && /\bdisk limit exceeded\b/i.test(error.message); +} + +// Thrown when Daytona still refuses new sandboxes after every quota retry. +// The message is shown to the organization in place of Daytona's, which +// asks the reader to upgrade Responder's Daytona plan. +export class DaytonaCapacityError extends Error { + constructor(options: { cause: unknown }) { + super( + "Responder's sandbox capacity is temporarily full. Try again in a few minutes.", + options, + ); + this.name = "DaytonaCapacityError"; + } +} + +// Archives the stopped Responder sandboxes that were stopped longest ago, so +// their disk no longer counts toward Daytona's limit. An archived sandbox +// still resumes; it takes longer to start. Returns how many were archived. +export async function archiveStoppedSandboxes( + config: DaytonaCleanupConfig, + options: { limit: number; minStoppedMs: number }, + dependencies: DaytonaCleanupDependencies = defaultCleanupDependencies, +): Promise { + const client = dependencies.createClient(config); + try { + const stoppedBefore = dependencies.now() - options.minStoppedMs; + const candidates: Sandbox[] = []; + for await (const sandbox of client.list({ name: "responder-", states: [SandboxState.STOPPED] })) { + const updatedAt = Date.parse(sandbox.updatedAt ?? ""); + if (Number.isFinite(updatedAt) && updatedAt < stoppedBefore) { + candidates.push(sandbox); + } + } + candidates.sort((left, right) => + Date.parse(left.updatedAt ?? "") - Date.parse(right.updatedAt ?? "")); + let archived = 0; + for (const sandbox of candidates.slice(0, options.limit)) { + try { + await sandbox.archive(); + archived += 1; + } catch (error) { + // A sandbox that restarted or was deleted meanwhile is skipped. + console.error(JSON.stringify({ + errorName: error instanceof Error ? error.name : typeof error, + event: "daytona_sandbox_archive_failed", + sandboxId: sandbox.name, + })); + } + } + return archived; + } finally { + await client[Symbol.asyncDispose]().catch(() => undefined); + } +} + +// Paused sandboxes are kept for a day. Archived and failed sandboxes older +// than that are deleted in case Daytona's own deletion interval missed them. +const sandboxRetentionMs = 24 * 60 * 60_000; +// Stopped sandboxes archived in one sweep. +const sweepArchiveLimit = 100; + +// Keeps Responder's sandboxes within Daytona's limits: archives sandboxes +// paused longer than the archive interval, including ones set up before it +// existed, deletes expired ones, and logs what is left. +export async function sweepDaytonaSandboxes( + config: DaytonaCleanupConfig, + dependencies: DaytonaCleanupDependencies = defaultCleanupDependencies, +): Promise { + const archived = await archiveStoppedSandboxes( + config, + { limit: sweepArchiveLimit, minStoppedMs: pausedSandboxArchiveMinutes * 60_000 }, + dependencies, + ); + const client = dependencies.createClient(config); + try { + const expiredBefore = dependencies.now() - sandboxRetentionMs; + const expired: Sandbox[] = []; + let sandboxes = 0; + let cpu = 0; + let diskGiB = 0; + let memoryGiB = 0; + for await (const sandbox of client.list({ name: "responder-" })) { + sandboxes += 1; + if ( + (sandbox.state === SandboxState.ARCHIVED || sandbox.state === SandboxState.ERROR) && + Date.parse(sandbox.updatedAt ?? "") < expiredBefore + ) { + expired.push(sandbox); + } + if (sandbox.state === SandboxState.ARCHIVED) continue; + diskGiB += sandbox.disk; + if (sandbox.state === SandboxState.STARTED) { + cpu += sandbox.cpu; + memoryGiB += sandbox.memory; + } + } + let deleted = 0; + for (const sandbox of expired) { + try { + await client.delete(sandbox, 60, false); + deleted += 1; + } catch (error) { + console.error(JSON.stringify({ + errorName: error instanceof Error ? error.name : typeof error, + event: "daytona_expired_sandbox_delete_failed", + sandboxId: sandbox.name, + })); + } + } + console.log(JSON.stringify({ + archived, + cpu, + deleted, + diskGiB, + event: "daytona_sandbox_sweep", + memoryGiB, + sandboxes, + })); + } finally { + await client[Symbol.asyncDispose]().catch(() => undefined); + } +} + // Cleanup errors for a sandbox that never appeared, already reported where // they were raised. const missingSandboxErrors = new WeakSet(); @@ -315,11 +462,17 @@ async function failCreation( const left = abandoned .filter((_sandbox, index) => deletionErrors[index] !== undefined) .map((sandbox) => sandbox.name); - const cleanupErrors = deletionErrors.filter((error) => error !== undefined); + // A sandbox that never appeared was already reported and must not hide + // why the creation failed. + const cleanupErrors = deletionErrors.filter((error) => + error !== undefined && + !(typeof error === "object" && error !== null && missingSandboxErrors.has(error))); const error = cleanupErrors.length > 0 ? new AggregateError( [createError, ...cleanupErrors], - `Unable to create or clean up Daytona sandbox ${sandboxName}`, + `Unable to create or clean up Daytona sandbox ${sandboxName}: ${ + createError instanceof Error ? createError.message : String(createError) + }`, ) : createError; if (typeof error === "object" && error !== null) { @@ -344,7 +497,9 @@ export async function createDaytonaSandboxSession( throw new Error(`Sandbox name ${sandboxName} is too long`); } const abandoned: AbandonedSandbox[] = []; - for (let attempt = 1; ; attempt += 1) { + let attempt = 1; + let quotaRetries = 0; + for (;;) { const attemptName = sandboxNameForAttempt(sandboxName, attempt); let session: DaytonaSandboxSession | undefined; try { @@ -362,6 +517,34 @@ export async function createDaytonaSandboxSession( }); } } catch (createError) { + // Daytona created nothing, so the same name is tried again once + // capacity frees up. + if (isDaytonaQuotaError(createError)) { + if (quotaRetries >= daytonaQuotaRetryDelaysMs.length || signal?.aborted) { + return await failCreation( + new DaytonaCapacityError({ cause: createError }), + sandboxName, + abandoned, + ); + } + const archived = isDaytonaDiskQuotaError(createError) + ? await archiveStoppedSandboxes( + config, + { limit: quotaEvictionBatchSize, minStoppedMs: quotaEvictionMinStoppedMs }, + dependencies, + ).catch(() => 0) + : 0; + console.error(JSON.stringify({ + archived, + error: createError instanceof Error ? createError.message.slice(0, 300) : String(createError), + event: "daytona_sandbox_quota_retry", + retry: quotaRetries + 1, + sandboxId: attemptName, + })); + await dependencies.sleep(daytonaQuotaRetryDelaysMs[quotaRetries]!); + quotaRetries += 1; + continue; + } abandoned.push(deleteAbandonedSandbox(attemptName, config, dependencies)); if ( attempt >= daytonaCreateAttempts || @@ -382,6 +565,7 @@ export async function createDaytonaSandboxSession( if (signal?.aborted) { return await failCreation(createError, sandboxName, abandoned); } + attempt += 1; continue; } // No session means the caller aborted before this attempt's creation. @@ -465,6 +649,12 @@ export async function configureDaytonaSandboxLifecycle( () => sandbox.setAutoDeleteInterval(autoDeleteInterval), dependencies, ); + if (autoDeleteInterval !== 0) { + await retryTransientDaytonaOperation( + () => sandbox.setAutoArchiveInterval(pausedSandboxArchiveMinutes), + dependencies, + ); + } } finally { await client[Symbol.asyncDispose]().catch(() => undefined); } diff --git a/apps/worker/src/worker.ts b/apps/worker/src/worker.ts index a1f4ed74..b0a12559 100644 --- a/apps/worker/src/worker.ts +++ b/apps/worker/src/worker.ts @@ -98,6 +98,7 @@ import { import { processGcpProjectSetupJob } from "@responder/core/integrations/gcp-setup-job"; import { purgeAutomationModelBrokerGrants } from "@responder/core/db/automation-model-broker"; import { settleUnbilledAutomationModelUsage } from "@responder/core/automations/model-usage-billing"; +import { requireDaytonaClientConfig } from "@responder/core/daytona-config"; import { settleUnbilledUsage } from "@responder/core/billing/usage-billing"; import { BILLING_NOTICE_RETRY_WINDOW_MS } from "@responder/core/billing/notifications"; import { defaultAlertAutomationDependencies, startAlertAutomations } from "./telemetry-alert-automations.js"; @@ -113,6 +114,7 @@ import { purgeSyntheticCheckRuns, } from "./synthetic-checks.js"; import { pullGcpTelemetrySources } from "./telemetry-sources.js"; +import { sweepDaytonaSandboxes } from "./sandbox.js"; import { sendUsageNotices } from "@responder/core/billing/usage-notices"; import { creditWaivedUsage, @@ -143,6 +145,7 @@ const pollers: { automationSuggestions?: NodeJS.Timeout; automationUsageBilling?: NodeJS.Timeout; brokerGrantCleanup?: NodeJS.Timeout; + daytonaSweep?: NodeJS.Timeout; gcpTelemetry?: NodeJS.Timeout; linearTicket?: NodeJS.Timeout; remediationRecovery?: NodeJS.Timeout; @@ -215,6 +218,21 @@ function pullGcpTelemetry(): Promise { return gcpTelemetryPass; } +let daytonaSweep: Promise | undefined; +// Runs at most one sandbox sweep at a time, when Daytona is configured. +function sweepSandboxes(): Promise { + if (!process.env.DAYTONA_API_KEY) return Promise.resolve(); + daytonaSweep ??= Promise.resolve() + .then(() => sweepDaytonaSandboxes(requireDaytonaClientConfig())) + .catch((error: unknown) => { + void reportWorkerException(error, { operation: "sandbox_cleanup" }); + }) + .finally(() => { + daytonaSweep = undefined; + }); + return daytonaSweep; +} + async function purgeExpiredAutomationBrokerGrants(): Promise { try { const deleted = await purgeAutomationModelBrokerGrants(); @@ -558,6 +576,7 @@ async function shutdown(signal: string): Promise { stopping = true; if (pollers.replayRequest) clearInterval(pollers.replayRequest); if (pollers.brokerGrantCleanup) clearInterval(pollers.brokerGrantCleanup); + if (pollers.daytonaSweep) clearInterval(pollers.daytonaSweep); if (pollers.automationUsageBilling) { clearInterval(pollers.automationUsageBilling); } @@ -1136,6 +1155,9 @@ pollers.brokerGrantCleanup = setInterval( 60 * 60 * 1_000, ); pollers.brokerGrantCleanup.unref(); +void sweepSandboxes(); +pollers.daytonaSweep = setInterval(() => void sweepSandboxes(), 10 * 60 * 1_000); +pollers.daytonaSweep.unref(); pollers.automationUsageBilling = setInterval( () => void settleAutomationUsageBilling(), 60 * 1_000, From 2f83ddd1a2d9692171c14520ff83f869476f3273 Mon Sep 17 00:00:00 2001 From: Arseniy Shishaev <35579920+arseniycodes@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:38:37 +0200 Subject: [PATCH 2/3] Delete expired build-failed sandboxes in the sweep --- apps/worker/src/sandbox.test.ts | 18 ++++++++++++++---- apps/worker/src/sandbox.ts | 7 ++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/worker/src/sandbox.test.ts b/apps/worker/src/sandbox.test.ts index ead04d33..4e53b004 100644 --- a/apps/worker/src/sandbox.test.ts +++ b/apps/worker/src/sandbox.test.ts @@ -923,8 +923,17 @@ describe("Daytona sandbox capacity", () => { const expiredArchive = sandboxStopped("responder-automation-expired", 25 * 60, "archived"); const freshArchive = sandboxStopped("responder-automation-archived", 2 * 60, "archived"); const failed = sandboxStopped("responder-automation-failed", 30 * 60, "error"); + const buildFailed = sandboxStopped("responder-automation-build-failed", 30 * 60, "build_failed"); const running = sandboxStopped("responder-automation-running", 0, "started"); - const harness = capacityHarness([paused, recent, expiredArchive, freshArchive, failed, running]); + const harness = capacityHarness([ + paused, + recent, + expiredArchive, + freshArchive, + failed, + buildFailed, + running, + ]); const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); await sweepDaytonaSandboxes({ daytonaApiKey: "daytona-test" }, harness.dependencies); @@ -934,15 +943,16 @@ describe("Daytona sandbox capacity", () => { expect(harness.deleteSandbox.mock.calls).toEqual([ [expiredArchive, 60, false], [failed, 60, false], + [buildFailed, 60, false], ]); expect(JSON.parse(consoleLog.mock.calls[0]![0] as string)).toEqual({ archived: 1, cpu: 2, - deleted: 2, - diskGiB: 40, + deleted: 3, + diskGiB: 50, event: "daytona_sandbox_sweep", memoryGiB: 4, - sandboxes: 6, + sandboxes: 7, }); consoleLog.mockRestore(); }); diff --git a/apps/worker/src/sandbox.ts b/apps/worker/src/sandbox.ts index 72a8b86d..22369d95 100644 --- a/apps/worker/src/sandbox.ts +++ b/apps/worker/src/sandbox.ts @@ -159,6 +159,11 @@ export async function archiveStoppedSandboxes( // Paused sandboxes are kept for a day. Archived and failed sandboxes older // than that are deleted in case Daytona's own deletion interval missed them. const sandboxRetentionMs = 24 * 60 * 60_000; +const expiringSandboxStates: ReadonlySet = new Set([ + SandboxState.ARCHIVED, + SandboxState.BUILD_FAILED, + SandboxState.ERROR, +]); // Stopped sandboxes archived in one sweep. const sweepArchiveLimit = 100; @@ -185,7 +190,7 @@ export async function sweepDaytonaSandboxes( for await (const sandbox of client.list({ name: "responder-" })) { sandboxes += 1; if ( - (sandbox.state === SandboxState.ARCHIVED || sandbox.state === SandboxState.ERROR) && + expiringSandboxStates.has(sandbox.state ?? "") && Date.parse(sandbox.updatedAt ?? "") < expiredBefore ) { expired.push(sandbox); From 8b02fd331a338efb192576b0c5be733cbf9b3d16 Mon Sep 17 00:00:00 2001 From: Arseniy Shishaev <35579920+arseniycodes@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:43:07 +0200 Subject: [PATCH 3/3] Treat only Daytona's total CPU, memory and disk limits as full capacity --- apps/worker/src/sandbox.test.ts | 37 +++++++++++++++++++++++++++++++++ apps/worker/src/sandbox.ts | 6 ++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/apps/worker/src/sandbox.test.ts b/apps/worker/src/sandbox.test.ts index 4e53b004..cb7c5848 100644 --- a/apps/worker/src/sandbox.test.ts +++ b/apps/worker/src/sandbox.test.ts @@ -917,6 +917,43 @@ describe("Daytona sandbox capacity", () => { consoleError.mockRestore(); }); + it("cleans up after a rate limit instead of treating it as full capacity", async () => { + const harness = capacityHarness([]); + harness.get.mockImplementation(async (name: string) => ({ id: name })); + const createdSession = { state: { sandboxId: "sandbox-2" } }; + const rateLimited = Object.assign( + new Error("DaytonaSandboxClient failed to create sandbox: Rate limit exceeded"), + { statusCode: 429 }, + ); + const creator = { + create: vi.fn() + .mockRejectedValueOnce(rateLimited) + .mockResolvedValueOnce(createdSession), + }; + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect( + createDaytonaSandboxSession( + creator, + { daytonaApiKey: "daytona-test" }, + "responder-automation-run-1", + harness.dependencies, + ), + ).resolves.toBe(createdSession); + + // The rate-limited sandbox may exist, so it is deleted and replaced. + expect(creator.create).toHaveBeenLastCalledWith({ + options: { createTimeoutSec: 120, name: "responder-automation-run-1-2" }, + }); + expect(harness.deleteSandbox).toHaveBeenCalledWith( + { id: "responder-automation-run-1" }, + 60, + true, + ); + expect(harness.list).not.toHaveBeenCalled(); + consoleError.mockRestore(); + }); + it("archives paused sandboxes, deletes expired ones and logs usage", async () => { const paused = sandboxStopped("responder-automation-paused", 45); const recent = sandboxStopped("responder-automation-recent", 10); diff --git a/apps/worker/src/sandbox.ts b/apps/worker/src/sandbox.ts index 22369d95..78f80b3b 100644 --- a/apps/worker/src/sandbox.ts +++ b/apps/worker/src/sandbox.ts @@ -94,9 +94,11 @@ function isDaytonaStartFailure(error: unknown): boolean { // Daytona rejects a new sandbox outright while the organization is at its // disk, CPU or memory limit, for example "Total disk limit exceeded. Maximum -// allowed: 2000GiB." Nothing is created. +// allowed: 2000GiB." Nothing is created. A rate limit is not a quota: it can +// arrive after the sandbox exists, so it keeps the normal cleanup. function isDaytonaQuotaError(error: unknown): boolean { - return error instanceof Error && /\blimit exceeded\b/i.test(error.message); + return error instanceof Error && + /\btotal (?:cpu|memory|disk) limit exceeded\b/i.test(error.message); } function isDaytonaDiskQuotaError(error: unknown): boolean {