From 86c4e78748df90c100e995c84aae1012988fe6f4 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 25 Oct 2026 09:35:26 +0100 Subject: [PATCH 1/5] =?UTF-8?q?test:=20reproduce=20#985=20=E2=80=94=20a=20?= =?UTF-8?q?cleanup=20run=20in=20flight=20at=20stop=20logs=20a=20warm=20poo?= =?UTF-8?q?l=20record=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/daemon/main.test.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 9ab9ae915..008800590 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -1523,6 +1523,37 @@ describe("startDaemon HTTP gateway startup readiness", () => { }); }); +describe("startDaemon stop with a cleanup run in flight (#985)", () => { + it("logs no warning or error once a stop lands while the cleanup run daemon.started began is still in flight", async () => { + // why: holds the cleanup run that `daemon.started` starts on its disk read, so the stop + // lands while the run is in flight, as an `http-start-failed` stop right after a start does. + class HeldDiskFree extends MemoryFilesystem { + readonly held: (() => void)[] = []; + override async diskFree(path: string): Promise { + await new Promise((resolve) => this.held.push(resolve)); + return super.diskFree(path); + } + } + const filesystem = new HeldDiskFree(); + const { daemon, sink } = await start({ filesystem }); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(filesystem.held).toHaveLength(1); + + await daemon.stop("http-start-failed"); + for (const release of filesystem.held.splice(0)) release(); + await new Promise((resolve) => setTimeout(resolve, 25)); + + const stopping = sink.records.findIndex((record) => record.message === "Daemon stopping"); + expect(stopping).toBeGreaterThanOrEqual(0); + expect( + sink.records + .slice(stopping) + .filter((record) => record.level === "warn" || record.level === "error") + .map(({ message, module, fields }) => ({ message, module, fields })), + ).toEqual([]); + }); +}); + describe("startDaemon stop with a warm target", () => { it("creates no further device for a target once a stop is asked for, though the creation in flight ends while the gateway stops", async () => { const directory = await mkdtemp(join(tmpdir(), "simlock-main-stop-target-")); From 49aee7cc07c761f8fc5a8c9b3cc5c84a12f9f113 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 25 Oct 2026 10:56:04 +0100 Subject: [PATCH 2/5] fix(daemon): a stop waits for the cleanup run in flight before closing state and drivers (#985) --- src/core/reaper.ts | 8 ++++++++ src/daemon/main.test.ts | 6 +++++- src/daemon/server.ts | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/core/reaper.ts b/src/core/reaper.ts index 493e322ae..ab5045476 100644 --- a/src/core/reaper.ts +++ b/src/core/reaper.ts @@ -93,6 +93,14 @@ export class CleanupReaper { } } + async settle(): Promise { + try { + await this.#running; + } catch { + // why: the run's failure is logged by whoever started it; a stop only waits for it to end. + } + } + async #runOnce({ dryRun = false, rule }: CleanupRunOptions = {}): Promise { // A dry run is a preview and must stay free of side effects: emitting the // pressure fact from one would wake this reaper through its own subscription diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 008800590..258207772 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -1539,8 +1539,12 @@ describe("startDaemon stop with a cleanup run in flight (#985)", () => { await new Promise((resolve) => setTimeout(resolve, 25)); expect(filesystem.held).toHaveLength(1); - await daemon.stop("http-start-failed"); + // why: the stop must wait for the held run, so the read is released while the stop is pending, + // after a stop that did not wait would already have closed the state database. + const stopped = daemon.stop("http-start-failed"); + await new Promise((resolve) => setTimeout(resolve, 25)); for (const release of filesystem.held.splice(0)) release(); + await stopped; await new Promise((resolve) => setTimeout(resolve, 25)); const stopping = sink.records.findIndex((record) => record.message === "Daemon stopping"); diff --git a/src/daemon/server.ts b/src/daemon/server.ts index 14659ce89..a2319dd14 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -637,6 +637,9 @@ export class DaemonServer { this.options.hostLoad?.dispose(); this.#workerDispatcher?.dispose(); this.#engine?.reaper.dispose(); + // why: a cleanup run in flight still reads the registry and calls drivers; it ends before + // anything below closes them. + await this.#engine?.reaper.settle(); this.#engine?.healthMonitor?.dispose(); this.#engine?.physicalDeviceWatcher?.dispose(); // why: a stop releases nothing; leases persist with their deadline. Only work already in From 62acac6185ee8ec07b3cf1f9dcbb76f424c91264 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 25 Oct 2026 10:59:03 +0100 Subject: [PATCH 3/5] test(daemon): the #985 stop test releases the held read by event-loop turns, not milliseconds --- src/daemon/main.test.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 258207772..46606f7e6 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -1524,6 +1524,8 @@ describe("startDaemon HTTP gateway startup readiness", () => { }); describe("startDaemon stop with a cleanup run in flight (#985)", () => { + const turnOfTheEventLoop = () => new Promise((resolve) => setImmediate(resolve)); + it("logs no warning or error once a stop lands while the cleanup run daemon.started began is still in flight", async () => { // why: holds the cleanup run that `daemon.started` starts on its disk read, so the stop // lands while the run is in flight, as an `http-start-failed` stop right after a start does. @@ -1536,16 +1538,22 @@ describe("startDaemon stop with a cleanup run in flight (#985)", () => { } const filesystem = new HeldDiskFree(); const { daemon, sink } = await start({ filesystem }); - await new Promise((resolve) => setTimeout(resolve, 25)); - expect(filesystem.held).toHaveLength(1); + await vi.waitFor(() => expect(filesystem.held).toHaveLength(1)); - // why: the stop must wait for the held run, so the read is released while the stop is pending, - // after a stop that did not wait would already have closed the state database. + // why: the stop must wait for the held run, so the read is released only once the stop has + // either ended (a stop that did not wait has closed the state database by then) or sat + // pending through many event-loop turns. Turns, not milliseconds: a loaded machine runs the + // same turns, only slower. const stopped = daemon.stop("http-start-failed"); - await new Promise((resolve) => setTimeout(resolve, 25)); + let stopEnded = false; + void stopped.then(() => { + stopEnded = true; + }); + for (let turn = 0; turn < 200 && !stopEnded; turn += 1) await turnOfTheEventLoop(); for (const release of filesystem.held.splice(0)) release(); await stopped; - await new Promise((resolve) => setTimeout(resolve, 25)); + // why: a run that outlived the stop surfaces its failure a few turns after its read returns. + for (let turn = 0; turn < 50; turn += 1) await turnOfTheEventLoop(); const stopping = sink.records.findIndex((record) => record.message === "Daemon stopping"); expect(stopping).toBeGreaterThanOrEqual(0); From f4374e20802337a1c21f0a36c14ef44195d66f2a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 25 Oct 2026 11:02:20 +0100 Subject: [PATCH 4/5] test(daemon): a stop ends the device deletion a cleanup run has in flight before the drivers are disposed (#985) --- src/daemon/main.test.ts | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 46606f7e6..dcac42830 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -1564,6 +1564,60 @@ describe("startDaemon stop with a cleanup run in flight (#985)", () => { .map(({ message, module, fields }) => ({ message, module, fields })), ).toEqual([]); }); + + it("ends the device deletion a cleanup run has in flight before the driver is disposed", async () => { + // why: records the order of the driver's calls, and holds the deletion the cleanup run proposes mid-call. + class HeldDestroyDriver extends FakeDriver { + readonly order: string[] = []; + readonly held: (() => void)[] = []; + override async destroy(device: Parameters[0]): Promise { + this.order.push("destroy started"); + await new Promise((resolve) => this.held.push(resolve)); + await super.destroy(device); + this.order.push("destroy finished"); + } + async dispose(): Promise { + this.order.push("dispose"); + } + } + const clock = new FakeClock(1_000); + const driver = new HeldDestroyDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const { daemon } = await start({ clock, drivers: [driver] }); + const agent = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent", + role: "agent", + } as const; + const request = { + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + requesterId: "agent-1", + }; + const grant = (await daemon.dispatch("lease.request", request, agent)) as { + readonly lease: { readonly id: string }; + }; + await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); + // why: the idle rules shut a released device down after 10 minutes and delete it after 60; each tick is the reaper's 60 s timer. + for (let minute = 0; minute < 70 && driver.held.length === 0; minute += 1) { + clock.advance(60_000); + for (let turn = 0; turn < 20; turn += 1) await turnOfTheEventLoop(); + } + expect(driver.held).toHaveLength(1); + + const stopped = daemon.stop("test-stop"); + let stopEnded = false; + void stopped.then(() => { + stopEnded = true; + }); + for (let turn = 0; turn < 200 && !stopEnded; turn += 1) await turnOfTheEventLoop(); + expect(driver.order).toEqual(["destroy started"]); + for (const release of driver.held.splice(0)) release(); + await stopped; + + expect(driver.order).toEqual(["destroy started", "destroy finished", "dispose"]); + }); }); describe("startDaemon stop with a warm target", () => { From 48919cad88495d9efa3f95f9e1018f9255056e3a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 25 Oct 2026 11:07:24 +0100 Subject: [PATCH 5/5] fix(daemon): a stop waits for dry-run and rule-scoped cleanup runs in flight too (#985) --- docs/internal/flows/startup.md | 5 +- src/core/reaper.ts | 17 ++++--- src/daemon/main.test.ts | 93 +++++++++++++++++++++------------- src/daemon/server.ts | 4 +- 4 files changed, 73 insertions(+), 46 deletions(-) diff --git a/docs/internal/flows/startup.md b/docs/internal/flows/startup.md index 78c7f2cd6..9cab62262 100644 --- a/docs/internal/flows/startup.md +++ b/docs/internal/flows/startup.md @@ -119,8 +119,9 @@ platform runs the deferred reclaim (step 21.2). 2. Stop auxiliary fronts: agent-device edge (cancel its waiting allocates), HTTP (after its start settles), the uplink and its redial loop. 3. Log, emit `daemon.stopping`. -4. Unsubscribe pushes; dispose host load, dispatcher, reaper, health monitor, - physical watcher. +4. Unsubscribe pushes; dispose host load, dispatcher, reaper, then wait for every + cleanup run in flight (scheduled, dry-run or rule-scoped) to end; dispose health + monitor, physical watcher. 5. Settle — later converge steps are skipped; leasing drains the physical watcher and background reclaims; core drains the warm pool. 6. Dispose — core and leasing timers, `ComponentInstaller.close`, agent-device diff --git a/src/core/reaper.ts b/src/core/reaper.ts index ab5045476..ae37ce010 100644 --- a/src/core/reaper.ts +++ b/src/core/reaper.ts @@ -45,6 +45,8 @@ export class CleanupReaper { readonly #unsubscribe: readonly (() => void)[]; #followUpRequested = false; #running: Promise | undefined; + /** why: runs a request started (a dry run, a rule-scoped run), which `#running` does not hold. */ + readonly #requestRuns = new Set>(); #tickTimer: TimerHandle | undefined; #disposed = false; // Tracks the previous tick's pressure state so disk.pressure-detected fires @@ -76,7 +78,13 @@ export class CleanupReaper { return this.#scheduleRun(); } - return this.#runOnce(rule === undefined ? { dryRun } : { dryRun, rule }); + const run = this.#runOnce(rule === undefined ? { dryRun } : { dryRun, rule }); + this.#requestRuns.add(run); + const forget = () => { + this.#requestRuns.delete(run); + }; + run.then(forget, forget); + return run; } dispose(): void { @@ -94,11 +102,8 @@ export class CleanupReaper { } async settle(): Promise { - try { - await this.#running; - } catch { - // why: the run's failure is logged by whoever started it; a stop only waits for it to end. - } + // why: the run's failure reaches whoever started it; a stop only waits for every run to end. + await Promise.allSettled([this.#running, ...this.#requestRuns]); } async #runOnce({ dryRun = false, rule }: CleanupRunOptions = {}): Promise { diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index dcac42830..a7b2e7924 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -1526,44 +1526,65 @@ describe("startDaemon HTTP gateway startup readiness", () => { describe("startDaemon stop with a cleanup run in flight (#985)", () => { const turnOfTheEventLoop = () => new Promise((resolve) => setImmediate(resolve)); - it("logs no warning or error once a stop lands while the cleanup run daemon.started began is still in flight", async () => { - // why: holds the cleanup run that `daemon.started` starts on its disk read, so the stop - // lands while the run is in flight, as an `http-start-failed` stop right after a start does. - class HeldDiskFree extends MemoryFilesystem { - readonly held: (() => void)[] = []; - override async diskFree(path: string): Promise { - await new Promise((resolve) => this.held.push(resolve)); - return super.diskFree(path); + it.each([ + { name: "the run daemon.started began", request: undefined }, + { name: "a dry run a client asked for", request: { dryRun: true } }, + { name: "a rule-scoped run a client asked for", request: { rule: "idle-destroy" } }, + ])( + "logs no warning or error once a stop lands while $name is still in flight", + async ({ request }) => { + // why: holds a cleanup run on its disk read, so the stop lands while the run is in flight, + // as an `http-start-failed` stop right after a start does. + class HeldDiskFree extends MemoryFilesystem { + readonly held: (() => void)[] = []; + override async diskFree(path: string): Promise { + await new Promise((resolve) => this.held.push(resolve)); + return super.diskFree(path); + } + } + const filesystem = new HeldDiskFree(); + const { daemon, sink } = await start({ filesystem }); + await vi.waitFor(() => expect(filesystem.held).toHaveLength(1)); + if (request !== undefined) { + // why: the run daemon.started began ends first, so the client's request is the only one held. + for (const release of filesystem.held.splice(0)) release(); + for (let turn = 0; turn < 50; turn += 1) await turnOfTheEventLoop(); + void daemon + .dispatch("cleanup.run", request, { + manageEventSubscription: () => undefined, + via: "socket", + principal: "admin", + role: "admin", + }) + .catch(() => undefined); + await vi.waitFor(() => expect(filesystem.held).toHaveLength(1)); } - } - const filesystem = new HeldDiskFree(); - const { daemon, sink } = await start({ filesystem }); - await vi.waitFor(() => expect(filesystem.held).toHaveLength(1)); - - // why: the stop must wait for the held run, so the read is released only once the stop has - // either ended (a stop that did not wait has closed the state database by then) or sat - // pending through many event-loop turns. Turns, not milliseconds: a loaded machine runs the - // same turns, only slower. - const stopped = daemon.stop("http-start-failed"); - let stopEnded = false; - void stopped.then(() => { - stopEnded = true; - }); - for (let turn = 0; turn < 200 && !stopEnded; turn += 1) await turnOfTheEventLoop(); - for (const release of filesystem.held.splice(0)) release(); - await stopped; - // why: a run that outlived the stop surfaces its failure a few turns after its read returns. - for (let turn = 0; turn < 50; turn += 1) await turnOfTheEventLoop(); - const stopping = sink.records.findIndex((record) => record.message === "Daemon stopping"); - expect(stopping).toBeGreaterThanOrEqual(0); - expect( - sink.records - .slice(stopping) - .filter((record) => record.level === "warn" || record.level === "error") - .map(({ message, module, fields }) => ({ message, module, fields })), - ).toEqual([]); - }); + // why: the stop must wait for the held run, so the read is released only once the stop has + // either ended (a stop that did not wait has closed the state database by then) or sat + // pending through many event-loop turns. Turns, not milliseconds: a loaded machine runs the + // same turns, only slower. + const stopped = daemon.stop("http-start-failed"); + let stopEnded = false; + void stopped.then(() => { + stopEnded = true; + }); + for (let turn = 0; turn < 200 && !stopEnded; turn += 1) await turnOfTheEventLoop(); + for (const release of filesystem.held.splice(0)) release(); + await stopped; + // why: a run that outlived the stop surfaces its failure a few turns after its read returns. + for (let turn = 0; turn < 50; turn += 1) await turnOfTheEventLoop(); + + const stopping = sink.records.findIndex((record) => record.message === "Daemon stopping"); + expect(stopping).toBeGreaterThanOrEqual(0); + expect( + sink.records + .slice(stopping) + .filter((record) => record.level === "warn" || record.level === "error") + .map(({ message, module, fields }) => ({ message, module, fields })), + ).toEqual([]); + }, + ); it("ends the device deletion a cleanup run has in flight before the driver is disposed", async () => { // why: records the order of the driver's calls, and holds the deletion the cleanup run proposes mid-call. diff --git a/src/daemon/server.ts b/src/daemon/server.ts index a2319dd14..cce24fca8 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -637,8 +637,8 @@ export class DaemonServer { this.options.hostLoad?.dispose(); this.#workerDispatcher?.dispose(); this.#engine?.reaper.dispose(); - // why: a cleanup run in flight still reads the registry and calls drivers; it ends before - // anything below closes them. + // why: every cleanup run in flight (scheduled, dry-run or rule-scoped) still reads the registry + // and calls drivers; each ends before anything below closes them. await this.#engine?.reaper.settle(); this.#engine?.healthMonitor?.dispose(); this.#engine?.physicalDeviceWatcher?.dispose();