diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b41b62..c328f78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 backgrounding or transport loss while retaining the same server session, shell state, working directory, and scrollback. Disconnect text is no longer written into the terminal. +- App-removal preparation now quiesces and fences automatic fleet care before + deleting owned Apple channel machines, so an in-flight reconciliation pass + cannot recreate a machine from stale pre-removal state. ## [0.0.3] - 2026-07-24 diff --git a/src/server/channel-computers.ts b/src/server/channel-computers.ts index 9d28d90..3862759 100644 --- a/src/server/channel-computers.ts +++ b/src/server/channel-computers.ts @@ -72,6 +72,7 @@ const CONTAINER_CANDIDATES = [process.env.HELM_CONTAINER_CLI, "/usr/local/bin/co const COMMAND_TIMEOUT_MS = Math.max(5_000, Number(process.env.HELM_MACHINE_COMMAND_TIMEOUT_MS || 120_000)); const IDLE_AFTER_MS = Math.max(60_000, Number(process.env.HELM_MACHINE_IDLE_MS || 15 * 60_000)); const RECONCILE_EVERY_MS = Math.max(15_000, Number(process.env.HELM_FLEET_INTERVAL_MS || 60_000)); +const INITIAL_RECONCILE_MS = Math.max(25, Number(process.env.HELM_FLEET_INITIAL_MS || 2_000)); const UPDATE_EVERY_MS = Math.max(24 * 60 * 60_000, Number(process.env.HELM_MACHINE_UPDATE_MS || 7 * 24 * 60 * 60_000)); const UPDATE_RETRY_MS = Math.max(60 * 60_000, Number(process.env.HELM_MACHINE_UPDATE_RETRY_MS || 6 * 60 * 60_000)); const MAX_WORKSPACE_SYNC_BYTES = Math.max(64 * 1024 ** 2, Number(process.env.HELM_WORKSPACE_SYNC_MAX_BYTES || 2 * 1024 ** 3)); @@ -85,7 +86,10 @@ const terminalSessions = new Map(); const channelLocks = new Map>(); const syncTimers = new Map(); let reconcileTimer: NodeJS.Timeout | null = null; +let reconcileStartupTimer: NodeJS.Timeout | null = null; let reconcileRunning = false; +let reconcileEnabled = false; +let reconcilePass: Promise | null = null; const installationId = (): string => { let id = String(q1("SELECT installation_id FROM workspace WHERE id=1")?.installation_id || ""); @@ -757,6 +761,10 @@ export async function appRemovalStatus(): Promise<{ backend: ChannelComputerBack export async function prepareAppRemoval(): Promise<{ backend: ChannelComputerBackend; deleted: number; remaining: number }> { const backend = configuredChannelBackend(); if (backend !== "apple") return { backend, deleted: 0, remaining: 0 }; + // Uninstall is a fleet-wide terminal state. Quiesce and fence the reconciler + // before enumerating machines so an already-running pass cannot recreate a + // machine from its stale pre-removal snapshot after deletion completes. + await shutdownChannelComputers(); const install = installationId(); const prefix = `1helm-${install}-channel-`; const machineIds = await ownedInstallationMachineIds(); @@ -1028,19 +1036,32 @@ export async function reconcileChannelComputers(channelIds?: Iterable): } export function startChannelComputerReconciler(): void { - if (reconcileTimer) return; + if (reconcileEnabled) return; + reconcileEnabled = true; const tick = (): void => { - if (reconcileRunning) return; + if (!reconcileEnabled || reconcileRunning) return; reconcileRunning = true; - void reconcileChannelComputers().catch((error) => console.error("channel computer reconcile failed:", (error as Error).message)).finally(() => { reconcileRunning = false; }); + const pass: Promise = reconcileChannelComputers() + .then(() => undefined) + .catch((error) => console.error("channel computer reconcile failed:", (error as Error).message)) + .finally(() => { + reconcileRunning = false; + if (reconcilePass === pass) reconcilePass = null; + }); + reconcilePass = pass; + void pass; }; - setTimeout(tick, 2_000).unref(); + reconcileStartupTimer = setTimeout(() => { reconcileStartupTimer = null; tick(); }, INITIAL_RECONCILE_MS); + reconcileStartupTimer.unref(); reconcileTimer = setInterval(tick, RECONCILE_EVERY_MS); reconcileTimer.unref(); } export async function shutdownChannelComputers(): Promise { + reconcileEnabled = false; + if (reconcileStartupTimer) { clearTimeout(reconcileStartupTimer); reconcileStartupTimer = null; } if (reconcileTimer) { clearInterval(reconcileTimer); reconcileTimer = null; } + await reconcilePass?.catch(() => undefined); for (const session of [...terminalSessions.values()]) closeChannelTerminal(session.id); for (const timer of syncTimers.values()) clearTimeout(timer); syncTimers.clear(); diff --git a/test/channel-computers.mjs b/test/channel-computers.mjs index 80ee9ae..f21eafe 100644 --- a/test/channel-computers.mjs +++ b/test/channel-computers.mjs @@ -19,6 +19,7 @@ process.env.HELM_CHANNEL_COMPUTER_BACKEND = "apple"; process.env.HELM_CONTAINER_CLI = fakeCli; process.env.FAKE_CONTAINER_STATE = fakeState; process.env.HELM_FLEET_INTERVAL_MS = "600000"; +process.env.HELM_FLEET_INITIAL_MS = "25"; process.env.HELM_MACHINE_IDLE_MS = "60000"; const db = await import("../src/server/db.ts"); @@ -143,9 +144,16 @@ test("Apple channel-computer contract preserves isolation, files, wakes, archive const backend = await readFile(join(root, "src", "server", "channel-computers.ts"), "utf8"); assert.match(backend, /machine", "run", "-it"[\s\S]*guestWords\("\/bin\/bash", "-l"\)/, "interactive Apple terminals request an explicit guest login shell"); + // Reproduce the real Apple race: uninstall begins while an automatic fleet + // pass is already inspecting a machine. Removal must wait for that pass, + // fence future ticks, and leave nothing for a stale snapshot to recreate. + process.env.FAKE_CONTAINER_INSPECT_DELAY_MS = "200"; + computers.startChannelComputerReconciler(); + await new Promise((resolveWait) => setTimeout(resolveWait, 50)); const removal = await computers.prepareAppRemoval(); assert.equal(removal.deleted, 1, "uninstall preparation deletes every remaining VM owned by this exact 1Helm installation"); assert.equal(removal.remaining, 0); + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); assert.equal(existsSync(join(fakeState, "machines", betaComputer.machine_id)), false, "no owned channel VM survives uninstall preparation"); computers.reactivateComputersAfterPreparedRemoval(); assert.equal(db.q1("SELECT desired_state FROM channel_computers WHERE channel_id=?", beta.channelId).desired_state, "auto", "a later reinstall can rebuild the removed VM from its preserved host mirror"); diff --git a/test/fake-container.mjs b/test/fake-container.mjs index 2d2cde4..1cd2d6a 100644 --- a/test/fake-container.mjs +++ b/test/fake-container.mjs @@ -63,6 +63,8 @@ if (action === "create") { const config = readConfig(name); if (!config) fail(`machine ${name} not found`); if (action === "inspect") { + const delay = Math.max(0, Number(process.env.FAKE_CONTAINER_INSPECT_DELAY_MS || 0)); + if (delay) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay); process.stdout.write(JSON.stringify(config)); process.exit(0); }