From 75a174a2cb01c624f6af99f9c3f7945c164d20cf Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:18:24 +0000 Subject: [PATCH] S26: wire execution-target placement in shadow mode (SHIP_PLACEMENT=shadow) SandboxPool.create/createFrom compute placeRun beside the existing least-loaded choice, and the C5 liveness failure records what onHostLoss would have done. Both only append to placement-shadow.jsonl; the chosen host and the failure are unchanged. Default off. Adds 'placement shadow-report'. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VqsBNqvaWezf1DwQrAnVgX --- src/cli.ts | 19 ++ src/durable.ts | 10 + src/placement-shadow.test.ts | 348 +++++++++++++++++++++ src/placement-shadow.ts | 590 +++++++++++++++++++++++++++++++++++ src/sandbox-pool.ts | 57 ++++ 5 files changed, 1024 insertions(+) create mode 100644 src/placement-shadow.test.ts create mode 100644 src/placement-shadow.ts diff --git a/src/cli.ts b/src/cli.ts index d6bf5aa..cec59d1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -44,6 +44,7 @@ import { attachEvidence, envAsked } from "./fix-evidence.js"; import type { Evidence } from "./verification.js"; import type { RepoRef } from "./git.js"; import { assertRepoAllowed, credentialFor, policyFromEnv } from "./repo-policy.js"; +import { placementShadowFile, placementShadowFromEnv, readPlacementSummary, renderPlacementReport } from "./placement-shadow.js"; import { readShadowSummary, renderShadowReport, shadowFile } from "./policy-shadow.js"; import { detectRepository } from "./stack-import.js"; import { renderProposal, stackDetectEnabled } from "./stack-propose.js"; @@ -136,6 +137,7 @@ Usage: teploy-ship policy window remove [--source ] (no --source = the global window) teploy-ship policy window check [--source ] is auto allowed right now? teploy-ship policy reviewers set [--users a,b] [--teams t] (both empty = remove) + teploy-ship placement shadow-report [--json] where the placement shadow (SHIP_PLACEMENT=shadow) disagreed with the pool teploy-ship audit export the run history (what ran, cost, PRs) [--format csv|json] [--since ] [--until ] teploy-ship runs list durable runs @@ -950,9 +952,12 @@ function durableProvider(args: ReturnType, config: Config): Ex fetch: longRequestFetch, }); if (urls.length <= 1) return build(urls[0] ?? sandbox.url); + // S26 shadow: undefined (and so absent below) unless SHIP_PLACEMENT=shadow. + const placementShadow = placementShadowFromEnv(process.env, (line) => process.stderr.write(`${line}\n`)); return new SandboxPool({ hosts: urls.map((url) => ({ url, provider: build(url) })), log: (line) => process.stderr.write(`${line}\n`), + ...(placementShadow !== undefined ? { placementShadow } : {}), }); } // Local durable runs: a persistent per-run workspace under the state @@ -1630,6 +1635,18 @@ async function projectCommand(rest: string[]): Promise { * The buyer half of P2-3 (governance.ts): per-user authority, auto windows, * required reviewers. The dashboard's Policies page edits the same store. */ +/** S26 placement shadow: `placement shadow-report [--json]`. Read-only. */ +async function placementCommand(rest: string[]): Promise { + const [sub] = rest; + const args = parseArgs(rest); + if (sub !== "shadow-report") { + fail("usage: teploy-ship placement shadow-report [--json] where the S26 placement shadow (SHIP_PLACEMENT=shadow) disagreed with the pool"); + } + const file = placementShadowFile(); + const summary = await readPlacementSummary(file); + process.stdout.write(args.flags.json === true ? `${JSON.stringify({ file, ...summary }, null, 2)}\n` : `${renderPlacementReport(summary, file)}\n`); +} + async function policyCommand(rest: string[]): Promise { const config = loadConfig(); const [sub, second, third] = rest; @@ -2406,6 +2423,8 @@ async function main(): Promise { return projectCommand(rest); case "policy": return policyCommand(rest); + case "placement": + return placementCommand(rest); case "audit": return auditCommand(rest); case "resume": diff --git a/src/durable.ts b/src/durable.ts index 7770b65..9054b09 100644 --- a/src/durable.ts +++ b/src/durable.ts @@ -717,6 +717,11 @@ export interface ExecutorProvider { * workflow). Providers must be honest here — false is the safe answer. */ isolated?: boolean; + /** + * Shadow observation only (S26): told when the C5 liveness probe found this + * handle's container gone. Must not throw and must not change anything. + */ + observeHostLoss?: (handle: string) => void; snapshot?: (handle: string) => Promise; createFrom?: (image: string, overrides?: SandboxOverrides) => Promise<{ handle: string }>; /** @@ -1103,6 +1108,11 @@ export function durableAgent( () => false, ); if (!alive) { + try { + config.executor.observeHostLoss?.(probed); + } catch { + // shadow evidence, never a dependency of the failure below + } throw new Error( `the sandbox this run recorded (${probed}) is no longer available — it has almost certainly outlived its TTL ` + `(SHIP_SANDBOX_TTL_SEC). A durable run replays its recorded container rather than creating a new one, and ` + diff --git a/src/placement-shadow.test.ts b/src/placement-shadow.test.ts new file mode 100644 index 0000000..4bd1c8e --- /dev/null +++ b/src/placement-shadow.test.ts @@ -0,0 +1,348 @@ +import assert from "node:assert/strict"; +import { appendFile, mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import type { AgentExecutor } from "@neutron-build/agents"; +import type { ModelAdapter } from "@neutron-build/ai"; +import { MemoryEventStore, executeRun } from "@neutron-build/workflow"; + +import { durableAgent } from "./durable.js"; +import type { ExecutorProvider } from "./durable.js"; +import type { RunRequirement } from "./execution-target.js"; +import { + MemoryPlacementSink, + PlacementShadow, + declaredRequirementFromEnv, + declaredTargetsFromEnv, + fileSink, + flushPlacementShadow, + parsePlacementRecords, + placementShadowFromEnv, + readPlacementSummary, + renderPlacementReport, + summarisePlacement, +} from "./placement-shadow.js"; +import type { DeclaredTarget } from "./placement-shadow.js"; +import { SandboxPool } from "./sandbox-pool.js"; + +/** A fake daemon: counts creates, can refuse, and can report its containers as gone. */ +function daemon(name: string, options: { snapshots?: boolean } = {}) { + let creates = 0; + let refusing = false; + let reaped = false; + const provider: ExecutorProvider = { + isolated: true, + async create() { + if (refusing) throw new Error(`${name} is down`); + creates += 1; + return { handle: `${name}-run-${creates}` }; + }, + attach(handle: string) { + return { + async exec() { + if (reaped) throw new Error(`run not found: ${handle}`); + return { exitCode: 0, stdout: "", stderr: "", timedOut: false, truncated: false }; + }, + } as unknown as AgentExecutor; + }, + async destroy() {}, + ...(options.snapshots === true + ? { + async snapshot(handle: string) { + return `${name}-snap-of-${handle}`; + }, + async createFrom(image: string) { + creates += 1; + return { handle: `${name}-from-${image}` }; + }, + } + : {}), + }; + return { provider, refuse: (on: boolean) => (refusing = on), reap: () => (reaped = true) }; +} + +interface Fleet { + pool: SandboxPool; + sink: MemoryPlacementSink; + daemons: ReturnType[]; +} + +function fleet( + declared: Array, + options: { require?: Partial; draining?: string[]; snapshots?: boolean } = {}, +): Fleet { + const sink = new MemoryPlacementSink(); + const daemons = declared.map((_, i) => daemon(`h${i}`, { snapshots: options.snapshots === true })); + const urls = declared.map((_, i) => `http://h${i}:7439`); + const shadow = new PlacementShadow({ + sink, + declared: Object.fromEntries(declared.flatMap((d, i) => (d === undefined ? [] : [[urls[i]!, d]]))), + ...(options.require !== undefined ? { require: options.require } : {}), + ...(options.draining !== undefined ? { draining: options.draining } : {}), + }); + const pool = new SandboxPool({ hosts: daemons.map((d, i) => ({ url: urls[i]!, provider: d.provider })), placementShadow: shadow }); + return { pool, sink, daemons }; +} + +const last = (f: Fleet) => f.sink.records[f.sink.records.length - 1]!; +const wouldTarget = (r: { would?: { ok: boolean; target?: string } }) => (r.would?.ok === true ? r.would.target : undefined); + +// ---- the naive first-healthy / least-loaded choice, run through the real pool ------- + +test("arch: the pool picks the idle amd64 host for an arm64 run; the shadow disagrees and names the arm64 host", async () => { + const f = fleet([{ arch: "amd64" }, { arch: "arm64" }], { require: { arch: "arm64" } }); + const { handle } = await f.pool.create(); + await flushPlacementShadow(); + assert.equal(handle.startsWith("0@"), true, "the pool's choice is untouched"); + const r = last(f); + assert.equal(r.disagreement, "chosen-unsuitable"); + assert.deepEqual(r.chosenRejections?.map((x) => x.code), ["arch"]); + assert.equal(wouldTarget(r), "http://h1:7439"); +}); + +test("arch: agreement when the pool's host is the arm64 one (the shadow is not just always disagreeing)", async () => { + const f = fleet([{ arch: "arm64" }, { arch: "amd64" }], { require: { arch: "arm64" } }); + await f.pool.create(); + await flushPlacementShadow(); + assert.equal(last(f).disagreement, null); +}); + +test("an undeclared host has unknown arch: a requirement naming one is not met by it; no requirement means no disagreement", async () => { + const named = fleet([undefined], { require: { arch: "amd64" } }); + await named.pool.create(); + const none = fleet([undefined]); + await none.pool.create(); + await flushPlacementShadow(); + assert.equal(last(named).disagreement, "chosen-unsuitable"); + assert.equal(last(named).would?.ok, false); + assert.equal(last(none).disagreement, null); + assert.deepEqual(last(none).defaulted, ["os=linux", "project=(unknown)"], "what was defaulted is stated, not hidden"); +}); + +test("browser: a headed-browser run on a pool that picks the browserless host first", async () => { + const f = fleet([{ arch: "amd64", browser: "none" }, { arch: "amd64", browser: "headed" }], { require: { browser: "headed" } }); + await f.pool.create(); + await flushPlacementShadow(); + const r = last(f); + assert.equal(r.disagreement, "chosen-unsuitable"); + assert.match(String(r.chosenRejections?.[0]?.detail), /needs headed browser, target has none/); + assert.equal(wouldTarget(r), "http://h1:7439"); +}); + +test("draining: the pool has no drain concept, so it places on a draining host; the shadow flags it", async () => { + const declared = fleet([{ draining: true }, {}]); + await declared.pool.create(); + const option = fleet([{}, {}], { draining: ["http://h0:7439/"] }); + await option.pool.create(); + await flushPlacementShadow(); + for (const f of [declared, option]) { + assert.equal(last(f).disagreement, "chosen-unsuitable"); + assert.deepEqual(last(f).chosenRejections?.map((x) => x.code), ["draining"]); + assert.equal(wouldTarget(last(f)), "http://h1:7439"); + } +}); + +test("quota: least-loaded still lands on a host at its declared run cap", async () => { + const f = fleet([{ quota: { maxRuns: 1 } }, { quota: { maxRuns: 5 } }]); + await f.pool.create(); // h0 (tie -> first) + await f.pool.create(); // h1 (least loaded) + await f.pool.create(); // h0 again (tie) but h0 is at 1/1 + await flushPlacementShadow(); + assert.deepEqual(f.sink.records.map((r) => r.disagreement), [null, null, "chosen-unsuitable"]); + assert.deepEqual(last(f).chosenRejections?.map((x) => x.code), ["quota"]); + assert.equal(wouldTarget(last(f)), "http://h1:7439"); +}); + +test("per-project quota counts this pool's live runs per project and drops them on destroy", async () => { + const f = fleet([{ quota: { maxRuns: 9, perProject: 1 } }]); + const ov = { warm: { repo: "acme/web" } }; + const first = await f.pool.create(ov); + await f.pool.create(ov); + await f.pool.destroy(first.handle); + await f.pool.create(ov); + await f.pool.create({ warm: { repo: "acme/other" } }); + await flushPlacementShadow(); + assert.deepEqual(f.sink.records.map((r) => r.disagreement), [null, "chosen-unsuitable", "chosen-unsuitable", null]); + assert.match(String(f.sink.records[1]!.chosenRejections?.[0]?.detail), /project acme\/web has 1\/1/); + assert.equal(f.sink.records[3]!.requirement.project, "acme/other"); +}); + +test("cooldown: a host placed on during the cooling pass is recorded as unhealthy", async () => { + const f = fleet([{}, {}]); + f.daemons[0]!.refuse(true); + f.daemons[1]!.refuse(true); + await assert.rejects(f.pool.create(), /no sandbox host could start a sandbox/); + f.daemons[0]!.refuse(false); + await f.pool.create(); + await flushPlacementShadow(); + assert.equal(f.sink.records.length, 1, "a create that failed everywhere records nothing"); + assert.equal(last(f).disagreement, "chosen-unsuitable"); + assert.deepEqual(last(f).chosenRejections?.map((x) => x.code), ["unhealthy"]); +}); + +test("failover: the host that finally took the run is the one judged (benign different-choice)", async () => { + const f = fleet([{}, {}]); + f.daemons[0]!.refuse(true); + const { handle } = await f.pool.create(); + await flushPlacementShadow(); + assert.equal(handle.startsWith("1@"), true); + assert.equal(last(f).host, "http://h1:7439"); + assert.equal(last(f).disagreement, "different-choice", "placeRun would have tried h0 first; the difference is the failover itself, and it is recorded as such"); + assert.equal(wouldTarget(last(f)), "http://h0:7439"); +}); + +test("createFrom: a restore onto a draining snapshot host is flagged; one onto a healthy host is not", async () => { + const f = fleet([{ draining: true }, {}], { snapshots: true }); + await f.pool.createFrom!("0@snap-a"); + await f.pool.createFrom!("1@snap-b"); + await flushPlacementShadow(); + assert.deepEqual(f.sink.records.map((r) => [r.point, r.disagreement]), [["createFrom", "chosen-unsuitable"], ["createFrom", null]]); + assert.equal(f.sink.records[0]!.would?.ok, false); +}); + +// ---- host loss ----------------------------------------------------------------------- + +test("observeHostLoss: records both bounds and never claims preservation it cannot show", async () => { + const f = fleet([{}, {}]); + const { handle } = await f.pool.create(); + f.pool.observeHostLoss(handle); + await flushPlacementShadow(); + const r = f.sink.records[1]!; + assert.equal(r.point, "host-loss"); + assert.equal(r.host, "http://h0:7439"); + assert.equal(r.hostLoss?.ifClean.action, "recover"); + assert.equal(r.hostLoss?.ifClean.action === "recover" && r.hostLoss.ifClean.target.id, "http://h1:7439"); + assert.deepEqual(r.hostLoss?.ifDirty, { + action: "fail", + reason: "host http://h0:7439 was lost and the run has no snapshot; the uncommitted working tree is gone and the run cannot restart from its committed state", + retryable: false, + }); + assert.equal(r.disagreement, "would-recover"); + assert.equal(r.hostLoss?.hostHealthyPerPool, true, "a healthy host with a dead container looks like TTL expiry, and is labelled as such"); +}); + +test("observeHostLoss on a one-host pool: nothing to recover onto, so no disagreement", async () => { + const f = fleet([{}]); + const { handle } = await f.pool.create(); + f.pool.observeHostLoss(handle); + await flushPlacementShadow(); + const r = last(f); + assert.equal(r.disagreement, null); + assert.equal(r.hostLoss?.ifClean.action, "fail"); +}); + +test("the real C5 path: a failed liveness probe is reported to the pool, and the run still fails exactly as before", async () => { + const f = fleet([{}, {}]); + f.daemons[0]!.reap(); + const model = { + provider: "t", + modelId: "t", + async doGenerate() { + throw new Error("model must not be reached"); + }, + } as unknown as ModelAdapter; + const wf = durableAgent({ model, executor: f.pool }); + const out = await executeRun({ workflow: wf, runId: "run-hostloss", store: new MemoryEventStore(), input: { task: "x" } }); + await flushPlacementShadow(); + assert.equal(out.status, "failed"); + assert.match(String(out.error?.detail ?? ""), /\(0@h0-run-1\) is no longer available/); + assert.deepEqual(f.sink.records.map((r) => r.point), ["create", "host-loss"]); + assert.equal(f.sink.records[1]!.hostLoss?.ifClean.action, "recover"); +}); + +// ---- off / equivalence --------------------------------------------------------------- + +test("shadow on or off, the pool places identically (handles, order, failover)", async () => { + const script = async (pool: SandboxPool, ds: ReturnType[]) => { + const out: string[] = []; + out.push((await pool.create()).handle); + out.push((await pool.create()).handle); + ds[0]!.refuse(true); + out.push((await pool.create()).handle); + out.push(JSON.stringify(pool.state())); + return out; + }; + const on = fleet([{ arch: "amd64", draining: true }, { arch: "arm64" }], { require: { arch: "arm64", browser: "headed" } }); + const offDaemons = [daemon("h0"), daemon("h1")]; + const off = new SandboxPool({ hosts: offDaemons.map((d, i) => ({ url: `http://h${i}:7439`, provider: d.provider })) }); + assert.deepEqual(await script(on.pool, on.daemons), await script(off, offDaemons)); +}); + +test("with no shadow the pool has no bookkeeping and observeHostLoss does nothing", async () => { + const d = daemon("h0"); + const pool = new SandboxPool({ hosts: [{ url: "http://h0:7439", provider: d.provider }] }); + const { handle } = await pool.create({ warm: { repo: "acme/web" } }); + pool.observeHostLoss(handle); + await pool.destroy(handle); + assert.equal(pool.state()[0]!.live, 0); +}); + +test("a sink that throws cannot fail a placement", async () => { + const logs: string[] = []; + const shadow = new PlacementShadow({ + sink: { + append: async () => { + throw new Error("disk full"); + }, + }, + log: (l) => logs.push(l), + }); + const d = daemon("h0"); + const pool = new SandboxPool({ hosts: [{ url: "http://h0:7439", provider: d.provider }], placementShadow: shadow }); + assert.equal((await pool.create()).handle, "0@h0-run-1"); + await flushPlacementShadow(); + assert.ok(logs.some((l) => /could not record: disk full/.test(l))); +}); + +test("SHIP_PLACEMENT: only 'shadow' turns it on; anything else is off and says so", () => { + const log: string[] = []; + assert.equal(placementShadowFromEnv({}, (l) => log.push(l)), undefined); + assert.equal(placementShadowFromEnv({ SHIP_PLACEMENT: "off" }, (l) => log.push(l)), undefined); + assert.equal(log.length, 0); + assert.equal(placementShadowFromEnv({ SHIP_PLACEMENT: "on" }, (l) => log.push(l)), undefined); + assert.match(log[0]!, /only "shadow" exists/); + assert.ok(placementShadowFromEnv({ SHIP_PLACEMENT: "Shadow", SHIP_PLACEMENT_SHADOW_FILE: "/dev/null" }) !== undefined); +}); + +test("declared config: malformed JSON, wrong types and unknown keys fall back to conservative defaults", () => { + const log: string[] = []; + assert.deepEqual(declaredTargetsFromEnv({ SHIP_PLACEMENT_TARGETS: "{nope" }, (l) => log.push(l)), {}); + assert.match(log[0]!, /not valid JSON/); + const t = declaredTargetsFromEnv({ + SHIP_PLACEMENT_TARGETS: JSON.stringify({ "http://a:1/": { arch: "sparc", browser: "headed", cpu: -3, quota: { maxRuns: 2 }, bogus: 1, services: ["pg", 4] } }), + }); + assert.deepEqual(t, { "http://a:1": { browser: "headed", quota: { maxRuns: 2 } } }); + assert.deepEqual(declaredRequirementFromEnv({ SHIP_PLACEMENT_REQUIRE: JSON.stringify({ arch: "arm64", browser: "none", gpu: 1, x: 1 }) }), { arch: "arm64", gpu: 1 }); +}); + +// ---- file + report ------------------------------------------------------------------- + +test("records land in JSONL and the report counts disagreements; a missing file is unknown, not zero", async () => { + const dir = await mkdtemp(join(tmpdir(), "placement-shadow-")); + const file = join(dir, "nested", "placement-shadow.jsonl"); + const missing = await readPlacementSummary(file); + assert.equal(missing.fileFound, false); + assert.match(renderPlacementReport(missing, file), /unknown, not zero disagreements/); + + const shadow = new PlacementShadow({ + sink: fileSink(file), + declared: { "http://h0:7439": { arch: "amd64" }, "http://h1:7439": { arch: "arm64" } }, + require: { arch: "arm64" }, + }); + const ds = [daemon("h0"), daemon("h1")]; + const pool = new SandboxPool({ hosts: ds.map((d, i) => ({ url: `http://h${i}:7439`, provider: d.provider })), placementShadow: shadow }); + await pool.create(); + await pool.create(); + await flushPlacementShadow(); + await appendFile(file, "not json\n"); + const s = await readPlacementSummary(file); + assert.equal(s.records, 2); + assert.equal(s.malformedLines, 1); + assert.equal(s.disagreements, 1); + assert.deepEqual(s.byRejection, { arch: 1 }); + assert.match(renderPlacementReport(s, file), /1x chosen-unsuitable \(create\) on http:\/\/h0:7439/); + assert.equal(parsePlacementRecords("").records.length, 0); + assert.equal(summarisePlacement([]).disagreements, 0); +}); diff --git a/src/placement-shadow.ts b/src/placement-shadow.ts new file mode 100644 index 0000000..d1e057b --- /dev/null +++ b/src/placement-shadow.ts @@ -0,0 +1,590 @@ +/** + * S26 in SHADOW mode: run `placeRun` / `onHostLoss` (execution-target.ts) next + * to the placement the pool actually made, record where they differ, change + * nothing. + * + * Flag `SHIP_PLACEMENT=shadow` (default off; any other value is off). Off + * constructs no shadow, so `SandboxPool` runs exactly the code it ran before. + * On, every `create`, every `createFrom` and every failed C5 liveness probe + * appends ONE JSON line to `placement-shadow.jsonl` in the state directory + * (`SHIP_PLACEMENT_SHADOW_FILE` overrides). `teploy-ship placement + * shadow-report` summarises it. Nothing here can change which host is used or + * how a run fails: the comparison runs on a snapshot taken before the real + * attempt, and a throw anywhere in it is swallowed and logged. + * + * WHAT THE TARGETS ARE. The pool knows a URL, a provider and (new, optional) + * what the operator DECLARED about it in `SHIP_PLACEMENT_TARGETS` (a JSON + * object keyed by host URL). Anything undeclared is the conservative answer: + * os linux (a sandbox daemon runs Linux containers), arch UNKNOWN (a sentinel + * that equals no real architecture, so a requirement naming an arch is never + * met by it), no browser, no desktop, no services, no gpu, no private + * networks, static credentials, no snapshot formats beyond oci-image. cpu, + * memory, disk and the run cap default to "very large": the daemon enforces + * the per-container limits itself at create, and an invented small number would + * manufacture disagreements. They are finite so utilisation (live / maxRuns) + * orders hosts exactly as the pool's own live count does. + * + * WHAT THE REQUIREMENT IS. Ship records no per-run arch/browser/etc today, so + * the requirement is: `limits` and `warm.repo` from the run's SandboxOverrides + * (cpu, memory, project/warm repo), merged over an operator-declared + * deployment-wide `SHIP_PLACEMENT_REQUIRE` (JSON; e.g. every run here needs + * arm64). `os` defaults to linux and every record says so. + * + * WHAT IS NOT KNOWN. Draining: the pool has no drain concept, so draining + * comes only from `SHIP_PLACEMENT_TARGETS` (`"draining": true`) or the + * `draining` option. Whether a failed liveness probe is host loss or just an + * expired container TTL: the pool cannot tell, so a host-loss record states the + * host's own health and the decision under both a clean and a dirty tree (a + * replay cannot see the tree). Per-project live counts exist only for runs + * this process placed. A create that fails on every host is not recorded. + */ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import type { SandboxOverrides } from "./durable.js"; +import { onHostLoss, placeRun } from "./execution-target.js"; +import type { + HostLossDecision, + Placement, + PlacementState, + Rejection, + RunOnHost, + RunRequirement, + SnapshotRef, + Target, + TargetCapabilities, + TargetState, +} from "./execution-target.js"; +import { stateDir } from "./run-store.js"; + +export const PLACEMENT_FLAG = "SHIP_PLACEMENT"; + +export function placementMode(env: NodeJS.ProcessEnv = process.env): "off" | "shadow" { + return env[PLACEMENT_FLAG]?.trim().toLowerCase() === "shadow" ? "shadow" : "off"; +} + +export const placementShadowFile = (env: NodeJS.ProcessEnv = process.env): string => + env.SHIP_PLACEMENT_SHADOW_FILE ?? join(stateDir(), "placement-shadow.jsonl"); + +/** Matches no real `TargetArch`: a host that never declared one fits no requirement that names one. */ +export const UNKNOWN_ARCH = "unknown" as unknown as TargetCapabilities["arch"]; + +const BIG = 1_000_000; + +// --------------------------------------------------------------------------- +// Declared capabilities +// --------------------------------------------------------------------------- + +/** What an operator may declare per host. Unknown keys and wrongly typed values are ignored. */ +export type DeclaredTarget = Partial> & { + quota?: Partial; + hardware?: Partial; + /** Operator-declared: no new work. The pool itself has no drain. */ + draining?: boolean; +}; + +export interface HostLike { + url: string; + isolated?: boolean; +} + +/** Conservative capabilities for a host, overlaid with whatever was declared. */ +export function declaredCaps(host: HostLike, declared: DeclaredTarget = {}): TargetCapabilities { + const q = declared.quota ?? {}; + const h = declared.hardware ?? {}; + return { + kind: declared.kind ?? "remote-container", + hosting: declared.hosting ?? "managed", + os: declared.os ?? "linux", + arch: declared.arch ?? UNKNOWN_ARCH, + cpu: declared.cpu ?? BIG, + memMB: declared.memMB ?? BIG, + diskMB: declared.diskMB ?? BIG, + browser: declared.browser ?? "none", + desktop: declared.desktop ?? false, + services: declared.services ?? [], + hardware: { gpu: h.gpu ?? 0, mobileSim: h.mobileSim ?? [] }, + network: declared.network ?? [], + snapshotFormats: declared.snapshotFormats ?? ["oci-image"], + isolated: declared.isolated ?? host.isolated === true, + credentialMode: declared.credentialMode ?? "static", + quota: { maxRuns: q.maxRuns ?? BIG, ...(q.perProject !== undefined ? { perProject: q.perProject } : {}) }, + }; +} + +const isStr = (v: unknown): v is string => typeof v === "string"; +const isNum = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v) && v >= 0; +const isBool = (v: unknown): v is boolean => typeof v === "boolean"; +const strList = (v: unknown): string[] | undefined => (Array.isArray(v) && v.every(isStr) ? (v as string[]) : undefined); +const oneOf = (v: unknown, options: readonly T[]): T | undefined => (isStr(v) && (options as readonly string[]).includes(v) ? (v as T) : undefined); + +function sanitizeDeclared(raw: unknown): DeclaredTarget { + if (raw === null || typeof raw !== "object") return {}; + const r = raw as Record; + const out: Record = {}; + const put = (k: string, v: unknown) => { + if (v !== undefined) out[k] = v; + }; + put("kind", oneOf(r.kind, ["local-container", "remote-container", "local-vm", "remote-vm", "customer-worker"] as const)); + put("hosting", oneOf(r.hosting, ["managed", "customer"] as const)); + put("os", oneOf(r.os, ["linux", "windows", "macos"] as const)); + put("arch", oneOf(r.arch, ["amd64", "arm64"] as const)); + for (const k of ["cpu", "memMB", "diskMB"]) put(k, isNum(r[k]) ? r[k] : undefined); + put("browser", oneOf(r.browser, ["none", "headless", "headed"] as const)); + put("desktop", isBool(r.desktop) ? r.desktop : undefined); + put("services", strList(r.services)); + put("network", strList(r.network)); + put("snapshotFormats", strList(r.snapshotFormats)); + put("isolated", isBool(r.isolated) ? r.isolated : undefined); + put("credentialMode", oneOf(r.credentialMode, ["disposable", "static", "none"] as const)); + put("draining", isBool(r.draining) ? r.draining : undefined); + if (r.quota !== null && typeof r.quota === "object") { + const q = r.quota as Record; + put("quota", { ...(isNum(q.maxRuns) ? { maxRuns: q.maxRuns } : {}), ...(isNum(q.perProject) ? { perProject: q.perProject } : {}) }); + } + if (r.hardware !== null && typeof r.hardware === "object") { + const hw = r.hardware as Record; + const sims = Array.isArray(hw.mobileSim) ? hw.mobileSim.filter((s): s is "android" | "ios" => s === "android" || s === "ios") : undefined; + put("hardware", { ...(isNum(hw.gpu) ? { gpu: hw.gpu } : {}), ...(sims !== undefined ? { mobileSim: sims } : {}) }); + } + return out as DeclaredTarget; +} + +function parseJson(raw: string | undefined, name: string, log: (line: string) => void): unknown { + if (raw === undefined || raw.trim() === "") return undefined; + try { + return JSON.parse(raw); + } catch (error) { + log(`[placement] ${name} is not valid JSON (${error instanceof Error ? error.message : String(error)}); using conservative defaults`); + return undefined; + } +} + +/** `SHIP_PLACEMENT_TARGETS`: `{ "": { ...declared } }`. URLs are matched without a trailing slash. */ +export function declaredTargetsFromEnv(env: NodeJS.ProcessEnv, log: (line: string) => void = () => {}): Record { + const raw = parseJson(env.SHIP_PLACEMENT_TARGETS, "SHIP_PLACEMENT_TARGETS", log); + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return {}; + const out: Record = {}; + for (const [url, v] of Object.entries(raw as Record)) out[url.replace(/\/+$/, "")] = sanitizeDeclared(v); + return out; +} + +/** `SHIP_PLACEMENT_REQUIRE`: a deployment-wide requirement merged under each run's own. */ +export function declaredRequirementFromEnv(env: NodeJS.ProcessEnv, log: (line: string) => void = () => {}): Partial { + const raw = parseJson(env.SHIP_PLACEMENT_REQUIRE, "SHIP_PLACEMENT_REQUIRE", log); + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return {}; + const r = raw as Record; + const out: Partial = {}; + const os = oneOf(r.os, ["linux", "windows", "macos"] as const); + if (os !== undefined) out.os = os; + const arch = oneOf(r.arch, ["amd64", "arm64"] as const); + if (arch !== undefined) out.arch = arch; + for (const k of ["cpu", "memMB", "diskMB", "gpu"] as const) if (isNum(r[k])) out[k] = r[k] as number; + const browser = oneOf(r.browser, ["headless", "headed"] as const); + if (browser !== undefined) out.browser = browser; + if (r.desktop === true) out.desktop = true; + const services = strList(r.services); + if (services !== undefined) out.services = services; + const network = strList(r.network); + if (network !== undefined) out.network = network; + const sim = oneOf(r.mobileSim, ["android", "ios"] as const); + if (sim !== undefined) out.mobileSim = sim; + const hosting = oneOf(r.hosting, ["managed", "customer"] as const); + if (hosting !== undefined) out.hosting = hosting; + if (r.isolated === true) out.isolated = true; + if (r.credentials === "disposable") out.credentials = "disposable"; + return out; +} + +/** The requirement for a create: deployment-wide declaration under the run's own overrides. */ +export function requirementOf(overrides: SandboxOverrides | undefined, base: Partial = {}): { req: RunRequirement; defaults: string[] } { + const defaults: string[] = []; + if (base.os === undefined) defaults.push("os=linux"); + const repo = overrides?.warm?.repo; + const cpu = overrides?.limits?.cpus; + const memMB = overrides?.limits?.memoryMb; + const req: RunRequirement = { + ...base, + project: repo ?? "(unknown)", + os: base.os ?? "linux", + ...(cpu !== undefined ? { cpu } : {}), + ...(memMB !== undefined ? { memMB } : {}), + ...(repo !== undefined ? { warmRepo: repo } : {}), + }; + if (repo === undefined) defaults.push("project=(unknown)"); + return { req, defaults }; +} + +// --------------------------------------------------------------------------- +// Records +// --------------------------------------------------------------------------- + +export type PlacementPoint = "create" | "createFrom" | "host-loss"; + +/** + * - `chosen-unsuitable`: the host the pool used would have been rejected. + * - `different-choice`: the pool's host was suitable but placeRun ranks another first. + * - `would-recover`: host loss, where the existing path fails and onHostLoss would recover. + */ +export type PlacementDisagreement = "chosen-unsuitable" | "different-choice" | "would-recover"; + +export interface PlacementRecord { + v: 1; + at: string; + point: PlacementPoint; + disagreement: PlacementDisagreement | null; + requirement: RunRequirement; + /** What the requirement filled in because nothing declared it. */ + defaulted: string[]; + /** The host the pool used (create/createFrom) or lost (host-loss). */ + host: string; + /** What the pool's own order made of it. */ + state: Record; + /** create / createFrom */ + would?: { ok: true; target: string; suitable: string[] } | { ok: false; refusal: "no-suitable-target" | "retry-later"; reason: string }; + chosenRejections?: Rejection[]; + /** host-loss */ + hostLoss?: { + cause: string; + existing: string; + hostHealthyPerPool: boolean; + ifClean: HostLossDecision; + ifDirty: HostLossDecision; + snapshot?: SnapshotRef; + }; +} + +export interface PlacementSink { + append(record: PlacementRecord): Promise; +} + +export function fileSink(path: string): PlacementSink { + return { + async append(record) { + await mkdir(dirname(path), { recursive: true }); + await appendFile(path, `${JSON.stringify(record)}\n`); + }, + }; +} + +export class MemoryPlacementSink implements PlacementSink { + readonly records: PlacementRecord[] = []; + async append(record: PlacementRecord): Promise { + this.records.push(structuredClone(record)); + } +} + +// --------------------------------------------------------------------------- +// The shadow +// --------------------------------------------------------------------------- + +/** What the pool hands over: its own view of each host at one instant. */ +export interface PoolView { + hosts: Array<{ url: string; isolated?: boolean; live: number; healthy: boolean }>; + /** Per host index: live runs per project that THIS pool placed. */ + projectRuns: Array>; +} + +export interface PlacementShadowOptions { + sink: PlacementSink; + declared?: Record; + /** Deployment-wide requirement (SHIP_PLACEMENT_REQUIRE). */ + require?: Partial; + /** Hosts (by URL) treated as draining in addition to the declared ones. */ + draining?: string[]; + log?: (line: string) => void; + now?: () => Date; +} + +const pending = new Set>(); + +/** Wait for in-flight shadow writes (tests, and a clean worker shutdown). */ +export async function flushPlacementShadow(): Promise { + while (pending.size > 0) await Promise.allSettled([...pending]); +} + +export class PlacementShadow { + readonly #sink: PlacementSink; + readonly #declared: Record; + readonly #require: Partial; + readonly #draining: Set; + readonly #log: (line: string) => void; + readonly #now: () => Date; + + constructor(options: PlacementShadowOptions) { + this.#sink = options.sink; + this.#declared = options.declared ?? {}; + this.#require = options.require ?? {}; + this.#draining = new Set((options.draining ?? []).map((u) => u.replace(/\/+$/, ""))); + this.#log = options.log ?? (() => {}); + this.#now = options.now ?? (() => new Date()); + } + + #targets(view: PoolView): { targets: Target[]; state: PlacementState; ids: string[] } { + const targets: Target[] = []; + const state: PlacementState = {}; + view.hosts.forEach((h, i) => { + const url = h.url.replace(/\/+$/, ""); + const d = this.#declared[url] ?? {}; + targets.push({ id: h.url, caps: declaredCaps(h, d) }); + const s: TargetState = { + healthy: h.healthy, + draining: d.draining === true || this.#draining.has(url), + live: h.live, + usedCpu: 0, + usedMemMB: 0, + usedDiskMB: 0, + projectRuns: view.projectRuns[i] ?? {}, + }; + state[h.url] = s; + }); + return { targets, state, ids: view.hosts.map((h) => h.url) }; + } + + #summary(state: PlacementState): PlacementRecord["state"] { + return Object.fromEntries(Object.entries(state).map(([id, s]) => [id, { live: s.live, healthy: s.healthy, draining: s.draining }])); + } + + #emit(record: PlacementRecord): void { + this.#log(`[placement] shadow ${record.point} host=${record.host} disagreement=${record.disagreement ?? "none"}`); + const p = this.#sink + .append(record) + .catch((error: unknown) => this.#log(`[placement] could not record: ${error instanceof Error ? error.message : String(error)}`)) + .finally(() => pending.delete(p)); + pending.add(p); + } + + /** + * Decide a create the way placeRun would, from the view taken BEFORE the real + * attempt, and return a function that records the comparison once the pool + * knows which host it used. Never throws. + */ + begin(point: "create" | "createFrom", view: PoolView, overrides: SandboxOverrides | undefined, restoreIndex?: number): (chosenIndex: number) => void { + let decided: { placement: Placement; targets: Target[]; state: PlacementState; req: RunRequirement; defaulted: string[] } | undefined; + try { + const { targets, state } = this.#targets(view); + const { req: base, defaults } = requirementOf(overrides, this.#require); + let req = base; + if (restoreIndex !== undefined) { + const t = targets[restoreIndex]; + if (t !== undefined) req = { ...base, restore: { format: "oci-image", os: t.caps.os, arch: t.caps.arch, host: t.id } }; + } + decided = { placement: placeRun(req, targets, state), targets, state, req, defaulted: defaults }; + } catch (error) { + this.#log(`[placement] shadow could not decide: ${error instanceof Error ? error.message : String(error)}`); + } + return (chosenIndex) => { + try { + if (decided === undefined) return; + const chosen = decided.targets[chosenIndex]; + if (chosen === undefined) return; + const p = decided.placement; + const verdict = p.verdicts.find((v) => v.targetId === chosen.id); + const rejections = verdict?.rejections ?? []; + const disagreement: PlacementDisagreement | null = + rejections.length > 0 ? "chosen-unsuitable" : p.ok && p.target.id !== chosen.id ? "different-choice" : null; + this.#emit({ + v: 1, + at: this.#now().toISOString(), + point, + disagreement, + requirement: decided.req, + defaulted: decided.defaulted, + host: chosen.id, + state: this.#summary(decided.state), + would: p.ok + ? { ok: true, target: p.target.id, suitable: p.suitable } + : { ok: false, refusal: p.refusal, reason: p.reason }, + ...(rejections.length > 0 ? { chosenRejections: rejections } : {}), + }); + } catch (error) { + this.#log(`[placement] shadow could not record: ${error instanceof Error ? error.message : String(error)}`); + } + }; + } + + /** + * A run's container failed its liveness probe on `lostIndex`. Record what + * onHostLoss would have done. The existing path always fails the run (the + * C5 error); the tree's state is unknown to a replay, so both bounds are + * recorded. Never throws. + */ + hostLoss(view: PoolView, lostIndex: number, requirement: { req: RunRequirement; defaults: string[] }, snapshotIndex?: number): void { + try { + const { targets, state } = this.#targets(view); + const lost = targets[lostIndex]; + if (lost === undefined) return; + const snapshot: SnapshotRef | undefined = + snapshotIndex !== undefined && targets[snapshotIndex] !== undefined + ? { format: "oci-image", os: targets[snapshotIndex]!.caps.os, arch: targets[snapshotIndex]!.caps.arch, host: targets[snapshotIndex]!.id } + : undefined; + const req = requirement.req; + const run = (uncommittedWork: boolean, restartableFromCommitted: boolean): RunOnHost => ({ + id: "run", + targetId: lost.id, + requirement: req, + ...(snapshot !== undefined ? { snapshot } : {}), + uncommittedWork, + restartableFromCommitted, + }); + const ifClean = onHostLoss(run(false, true), targets, state); + const ifDirty = onHostLoss(run(true, false), targets, state); + this.#emit({ + v: 1, + at: this.#now().toISOString(), + point: "host-loss", + disagreement: ifClean.action === "recover" || ifDirty.action === "recover" ? "would-recover" : null, + requirement: req, + defaulted: requirement.defaults, + host: lost.id, + state: this.#summary(state), + hostLoss: { + cause: "c5 liveness probe failed: host loss and container TTL expiry are indistinguishable here", + existing: "fail: re-enqueue (sandbox no longer available)", + hostHealthyPerPool: state[lost.id]?.healthy === true, + ifClean, + ifDirty, + ...(snapshot !== undefined ? { snapshot } : {}), + }, + }); + } catch (error) { + this.#log(`[placement] shadow could not record host loss: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +/** The shadow for this process, or undefined when `SHIP_PLACEMENT` is not `shadow`. */ +export function placementShadowFromEnv(env: NodeJS.ProcessEnv = process.env, log: (line: string) => void = () => {}): PlacementShadow | undefined { + const raw = env[PLACEMENT_FLAG]?.trim(); + if (raw !== undefined && raw !== "" && placementMode(env) === "off" && raw.toLowerCase() !== "off") { + log(`[placement] ${PLACEMENT_FLAG}=${raw} is not a mode (only "shadow" exists); placement shadow is off`); + } + if (placementMode(env) !== "shadow") return undefined; + const draining = env.SHIP_PLACEMENT_DRAINING?.split(",").map((s) => s.trim()).filter((s) => s !== ""); + return new PlacementShadow({ + sink: fileSink(placementShadowFile(env)), + declared: declaredTargetsFromEnv(env, log), + require: declaredRequirementFromEnv(env, log), + ...(draining !== undefined ? { draining } : {}), + log, + }); +} + +// --------------------------------------------------------------------------- +// Report +// --------------------------------------------------------------------------- + +export interface PlacementSummary { + fileFound: boolean; + records: number; + malformedLines: number; + byPoint: Record; + disagreements: number; + byKind: Record; + /** Rejection code -> count, across chosen-unsuitable records. */ + byRejection: Record; + groups: Array<{ kind: PlacementDisagreement; point: PlacementPoint; host: string; detail: string; count: number; first: string; last: string }>; + notes: string[]; +} + +export const PLACEMENT_REPORT_NOTES: readonly string[] = [ + "shadow only: no placement or failure was changed by this module", + "undeclared hosts have unknown architecture: a requirement naming an arch disagrees with them by design", + "host-loss records cannot tell host loss from container TTL expiry, and assume nothing about the working tree (both bounds are recorded)", +]; + +export function parsePlacementRecords(text: string): { records: PlacementRecord[]; malformed: number } { + const records: PlacementRecord[] = []; + let malformed = 0; + for (const line of text.split("\n")) { + if (line.trim() === "") continue; + try { + const r = JSON.parse(line) as PlacementRecord; + if (r === null || typeof r !== "object" || r.v !== 1 || typeof r.point !== "string") malformed += 1; + else records.push(r); + } catch { + malformed += 1; + } + } + return { records, malformed }; +} + +function detailOf(r: PlacementRecord): string { + if (r.point === "host-loss") { + const h = r.hostLoss!; + const d = (x: HostLossDecision) => (x.action === "recover" ? `recover on ${x.target.id} from ${x.from}` : `fail (${x.retryable ? "retryable" : "not retryable"})`); + return `clean tree: ${d(h.ifClean)}; dirty tree: ${d(h.ifDirty)}`; + } + const why = (r.chosenRejections ?? []).map((x) => `${x.code} (${x.detail})`).join("; "); + const would = r.would === undefined ? "" : r.would.ok ? `placeRun would use ${r.would.target}` : `placeRun would refuse (${r.would.refusal})`; + return [why, would].filter((s) => s !== "").join(" -> "); +} + +export function summarisePlacement(records: readonly PlacementRecord[], malformed = 0): PlacementSummary { + const s: PlacementSummary = { + fileFound: true, + records: records.length, + malformedLines: malformed, + byPoint: { create: 0, createFrom: 0, "host-loss": 0 }, + disagreements: 0, + byKind: { "chosen-unsuitable": 0, "different-choice": 0, "would-recover": 0 }, + byRejection: {}, + groups: [], + notes: [...PLACEMENT_REPORT_NOTES], + }; + const groups = new Map(); + for (const r of records) { + if (r.point in s.byPoint) s.byPoint[r.point] += 1; + if (r.disagreement === null || r.disagreement === undefined) continue; + s.disagreements += 1; + s.byKind[r.disagreement] = (s.byKind[r.disagreement] ?? 0) + 1; + for (const x of r.chosenRejections ?? []) s.byRejection[x.code] = (s.byRejection[x.code] ?? 0) + 1; + const detail = detailOf(r); + const key = [r.disagreement, r.point, r.host, detail].join("\u0000"); + const g = groups.get(key); + if (g === undefined) groups.set(key, { kind: r.disagreement, point: r.point, host: r.host, detail, count: 1, first: r.at, last: r.at }); + else { + g.count += 1; + if (r.at < g.first) g.first = r.at; + if (r.at > g.last) g.last = r.at; + } + } + s.groups = [...groups.values()].sort((a, b) => b.count - a.count); + return s; +} + +export function renderPlacementReport(s: PlacementSummary, file: string): string { + const out: string[] = []; + out.push(`placement shadow report (${file})`); + if (!s.fileFound) out.push(`no record file: the shadow has not recorded anything here (${PLACEMENT_FLAG} is not "shadow", or never ran). This is unknown, not zero disagreements.`); + out.push(`${s.records} decision(s): ${s.byPoint.create} create, ${s.byPoint.createFrom} createFrom, ${s.byPoint["host-loss"]} host-loss`); + out.push( + `${s.disagreements} disagreement(s): ${s.byKind["chosen-unsuitable"]} chosen host unsuitable, ` + + `${s.byKind["different-choice"]} different choice, ${s.byKind["would-recover"]} would recover where the run fails today`, + ); + if (s.malformedLines > 0) out.push(`${s.malformedLines} unreadable line(s) skipped`); + const codes = Object.entries(s.byRejection).sort((a, b) => b[1] - a[1]); + if (codes.length > 0) out.push(`rejections of the chosen host: ${codes.map(([c, n]) => `${c}=${n}`).join(", ")}`); + for (const g of s.groups) { + out.push(""); + out.push(`${g.count}x ${g.kind} (${g.point}) on ${g.host}`); + out.push(` ${g.detail}`); + out.push(` seen ${g.first} .. ${g.last}`); + } + out.push(""); + for (const n of s.notes) out.push(`note: ${n}`); + return out.join("\n"); +} + +export async function readPlacementSummary(path: string): Promise { + let text = ""; + let fileFound = true; + try { + text = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + fileFound = false; + } + const { records, malformed } = parsePlacementRecords(text); + return { ...summarisePlacement(records, malformed), fileFound }; +} diff --git a/src/sandbox-pool.ts b/src/sandbox-pool.ts index 1573bf8..d28cb9d 100644 --- a/src/sandbox-pool.ts +++ b/src/sandbox-pool.ts @@ -33,6 +33,8 @@ import type { AgentExecutor, ExecResult } from "@neutron-build/agents"; import type { ExecutorProvider, SandboxOverrides } from "./durable.js"; +import { requirementOf } from "./placement-shadow.js"; +import type { PlacementShadow, PoolView } from "./placement-shadow.js"; import type { WarmState } from "./warm.js"; /** Separator between the host tag and the daemon's own run id. */ @@ -102,6 +104,12 @@ export interface SandboxPoolOptions { log?: (line: string) => void; now?: () => number; cooldownMs?: number; + /** + * S26 shadow (placement-shadow.ts). Absent unless `SHIP_PLACEMENT=shadow`. + * Compares each placement with execution-target.ts and records the + * difference; it never influences which host is used. + */ + placementShadow?: PlacementShadow; } /** @@ -121,6 +129,10 @@ export class SandboxPool implements ExecutorProvider { #log: (line: string) => void; #now: () => number; #cooldownMs: number; + #shadow: PlacementShadow | undefined; + /** Shadow bookkeeping only: live runs per project per host, and which project/host each handle is. */ + #projectRuns: Array>; + #handleProject = new Map(); constructor(options: SandboxPoolOptions) { if (options.hosts.length === 0) throw new Error("a sandbox pool needs at least one host"); @@ -131,6 +143,8 @@ export class SandboxPool implements ExecutorProvider { this.#log = options.log ?? (() => {}); this.#now = options.now ?? Date.now; this.#cooldownMs = options.cooldownMs ?? UNHEALTHY_COOLDOWN_MS; + this.#shadow = options.placementShadow; + this.#projectRuns = options.hosts.map(() => ({})); // Every host must isolate, or the pool does not. A run whose task came from // outside refuses to execute on a non-isolating provider, and that check // reads ONE boolean — so the honest answer for a mixed pool is false. @@ -149,8 +163,11 @@ export class SandboxPool implements ExecutorProvider { if (host?.provider.createFrom === undefined) { throw new Error(`this run's snapshot is on host #${index}, which is no longer configured`); } + const shadowDone = this.#shadow?.begin("createFrom", this.#view(), overrides, index); const created = await host.provider.createFrom(inner, overrides); this.#live[index] += 1; + shadowDone?.(index); + this.#track(poolHandle(index, created.handle), index, overrides); return { handle: poolHandle(index, created.handle) }; }; } @@ -231,13 +248,47 @@ export class SandboxPool implements ExecutorProvider { } async create(overrides?: SandboxOverrides): Promise<{ handle: string }> { + // Shadow: decided from the state BEFORE the attempt, recorded after it. + const shadowDone = this.#shadow?.begin("create", this.#view(), overrides); return this.#place("start a sandbox", async (host, index) => { const created = await host.provider.create(overrides); this.#live[index] += 1; + shadowDone?.(index); + this.#track(poolHandle(index, created.handle), index, overrides); return { handle: poolHandle(index, created.handle) }; }); } + /** Shadow only: what the pool believes about each host right now. */ + #view(): PoolView { + const state = this.state(); + return { + hosts: this.#hosts.map((h, i) => ({ url: h.url, isolated: h.provider.isolated === true, live: state[i]!.live, healthy: state[i]!.healthy })), + projectRuns: this.#projectRuns.map((r) => ({ ...r })), + }; + } + + #track(handle: string, index: number, overrides: SandboxOverrides | undefined): void { + if (this.#shadow === undefined) return; + const project = requirementOf(overrides).req.project; + this.#handleProject.set(handle, project); + this.#projectRuns[index]![project] = (this.#projectRuns[index]![project] ?? 0) + 1; + } + + /** + * C5 liveness probe failed for `handle` (durable.ts). Shadow only: record + * what execution-target.ts would have done. Does nothing, and returns, when + * the shadow is off. The caller's failure path is unchanged either way. + */ + observeHostLoss(handle: string): void { + if (this.#shadow === undefined) return; + const { index } = parsePoolHandle(handle); + if (this.#hosts[index] === undefined) return; + const project = this.#handleProject.get(handle); + const req = requirementOf(project !== undefined && project !== "(unknown)" ? { warm: { repo: project } } : undefined, {}); + this.#shadow.hostLoss(this.#view(), index, req); + } + attach(handle: string): AgentExecutor { const { index, handle: inner } = parsePoolHandle(handle); const host = this.#hosts[index]; @@ -260,6 +311,12 @@ export class SandboxPool implements ExecutorProvider { if (host?.provider.destroy === undefined) return; await host.provider.destroy(inner); this.#live[index] = Math.max(0, this.#live[index]! - 1); + const project = this.#handleProject.get(handle); + if (project !== undefined) { + this.#handleProject.delete(handle); + const runs = this.#projectRuns[index]!; + runs[project] = Math.max(0, (runs[project] ?? 1) - 1); + } } /**