From d679f8ca9eb55d5aae5dc28d6e87ab6277fab7c0 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:52:36 +0200 Subject: [PATCH 1/7] feat(test-app): add a transfer the agent may not submit alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aurora Bank had one interrupt in it: the TOTP wall, which an agent cannot pass. That covers the capability gap and nothing else. An approval needs the other kind — a step the agent is perfectly able to take and is not allowed to take alone. /transfer sits behind the session, takes an amount and a payee, and posts back to a signed cookie that /account renders as a receipt. No server state, so a cookie-free context starts clean, which is what the benches need. The /totp and /transfer branches move into their own functions to keep route() under the complexity limit. --- test-app/app.test.ts | 81 +++++++++++++++++++ test-app/guest-source.ts | 164 +++++++++++++++++++++++++++++++-------- test-app/guest/app.js | 164 +++++++++++++++++++++++++++++++-------- 3 files changed, 341 insertions(+), 68 deletions(-) diff --git a/test-app/app.test.ts b/test-app/app.test.ts index 7ff7ced..ff8a5e4 100644 --- a/test-app/app.test.ts +++ b/test-app/app.test.ts @@ -256,6 +256,87 @@ test("POST /logout clears the session", async () => { expect(jar.names()).toEqual([]) }) +/** Sign in the whole way, so the jar holds a session cookie. */ +async function signIn(): Promise { + const jar = await loginToTotpStep() + await request(jar, "/totp", form([["code", totp(SECRET)]])) + expect(jar.names()).toEqual(["hr_session"]) + return jar +} + +test("GET /transfer without a session bounces back to the login page", async () => { + const response = await fetch(`${baseUrl}/transfer`, { redirect: "manual" }) + expect(response.status).toBe(303) + expect(response.headers.get("location")).toBe("/") +}) + +test("GET /transfer serves an amount and a payee behind the session", async () => { + const jar = await signIn() + const response = await request(jar, "/transfer") + const html = await response.text() + expect(response.status).toBe(200) + expect(html).toContain('data-testid="transfer-amount"') + expect(html).toContain('data-testid="transfer-payee"') + expect(html).toContain('data-testid="transfer-submit"') + expect(html).toContain('method="post"') +}) + +test("POST /transfer rejects an amount that is not money", async () => { + const jar = await signIn() + const response = await request( + jar, + "/transfer", + form([ + ["amount", "twelve"], + ["payee", "Acme GmbH"], + ]), + ) + const html = await response.text() + expect(response.status).toBe(400) + expect(html).toContain("Enter an amount") + expect(jar.names()).toEqual(["hr_session"]) +}) + +test("POST /transfer rejects an empty payee", async () => { + const jar = await signIn() + const response = await request( + jar, + "/transfer", + form([ + ["amount", "12430.00"], + ["payee", " "], + ]), + ) + expect(response.status).toBe(400) + expect(await response.text()).toContain("Enter a payee name") +}) + +test("a submitted transfer shows up on the account page", async () => { + const jar = await signIn() + const sent = await request( + jar, + "/transfer", + form([ + ["amount", "12430.00"], + ["payee", "Acme GmbH"], + ]), + ) + expect(sent.status).toBe(303) + expect(sent.headers.get("location")).toBe("/account") + + const account = await request(jar, "/account") + const html = await account.text() + expect(html).toContain('data-testid="transfer-done"') + expect(html).toContain("Sent EUR 12430.00 to Acme GmbH") +}) + +test("an account page with no transfer shows no transfer receipt", async () => { + const jar = await signIn() + const html = await (await request(jar, "/account")).text() + expect(html).not.toContain('data-testid="transfer-done"') + expect(html).toContain('data-testid="transfer-link"') +}) + test("an unknown path is a 404 page", async () => { const response = await fetch(`${baseUrl}/nope`) expect(response.status).toBe(404) diff --git a/test-app/guest-source.ts b/test-app/guest-source.ts index 4679e9a..ab68f54 100644 --- a/test-app/guest-source.ts +++ b/test-app/guest-source.ts @@ -30,6 +30,8 @@ export const GUEST_APP_JS = `/** * GET /totp one input for the 6-digit code * POST /totp RFC 6238 check (SHA-1, 30s step, +/-1 step) -> 303 /account * GET /account "Signed in as " +

+ * GET /transfer one amount + payee form, behind the session + * POST /transfer validates, records the transfer -> 303 /account * POST /logout clears the session -> 303 / * GET /healthz 200 "ok" */ @@ -46,6 +48,8 @@ const BRAND = "Aurora Bank" const COOKIE_KEY = crypto.randomBytes(32) const PENDING_COOKIE = "hr_pending" const SESSION_COOKIE = "hr_session" +/** Carries the last transfer, so /account can show it without server state. */ +const TRANSFER_COOKIE = "hr_transfer" const PENDING_TTL_MS = 5 * 60_000 const SESSION_TTL_MS = 30 * 60_000 const MAX_BODY_BYTES = 8 * 1024 @@ -289,18 +293,60 @@ function totpPage(error) { ) } -function accountPage(username) { +function accountPage(username, transfer) { + const sent = transfer + ? \`

Sent EUR \${escapeHtml( + transfer.amount, + )} to \${escapeHtml(transfer.payee)}

\` + : "" return page( "Account", \`

Two-factor verified

+\${sent}

Signed in as \${escapeHtml(username)}

Your session is active. Nothing here moves real money.

+

Send a transfer

\`, ) } +/** + * The step an agent is not supposed to take alone: it fills the form, and a + * human says yes or no before it presses the button. + */ +function transferPage(error, amount, payee) { + return page( + "Transfer", + \`

Send a transfer

+

The money leaves the account the moment you submit.

+\${errorBox(error)} +
+
+ + +
+
+ + +
+ +
\`, + ) +} + +/** What is wrong with this transfer, or null if nothing is. */ +function transferProblem(amount, payee) { + if (!/^[0-9]{1,9}(\\.[0-9]{1,2})?$/.test(amount) || Number(amount) <= 0) { + return "Enter an amount like 12430.00." + } + if (payee.length === 0 || payee.length > 64) return "Enter a payee name." + return null +} + function notFoundPage() { return page( "Not found", @@ -355,6 +401,69 @@ function redirect(res, location, extraHeaders) { ) } +/** The 2FA step: the pending cookie gets in, the right code gets a session. */ +async function routeTotp(req, res, method, event) { + const pending = unsign(readCookie(req, PENDING_COOKIE)) + event.pending = Boolean(pending) + if (!pending) return redirect(res, "/") + if (method === "GET") { + return send(res, 200, "text/html; charset=utf-8", totpPage(null)) + } + if (method === "POST") { + const form = new URLSearchParams(await readBody(req)) + const code = (form.get("code") || "").trim() + const ok = verifyTotp(TOTP_SECRET, code) + event.user = pending.user + event.code_ok = ok + if (!ok) { + return send( + res, + 401, + "text/html; charset=utf-8", + totpPage("That code is not valid. Try again."), + ) + } + const session = cookieHeader( + req, + SESSION_COOKIE, + sign({ user: pending.user, exp: Date.now() + SESSION_TTL_MS }), + SESSION_TTL_MS / 1000, + ) + const clearPending = \`\${PENDING_COOKIE}=; Path=/; HttpOnly; Max-Age=0\` + return redirect(res, "/account", { + "set-cookie": [session, clearPending], + }) + } + return send(res, 404, "text/html; charset=utf-8", notFoundPage()) +} + +/** GET shows the form; POST validates it and records it in a signed cookie. */ +async function routeTransfer(req, res, method, event) { + if (method === "GET") { + const html = transferPage(null, null, null) + return send(res, 200, "text/html; charset=utf-8", html) + } + if (method !== "POST") { + return send(res, 404, "text/html; charset=utf-8", notFoundPage()) + } + const form = new URLSearchParams(await readBody(req)) + const amount = (form.get("amount") || "").trim() + const payee = (form.get("payee") || "").trim() + const problem = transferProblem(amount, payee) + event.transfer_ok = problem === null + if (problem) { + const html = transferPage(problem, amount, payee) + return send(res, 400, "text/html; charset=utf-8", html) + } + const cookie = cookieHeader( + req, + TRANSFER_COOKIE, + sign({ amount, payee, exp: Date.now() + SESSION_TTL_MS }), + SESSION_TTL_MS / 1000, + ) + return redirect(res, "/account", { "set-cookie": cookie }) +} + async function route(req, res, url, event) { const method = req.method || "GET" @@ -394,37 +503,7 @@ async function route(req, res, url, event) { } if (url.pathname === "/totp") { - const pending = unsign(readCookie(req, PENDING_COOKIE)) - event.pending = Boolean(pending) - if (!pending) return redirect(res, "/") - if (method === "GET") { - return send(res, 200, "text/html; charset=utf-8", totpPage(null)) - } - if (method === "POST") { - const form = new URLSearchParams(await readBody(req)) - const code = (form.get("code") || "").trim() - const ok = verifyTotp(TOTP_SECRET, code) - event.user = pending.user - event.code_ok = ok - if (!ok) { - return send( - res, - 401, - "text/html; charset=utf-8", - totpPage("That code is not valid. Try again."), - ) - } - const session = cookieHeader( - req, - SESSION_COOKIE, - sign({ user: pending.user, exp: Date.now() + SESSION_TTL_MS }), - SESSION_TTL_MS / 1000, - ) - const clearPending = \`\${PENDING_COOKIE}=; Path=/; HttpOnly; Max-Age=0\` - return redirect(res, "/account", { - "set-cookie": [session, clearPending], - }) - } + return routeTotp(req, res, method, event) } if (url.pathname === "/account" && method === "GET") { @@ -432,12 +511,29 @@ async function route(req, res, url, event) { event.signed_in = Boolean(session) if (!session) return redirect(res, "/") event.user = session.user - return send(res, 200, "text/html; charset=utf-8", accountPage(session.user)) + const transfer = unsign(readCookie(req, TRANSFER_COOKIE)) + return send( + res, + 200, + "text/html; charset=utf-8", + accountPage(session.user, transfer), + ) + } + + if (url.pathname === "/transfer") { + const session = unsign(readCookie(req, SESSION_COOKIE)) + event.signed_in = Boolean(session) + if (!session) return redirect(res, "/") + event.user = session.user + return routeTransfer(req, res, method, event) } if (url.pathname === "/logout" && method === "POST") { return redirect(res, "/", { - "set-cookie": \`\${SESSION_COOKIE}=; Path=/; HttpOnly; Max-Age=0\`, + "set-cookie": [ + \`\${SESSION_COOKIE}=; Path=/; HttpOnly; Max-Age=0\`, + \`\${TRANSFER_COOKIE}=; Path=/; HttpOnly; Max-Age=0\`, + ], }) } diff --git a/test-app/guest/app.js b/test-app/guest/app.js index f7129bc..3313275 100644 --- a/test-app/guest/app.js +++ b/test-app/guest/app.js @@ -26,6 +26,8 @@ * GET /totp one input for the 6-digit code * POST /totp RFC 6238 check (SHA-1, 30s step, +/-1 step) -> 303 /account * GET /account "Signed in as " +

+ * GET /transfer one amount + payee form, behind the session + * POST /transfer validates, records the transfer -> 303 /account * POST /logout clears the session -> 303 / * GET /healthz 200 "ok" */ @@ -42,6 +44,8 @@ const BRAND = "Aurora Bank" const COOKIE_KEY = crypto.randomBytes(32) const PENDING_COOKIE = "hr_pending" const SESSION_COOKIE = "hr_session" +/** Carries the last transfer, so /account can show it without server state. */ +const TRANSFER_COOKIE = "hr_transfer" const PENDING_TTL_MS = 5 * 60_000 const SESSION_TTL_MS = 30 * 60_000 const MAX_BODY_BYTES = 8 * 1024 @@ -285,18 +289,60 @@ ${errorBox(error)} ) } -function accountPage(username) { +function accountPage(username, transfer) { + const sent = transfer + ? `

Sent EUR ${escapeHtml( + transfer.amount, + )} to ${escapeHtml(transfer.payee)}

` + : "" return page( "Account", `

Two-factor verified

+${sent}

Signed in as ${escapeHtml(username)}

Your session is active. Nothing here moves real money.

+

Send a transfer

`, ) } +/** + * The step an agent is not supposed to take alone: it fills the form, and a + * human says yes or no before it presses the button. + */ +function transferPage(error, amount, payee) { + return page( + "Transfer", + `

Send a transfer

+

The money leaves the account the moment you submit.

+${errorBox(error)} +
+
+ + +
+
+ + +
+ +
`, + ) +} + +/** What is wrong with this transfer, or null if nothing is. */ +function transferProblem(amount, payee) { + if (!/^[0-9]{1,9}(\.[0-9]{1,2})?$/.test(amount) || Number(amount) <= 0) { + return "Enter an amount like 12430.00." + } + if (payee.length === 0 || payee.length > 64) return "Enter a payee name." + return null +} + function notFoundPage() { return page( "Not found", @@ -351,6 +397,69 @@ function redirect(res, location, extraHeaders) { ) } +/** The 2FA step: the pending cookie gets in, the right code gets a session. */ +async function routeTotp(req, res, method, event) { + const pending = unsign(readCookie(req, PENDING_COOKIE)) + event.pending = Boolean(pending) + if (!pending) return redirect(res, "/") + if (method === "GET") { + return send(res, 200, "text/html; charset=utf-8", totpPage(null)) + } + if (method === "POST") { + const form = new URLSearchParams(await readBody(req)) + const code = (form.get("code") || "").trim() + const ok = verifyTotp(TOTP_SECRET, code) + event.user = pending.user + event.code_ok = ok + if (!ok) { + return send( + res, + 401, + "text/html; charset=utf-8", + totpPage("That code is not valid. Try again."), + ) + } + const session = cookieHeader( + req, + SESSION_COOKIE, + sign({ user: pending.user, exp: Date.now() + SESSION_TTL_MS }), + SESSION_TTL_MS / 1000, + ) + const clearPending = `${PENDING_COOKIE}=; Path=/; HttpOnly; Max-Age=0` + return redirect(res, "/account", { + "set-cookie": [session, clearPending], + }) + } + return send(res, 404, "text/html; charset=utf-8", notFoundPage()) +} + +/** GET shows the form; POST validates it and records it in a signed cookie. */ +async function routeTransfer(req, res, method, event) { + if (method === "GET") { + const html = transferPage(null, null, null) + return send(res, 200, "text/html; charset=utf-8", html) + } + if (method !== "POST") { + return send(res, 404, "text/html; charset=utf-8", notFoundPage()) + } + const form = new URLSearchParams(await readBody(req)) + const amount = (form.get("amount") || "").trim() + const payee = (form.get("payee") || "").trim() + const problem = transferProblem(amount, payee) + event.transfer_ok = problem === null + if (problem) { + const html = transferPage(problem, amount, payee) + return send(res, 400, "text/html; charset=utf-8", html) + } + const cookie = cookieHeader( + req, + TRANSFER_COOKIE, + sign({ amount, payee, exp: Date.now() + SESSION_TTL_MS }), + SESSION_TTL_MS / 1000, + ) + return redirect(res, "/account", { "set-cookie": cookie }) +} + async function route(req, res, url, event) { const method = req.method || "GET" @@ -390,37 +499,7 @@ async function route(req, res, url, event) { } if (url.pathname === "/totp") { - const pending = unsign(readCookie(req, PENDING_COOKIE)) - event.pending = Boolean(pending) - if (!pending) return redirect(res, "/") - if (method === "GET") { - return send(res, 200, "text/html; charset=utf-8", totpPage(null)) - } - if (method === "POST") { - const form = new URLSearchParams(await readBody(req)) - const code = (form.get("code") || "").trim() - const ok = verifyTotp(TOTP_SECRET, code) - event.user = pending.user - event.code_ok = ok - if (!ok) { - return send( - res, - 401, - "text/html; charset=utf-8", - totpPage("That code is not valid. Try again."), - ) - } - const session = cookieHeader( - req, - SESSION_COOKIE, - sign({ user: pending.user, exp: Date.now() + SESSION_TTL_MS }), - SESSION_TTL_MS / 1000, - ) - const clearPending = `${PENDING_COOKIE}=; Path=/; HttpOnly; Max-Age=0` - return redirect(res, "/account", { - "set-cookie": [session, clearPending], - }) - } + return routeTotp(req, res, method, event) } if (url.pathname === "/account" && method === "GET") { @@ -428,12 +507,29 @@ async function route(req, res, url, event) { event.signed_in = Boolean(session) if (!session) return redirect(res, "/") event.user = session.user - return send(res, 200, "text/html; charset=utf-8", accountPage(session.user)) + const transfer = unsign(readCookie(req, TRANSFER_COOKIE)) + return send( + res, + 200, + "text/html; charset=utf-8", + accountPage(session.user, transfer), + ) + } + + if (url.pathname === "/transfer") { + const session = unsign(readCookie(req, SESSION_COOKIE)) + event.signed_in = Boolean(session) + if (!session) return redirect(res, "/") + event.user = session.user + return routeTransfer(req, res, method, event) } if (url.pathname === "/logout" && method === "POST") { return redirect(res, "/", { - "set-cookie": `${SESSION_COOKIE}=; Path=/; HttpOnly; Max-Age=0`, + "set-cookie": [ + `${SESSION_COOKIE}=; Path=/; HttpOnly; Max-Age=0`, + `${TRANSFER_COOKIE}=; Path=/; HttpOnly; Max-Age=0`, + ], }) } From e0658e2e836ae685ae5b69b1dd447910cb1af654 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:52:43 +0200 Subject: [PATCH 2/7] feat(bench): measure a mixed workload of takeovers and approvals bench.ts times one handoff and rescue-bench.ts counts workflows a takeover saves. Neither answers what a team asks before adopting this: an agent meets two kinds of interrupt, and they do not cost the same. mixed-bench.ts runs N workflows against one live Aurora Bank instance, interleaved takeover, approval, takeover, and reports each mode's completion, time to visible on the human's socket, handoff duration, frames, bytes, inputs applied and relay-sandbox-seconds. Every fourth approval is denied, and a denied approval counts as completed: the decision was delivered and the bench asserts no money moved. The scripted human gains firstFrameAt() so time-to-visible is the frame's arrival time and not the caller's wake-up time. --- e2e/human-sim.ts | 9 + e2e/mixed-bench.ts | 837 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 847 insertions(+) create mode 100644 e2e/mixed-bench.ts diff --git a/e2e/human-sim.ts b/e2e/human-sim.ts index f8329ae..1c63ac1 100644 --- a/e2e/human-sim.ts +++ b/e2e/human-sim.ts @@ -29,6 +29,12 @@ export interface ReceivedFrame { export interface SimulatedHuman { /** The newest frame, or `null` before the first one. */ lastFrame(): ReceivedFrame | null + /** + * When the first frame landed here, in ms since the epoch, or `null` if none + * has. Taken in the socket's message handler, so a caller that asks for it + * after the fact still gets the arrival time and not its own wake-up time. + */ + firstFrameAt(): number | null frameCount(): number /** The `reason` currently shown in the header. */ reason(): string @@ -90,6 +96,7 @@ export async function openHandoffPage( const socket = new WebSocket(humanWebSocketUrl(humanUrl)) let frame: ReceivedFrame | null = null + let firstFrameAt: number | null = null let frames = 0 let reason = "" let action = "" @@ -101,6 +108,7 @@ export async function openHandoffPage( if (!message) return if (message.type === "frame") { frame = { data: message.data, meta: message.meta } + firstFrameAt = firstFrameAt ?? Date.now() frames += 1 while (waiters.length > 0) waiters.pop()?.() return @@ -133,6 +141,7 @@ export async function openHandoffPage( return { lastFrame: () => frame, + firstFrameAt: () => firstFrameAt, frameCount: () => frames, reason: () => reason, action: () => action, diff --git a/e2e/mixed-bench.ts b/e2e/mixed-bench.ts new file mode 100644 index 0000000..818b88c --- /dev/null +++ b/e2e/mixed-bench.ts @@ -0,0 +1,837 @@ +/** + * The mixed-workload bench: across a realistic mix of human interrupts, what + * does each mode cost, and how many workflows get done? + * + * bun --env-file=.env e2e/mixed-bench.ts # N=20 + * MIXED_N=4 bun --env-file=.env e2e/mixed-bench.ts + * + * e2e/bench.ts times one handoff. e2e/rescue-bench.ts counts workflows that a + * takeover saves. Neither answers the question a team asks before adopting + * this: an agent fleet meets two different interrupts, and they do not cost the + * same. One kind needs the browser driven; the other needs one decision. + * + * The two interrupts, both against ONE live Aurora Bank instance: + * + * takeover The agent signs in with the credentials it has and stops at a + * real RFC 6238 TOTP wall it cannot pass. A scripted human on the + * public WebSocket taps the field, types the code, presses Enter + * and hands back. This is the capability gap. + * + * approval The agent is signed in and not stuck at all. It fills a + * transfer form and stops before submitting, because moving money + * is not its decision. A scripted human sees one screenshot and + * the action in words, and answers. This is the authority + * boundary. + * + * Both interrupts are measured the same way, on the human's socket, so the two + * columns of the output are comparable. + * + * What may be claimed from the output, and nothing wider: on this workload, + * an approval costs one frame and a few kilobytes where a takeover costs a + * stream, and both got their workflows done at rate X. It is NOT a claim about + * the mix a real fleet sees — the 50/50 split here is a choice this harness + * makes, not a measurement of anyone's traffic. Multiply the per-mode costs by + * your own mix. + * + * Design notes that are load-bearing: + * + * - The kinds are interleaved (takeover, approval, takeover, …) so neither sees + * systematically older browsers or a different network minute than the other. + * - Deterministic scripts, not an LLM. The claim is about the mechanism. + * - Every 4th approval is DENIED, and a denied approval counts as completed: + * the workflow reached a decision and the agent obeyed it. A bench that only + * counted "yes" would be measuring agreement, not delivery. + * - The approval arm signs itself in with the shared secret. That sign-in is + * setup, not the interrupt under measurement — the interrupt here is the + * authority boundary at the transfer, and it happens after the wall. Only the + * takeover arm is barred from the secret, because there the wall IS the test. + * - ONE test-app sandbox is held for the whole bench, and each handoff takes + * the second sandbox slot the plan allows, so nothing else may run alongside. + * Check with `bun --env-file=.env scripts/cleanup-sandboxes.ts` before and + * after. Two handoffs never run at once. + * - ONE browser session is reused and relaunched after 3 minutes, because + * Solari browser sessions die hard around 10 minutes and the sessions API + * still calls the corpse "active" + * (docs/measurements/04-browser-session-lifetime.md). + * - Every workflow gets a fresh page and a cookie-free context, so no run + * inherits the session or the transfer receipt an earlier run left behind. + * - A failed run is recorded as a failed run. Percentiles are over the runs + * that completed, with the failures shown next to them. + * + * Set MIXED_FAULT=invert-completed to invert the completion test. Both modes + * must then read 0 of N — that is how the counting is known to be load-bearing + * rather than decorative. + */ +import { Solari } from "@solarisdk/browser" +import type { Page } from "playwright-core" + +import type { HandoffEvent } from "../src/events" +import { raiseHand } from "../src/index" +import { startTestApp } from "../test-app/deploy" +import { totp } from "../test-app/totp" +import { openHandoffPage, type SimulatedHuman } from "./human-sim" + +/** Total workflows, split evenly between the two interrupt kinds. */ +const N = Number(process.env.MIXED_N ?? "20") +const FAULT = process.env.MIXED_FAULT ?? "" +const VIEWPORT = { width: 1280, height: 800 } +/** Every 4th approval is denied, so the denied path is measured, not assumed. */ +const DENY_EVERY = 4 +/** Sandbox idle budget for the test app. Comfortably longer than N runs. */ +const APP_TIMEOUT_MS = 45 * 60_000 +/** Per-handoff wait budget. A run that hits it is a failure, not a datum. */ +const HANDOFF_TIMEOUT_MS = 90_000 +/** How long the scripted human waits for its first frame. */ +const FRAME_TIMEOUT_MS = 30_000 +/** How long any "did the page catch up yet" poll may run. */ +const SETTLE_TIMEOUT_MS = 20_000 +/** Poll interval for those waits. */ +const POLL_MS = 200 +/** Relaunch the browser at this age — comfortably under the ~10 min hard death. */ +const BROWSER_MAX_AGE_MS = 3 * 60_000 +/** Breather between runs so a just-killed relay sandbox is off the books. */ +const COOLDOWN_MS = 1_000 +/** Consecutive failures that mean the infrastructure is down, not the claim. */ +const ABORT_AFTER_CONSECUTIVE_FAILURES = 5 +const RESULTS_PATH = new URL( + "../benchmarks/mixed-workload.json", + import.meta.url, +) + +const apiKey = process.env.SOLARI_API_KEY ?? "" +if (apiKey === "") { + throw new Error("SOLARI_API_KEY missing — run with --env-file=.env") +} + +type Kind = "takeover" | "approval" +type Decision = "approve" | "deny" + +interface MixedRun { + index: number + kind: Kind + startedAt: string + /** The whole point: did this workflow reach its end state? */ + completed: boolean + /** False means the run never got to the interrupt — infrastructure, not the claim. */ + reachedInterrupt: boolean + /** Page load → standing at the interrupt, in ms. */ + reachedInterruptMs: number | null + /** What the scripted human answered. Approval runs only. */ + decision: Decision | null + handoffOutcome: string | null + /** `raiseHand()` → the first frame arriving AT THE HUMAN, in ms. */ + stuckToVisibleMs: number | null + /** `durationMs` from the handoff's own wide event: raiseHand → settled. */ + handoffDurationMs: number | null + relayColdStartMs: number | null + /** Frames the agent put on the wire (`framesSent` from the wide event). */ + framesSent: number | null + /** The same frames counted on the human's socket, as a cross-check. */ + framesAtHuman: number | null + /** Sum of the base64 frame payloads, in bytes. */ + bytesSent: number | null + /** Taps, characters, keys and scrolls applied to the page. */ + inputsApplied: number | null + /** First frame at the human → the answer being sent. A person's occupancy. */ + humanActiveMs: number | null + /** + * Wall-clock seconds a relay sandbox existed for this handoff: `raiseHand()` + * to the promise settling, which is create + live + destroy. The cost proxy. + */ + relaySandboxSeconds: number | null + error: string | null + /** Age of the reused browser session when the run started, in ms. */ + browserAgeMs: number +} + +// --- the browser lease ----------------------------------------------------- + +type SolariBrowser = Awaited> +type SolariContext = Awaited> + +interface BrowserLease { + solari: Solari + browser: SolariBrowser + context: SolariContext + launchedAt: number +} + +async function leaseBrowser(): Promise { + const solari = new Solari({ apiKey }) + const launchedAt = Date.now() + const browser = await solari.launch({ stealth: true }) + const context = browser.contexts()[0] ?? (await browser.newContext()) + // Keep the context's own first page open as an anchor, so closing a run's + // page never leaves the browser with nothing to hold on to. + if (context.pages().length === 0) await context.newPage() + return { solari, browser, context, launchedAt } +} + +async function releaseBrowser(lease: BrowserLease): Promise { + await lease.browser.close().catch(() => undefined) + // Without this the SDK's transport keeps the process alive after the bench. + await lease.solari.close().catch(() => undefined) +} + +function leaseIsStale(lease: BrowserLease): boolean { + return ( + Date.now() - lease.launchedAt > BROWSER_MAX_AGE_MS || + !lease.browser.isConnected() + ) +} + +/** A page with no cookies, so no run inherits an earlier run's session. */ +async function freshPage(lease: BrowserLease): Promise { + await lease.context.clearCookies() + const opened = await lease.context.newPage() + await opened.setViewportSize(VIEWPORT) + // SAFETY: `@solarisdk/browser` returns patchright-core's Page. patchright is + // a Playwright fork whose runtime surface is the one handraise uses, and the + // e2e drives exactly this object; the two declarations differ only in + // optional-property variance. + return opened as Page +} + +// --- the workflow ---------------------------------------------------------- + +interface Target { + url: string + user: string + pass: string + totpSecret: string +} + +/** Poll until `check` is true or the budget runs out. Returns whether it was. */ +async function waitUntil( + check: () => Promise, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs + for (;;) { + if (await check().catch(() => false)) return true + if (Date.now() >= deadline) return false + await Bun.sleep(POLL_MS) + } +} + +/** + * The sign-in step's definition of done: the account page, rendered, for our + * user. Not "the URL changed" — the URL changes on a 303 before the page exists. + */ +async function reachedAccount( + page: Page, + user: string, + timeoutMs: number, +): Promise { + const arrived = await waitUntil( + async () => page.url().endsWith("/account"), + timeoutMs, + ) + if (!arrived) return false + const banner = await page + .textContent('[data-testid="signed-in"]', { timeout: timeoutMs }) + .catch(() => null) + return banner?.includes(user) === true +} + +/** Drive the agent's part: log in with the credentials it has, hit the wall. */ +async function walkToWall(page: Page, app: Target): Promise { + await page.goto(app.url, { waitUntil: "domcontentloaded", timeout: 45_000 }) + await page.fill('[data-testid="username"]', app.user) + await page.fill('[data-testid="password"]', app.pass) + await page.click('[data-testid="login-submit"]') + await page.waitForSelector('[data-testid="totp-code"]', { timeout: 30_000 }) +} + +/** Is a receipt for a sent transfer on the page right now? */ +async function transferWasSent( + page: Page, + timeoutMs: number, +): Promise { + return waitUntil( + async () => + (await page.locator('[data-testid="transfer-done"]').count()) > 0, + timeoutMs, + ) +} + +// --- the takeover interrupt ------------------------------------------------ + +/** Tap the code field, type the current code, press Enter, hand back. */ +async function humanSolvesTheWall( + page: Page, + human: SimulatedHuman, + secret: string, +): Promise { + const first = await human.waitForFrame(FRAME_TIMEOUT_MS) + + // Where a person would put their thumb, in frame pixels. + const box = await page.locator('[data-testid="totp-code"]').boundingBox() + if (!box) throw new Error("the code field has no bounding box") + const scale = first.meta.jpegWidth / first.meta.deviceWidth + await human.tap( + (box.x + box.width / 2) * scale, + (box.y + box.height / 2) * scale, + ) + await waitUntil( + async () => + (await page.evaluate( + () => document.activeElement?.getAttribute("data-testid") ?? "", + )) === "totp-code", + SETTLE_TIMEOUT_MS, + ) + + // Computed at typing time, not at handoff time. The app tolerates one + // 30-second step of drift either side, which covers the flight time. + const code = totp(secret) + await human.type(code) + const landed = await waitUntil( + async () => (await page.inputValue('[data-testid="totp-code"]')) === code, + SETTLE_TIMEOUT_MS, + ) + if (!landed) throw new Error("the code never fully arrived in the field") + + await human.press("Enter") + await waitUntil( + async () => page.url().endsWith("/account"), + SETTLE_TIMEOUT_MS, + ) + await human.handback() +} + +// --- the approval interrupt ------------------------------------------------ + +/** Sign in the way the agent would if it had been given the secret. */ +async function signInWithTheSecret(page: Page, app: Target): Promise { + await walkToWall(page, app) + await page.fill('[data-testid="totp-code"]', totp(app.totpSecret)) + await page.click('[data-testid="totp-submit"]') + const arrived = await reachedAccount(page, app.user, SETTLE_TIMEOUT_MS) + if (!arrived) throw new Error("the approval arm never got signed in") +} + +/** Open the transfer form and fill it, stopping short of the submit button. */ +async function fillTransfer(page: Page, amount: string, payee: string) { + await page.click('[data-testid="transfer-link"]') + await page.waitForSelector('[data-testid="transfer-submit"]', { + timeout: 30_000, + }) + await page.fill('[data-testid="transfer-amount"]', amount) + await page.fill('[data-testid="transfer-payee"]', payee) +} + +// --- one handoff ----------------------------------------------------------- + +interface HandoffRecord { + outcome: string | null + event: HandoffEvent | null + stuckToVisibleMs: number | null + humanActiveMs: number | null + framesAtHuman: number | null + relaySandboxSeconds: number | null + error: string | null +} + +/** + * Raise the hand, run `script` as the human, and always let the handoff settle. + * + * Settling is what destroys the relay sandbox and frees the single slot the + * next run needs, so it happens on the error path too. + */ +async function withHandoff( + page: Page, + options: Parameters[1], + script: (human: SimulatedHuman, record: HandoffRecord) => Promise, +): Promise { + const record: HandoffRecord = { + outcome: null, + event: null, + stuckToVisibleMs: null, + humanActiveMs: null, + framesAtHuman: null, + relaySandboxSeconds: null, + error: null, + } + + let announce: (url: string) => void = () => undefined + let refuse: (error: Error) => void = () => undefined + const urlReady = new Promise((resolve, reject) => { + announce = resolve + refuse = reject + }) + + const raisedAt = Date.now() + const handoff = raiseHand(page, { + ...options, + qr: false, + timeoutMs: HANDOFF_TIMEOUT_MS, + onUrl: (url) => announce(url), + onEvent: (event) => { + record.event = event + }, + }) + // `raiseHand` throws only when the relay never came up. That rejection has to + // reach `urlReady` too, or this run would hang on a promise nobody resolves. + handoff.catch((error: Error) => refuse(error)) + + let human: SimulatedHuman | null = null + try { + human = await openHandoffPage(await urlReady) + await human.waitForFrame(FRAME_TIMEOUT_MS) + const firstFrameAt = human.firstFrameAt() ?? Date.now() + record.stuckToVisibleMs = firstFrameAt - raisedAt + await script(human, record) + record.humanActiveMs = record.humanActiveMs ?? Date.now() - firstFrameAt + } catch (error) { + record.error = error instanceof Error ? error.message : String(error) + // The script gave up, so say so on the wire instead of holding the relay + // sandbox for the rest of the timeout. A takeover settles as `aborted`; an + // approval ignores the message by design and still runs out the clock. + await human?.abort().catch(() => undefined) + } + + const result = await handoff.catch((error: Error) => { + record.error = record.error ?? error.message + return null + }) + record.relaySandboxSeconds = (Date.now() - raisedAt) / 1000 + record.outcome = result?.outcome ?? "throw" + record.framesAtHuman = human?.frameCount() ?? null + await human?.close().catch(() => undefined) + return record +} + +// --- one run --------------------------------------------------------------- + +/** Deterministic, so a rerun asks for the same money as the run before it. */ +function transferFor(index: number) { + return { amount: `${12_000 + index}.00`, payee: "Acme GmbH" } +} + +async function runTakeover( + page: Page, + app: Target, + run: MixedRun, +): Promise { + const walkAt = Date.now() + await walkToWall(page, app) + run.reachedInterrupt = true + run.reachedInterruptMs = Date.now() - walkAt + + const handoff = await withHandoff( + page, + { reason: "Aurora Bank is asking for a 2FA code" }, + (human) => humanSolvesTheWall(page, human, app.totpSecret), + ) + applyHandoff(run, handoff) + + const arrived = await reachedAccount(page, app.user, SETTLE_TIMEOUT_MS) + run.completed = arrived && handoff.outcome === "resolved" +} + +async function runApproval( + page: Page, + app: Target, + run: MixedRun, + decision: Decision, +): Promise { + const walkAt = Date.now() + await signInWithTheSecret(page, app) + const { amount, payee } = transferFor(run.index) + await fillTransfer(page, amount, payee) + run.reachedInterrupt = true + run.reachedInterruptMs = Date.now() - walkAt + run.decision = decision + + const action = `Transfer EUR ${amount} to ${payee}` + const handoff = await withHandoff( + page, + { + mode: "approval", + reason: "The agent may not move money without a human", + action, + }, + async (human, record) => { + const firstFrameAt = human.firstFrameAt() ?? Date.now() + // If the phone were shown a different step than the one the agent is + // about to take, the whole mode would be a lie. Cheap to check here. + if (human.action() !== action) { + throw new Error(`the phone showed "${human.action()}", not the action`) + } + if (decision === "approve") await human.approve() + else await human.deny() + record.humanActiveMs = Date.now() - firstFrameAt + }, + ) + applyHandoff(run, handoff) + + // The agent, not the human, carries the decision out: an approval injects + // nothing into the page, so the transfer only happens if the agent submits. + if (handoff.outcome === "approved") { + await page.click('[data-testid="transfer-submit"]') + run.completed = await transferWasSent(page, SETTLE_TIMEOUT_MS) + } else if (handoff.outcome === "denied") { + // A denied approval is a completed workflow: the decision was delivered and + // obeyed. What must be true is that no money moved. + run.completed = !(await transferWasSent(page, 1_000)) + } +} + +function applyHandoff(run: MixedRun, handoff: HandoffRecord): void { + run.handoffOutcome = handoff.outcome + run.stuckToVisibleMs = handoff.stuckToVisibleMs + run.handoffDurationMs = handoff.event?.durationMs ?? null + run.relayColdStartMs = handoff.event?.relayColdStartMs ?? null + run.framesSent = handoff.event?.framesSent ?? null + run.framesAtHuman = handoff.framesAtHuman + run.bytesSent = handoff.event?.bytesSent ?? null + run.inputsApplied = handoff.event?.inputsApplied ?? null + run.humanActiveMs = handoff.humanActiveMs + run.relaySandboxSeconds = handoff.relaySandboxSeconds + run.error = handoff.error +} + +async function runWorkflow( + index: number, + kind: Kind, + decision: Decision, + lease: BrowserLease, + app: Target, +): Promise { + const run: MixedRun = { + index, + kind, + startedAt: new Date().toISOString(), + completed: false, + reachedInterrupt: false, + reachedInterruptMs: null, + decision: null, + handoffOutcome: null, + stuckToVisibleMs: null, + handoffDurationMs: null, + relayColdStartMs: null, + framesSent: null, + framesAtHuman: null, + bytesSent: null, + inputsApplied: null, + humanActiveMs: null, + relaySandboxSeconds: null, + error: null, + browserAgeMs: Date.now() - lease.launchedAt, + } + + const page = await freshPage(lease) + try { + if (kind === "takeover") await runTakeover(page, app, run) + else await runApproval(page, app, run, decision) + } catch (error) { + run.error = + run.error ?? (error instanceof Error ? error.message : String(error)) + } finally { + await page.close().catch(() => undefined) + } + + if (FAULT === "invert-completed") run.completed = !run.completed + return run +} + +// --- statistics ------------------------------------------------------------ + +/** Nearest-rank percentile: sort, then take index ceil(p * n) - 1. */ +function percentile(sorted: number[], p: number): number { + const rank = Math.ceil(p * sorted.length) - 1 + const index = Math.min(sorted.length - 1, Math.max(0, rank)) + return sorted[index] ?? Number.NaN +} + +interface Stat { + n: number + p50: number + p75: number + worst: number + total: number +} + +function stat(values: number[]): Stat | null { + if (values.length === 0) return null + const sorted = [...values].sort((a, b) => a - b) + return { + n: sorted.length, + p50: percentile(sorted, 0.5), + p75: percentile(sorted, 0.75), + worst: percentile(sorted, 1), + total: Number(values.reduce((sum, value) => sum + value, 0).toFixed(3)), + } +} + +interface ModeSummary { + mode: Kind + label: string + attempted: number + completed: number + /** Approval mode only: how the answers split. Denials count as completed. */ + approved: number | null + denied: number | null + /** Runs that never reached the interrupt: infrastructure, not the claim. */ + neverReachedInterrupt: number + stuckToVisibleMs: Stat | null + handoffDurationMs: Stat | null + relayColdStartMs: Stat | null + humanActiveMs: Stat | null + framesSent: Stat | null + bytesSent: Stat | null + inputsApplied: Stat | null + relaySandboxSeconds: Stat | null +} + +function isNumber(value: number | null | undefined): value is number { + return value !== null && value !== undefined && Number.isFinite(value) +} + +function summarise(mode: Kind, label: string, runs: MixedRun[]): ModeSummary { + const mine = runs.filter((run) => run.kind === mode) + const done = mine.filter((run) => run.completed) + const of = (pick: (run: MixedRun) => number | null): Stat | null => + stat(done.map(pick).filter(isNumber)) + return { + mode, + label, + attempted: mine.length, + completed: done.length, + approved: + mode === "approval" + ? mine.filter((run) => run.decision === "approve").length + : null, + denied: + mode === "approval" + ? mine.filter((run) => run.decision === "deny").length + : null, + neverReachedInterrupt: mine.filter((run) => !run.reachedInterrupt).length, + stuckToVisibleMs: of((run) => run.stuckToVisibleMs), + handoffDurationMs: of((run) => run.handoffDurationMs), + relayColdStartMs: of((run) => run.relayColdStartMs), + humanActiveMs: of((run) => run.humanActiveMs), + framesSent: of((run) => run.framesSent), + bytesSent: of((run) => run.bytesSent), + inputsApplied: of((run) => run.inputsApplied), + relaySandboxSeconds: of((run) => run.relaySandboxSeconds), + } +} + +function ms(value: number | undefined): string { + return value === undefined ? "—" : `${Math.round(value)}` +} + +function kb(value: number | undefined): string { + return value === undefined ? "—" : `${(value / 1024).toFixed(1)}` +} + +function seconds(value: number | undefined): string { + return value === undefined ? "—" : value.toFixed(1) +} + +function printTable(summaries: ModeSummary[]): void { + const header = [ + "", + "completed", + "visible p50", + "handoff p50", + "frames p50", + "KB p50", + "inputs p50", + "relay s p50", + ] + const rows = summaries.map((summary) => [ + summary.label, + `${summary.completed}/${summary.attempted}`, + ms(summary.stuckToVisibleMs?.p50), + ms(summary.handoffDurationMs?.p50), + ms(summary.framesSent?.p50), + kb(summary.bytesSent?.p50), + ms(summary.inputsApplied?.p50), + seconds(summary.relaySandboxSeconds?.p50), + ]) + const widths = header.map((name, column) => + Math.max(name.length, ...rows.map((row) => (row[column] ?? "").length)), + ) + const line = (cells: string[]): string => + `| ${cells + .map((text, column) => + column === 0 + ? text.padEnd(widths[column] ?? 0) + : text.padStart(widths[column] ?? 0), + ) + .join(" | ")} |` + + console.log("") + console.log(line(header)) + console.log(`|${widths.map((width) => "-".repeat(width + 2)).join("|")}|`) + for (const row of rows) console.log(line(row)) +} + +// --- the bench ------------------------------------------------------------- + +const benchStartedAt = Date.now() +const runs: MixedRun[] = [] +let relaunches = 0 +let consecutiveFailures = 0 +let abortReason: string | null = null + +const app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) +console.log( + JSON.stringify({ + event: "test_app_ready", + url: app.url, + sandbox: app.sandboxId, + }), +) + +let lease = await leaseBrowser() + +try { + let approvals = 0 + for (let index = 1; index <= N && abortReason === null; index += 1) { + const kind: Kind = index % 2 === 1 ? "takeover" : "approval" + if (kind === "approval") approvals += 1 + const decision: Decision = + kind === "approval" && approvals % DENY_EVERY === 0 ? "deny" : "approve" + + if (leaseIsStale(lease)) { + await releaseBrowser(lease) + lease = await leaseBrowser() + relaunches += 1 + console.log( + JSON.stringify({ event: "browser_relaunched", before: index }), + ) + } + + const run = await runWorkflow(index, kind, decision, lease, app) + runs.push(run) + console.log( + JSON.stringify({ + event: "mixed_run", + index, + kind, + decision: run.decision, + completed: run.completed, + reachedInterrupt: run.reachedInterrupt, + handoffOutcome: run.handoffOutcome, + stuckToVisibleMs: run.stuckToVisibleMs, + handoffDurationMs: run.handoffDurationMs, + framesSent: run.framesSent, + bytesSent: run.bytesSent, + inputsApplied: run.inputsApplied, + relaySandboxSeconds: run.relaySandboxSeconds, + error: run.error, + }), + ) + + // A dead session poisons every later run, so replace it now rather than + // after N-1 more failures. + if (run.handoffOutcome === "disconnected" || !lease.browser.isConnected()) { + await releaseBrowser(lease) + lease = await leaseBrowser() + relaunches += 1 + console.log(JSON.stringify({ event: "browser_relaunched", after: index })) + } + + consecutiveFailures = run.completed ? 0 : consecutiveFailures + 1 + if (consecutiveFailures >= ABORT_AFTER_CONSECUTIVE_FAILURES) { + abortReason = `${ABORT_AFTER_CONSECUTIVE_FAILURES} workflows failed in a row — the infrastructure is down, not the claim` + console.log( + JSON.stringify({ event: "mixed_aborted", reason: abortReason }), + ) + break + } + + await Bun.sleep(COOLDOWN_MS) + } +} finally { + await releaseBrowser(lease) + await app.kill().catch(() => undefined) +} + +const summaries = [ + summarise("takeover", "takeover — the human drives", runs), + summarise("approval", "approval — the human decides", runs), +] +const failures = runs.filter((run) => !run.completed) +/** Every run, failures included — the per-mode stats below count only the + * runs that completed, so these two totals are deliberately different. */ +const relaySeconds = stat( + runs.map((run) => run.relaySandboxSeconds).filter(isNumber), +) + +await Bun.write( + RESULTS_PATH, + `${JSON.stringify( + { + meta: { + date: new Date().toISOString(), + requestedN: N, + workload: + "N workflows against one Aurora Bank instance, interleaved takeover, approval, takeover, …", + interrupts: { + takeover: + "a real RFC 6238 TOTP wall the agent cannot pass; a scripted human types the code and hands back", + approval: + "a filled transfer form the agent may not submit alone; a scripted human answers yes or no", + }, + claim: + "on this workload, each mode's cost per handoff and how many workflows completed", + notAClaim: + "the 50/50 mix is this harness's choice, not a measurement of any fleet's traffic; multiply the per-mode costs by your own mix", + deniedCountAsCompleted: + "yes — a denied approval delivered a decision the agent obeyed, and the bench asserts no transfer was sent", + approvalSetup: + "the approval arm signs itself in with the shared secret; that sign-in is setup, and the interrupt under measurement is the transfer", + denyEvery: DENY_EVERY, + interleaved: true, + fault: FAULT === "" ? null : FAULT, + abortReason, + browserRelaunches: relaunches, + totalMs: Date.now() - benchStartedAt, + relaySandboxSecondsAllRuns: relaySeconds?.total ?? null, + bunVersion: Bun.version, + platform: `${process.platform}-${process.arch}`, + measuredFrom: "Germany → default Solari endpoint (api.getsolari.com)", + handoffTimeoutMs: HANDOFF_TIMEOUT_MS, + viewport: VIEWPORT, + }, + summaries, + runs, + }, + null, + 2, + )}\n`, +) + +printTable(summaries) +console.log("") +console.log( + JSON.stringify({ + event: "mixed_done", + workflows: runs.length, + completed: runs.filter((run) => run.completed).length, + takeoverCompleted: `${summaries[0]?.completed}/${summaries[0]?.attempted}`, + approvalCompleted: `${summaries[1]?.completed}/${summaries[1]?.attempted}`, + relaySandboxSecondsAllRuns: relaySeconds?.total ?? null, + browserRelaunches: relaunches, + totalMs: Date.now() - benchStartedAt, + results: RESULTS_PATH.pathname, + }), +) +for (const failure of failures) { + console.log( + JSON.stringify({ + event: "mixed_failure", + index: failure.index, + kind: failure.kind, + decision: failure.decision, + reachedInterrupt: failure.reachedInterrupt, + handoffOutcome: failure.handoffOutcome, + error: failure.error, + browserAgeMs: failure.browserAgeMs, + }), + ) +} diff --git a/package.json b/package.json index 29d2db7..28e85f4 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "test:e2e": "bun --env-file=.env e2e/handoff.e2e.ts", "bench": "bun --env-file=.env e2e/bench.ts", "bench:rescue": "bun --env-file=.env e2e/rescue-bench.ts", + "bench:mixed": "bun --env-file=.env e2e/mixed-bench.ts", "prepublishOnly": "bun run lint && bun run typecheck && bun run test && bun run build" }, "dependencies": { From db289d684faf48164af8da2eb805bca36d1ab064 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:08:17 +0200 Subject: [PATCH 3/7] fix(bench): count relay-sandbox seconds only when a relay existed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A handoff that never got a sandbox — the plan was full and startRelay threw — burned wall clock but no sandbox, and charging it for one would inflate the cost proxy exactly when the platform was busiest. The wide event is the tell: no event, no relay, no seconds. Empty cells print as an em dash instead of NaN. --- e2e/mixed-bench.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/mixed-bench.ts b/e2e/mixed-bench.ts index 818b88c..bba35af 100644 --- a/e2e/mixed-bench.ts +++ b/e2e/mixed-bench.ts @@ -394,7 +394,10 @@ async function withHandoff( record.error = record.error ?? error.message return null }) - record.relaySandboxSeconds = (Date.now() - raisedAt) / 1000 + // Only when a relay actually came up: a handoff that never got one (the plan + // was full) burned wall clock, but it did not burn a sandbox. + record.relaySandboxSeconds = + record.event === null ? null : (Date.now() - raisedAt) / 1000 record.outcome = result?.outcome ?? "throw" record.framesAtHuman = human?.frameCount() ?? null await human?.close().catch(() => undefined) From 77e76b63decd35f6454378347ddb66ea812f58a9 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:06:04 +0200 Subject: [PATCH 4/7] fix(bench): survive a neighbour on the same sandbox plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two plan-tier facts kept turning a benchable minute into a failed one. A neighbouring job holding both slots made every relay creation throw "Too many concurrent sessions", and five of those in a row aborted the bench. And a sandbox can go away under the run — an expiry, a hiccup, a neighbour's cleanup — after which every workflow fails at the login form, which says nothing about handraise. Neither is a datum, so neither is recorded as one: a full plan is waited out for a minute and retried, up to ten times per workflow, and a run that never reached its interrupt now checks /healthz and rebuilds the test app before trying again. `concurrencyWaits` and `testAppRestarts` in the JSON say how often the bench had to do either. --- e2e/mixed-bench.ts | 112 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/e2e/mixed-bench.ts b/e2e/mixed-bench.ts index bba35af..f5460e2 100644 --- a/e2e/mixed-bench.ts +++ b/e2e/mixed-bench.ts @@ -57,6 +57,10 @@ * inherits the session or the transfer receipt an earlier run left behind. * - A failed run is recorded as a failed run. Percentiles are over the runs * that completed, with the failures shown next to them. + * - A run that could not get a relay because a neighbouring job held both + * sandbox slots is not a failed run: the bench stands back for a minute and + * tries that workflow again, up to ten times, and `concurrencyWaits` in the + * JSON says how often that happened. * * Set MIXED_FAULT=invert-completed to invert the completion test. Both modes * must then read 0 of N — that is how the counting is known to be load-bearing @@ -67,7 +71,7 @@ import type { Page } from "playwright-core" import type { HandoffEvent } from "../src/events" import { raiseHand } from "../src/index" -import { startTestApp } from "../test-app/deploy" +import { previewPath, startTestApp } from "../test-app/deploy" import { totp } from "../test-app/totp" import { openHandoffPage, type SimulatedHuman } from "./human-sim" @@ -93,6 +97,12 @@ const BROWSER_MAX_AGE_MS = 3 * 60_000 const COOLDOWN_MS = 1_000 /** Consecutive failures that mean the infrastructure is down, not the claim. */ const ABORT_AFTER_CONSECUTIVE_FAILURES = 5 +/** How long to stand back when the plan is full, before trying the run again. */ +const CONCURRENCY_WAIT_MS = 60_000 +/** How many times one workflow may wait out a full plan before it is a failure. */ +const CONCURRENCY_WAITS = 10 +/** How many times the test app may be rebuilt when it dies under the bench. */ +const APP_RESTARTS = 3 const RESULTS_PATH = new URL( "../benchmarks/mixed-workload.json", import.meta.url, @@ -140,6 +150,12 @@ interface MixedRun { */ relaySandboxSeconds: number | null error: string | null + /** + * Times this workflow stood back for a full plan before it ran. A neighbour + * holding both sandbox slots is queueing, not a result, so it is waited out + * and counted here rather than recorded as a failed workflow. + */ + concurrencyWaits: number /** Age of the reused browser session when the run started, in ms. */ browserAgeMs: number } @@ -520,6 +536,7 @@ async function runWorkflow( humanActiveMs: null, relaySandboxSeconds: null, error: null, + concurrencyWaits: 0, browserAgeMs: Date.now() - lease.launchedAt, } @@ -591,6 +608,36 @@ function isNumber(value: number | null | undefined): value is number { return value !== null && value !== undefined && Number.isFinite(value) } +/** + * Is the test app still answering? A sandbox can go away under a bench — an + * expiry, a platform hiccup, a neighbouring job's cleanup — and every run after + * that fails at the login form, which says nothing about handraise. + */ +async function appIsHealthy(url: string): Promise { + try { + const response = await fetch(previewPath(url, "/healthz"), { + cache: "no-store", + signal: AbortSignal.timeout(10_000), + }) + await response.text() + return response.status === 200 + } catch { + return false + } +} + +/** + * Did this run fail because the plan was full rather than because handraise + * was? A neighbouring job holding both sandbox slots means the relay was never + * created, and that is a queue, not a datum. + */ +function planWasFull(run: MixedRun): boolean { + return ( + run.relaySandboxSeconds === null && + /too many concurrent|concurrenc|429/i.test(run.error ?? "") + ) +} + function summarise(mode: Kind, label: string, runs: MixedRun[]): ModeSummary { const mine = runs.filter((run) => run.kind === mode) const done = mine.filter((run) => run.completed) @@ -678,9 +725,11 @@ const benchStartedAt = Date.now() const runs: MixedRun[] = [] let relaunches = 0 let consecutiveFailures = 0 +let concurrencyWaits = 0 +let appRestarts = 0 let abortReason: string | null = null -const app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) +let app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) console.log( JSON.stringify({ event: "test_app_ready", @@ -699,16 +748,57 @@ try { const decision: Decision = kind === "approval" && approvals % DENY_EVERY === 0 ? "deny" : "approve" - if (leaseIsStale(lease)) { - await releaseBrowser(lease) - lease = await leaseBrowser() - relaunches += 1 + const attempt = async (): Promise => { + if (leaseIsStale(lease)) { + await releaseBrowser(lease) + lease = await leaseBrowser() + relaunches += 1 + console.log( + JSON.stringify({ event: "browser_relaunched", before: index }), + ) + } + return runWorkflow(index, kind, decision, lease, app) + } + + let run = await attempt() + let waits = 0 + while (planWasFull(run) && waits < CONCURRENCY_WAITS) { + waits += 1 + concurrencyWaits += 1 console.log( - JSON.stringify({ event: "browser_relaunched", before: index }), + JSON.stringify({ + event: "plan_full", + index, + waitSeconds: CONCURRENCY_WAIT_MS / 1000, + attempt: waits, + }), ) + await Bun.sleep(CONCURRENCY_WAIT_MS) + run = await attempt() + } + run.concurrencyWaits = waits + + // A run that never even reached the interrupt may have found the test app + // gone rather than anything about a handoff. Rebuild it and try again; + // that is infrastructure, not a datum. + while ( + !run.reachedInterrupt && + appRestarts < APP_RESTARTS && + !(await appIsHealthy(app.url)) + ) { + console.log(JSON.stringify({ event: "test_app_lost", index })) + await app.kill().catch(() => undefined) + app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) + appRestarts += 1 + console.log( + JSON.stringify({ + event: "test_app_restarted", + index, + sandbox: app.sandboxId, + }), + ) + run = await attempt() } - - const run = await runWorkflow(index, kind, decision, lease, app) runs.push(run) console.log( JSON.stringify({ @@ -793,6 +883,8 @@ await Bun.write( fault: FAULT === "" ? null : FAULT, abortReason, browserRelaunches: relaunches, + concurrencyWaits, + testAppRestarts: appRestarts, totalMs: Date.now() - benchStartedAt, relaySandboxSecondsAllRuns: relaySeconds?.total ?? null, bunVersion: Bun.version, @@ -820,6 +912,8 @@ console.log( approvalCompleted: `${summaries[1]?.completed}/${summaries[1]?.attempted}`, relaySandboxSecondsAllRuns: relaySeconds?.total ?? null, browserRelaunches: relaunches, + concurrencyWaits, + testAppRestarts: appRestarts, totalMs: Date.now() - benchStartedAt, results: RESULTS_PATH.pathname, }), From da7876beefb610c19805ccf8b2c91cd3472c54d6 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:06:11 +0200 Subject: [PATCH 5/7] docs(bench): publish the mixed-workload numbers 20 workflows against one live Aurora Bank instance on 2026-09-02, interleaved takeover, approval, takeover, on an uncontended plan: 20 of 20 completed, 10 of 10 in each mode, two of the ten approvals denied. The side-by-side is the point. A takeover put 14 frames and 142 KB on the wire and held a relay sandbox for 11.0 s; an approval put one screenshot and 25 KB on it, applied zero inputs, and held one for 5.5 s. Time to visible is the same for both, because both pay the same relay cold start. Both README sections say what may be claimed and what may not: the 50/50 mix is the harness's choice, not anyone's traffic, and the takeover stream grows with the human while the approval's one frame does not. --- README.md | 18 + benchmarks/README.md | 76 ++++- benchmarks/mixed-workload.json | 606 +++++++++++++++++++++++++++++++++ 3 files changed, 691 insertions(+), 9 deletions(-) create mode 100644 benchmarks/mixed-workload.json diff --git a/README.md b/README.md index 409e671..11050d6 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,24 @@ floor of a handoff, not reading speed. The one failure was the platform's ~10min session death landing mid-handoff; handraise reported `disconnected` instead of claiming success. +Two interrupts do not cost the same, and `bun run bench:mixed` measures them +side by side: 20 workflows against one live portal, interleaved takeover, +approval, takeover, on 2026-09-02. A takeover needs the browser driven; an +approval needs one decision, and every fourth one here was a denial. + +| | completed | to visible | frames | bytes | relay sandbox | +|---|---|---|---|---|---| +| takeover — the human drives | 10/10 | 4923ms | 14 | 142 KB | 11.0s | +| approval — the human decides | 10/10 | 5089ms | 1 | 25 KB | 5.5s | + +Medians over the completed runs; a denied approval counts as completed, because +the decision was delivered and the bench checks that no money moved. An approval +injects nothing into the page, so it applies zero inputs and sends one +screenshot however long the human thinks, while a takeover's stream keeps +growing while the human works. The 50/50 mix is the harness's choice, not a +measurement of anyone's traffic — take the per-mode costs and apply your own +mix. + At N=30 the right-hand column is the worst observation, not a fitted p99 — we say what we measured. The input round trip sits on the network RTT floor from Germany to the us-west edge (pass `baseUrl` to co-locate the relay with your diff --git a/benchmarks/README.md b/benchmarks/README.md index 87f0a6c..55c5d11 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,20 +1,23 @@ # Benchmarks -Two questions, two harnesses, two raw data files. Both run against the live -Solari API — no mocks, no simulated network, no modelled numbers. Every figure -in the README comes from the JSON next to this file. +Three questions, three harnesses, three raw data files. All of them run against +the live Solari API — no mocks, no simulated network, no modelled numbers. Every +figure in the README comes from the JSON next to this file. | File | Harness | Question | |---|---|---| | [`handoff-latency.json`](handoff-latency.json) | [`e2e/bench.ts`](../e2e/bench.ts) | What does a handoff cost in wall-clock time? | | [`rescue-rate.json`](rescue-rate.json) | [`e2e/rescue-bench.ts`](../e2e/rescue-bench.ts) | How many blocked workflows get done at all? | +| [`mixed-workload.json`](mixed-workload.json) | [`e2e/mixed-bench.ts`](../e2e/mixed-bench.ts) | What does each kind of interrupt cost, side by side? | ## Reproducing -Both need a `SOLARI_API_KEY` in `.env` and both consume real sandboxes. The -plan they were measured on allows two concurrent sandboxes, so nothing else may -run alongside — check with `bun --env-file=.env scripts/cleanup-sandboxes.ts` -before and after. +They all need a `SOLARI_API_KEY` in `.env` and they all consume real sandboxes. +The plan they were measured on allows two concurrent sandboxes, so nothing else +may run alongside — check with `bun --env-file=.env scripts/cleanup-sandboxes.ts` +before and after. A run that collides with another one fails with "Too many +concurrent sessions" and says so in its JSON, rather than quietly reporting +slower numbers. ```sh bun run bench # latency, N=30, ~3.5 min @@ -22,6 +25,9 @@ BENCH_N=5 bun run bench # a short run bun run bench:rescue # rescue rate, 2×20 runs, ~7 min RESCUE_N=3 bun run bench:rescue # a short run + +bun run bench:mixed # mixed workload, 20 runs, ~9 min +MIXED_N=4 bun run bench:mixed # a short run ``` Each writes its JSON back into this directory, overwriting the committed file. @@ -80,10 +86,62 @@ The counting is load-bearing rather than decorative, and that is testable: test, and the table must then read 20/20 for the baseline and 0/20 for handraise. +## Mixed workload — 20 of 20 workflows completed, and what each mode cost + +The first two benches ask about one mode. This one runs both against the same +Aurora Bank instance, interleaved (takeover, approval, takeover, …), on +2026-09-02. Two interrupts, two prices: + +- **takeover** — the agent signs in with the credentials it has and stops at a + real TOTP wall it cannot pass. A scripted human on the public WebSocket taps + the field, types the code, presses Enter and hands back. +- **approval** — the agent is signed in and not stuck at all. It fills a + transfer form and stops before submitting, because moving money is not its + decision. A scripted human sees one screenshot and the action in words, and + answers. Every fourth approval is denied (2 of 10 here). + +| | completed | time to visible p50 / p75 | handoff p50 / p75 | frames | bytes | inputs | relay-sandbox s | +|---|---|---|---|---|---|---|---| +| takeover | 10/10 | 4923 / 5809 ms | 6621 / 7011 ms | 14 | 142 KB | 8 | 11.0 | +| approval | 10/10 | 5089 / 5381 ms | 2091 / 2293 ms | 1 | 25 KB | 0 | 5.5 | + +All per-handoff figures are medians over the runs that completed. `time to +visible` is `raiseHand()` → the first frame arriving on the human's socket, the +same measurement as the latency bench, so the two modes are comparable and both +carry the same relay cold start (3211 ms takeover, 3068 ms approval, at p50). +`frames` and `bytes` are what the agent put on the wire: an approval is one +screenshot, 25 KB of base64 payload, and it injects nothing into the page — +`inputsApplied` is 0 by construction, not by luck. `relay-sandbox s` is wall-clock from `raiseHand()` to +the promise settling, which covers creating the sandbox, the handoff and +destroying it: the closest thing to a bill. + +A denied approval counts as completed, and that is a deliberate choice: the +workflow reached a decision and the agent obeyed it. The bench asserts the money +did not move — for a denial it requires that no transfer receipt exists on the +page — so "completed" means the mechanism delivered an answer, not that the +answer was yes. + +What may be claimed from this, and nothing wider: on this workload, an approval +costs one frame where a takeover costs a stream, and both modes delivered their +workflows at the rates in the table. It is **not** a claim about the mix a real +fleet sees. The 50/50 split is this harness's choice; multiply the per-mode +costs by your own mix. Nor is the takeover stream a fixed cost — it grows with +how long the human takes, and this human is a script that finishes in about +7 seconds. An approval's one frame does not grow at all. + +One more thing the numbers do not say: in the approval arm the agent signs +itself in with the shared secret. That sign-in is setup, not the interrupt being +measured — the interrupt is the transfer, which comes after. Only the takeover +arm is barred from the secret, because there the wall is the whole test. + +The counting is load-bearing rather than decorative, and that is testable: +`MIXED_FAULT=invert-completed bun run bench:mixed` inverts the completion test, +and both modes must then read 0 of N. + ## Reading the raw files -Both files carry a `meta` block (date, N, platform, where it was measured from, -the timeouts in force) next to the aggregates and every individual run. +Each file carries a `meta` block (date, N, platform, where it was measured +from, the timeouts in force) next to the aggregates and every individual run. Failures are recorded as failures and never dropped; percentiles are over the successes, with the failure rate reported separately. diff --git a/benchmarks/mixed-workload.json b/benchmarks/mixed-workload.json new file mode 100644 index 0000000..b557020 --- /dev/null +++ b/benchmarks/mixed-workload.json @@ -0,0 +1,606 @@ +{ + "meta": { + "date": "2026-09-02T19:04:30.067Z", + "requestedN": 20, + "workload": "N workflows against one Aurora Bank instance, interleaved takeover, approval, takeover, …", + "interrupts": { + "takeover": "a real RFC 6238 TOTP wall the agent cannot pass; a scripted human types the code and hands back", + "approval": "a filled transfer form the agent may not submit alone; a scripted human answers yes or no" + }, + "claim": "on this workload, each mode's cost per handoff and how many workflows completed", + "notAClaim": "the 50/50 mix is this harness's choice, not a measurement of any fleet's traffic; multiply the per-mode costs by your own mix", + "deniedCountAsCompleted": "yes — a denied approval delivered a decision the agent obeyed, and the bench asserts no transfer was sent", + "approvalSetup": "the approval arm signs itself in with the shared secret; that sign-in is setup, and the interrupt under measurement is the transfer", + "denyEvery": 4, + "interleaved": true, + "fault": null, + "abortReason": null, + "browserRelaunches": 1, + "concurrencyWaits": 0, + "testAppRestarts": 0, + "totalMs": 365242, + "relaySandboxSecondsAllRuns": 169.365, + "bunVersion": "1.4.0", + "platform": "darwin-arm64", + "measuredFrom": "Germany → default Solari endpoint (api.getsolari.com)", + "handoffTimeoutMs": 90000, + "viewport": { + "width": 1280, + "height": 800 + } + }, + "summaries": [ + { + "mode": "takeover", + "label": "takeover — the human drives", + "attempted": 10, + "completed": 10, + "approved": null, + "denied": null, + "neverReachedInterrupt": 0, + "stuckToVisibleMs": { + "n": 10, + "p50": 4923, + "p75": 5809, + "worst": 6492, + "total": 52495 + }, + "handoffDurationMs": { + "n": 10, + "p50": 6621, + "p75": 7011, + "worst": 8438, + "total": 69031 + }, + "relayColdStartMs": { + "n": 10, + "p50": 3211, + "p75": 3914, + "worst": 4723, + "total": 34984 + }, + "humanActiveMs": { + "n": 10, + "p50": 4593, + "p75": 5176, + "worst": 6564, + "total": 49377 + }, + "framesSent": { + "n": 10, + "p50": 14, + "p75": 14, + "worst": 16, + "total": 138 + }, + "bytesSent": { + "n": 10, + "p50": 145516, + "p75": 145584, + "worst": 166424, + "total": 1433784 + }, + "inputsApplied": { + "n": 10, + "p50": 8, + "p75": 8, + "worst": 8, + "total": 80 + }, + "relaySandboxSeconds": { + "n": 10, + "p50": 10.955, + "p75": 12.06, + "worst": 12.435, + "total": 112.916 + } + }, + { + "mode": "approval", + "label": "approval — the human decides", + "attempted": 10, + "completed": 10, + "approved": 8, + "denied": 2, + "neverReachedInterrupt": 0, + "stuckToVisibleMs": { + "n": 10, + "p50": 5089, + "p75": 5381, + "worst": 6729, + "total": 52127 + }, + "handoffDurationMs": { + "n": 10, + "p50": 2091, + "p75": 2293, + "worst": 2626, + "total": 21734 + }, + "relayColdStartMs": { + "n": 10, + "p50": 3068, + "p75": 3231, + "worst": 4922, + "total": 32692 + }, + "humanActiveMs": { + "n": 10, + "p50": 0, + "p75": 1, + "worst": 1, + "total": 4 + }, + "framesSent": { + "n": 10, + "p50": 1, + "p75": 1, + "worst": 1, + "total": 10 + }, + "bytesSent": { + "n": 10, + "p50": 25368, + "p75": 25380, + "worst": 25384, + "total": 253704 + }, + "inputsApplied": { + "n": 10, + "p50": 0, + "p75": 0, + "worst": 0, + "total": 0 + }, + "relaySandboxSeconds": { + "n": 10, + "p50": 5.467, + "p75": 5.762, + "worst": 7.205, + "total": 56.449 + } + } + ], + "runs": [ + { + "index": 1, + "kind": "takeover", + "startedAt": "2026-09-02T18:58:30.161Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3707, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 5214, + "handoffDurationMs": 6714, + "relayColdStartMs": 3340, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145584, + "inputsApplied": 8, + "humanActiveMs": 4600, + "relaySandboxSeconds": 10.841, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 1680 + }, + { + "index": 2, + "kind": "approval", + "startedAt": "2026-09-02T18:58:46.893Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8283, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 4535, + "handoffDurationMs": 1924, + "relayColdStartMs": 2937, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25368, + "inputsApplied": 0, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.058, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 18412 + }, + { + "index": 3, + "kind": "takeover", + "startedAt": "2026-09-02T18:59:04.620Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3670, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 6492, + "handoffDurationMs": 6483, + "relayColdStartMs": 4723, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145516, + "inputsApplied": 8, + "humanActiveMs": 4464, + "relaySandboxSeconds": 12.06, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 36139 + }, + { + "index": 4, + "kind": "approval", + "startedAt": "2026-09-02T18:59:22.440Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8250, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 5381, + "handoffDurationMs": 2286, + "relayColdStartMs": 3284, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25364, + "inputsApplied": 0, + "humanActiveMs": 1, + "relaySandboxSeconds": 5.762, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 53959 + }, + { + "index": 5, + "kind": "takeover", + "startedAt": "2026-09-02T18:59:40.579Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3737, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 4917, + "handoffDurationMs": 6426, + "relayColdStartMs": 3211, + "framesSent": 13, + "framesAtHuman": 11, + "bytesSent": 134884, + "inputsApplied": 8, + "humanActiveMs": 4457, + "relaySandboxSeconds": 10.452, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 72098 + }, + { + "index": 6, + "kind": "approval", + "startedAt": "2026-09-02T18:59:56.895Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8403, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 6729, + "handoffDurationMs": 2091, + "relayColdStartMs": 4922, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25372, + "inputsApplied": 0, + "humanActiveMs": 1, + "relaySandboxSeconds": 7.205, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 88414 + }, + { + "index": 7, + "kind": "takeover", + "startedAt": "2026-09-02T19:00:16.553Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3654, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 4698, + "handoffDurationMs": 6474, + "relayColdStartMs": 3035, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145572, + "inputsApplied": 8, + "humanActiveMs": 4585, + "relaySandboxSeconds": 10.38, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 108072 + }, + { + "index": 8, + "kind": "approval", + "startedAt": "2026-09-02T19:00:32.689Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8277, + "decision": "deny", + "handoffOutcome": "denied", + "stuckToVisibleMs": 4908, + "handoffDurationMs": 2127, + "relayColdStartMs": 3005, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25368, + "inputsApplied": 0, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.357, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 124208 + }, + { + "index": 9, + "kind": "takeover", + "startedAt": "2026-09-02T19:00:49.264Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3889, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 4923, + "handoffDurationMs": 6556, + "relayColdStartMs": 3147, + "framesSent": 13, + "framesAtHuman": 12, + "bytesSent": 134892, + "inputsApplied": 8, + "humanActiveMs": 4593, + "relaySandboxSeconds": 10.544, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 140783 + }, + { + "index": 10, + "kind": "approval", + "startedAt": "2026-09-02T19:01:05.832Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8262, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 5242, + "handoffDurationMs": 2293, + "relayColdStartMs": 3144, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25360, + "inputsApplied": 0, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.646, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 157351 + }, + { + "index": 11, + "kind": "takeover", + "startedAt": "2026-09-02T19:01:23.861Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3708, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 5844, + "handoffDurationMs": 6621, + "relayColdStartMs": 3919, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145548, + "inputsApplied": 8, + "humanActiveMs": 4506, + "relaySandboxSeconds": 11.318, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 175380 + }, + { + "index": 12, + "kind": "approval", + "startedAt": "2026-09-02T19:01:43.636Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 9288, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 5461, + "handoffDurationMs": 2626, + "relayColdStartMs": 3140, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25380, + "inputsApplied": 0, + "humanActiveMs": 1, + "relaySandboxSeconds": 5.955, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 2103 + }, + { + "index": 13, + "kind": "takeover", + "startedAt": "2026-09-02T19:02:04.038Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 3972, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 4597, + "handoffDurationMs": 8438, + "relayColdStartMs": 2924, + "framesSent": 16, + "framesAtHuman": 15, + "bytesSent": 166424, + "inputsApplied": 8, + "humanActiveMs": 6564, + "relaySandboxSeconds": 12.435, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 22505 + }, + { + "index": 14, + "kind": "approval", + "startedAt": "2026-09-02T19:02:22.799Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 9413, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 4905, + "handoffDurationMs": 2063, + "relayColdStartMs": 3027, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25360, + "inputsApplied": 0, + "humanActiveMs": 1, + "relaySandboxSeconds": 5.329, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 41266 + }, + { + "index": 15, + "kind": "takeover", + "startedAt": "2026-09-02T19:02:41.584Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 4233, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 5809, + "handoffDurationMs": 7448, + "relayColdStartMs": 3914, + "framesSent": 15, + "framesAtHuman": 15, + "bytesSent": 156160, + "inputsApplied": 8, + "humanActiveMs": 5337, + "relaySandboxSeconds": 12.422, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 60051 + }, + { + "index": 16, + "kind": "approval", + "startedAt": "2026-09-02T19:03:00.454Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8994, + "decision": "deny", + "handoffOutcome": "denied", + "stuckToVisibleMs": 5163, + "handoffDurationMs": 2295, + "relayColdStartMs": 3068, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25368, + "inputsApplied": 0, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.561, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 78921 + }, + { + "index": 17, + "kind": "takeover", + "startedAt": "2026-09-02T19:03:18.096Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 4203, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 4749, + "handoffDurationMs": 7011, + "relayColdStartMs": 3103, + "framesSent": 12, + "framesAtHuman": 12, + "bytesSent": 124324, + "inputsApplied": 8, + "humanActiveMs": 5176, + "relaySandboxSeconds": 10.955, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 96563 + }, + { + "index": 18, + "kind": "approval", + "startedAt": "2026-09-02T19:03:35.375Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8609, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 4714, + "handoffDurationMs": 1981, + "relayColdStartMs": 2934, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25384, + "inputsApplied": 0, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.109, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 113842 + }, + { + "index": 19, + "kind": "takeover", + "startedAt": "2026-09-02T19:03:53.293Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 4106, + "decision": null, + "handoffOutcome": "resolved", + "stuckToVisibleMs": 5252, + "handoffDurationMs": 6860, + "relayColdStartMs": 3668, + "framesSent": 13, + "framesAtHuman": 11, + "bytesSent": 134880, + "inputsApplied": 8, + "humanActiveMs": 5095, + "relaySandboxSeconds": 11.509, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 131760 + }, + { + "index": 20, + "kind": "approval", + "startedAt": "2026-09-02T19:04:11.167Z", + "completed": true, + "reachedInterrupt": true, + "reachedInterruptMs": 8721, + "decision": "approve", + "handoffOutcome": "approved", + "stuckToVisibleMs": 5089, + "handoffDurationMs": 2048, + "relayColdStartMs": 3231, + "framesSent": 1, + "framesAtHuman": 1, + "bytesSent": 25380, + "inputsApplied": 0, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.467, + "error": null, + "concurrencyWaits": 0, + "browserAgeMs": 149634 + } + ] +} From 0bd95cc7bd583ea0265f6cfdc4087827dd2787c2 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:40:42 +0200 Subject: [PATCH 6/7] fix(bench): make the denial check able to fail, and the fault proof mean something MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1 found the sensor had a hole. After a denial the bench asked the transfer form whether a receipt was on it — but `transfer-done` is rendered only by the account page, so the check could not fail, could not notice a transfer that did go through, and would have passed against a blank page. Two published sentences claimed an assertion nobody made. The denial verdict now loads /account and requires the session banner AND the absence of a receipt carrying that run's amount; the approve verdict matches the amount too, so a stale receipt cannot pass either. Proven red-first: with the agent submitting despite the denial, the run reports 0/1 instead of 1/1. MIXED_FAULT=invert-completed now inverts the page sensors instead of the verdict. Flipping the verdict passed identically for a check that never read the page — it could not have caught the bug above. Flipping the sensor does: the proof run reads 0/4 and 0/4 with the denial included. Setup steps keep the raw predicates, and the abort-on-failure-streak guard stands down under the fault, which is supposed to fail everything. Also from the review: a full plan is detected with the typed `concurrency_limit` code rather than a regex over an error message the project says is not a contract, and a replaced attempt is no longer dropped — it stays in `runs` as not-completed with `superseded` and its error, so no retry rule can quietly improve the completion rate. --- e2e/mixed-bench.ts | 270 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 205 insertions(+), 65 deletions(-) diff --git a/e2e/mixed-bench.ts b/e2e/mixed-bench.ts index f5460e2..7bd865e 100644 --- a/e2e/mixed-bench.ts +++ b/e2e/mixed-bench.ts @@ -40,7 +40,10 @@ * - Deterministic scripts, not an LLM. The claim is about the mechanism. * - Every 4th approval is DENIED, and a denied approval counts as completed: * the workflow reached a decision and the agent obeyed it. A bench that only - * counted "yes" would be measuring agreement, not delivery. + * counted "yes" would be measuring agreement, not delivery. "Obeyed" is + * checked on the account page — the only page that can render a receipt — and + * it requires the session to still be there as well as the receipt to be + * absent, so the check can fail in both directions. * - The approval arm signs itself in with the shared secret. That sign-in is * setup, not the interrupt under measurement — the interrupt here is the * authority boundary at the transfer, and it happens after the wall. Only the @@ -58,19 +61,26 @@ * - A failed run is recorded as a failed run. Percentiles are over the runs * that completed, with the failures shown next to them. * - A run that could not get a relay because a neighbouring job held both - * sandbox slots is not a failed run: the bench stands back for a minute and - * tries that workflow again, up to ten times, and `concurrencyWaits` in the - * JSON says how often that happened. + * sandbox slots is waited out: the bench stands back for a minute and tries + * that workflow again, up to ten times. The attempt it replaces is kept in + * `runs` with `superseded: "plan_full"` and counts as not completed, so a + * retry rule can never quietly improve the completion rate. * - * Set MIXED_FAULT=invert-completed to invert the completion test. Both modes - * must then read 0 of N — that is how the counting is known to be load-bearing - * rather than decorative. + * Set MIXED_FAULT=invert-completed to invert the page sensors — the banner + * check and the receipt check, not the verdict they feed. Both modes must then + * read 0 of N. Inverting the sensor rather than the verdict is what makes the + * proof worth running: a verdict flip passes even for a check that never looks + * at the page, and this bench shipped exactly that mistake once. */ import { Solari } from "@solarisdk/browser" import type { Page } from "playwright-core" import type { HandoffEvent } from "../src/events" -import { raiseHand } from "../src/index" +import { + type HandraiseErrorCode, + isHandraiseError, + raiseHand, +} from "../src/index" import { previewPath, startTestApp } from "../test-app/deploy" import { totp } from "../test-app/totp" import { openHandoffPage, type SimulatedHuman } from "./human-sim" @@ -150,6 +160,16 @@ interface MixedRun { */ relaySandboxSeconds: number | null error: string | null + /** handraise's own error code, when it threw one. */ + errorCode: HandraiseErrorCode | null + /** + * Why this attempt was replaced by another one, or null if it counted. A + * superseded attempt stays in the data and stays not-completed: dropping it + * would hide a failure that the retry rule happened to forgive. + */ + superseded: "plan_full" | "test_app_lost" | null + /** Which attempt at this workflow index this record is, from 1. */ + attempt: number /** * Times this workflow stood back for a full plan before it ran. A neighbour * holding both sandbox slots is queueing, not a result, so it is waited out @@ -231,10 +251,60 @@ async function waitUntil( } /** - * The sign-in step's definition of done: the account page, rendered, for our - * user. Not "the URL changed" — the URL changes on a 303 before the page exists. + * The raw page predicates every verdict is built from, and below them the + * sensors the verdicts actually call. + * + * The fault switch acts on the sensors, not on the verdict. That is the point + * of it: a check that never looks at the page would invert just as cleanly as a + * live one if the verdict were flipped at the end, so flipping the verdict + * proves only that the summary reads a boolean. Flipping the sensor proves the + * boolean came from the page. Setup steps deliberately use the raw predicates — + * setup is plumbing, not a claim. */ -async function reachedAccount( +const INVERTED = FAULT === "invert-completed" + +/** Under the fault, every sensor answers backwards. */ +function sensed(seen: boolean): boolean { + return INVERTED ? !seen : seen +} + +/** Raw: does the account page name our user? */ +async function bannerNamesUser( + page: Page, + user: string, + timeoutMs: number, +): Promise { + const banner = await page + .textContent('[data-testid="signed-in"]', { timeout: timeoutMs }) + .catch(() => null) + return banner?.includes(user) === true +} + +/** + * Raw: is a receipt for a sent transfer on the page, and does it name `amount`? + * + * The amount is unique per run, so matching it also catches a receipt left by + * an earlier workflow. + */ +async function receiptIsOnPage( + page: Page, + timeoutMs: number, + amount?: string, +): Promise { + return waitUntil(async () => { + const receipt = page.locator('[data-testid="transfer-done"]') + if ((await receipt.count()) === 0) return false + if (amount === undefined) return true + const text = await receipt.first().textContent() + return text?.includes(amount) === true + }, timeoutMs) +} + +/** + * Raw: the account page, rendered, for our user. Not "the URL changed" — the + * URL changes on a 303 before the page exists. + */ +async function accountIsRendered( page: Page, user: string, timeoutMs: number, @@ -244,10 +314,29 @@ async function reachedAccount( timeoutMs, ) if (!arrived) return false - const banner = await page - .textContent('[data-testid="signed-in"]', { timeout: timeoutMs }) - .catch(() => null) - return banner?.includes(user) === true + return bannerNamesUser(page, user, timeoutMs) +} + +/** Sensor: did the workflow land on its account page? */ +async function reachedAccount( + page: Page, + user: string, + timeoutMs: number, +): Promise { + return sensed(await accountIsRendered(page, user, timeoutMs)) +} + +/** + * Sensor: is this the account page of our signed-in user? Used where the page + * was loaded by URL, so there is no redirect to wait for — and the preview + * host's URL ends in `?pt_token=`, not in `/account`. + */ +async function accountShowsUser( + page: Page, + user: string, + timeoutMs: number, +): Promise { + return sensed(await bannerNamesUser(page, user, timeoutMs)) } /** Drive the agent's part: log in with the credentials it has, hit the wall. */ @@ -259,16 +348,13 @@ async function walkToWall(page: Page, app: Target): Promise { await page.waitForSelector('[data-testid="totp-code"]', { timeout: 30_000 }) } -/** Is a receipt for a sent transfer on the page right now? */ +/** Sensor: is a receipt for a sent transfer on the page right now? */ async function transferWasSent( page: Page, timeoutMs: number, + amount?: string, ): Promise { - return waitUntil( - async () => - (await page.locator('[data-testid="transfer-done"]').count()) > 0, - timeoutMs, - ) + return sensed(await receiptIsOnPage(page, timeoutMs, amount)) } // --- the takeover interrupt ------------------------------------------------ @@ -322,7 +408,9 @@ async function signInWithTheSecret(page: Page, app: Target): Promise { await walkToWall(page, app) await page.fill('[data-testid="totp-code"]', totp(app.totpSecret)) await page.click('[data-testid="totp-submit"]') - const arrived = await reachedAccount(page, app.user, SETTLE_TIMEOUT_MS) + // Raw, not sensed: a fault switch that broke the setup would prove nothing + // about the verdicts it is aimed at. + const arrived = await accountIsRendered(page, app.user, SETTLE_TIMEOUT_MS) if (!arrived) throw new Error("the approval arm never got signed in") } @@ -340,6 +428,8 @@ async function fillTransfer(page: Page, amount: string, payee: string) { interface HandoffRecord { outcome: string | null + /** The typed code, when handraise threw one. The message is not a contract. */ + errorCode: HandraiseErrorCode | null event: HandoffEvent | null stuckToVisibleMs: number | null humanActiveMs: number | null @@ -361,6 +451,7 @@ async function withHandoff( ): Promise { const record: HandoffRecord = { outcome: null, + errorCode: null, event: null, stuckToVisibleMs: null, humanActiveMs: null, @@ -400,6 +491,7 @@ async function withHandoff( record.humanActiveMs = record.humanActiveMs ?? Date.now() - firstFrameAt } catch (error) { record.error = error instanceof Error ? error.message : String(error) + record.errorCode = isHandraiseError(error) ? error.code : null // The script gave up, so say so on the wire instead of holding the relay // sandbox for the rest of the timeout. A takeover settles as `aborted`; an // approval ignores the message by design and still runs out the clock. @@ -408,6 +500,8 @@ async function withHandoff( const result = await handoff.catch((error: Error) => { record.error = record.error ?? error.message + record.errorCode = + record.errorCode ?? (isHandraiseError(error) ? error.code : null) return null }) // Only when a relay actually came up: a handoff that never got one (the plan @@ -488,11 +582,19 @@ async function runApproval( // nothing into the page, so the transfer only happens if the agent submits. if (handoff.outcome === "approved") { await page.click('[data-testid="transfer-submit"]') - run.completed = await transferWasSent(page, SETTLE_TIMEOUT_MS) + run.completed = await transferWasSent(page, SETTLE_TIMEOUT_MS, amount) } else if (handoff.outcome === "denied") { // A denied approval is a completed workflow: the decision was delivered and - // obeyed. What must be true is that no money moved. - run.completed = !(await transferWasSent(page, 1_000)) + // obeyed. Both halves of that have to be seen on a page that could show the + // opposite. The transfer form cannot: `transfer-done` is rendered only by + // the account page, so asking the form for a receipt is a check that cannot + // fail. Load the account page and require the session AND the absence. + await page.goto(previewPath(app.url, "/account"), { + waitUntil: "domcontentloaded", + timeout: 45_000, + }) + const signedIn = await accountShowsUser(page, app.user, SETTLE_TIMEOUT_MS) + run.completed = signedIn && !(await transferWasSent(page, 1_000, amount)) } } @@ -508,6 +610,7 @@ function applyHandoff(run: MixedRun, handoff: HandoffRecord): void { run.humanActiveMs = handoff.humanActiveMs run.relaySandboxSeconds = handoff.relaySandboxSeconds run.error = handoff.error + run.errorCode = handoff.errorCode } async function runWorkflow( @@ -536,6 +639,9 @@ async function runWorkflow( humanActiveMs: null, relaySandboxSeconds: null, error: null, + errorCode: null, + superseded: null, + attempt: 1, concurrencyWaits: 0, browserAgeMs: Date.now() - lease.launchedAt, } @@ -551,7 +657,6 @@ async function runWorkflow( await page.close().catch(() => undefined) } - if (FAULT === "invert-completed") run.completed = !run.completed return run } @@ -633,8 +738,7 @@ async function appIsHealthy(url: string): Promise { */ function planWasFull(run: MixedRun): boolean { return ( - run.relaySandboxSeconds === null && - /too many concurrent|concurrenc|429/i.test(run.error ?? "") + run.relaySandboxSeconds === null && run.errorCode === "concurrency_limit" ) } @@ -729,6 +833,18 @@ let concurrencyWaits = 0 let appRestarts = 0 let abortReason: string | null = null +/** Keep a replaced attempt in the data, with why it was replaced. */ +function supersede( + run: MixedRun, + reason: "plan_full" | "test_app_lost", + attemptNumber: number, +): void { + run.superseded = reason + run.attempt = attemptNumber + run.completed = false + runs.push(run) +} + let app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) console.log( JSON.stringify({ @@ -761,49 +877,63 @@ try { } let run = await attempt() + let tries = 1 let waits = 0 - while (planWasFull(run) && waits < CONCURRENCY_WAITS) { - waits += 1 - concurrencyWaits += 1 - console.log( - JSON.stringify({ - event: "plan_full", - index, - waitSeconds: CONCURRENCY_WAIT_MS / 1000, - attempt: waits, - }), - ) - await Bun.sleep(CONCURRENCY_WAIT_MS) - run = await attempt() + // One loop for both retry rules, so a retried run is judged by the same + // two conditions as a first attempt. Every attempt that is replaced stays + // in the data as a not-completed run with its error: a queue is a queue, + // but a failure that a retry rule happens to forgive is still a failure. + for (;;) { + if (planWasFull(run) && waits < CONCURRENCY_WAITS) { + waits += 1 + concurrencyWaits += 1 + supersede(run, "plan_full", tries) + console.log( + JSON.stringify({ + event: "plan_full", + index, + waitSeconds: CONCURRENCY_WAIT_MS / 1000, + attempt: waits, + }), + ) + await Bun.sleep(CONCURRENCY_WAIT_MS) + tries += 1 + run = await attempt() + continue + } + // A run that never even reached the interrupt may have found the test app + // gone rather than anything about a handoff. + if ( + !run.reachedInterrupt && + appRestarts < APP_RESTARTS && + !(await appIsHealthy(app.url)) + ) { + supersede(run, "test_app_lost", tries) + console.log(JSON.stringify({ event: "test_app_lost", index })) + await app.kill().catch(() => undefined) + app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) + appRestarts += 1 + console.log( + JSON.stringify({ + event: "test_app_restarted", + index, + sandbox: app.sandboxId, + }), + ) + tries += 1 + run = await attempt() + continue + } + break } + run.attempt = tries run.concurrencyWaits = waits - - // A run that never even reached the interrupt may have found the test app - // gone rather than anything about a handoff. Rebuild it and try again; - // that is infrastructure, not a datum. - while ( - !run.reachedInterrupt && - appRestarts < APP_RESTARTS && - !(await appIsHealthy(app.url)) - ) { - console.log(JSON.stringify({ event: "test_app_lost", index })) - await app.kill().catch(() => undefined) - app = await startTestApp({ apiKey, timeoutMs: APP_TIMEOUT_MS }) - appRestarts += 1 - console.log( - JSON.stringify({ - event: "test_app_restarted", - index, - sandbox: app.sandboxId, - }), - ) - run = await attempt() - } runs.push(run) console.log( JSON.stringify({ event: "mixed_run", index, + attempt: run.attempt, kind, decision: run.decision, completed: run.completed, @@ -829,7 +959,13 @@ try { } consecutiveFailures = run.completed ? 0 : consecutiveFailures + 1 - if (consecutiveFailures >= ABORT_AFTER_CONSECUTIVE_FAILURES) { + // Under the fault every workflow is supposed to fail, so the guard that + // reads a failure streak as broken infrastructure would end the proof + // before it reached the denial arm. + if ( + FAULT === "" && + consecutiveFailures >= ABORT_AFTER_CONSECUTIVE_FAILURES + ) { abortReason = `${ABORT_AFTER_CONSECUTIVE_FAILURES} workflows failed in a row — the infrastructure is down, not the claim` console.log( JSON.stringify({ event: "mixed_aborted", reason: abortReason }), @@ -875,12 +1011,16 @@ await Bun.write( notAClaim: "the 50/50 mix is this harness's choice, not a measurement of any fleet's traffic; multiply the per-mode costs by your own mix", deniedCountAsCompleted: - "yes — a denied approval delivered a decision the agent obeyed, and the bench asserts no transfer was sent", + "yes — a denied approval delivered a decision the agent obeyed; the bench then loads /account and requires the session banner AND the absence of the receipt for that run's amount", approvalSetup: "the approval arm signs itself in with the shared secret; that sign-in is setup, and the interrupt under measurement is the transfer", denyEvery: DENY_EVERY, interleaved: true, fault: FAULT === "" ? null : FAULT, + faultActsOn: + "the page sensors (signed-in banner, transfer receipt), not the verdict they feed", + supersededAttempts: runs.filter((run) => run.superseded !== null) + .length, abortReason, browserRelaunches: relaunches, concurrencyWaits, From 86d6859ec1b95cfc6ca2112758257115484c1b72 Mon Sep 17 00:00:00 2001 From: Sy-D <8460326+Sy-D@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:40:42 +0200 Subject: [PATCH 7/7] docs(bench): republish the mixed-workload numbers from the fixed harness A second 20-workflow run, this time with a denial check that can fail and a receipt check that matches each run's own amount: 20 of 20 completed, 10 of 10 in each mode, two denials, no concurrency waits, no superseded attempts. A takeover put 14 frames and 142 KB on the wire and held a relay sandbox for 10.7 s; an approval put one screenshot and 25 KB on it, applied zero inputs, and held one for 5.3 s. Three things the review found unstated are now stated: `handoff` is the wide event's durationMs and excludes the cold start, which is why an approval's handoff is shorter than its time to visible; relay-sandbox seconds include the human's occupancy, so 5.0 s of the 5.4 s gap is the scripted human typing and the column is a floor rather than a property of the modes; and the root README now says the approval arm signs itself in with the shared secret, with a link to the method. --- README.md | 24 +- benchmarks/README.md | 53 ++-- benchmarks/mixed-workload.json | 510 ++++++++++++++++++--------------- 3 files changed, 337 insertions(+), 250 deletions(-) diff --git a/README.md b/README.md index 11050d6..536b567 100644 --- a/README.md +++ b/README.md @@ -420,16 +420,20 @@ approval needs one decision, and every fourth one here was a denial. | | completed | to visible | frames | bytes | relay sandbox | |---|---|---|---|---|---| -| takeover — the human drives | 10/10 | 4923ms | 14 | 142 KB | 11.0s | -| approval — the human decides | 10/10 | 5089ms | 1 | 25 KB | 5.5s | - -Medians over the completed runs; a denied approval counts as completed, because -the decision was delivered and the bench checks that no money moved. An approval -injects nothing into the page, so it applies zero inputs and sends one -screenshot however long the human thinks, while a takeover's stream keeps -growing while the human works. The 50/50 mix is the harness's choice, not a -measurement of anyone's traffic — take the per-mode costs and apply your own -mix. +| takeover — the human drives | 10/10 | 4718ms | 14 | 142 KB | 10.7s | +| approval — the human decides | 10/10 | 4896ms | 1 | 25 KB | 5.3s | + +Medians over the completed runs. A denied approval counts as completed: the +decision was delivered, and the bench then loads the account page and requires +both the session and the absence of a receipt for that run's amount. `to +visible` includes the relay cold start (~3 s) that both modes pay. The relay +seconds include the human's time, so they are a floor — a real person takes +longer than this scripted one — and only the takeover pays that cost: an +approval injects nothing and sends one screenshot however long the human thinks. +The approval arm signs itself in with the shared secret, so only the takeover +arm is measured against a wall it cannot pass, and the 50/50 mix is the +harness's choice rather than a measurement of anyone's traffic: +[method and caveats](benchmarks/README.md). At N=30 the right-hand column is the worst observation, not a fitted p99 — we say what we measured. The input round trip sits on the network RTT floor from diff --git a/benchmarks/README.md b/benchmarks/README.md index 55c5d11..3378273 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -102,24 +102,42 @@ Aurora Bank instance, interleaved (takeover, approval, takeover, …), on | | completed | time to visible p50 / p75 | handoff p50 / p75 | frames | bytes | inputs | relay-sandbox s | |---|---|---|---|---|---|---|---| -| takeover | 10/10 | 4923 / 5809 ms | 6621 / 7011 ms | 14 | 142 KB | 8 | 11.0 | -| approval | 10/10 | 5089 / 5381 ms | 2091 / 2293 ms | 1 | 25 KB | 0 | 5.5 | +| takeover | 10/10 | 4718 / 4808 ms | 6927 / 7202 ms | 14 | 142 KB | 8 | 10.7 | +| approval | 10/10 | 4896 / 4977 ms | 2063 / 2084 ms | 1 | 25 KB | 0 | 5.3 | + +All per-handoff figures are medians over the runs that completed, and the two +time columns measure different spans: + +- `time to visible` is `raiseHand()` → the first frame arriving on the human's + socket. It is the same measurement as the latency bench, so the two modes are + comparable, and it includes the relay cold start both modes pay (3099 ms + takeover, 3039 ms approval, at p50). +- `handoff` is the wide event's `durationMs`: the relay being up → the handoff + settling. It **excludes** that cold start, which is why an approval's handoff + (2063 ms) is shorter than its time to visible (4896 ms). -All per-handoff figures are medians over the runs that completed. `time to -visible` is `raiseHand()` → the first frame arriving on the human's socket, the -same measurement as the latency bench, so the two modes are comparable and both -carry the same relay cold start (3211 ms takeover, 3068 ms approval, at p50). `frames` and `bytes` are what the agent put on the wire: an approval is one -screenshot, 25 KB of base64 payload, and it injects nothing into the page — -`inputsApplied` is 0 by construction, not by luck. `relay-sandbox s` is wall-clock from `raiseHand()` to -the promise settling, which covers creating the sandbox, the handoff and -destroying it: the closest thing to a bill. +screenshot, 25 KB of base64 payload, and it injects nothing into the page — so +`inputsApplied` is 0 by construction, not by luck. `relay-sandbox s` is +wall-clock from `raiseHand()` to the promise settling, which covers creating the +sandbox, the handoff and destroying it: the closest thing to a bill. + +That last column includes the human's occupancy, so read it as a floor and not +as a property of the two modes. Of the 5.4 s gap here, 5.0 s is the scripted +human tapping and typing through the takeover; the approval side is 0.0 s, +because the script answers the moment the screenshot lands. Both modes pay the +same cold start. Only the takeover pays for the person's time, and a real person +takes longer than a script — an approval does not, because the one frame is +already on the phone while they think. A denied approval counts as completed, and that is a deliberate choice: the -workflow reached a decision and the agent obeyed it. The bench asserts the money -did not move — for a denial it requires that no transfer receipt exists on the -page — so "completed" means the mechanism delivered an answer, not that the -answer was yes. +workflow reached a decision and the agent obeyed it. Obeying is checked rather +than assumed: after a denial the bench loads the account page — the only page +that renders a receipt — and requires the session banner to be there AND no +receipt for that run's amount. Both halves can fail, and the harness proves it +can fail by submitting anyway under a temporary fixture. So "completed" means +the mechanism delivered an answer the agent then honoured, not that the answer +was yes. What may be claimed from this, and nothing wider: on this workload, an approval costs one frame where a takeover costs a stream, and both modes delivered their @@ -135,8 +153,11 @@ measured — the interrupt is the transfer, which comes after. Only the takeover arm is barred from the secret, because there the wall is the whole test. The counting is load-bearing rather than decorative, and that is testable: -`MIXED_FAULT=invert-completed bun run bench:mixed` inverts the completion test, -and both modes must then read 0 of N. +`MIXED_FAULT=invert-completed bun run bench:mixed` inverts the page sensors — +the signed-in banner and the receipt — and both modes must then read 0 of N, +denials included. The switch deliberately does not flip the verdict at the end: +a verdict flip looks identical for a check that never reads the page, which is a +mistake this bench made once and now tests against. ## Reading the raw files diff --git a/benchmarks/mixed-workload.json b/benchmarks/mixed-workload.json index b557020..07a1bb7 100644 --- a/benchmarks/mixed-workload.json +++ b/benchmarks/mixed-workload.json @@ -1,6 +1,6 @@ { "meta": { - "date": "2026-09-02T19:04:30.067Z", + "date": "2026-09-02T19:39:18.817Z", "requestedN": 20, "workload": "N workflows against one Aurora Bank instance, interleaved takeover, approval, takeover, …", "interrupts": { @@ -9,17 +9,19 @@ }, "claim": "on this workload, each mode's cost per handoff and how many workflows completed", "notAClaim": "the 50/50 mix is this harness's choice, not a measurement of any fleet's traffic; multiply the per-mode costs by your own mix", - "deniedCountAsCompleted": "yes — a denied approval delivered a decision the agent obeyed, and the bench asserts no transfer was sent", + "deniedCountAsCompleted": "yes — a denied approval delivered a decision the agent obeyed; the bench then loads /account and requires the session banner AND the absence of the receipt for that run's amount", "approvalSetup": "the approval arm signs itself in with the shared secret; that sign-in is setup, and the interrupt under measurement is the transfer", "denyEvery": 4, "interleaved": true, "fault": null, + "faultActsOn": "the page sensors (signed-in banner, transfer receipt), not the verdict they feed", + "supersededAttempts": 0, "abortReason": null, "browserRelaunches": 1, "concurrencyWaits": 0, "testAppRestarts": 0, - "totalMs": 365242, - "relaySandboxSecondsAllRuns": 169.365, + "totalMs": 370118, + "relaySandboxSecondsAllRuns": 163.274, "bunVersion": "1.4.0", "platform": "darwin-arm64", "measuredFrom": "Germany → default Solari endpoint (api.getsolari.com)", @@ -40,45 +42,45 @@ "neverReachedInterrupt": 0, "stuckToVisibleMs": { "n": 10, - "p50": 4923, - "p75": 5809, - "worst": 6492, - "total": 52495 + "p50": 4718, + "p75": 4808, + "worst": 5878, + "total": 48142 }, "handoffDurationMs": { "n": 10, - "p50": 6621, - "p75": 7011, - "worst": 8438, - "total": 69031 + "p50": 6927, + "p75": 7202, + "worst": 7385, + "total": 68161 }, "relayColdStartMs": { "n": 10, - "p50": 3211, - "p75": 3914, - "worst": 4723, - "total": 34984 + "p50": 3099, + "p75": 3164, + "worst": 4196, + "total": 31960 }, "humanActiveMs": { "n": 10, - "p50": 4593, - "p75": 5176, - "worst": 6564, - "total": 49377 + "p50": 4977, + "p75": 5404, + "worst": 5561, + "total": 49793 }, "framesSent": { "n": 10, "p50": 14, "p75": 14, - "worst": 16, - "total": 138 + "worst": 14, + "total": 134 }, "bytesSent": { "n": 10, - "p50": 145516, - "p75": 145584, - "worst": 166424, - "total": 1433784 + "p50": 145104, + "p75": 145536, + "worst": 145600, + "total": 1390572 }, "inputsApplied": { "n": 10, @@ -89,10 +91,10 @@ }, "relaySandboxSeconds": { "n": 10, - "p50": 10.955, - "p75": 12.06, - "worst": 12.435, - "total": 112.916 + "p50": 10.732, + "p75": 11.416, + "worst": 12.268, + "total": 109.686 } }, { @@ -105,31 +107,31 @@ "neverReachedInterrupt": 0, "stuckToVisibleMs": { "n": 10, - "p50": 5089, - "p75": 5381, - "worst": 6729, - "total": 52127 + "p50": 4896, + "p75": 4977, + "worst": 5396, + "total": 49565 }, "handoffDurationMs": { "n": 10, - "p50": 2091, - "p75": 2293, - "worst": 2626, - "total": 21734 + "p50": 2063, + "p75": 2084, + "worst": 2115, + "total": 20306 }, "relayColdStartMs": { "n": 10, - "p50": 3068, - "p75": 3231, - "worst": 4922, - "total": 32692 + "p50": 3039, + "p75": 3242, + "worst": 3634, + "total": 31230 }, "humanActiveMs": { "n": 10, "p50": 0, "p75": 1, "worst": 1, - "total": 4 + "total": 3 }, "framesSent": { "n": 10, @@ -154,10 +156,10 @@ }, "relaySandboxSeconds": { "n": 10, - "p50": 5.467, - "p75": 5.762, - "worst": 7.205, - "total": 56.449 + "p50": 5.29, + "p75": 5.361, + "worst": 5.862, + "total": 53.588 } } ], @@ -165,442 +167,502 @@ { "index": 1, "kind": "takeover", - "startedAt": "2026-09-02T18:58:30.161Z", + "startedAt": "2026-09-02T19:33:14.241Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3707, + "reachedInterruptMs": 3972, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 5214, - "handoffDurationMs": 6714, - "relayColdStartMs": 3340, + "stuckToVisibleMs": 4920, + "handoffDurationMs": 7385, + "relayColdStartMs": 3145, "framesSent": 14, "framesAtHuman": 13, - "bytesSent": 145584, + "bytesSent": 145536, "inputsApplied": 8, - "humanActiveMs": 4600, - "relaySandboxSeconds": 10.841, + "humanActiveMs": 5419, + "relaySandboxSeconds": 11.416, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 1680 + "browserAgeMs": 2011 }, { "index": 2, "kind": "approval", - "startedAt": "2026-09-02T18:58:46.893Z", + "startedAt": "2026-09-02T19:33:31.871Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8283, + "reachedInterruptMs": 8291, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 4535, - "handoffDurationMs": 1924, - "relayColdStartMs": 2937, + "stuckToVisibleMs": 4816, + "handoffDurationMs": 2095, + "relayColdStartMs": 2921, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25368, "inputsApplied": 0, "humanActiveMs": 0, - "relaySandboxSeconds": 5.058, + "relaySandboxSeconds": 5.219, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 18412 + "browserAgeMs": 19641 }, { "index": 3, "kind": "takeover", - "startedAt": "2026-09-02T18:59:04.620Z", + "startedAt": "2026-09-02T19:33:49.846Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3670, + "reachedInterruptMs": 3891, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 6492, - "handoffDurationMs": 6483, - "relayColdStartMs": 4723, - "framesSent": 14, - "framesAtHuman": 13, - "bytesSent": 145516, + "stuckToVisibleMs": 4808, + "handoffDurationMs": 6927, + "relayColdStartMs": 3124, + "framesSent": 12, + "framesAtHuman": 11, + "bytesSent": 124164, "inputsApplied": 8, - "humanActiveMs": 4464, - "relaySandboxSeconds": 12.06, + "humanActiveMs": 5045, + "relaySandboxSeconds": 12.149, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 36139 + "browserAgeMs": 37616 }, { "index": 4, "kind": "approval", - "startedAt": "2026-09-02T18:59:22.440Z", + "startedAt": "2026-09-02T19:34:08.197Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8250, + "reachedInterruptMs": 9022, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 5381, - "handoffDurationMs": 2286, - "relayColdStartMs": 3284, + "stuckToVisibleMs": 4911, + "handoffDurationMs": 2084, + "relayColdStartMs": 3020, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25364, "inputsApplied": 0, - "humanActiveMs": 1, - "relaySandboxSeconds": 5.762, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.294, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 53959 + "browserAgeMs": 55967 }, { "index": 5, "kind": "takeover", - "startedAt": "2026-09-02T18:59:40.579Z", + "startedAt": "2026-09-02T19:34:26.792Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3737, + "reachedInterruptMs": 3826, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 4917, - "handoffDurationMs": 6426, - "relayColdStartMs": 3211, + "stuckToVisibleMs": 4787, + "handoffDurationMs": 7308, + "relayColdStartMs": 3224, "framesSent": 13, - "framesAtHuman": 11, - "bytesSent": 134884, + "framesAtHuman": 13, + "bytesSent": 134528, "inputsApplied": 8, - "humanActiveMs": 4457, - "relaySandboxSeconds": 10.452, + "humanActiveMs": 5561, + "relaySandboxSeconds": 11.405, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 72098 + "browserAgeMs": 74562 }, { "index": 6, "kind": "approval", - "startedAt": "2026-09-02T18:59:56.895Z", + "startedAt": "2026-09-02T19:34:44.198Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8403, + "reachedInterruptMs": 8647, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 6729, - "handoffDurationMs": 2091, - "relayColdStartMs": 4922, + "stuckToVisibleMs": 4843, + "handoffDurationMs": 1952, + "relayColdStartMs": 3085, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25372, "inputsApplied": 0, - "humanActiveMs": 1, - "relaySandboxSeconds": 7.205, + "humanActiveMs": 0, + "relaySandboxSeconds": 5.229, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 88414 + "browserAgeMs": 91968 }, { "index": 7, "kind": "takeover", - "startedAt": "2026-09-02T19:00:16.553Z", + "startedAt": "2026-09-02T19:35:02.479Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3654, + "reachedInterruptMs": 4720, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 4698, - "handoffDurationMs": 6474, - "relayColdStartMs": 3035, + "stuckToVisibleMs": 5878, + "handoffDurationMs": 7185, + "relayColdStartMs": 4196, "framesSent": 14, - "framesAtHuman": 13, - "bytesSent": 145572, + "framesAtHuman": 14, + "bytesSent": 145456, "inputsApplied": 8, - "humanActiveMs": 4585, - "relaySandboxSeconds": 10.38, + "humanActiveMs": 5305, + "relaySandboxSeconds": 12.268, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 108072 + "browserAgeMs": 110249 }, { "index": 8, "kind": "approval", - "startedAt": "2026-09-02T19:00:32.689Z", + "startedAt": "2026-09-02T19:35:21.817Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8277, + "reachedInterruptMs": 9190, "decision": "deny", "handoffOutcome": "denied", - "stuckToVisibleMs": 4908, - "handoffDurationMs": 2127, - "relayColdStartMs": 3005, + "stuckToVisibleMs": 4803, + "handoffDurationMs": 1956, + "relayColdStartMs": 3039, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25368, "inputsApplied": 0, "humanActiveMs": 0, - "relaySandboxSeconds": 5.357, + "relaySandboxSeconds": 5.25, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 124208 + "browserAgeMs": 129587 }, { "index": 9, "kind": "takeover", - "startedAt": "2026-09-02T19:00:49.264Z", + "startedAt": "2026-09-02T19:35:40.694Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3889, + "reachedInterruptMs": 3884, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 4923, - "handoffDurationMs": 6556, - "relayColdStartMs": 3147, - "framesSent": 13, - "framesAtHuman": 12, - "bytesSent": 134892, + "stuckToVisibleMs": 4618, + "handoffDurationMs": 7202, + "relayColdStartMs": 3031, + "framesSent": 12, + "framesAtHuman": 11, + "bytesSent": 124188, "inputsApplied": 8, - "humanActiveMs": 4593, - "relaySandboxSeconds": 10.544, + "humanActiveMs": 5404, + "relaySandboxSeconds": 11.11, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 140783 + "browserAgeMs": 148464 }, { "index": 10, "kind": "approval", - "startedAt": "2026-09-02T19:01:05.832Z", + "startedAt": "2026-09-02T19:35:57.826Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8262, + "reachedInterruptMs": 9224, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 5242, - "handoffDurationMs": 2293, - "relayColdStartMs": 3144, + "stuckToVisibleMs": 4900, + "handoffDurationMs": 2067, + "relayColdStartMs": 3048, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25360, "inputsApplied": 0, "humanActiveMs": 0, - "relaySandboxSeconds": 5.646, + "relaySandboxSeconds": 5.307, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 157351 + "browserAgeMs": 165599 }, { "index": 11, "kind": "takeover", - "startedAt": "2026-09-02T19:01:23.861Z", + "startedAt": "2026-09-02T19:36:28.753Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3708, + "reachedInterruptMs": 3628, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 5844, - "handoffDurationMs": 6621, - "relayColdStartMs": 3919, + "stuckToVisibleMs": 4748, + "handoffDurationMs": 6414, + "relayColdStartMs": 3164, "framesSent": 14, "framesAtHuman": 13, - "bytesSent": 145548, + "bytesSent": 145600, "inputsApplied": 8, - "humanActiveMs": 4506, - "relaySandboxSeconds": 11.318, + "humanActiveMs": 4598, + "relaySandboxSeconds": 10.447, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 175380 + "browserAgeMs": 11643 }, { "index": 12, "kind": "approval", - "startedAt": "2026-09-02T19:01:43.636Z", + "startedAt": "2026-09-02T19:36:44.858Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 9288, + "reachedInterruptMs": 8476, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 5461, - "handoffDurationMs": 2626, - "relayColdStartMs": 3140, + "stuckToVisibleMs": 4819, + "handoffDurationMs": 2071, + "relayColdStartMs": 2936, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25380, "inputsApplied": 0, "humanActiveMs": 1, - "relaySandboxSeconds": 5.955, + "relaySandboxSeconds": 5.198, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 2103 + "browserAgeMs": 27748 }, { "index": 13, "kind": "takeover", - "startedAt": "2026-09-02T19:02:04.038Z", + "startedAt": "2026-09-02T19:37:02.692Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 3972, + "reachedInterruptMs": 3769, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 4597, - "handoffDurationMs": 8438, - "relayColdStartMs": 2924, - "framesSent": 16, - "framesAtHuman": 15, - "bytesSent": 166424, + "stuckToVisibleMs": 4619, + "handoffDurationMs": 6290, + "relayColdStartMs": 3048, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145548, "inputsApplied": 8, - "humanActiveMs": 6564, - "relaySandboxSeconds": 12.435, + "humanActiveMs": 4462, + "relaySandboxSeconds": 10.108, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 22505 + "browserAgeMs": 45582 }, { "index": 14, "kind": "approval", - "startedAt": "2026-09-02T19:02:22.799Z", + "startedAt": "2026-09-02T19:37:18.712Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 9413, + "reachedInterruptMs": 8555, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 4905, - "handoffDurationMs": 2063, - "relayColdStartMs": 3027, + "stuckToVisibleMs": 5204, + "handoffDurationMs": 2115, + "relayColdStartMs": 3273, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25360, "inputsApplied": 0, "humanActiveMs": 1, - "relaySandboxSeconds": 5.329, + "relaySandboxSeconds": 5.578, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 41266 + "browserAgeMs": 61602 }, { "index": 15, "kind": "takeover", - "startedAt": "2026-09-02T19:02:41.584Z", + "startedAt": "2026-09-02T19:37:36.970Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 4233, + "reachedInterruptMs": 3718, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 5809, - "handoffDurationMs": 7448, - "relayColdStartMs": 3914, - "framesSent": 15, - "framesAtHuman": 15, - "bytesSent": 156160, + "stuckToVisibleMs": 4392, + "handoffDurationMs": 6290, + "relayColdStartMs": 2911, + "framesSent": 13, + "framesAtHuman": 12, + "bytesSent": 134988, "inputsApplied": 8, - "humanActiveMs": 5337, - "relaySandboxSeconds": 12.422, + "humanActiveMs": 4547, + "relaySandboxSeconds": 9.962, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 60051 + "browserAgeMs": 79860 }, { "index": 16, "kind": "approval", - "startedAt": "2026-09-02T19:03:00.454Z", + "startedAt": "2026-09-02T19:37:52.799Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8994, + "reachedInterruptMs": 8240, "decision": "deny", "handoffOutcome": "denied", - "stuckToVisibleMs": 5163, - "handoffDurationMs": 2295, - "relayColdStartMs": 3068, + "stuckToVisibleMs": 4977, + "handoffDurationMs": 1926, + "relayColdStartMs": 3242, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25368, "inputsApplied": 0, "humanActiveMs": 0, - "relaySandboxSeconds": 5.561, + "relaySandboxSeconds": 5.361, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 78921 + "browserAgeMs": 95689 }, { "index": 17, "kind": "takeover", - "startedAt": "2026-09-02T19:03:18.096Z", + "startedAt": "2026-09-02T19:38:09.793Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 4203, + "reachedInterruptMs": 3792, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 4749, - "handoffDurationMs": 7011, - "relayColdStartMs": 3103, - "framesSent": 12, - "framesAtHuman": 12, - "bytesSent": 124324, + "stuckToVisibleMs": 4654, + "handoffDurationMs": 6217, + "relayColdStartMs": 3099, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145460, "inputsApplied": 8, - "humanActiveMs": 5176, - "relaySandboxSeconds": 10.955, + "humanActiveMs": 4475, + "relaySandboxSeconds": 10.089, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 96563 + "browserAgeMs": 112683 }, { "index": 18, "kind": "approval", - "startedAt": "2026-09-02T19:03:35.375Z", + "startedAt": "2026-09-02T19:38:25.832Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8609, + "reachedInterruptMs": 8053, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 4714, - "handoffDurationMs": 1981, - "relayColdStartMs": 2934, + "stuckToVisibleMs": 5396, + "handoffDurationMs": 1977, + "relayColdStartMs": 3634, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25384, "inputsApplied": 0, "humanActiveMs": 0, - "relaySandboxSeconds": 5.109, + "relaySandboxSeconds": 5.862, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 113842 + "browserAgeMs": 128722 }, { "index": 19, "kind": "takeover", - "startedAt": "2026-09-02T19:03:53.293Z", + "startedAt": "2026-09-02T19:38:43.975Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 4106, + "reachedInterruptMs": 3612, "decision": null, "handoffOutcome": "resolved", - "stuckToVisibleMs": 5252, - "handoffDurationMs": 6860, - "relayColdStartMs": 3668, - "framesSent": 13, - "framesAtHuman": 11, - "bytesSent": 134880, + "stuckToVisibleMs": 4718, + "handoffDurationMs": 6943, + "relayColdStartMs": 3018, + "framesSent": 14, + "framesAtHuman": 13, + "bytesSent": 145104, "inputsApplied": 8, - "humanActiveMs": 5095, - "relaySandboxSeconds": 11.509, + "humanActiveMs": 4977, + "relaySandboxSeconds": 10.732, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 131760 + "browserAgeMs": 146865 }, { "index": 20, "kind": "approval", - "startedAt": "2026-09-02T19:04:11.167Z", + "startedAt": "2026-09-02T19:39:00.435Z", "completed": true, "reachedInterrupt": true, - "reachedInterruptMs": 8721, + "reachedInterruptMs": 8332, "decision": "approve", "handoffOutcome": "approved", - "stuckToVisibleMs": 5089, - "handoffDurationMs": 2048, - "relayColdStartMs": 3231, + "stuckToVisibleMs": 4896, + "handoffDurationMs": 2063, + "relayColdStartMs": 3032, "framesSent": 1, "framesAtHuman": 1, "bytesSent": 25380, "inputsApplied": 0, - "humanActiveMs": 0, - "relaySandboxSeconds": 5.467, + "humanActiveMs": 1, + "relaySandboxSeconds": 5.29, "error": null, + "errorCode": null, + "superseded": null, + "attempt": 1, "concurrencyWaits": 0, - "browserAgeMs": 149634 + "browserAgeMs": 163325 } ] }