From 6f4b46e6e3c0e27d56f28a211d4ce84c1f80c8f2 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 11:24:26 -0700 Subject: [PATCH 1/7] feat(cli): mirror a local run onto the Cloud dashboard, by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run Cloud launched is watchable because something inside the sandbox reads its journal every few seconds and pushes what it finds. Nothing did that for a run started in a terminal, so the same flow, the same journal and the same evidence had no page to look at — `flows status --cloud` did not know it existed, because no run row did. `flows run` and `flows resume` now register the run with Cloud (`POST /api/v1/workflows/local-run`), take a credential scoped to that one run, and report through exactly the endpoints a sandbox reports through: the live step view while it runs, the final step rows, the per-step transcripts assembled in Cloud's own `relayflow.attempt` vocabulary, the runner log, and the terminal status. The control plane cannot tell a mirrored run from a sandboxed one, so no route needed a new case. Three properties this is built around: - **It reads only this run's journals.** The sandbox reporter scans its whole data directory because a sandbox holds one run; `~/.relayflowd` holds every run you have ever started, and in a shared checkout, other people's. So the journal set is derived: the root, plus the children the root's own authored step index names. - **It cannot fail a run.** Every push collapses to a boolean at the transport, each poll is bounded by its own deadline, the finish is bounded, and a journal that cannot be read keeps its cached view rather than publishing an emptier one. A Cloud outage costs a local run its page and nothing else. - **The deployment is pinned by the registration.** `cloudConnection` falls back to the production default once an explicit token is supplied, and every call after registration supplies one — so without the pin a CLI signed in to a staging deployment would send that deployment's run token to agentrelay.com. On by default, conditional on a Cloud credential already resolving. A machine that has never signed in prints one line and runs exactly as before: a local run that joins no workspace is not a defect (RFC-0001 settled decision 7). `--no-cloud-mirror` opts out for a run, `FLOWS_CLOUD_MIRROR=0` for a shell. The flag is refused with `--cloud` and on `check`, where it would describe nothing, rather than accepted and ignored. The terminal callback is last, and that ordering is load-bearing: Cloud revokes the run's credential at the terminal transition, so the transcripts and the final rows have to land before it. No credential is written to disk. A resume registers its own row — Cloud refuses to move a terminal run back to `running`, and its own v2 resume is likewise a new attempt — and mirrors the kernel spec its journal recorded, since the flow file may have been edited or deleted since. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 6 + docs/CLOUD.md | 63 ++ packages/sdk/src/cli-commands.ts | 1 + packages/sdk/src/cli.ts | 64 +- packages/sdk/src/cli/cloud-mirror-session.ts | 210 +++++++ packages/sdk/src/cloud-mirror-step.ts | 567 ++++++++++++++++++ packages/sdk/src/cloud-mirror-transport.ts | 238 ++++++++ packages/sdk/src/cloud-mirror.ts | 469 +++++++++++++++ .../sdk/tests/cli-cloud-mirror-flag.test.ts | 38 ++ .../sdk/tests/cloud-mirror-session.test.ts | 172 ++++++ packages/sdk/tests/cloud-mirror-step.test.ts | 262 ++++++++ .../sdk/tests/cloud-mirror-transport.test.ts | 144 +++++ packages/sdk/tests/cloud-mirror.test.ts | 293 +++++++++ packages/sdk/tests/relay-cli-surface.test.ts | 4 +- 14 files changed, 2517 insertions(+), 14 deletions(-) create mode 100644 packages/sdk/src/cli/cloud-mirror-session.ts create mode 100644 packages/sdk/src/cloud-mirror-step.ts create mode 100644 packages/sdk/src/cloud-mirror-transport.ts create mode 100644 packages/sdk/src/cloud-mirror.ts create mode 100644 packages/sdk/tests/cli-cloud-mirror-flag.test.ts create mode 100644 packages/sdk/tests/cloud-mirror-session.test.ts create mode 100644 packages/sdk/tests/cloud-mirror-step.test.ts create mode 100644 packages/sdk/tests/cloud-mirror-transport.test.ts create mode 100644 packages/sdk/tests/cloud-mirror.test.ts diff --git a/AGENTS.md b/AGENTS.md index e1687ee70..ef699a887 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,12 @@ truth*: the journal is the record, and the workspace is one view onto it. A run that never joins a workspace is harder to watch; it is not less durable, less resumable, or less correct. +A local run also mirrors itself onto the Cloud dashboard by default whenever a +Cloud login resolves, and prints the page's URL (`docs/CLOUD.md`, "Local runs +on the dashboard"). `--no-cloud-mirror`, or `FLOWS_CLOUD_MIRROR=0`, turns that +off. It is the same kind of projection as the observer link — watchability, not +authority — and it cannot fail a run. + This paragraph previously said every run MUST join the canonical workspace and that anything else was a defect. That predates decision 7 and outlived it — it caused a review to flag a local demo as a P1 defect when the demo was fine. diff --git a/docs/CLOUD.md b/docs/CLOUD.md index d0418c1d0..00ac3f8a4 100644 --- a/docs/CLOUD.md +++ b/docs/CLOUD.md @@ -132,6 +132,69 @@ A synced run and a Cloud repository grant are mutually exclusive on the server: `--sync-code` is the local-driven development loop, and webhook-triggered deployments keep cloning through the grant. +## Local runs on the dashboard + +A run started in a terminal mirrors itself onto the same dashboard page a +hosted run gets, and does so by default: + +```sh +flows run review.flow.ts --input '{"pr":7}' +# Dashboard: https://agentrelay.com/cloud/dashboard/workflow//runner +``` + +Nothing about the run changes. It executes locally, against the local daemon, +under your own credentials; the journal is still the record. What is new is a +reader beside it that polls this run's journals every ten seconds and pushes +what it finds — the same live step view, the same final step rows, the same +per-step transcripts and the same terminal status a sandbox reports. The run +row is marked `dispatchType: "local"`, so the run page says it ran on your +machine rather than promising a sandbox that is never coming. + +Opting out: + +```sh +flows run --no-cloud-mirror flow.yaml # this run only +FLOWS_CLOUD_MIRROR=0 flows run flow.yaml # this shell: a CI job, a shared checkout +``` + +`--no-cloud-mirror` is refused with `--cloud` (which *is* the hosted run) and +on `check` (which starts nothing), rather than being accepted and ignored. + +The mirror is on by default *when a Cloud credential resolves* — the same +credential every other hosted verb uses (see [Credentials](#credentials)). A +machine that has never signed in prints one line saying the run stays local +and runs exactly as before. A local run that joins no workspace is not a +defect: RFC-0001 settled decision 7 makes the projection a view, not an +authority. + +What it does and does not do: + +- **Registers before it reports.** `POST /api/v1/workflows/local-run` creates + the run row and returns a credential scoped to that one run. Registration + happens once the run id exists, so a flow `flows run` refuses at check time + never reaches Cloud at all. +- **Reads only this run's journals.** The root, plus the child journals the + root's own authored-step index names. A shared `~/.relayflowd` holding other + people's runs contributes nothing. +- **Cannot fail a run.** Every push collapses to a boolean; each poll is + bounded by its own deadline; the whole finish is bounded. A Cloud outage + costs a local run its dashboard page and nothing else. +- **Publishes what a hosted run publishes, redacted the same way.** Free text + goes through `flows status`'s redactor before it is bounded, and identifier + fields are normalized into the shape Cloud's parser accepts. +- **Does not make Cloud the authority.** Cancel is refused for a local run: + Cloud mirrors it and does not control it, and a cancel button that stopped + the *reporting* while the flow kept running would be a cancellation that did + not happen. Stop it where it is running. +- **One dashboard row per invocation.** A mirrored run goes terminal on Cloud + when the CLI exits, and Cloud refuses to move a terminal run back to + `running`, so `flows resume` registers its own row — the same shape Cloud's + own v2 resume already has. A resume mirrors the kernel spec its journal + recorded, since the flow file may have been edited or deleted since. + +No credential is written to disk between invocations: the run token lives only +for the process that holds it. + ## Reading a hosted run Three read-only verbs answer "what did that run do" from the Cloud API, so an diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index f9104439f..5a164adfd 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -96,6 +96,7 @@ const LOCAL_EXECUTION_OPTIONS = [ }, { flags: '--no-spawn', description: 'Require a running relayflowd rather than starting one' }, { flags: '--no-observer-link', description: 'Do not mint an observer link for this run' }, + { flags: '--no-cloud-mirror', description: 'Do not mirror this run onto the Cloud dashboard (also FLOWS_CLOUD_MIRROR=0)' }, { flags: '--allow-human-influenced', description: 'Proceed even though the run carries human-influenced state', diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 181b593a9..ac1c6dc74 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -9,6 +9,9 @@ import type { CliModelSource } from './cli-adapter.js'; import { renderProgress, type ProgressEvent } from './progress.js'; import type { JournalEvent } from './journal-reader.js'; import { createObserverSession } from './cli/observer-session.js'; +import { + cloudMirrorEnabled, createCloudMirrorSession, mirrorSourceFromJournal, mirrorSourceFromPath, +} from './cli/cloud-mirror-session.js'; import { realpathSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; import { @@ -95,8 +98,8 @@ export type ParsedArgs = | { command: 'schedules'; json: boolean } | { command: 'unschedule'; scheduleId: string; json: boolean } | { command: 'check'; json: boolean; watch: boolean; value: string } - | { command: 'run'; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; allowHumanInfluenced: boolean; value: string } - | { command: 'resume'; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'run'; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; noCloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'resume'; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; noCloudMirror: boolean; allowHumanInfluenced: boolean; value: string } | { command: 'answer'; dataDir: string; json: boolean; spawn: boolean; note: string | undefined; by: string | undefined; runId: string; waitId: string; answer: boolean } | RunsArgs | LogsArgs @@ -126,13 +129,13 @@ const USAGE = [ 'flows run @sha256: [--bucket ] [--data-dir ] [--json]', 'flows check [--watch] [--json] ', 'flows serve-webhook --data-dir --port

[--allow [,]]', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--data-dir

] [--local-agent [--agent-capacity ]] [--reuse-from ] ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] [--reuse-from ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] --input ', 'flows sync [--json] [--dry-run] [--dir ] ', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--data-dir ] [--local-agent [--agent-capacity ]] --input ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] --input ', 'flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] ', - 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--data-dir ] [--local-agent [--agent-capacity ]] ', + 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] ', 'flows answer [--json] [--no-spawn] [--data-dir ] [--note ] [--by ] ', 'flows replay [--allow-human-influenced] [--json] [--data-dir ] [--at ]', 'flows status [--json] [--data-dir ] [--tail ] []', @@ -315,10 +318,28 @@ export async function runCli( // effect of an invocation that is about to be refused for bad input. const startedSteps = new Map(); const observer = parsed.noObserverLink ? undefined : createObserverSession(parsed.command, io); + // What the run printed about itself, kept so the mirror can upload it as the + // run's `runner.log` — the object the dashboard's log pane reads. Bounded by + // the mirror before it is sent; kept whole here because the same lines are + // what a reader sees on stderr. + const runnerLog: string[] = []; + const mirror = parsed.noCloudMirror || !cloudMirrorEnabled(process.env) + ? undefined + : createCloudMirrorSession({ + source: parsed.command === 'run' + ? mirrorSourceFromPath(parsed.value, parsed.input) + : mirrorSourceFromJournal(parsed.dataDir), + dataDir: parsed.dataDir, + log: () => runnerLog, + }, io); const showProgress = (event: ProgressEvent): void => { if (event.type === 'step.started') startedSteps.set(event.stepId, performance.now()); - if (!parsed.json) for (const line of renderProgress([event])) io.stderr(line); + for (const line of renderProgress([event])) { + runnerLog.push(line); + if (!parsed.json) io.stderr(line); + } observer?.onProgress(event); + mirror?.onProgress(event); }; const lifecycle = { ...(parsed.command === 'run' ? { bucket: parsed.bucket } : {}), @@ -328,9 +349,15 @@ export async function runCli( localAgent: parsed.localAgent, ...(parsed.agentCapacity === undefined ? {} : { agentCapacity: parsed.agentCapacity }), onProgress: showProgress, - ...(observer === undefined ? {} : { - onJournalEntry: (entry: JournalEvent) => observer.onJournalEntry(entry), - onRunStarted: (run: { runId: string; flow: string; resumed?: boolean }) => observer.onRunStarted(run), + ...(observer === undefined && mirror === undefined ? {} : { + onJournalEntry: (entry: JournalEvent) => { + observer?.onJournalEntry(entry); + mirror?.onJournalEntry(entry); + }, + onRunStarted: (run: { runId: string; flow: string; resumed?: boolean }) => { + observer?.onRunStarted(run); + mirror?.onRunStarted(run); + }, }), onWait: (progress: RunProgress) => { emitWait(progress, io); @@ -350,6 +377,9 @@ export async function runCli( // does. `finish` drains the projection and settles the mint, both bounded; // it never rejects (see `createObserverSession`). const observerMint = observer?.finish(execution.report); + // Bounded by the mirror itself, and it never rejects: a run's exit code has + // never waited on Cloud and does not start now. + await mirror?.finish(execution.report); // In `--json` mode the report is a single machine-readable object that // MUST carry `observerUrl` when one is available, so a consumer sees one // authoritative signal. That justifies blocking up to `MINT_TIMEOUT_MS` @@ -593,6 +623,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { let sawDataDir = false; let spawn = true; let noObserverLink = false; + let noCloudMirror = false; let input: string | undefined; let sawInput = false; let reuseFromRunId: string | undefined; @@ -651,6 +682,14 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { noObserverLink = true; continue; } + if (argument === '--no-cloud-mirror') { + // Only meaningful where a local run exists to mirror. Refused on `check` + // (which starts nothing) and, below, on `--cloud` (which IS the hosted + // run), so the flag never silently no-ops. + if (command === 'check' || noCloudMirror) return undefined; + noCloudMirror = true; + continue; + } if (argument === '--data-dir') { const value = args[index + 1]; if (command === 'check' || sawDataDir || value === undefined || value.startsWith('-')) return undefined; @@ -694,7 +733,8 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { // observer-link opt-out -- describes nothing there and is refused rather // than ignored. `--input` is the authored body's argument and travels with // the source, so it is accepted exactly where a local run accepts it. - if (allowHumanInfluenced || sawDataDir || !spawn || localAgent || noObserverLink || reuseFromRunId !== undefined) return undefined; + if (allowHumanInfluenced || sawDataDir || !spawn || localAgent || noObserverLink || noCloudMirror + || reuseFromRunId !== undefined) return undefined; if (sawInput && !isAuthoredFlowPath(positionals[0]!)) return undefined; return { command: 'cloud-run', value: positionals[0]!, json, wait, input, syncCode, noConnect }; } @@ -705,8 +745,8 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { return command === 'check' ? { command, json, watch, value: positionals[0]! } : command === 'run' - ? { command, bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, allowHumanInfluenced, value: positionals[0]! } - : { command, localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, allowHumanInfluenced, value: positionals[0]! }; + ? { command, bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, noCloudMirror, allowHumanInfluenced, value: positionals[0]! } + : { command, localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, noCloudMirror, allowHumanInfluenced, value: positionals[0]! }; } /** diff --git a/packages/sdk/src/cli/cloud-mirror-session.ts b/packages/sdk/src/cli/cloud-mirror-session.ts new file mode 100644 index 000000000..7be2aca8a --- /dev/null +++ b/packages/sdk/src/cli/cloud-mirror-session.ts @@ -0,0 +1,210 @@ +// The Cloud half of `flows run` / `flows resume`: mirror this local run onto +// the dashboard, so a run started in a terminal is as watchable as one Cloud +// launched for you. +// +// On by default, and the default is conditional on exactly one thing: a Cloud +// credential this CLI can already resolve. A machine that has never run +// `agent-relay cloud login` has nothing to upload with, and refusing the run +// over that would be absurd — a local run that joins no workspace is not a +// defect (RFC-0001 settled decision 7; the journal is the record, and Cloud is +// one view onto it). So a missing login prints one line saying the run is +// local-only, and the run proceeds exactly as it always has. +// +// `--no-cloud-mirror` opts out, and `FLOWS_CLOUD_MIRROR=0` opts out for a +// whole shell — a CI job, a machine running someone else's flows, a checkout +// whose runs should not leave it. +// +// Best-effort throughout, like `observer-session.ts` beside it: registration +// failure is one labeled stderr line, and nothing here can change a run's +// exit code or its journal. + +import { readFile } from 'node:fs/promises'; +import type { CliIo } from '../cli.js'; +import { canonicalize } from '../canonical.js'; +import { CloudFlowError } from '../cloud-http.js'; +import { createRunMirror, type RunMirror } from '../cloud-mirror.js'; +import { MirrorClient, registerLocalRun, type MirrorRunSource } from '../cloud-mirror-transport.js'; +import { isAuthoredFlowPath, parseDirectInput } from '../direct-input.js'; +import { walkJournal } from '../journal-reader.js'; +import type { ProgressEvent } from '../progress.js'; +import type { RunReport } from './run.js'; + +/** `FLOWS_CLOUD_MIRROR=0|false|off` turns the mirror off for a whole shell. */ +export const MIRROR_ENV = 'FLOWS_CLOUD_MIRROR'; + +export interface CloudMirrorSession { + /** The root run is admitted: the mirror can start reading its journal. */ + onRunStarted(run: { runId: string }): void; + /** A journaled entry; the first one names the run for a declarative flow. */ + onJournalEntry(entry: { run_id: string }): void; + /** Step transitions, pushed as lifecycle events the run page's stream shows. */ + onProgress(event: ProgressEvent): void; + /** Publish the final report and the terminal status. Never rejects. */ + finish(report: RunReport): Promise; +} + +export interface CloudMirrorDeps { + register?: typeof registerLocalRun; + createMirror?: typeof createRunMirror; +} + +export interface CloudMirrorRequest { + /** + * The flow's source and input, exactly as `flows run --cloud` would send + * them. Resolved lazily, once the run id exists: a `resume` has no flow + * path at all and has to read what ran out of the journal. + */ + source: (runId: string) => Promise; + dataDir: string; + /** Lines the CLI has printed for this run, uploaded as the run's `runner.log`. */ + log: () => readonly string[]; +} + +/** True unless the operator turned the mirror off for this shell. */ +export function cloudMirrorEnabled(env: NodeJS.ProcessEnv): boolean { + const value = env[MIRROR_ENV]?.trim().toLowerCase(); + return value !== '0' && value !== 'false' && value !== 'off' && value !== 'no'; +} + +/** + * Open the session. Registration is deferred until the run id exists, so a + * refused flow never reaches Cloud at all: `flows run` on a spec that does not + * compile creates no dashboard row, the same as today. + */ +export function createCloudMirrorSession( + request: CloudMirrorRequest, + io: CliIo, + env: NodeJS.ProcessEnv = process.env, + deps: CloudMirrorDeps = {}, +): CloudMirrorSession { + const register = deps.register ?? registerLocalRun; + const createMirror = deps.createMirror ?? createRunMirror; + let opening: Promise | undefined; + + const open = (runId: string): Promise => opening ??= request.source(runId) + .then(register) + .then(registration => { + const mirror = createMirror({ + client: new MirrorClient(registration), + dataDir: request.dataDir, + env, + diagnostic: message => io.stderr(`[cloud] ${message}`), + }); + io.stderr(`Dashboard: ${mirror.runUrl}`); + mirror.start(runId); + return mirror; + }) + .catch((error: unknown) => { + io.stderr(`[cloud] ${mirrorRefusal(error)}`); + return undefined; + }); + + return { + onRunStarted(run) { + void open(run.runId); + }, + onJournalEntry(entry) { + void open(entry.run_id); + }, + onProgress(event) { + // Fire-and-forget, and only for transitions: a `step.running` tick + // arrives every second and would say nothing the snapshot does not. + if (opening === undefined) return; + if (event.type !== 'step.started' && event.type !== 'step.completed') return; + void opening.then(mirror => mirror?.event({ + eventType: `relayflow.${event.type}`, + stepName: event.stepId, + payload: { stepType: event.stepType }, + })); + }, + async finish(report) { + const mirror = opening === undefined ? undefined : await opening; + if (mirror === undefined) return; + await mirror.finish({ + status: report.completionReason === 'canceled' + ? 'cancelled' + : report.ok ? 'completed' : 'failed', + ...(report.completionReason === undefined ? {} : { completionReason: report.completionReason }), + ...(terminalError(report) === undefined ? {} : { error: terminalError(report)! }), + log: request.log(), + }); + }, + }; +} + +/** + * What `flows run` mirrors: the flow file's exact bytes, and — for an authored + * flow — the input it was invoked with. + * + * The same source `flows run --cloud` would submit, so a run mirrored from a + * terminal and the same run launched hosted are recorded identically. It is + * read here rather than taken from the compiled spec because the compiled + * form is not what the author wrote, and the run page shows source. + */ +export function mirrorSourceFromPath( + path: string, + inputArgument: string | undefined, +): () => Promise { + return async () => { + const workflow = await readFile(path, 'utf8'); + if (!isAuthoredFlowPath(path)) return { workflow, fileType: 'yaml' }; + return { workflow, fileType: 'ts', inputs: parseDirectInput(inputArgument) }; + }; +} + +/** + * What `flows resume` mirrors: the kernel spec the journal recorded at + * `run.spawned`, canonicalized. + * + * A resume names a run id, not a file — the flow it continues may have been + * edited or deleted since. The journal is the record of what actually ran, so + * that is what the mirror publishes, as canonical JSON (a YAML subset, the + * same representation `runInCloud` sends for a declarative flow). + * + * Each resume registers its own Cloud run, and that is deliberate rather than + * a shortcut: a mirrored run goes terminal on Cloud when the CLI exits, and + * Cloud refuses to move a terminal run back to `running`. Cloud's own v2 + * resume is likewise a new attempt row, so one dashboard row per invocation is + * the shape the control plane already has — and it costs no credential stored + * on disk between invocations. + */ +export function mirrorSourceFromJournal(dataDir: string): (runId: string) => Promise { + return async (runId) => { + for await (const event of walkJournal(runId, dataDir)) { + if (event.entry_type !== 'run.spawned') continue; + const payload = event.payload as { spec?: unknown } | null; + if (payload?.spec === undefined) break; + return { workflow: canonicalize(payload.spec), fileType: 'yaml' }; + } + throw new CloudFlowError('invalid_input', + `Run "${runId}" journals no spec to mirror; the run is unaffected.`); + }; +} + +/** + * Why the mirror is not running, in one line a reader can act on. + * + * A missing login is the ordinary case and reads as a fact plus the command + * that changes it — never as an error, because a local-only run is not one. + */ +function mirrorRefusal(error: unknown): string { + if (error instanceof CloudFlowError && error.code === 'configuration') { + return error.reason === 'auth_missing' + ? 'no Cloud login, so this run stays local. `agent-relay cloud login` puts future runs on the dashboard; ' + + `${MIRROR_ENV}=0 stops this line.` + : `this run stays local: ${error.message}`; + } + if (error instanceof CloudFlowError && error.status === 404) { + return 'this Cloud deployment does not accept local runs yet; the run is unaffected'; + } + return `could not register this run with Cloud, so it stays local (${ + error instanceof Error ? error.message : String(error)}); the run is unaffected`; +} + +/** The run's own failure text, bounded by the mirror's transport, or nothing. */ +function terminalError(report: RunReport): string | undefined { + if (report.ok) return undefined; + const diagnostic = report.diagnostics.find(entry => + 'severity' in entry && (entry.severity === 'failure' || entry.severity === 'refusal')); + return diagnostic?.message; +} diff --git a/packages/sdk/src/cloud-mirror-step.ts b/packages/sdk/src/cloud-mirror-step.ts new file mode 100644 index 000000000..70af173ee --- /dev/null +++ b/packages/sdk/src/cloud-mirror-step.ts @@ -0,0 +1,567 @@ +// One journal, folded into the two step shapes Cloud stores for a run. +// +// A hosted run's steps reach Cloud twice: as a *live snapshot* while it runs +// (`POST /runs//steps/snapshot`) and as the *final report* once it is over +// (`POST /runs//steps`). The sandbox derives both from `flows status`; +// this module derives them from the same journal fold `flows status` uses +// (`run-state.ts`), so a local run and a hosted one put the same facts in the +// same fields and the dashboard cannot tell which produced a row. +// +// Nothing here does I/O or reads a clock: the caller passes the events and +// `now_ms`, exactly as `foldRunState` does. That is what lets a test fold a +// hand-built journal and assert the exact bytes a push would carry. +// +// ## The bounds are the server's, restated +// +// Every cap below is the one Cloud's own parser enforces +// (`@cloud/core` `storage/step-snapshot.ts` and `storage/step-detail.ts`). +// They are restated rather than imported because this package cannot depend on +// the Cloud app — the same reason `cloud-transcript.ts` restates the +// dashboard's transcript vocabulary. A push that violates one is refused +// whole, so producing a bounded value here is not politeness: it is the +// difference between a run that appears on the dashboard and one that does +// not. +// +// ## Redaction +// +// Every dynamic string goes through `redact` before it is bounded, never +// after: clipping first can leave the head of a secret standing where the +// redactor would have replaced the whole value. Identifier-shaped fields are +// then normalized into the shape Cloud's parser accepts, because the redactor +// emits `[redacted:NAME]`, which no identifier pattern admits. + +import { redact } from './redact.js'; +import { foldRunState, type RunView, type StepView } from './run-state.js'; +import type { JournalEvent } from './journal-reader.js'; +import { AUTHORED_STEP_STREAM, foldAuthoredStepRecords } from './authored-step-index.js'; + +/** Steps one snapshot may carry; a longer run publishes its live tail. */ +export const SNAPSHOT_MAX_STEPS = 64; +/** Serialized snapshot envelope cap, against the real UTF-8 byte count. */ +export const SNAPSHOT_MAX_BYTES = 64 * 1024; +/** Artifact paths named per step; the rest are counted. */ +export const SNAPSHOT_ARTIFACTS_MAX = 5; +export const SNAPSHOT_ARTIFACT_MAX_CHARS = 200; +/** `stepName`, `journalRunId`, `stepType`, `completionReason`, gate, model. */ +export const IDENTIFIER_MAX_CHARS = 128; +/** Predecessors one live step lists. */ +export const SNAPSHOT_DEPENDS_ON_MAX = 32; +/** Predecessors a final row lists. */ +export const DEPENDS_ON_MAX_ENTRIES = 256; +/** The backing columns are PostgreSQL 32-bit integers. */ +export const MAX_INT32 = 2_147_483_647; +/** `workflow_steps.display_name`, in code points. */ +export const LABEL_MAX_CHARS = 120; +/** `workflow_steps.output_summary`. */ +export const OUTPUT_SUMMARY_MAX_CHARS = 1000; +/** `workflow_steps.error`. */ +export const ERROR_MAX_CHARS = 1024; +/** Serialized `workflow_steps.detail`. */ +export const DETAIL_MAX_BYTES = 16 * 1024; +/** Attempt rows kept on `detail.attempts`; older ones are counted, not listed. */ +export const DETAIL_MAX_ATTEMPTS = 20; +/** + * The one completion reason the kernel uses for success. Cloud keys a step's + * status off exactly this value, so the live derivation and the final row + * agree about what "completed" means. + */ +export const SUCCESS_COMPLETION_REASON = 'success'; +/** The authored root step spans the whole flow; its children carry the steps. */ +export const AUTHORED_ROOT_STEP = 'authored-root'; + +const IDENTIFIER = /^[A-Za-z0-9_.:/-]+$/u; +const IDENTIFIER_UNSAFE = /[^A-Za-z0-9_.:/-]+/gu; + +/** Live kernel states Cloud publishes. A step that has not begun has no honest rendering. */ +export type SnapshotState = 'running' | 'backoff' | 'waiting' | 'needs_human' | 'done'; + +export interface SnapshotStep { + stepName: string; + /** Child runs may repeat a step id, so identity is the pair. */ + journalRunId: string; + stepType: string; + state: SnapshotState; + attempt: number; + elapsedMs: number; + completionReason?: string; + gate?: { name: string; verdict: string }; + model?: string; + turns?: number; + toolCalls?: number; + costUsd?: number; + artifactCount?: number; + artifacts?: string[]; + label?: string; + dependsOn?: string[]; +} + +/** One row of the final report; the field names are `workflow_steps`'s own. */ +export interface FinalStep { + stepName: string; + stepType: string; + agent: string; + preset: string; + cli: string; + sandboxId: string; + startTime: string; + endTime: string; + durationMs: number; + exitCode: number; + outputSummary: string; + status: 'completed' | 'failed'; + completionReason: string; + retryCount: number; + model?: string; + tokensInput?: number; + tokensOutput?: number; + costUsd?: number; + error?: string; + detail?: Record; + detailTruncated?: true; + label?: string; + dependsOn?: string[]; +} + +/** One step's per-attempt transcript files, for the `/agent.log` upload. */ +export interface StepTranscriptRef { + stepName: string; + attempts: Array<{ attempt: number; path: string }>; +} + +/** What one journal contributes, keyed for the caller's cross-journal cache. */ +export interface MirroredJournal { + runId: string; + status: RunView['status']; + terminal: boolean; + steps: SnapshotStep[]; + finals: FinalStep[]; + transcripts: StepTranscriptRef[]; + /** Child journals this run admitted, from an authored root's step index. */ + children: string[]; + /** Authored graph hints by step id, from an authored root's step index. */ + hints: Map; +} + +type Payload = Record; + +function record(value: unknown): Payload | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Payload : undefined; +} + +function int32(value: unknown, floor = 0): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value)) return undefined; + const rounded = Math.round(value); + if (rounded < floor) return undefined; + return Math.min(rounded, MAX_INT32); +} + +/** + * Redact, then normalize, then clip — in that order. Clipping first could + * leave the head of a secret standing; normalizing first would let a + * multi-character replacement push the result back over the cap. + */ +export function identifier( + value: unknown, + env: NodeJS.ProcessEnv, + maxChars: number = IDENTIFIER_MAX_CHARS, +): string | undefined { + if (typeof value !== 'string' || value.length === 0) return undefined; + const clipped = redact(value, env).replace(IDENTIFIER_UNSAFE, '_').slice(0, Math.max(1, maxChars)); + return IDENTIFIER.test(clipped) ? clipped : undefined; +} + +/** Free text, redacted then bounded by code points. Empty becomes undefined. */ +export function text(value: unknown, env: NodeJS.ProcessEnv, maxChars: number): string | undefined { + if (typeof value !== 'string' || value.length === 0) return undefined; + const cleaned = redact(value, env); + const bounded = [...cleaned].slice(0, maxChars).join(''); + return bounded.length === 0 ? undefined : bounded; +} + +/** + * A step label as `workflow_steps.display_name` accepts it: control characters + * become spaces, whitespace runs collapse, the result is trimmed and clipped. + */ +export function label(value: unknown, env: NodeJS.ProcessEnv): string | undefined { + if (typeof value !== 'string') return undefined; + const normalized = redact(value, env) + .replace(/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]+/gu, ' ') + .replace(/\s+/gu, ' ') + .trim(); + if (normalized.length === 0) return undefined; + return [...normalized].slice(0, LABEL_MAX_CHARS).join('').trimEnd(); +} + +function snapshotState(state: StepView['state']): SnapshotState | undefined { + // `pending` and `runnable` are states, but not publishable ones: a step that + // has not begun is neither completed nor running. + return state === 'pending' || state === 'runnable' ? undefined : state; +} + +/** Cloud's own vocabulary for a live step, derived rather than carried. */ +export function snapshotStepStatus(step: Pick): +'running' | 'completed' | 'failed' { + if (step.state !== 'done') return 'running'; + return step.completionReason === SUCCESS_COMPLETION_REASON ? 'completed' : 'failed'; +} + +function snapshotStep(journalRunId: string, step: StepView, env: NodeJS.ProcessEnv): SnapshotStep | undefined { + const state = snapshotState(step.state); + if (state === undefined) return undefined; + const stepName = identifier(step.id, env); + const journal = identifier(journalRunId, env); + const stepType = identifier(step.type, env); + if (stepName === undefined || journal === undefined || stepType === undefined) return undefined; + const transcript = step.last_attempt?.transcript ?? null; + const verification = step.last_attempt?.verification ?? null; + const gateName = verification === null ? undefined : identifier(verification.gate, env); + const gateVerdict = verification === null ? undefined : identifier(verification.verdict, env); + const paths = step.artifacts.paths + .slice(0, SNAPSHOT_ARTIFACTS_MAX) + .map(path => identifier(path, env, SNAPSHOT_ARTIFACT_MAX_CHARS)) + .filter((path): path is string => path !== undefined); + const completionReason = identifier(step.completion_reason ?? step.last_attempt?.completion_reason, env); + const cost = transcript?.total_cost_usd; + return { + stepName, + journalRunId: journal, + stepType, + state, + attempt: int32(step.attempt) ?? 0, + elapsedMs: int32(step.elapsed_ms) ?? 0, + ...(completionReason === undefined ? {} : { completionReason }), + ...(gateName === undefined || gateVerdict === undefined ? {} : { gate: { name: gateName, verdict: gateVerdict } }), + ...(identifier(transcript?.model, env) === undefined ? {} : { model: identifier(transcript?.model, env)! }), + ...(int32(transcript?.num_turns) === undefined ? {} : { turns: int32(transcript?.num_turns)! }), + ...(int32(transcript?.tool_calls) === undefined ? {} : { toolCalls: int32(transcript?.tool_calls)! }), + ...(typeof cost === 'number' && Number.isFinite(cost) && cost >= 0 ? { costUsd: cost } : {}), + ...(step.artifacts.journaled ? { artifactCount: int32(step.artifacts.paths.length) ?? 0 } : {}), + ...(paths.length === 0 ? {} : { artifacts: paths }), + }; +} + +function isoAt(ms: number | null | undefined): string { + if (typeof ms !== 'number' || !Number.isFinite(ms)) return ''; + const at = new Date(ms); + return Number.isNaN(at.getTime()) ? '' : at.toISOString(); +} + +/** + * The bounded `workflow_steps.detail` for one step: the attempt's journaled + * transcript digest, plus the attempt roster. The digest is already redacted + * and bounded where the worker wrote it (`agent-transcript.ts`); it is + * re-bounded here because this process, not that one, is what Cloud's parser + * will judge. + */ +function stepDetail( + attempts: AttemptRecord[], + env: NodeJS.ProcessEnv, +): { detail?: Record; truncated?: true } { + const last = attempts.at(-1); + const digest = record(record(last?.trajectoryTail)?.['transcript']); + const roster = attempts.slice(-DETAIL_MAX_ATTEMPTS).map(entry => ({ + attempt: entry.attempt, + atMs: entry.atMs, + completionReason: identifier(entry.completionReason, env) ?? 'unknown', + ...(identifier(entry.disposition, env) === undefined ? {} : { disposition: identifier(entry.disposition, env)! }), + ...(entry.transcriptBytes === undefined ? {} : { transcriptBytes: entry.transcriptBytes }), + })); + if (digest === undefined && roster.length <= 1) return {}; + // The digest carries a sandbox-free copy of the transcript's facts. `file` + // names a path on *this* machine, which is neither useful to a reader of the + // dashboard nor something to publish; the byte counts beside it are. + const file = record(digest?.['file']); + const detail: Record = { + ...(digest === undefined ? {} : { + transcript: { + ...digest, + ...(file === undefined ? {} : { + file: Object.fromEntries(Object.entries(file).filter(([key]) => key !== 'path')), + }), + }, + }), + ...(roster.length === 0 ? {} : { attempts: roster }), + ...(attempts.length > roster.length ? { attemptsOmitted: attempts.length - roster.length } : {}), + }; + if (Buffer.byteLength(JSON.stringify(detail), 'utf8') <= DETAIL_MAX_BYTES) return { detail }; + // Over the cap: drop the transcript's free text and its tool roster — the + // largest fields by far — and say the detail was cut rather than dropping it + // whole, which would take the attempt history with it. + const transcript = record(detail['transcript']); + const reduced: Record = { + ...(transcript === undefined ? {} : { + transcript: Object.fromEntries( + Object.entries(transcript).filter(([key]) => key === 'attempt' || key === 'exit_code' + || key === 'file' || key === 'result'), + ), + }), + ...(roster.length === 0 ? {} : { attempts: roster }), + }; + if (Buffer.byteLength(JSON.stringify(reduced), 'utf8') <= DETAIL_MAX_BYTES) { + return { detail: reduced, truncated: true }; + } + return { detail: { attempts: roster.slice(-1) }, truncated: true }; +} + +interface AttemptRecord { + attempt: number; + atMs: number; + completionReason: string; + disposition: string; + trajectoryTail: unknown; + transcriptBytes?: number; + transcriptPath?: string; + tokensIn?: number; + tokensOut?: number; +} + +/** + * Every `step.completed` the journal holds, in order, by step id. + * + * `foldRunState` keeps only the *last* attempt, because that is what a status + * view shows. The final report's `detail.attempts` is the whole roster, and + * its token totals are the sum across attempts, so both need the raw entries. + */ +function attemptsByStep(events: readonly JournalEvent[]): Map { + const byStep = new Map(); + for (const event of events) { + if (event.entry_type !== 'step.completed' || event.step_id === null) continue; + const payload = record(event.payload) ?? {}; + const budget = record(payload['budget']) ?? {}; + const digest = record(record(payload['trajectory_tail'])?.['transcript']); + const file = record(digest?.['file']); + const entries = byStep.get(event.step_id) ?? []; + entries.push({ + attempt: event.attempt ?? entries.length + 1, + atMs: event.at_ms, + completionReason: typeof payload['completionReason'] === 'string' ? payload['completionReason'] : 'unknown', + disposition: typeof payload['disposition'] === 'string' ? payload['disposition'] : 'unknown', + trajectoryTail: payload['trajectory_tail'], + ...(int32(file?.['bytes_kept']) === undefined ? {} : { transcriptBytes: int32(file?.['bytes_kept'])! }), + ...(typeof file?.['path'] === 'string' ? { transcriptPath: file['path'] } : {}), + ...(int32(budget['tokens_in']) === undefined ? {} : { tokensIn: int32(budget['tokens_in'])! }), + ...(int32(budget['tokens_out']) === undefined ? {} : { tokensOut: int32(budget['tokens_out'])! }), + }); + byStep.set(event.step_id, entries); + } + return byStep; +} + +function finalStep( + step: StepView, + attempts: AttemptRecord[], + env: NodeJS.ProcessEnv, +): FinalStep | undefined { + const stepName = identifier(step.id, env); + const stepType = identifier(step.type, env); + if (stepName === undefined || stepType === undefined) return undefined; + const completionReason = step.completion_reason ?? step.last_attempt?.completion_reason ?? 'unknown'; + const succeeded = completionReason === SUCCESS_COMPLETION_REASON; + const transcript = step.last_attempt?.transcript ?? null; + const startMs = step.started_at_ms; + const endMs = step.last_attempt?.ended_at_ms ?? null; + const duration = startMs !== null && endMs !== null ? Math.max(0, endMs - startMs) : (step.elapsed_ms ?? 0); + const tokensIn = attempts.reduce((total, entry) => total + (entry.tokensIn ?? 0), 0); + const tokensOut = attempts.reduce((total, entry) => total + (entry.tokensOut ?? 0), 0); + const failure = transcript?.failure ?? null; + const detail = stepDetail(attempts, env); + const verification = step.last_attempt?.verification ?? null; + // What the step said about itself. A gate's verdict detail is the nearest + // thing a deterministic step has to an agent's final message; an agent step + // has neither in the status view, so the completion reason stands alone + // rather than being invented. + const summary = text(verification?.detail, env, OUTPUT_SUMMARY_MAX_CHARS) + ?? `step ${succeeded ? 'completed' : 'ended'}: ${identifier(completionReason, env) ?? 'unknown'}`; + const model = identifier(transcript?.model, env); + const cost = transcript?.total_cost_usd; + const error = succeeded ? undefined : text(failure?.excerpt, env, ERROR_MAX_CHARS); + return { + stepName, + stepType, + // Cloud's legacy v1 columns. The kernel has no agent/preset/CLI identity + // per step, and an empty string is the honest answer the hosted v2 + // executor gives too — never a guess that would render as a real name. + agent: '', + preset: '', + cli: '', + // Named after the uploaded transcript object once one is written, exactly + // as the sandbox does; empty until then, never a fabricated sandbox id. + sandboxId: '', + startTime: isoAt(startMs), + endTime: isoAt(endMs), + durationMs: int32(duration) ?? 0, + exitCode: succeeded ? 0 : 1, + outputSummary: summary, + status: succeeded ? 'completed' : 'failed', + completionReason: identifier(completionReason, env) ?? 'unknown', + retryCount: Math.max(0, attempts.length - 1), + ...(model === undefined ? {} : { model }), + ...(tokensIn > 0 ? { tokensInput: Math.min(tokensIn, MAX_INT32) } : {}), + ...(tokensOut > 0 ? { tokensOutput: Math.min(tokensOut, MAX_INT32) } : {}), + ...(typeof cost === 'number' && Number.isFinite(cost) && cost >= 0 ? { costUsd: cost } : {}), + ...(error === undefined ? {} : { error }), + ...(detail.detail === undefined ? {} : { detail: detail.detail }), + ...(detail.truncated === undefined ? {} : { detailTruncated: detail.truncated }), + }; +} + +/** Per-attempt transcript files this step left on disk, in attempt order. */ +function transcriptRef(stepName: string, attempts: AttemptRecord[]): StepTranscriptRef | undefined { + const files = attempts + .filter((entry): entry is AttemptRecord & { transcriptPath: string } => entry.transcriptPath !== undefined) + .map(entry => ({ attempt: entry.attempt, path: entry.transcriptPath })); + return files.length === 0 ? undefined : { stepName, attempts: files }; +} + +/** + * Fold one journal into everything the mirror can say about it. + * + * The authored root is deliberately not published as a step: it spans the + * whole flow, so a snapshot carrying it would show one node called + * `authored-root` sitting at 100% for the run's entire duration beside the + * steps that actually did the work. Its journal is still read, for the step + * index that names the child journals and their graph edges. + */ +export function mirrorJournal( + runId: string, + events: readonly JournalEvent[], + nowMs: number, + env: NodeJS.ProcessEnv = process.env, +): MirroredJournal { + const view = foldRunState(events, nowMs); + const attempts = attemptsByStep(events); + const steps: SnapshotStep[] = []; + const finals: FinalStep[] = []; + const transcripts: StepTranscriptRef[] = []; + for (const step of view.steps) { + if (step.id === AUTHORED_ROOT_STEP) continue; + const live = snapshotStep(runId, step, env); + if (live !== undefined) steps.push(live); + const stepAttempts = attempts.get(step.id) ?? []; + if (step.state === 'done') { + const row = finalStep(step, stepAttempts, env); + if (row !== undefined) { + finals.push(row); + const ref = transcriptRef(row.stepName, stepAttempts); + if (ref !== undefined) transcripts.push(ref); + } + } + } + const index = authoredIndex(events, env); + return { + runId, + status: view.status, + terminal: view.status === 'completed' || view.status === 'failed' || view.status === 'cancelled', + steps, + finals, + transcripts, + children: index.children, + hints: index.hints, + }; +} + +/** + * An authored root's step index: which child journal runs each authored step, + * what the author called it, and what it came after. + * + * This is the only place those facts exist. A child journal's own view knows + * its kernel step ids and nothing about the authored graph above it, so + * without the root's index a mirrored authored run would show its steps with + * no names and no edges — which is exactly what the run graph draws from. + */ +function authoredIndex( + events: readonly JournalEvent[], + env: NodeJS.ProcessEnv, +): { children: string[]; hints: Map } { + const messages: unknown[] = []; + for (const event of events) { + if (event.entry_type !== 'stream.appended') continue; + const payload = record(event.payload); + if (payload?.['stream'] === AUTHORED_STEP_STREAM) messages.push(payload['message']); + } + const children: string[] = []; + const hints = new Map(); + for (const entry of foldAuthoredStepRecords(messages).values()) { + if (/^[A-Za-z0-9][A-Za-z0-9_-]*$/u.test(entry.runId)) children.push(entry.runId); + const name = label(entry.label, env); + const after = entry.after === undefined + ? undefined + : [...entry.after] + .slice(0, DEPENDS_ON_MAX_ENTRIES) + .map(value => identifier(value, env)) + .filter((value): value is string => value !== undefined); + if (name === undefined && after === undefined) continue; + hints.set(`${entry.runId}/${entry.step}`, { + ...(name === undefined ? {} : { label: name }), + ...(after === undefined ? {} : { after }), + }); + } + return { children, hints }; +} + +/** + * Build a snapshot envelope that actually fits both bounds. + * + * The count cap applies unconditionally — 65 tiny steps are 65 steps. Only + * then does the byte cap bite: first the artifact *names* go (their count + * survives), then the `dependsOn` edges, then whole entries from the head of + * the display order, because the tail is the newest work and that is what a + * live view exists to show. A capped view announces itself as incomplete; + * silently showing fewer steps than ran is the one thing it must not do. + */ +export function fitSnapshot( + steps: readonly SnapshotStep[], + envelope: { sequence: number; capturedAt: string; alreadyOmitted?: number }, +): { sequence: number; capturedAt: string; truncated?: true; omittedStepCount?: number; steps: SnapshotStep[] } { + let kept = [...steps]; + let omitted = envelope.alreadyOmitted ?? 0; + let truncated = omitted > 0; + if (kept.length > SNAPSHOT_MAX_STEPS) { + omitted += kept.length - SNAPSHOT_MAX_STEPS; + kept = kept.slice(kept.length - SNAPSHOT_MAX_STEPS); + truncated = true; + } + const build = (): { sequence: number; capturedAt: string; truncated?: true; omittedStepCount?: number; steps: SnapshotStep[] } => ({ + sequence: envelope.sequence, + capturedAt: envelope.capturedAt, + ...(truncated ? { truncated: true as const } : {}), + ...(omitted > 0 ? { omittedStepCount: Math.min(omitted, MAX_INT32) } : {}), + steps: kept, + }); + const bytes = (): number => Buffer.byteLength(JSON.stringify(build()), 'utf8'); + if (bytes() <= SNAPSHOT_MAX_BYTES) return build(); + kept = kept.map(({ artifacts: _artifacts, ...step }) => step); + truncated = true; + if (bytes() <= SNAPSHOT_MAX_BYTES) return build(); + kept = kept.map(({ dependsOn: _dependsOn, ...step }) => step); + if (bytes() <= SNAPSHOT_MAX_BYTES) return build(); + while (kept.length > 1 && bytes() > SNAPSHOT_MAX_BYTES) { + kept = kept.slice(1); + omitted += 1; + } + return build(); +} + +/** Apply an authored root's graph hints to a step, bounded for the live view. */ +export function withGraphHints( + step: T, + hints: Map, + journalRunId: string, + maxDependsOn: number, + knownSteps: ReadonlySet, +): T & { label?: string; dependsOn?: string[] } { + const hint = hints.get(`${journalRunId}/${step.stepName}`); + if (hint === undefined) return step; + // Only predecessors that are themselves steps of this report: an edge to a + // node the graph does not contain draws nothing and reads as data loss. + const after = hint.after + ?.filter(name => name !== step.stepName && knownSteps.has(name)) + .slice(0, maxDependsOn); + return { + ...step, + ...(hint.label === undefined ? {} : { label: hint.label }), + ...(after === undefined ? {} : { dependsOn: after }), + }; +} + +export type { RunView }; diff --git a/packages/sdk/src/cloud-mirror-transport.ts b/packages/sdk/src/cloud-mirror-transport.ts new file mode 100644 index 000000000..592519481 --- /dev/null +++ b/packages/sdk/src/cloud-mirror-transport.ts @@ -0,0 +1,238 @@ +// The Cloud calls a mirrored local run makes, and nothing else. +// +// A hosted run reports through five endpoints. A local run reports through the +// same five, with the same bodies, using a credential Cloud issued for that +// one run — so the control plane cannot tell a mirrored run from a sandboxed +// one, and no route needed a new case to admit it. +// +// POST /api/v1/workflows/local-run register; returns the credential +// POST /api/v1/workflows/runs//events lifecycle events +// POST /api/v1/workflows/runs//steps/snapshot the live view, while it runs +// POST /api/v1/workflows/runs//steps the final report +// PUT /api/v1/workflows/runs//storage/ runner log and step transcripts +// POST /api/v1/workflows/callback the terminal status +// +// ## Two credentials, never confused +// +// Registration authenticates as the *operator*: the `agent-relay cloud login` +// token or `FLOWS_CLOUD_TOKEN`, the same credential `flows run --cloud` uses. +// Everything after it authenticates as the *run*, with the run-bound token the +// registration returned. That token can write this run's steps and read this +// run, and can do nothing else in the workspace — so a mirror that is somehow +// coerced into pushing elsewhere has nothing to push with. +// +// ## Everything here is best-effort by construction +// +// Except `register`, which the caller needs an answer from, every function +// resolves to a boolean and throws nothing. A mirror is an observer: a run's +// correctness cannot depend on whether a report landed, and a Cloud outage +// must cost a local run nothing but its dashboard page. The caller decides +// what to retry; this module decides nothing. + +import { cloudFetch, CloudFlowError, cloudConnection, cloudRunId, isCloudRecord, type CloudConnectionOptions } from './cloud-http.js'; +import type { FinalStep, SnapshotStep } from './cloud-mirror-step.js'; + +/** What registration returns: the run, and the credential that may report on it. */ +export interface MirrorRegistration { + runId: string; + /** The run-bound token every later call authenticates with. */ + token: string; + /** Proves the terminal callback came from this run's executor. */ + callbackToken: string; + /** The dashboard page for this run, as Cloud named it. */ + runUrl: string; + /** + * The deployment that issued the credential, pinned. + * + * Load-bearing, not bookkeeping. `cloudConnection` resolves the base URL + * from the *login store* only while no explicit token is given — and every + * call after registration gives one. Without pinning it here, a CLI signed + * in to a staging deployment would register there and then send that + * deployment's run token to the production default. + */ + apiUrl: string; +} + +export interface MirrorRunSource { + /** The exact source bytes that ran: YAML/JSON text, or authored `.flow.ts`. */ + workflow: string; + fileType: 'yaml' | 'ts'; + /** Authored runs only: the input the flow was invoked with. */ + inputs?: unknown; +} + +/** One lifecycle event, in the vocabulary Cloud's session event stream uses. */ +export interface MirrorEvent { + eventType: string; + stepName?: string; + payload?: Record; +} + +const REGISTER_PATH = '/api/v1/workflows/local-run'; + +/** + * Register the run and take its credential. + * + * The only call here that throws. Everything the mirror does afterwards + * depends on this receipt, so a caller has to be able to tell "Cloud refused" + * from "Cloud accepted" — and the CLI turns that difference into one line on + * stderr rather than a failed run. + */ +export async function registerLocalRun( + source: MirrorRunSource, + options: CloudConnectionOptions = {}, +): Promise { + const result = await cloudFetch(REGISTER_PATH, options, { + method: 'POST', + detail: true, + body: JSON.stringify({ + workflow: source.workflow, + fileType: source.fileType, + relayflowVersion: 'v2', + ...(source.inputs === undefined ? {} : { inputs: source.inputs }), + }), + }); + if (!isCloudRecord(result)) { + throw new CloudFlowError('invalid_response', 'Cloud did not return a registered local run.'); + } + const runId = cloudRunId(result['runId']); + const token = result['accessToken']; + const callbackToken = result['callbackToken']; + if (typeof token !== 'string' || token.trim().length === 0 + || typeof callbackToken !== 'string' || callbackToken.length === 0) { + throw new CloudFlowError('invalid_response', 'Cloud registered the run without a usable credential.'); + } + const { baseUrl } = cloudConnection(options); + const runUrl = typeof result['runUrl'] === 'string' && /^https:\/\//u.test(result['runUrl']) + ? result['runUrl'] + : `${baseUrl}/dashboard/workflow/${encodeURIComponent(runId)}/runner`; + return { runId, token, callbackToken, runUrl, apiUrl: baseUrl }; +} + +/** A mirror's authenticated view of one run. Built once, from the registration. */ +export class MirrorClient { + /** Connection options with the run's credential and its deployment pinned. */ + private readonly bound: CloudConnectionOptions; + + constructor( + private readonly registration: MirrorRegistration, + options: CloudConnectionOptions = {}, + ) { + this.bound = { ...options, token: registration.token, apiUrl: registration.apiUrl }; + } + + get runId(): string { + return this.registration.runId; + } + + get runUrl(): string { + return this.registration.runUrl; + } + + /** + * Every call but registration goes through here. The run token replaces the + * operator credential, the outcome collapses to a boolean, and nothing — + * not a refusal, not a timeout, not a thrown transport error — escapes. + */ + private async post( + path: string, + body: string, + init: { signal?: AbortSignal; timeoutMs?: number } = {}, + ): Promise { + try { + await cloudFetch(path, { + ...this.bound, + ...(init.signal === undefined ? {} : { signal: init.signal }), + ...(init.timeoutMs === undefined ? {} : { requestTimeoutMs: init.timeoutMs }), + }, { method: 'POST', body }); + return true; + } catch { + return false; + } + } + + async publishEvent(event: MirrorEvent, signal?: AbortSignal): Promise { + return this.post( + `/api/v1/workflows/runs/${this.registration.runId}/events`, + JSON.stringify({ + eventType: event.eventType, + ...(event.stepName === undefined ? {} : { stepName: event.stepName }), + payload: event.payload ?? {}, + }), + { ...(signal === undefined ? {} : { signal }) }, + ); + } + + async publishSnapshot( + snapshot: { sequence: number; capturedAt: string; truncated?: true; omittedStepCount?: number; steps: SnapshotStep[] }, + limits: { signal?: AbortSignal; timeoutMs?: number } = {}, + ): Promise { + return this.post( + `/api/v1/workflows/runs/${this.registration.runId}/steps/snapshot`, + JSON.stringify(snapshot), + limits, + ); + } + + async publishSteps( + steps: readonly FinalStep[], + omittedStepCount: number, + signal?: AbortSignal, + ): Promise { + return this.post( + `/api/v1/workflows/runs/${this.registration.runId}/steps`, + JSON.stringify({ steps, ...(omittedStepCount > 0 ? { omittedStepCount } : {}) }), + { ...(signal === undefined ? {} : { signal }) }, + ); + } + + /** + * Write one object under the run's storage prefix. The keys are the ones the + * `/logs` route already reads: `runner.log` for the run, and + * `/agent.log` for a step's assembled transcript. + */ + async putObject(key: string, bytes: Uint8Array, signal?: AbortSignal): Promise { + if (!/^[A-Za-z0-9][A-Za-z0-9._/-]{0,511}$/u.test(key) || key.includes('..')) return false; + try { + await cloudFetch( + `/api/v1/workflows/runs/${this.registration.runId}/storage/${key}`, + { ...this.bound, ...(signal === undefined ? {} : { signal }) }, + { method: 'PUT', body: bytes, contentType: 'text/plain' }, + ); + return true; + } catch { + return false; + } + } + + /** + * The terminal status. Authenticated by the callback token rather than the + * run token, because that is the credential this route checks — and because + * the run token is revoked by the transition this very call performs. + */ + async reportTerminal( + status: 'completed' | 'failed' | 'cancelled', + result: Record, + error?: string, + signal?: AbortSignal, + ): Promise { + try { + await cloudFetch('/api/v1/workflows/callback', { + ...this.bound, + ...(signal === undefined ? {} : { signal }), + }, { + method: 'POST', + body: JSON.stringify({ + runId: this.registration.runId, + callbackToken: this.registration.callbackToken, + status, + result, + ...(error === undefined ? {} : { error }), + }), + }); + return true; + } catch { + return false; + } + } +} diff --git a/packages/sdk/src/cloud-mirror.ts b/packages/sdk/src/cloud-mirror.ts new file mode 100644 index 000000000..d5d180b96 --- /dev/null +++ b/packages/sdk/src/cloud-mirror.ts @@ -0,0 +1,469 @@ +// Mirroring a local run onto the Cloud dashboard, while it runs. +// +// A hosted run is watchable because something inside the sandbox reads its +// journal every few seconds and pushes what it finds. Nothing did that for a +// run started in a terminal, so a local run was invisible: the same flow, the +// same journal, the same evidence, and no page to look at. This is that +// reader, on the other side of the boundary. +// +// ## What it reads +// +// Only this run's journals. The sandbox reporter scans its whole data +// directory because a sandbox holds exactly one run; a developer's +// `~/.relayflowd` holds every run they have ever started, including other +// people's work in a shared checkout. So the set of journals is *derived*: the +// root, plus the child journals the root's own authored-step index names. A +// journal this run did not create is never opened, let alone uploaded. +// +// ## What it costs a run +// +// Nothing that can fail the run. Every push collapses to a boolean at the +// transport (`cloud-mirror-transport.ts`), each poll is bounded by its own +// deadline, and a poll that overruns is abandoned rather than allowed to grow +// with the flow. A journal that cannot be read keeps the previously cached +// view for that journal: a snapshot never gets emptier because a read failed. +// +// ## Ordering at the end +// +// The terminal callback is last, and that is load-bearing. Cloud revokes the +// run's credential when the run goes terminal, so the transcripts and the +// final step rows have to land before it — otherwise the mirror would report +// the run finished and then find itself unable to say what it did. + +import { readFile, stat } from 'node:fs/promises'; +import { walkJournal, JournalReadError, type JournalEvent } from './journal-reader.js'; +import { + fitSnapshot, mirrorJournal, withGraphHints, + DEPENDS_ON_MAX_ENTRIES, SNAPSHOT_DEPENDS_ON_MAX, MAX_INT32, + type FinalStep, type SnapshotStep, +} from './cloud-mirror-step.js'; +import { MirrorClient, type MirrorEvent } from './cloud-mirror-transport.js'; + +/** How often the journals are re-read. The sandbox reporter's own cadence. */ +export const MIRROR_POLL_INTERVAL_MS = 10_000; +/** Whole-poll budget, independent of how many journals exist. */ +export const MIRROR_POLL_BUDGET_MS = 8_000; +/** Budget for the final poll and every push that follows it. */ +export const MIRROR_FINISH_BUDGET_MS = 30_000; +/** Republish an unchanged view this often, so a stalled view still looks fresh. */ +export const MIRROR_SNAPSHOT_HEARTBEAT_MS = 120_000; +/** Final step rows one report carries; the rest are counted, not sent. */ +export const MIRROR_MAX_FINAL_STEPS = 256; +/** Step views held across polls. Bounds the mirror, not the run. */ +export const MIRROR_STEP_CACHE_MAX = 512; +/** Assembled `/agent.log` cap, as the hosted executor uses. */ +export const MIRROR_TRANSCRIPT_MAX_BYTES = 1024 * 1024; +/** Steps whose transcripts are uploaded. Beyond this the rows still land. */ +export const MIRROR_MAX_TRANSCRIPT_UPLOADS = 64; +/** The run's own `runner.log`, as the `/logs` route serves it. */ +export const MIRROR_RUNNER_LOG_MAX_BYTES = 256 * 1024; +/** Journals one mirror will follow: the root, plus the children it admitted. */ +export const MIRROR_MAX_JOURNALS = 4096; + +export interface RunMirrorOptions { + client: MirrorClient; + /** The daemon data directory holding this run's journals. */ + dataDir: string; + /** One line about the mirror itself; never about the run. */ + diagnostic?: (message: string) => void; + env?: NodeJS.ProcessEnv; + now?: () => number; + intervalMs?: number; + pollBudgetMs?: number; + snapshotHeartbeatMs?: number; + /** Test seam: read one journal's events. */ + readJournal?: (runId: string, dataDir: string) => Promise; + /** Test seam: read a transcript file. */ + readTranscript?: (path: string) => Promise<{ bytes: Buffer; size: number }>; +} + +/** What the mirror was told about the run when it ended. */ +export interface RunMirrorOutcome { + status: 'completed' | 'failed' | 'cancelled'; + completionReason?: string; + error?: string; + /** Lines the CLI printed for this run, uploaded as `runner.log`. */ + log?: readonly string[]; +} + +export interface RunMirror { + readonly runId: string; + readonly runUrl: string; + /** Begin polling. The first poll runs on the first interval, not now. */ + start(rootRunId: string): void; + /** One lifecycle event, pushed without waiting for the next poll. */ + event(event: MirrorEvent): void; + /** + * Stop polling, take one last reading, and publish the final report, the + * transcripts and the terminal status — in that order. Never throws, and is + * bounded by {@link MIRROR_FINISH_BUDGET_MS} however much is outstanding. + */ + finish(outcome: RunMirrorOutcome): Promise; +} + +interface CachedStep { + step: SnapshotStep; + /** Journal order within its journal, so the published view reads top to bottom. */ + order: number; + /** When this step's meaningful content last changed, for eviction. */ + changedAt: number; +} + +async function defaultReadJournal(runId: string, dataDir: string): Promise { + const events: JournalEvent[] = []; + for await (const event of walkJournal(runId, dataDir)) events.push(event); + return events; +} + +async function defaultReadTranscript(path: string): Promise<{ bytes: Buffer; size: number }> { + const info = await stat(path); + if (!info.isFile()) throw new Error('transcript is not a regular file'); + // Bound the read, not just the result: materializing a huge file only to + // discard all but its tail can exhaust the CLI before it reports at all. + if (info.size <= MIRROR_TRANSCRIPT_MAX_BYTES) { + return { bytes: await readFile(path), size: info.size }; + } + const handle = await (await import('node:fs/promises')).open(path, 'r'); + try { + const buffer = Buffer.alloc(MIRROR_TRANSCRIPT_MAX_BYTES); + const { bytesRead } = await handle.read( + buffer, 0, MIRROR_TRANSCRIPT_MAX_BYTES, info.size - MIRROR_TRANSCRIPT_MAX_BYTES, + ); + return { bytes: buffer.subarray(0, bytesRead), size: info.size }; + } finally { + await handle.close(); + } +} + +/** + * Assemble one step's attempts into the single object `/logs` serves. + * + * The frame vocabulary is Cloud's: each kept attempt is preceded by a + * `relayflow.attempt` marker and a dropped one leaves a + * `relayflow.attempt.omitted` marker in its place, so the dashboard's renderer + * and `flows logs --step` read a mirrored transcript exactly as they read a + * hosted one. A reader that sees fewer attempts than ran has been lied to. + */ +export async function assembleTranscript( + attempts: ReadonlyArray<{ attempt: number; path: string }>, + read: (path: string) => Promise<{ bytes: Buffer; size: number }>, + maxBytes = MIRROR_TRANSCRIPT_MAX_BYTES, +): Promise<{ bytes: Buffer; kept: number; omitted: number; truncated: boolean }> { + const ordered = [...attempts].sort((left, right) => left.attempt - right.attempt); + const marker = (value: Record): Buffer => Buffer.from(`${JSON.stringify(value)}\n`, 'utf8'); + const parts: Buffer[] = []; + let total = 0; + let kept = 0; + let omitted = 0; + let truncated = false; + // Newest first: the last attempt is what the run's outcome rests on. + for (let index = ordered.length - 1; index >= 0; index -= 1) { + const { attempt, path } = ordered[index]!; + let read_: { bytes: Buffer; size: number }; + try { + read_ = await read(path); + } catch { + // A missing attempt file is still an attempt that happened. + parts.unshift(marker({ type: 'relayflow.attempt.omitted', attempt, bytes: 0 })); + total += parts[0]!.length; + omitted += 1; + continue; + } + const reserve = 128 * (index + 1); + if (kept > 0 && read_.bytes.length > maxBytes - total - reserve) { + parts.unshift(marker({ type: 'relayflow.attempt.omitted', attempt, bytes: read_.size })); + total += parts[0]!.length; + omitted += 1; + continue; + } + let content = read_.bytes; + let cut = read_.size > content.length; + const cap = maxBytes - reserve; + if (content.length > cap) { + content = content.subarray(content.length - cap); + cut = true; + } + if (cut) { + // A tail cut lands mid-frame and this object is parsed as JSONL: open on + // the first WHOLE frame rather than a broken one. A file with no newline + // is one frame, so there is nothing to snap to and it is left as it is. + const firstFrame = content.indexOf(0x0a); + if (firstFrame >= 0 && firstFrame + 1 < content.length) content = content.subarray(firstFrame + 1); + truncated = true; + } + const head = marker({ type: 'relayflow.attempt', attempt, bytes: content.length, truncated: cut }); + const body = content.length > 0 && content[content.length - 1] !== 0x0a + ? Buffer.concat([content, Buffer.from('\n')]) + : content; + parts.unshift(head, body); + total += head.length + body.length; + kept += 1; + } + return { bytes: Buffer.concat(parts), kept, omitted, truncated }; +} + +export function createRunMirror(options: RunMirrorOptions): RunMirror { + const env = options.env ?? process.env; + const now = options.now ?? Date.now; + const intervalMs = options.intervalMs ?? MIRROR_POLL_INTERVAL_MS; + const pollBudgetMs = options.pollBudgetMs ?? MIRROR_POLL_BUDGET_MS; + const heartbeatMs = options.snapshotHeartbeatMs ?? MIRROR_SNAPSHOT_HEARTBEAT_MS; + const readJournal = options.readJournal ?? defaultReadJournal; + const readTranscript = options.readTranscript ?? defaultReadTranscript; + const diagnostic = options.diagnostic ?? ((): void => {}); + + /** The journals this run owns: the root, and every child its index names. */ + const journals: string[] = []; + const known = new Set(); + const finished = new Set(); + const steps = new Map(); + /** Final rows by `/`, so two journals cannot collide on a step id. */ + const finals = new Map(); + const transcripts = new Map }>(); + const hints = new Map(); + /** Live steps the cache cap dropped, by identity, so a capped view owns up to it. */ + const evicted = new Set(); + /** Finished steps past the report cap, by identity. Counted, never sent. */ + const unreported = new Set(); + let timer: ReturnType | undefined; + let polling = false; + let stopped = false; + let sequence = 0; + let acknowledged: string | undefined; + let acknowledgedAt = now(); + + const admit = (runId: string): void => { + // Far above any real authored run, and a bound all the same: the child + // list comes out of a journal, and a reader of a journal bounds what it + // will grow to hold. + if (known.has(runId) || journals.length >= MIRROR_MAX_JOURNALS) return; + known.add(runId); + journals.push(runId); + }; + + const remember = (journalRunId: string, step: SnapshotStep, order: number): void => { + const key = `${journalRunId}/${step.stepName}`; + const existing = steps.get(key); + const changed = existing === undefined || fingerprint(existing.step) !== fingerprint(step); + steps.set(key, { step, order, changedAt: changed ? now() : existing!.changedAt }); + evicted.delete(key); + while (steps.size > MIRROR_STEP_CACHE_MAX) { + // Evict the least recently changed *completed* step first: the live work + // is the whole point of a live view. + const victim = [...steps.entries()].sort(comparePriority).pop(); + if (victim === undefined) break; + steps.delete(victim[0]); + evicted.add(victim[0]); + } + }; + + const scan = async (deadline: number): Promise => { + // Index-based, over the live array: a child journal the root's index names + // is appended during this very pass, and a run whose only steps are its + // children would otherwise publish nothing until the next poll — which for + // a short run is never, because `finish` scans once. + for (let index = 0; index < journals.length; index += 1) { + const runId = journals[index]!; + if (now() > deadline) break; + if (finished.has(runId)) continue; + let events: JournalEvent[]; + try { + events = await readJournal(runId, options.dataDir); + } catch (error) { + // `run_not_found` is ordinary: a child journal is named in the index + // the moment the step is admitted, which can be before its journal + // exists on disk. Everything else keeps the cached view for it. + if (!(error instanceof JournalReadError) || error.code !== 'run_not_found') { + diagnostic(`could not read journal ${runId}: ${error instanceof Error ? error.message : String(error)}`); + } + continue; + } + let folded; + try { + folded = mirrorJournal(runId, events, now(), env); + } catch (error) { + diagnostic(`could not fold journal ${runId}: ${error instanceof Error ? error.message : String(error)}`); + continue; + } + for (const child of folded.children) admit(child); + for (const [key, hint] of folded.hints) { + // Merge, never replace: a completion record can omit a label or an + // edge the admission carried, and the view must not lose a step's + // name between them. + const existing = hints.get(key); + hints.set(key, { + ...(hint.label ?? existing?.label ? { label: hint.label ?? existing?.label } : {}), + ...(hint.after ?? existing?.after ? { after: hint.after ?? existing?.after } : {}), + }); + } + folded.steps.forEach((step, order) => remember(runId, step, order)); + for (const row of folded.finals) { + const key = `${runId}/${row.stepName}`; + // The report cap bounds what this process holds, not just what it + // sends: a run with ten thousand steps must not grow its own mirror. + // Steps past it are counted by identity, so a step read twice is one + // missing step and not two. + if (!finals.has(key) && finals.size >= MIRROR_MAX_FINAL_STEPS) { + unreported.add(key); + continue; + } + finals.set(key, { journalRunId: runId, row }); + } + for (const ref of folded.transcripts) { + if (!finals.has(`${runId}/${ref.stepName}`)) continue; + transcripts.set(`${runId}/${ref.stepName}`, { stepName: ref.stepName, attempts: ref.attempts }); + } + if (folded.terminal) finished.add(runId); + } + }; + + /** The cached view in display order: journal order, journal by journal. */ + const ordered = (): SnapshotStep[] => { + const names = new Set([...steps.values()].map(entry => entry.step.stepName)); + return [...steps.entries()] + .sort(([leftKey, left], [rightKey, right]) => { + const journal = leftKey.split('/')[0]!.localeCompare(rightKey.split('/')[0]!); + return journal !== 0 ? journal : left.order - right.order; + }) + .map(([, entry]) => withGraphHints( + entry.step, hints, entry.step.journalRunId, SNAPSHOT_DEPENDS_ON_MAX, names, + )); + }; + + const publishSnapshot = async (deadline: number): Promise => { + const view = ordered(); + if (view.length === 0) return; + // `elapsedMs` moves every poll and is excluded from the comparison, or the + // throttle would degenerate into "push every interval". + const print = view.map(fingerprint).join('|'); + if (print === acknowledged && now() - acknowledgedAt < heartbeatMs) return; + sequence += 1; + const snapshot = fitSnapshot(view, { + sequence, + capturedAt: new Date(now()).toISOString(), + alreadyOmitted: evicted.size, + }); + const budget = deadline - now(); + if (budget <= 0) return; + if (await options.client.publishSnapshot(snapshot, { timeoutMs: budget })) { + acknowledged = print; + acknowledgedAt = now(); + } + }; + + const poll = async (budgetMs: number): Promise => { + if (polling || stopped) return; + polling = true; + try { + const deadline = now() + budgetMs; + await scan(deadline); + await publishSnapshot(deadline); + } catch (error) { + diagnostic(`poll failed: ${error instanceof Error ? error.message : String(error)}`); + } finally { + polling = false; + } + }; + + const uploadTranscripts = async (deadline: number): Promise => { + let uploaded = 0; + for (const [key, ref] of transcripts) { + if (uploaded >= MIRROR_MAX_TRANSCRIPT_UPLOADS || now() > deadline) break; + const entry = finals.get(key); + if (entry === undefined) continue; + let assembled; + try { + assembled = await assembleTranscript(ref.attempts, readTranscript); + } catch (error) { + diagnostic(`could not assemble the transcript for ${ref.stepName}: ${error instanceof Error ? error.message : String(error)}`); + continue; + } + if (assembled.bytes.length === 0) continue; + // Name the row after the object only once the object is there, so no row + // ever points at a transcript that was never written. + if (await options.client.putObject(`${ref.stepName}/agent.log`, assembled.bytes)) { + finals.set(key, { ...entry, row: { ...entry.row, sandboxId: ref.stepName } }); + uploaded += 1; + } + } + }; + + const publishFinal = async (deadline: number): Promise => { + const sent = [...finals.values()]; + // A capped report says how many steps it left out. A run page that shows + // 256 of 400 steps without saying so is worse than one that shows none. + const omitted = Math.min(unreported.size, MAX_INT32); + const names = new Set(sent.map(entry => entry.row.stepName)); + const withGraph = sent.map(entry => + withGraphHints(entry.row, hints, entry.journalRunId, DEPENDS_ON_MAX_ENTRIES, names)); + if (now() > deadline) return; + if (!await options.client.publishSteps(withGraph, omitted)) { + diagnostic('Cloud did not accept the final step report; the run page keeps its live view'); + } + }; + + return { + get runId() { return options.client.runId; }, + get runUrl() { return options.client.runUrl; }, + start(rootRunId) { + admit(rootRunId); + if (timer !== undefined || stopped) return; + timer = setInterval(() => { void poll(pollBudgetMs); }, intervalMs); + // Never hold the process open for an observation. + timer.unref?.(); + }, + event(event) { + void options.client.publishEvent(event); + }, + async finish(outcome) { + if (timer !== undefined) { clearInterval(timer); timer = undefined; } + const deadline = now() + MIRROR_FINISH_BUDGET_MS; + try { + // One last reading, so the page shows the run's actual last moments + // rather than whatever the previous poll happened to catch. + await scan(deadline); + await publishSnapshot(deadline); + if (outcome.log !== undefined && outcome.log.length > 0) { + const text = outcome.log.join('\n'); + const bytes = Buffer.from(text, 'utf8'); + await options.client.putObject('runner.log', + bytes.length > MIRROR_RUNNER_LOG_MAX_BYTES + ? bytes.subarray(bytes.length - MIRROR_RUNNER_LOG_MAX_BYTES) + : bytes); + } + await uploadTranscripts(deadline); + await publishFinal(deadline); + // Last, always: this transition revokes the credential every call + // above depends on. + await options.client.reportTerminal( + outcome.status, + { + status: outcome.status === 'completed' ? 'completed' : outcome.status, + ...(outcome.completionReason === undefined ? {} : { completionReason: outcome.completionReason }), + ...(outcome.error === undefined ? {} : { error: outcome.error }), + }, + outcome.error, + ); + } catch (error) { + diagnostic(`could not finish the mirror: ${error instanceof Error ? error.message : String(error)}`); + } finally { + stopped = true; + } + }, + }; +} + +/** Everything about a step but its ever-moving elapsed time. */ +function fingerprint(step: SnapshotStep): string { + const { elapsedMs: _elapsedMs, ...rest } = step; + return JSON.stringify(rest); +} + +/** Least recently changed first, and a running step is never the first victim. */ +function comparePriority( + [, left]: [string, CachedStep], + [, right]: [string, CachedStep], +): number { + const liveness = Number(left.step.state === 'done') - Number(right.step.state === 'done'); + return liveness !== 0 ? liveness : right.changedAt - left.changedAt; +} diff --git a/packages/sdk/tests/cli-cloud-mirror-flag.test.ts b/packages/sdk/tests/cli-cloud-mirror-flag.test.ts new file mode 100644 index 000000000..993bd0785 --- /dev/null +++ b/packages/sdk/tests/cli-cloud-mirror-flag.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { runCli, type CliIo } from '../src/cli.js'; + +function capture(): { io: CliIo; out: string[]; err: string[] } { + const out: string[] = []; + const err: string[] = []; + return { io: { stdout: line => out.push(line), stderr: line => err.push(line) }, out, err }; +} + +/** + * `--no-cloud-mirror` is refused wherever it would describe nothing, rather + * than being accepted and ignored. A flag that silently no-ops is worse than + * one that is rejected: it reads as an opt-out that was honoured. + */ +describe('flows --no-cloud-mirror', () => { + it('is refused on `check`, which starts no run to mirror', async () => { + const { io } = capture(); + expect(await runCli(['check', '--no-cloud-mirror', 'flow.yaml'], io)).toBe(2); + }); + + it('is refused with --cloud, which IS the hosted run', async () => { + const { io } = capture(); + expect(await runCli(['run', '--cloud', '--no-cloud-mirror', 'flow.yaml'], io)).toBe(2); + }); + + it('is refused twice over, like every other flag here', async () => { + const { io } = capture(); + expect(await runCli(['run', '--no-cloud-mirror', '--no-cloud-mirror', 'flow.yaml'], io)).toBe(2); + }); + + it('is listed in the usage for the verbs that accept it', async () => { + const { io, out } = capture(); + await runCli(['--help'], io); + const usage = out.join('\n'); + expect(usage).toContain('flows run [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror]'); + expect(usage).toContain('flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror]'); + }); +}); diff --git a/packages/sdk/tests/cloud-mirror-session.test.ts b/packages/sdk/tests/cloud-mirror-session.test.ts new file mode 100644 index 000000000..2c6a4d291 --- /dev/null +++ b/packages/sdk/tests/cloud-mirror-session.test.ts @@ -0,0 +1,172 @@ +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { + cloudMirrorEnabled, createCloudMirrorSession, mirrorSourceFromPath, MIRROR_ENV, +} from '../src/cli/cloud-mirror-session.js'; +import { CloudFlowError } from '../src/cloud-http.js'; +import type { RunReport } from '../src/cli/run.js'; +import type { CliIo } from '../src/cli.js'; + +function io(): { io: CliIo; out: string[]; err: string[] } { + const out: string[] = []; + const err: string[] = []; + return { io: { stdout: line => out.push(line), stderr: line => err.push(line) }, out, err }; +} + +const okReport: RunReport = { ok: true, command: 'run', runId: '01RUN', resolutions: [], diagnostics: [] }; + +function registration() { + return { + runId: 'cloud-run', + token: 'cld_at_x', + callbackToken: 'cb', + runUrl: 'https://agentrelay.com/cloud/dashboard/workflow/cloud-run/runner', + }; +} + +describe('cloudMirrorEnabled', () => { + it('is on unless the operator turned it off for this shell', () => { + expect(cloudMirrorEnabled({})).toBe(true); + expect(cloudMirrorEnabled({ [MIRROR_ENV]: '1' })).toBe(true); + for (const value of ['0', 'false', 'off', 'no', 'OFF']) { + expect(cloudMirrorEnabled({ [MIRROR_ENV]: value })).toBe(false); + } + }); +}); + +describe('createCloudMirrorSession', () => { + it('registers once the run id exists, prints the page, and starts on that run', async () => { + const { io: cli, err } = io(); + const register = vi.fn(async () => registration()); + const mirror = { runId: 'cloud-run', runUrl: registration().runUrl, start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => ['RUN 01RUN'], + }, cli, {}, { register, createMirror: vi.fn(() => mirror) }); + + session.onRunStarted({ runId: '01RUN' }); + await session.finish(okReport); + + expect(register).toHaveBeenCalledOnce(); + expect(mirror.start).toHaveBeenCalledWith('01RUN'); + expect(err).toContain(`Dashboard: ${registration().runUrl}`); + expect(mirror.finish).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed', log: ['RUN 01RUN'] })); + }); + + it('registers exactly once however many entries arrive', async () => { + const { io: cli } = io(); + const register = vi.fn(async () => registration()); + const mirror = { runId: 'cloud-run', runUrl: 'u', start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register, createMirror: vi.fn(() => mirror) }); + + session.onJournalEntry({ run_id: '01RUN' }); + session.onJournalEntry({ run_id: '01RUN' }); + session.onRunStarted({ runId: '01RUN' }); + await session.finish(okReport); + + expect(register).toHaveBeenCalledOnce(); + expect(mirror.start).toHaveBeenCalledOnce(); + }); + + it('says the run stays local when there is no Cloud login, and finishes cleanly', async () => { + const { io: cli, err } = io(); + const missing = new CloudFlowError('configuration', 'Set FLOWS_CLOUD_TOKEN …'); + missing.reason = 'auth_missing'; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register: vi.fn(async () => { throw missing; }) }); + + session.onRunStarted({ runId: '01RUN' }); + await expect(session.finish(okReport)).resolves.toBeUndefined(); + + expect(err).toHaveLength(1); + expect(err[0]).toContain('no Cloud login, so this run stays local'); + expect(err[0]).toContain('agent-relay cloud login'); + }); + + it('names a deployment that does not serve the route yet, without blaming the run', async () => { + const { io: cli, err } = io(); + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { + register: vi.fn(async () => { throw new CloudFlowError('http_error', 'HTTP 404', 404); }), + }); + + session.onRunStarted({ runId: '01RUN' }); + await session.finish(okReport); + + expect(err[0]).toContain('does not accept local runs yet'); + expect(err[0]).toContain('the run is unaffected'); + }); + + it('never registers a run that was refused before it started', async () => { + const { io: cli } = io(); + const register = vi.fn(async () => registration()); + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register }); + + // No run id ever arrived: `flows run` refused the flow at check time. + await session.finish({ ...okReport, ok: false, runId: undefined }); + + expect(register).not.toHaveBeenCalled(); + }); + + it('reports a park as a failed run carrying its completion reason', async () => { + const { io: cli } = io(); + const mirror = { runId: 'cloud-run', runUrl: 'u', start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); + + session.onRunStarted({ runId: '01RUN' }); + await session.finish({ + ok: false, + command: 'run', + runId: '01RUN', + status: 'parked', + completionReason: 'needs_human', + resolutions: [], + diagnostics: [{ severity: 'parked', kind: 'needs_human', message: 'waiting on a human' }], + }); + + expect(mirror.finish).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', completionReason: 'needs_human', + })); + }); +}); + +describe('mirrorSourceFromPath', () => { + it('sends a declarative flow as the bytes on disk', async () => { + const dir = await mkdtemp(join(tmpdir(), 'mirror-source-')); + const path = join(dir, 'flow.yaml'); + await writeFile(path, 'name: demo\nsteps: []\n'); + await expect(mirrorSourceFromPath(path, undefined)()).resolves.toEqual({ + workflow: 'name: demo\nsteps: []\n', fileType: 'yaml', + }); + }); + + it('sends an authored flow with the input it was invoked with', async () => { + const dir = await mkdtemp(join(tmpdir(), 'mirror-source-')); + const path = join(dir, 'review.flow.ts'); + await writeFile(path, 'export default flow("review", () => {});\n'); + await expect(mirrorSourceFromPath(path, '{"pr":7}')()).resolves.toEqual({ + workflow: 'export default flow("review", () => {});\n', fileType: 'ts', inputs: { pr: 7 }, + }); + }); +}); diff --git a/packages/sdk/tests/cloud-mirror-step.test.ts b/packages/sdk/tests/cloud-mirror-step.test.ts new file mode 100644 index 000000000..3c3005f34 --- /dev/null +++ b/packages/sdk/tests/cloud-mirror-step.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it } from 'vitest'; +import { + fitSnapshot, identifier, label, mirrorJournal, snapshotStepStatus, withGraphHints, + IDENTIFIER_MAX_CHARS, LABEL_MAX_CHARS, SNAPSHOT_MAX_BYTES, SNAPSHOT_MAX_STEPS, +} from '../src/cloud-mirror-step.js'; +import type { JournalEvent } from '../src/journal-reader.js'; + +let seq = 0; +function entry(partial: Partial & { entry_type: string }): JournalEvent { + seq += 1; + return { + seq, + segment_id: 1, + run_id: '01RUN', + step_id: null, + attempt: null, + at_ms: 1_700_000_000_000 + seq * 1_000, + payload: null, + ...partial, + }; +} + +/** A two-step declarative run: one agent step that retried once, then a gate. */ +function journal(): JournalEvent[] { + seq = 0; + return [ + entry({ + entry_type: 'run.spawned', + payload: { + spec: { + name: 'review', + steps: [ + { id: 'write', type: 'agent', after: [] }, + { id: 'verify', type: 'deterministic', after: ['write'] }, + ], + }, + }, + }), + entry({ entry_type: 'step.attempt.started', step_id: 'write', attempt: 1, payload: { lease_deadline_ms: 1 } }), + entry({ + entry_type: 'step.completed', + step_id: 'write', + attempt: 1, + payload: { + completionReason: 'agent_error', + disposition: 'retry', + next_attempt_at_ms: 1_700_000_004_000, + budget: { tokens_in: 10, tokens_out: 5, dollars: '0.01' }, + trajectory_tail: { + transcript: { + file: { path: '/home/dev/.relayflowd/runs/01RUN/steps/write/attempt-1.transcript.jsonl', bytes_kept: 40, truncated: false }, + result: { model: 'claude-opus-5', num_turns: 2, total_cost_usd: 0.02 }, + failure: { kind: 'result', excerpt: 'the tool call failed' }, + }, + }, + }, + }), + entry({ entry_type: 'step.attempt.started', step_id: 'write', attempt: 2, payload: { lease_deadline_ms: 1 } }), + entry({ + entry_type: 'step.completed', + step_id: 'write', + attempt: 2, + payload: { + completionReason: 'success', + disposition: 'step_done', + budget: { tokens_in: 20, tokens_out: 7, dollars: '0.03' }, + output: { artifacts: ['out/report.md'] }, + trajectory_tail: { + transcript: { + file: { path: '/home/dev/.relayflowd/runs/01RUN/steps/write/attempt-2.transcript.jsonl', bytes_kept: 90, truncated: false }, + result: { model: 'claude-opus-5', num_turns: 5, total_cost_usd: 0.05 }, + tools: { total_calls: 7, counts: [], last_calls: [], shown_calls: 0, complete: true }, + }, + }, + }, + }), + entry({ entry_type: 'step.attempt.started', step_id: 'verify', attempt: 1, payload: { lease_deadline_ms: 1 } }), + ]; +} + +describe('mirrorJournal', () => { + it('publishes a live view and a final row for every step the journal shows', () => { + const mirrored = mirrorJournal('01RUN', journal(), 1_700_000_010_000, {}); + + expect(mirrored.status).toBe('running'); + expect(mirrored.terminal).toBe(false); + expect(mirrored.steps.map(step => [step.stepName, step.state])).toEqual([ + ['write', 'done'], + ['verify', 'running'], + ]); + // Only the finished step is reported as final; a running one has no outcome. + expect(mirrored.finals).toHaveLength(1); + expect(mirrored.finals[0]).toMatchObject({ + stepName: 'write', + stepType: 'agent', + status: 'completed', + completionReason: 'success', + exitCode: 0, + // Two attempts ran, so one was a retry. + retryCount: 1, + model: 'claude-opus-5', + // Summed across attempts, not taken from the last one. + tokensInput: 30, + tokensOutput: 12, + costUsd: 0.05, + // Named after the transcript object only once one is uploaded. + sandboxId: '', + }); + }); + + it('carries the whole attempt roster in detail, and no local path with it', () => { + const [step] = mirrorJournal('01RUN', journal(), 1_700_000_010_000, {}).finals; + const detail = step!.detail as { attempts: unknown[]; transcript: { file: Record } }; + + expect(detail.attempts).toHaveLength(2); + expect(detail.attempts[0]).toMatchObject({ attempt: 1, completionReason: 'agent_error', disposition: 'retry' }); + // The digest's byte counts are useful; the path names a file on this + // machine that no dashboard reader can open. + expect(detail.transcript.file).toMatchObject({ bytes_kept: 90 }); + expect(detail.transcript.file).not.toHaveProperty('path'); + expect(JSON.stringify(detail)).not.toContain('/home/dev'); + }); + + it('names every attempt transcript on disk, in attempt order', () => { + const { transcripts } = mirrorJournal('01RUN', journal(), 1_700_000_010_000, {}); + expect(transcripts).toEqual([{ + stepName: 'write', + attempts: [ + { attempt: 1, path: '/home/dev/.relayflowd/runs/01RUN/steps/write/attempt-1.transcript.jsonl' }, + { attempt: 2, path: '/home/dev/.relayflowd/runs/01RUN/steps/write/attempt-2.transcript.jsonl' }, + ], + }]); + }); + + it('reports a failed step with its own failure excerpt, redacted', () => { + const events = journal().slice(0, 3); + events[2] = entry({ + ...events[2]!, + payload: { + ...(events[2]!.payload as Record), + disposition: 'step_done', + trajectory_tail: { + transcript: { + result: { model: 'claude-opus-5' }, + failure: { kind: 'result', excerpt: 'refused: Bearer sk-ant-abcdefghijkl' }, + }, + }, + }, + }); + const [step] = mirrorJournal('01RUN', events, 1_700_000_010_000, {}).finals; + + expect(step).toMatchObject({ status: 'failed', exitCode: 1, completionReason: 'agent_error' }); + expect(step!.error).toContain('[redacted]'); + expect(step!.error).not.toContain('sk-ant-abcdefghijkl'); + }); + + it('never publishes the authored root as a step of its own flow', () => { + seq = 0; + const events = [ + entry({ + entry_type: 'run.spawned', + payload: { spec: { name: 'authored', steps: [{ id: 'authored-root', type: 'deterministic', after: [] }] } }, + }), + entry({ entry_type: 'step.attempt.started', step_id: 'authored-root', attempt: 1, payload: {} }), + ]; + expect(mirrorJournal('01ROOT', events, 1_700_000_010_000, {}).steps).toEqual([]); + }); + + it('reads the child journals and graph edges out of an authored root index', () => { + seq = 0; + const events = [ + entry({ entry_type: 'run.spawned', payload: { spec: { name: 'authored', steps: [] } } }), + entry({ + entry_type: 'stream.appended', + payload: { + stream: 'authored-steps', + message: { index: 'relayflows.authored-step.v1', step: 'review', runId: '01CHILD', state: 'admitted', label: 'Review the PR' }, + }, + }), + entry({ + entry_type: 'stream.appended', + payload: { + stream: 'authored-steps', + // A completion record that carries no label must not erase the one + // the admission carried. + message: { index: 'relayflows.authored-step.v1', step: 'review', runId: '01CHILD', state: 'completed', completionReason: 'success', after: ['plan'] }, + }, + }), + ]; + const mirrored = mirrorJournal('01ROOT', events, 1_700_000_010_000, {}); + + expect(mirrored.children).toEqual(['01CHILD']); + expect(mirrored.hints.get('01CHILD/review')).toEqual({ label: 'Review the PR', after: ['plan'] }); + }); +}); + +describe('the bounds Cloud enforces', () => { + it('redacts before it clips, so no secret survives as a prefix', () => { + const secret = 'sk-ant-0123456789abcdefghijklmnopqrstuvwxyz'; + expect(identifier(secret, {})).not.toContain('sk-ant-0123'); + expect(identifier('a'.repeat(400), {})).toHaveLength(IDENTIFIER_MAX_CHARS); + // `[redacted]` is not identifier-shaped, so it is normalized rather than dropped. + expect(identifier(secret, {})).toMatch(/^[A-Za-z0-9_.:/-]+$/u); + }); + + it('normalizes a label the way workflow_steps.display_name is normalized', () => { + expect(label(' Review the PR \n', {})).toBe('Review the PR'); + expect(label('', {})).toBeUndefined(); + expect(label('x'.repeat(400), {})?.length).toBe(LABEL_MAX_CHARS); + }); + + it('derives a live step status the final report will agree with', () => { + expect(snapshotStepStatus({ state: 'running' })).toBe('running'); + expect(snapshotStepStatus({ state: 'done', completionReason: 'success' })).toBe('completed'); + expect(snapshotStepStatus({ state: 'done', completionReason: 'agent_error' })).toBe('failed'); + // A `done` step whose reason is missing is NOT quietly called a success. + expect(snapshotStepStatus({ state: 'done' })).toBe('failed'); + }); + + it('caps a long run to its live tail and says how many steps it left out', () => { + const steps = Array.from({ length: SNAPSHOT_MAX_STEPS + 10 }, (_unused, index) => ({ + stepName: `step-${index}`, + journalRunId: '01RUN', + stepType: 'agent', + state: 'done' as const, + attempt: 1, + elapsedMs: 1, + })); + const snapshot = fitSnapshot(steps, { sequence: 1, capturedAt: '2026-09-24T00:00:00.000Z' }); + + expect(snapshot.steps).toHaveLength(SNAPSHOT_MAX_STEPS); + expect(snapshot.steps[0]!.stepName).toBe('step-10'); + expect(snapshot.truncated).toBe(true); + expect(snapshot.omittedStepCount).toBe(10); + }); + + it('sheds artifact names before whole steps to fit the byte cap', () => { + const steps = Array.from({ length: SNAPSHOT_MAX_STEPS }, (_unused, index) => ({ + stepName: `step-${index}`, + journalRunId: '01RUN', + stepType: 'agent', + state: 'done' as const, + attempt: 1, + elapsedMs: 1, + artifacts: Array.from({ length: 5 }, (_u, a) => `out/${'p'.repeat(190)}-${index}-${a}`), + })); + const snapshot = fitSnapshot(steps, { sequence: 2, capturedAt: '2026-09-24T00:00:00.000Z' }); + + expect(Buffer.byteLength(JSON.stringify(snapshot), 'utf8')).toBeLessThanOrEqual(SNAPSHOT_MAX_BYTES); + expect(snapshot.steps).toHaveLength(SNAPSHOT_MAX_STEPS); + expect(snapshot.steps.every(step => step.artifacts === undefined)).toBe(true); + expect(snapshot.truncated).toBe(true); + }); + + it('drops an edge to a step the report does not contain', () => { + const hints = new Map([['01RUN/verify', { label: 'Verify', after: ['write', 'gone'] }]]); + const applied = withGraphHints( + { stepName: 'verify', journalRunId: '01RUN' }, hints, '01RUN', 32, new Set(['write', 'verify']), + ); + expect(applied).toMatchObject({ label: 'Verify', dependsOn: ['write'] }); + }); +}); diff --git a/packages/sdk/tests/cloud-mirror-transport.test.ts b/packages/sdk/tests/cloud-mirror-transport.test.ts new file mode 100644 index 000000000..d75804a07 --- /dev/null +++ b/packages/sdk/tests/cloud-mirror-transport.test.ts @@ -0,0 +1,144 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { CloudFlowError } from '../src/cloud-http.js'; +import { MirrorClient, registerLocalRun } from '../src/cloud-mirror-transport.js'; + +type Seen = { url: string; method: string; authorization: string | null; body: string | undefined }; + +/** A Cloud stand-in at the fetch boundary, so the URL a credential reaches is observable. */ +function server(response: (url: string) => { status: number; body: unknown }) { + const seen: Seen[] = []; + const fetchMock = vi.fn(async (url: string | URL, init: RequestInit = {}) => { + const target = String(url); + const headers = init.headers as Record | undefined; + seen.push({ + url: target, + method: init.method ?? 'GET', + authorization: headers?.['authorization'] ?? null, + body: typeof init.body === 'string' ? init.body : undefined, + }); + const { status, body } = response(target); + return { + ok: status < 300, + status, + json: async () => body, + } as unknown as Response; + }); + vi.stubGlobal('fetch', fetchMock); + return seen; +} + +const REGISTERED = { + runId: 'cloud-run', + status: 'running', + dispatchType: 'local', + callbackToken: 'cb-token', + accessToken: 'cld_at_run', + refreshToken: 'cld_rt_run', + runUrl: 'https://staging.example.com/cloud/dashboard/workflow/cloud-run/runner', +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('registerLocalRun', () => { + it('sends the exact source and takes the run-bound credential back', async () => { + const seen = server(() => ({ status: 201, body: REGISTERED })); + + const registration = await registerLocalRun( + { workflow: 'name: demo\n', fileType: 'yaml' }, + { token: 'cld_at_operator', apiUrl: 'https://staging.example.com/cloud' }, + ); + + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ + url: 'https://staging.example.com/cloud/api/v1/workflows/local-run', + method: 'POST', + // Registration is the operator's call; everything after it is the run's. + authorization: 'Bearer cld_at_operator', + }); + expect(JSON.parse(seen[0]!.body!)).toEqual({ + workflow: 'name: demo\n', fileType: 'yaml', relayflowVersion: 'v2', + }); + expect(registration).toMatchObject({ + runId: 'cloud-run', token: 'cld_at_run', callbackToken: 'cb-token', + apiUrl: 'https://staging.example.com/cloud', + }); + }); + + it('refuses a receipt with no usable credential rather than reporting nowhere', async () => { + server(() => ({ status: 201, body: { ...REGISTERED, accessToken: ' ' } })); + await expect(registerLocalRun( + { workflow: 'name: demo\n', fileType: 'yaml' }, + { token: 'cld_at_operator', apiUrl: 'https://staging.example.com/cloud' }, + )).rejects.toBeInstanceOf(CloudFlowError); + }); +}); + +describe('MirrorClient', () => { + /** + * The deployment is pinned by the registration, not re-resolved per call. + * + * `cloudConnection` falls back to the production default once an explicit + * token is supplied — and every call after registration supplies one — so + * without the pin a CLI signed in to a staging deployment would send that + * deployment's run token to `agentrelay.com`. + */ + it('sends every later call to the deployment that issued the credential', async () => { + const seen = server(() => ({ status: 200, body: { ok: true } })); + const client = new MirrorClient({ + runId: 'cloud-run', + token: 'cld_at_run', + callbackToken: 'cb-token', + runUrl: REGISTERED.runUrl, + apiUrl: 'https://staging.example.com/cloud', + }); + + await client.publishEvent({ eventType: 'relayflow.step.started', stepName: 'write' }); + await client.publishSnapshot({ sequence: 1, capturedAt: '2026-09-24T00:00:00.000Z', steps: [] }); + await client.publishSteps([], 0); + await client.putObject('write/agent.log', Buffer.from('{}\n')); + await client.reportTerminal('completed', { status: 'completed' }); + + expect(seen.map(call => call.url)).toEqual([ + 'https://staging.example.com/cloud/api/v1/workflows/runs/cloud-run/events', + 'https://staging.example.com/cloud/api/v1/workflows/runs/cloud-run/steps/snapshot', + 'https://staging.example.com/cloud/api/v1/workflows/runs/cloud-run/steps', + 'https://staging.example.com/cloud/api/v1/workflows/runs/cloud-run/storage/write/agent.log', + 'https://staging.example.com/cloud/api/v1/workflows/callback', + ]); + expect(seen.every(call => call.authorization === 'Bearer cld_at_run')).toBe(true); + expect(seen[3]!.method).toBe('PUT'); + }); + + it('answers false instead of throwing when Cloud refuses a push', async () => { + server(() => ({ status: 503, body: { error: 'unavailable' } })); + const client = new MirrorClient({ + runId: 'cloud-run', + token: 'cld_at_run', + callbackToken: 'cb-token', + runUrl: REGISTERED.runUrl, + apiUrl: 'https://staging.example.com/cloud', + }); + + await expect(client.publishSnapshot({ sequence: 1, capturedAt: 'x', steps: [] })).resolves.toBe(false); + await expect(client.publishSteps([], 0)).resolves.toBe(false); + await expect(client.reportTerminal('failed', {})).resolves.toBe(false); + }); + + it('refuses a storage key that could escape the run prefix, without a request', async () => { + const seen = server(() => ({ status: 200, body: { ok: true } })); + const client = new MirrorClient({ + runId: 'cloud-run', + token: 'cld_at_run', + callbackToken: 'cb-token', + runUrl: REGISTERED.runUrl, + apiUrl: 'https://staging.example.com/cloud', + }); + + for (const key of ['../other/agent.log', '/runner.log', 'a/../../b.log', '']) { + await expect(client.putObject(key, Buffer.from('x'))).resolves.toBe(false); + } + expect(seen).toHaveLength(0); + }); +}); diff --git a/packages/sdk/tests/cloud-mirror.test.ts b/packages/sdk/tests/cloud-mirror.test.ts new file mode 100644 index 000000000..de731940f --- /dev/null +++ b/packages/sdk/tests/cloud-mirror.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, it, vi } from 'vitest'; +import { assembleTranscript, createRunMirror, MIRROR_MAX_FINAL_STEPS } from '../src/cloud-mirror.js'; +import { MirrorClient } from '../src/cloud-mirror-transport.js'; +import { JournalReadError, type JournalEvent } from '../src/journal-reader.js'; + +type Call = { kind: string; body: unknown }; + +/** A Cloud stand-in: records every push, answers however the test says. */ +function cloud(accept: (kind: string) => boolean = () => true) { + const calls: Call[] = []; + const client = { + runId: 'cloud-run', + runUrl: 'https://agentrelay.com/cloud/dashboard/workflow/cloud-run/runner', + publishEvent: vi.fn(async (event: unknown) => { calls.push({ kind: 'event', body: event }); return accept('event'); }), + publishSnapshot: vi.fn(async (snapshot: unknown) => { calls.push({ kind: 'snapshot', body: snapshot }); return accept('snapshot'); }), + publishSteps: vi.fn(async (steps: unknown, omitted: unknown) => { + calls.push({ kind: 'steps', body: { steps, omitted } }); + return accept('steps'); + }), + putObject: vi.fn(async (key: string, bytes: Uint8Array) => { + calls.push({ kind: 'object', body: { key, bytes: Buffer.from(bytes).toString('utf8') } }); + return accept('object'); + }), + reportTerminal: vi.fn(async (status: unknown, result: unknown) => { + calls.push({ kind: 'terminal', body: { status, result } }); + return accept('terminal'); + }), + }; + return { client: client as unknown as MirrorClient, calls, spies: client }; +} + +let seq = 0; +function entry(partial: Partial & { entry_type: string; run_id: string }): JournalEvent { + seq += 1; + return { seq, segment_id: 1, step_id: null, attempt: null, at_ms: 1_700_000_000_000 + seq, payload: null, ...partial }; +} + +function rootJournal(runId: string, childRunId: string): JournalEvent[] { + return [ + entry({ entry_type: 'run.spawned', run_id: runId, payload: { spec: { name: 'authored', steps: [] } } }), + entry({ + entry_type: 'stream.appended', + run_id: runId, + payload: { + stream: 'authored-steps', + message: { + index: 'relayflows.authored-step.v1', + step: 'write', runId: childRunId, state: 'admitted', label: 'Write the patch', + }, + }, + }), + ]; +} + +function childJournal(runId: string, transcriptPath?: string): JournalEvent[] { + return [ + entry({ + entry_type: 'run.spawned', + run_id: runId, + payload: { spec: { name: 'child', steps: [{ id: 'write', type: 'agent', after: [] }] } }, + }), + entry({ entry_type: 'step.attempt.started', run_id: runId, step_id: 'write', attempt: 1, payload: {} }), + entry({ + entry_type: 'step.completed', + run_id: runId, + step_id: 'write', + attempt: 1, + payload: { + completionReason: 'success', + disposition: 'step_done', + budget: { tokens_in: 4, tokens_out: 2, dollars: '0.01' }, + ...(transcriptPath === undefined ? {} : { + trajectory_tail: { transcript: { file: { path: transcriptPath, bytes_kept: 10 } } }, + }), + }, + }), + entry({ entry_type: 'run.completed', run_id: runId, payload: { completionReason: 'success' } }), + ]; +} + +describe('createRunMirror', () => { + it('reads only the journals this run owns', async () => { + seq = 0; + const { client } = cloud(); + const read = vi.fn(async (runId: string) => { + if (runId === '01ROOT') return rootJournal('01ROOT', '01CHILD'); + if (runId === '01CHILD') return childJournal('01CHILD'); + throw new JournalReadError('run_not_found', 'not this run'); + }); + const mirror = createRunMirror({ client, dataDir: '/data', readJournal: read, env: {} }); + + mirror.start('01ROOT'); + await mirror.finish({ status: 'completed' }); + + // The root, and the one child its index names. A data directory holding a + // hundred other runs contributes nothing. + expect(read.mock.calls.map(call => call[0])).toEqual(['01ROOT', '01CHILD']); + }); + + it('publishes the child steps under the authored names the root gave them', async () => { + seq = 0; + const { client, calls } = cloud(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + readJournal: async (runId: string) => + runId === '01ROOT' ? rootJournal('01ROOT', '01CHILD') : childJournal('01CHILD'), + }); + + mirror.start('01ROOT'); + await mirror.finish({ status: 'completed' }); + + const snapshot = calls.find(call => call.kind === 'snapshot')!.body as { steps: Array> }; + expect(snapshot.steps).toHaveLength(1); + expect(snapshot.steps[0]).toMatchObject({ + stepName: 'write', journalRunId: '01CHILD', state: 'done', label: 'Write the patch', + }); + const report = calls.find(call => call.kind === 'steps')!.body as { steps: Array> }; + expect(report.steps[0]).toMatchObject({ stepName: 'write', status: 'completed', label: 'Write the patch' }); + }); + + it('reports the terminal status last, after the transcripts and the final rows', async () => { + seq = 0; + const { client, calls } = cloud(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + readJournal: async (runId: string) => + runId === '01ROOT' ? rootJournal('01ROOT', '01CHILD') : childJournal('01CHILD', '/tmp/attempt-1.jsonl'), + readTranscript: async () => ({ bytes: Buffer.from('{"type":"result"}\n'), size: 18 }), + }); + + mirror.start('01ROOT'); + await mirror.finish({ status: 'completed', completionReason: 'success', log: ['RUN 01ROOT'] }); + + // Cloud revokes the run's credential at the terminal transition, so every + // write has to be in before it. + const kinds = calls.map(call => call.kind); + expect(kinds.at(-1)).toBe('terminal'); + expect(kinds.indexOf('object')).toBeLessThan(kinds.indexOf('steps')); + const transcript = calls.find(call => + call.kind === 'object' && (call.body as { key: string }).key.endsWith('agent.log'))!; + expect((transcript.body as { key: string }).key).toBe('write/agent.log'); + // Marked up in Cloud's own attempt vocabulary, so the dashboard renders a + // mirrored transcript exactly as it renders a hosted one. + expect((transcript.body as { bytes: string }).bytes) + .toBe('{"type":"relayflow.attempt","attempt":1,"bytes":18,"truncated":false}\n{"type":"result"}\n'); + // The row now names the object, and only because the object landed. + const report = calls.find(call => call.kind === 'steps')!.body as { steps: Array> }; + expect(report.steps[0]!.sandboxId).toBe('write'); + expect(calls.some(call => call.kind === 'object' && (call.body as { key: string }).key === 'runner.log')).toBe(true); + }); + + it('leaves the row without a transcript when the upload is refused', async () => { + seq = 0; + const { client, calls } = cloud(kind => kind !== 'object'); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + readJournal: async () => childJournal('01RUN', '/tmp/attempt-1.jsonl'), + readTranscript: async () => ({ bytes: Buffer.from('{}\n'), size: 3 }), + }); + + mirror.start('01RUN'); + await mirror.finish({ status: 'completed' }); + + const report = calls.find(call => call.kind === 'steps')!.body as { steps: Array> }; + expect(report.steps[0]!.sandboxId).toBe(''); + }); + + it('keeps a journal it could not read, rather than publishing an emptier view', async () => { + seq = 0; + const { client, calls } = cloud(); + let reads = 0; + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + intervalMs: 1, + readJournal: async () => { + reads += 1; + if (reads === 1) return childJournal('01RUN'); + throw new JournalReadError('journal_busy', 'a writer was mid-flight'); + }, + }); + + mirror.start('01RUN'); + await mirror.finish({ status: 'completed' }); + + const report = calls.find(call => call.kind === 'steps')!.body as { steps: unknown[] }; + expect(report.steps).toHaveLength(1); + }); + + it('caps the final report and says how many steps it left out', async () => { + seq = 0; + const overCap = MIRROR_MAX_FINAL_STEPS + 12; + const { client, calls } = cloud(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + readJournal: async (runId: string) => [ + entry({ + entry_type: 'run.spawned', + run_id: runId, + payload: { + spec: { + name: 'wide', + steps: Array.from({ length: overCap }, (_unused, index) => ({ + id: `step-${index}`, type: 'deterministic', after: [], + })), + }, + }, + }), + ...Array.from({ length: overCap }, (_unused, index) => [ + entry({ entry_type: 'step.attempt.started', run_id: runId, step_id: `step-${index}`, attempt: 1, payload: {} }), + entry({ + entry_type: 'step.completed', + run_id: runId, + step_id: `step-${index}`, + attempt: 1, + payload: { completionReason: 'success', disposition: 'step_done' }, + }), + ]).flat(), + ], + }); + + mirror.start('01RUN'); + await mirror.finish({ status: 'completed' }); + + const report = calls.find(call => call.kind === 'steps')!.body as { steps: unknown[]; omitted: number }; + expect(report.steps).toHaveLength(MIRROR_MAX_FINAL_STEPS); + // The count is by identity, so re-reading the same journal cannot inflate it. + expect(report.omitted).toBe(12); + }); + + it('never throws out of finish, whatever Cloud answers', async () => { + seq = 0; + const { client } = cloud(() => false); + const diagnostic = vi.fn(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + diagnostic, + readJournal: async () => { throw new Error('the disk is on fire'); }, + }); + + mirror.start('01RUN'); + await expect(mirror.finish({ status: 'failed', error: 'step failed' })).resolves.toBeUndefined(); + expect(diagnostic).toHaveBeenCalled(); + }); +}); + +describe('assembleTranscript', () => { + const read = async (path: string) => { + const body = `${path}\n`; + return { bytes: Buffer.from(body), size: body.length }; + }; + + it('orders attempts oldest-first, each behind its own marker', async () => { + const assembled = await assembleTranscript( + [{ attempt: 2, path: 'b' }, { attempt: 1, path: 'a' }], read, + ); + expect(assembled.bytes.toString('utf8').split('\n').filter(Boolean)).toEqual([ + '{"type":"relayflow.attempt","attempt":1,"bytes":2,"truncated":false}', + 'a', + '{"type":"relayflow.attempt","attempt":2,"bytes":2,"truncated":false}', + 'b', + ]); + expect(assembled).toMatchObject({ kept: 2, omitted: 0, truncated: false }); + }); + + it('marks an attempt it had to drop rather than leaving the reader to count', async () => { + const big = async () => ({ bytes: Buffer.alloc(400, 0x61), size: 400 }); + const assembled = await assembleTranscript( + [{ attempt: 1, path: 'a' }, { attempt: 2, path: 'b' }], big, 600, + ); + expect(assembled.omitted).toBe(1); + expect(assembled.kept).toBe(1); + expect(assembled.bytes.toString('utf8')).toContain('"type":"relayflow.attempt.omitted","attempt":1'); + }); + + it('counts an attempt whose file has gone, instead of pretending it never ran', async () => { + const missing = async () => { throw new Error('ENOENT'); }; + const assembled = await assembleTranscript([{ attempt: 1, path: 'a' }], missing); + expect(assembled).toMatchObject({ kept: 0, omitted: 1 }); + expect(assembled.bytes.toString('utf8')).toContain('relayflow.attempt.omitted'); + }); +}); diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index 018feb6b8..ea5fec6da 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -108,14 +108,14 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa { verb: 'resume', argv: ['resume', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', - '--no-observer-link', '--allow-human-influenced', RUN_ID], + '--no-observer-link', '--no-cloud-mirror', '--allow-human-influenced', RUN_ID], variant: 'resume', }, { verb: 'run', argv: ['run', 'flow.yaml'], variant: 'run' }, { verb: 'run', argv: ['run', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', - '--no-observer-link', '--allow-human-influenced', '--input', '{"a":1}', 'review.flow.ts'], + '--no-observer-link', '--no-cloud-mirror', '--allow-human-influenced', '--input', '{"a":1}', 'review.flow.ts'], variant: 'run', }, // `--input` is the authored body's argument and `--reuse-from` memoizes a From cc9252c569c7396684652b6af6903cb727f67335 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 11:28:04 -0700 Subject: [PATCH 2/7] feat(cli): upload the run's own output as runner.log, and prove the mirror end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things the first pass got wrong, both found by running it: The runner log was built from progress events, and a declarative run emits none — so the dashboard's log pane was empty for exactly the runs that are easiest to follow. It is now what this invocation actually printed, captured at the CLI's own IO seam, redacted on the way out: it is not a transcript a worker already scrubbed, it is whatever a developer's machine put on stderr, including diagnostics that can quote a command line or an environment value. The mirror also finished before the report was emitted, so the `RUN` line — the one line a reader most wants in the log — was written after the log was uploaded. It now settles last, after the report and the observer line, which is also what the surrounding comments already argue for: a run's exit code has never waited on Cloud. `cloud-mirror-live.test.ts` runs the built CLI against the real kernel on a real flow, with a local HTTPS server standing where Cloud would be, and asserts the bytes a mirrored run puts on the wire and the order it puts them in — registration under the operator credential, everything after it under the run's, the live view, the final rows, the log, and the terminal callback last. A second case proves both opt-outs send nothing at all. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/cli.ts | 39 ++-- packages/sdk/src/cloud-mirror.ts | 8 +- packages/sdk/tests/cloud-mirror-live.test.ts | 177 +++++++++++++++++++ 3 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 packages/sdk/tests/cloud-mirror-live.test.ts diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index ac1c6dc74..5c6cd9d13 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -318,10 +318,6 @@ export async function runCli( // effect of an invocation that is about to be refused for bad input. const startedSteps = new Map(); const observer = parsed.noObserverLink ? undefined : createObserverSession(parsed.command, io); - // What the run printed about itself, kept so the mirror can upload it as the - // run's `runner.log` — the object the dashboard's log pane reads. Bounded by - // the mirror before it is sent; kept whole here because the same lines are - // what a reader sees on stderr. const runnerLog: string[] = []; const mirror = parsed.noCloudMirror || !cloudMirrorEnabled(process.env) ? undefined @@ -332,19 +328,26 @@ export async function runCli( dataDir: parsed.dataDir, log: () => runnerLog, }, io); + // Everything this invocation prints, kept in order, so the mirror can upload + // it as the run's `runner.log` — the object the dashboard's log pane reads, + // and the sandbox's own equivalent. A declarative run emits no progress + // events at all, so building the log out of those would have left the pane + // empty for exactly the runs that are easiest to follow. + const logged: CliIo = mirror === undefined ? io : { + ...io, + stdout: line => { runnerLog.push(line); io.stdout(line); }, + stderr: line => { runnerLog.push(line); io.stderr(line); }, + }; const showProgress = (event: ProgressEvent): void => { if (event.type === 'step.started') startedSteps.set(event.stepId, performance.now()); - for (const line of renderProgress([event])) { - runnerLog.push(line); - if (!parsed.json) io.stderr(line); - } + if (!parsed.json) for (const line of renderProgress([event])) logged.stderr(line); observer?.onProgress(event); mirror?.onProgress(event); }; const lifecycle = { ...(parsed.command === 'run' ? { bucket: parsed.bucket } : {}), allowHumanInfluenced: parsed.allowHumanInfluenced, - onPtyReady: (path: string) => io.stderr(`PTY ${path}`), + onPtyReady: (path: string) => logged.stderr(`PTY ${path}`), ...(parsed.command === 'run' && parsed.reuseFromRunId !== undefined ? { reuseFromRunId: parsed.reuseFromRunId } : {}), localAgent: parsed.localAgent, ...(parsed.agentCapacity === undefined ? {} : { agentCapacity: parsed.agentCapacity }), @@ -360,7 +363,7 @@ export async function runCli( }, }), onWait: (progress: RunProgress) => { - emitWait(progress, io); + emitWait(progress, logged); const now = performance.now(); if (!startedSteps.has(progress.stepId)) startedSteps.set(progress.stepId, now); showProgress({ type: 'step.running', stepId: progress.stepId, stepType: progress.stepType, @@ -377,9 +380,6 @@ export async function runCli( // does. `finish` drains the projection and settles the mint, both bounded; // it never rejects (see `createObserverSession`). const observerMint = observer?.finish(execution.report); - // Bounded by the mirror itself, and it never rejects: a run's exit code has - // never waited on Cloud and does not start now. - await mirror?.finish(execution.report); // In `--json` mode the report is a single machine-readable object that // MUST carry `observerUrl` when one is available, so a consumer sees one // authoritative signal. That justifies blocking up to `MINT_TIMEOUT_MS` @@ -391,12 +391,17 @@ export async function runCli( // preflight) is worse than printing `Observer:` on a later line, so we // emit the run report immediately and finalize the observer link after. if (parsed.json) { - const observerUrl = await observerUrlFrom(observerMint, io); - emitRunReport(execution, parsed.json, io, observerUrl); + const observerUrl = await observerUrlFrom(observerMint, logged); + emitRunReport(execution, parsed.json, logged, observerUrl); + // Last, and after the report: the run's own output is what the mirror + // uploads, and a run's exit code has never waited on Cloud. The mirror + // bounds itself and never rejects. + await mirror?.finish(execution.report); return execution.exitCode; } - emitRunReport(execution, parsed.json, io); - await finalizeObserverLine(observerMint, io); + emitRunReport(execution, parsed.json, logged); + await finalizeObserverLine(observerMint, logged); + await mirror?.finish(execution.report); return execution.exitCode; } diff --git a/packages/sdk/src/cloud-mirror.ts b/packages/sdk/src/cloud-mirror.ts index d5d180b96..194a1c864 100644 --- a/packages/sdk/src/cloud-mirror.ts +++ b/packages/sdk/src/cloud-mirror.ts @@ -38,6 +38,7 @@ import { type FinalStep, type SnapshotStep, } from './cloud-mirror-step.js'; import { MirrorClient, type MirrorEvent } from './cloud-mirror-transport.js'; +import { redact } from './redact.js'; /** How often the journals are re-read. The sandbox reporter's own cadence. */ export const MIRROR_POLL_INTERVAL_MS = 10_000; @@ -424,8 +425,11 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { await scan(deadline); await publishSnapshot(deadline); if (outcome.log !== undefined && outcome.log.length > 0) { - const text = outcome.log.join('\n'); - const bytes = Buffer.from(text, 'utf8'); + // The CLI's own output, redacted on the way out. It is not a + // transcript the worker already scrubbed: it is whatever this + // invocation printed on a developer's machine, including diagnostics + // that can quote a command line or an environment value. + const bytes = Buffer.from(redact(outcome.log.join('\n'), env), 'utf8'); await options.client.putObject('runner.log', bytes.length > MIRROR_RUNNER_LOG_MAX_BYTES ? bytes.subarray(bytes.length - MIRROR_RUNNER_LOG_MAX_BYTES) diff --git a/packages/sdk/tests/cloud-mirror-live.test.ts b/packages/sdk/tests/cloud-mirror-live.test.ts new file mode 100644 index 000000000..4274474b6 --- /dev/null +++ b/packages/sdk/tests/cloud-mirror-live.test.ts @@ -0,0 +1,177 @@ +// The mirror against a real local run, end to end. +// +// Every other test here folds a hand-built journal or stubs the transport. +// This one runs the built CLI against the real kernel, on a real flow, and +// stands a local HTTPS server where Cloud would be — so what is asserted is +// the bytes a mirrored run actually puts on the wire, in the order it puts +// them, not a reconstruction of them. +// +// Local HTTPS only. No hosted run is claimed and nothing leaves this machine. + +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createServer, type Server } from 'node:https'; +import { homedir, tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const BUILT_CLI = join(ROOT, 'packages', 'sdk', 'dist', 'cli.js'); +const FLOW = join(ROOT, 'testdata', 'hello-deterministic.flow.yaml'); +const TOOLCHAIN_TARGET = process.env['CARGO_TARGET_DIR'] + ?? join(process.env['RELAYFLOWS_TOOLCHAIN_HOME'] ?? join(homedir(), '.relayflows-toolchain'), 'target'); + +/** Same resolution `live-kernel.test.ts` uses: ops/cargo.sh builds outside the repo. */ +function locateRelayflowd(): string { + const direct = join(TOOLCHAIN_TARGET, 'debug', 'relayflowd'); + if (existsSync(direct)) return direct; + const keyed = existsSync(TOOLCHAIN_TARGET) + ? readdirSync(TOOLCHAIN_TARGET) + .map(entry => join(TOOLCHAIN_TARGET, entry, 'debug', 'relayflowd')) + .filter(candidate => existsSync(candidate)) + .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs) + : []; + return keyed[0] ?? join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'); +} + +const RELAYFLOWD = resolve(process.env['RELAYFLOWD_BIN'] ?? locateRelayflowd()); + +interface Seen { + method: string; + url: string; + authorization: string | undefined; + body: unknown; +} + +let work: string; +let server: Server; +let origin: string; +let seen: Seen[] = []; + +/** One `flows run`, with the stub Cloud configured. */ +function runCli(args: readonly string[], extraEnv: NodeJS.ProcessEnv = {}): Promise<{ stderr: string; code: number }> { + return new Promise((done, reject) => { + const child = spawn(process.execPath, [BUILT_CLI, ...args], { + cwd: work, + env: { + ...process.env, + NODE_EXTRA_CA_CERTS: join(work, 'cert.pem'), + FLOWS_CLOUD_URL: origin, + FLOWS_CLOUD_TOKEN: 'operator-token', + RELAYFLOWD_BIN: RELAYFLOWD, + ...extraEnv, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', chunk => { stderr += chunk; }); + child.stdout.resume(); + child.on('error', reject); + child.on('close', code => done({ stderr, code: code ?? 1 })); + }); +} + +beforeAll(async () => { + expect(existsSync(RELAYFLOWD), `relayflowd at ${RELAYFLOWD} — build it with (cd kernel && ../ops/cargo.sh build)`).toBe(true); + expect(existsSync(BUILT_CLI), `built CLI at ${BUILT_CLI} — build it with (cd packages/sdk && npm run build)`).toBe(true); + + work = await mkdtemp(join(tmpdir(), 'cloud-mirror-live-')); + const config = join(work, 'openssl.cnf'); + await writeFile(config, '[req]\ndistinguished_name=dn\nx509_extensions=ext\n[dn]\n[ext]\n' + + 'subjectAltName=IP:127.0.0.1\nbasicConstraints=critical,CA:TRUE\n'); + const openssl = spawnSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-subj', '/CN=cloud-mirror-live', '-config', config, + '-keyout', join(work, 'key.pem'), '-out', join(work, 'cert.pem')], { cwd: work }); + expect(openssl.status, String(openssl.stderr)).toBe(0); + + server = createServer( + { key: readFileSync(join(work, 'key.pem')), cert: readFileSync(join(work, 'cert.pem')) }, + async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(chunk as Buffer); + const raw = Buffer.concat(chunks).toString('utf8'); + let body: unknown; + try { body = JSON.parse(raw); } catch { body = raw; } + seen.push({ + method: request.method ?? '', + url: request.url ?? '', + authorization: request.headers.authorization, + body, + }); + response.writeHead(request.url === '/api/v1/workflows/local-run' ? 201 : 200, + { 'content-type': 'application/json' }); + response.end(request.url === '/api/v1/workflows/local-run' + ? JSON.stringify({ + runId: 'live-cloud-run', status: 'running', dispatchType: 'local', + callbackToken: 'cb', accessToken: 'cld_at_run', refreshToken: 'cld_rt_run', + runUrl: `${origin}/dashboard/workflow/live-cloud-run/runner`, + }) + : JSON.stringify({ ok: true })); + }, + ); + await new Promise(done => server.listen(0, '127.0.0.1', done)); + const address = server.address(); + origin = `https://127.0.0.1:${typeof address === 'object' && address !== null ? address.port : 0}`; +}, 60_000); + +afterAll(async () => { + server?.closeAllConnections(); + await new Promise(done => server?.close(() => done())); + if (work) await rm(work, { recursive: true, force: true }); +}); + +describe('a local run on the Cloud dashboard', () => { + it('registers, publishes its steps and its log, and reports terminal last', async () => { + seen = []; + const run = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW]); + expect(run.code).toBe(0); + expect(run.stderr).toContain(`Dashboard: ${origin}/dashboard/workflow/live-cloud-run/runner`); + + // Registration is the operator's call; everything after it is the run's. + expect(seen[0]).toMatchObject({ + method: 'POST', url: '/api/v1/workflows/local-run', authorization: 'Bearer operator-token', + }); + const registration = seen[0]!.body as { workflow: string; fileType: string; relayflowVersion: string }; + expect(registration.fileType).toBe('yaml'); + expect(registration.relayflowVersion).toBe('v2'); + expect(registration.workflow).toContain('name: hello-deterministic'); + + const reports = seen.slice(1); + expect(reports.every(call => call.authorization === 'Bearer cld_at_run')).toBe(true); + + const snapshot = reports.find(call => call.url.endsWith('/steps/snapshot'))!; + const live = snapshot.body as { steps: Array<{ stepName: string; state: string; journalRunId: string }> }; + expect(live.steps.map(step => [step.stepName, step.state])) + .toEqual([['greet', 'done'], ['shout', 'done']]); + expect(live.steps[0]!.journalRunId).toMatch(/^[A-Za-z0-9][A-Za-z0-9_-]*$/u); + + const final = reports.find(call => call.url === '/api/v1/workflows/runs/live-cloud-run/steps')!; + const rows = (final.body as { steps: Array<{ stepName: string; status: string; completionReason: string }> }).steps; + expect(rows.map(row => [row.stepName, row.status, row.completionReason])) + .toEqual([['greet', 'completed', 'success'], ['shout', 'completed', 'success']]); + + const log = reports.find(call => call.url.endsWith('/storage/runner.log'))!; + expect(String(log.body)).toContain('greet'); + + // Cloud revokes the run credential at the terminal transition, so every + // write has to be in before it. + const terminal = reports.at(-1)!; + expect(terminal.url).toBe('/api/v1/workflows/callback'); + expect(terminal.body).toMatchObject({ status: 'completed', callbackToken: 'cb' }); + }, 120_000); + + it('sends nothing at all under either opt-out', async () => { + seen = []; + const flagged = await runCli(['run', '--no-cloud-mirror', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW]); + const shell = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW], + { FLOWS_CLOUD_MIRROR: '0' }); + + expect(flagged.code).toBe(0); + expect(shell.code).toBe(0); + expect(seen).toEqual([]); + expect(flagged.stderr).not.toContain('Dashboard:'); + expect(shell.stderr).not.toContain('Dashboard:'); + }, 120_000); +}); From 08447d9ccba8f651161894bdf8dcb02b4a4cd82e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 11:44:32 -0700 Subject: [PATCH 3/7] fix(cli): carry the completion report on the mirror's terminal callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A finished local run showed up red on the dashboard. Cloud reconciles a v2 run's reported status against the report the callback carries: `reconcileTerminalStatus` records `failed` for any `completed` callback whose result does not prove success with `ok`, `status`, `completionReason` and `runId` together. That guard exists because a sandbox whose bootstrap exits cleanly must not report a run that died at step 1 as green — and it cuts the other way too. The mirror sent a bare `{status, completionReason}`, so every successful mirrored run was reconciled to `failed`. The callback now carries the run's own report, built from the CLI's `RunReport` — which is where those four fields already live, and which is what the hosted path posts. The run list reads `completionReason` out of the same document, so a parked local run now renders as "Needs review" rather than a bare failure. Diagnostics are the one unbounded free-text field in a report that is stored whole, so they are capped at 20 and each message clipped, with the drop counted. The whole document is redacted leaf by leaf rather than as serialized text: the redactor's value patterns end in `\S+`, which across JSON would swallow the closing quote and the next key and hand Cloud something it cannot parse. Found by running the thing end to end against the real Cloud route handlers rather than a stub — the stub answered 200 to anything and had no opinion about what it was sent. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/cli/cloud-mirror-session.ts | 46 +++++++++++++ packages/sdk/src/cloud-mirror.ts | 37 +++++++++-- .../sdk/tests/cloud-mirror-session.test.ts | 65 +++++++++++++++++++ packages/sdk/tests/cloud-mirror.test.ts | 14 ++-- 4 files changed, 150 insertions(+), 12 deletions(-) diff --git a/packages/sdk/src/cli/cloud-mirror-session.ts b/packages/sdk/src/cli/cloud-mirror-session.ts index 7be2aca8a..d262768c0 100644 --- a/packages/sdk/src/cli/cloud-mirror-session.ts +++ b/packages/sdk/src/cli/cloud-mirror-session.ts @@ -124,6 +124,7 @@ export function createCloudMirrorSession( status: report.completionReason === 'canceled' ? 'cancelled' : report.ok ? 'completed' : 'failed', + result: completionReport(report), ...(report.completionReason === undefined ? {} : { completionReason: report.completionReason }), ...(terminalError(report) === undefined ? {} : { error: terminalError(report)! }), log: request.log(), @@ -181,6 +182,51 @@ export function mirrorSourceFromJournal(dataDir: string): (runId: string) => Pro }; } +/** Diagnostics one completion report carries; the rest are counted. */ +const MAX_REPORT_DIAGNOSTICS = 20; +/** One diagnostic message, in code points. */ +const MAX_DIAGNOSTIC_CHARS = 2_000; + +/** + * The run's completion report, as the terminal callback stores it. + * + * Cloud reconciles a v2 run's reported status against this document: a + * `completed` callback whose report does not carry `ok`, `status`, + * `completionReason` and `runId` together is recorded as `failed`. That guard + * exists so a sandbox whose bootstrap exits cleanly cannot report a run that + * died at step 1 as green — and it cuts the other way too, so a mirror that + * omits the report turns a finished local run red. Those four fields are the + * contract; everything else here is what a reader of the run page gets for + * free, since the run list extracts `completionReason` from the same document. + * + * Diagnostics are bounded and their messages clipped. They are the one part of + * a report that is unbounded free text, and this document is stored whole. + */ +function completionReport(report: RunReport): Record { + const diagnostics = report.diagnostics.slice(0, MAX_REPORT_DIAGNOSTICS).map(entry => ({ + severity: 'severity' in entry ? entry.severity : 'refusal', + kind: entry.kind, + message: [...entry.message].slice(0, MAX_DIAGNOSTIC_CHARS).join(''), + })); + return { + ok: report.ok, + command: report.command, + ...(report.runId === undefined ? {} : { runId: report.runId }), + ...(report.rootRunId === undefined ? {} : { rootRunId: report.rootRunId }), + ...(report.status === undefined ? {} : { status: report.status }), + ...(report.completionReason === undefined ? {} : { completionReason: report.completionReason }), + ...(report.completionDetail === undefined ? {} : { completionDetail: report.completionDetail }), + ...(report.completedSteps === undefined ? {} : { completedSteps: report.completedSteps }), + ...(report.parkedStep === undefined ? {} : { parkedStep: report.parkedStep }), + ...(report.parkCause === undefined ? {} : { parkCause: report.parkCause }), + ...(report.next === undefined ? {} : { next: report.next }), + ...(diagnostics.length === 0 ? {} : { diagnostics }), + ...(report.diagnostics.length > diagnostics.length + ? { diagnosticsOmitted: report.diagnostics.length - diagnostics.length } + : {}), + }; +} + /** * Why the mirror is not running, in one line a reader can act on. * diff --git a/packages/sdk/src/cloud-mirror.ts b/packages/sdk/src/cloud-mirror.ts index 194a1c864..b785933d8 100644 --- a/packages/sdk/src/cloud-mirror.ts +++ b/packages/sdk/src/cloud-mirror.ts @@ -81,6 +81,18 @@ export interface RunMirrorOptions { /** What the mirror was told about the run when it ended. */ export interface RunMirrorOutcome { status: 'completed' | 'failed' | 'cancelled'; + /** + * The run's own completion report, as the terminal callback stores it. + * + * Not decoration. Cloud reconciles a v2 run's reported status against this + * document and records `failed` for anything that does not prove success — + * the guard that stops a sandbox whose bootstrap exited cleanly from + * reporting a run that died at step 1 as green. A mirror that sent a status + * without the report was reconciled the other way, and a finished local run + * showed up red. The run list also reads `completionReason` and + * `pullRequestUrl` straight out of it. + */ + result: Record; completionReason?: string; error?: string; /** Lines the CLI printed for this run, uploaded as `runner.log`. */ @@ -441,11 +453,7 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { // above depends on. await options.client.reportTerminal( outcome.status, - { - status: outcome.status === 'completed' ? 'completed' : outcome.status, - ...(outcome.completionReason === undefined ? {} : { completionReason: outcome.completionReason }), - ...(outcome.error === undefined ? {} : { error: outcome.error }), - }, + redactJson(outcome.result, env) as Record, outcome.error, ); } catch (error) { @@ -457,6 +465,25 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { }; } +/** + * Redact every string the report carries, leaf by leaf. + * + * Not `redact(JSON.stringify(...))`: the redactor's value patterns end in + * `\\S+`, which across serialized JSON would swallow the closing quote and the + * next key, and hand Cloud a document it cannot parse. Redacting leaves keeps + * the shape intact and still scrubs every free-text field. + */ +function redactJson(value: unknown, env: NodeJS.ProcessEnv): unknown { + if (typeof value === 'string') return redact(value, env); + if (Array.isArray(value)) return value.map(entry => redactJson(entry, env)); + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record).map(([key, entry]) => [key, redactJson(entry, env)]), + ); + } + return value; +} + /** Everything about a step but its ever-moving elapsed time. */ function fingerprint(step: SnapshotStep): string { const { elapsedMs: _elapsedMs, ...rest } = step; diff --git a/packages/sdk/tests/cloud-mirror-session.test.ts b/packages/sdk/tests/cloud-mirror-session.test.ts index 2c6a4d291..c9adfcf18 100644 --- a/packages/sdk/tests/cloud-mirror-session.test.ts +++ b/packages/sdk/tests/cloud-mirror-session.test.ts @@ -125,6 +125,71 @@ describe('createCloudMirrorSession', () => { expect(register).not.toHaveBeenCalled(); }); + /** + * Cloud reconciles a v2 run's reported status against the report it carries: + * a `completed` callback whose result lacks `ok`, `status`, + * `completionReason` or `runId` is recorded as `failed`. The first version of + * this mirror sent a status with no report, and every finished local run + * showed up red on the dashboard. + */ + it('carries the four fields Cloud reconciles a successful v2 run against', async () => { + const { io: cli } = io(); + const mirror = { runId: 'cloud-run', runUrl: 'u', start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); + + session.onRunStarted({ runId: '01RUN' }); + await session.finish({ + ok: true, + command: 'run', + runId: '01RUN', + status: 'completed', + completionReason: 'success', + completedSteps: 2, + resolutions: [], + diagnostics: [], + }); + + const outcome = mirror.finish.mock.calls[0]![0] as { status: string; result: Record }; + expect(outcome.status).toBe('completed'); + expect(outcome.result).toMatchObject({ + ok: true, status: 'completed', completionReason: 'success', runId: '01RUN', completedSteps: 2, + }); + }); + + it('bounds the diagnostics the report carries, and says how many it dropped', async () => { + const { io: cli } = io(); + const mirror = { runId: 'cloud-run', runUrl: 'u', start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); + + session.onRunStarted({ runId: '01RUN' }); + await session.finish({ + ok: false, + command: 'run', + runId: '01RUN', + resolutions: [], + diagnostics: Array.from({ length: 25 }, (_unused, index) => ({ + severity: 'failure' as const, + kind: 'step_failed' as const, + message: `${index}:${'x'.repeat(5_000)}`, + })), + }); + + const result = (mirror.finish.mock.calls[0]![0] as { result: Record }).result; + const diagnostics = result.diagnostics as Array<{ message: string }>; + expect(diagnostics).toHaveLength(20); + expect(result.diagnosticsOmitted).toBe(5); + // The report is stored whole, and a diagnostic is the one unbounded field. + expect(diagnostics.every(entry => [...entry.message].length <= 2_000)).toBe(true); + }); + it('reports a park as a failed run carrying its completion reason', async () => { const { io: cli } = io(); const mirror = { runId: 'cloud-run', runUrl: 'u', start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; diff --git a/packages/sdk/tests/cloud-mirror.test.ts b/packages/sdk/tests/cloud-mirror.test.ts index de731940f..9c26099e9 100644 --- a/packages/sdk/tests/cloud-mirror.test.ts +++ b/packages/sdk/tests/cloud-mirror.test.ts @@ -90,7 +90,7 @@ describe('createRunMirror', () => { const mirror = createRunMirror({ client, dataDir: '/data', readJournal: read, env: {} }); mirror.start('01ROOT'); - await mirror.finish({ status: 'completed' }); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); // The root, and the one child its index names. A data directory holding a // hundred other runs contributes nothing. @@ -109,7 +109,7 @@ describe('createRunMirror', () => { }); mirror.start('01ROOT'); - await mirror.finish({ status: 'completed' }); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); const snapshot = calls.find(call => call.kind === 'snapshot')!.body as { steps: Array> }; expect(snapshot.steps).toHaveLength(1); @@ -133,7 +133,7 @@ describe('createRunMirror', () => { }); mirror.start('01ROOT'); - await mirror.finish({ status: 'completed', completionReason: 'success', log: ['RUN 01ROOT'] }); + await mirror.finish({ status: 'completed', completionReason: 'success', result: { ok: true, status: 'completed', completionReason: 'success', runId: '01ROOT' }, log: ['RUN 01ROOT'] }); // Cloud revokes the run's credential at the terminal transition, so every // write has to be in before it. @@ -165,7 +165,7 @@ describe('createRunMirror', () => { }); mirror.start('01RUN'); - await mirror.finish({ status: 'completed' }); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); const report = calls.find(call => call.kind === 'steps')!.body as { steps: Array> }; expect(report.steps[0]!.sandboxId).toBe(''); @@ -188,7 +188,7 @@ describe('createRunMirror', () => { }); mirror.start('01RUN'); - await mirror.finish({ status: 'completed' }); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); const report = calls.find(call => call.kind === 'steps')!.body as { steps: unknown[] }; expect(report.steps).toHaveLength(1); @@ -229,7 +229,7 @@ describe('createRunMirror', () => { }); mirror.start('01RUN'); - await mirror.finish({ status: 'completed' }); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); const report = calls.find(call => call.kind === 'steps')!.body as { steps: unknown[]; omitted: number }; expect(report.steps).toHaveLength(MIRROR_MAX_FINAL_STEPS); @@ -250,7 +250,7 @@ describe('createRunMirror', () => { }); mirror.start('01RUN'); - await expect(mirror.finish({ status: 'failed', error: 'step failed' })).resolves.toBeUndefined(); + await expect(mirror.finish({ status: 'failed', error: 'step failed', result: { ok: false } })).resolves.toBeUndefined(); expect(diagnostic).toHaveBeenCalled(); }); }); From a1af4767713213c06fc0fa715571ee52a8eae92c Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 12:03:25 -0700 Subject: [PATCH 4/7] feat(cli): name the Cloud run, and link a resume to the attempt it continues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps a developer hits the moment they use this for real. **The run had no handle.** The terminal printed a dashboard URL and nothing else, while the report's own `runId` is the *journal's* — and every hosted read verb (`flows status --cloud`, `flows logs`, `flows runs`) takes Cloud's. A human had to pick one out of a URL; a script had none at all, since `--json` carried `observerUrl` but nothing about the dashboard. The terminal line now ends with the command that follows the same run, and `--json` carries `cloudRunId` and `dashboardUrl` beside `observerUrl`. **A resumed run read as an unrelated second run.** Cloud refuses to move a terminal run back to `running`, so a resume is a new row — its own hosted resume mints a fresh id too. What made that read badly is that the resuming process had no idea the first attempt existed. `/cloud-runs/` now records which Cloud run mirrored which journal, so a resume can name its predecessor; Cloud verifies the caller can read it and stores it, and the run page links the two. The ledger holds the run id and the deployment and **nothing else** — no credential, because a resume mints its own and a token per run in a dotfile under someone's home directory is a worse trade than the feature is worth. Everything in it was already in the URL the first attempt printed. Entries age out at 30 days and 500 rows. Proving that second one found a third bug: a resume reads its journal while the daemon is writing it, hit `journal_busy`, and the mirror gave up — so resumes largely did not reach the dashboard at all. `flows status` has always retried that case; both journal readers now do, and a busy journal no longer prints a diagnostic once per poll, because a hot journal is what a running flow looks like. Co-Authored-By: Claude Opus 5 (1M context) --- docs/CLOUD.md | 26 ++- packages/sdk/src/cli.ts | 16 +- packages/sdk/src/cli/cloud-mirror-session.ts | 60 ++++++- packages/sdk/src/cloud-mirror-ledger.ts | 161 ++++++++++++++++++ packages/sdk/src/cloud-mirror-transport.ts | 11 ++ packages/sdk/src/cloud-mirror.ts | 53 +++++- .../sdk/tests/cloud-mirror-ledger.test.ts | 88 ++++++++++ .../sdk/tests/cloud-mirror-session.test.ts | 23 ++- packages/sdk/tests/cloud-mirror.test.ts | 65 ++++++- 9 files changed, 477 insertions(+), 26 deletions(-) create mode 100644 packages/sdk/src/cloud-mirror-ledger.ts create mode 100644 packages/sdk/tests/cloud-mirror-ledger.test.ts diff --git a/docs/CLOUD.md b/docs/CLOUD.md index 00ac3f8a4..17bba742f 100644 --- a/docs/CLOUD.md +++ b/docs/CLOUD.md @@ -139,9 +139,14 @@ hosted run gets, and does so by default: ```sh flows run review.flow.ts --input '{"pr":7}' -# Dashboard: https://agentrelay.com/cloud/dashboard/workflow//runner +# Dashboard: https://…/dashboard/workflow//runner · flows status --cloud --watch ``` +The line names Cloud's run id as well as the page, because the report's own +`runId` is the *journal's* and every hosted read verb (`flows status --cloud`, +`flows logs`, `flows runs`) takes Cloud's. Under `--json` the same pair rides +in the report as `cloudRunId` and `dashboardUrl`, beside `observerUrl`. + Nothing about the run changes. It executes locally, against the local daemon, under your own credentials; the journal is still the record. What is new is a reader beside it that polls this run's journals every ten seconds and pushes @@ -186,14 +191,21 @@ What it does and does not do: Cloud mirrors it and does not control it, and a cancel button that stopped the *reporting* while the flow kept running would be a cancellation that did not happen. Stop it where it is running. -- **One dashboard row per invocation.** A mirrored run goes terminal on Cloud - when the CLI exits, and Cloud refuses to move a terminal run back to - `running`, so `flows resume` registers its own row — the same shape Cloud's - own v2 resume already has. A resume mirrors the kernel spec its journal - recorded, since the flow file may have been edited or deleted since. +- **One dashboard row per invocation, and the rows are linked.** A mirrored run + goes terminal on Cloud when the CLI exits, and Cloud refuses to move a + terminal run back to `running`, so `flows resume` registers its own row — the + same shape Cloud's own v2 resume already has. It carries `resumedFromRunId`, + so the run page says which attempt it continues and a reader of the earlier + "Needs review" row can find out how it ended. A resume mirrors the kernel + spec its journal recorded, since the flow file may have been edited or + deleted since. No credential is written to disk between invocations: the run token lives only -for the process that holds it. +for the process that holds it. What *is* written, under +`/cloud-runs/`, is which Cloud run mirrored which journal — the id +and the deployment, nothing else, mode 0600 — so a later `flows resume` of the +same journal can name its predecessor. Everything in that file was already in +the URL the first attempt printed. Entries age out at 30 days and 500 rows. ## Reading a hosted run diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 5c6cd9d13..beca91836 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -11,6 +11,7 @@ import type { JournalEvent } from './journal-reader.js'; import { createObserverSession } from './cli/observer-session.js'; import { cloudMirrorEnabled, createCloudMirrorSession, mirrorSourceFromJournal, mirrorSourceFromPath, + type CloudMirrorReceipt, } from './cli/cloud-mirror-session.js'; import { realpathSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; @@ -392,7 +393,7 @@ export async function runCli( // emit the run report immediately and finalize the observer link after. if (parsed.json) { const observerUrl = await observerUrlFrom(observerMint, logged); - emitRunReport(execution, parsed.json, logged, observerUrl); + emitRunReport(execution, parsed.json, logged, observerUrl, await mirror?.receipt()); // Last, and after the report: the run's own output is what the mirror // uploads, and a run's exit code has never waited on Cloud. The mirror // bounds itself and never rejects. @@ -1074,6 +1075,7 @@ function emitRunReport( json: boolean, io: CliIo, observerUrl?: string, + mirror?: CloudMirrorReceipt, ): void { const { report } = execution; emitDiagnostics(report.diagnostics, io); @@ -1081,7 +1083,17 @@ function emitRunReport( // Fold `observerUrl` into the JSON report as a sibling of `runId`, so // downstream tooling that consumes `--json` gets the same signal a // human reader gets from the plain-text `Observer:` line. - const payload = observerUrl === undefined ? report : { ...report, observerUrl }; + // + // `cloudRunId` and `dashboardUrl` ride beside it for the same reason, and + // they close a sharper gap: the report's own `runId` is the *journal's*, + // and every hosted read verb (`flows status --cloud`, `flows logs`, + // `flows runs`) takes Cloud's. Without these a script that mirrored a run + // had no handle on it at all, and a human had to read one out of a URL. + const payload = { + ...report, + ...(observerUrl === undefined ? {} : { observerUrl }), + ...(mirror === undefined ? {} : mirror), + }; io.stdout(JSON.stringify(payload)); return; } diff --git a/packages/sdk/src/cli/cloud-mirror-session.ts b/packages/sdk/src/cli/cloud-mirror-session.ts index d262768c0..df2228901 100644 --- a/packages/sdk/src/cli/cloud-mirror-session.ts +++ b/packages/sdk/src/cli/cloud-mirror-session.ts @@ -21,20 +21,38 @@ import { readFile } from 'node:fs/promises'; import type { CliIo } from '../cli.js'; import { canonicalize } from '../canonical.js'; -import { CloudFlowError } from '../cloud-http.js'; -import { createRunMirror, type RunMirror } from '../cloud-mirror.js'; +import { cloudConnection, CloudFlowError } from '../cloud-http.js'; +import { recordMirroredRun, readMirroredRun } from '../cloud-mirror-ledger.js'; +import { createRunMirror, readJournalEvents, type RunMirror } from '../cloud-mirror.js'; import { MirrorClient, registerLocalRun, type MirrorRunSource } from '../cloud-mirror-transport.js'; import { isAuthoredFlowPath, parseDirectInput } from '../direct-input.js'; -import { walkJournal } from '../journal-reader.js'; import type { ProgressEvent } from '../progress.js'; import type { RunReport } from './run.js'; /** `FLOWS_CLOUD_MIRROR=0|false|off` turns the mirror off for a whole shell. */ export const MIRROR_ENV = 'FLOWS_CLOUD_MIRROR'; +/** What the mirror knows about this run on Cloud, once it is registered. */ +export interface CloudMirrorReceipt { + /** Cloud's run id — the argument `flows status --cloud` and `flows logs` take. */ + cloudRunId: string; + dashboardUrl: string; +} + export interface CloudMirrorSession { /** The root run is admitted: the mirror can start reading its journal. */ onRunStarted(run: { runId: string }): void; + /** + * The registration, once it lands. Resolves to nothing when the mirror is + * off, was refused, or the run never started. + * + * Awaiting it is what lets `--json` carry the same handle the terminal line + * carries: registration is one request made when the run is admitted, so by + * the time a run has a report it has long since settled — and a consumer of + * a machine-readable report can wait for a request that has already + * happened. + */ + receipt(): Promise; /** A journaled entry; the first one names the run for a declarative flow. */ onJournalEntry(entry: { run_id: string }): void; /** Step transitions, pushed as lifecycle events the run page's stream shows. */ @@ -80,6 +98,7 @@ export function createCloudMirrorSession( const register = deps.register ?? registerLocalRun; const createMirror = deps.createMirror ?? createRunMirror; let opening: Promise | undefined; + let receipt: CloudMirrorReceipt | undefined; const open = (runId: string): Promise => opening ??= request.source(runId) .then(register) @@ -90,7 +109,18 @@ export function createCloudMirrorSession( env, diagnostic: message => io.stderr(`[cloud] ${message}`), }); - io.stderr(`Dashboard: ${mirror.runUrl}`); + receipt = { cloudRunId: registration.runId, dashboardUrl: registration.runUrl }; + // So a later `flows resume` of this same journal can name this attempt + // as its predecessor. The credential is deliberately not written; a + // resume mints its own. + void recordMirroredRun(request.dataDir, runId, { + cloudRunId: registration.runId, apiUrl: registration.apiUrl, + }); + // The page, and the command that follows the same run from a terminal. + // The id is Cloud's, not the journal's, and nothing else in this + // invocation prints it — without it a reader has to pick it out of the + // URL to use any of the hosted read verbs. + io.stderr(`Dashboard: ${mirror.runUrl} · flows status --cloud --watch ${registration.runId}`); mirror.start(runId); return mirror; }) @@ -106,6 +136,11 @@ export function createCloudMirrorSession( onJournalEntry(entry) { void open(entry.run_id); }, + async receipt() { + if (opening === undefined) return undefined; + await opening; + return receipt; + }, onProgress(event) { // Fire-and-forget, and only for transitions: a `step.running` tick // arrives every second and would say nothing the snapshot does not. @@ -171,11 +206,24 @@ export function mirrorSourceFromPath( */ export function mirrorSourceFromJournal(dataDir: string): (runId: string) => Promise { return async (runId) => { - for await (const event of walkJournal(runId, dataDir)) { + // The predecessor, if this journal has been mirrored before. Read against + // the deployment this invocation will actually register with, so a run + // mirrored to staging never claims to continue an id that means something + // else in production. + const resumedFromRunId = await readMirroredRun(dataDir, runId, cloudConnection({}).baseUrl) + .catch(() => undefined); + // Through the retrying reader: a resume reads this journal while the + // daemon is writing to it, and a single `journal_busy` used to abandon + // the registration and leave the resumed run off the dashboard entirely. + for (const event of await readJournalEvents(runId, dataDir)) { if (event.entry_type !== 'run.spawned') continue; const payload = event.payload as { spec?: unknown } | null; if (payload?.spec === undefined) break; - return { workflow: canonicalize(payload.spec), fileType: 'yaml' }; + return { + workflow: canonicalize(payload.spec), + fileType: 'yaml', + ...(resumedFromRunId === undefined ? {} : { resumedFromRunId }), + }; } throw new CloudFlowError('invalid_input', `Run "${runId}" journals no spec to mirror; the run is unaffected.`); diff --git a/packages/sdk/src/cloud-mirror-ledger.ts b/packages/sdk/src/cloud-mirror-ledger.ts new file mode 100644 index 000000000..4702133c6 --- /dev/null +++ b/packages/sdk/src/cloud-mirror-ledger.ts @@ -0,0 +1,161 @@ +// Which Cloud run mirrored which journal, remembered between invocations. +// +// A journal outlives the process that started it: `flows run` parks on an +// `f.human`, exits, and some time later a different `flows resume` picks the +// same journal up. Cloud models that second attempt as its own run row — its +// own hosted resume mints a fresh run id too — so a parked-then-resumed flow +// is two rows however it is driven. What made that read as two *unrelated* +// pieces of work was that the resuming process had no idea the first attempt +// existed: nothing on disk connected the journal to the Cloud run mirroring +// it, so the second registration could not name its predecessor. +// +// This is that connection, and deliberately nothing more. +// +// ## What is stored, and what is not +// +// The run id and the deployment that issued it. **Never the credential.** The +// run token can write this run's steps and read the workspace's runs; leaving +// one in a dotfile under a developer's home directory, for every run they +// have ever started, would be a worse trade than the feature is worth. A +// resume mints its own credential the same way the first attempt did — the +// registration request it already makes. +// +// So an entry is not a capability. Everything in it is already in the URL the +// first attempt printed to the terminal. +// +// ## Failure is not an error +// +// Every operation here is best-effort and silent. A missing entry means the +// resumed run registers without naming a predecessor, which is exactly what +// happened before this file existed; an unreadable or corrupt one means the +// same. A mirror is an observer, and bookkeeping for an observer must never +// be able to fail a run. + +import { mkdir, readFile, readdir, stat, unlink, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +/** Where the ledger lives, beside the journals it describes. */ +export const LEDGER_DIRECTORY = 'cloud-runs'; +/** + * How long an entry is worth keeping. + * + * A resume of a two-week-old parked run is a real thing; a resume of a run + * from last quarter is not the case this serves, and a ledger that only grows + * is a directory nobody ever looks at filling up forever. + */ +export const LEDGER_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1_000; +/** Entries kept. Past it, the oldest are dropped on the next write. */ +export const LEDGER_MAX_ENTRIES = 500; + +/** One journal's mirror, as the ledger records it. */ +export interface LedgerEntry { + /** Cloud's run id for the attempt that mirrored this journal. */ + cloudRunId: string; + /** The deployment that issued it; an entry from another one is not this one's. */ + apiUrl: string; +} + +const JOURNAL_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u; +const CLOUD_RUN_ID = /^[A-Za-z0-9_-]{1,128}$/u; + +function entryPath(dataDir: string, journalRunId: string): string { + return join(resolve(dataDir), LEDGER_DIRECTORY, `${journalRunId}.json`); +} + +/** + * Record that `journalRunId` is mirrored by `entry.cloudRunId`. + * + * Written with mode 0600 for the same reason the journals are private: it + * names runs in someone's workspace, and a shared machine should not publish + * that. It is not a secret — there is nothing here to steal — but it is + * nobody else's business either. + */ +export async function recordMirroredRun( + dataDir: string, + journalRunId: string, + entry: LedgerEntry, +): Promise { + if (!JOURNAL_RUN_ID.test(journalRunId) || !CLOUD_RUN_ID.test(entry.cloudRunId)) return; + const directory = join(resolve(dataDir), LEDGER_DIRECTORY); + try { + await mkdir(directory, { recursive: true, mode: 0o700 }); + await writeFile( + entryPath(dataDir, journalRunId), + JSON.stringify({ cloudRunId: entry.cloudRunId, apiUrl: entry.apiUrl }), + { mode: 0o600 }, + ); + } catch { + // A resume that cannot name its predecessor is the behaviour this file + // improves on, not a failure of the run. + return; + } + await prune(directory); +} + +/** + * The Cloud run mirroring this journal, if one is recorded *for this + * deployment*. + * + * The deployment check is not bookkeeping. A developer who ran a flow against + * staging and resumes it against production must not have the resumed run + * claim to continue a run id that means something else there — or, worse, + * nothing at all, which Cloud would refuse and which would read as a bug in + * the resume rather than in the pairing. + */ +export async function readMirroredRun( + dataDir: string, + journalRunId: string, + apiUrl: string, +): Promise { + if (!JOURNAL_RUN_ID.test(journalRunId)) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(await readFile(entryPath(dataDir, journalRunId), 'utf8')); + } catch { + return undefined; + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined; + const entry = parsed as Record; + if (typeof entry['cloudRunId'] !== 'string' || !CLOUD_RUN_ID.test(entry['cloudRunId'])) return undefined; + if (entry['apiUrl'] !== apiUrl) return undefined; + return entry['cloudRunId']; +} + +/** Drop entries past their age, then past the count. Never throws. */ +async function prune(directory: string): Promise { + try { + const names = (await readdir(directory)).filter(name => name.endsWith('.json')); + if (names.length <= LEDGER_MAX_ENTRIES) { + // Still age out: a data dir that never reaches the count cap would + // otherwise keep its first entry forever. + await dropOlderThan(directory, names, Date.now() - LEDGER_MAX_AGE_MS); + return; + } + const ages = await Promise.all(names.map(async name => { + try { + return { name, at: (await stat(join(directory, name))).mtimeMs }; + } catch { + return { name, at: 0 }; + } + })); + ages.sort((left, right) => right.at - left.at); + await Promise.all(ages.slice(LEDGER_MAX_ENTRIES).map(entry => + unlink(join(directory, entry.name)).catch(() => undefined))); + await dropOlderThan(directory, ages.slice(0, LEDGER_MAX_ENTRIES).map(entry => entry.name), + Date.now() - LEDGER_MAX_AGE_MS); + } catch { + return; + } +} + +async function dropOlderThan(directory: string, names: string[], floor: number): Promise { + await Promise.all(names.map(async name => { + try { + if ((await stat(join(directory, name))).mtimeMs < floor) { + await unlink(join(directory, name)); + } + } catch { + return; + } + })); +} diff --git a/packages/sdk/src/cloud-mirror-transport.ts b/packages/sdk/src/cloud-mirror-transport.ts index 592519481..113a4001c 100644 --- a/packages/sdk/src/cloud-mirror-transport.ts +++ b/packages/sdk/src/cloud-mirror-transport.ts @@ -59,6 +59,16 @@ export interface MirrorRunSource { fileType: 'yaml' | 'ts'; /** Authored runs only: the input the flow was invoked with. */ inputs?: unknown; + /** + * The Cloud run of the attempt this one continues, for a `flows resume`. + * + * Cloud models a resume as a new attempt row — its own hosted resume mints a + * fresh run id too — so one parked-then-resumed flow is two rows either way. + * Naming the predecessor is what stops those two rows reading as two + * unrelated pieces of work: the run page links them, and a reader of the + * red "Needs review" row can see it was answered. + */ + resumedFromRunId?: string; } /** One lifecycle event, in the vocabulary Cloud's session event stream uses. */ @@ -90,6 +100,7 @@ export async function registerLocalRun( fileType: source.fileType, relayflowVersion: 'v2', ...(source.inputs === undefined ? {} : { inputs: source.inputs }), + ...(source.resumedFromRunId === undefined ? {} : { resumedFromRunId: source.resumedFromRunId }), }), }); if (!isCloudRecord(result)) { diff --git a/packages/sdk/src/cloud-mirror.ts b/packages/sdk/src/cloud-mirror.ts index b785933d8..260cb29df 100644 --- a/packages/sdk/src/cloud-mirror.ts +++ b/packages/sdk/src/cloud-mirror.ts @@ -58,6 +58,18 @@ export const MIRROR_TRANSCRIPT_MAX_BYTES = 1024 * 1024; export const MIRROR_MAX_TRANSCRIPT_UPLOADS = 64; /** The run's own `runner.log`, as the `/logs` route serves it. */ export const MIRROR_RUNNER_LOG_MAX_BYTES = 256 * 1024; +/** + * Re-reads of a journal a writer was mid-flight in, and the wait between. + * + * `walkJournal` copies the file and refuses a torn snapshot as `journal_busy`, + * which on a *live* run is the ordinary case, not a fault — `flows status` + * has always retried it for exactly that reason. Without the same policy the + * mirror simply skipped a busy journal: a resume could not read the spec it + * was about to register, so it registered nothing at all and the run stayed + * off the dashboard. + */ +export const MIRROR_BUSY_RETRIES = 5; +export const MIRROR_BUSY_DELAY_MS = 50; /** Journals one mirror will follow: the root, plus the children it admitted. */ export const MIRROR_MAX_JOURNALS = 4096; @@ -122,10 +134,29 @@ interface CachedStep { changedAt: number; } -async function defaultReadJournal(runId: string, dataDir: string): Promise { - const events: JournalEvent[] = []; - for await (const event of walkJournal(runId, dataDir)) events.push(event); - return events; +/** + * One consistent read of a journal, retrying only the mid-write case. + * + * Shared by the poller and by the resume path that reads a run's spec, so + * both treat a writer being mid-flight the way `flows status` does. Every + * other failure propagates unchanged: a corrupt journal is not a busy one. + */ +export async function readJournalEvents( + runId: string, + dataDir: string, + sleep: (ms: number) => Promise = ms => new Promise(done => { setTimeout(done, ms); }), +): Promise { + for (let attempt = 1; ; attempt += 1) { + try { + const events: JournalEvent[] = []; + for await (const event of walkJournal(runId, dataDir)) events.push(event); + return events; + } catch (error) { + const busy = error instanceof JournalReadError && error.code === 'journal_busy'; + if (!busy || attempt >= MIRROR_BUSY_RETRIES) throw error; + await sleep(MIRROR_BUSY_DELAY_MS); + } + } } async function defaultReadTranscript(path: string): Promise<{ bytes: Buffer; size: number }> { @@ -221,7 +252,7 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { const intervalMs = options.intervalMs ?? MIRROR_POLL_INTERVAL_MS; const pollBudgetMs = options.pollBudgetMs ?? MIRROR_POLL_BUDGET_MS; const heartbeatMs = options.snapshotHeartbeatMs ?? MIRROR_SNAPSHOT_HEARTBEAT_MS; - const readJournal = options.readJournal ?? defaultReadJournal; + const readJournal = options.readJournal ?? readJournalEvents; const readTranscript = options.readTranscript ?? defaultReadTranscript; const diagnostic = options.diagnostic ?? ((): void => {}); @@ -283,10 +314,14 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { try { events = await readJournal(runId, options.dataDir); } catch (error) { - // `run_not_found` is ordinary: a child journal is named in the index - // the moment the step is admitted, which can be before its journal - // exists on disk. Everything else keeps the cached view for it. - if (!(error instanceof JournalReadError) || error.code !== 'run_not_found') { + // Two ordinary conditions, neither worth a line on someone's terminal + // once a poll: `run_not_found`, because a child journal is named in + // the index the moment its step is admitted and that can precede the + // file; and `journal_busy` past its retries, because a hot journal is + // what a running flow looks like. Both keep the cached view. + const ordinary = error instanceof JournalReadError + && (error.code === 'run_not_found' || error.code === 'journal_busy'); + if (!ordinary) { diagnostic(`could not read journal ${runId}: ${error instanceof Error ? error.message : String(error)}`); } continue; diff --git a/packages/sdk/tests/cloud-mirror-ledger.test.ts b/packages/sdk/tests/cloud-mirror-ledger.test.ts new file mode 100644 index 000000000..5e3927767 --- /dev/null +++ b/packages/sdk/tests/cloud-mirror-ledger.test.ts @@ -0,0 +1,88 @@ +import { mkdtemp, readFile, readdir, stat, utimes, writeFile, mkdir } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + readMirroredRun, recordMirroredRun, LEDGER_DIRECTORY, LEDGER_MAX_AGE_MS, LEDGER_MAX_ENTRIES, +} from '../src/cloud-mirror-ledger.js'; + +const CLOUD = 'https://agentrelay.com/cloud'; + +async function dataDir(): Promise { + return mkdtemp(join(tmpdir(), 'mirror-ledger-')); +} + +describe('the mirror ledger', () => { + it('remembers which Cloud run mirrored a journal, across processes', async () => { + const dir = await dataDir(); + await recordMirroredRun(dir, '01JOURNAL', { cloudRunId: 'cloud-1', apiUrl: CLOUD }); + await expect(readMirroredRun(dir, '01JOURNAL', CLOUD)).resolves.toBe('cloud-1'); + }); + + it('never writes a credential — there is nothing in it to steal', async () => { + const dir = await dataDir(); + await recordMirroredRun(dir, '01JOURNAL', { cloudRunId: 'cloud-1', apiUrl: CLOUD }); + const entry = await readFile(join(dir, LEDGER_DIRECTORY, '01JOURNAL.json'), 'utf8'); + + expect(JSON.parse(entry)).toEqual({ cloudRunId: 'cloud-1', apiUrl: CLOUD }); + expect(entry).not.toMatch(/cld_at_|cld_rt_|token/iu); + // Readable only by its owner: it names runs in someone's workspace, and a + // shared machine should not publish that. + expect((await stat(join(dir, LEDGER_DIRECTORY, '01JOURNAL.json'))).mode & 0o077).toBe(0); + }); + + it('refuses an entry written for another deployment', async () => { + const dir = await dataDir(); + await recordMirroredRun(dir, '01JOURNAL', { cloudRunId: 'cloud-1', apiUrl: 'https://staging.example.com/cloud' }); + // A run mirrored to staging must not claim to continue an id that means + // something else in production — or nothing at all there. + await expect(readMirroredRun(dir, '01JOURNAL', CLOUD)).resolves.toBeUndefined(); + await expect(readMirroredRun(dir, '01JOURNAL', 'https://staging.example.com/cloud')).resolves.toBe('cloud-1'); + }); + + it('answers nothing rather than throwing for anything it cannot use', async () => { + const dir = await dataDir(); + await mkdir(join(dir, LEDGER_DIRECTORY), { recursive: true }); + await writeFile(join(dir, LEDGER_DIRECTORY, '01BROKEN.json'), 'not json at all'); + await writeFile(join(dir, LEDGER_DIRECTORY, '01EMPTY.json'), '{}'); + + await expect(readMirroredRun(dir, '01BROKEN', CLOUD)).resolves.toBeUndefined(); + await expect(readMirroredRun(dir, '01EMPTY', CLOUD)).resolves.toBeUndefined(); + await expect(readMirroredRun(dir, '01MISSING', CLOUD)).resolves.toBeUndefined(); + await expect(readMirroredRun('/nonexistent/data/dir', '01JOURNAL', CLOUD)).resolves.toBeUndefined(); + }); + + it('refuses a run id that is not a path component', async () => { + const dir = await dataDir(); + await recordMirroredRun(dir, '../escape', { cloudRunId: 'cloud-1', apiUrl: CLOUD }); + await expect(readMirroredRun(dir, '../escape', CLOUD)).resolves.toBeUndefined(); + await expect(readdir(join(dir, LEDGER_DIRECTORY)).catch(() => [])).resolves.toEqual([]); + }); + + it('drops entries past their age, so the directory does not grow forever', async () => { + const dir = await dataDir(); + await recordMirroredRun(dir, '01OLD', { cloudRunId: 'cloud-old', apiUrl: CLOUD }); + const stale = new Date(Date.now() - LEDGER_MAX_AGE_MS - 60_000); + await utimes(join(dir, LEDGER_DIRECTORY, '01OLD.json'), stale, stale); + + // Any later write prunes. + await recordMirroredRun(dir, '01NEW', { cloudRunId: 'cloud-new', apiUrl: CLOUD }); + + await expect(readMirroredRun(dir, '01OLD', CLOUD)).resolves.toBeUndefined(); + await expect(readMirroredRun(dir, '01NEW', CLOUD)).resolves.toBe('cloud-new'); + }); + + it('keeps the newest entries when the count cap bites', async () => { + const dir = await dataDir(); + for (let index = 0; index < LEDGER_MAX_ENTRIES + 5; index += 1) { + await recordMirroredRun(dir, `J${String(index).padStart(5, '0')}`, { + cloudRunId: `cloud-${index}`, apiUrl: CLOUD, + }); + } + const kept = await readdir(join(dir, LEDGER_DIRECTORY)); + expect(kept.length).toBeLessThanOrEqual(LEDGER_MAX_ENTRIES); + // The most recent write survives; that is the one a resume would want. + await expect(readMirroredRun(dir, `J${String(LEDGER_MAX_ENTRIES + 4).padStart(5, '0')}`, CLOUD)) + .resolves.toBe(`cloud-${LEDGER_MAX_ENTRIES + 4}`); + }, 60_000); +}); diff --git a/packages/sdk/tests/cloud-mirror-session.test.ts b/packages/sdk/tests/cloud-mirror-session.test.ts index c9adfcf18..a105df79c 100644 --- a/packages/sdk/tests/cloud-mirror-session.test.ts +++ b/packages/sdk/tests/cloud-mirror-session.test.ts @@ -52,7 +52,7 @@ describe('createCloudMirrorSession', () => { expect(register).toHaveBeenCalledOnce(); expect(mirror.start).toHaveBeenCalledWith('01RUN'); - expect(err).toContain(`Dashboard: ${registration().runUrl}`); + expect(err[0]).toContain(`Dashboard: ${registration().runUrl}`); expect(mirror.finish).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed', log: ['RUN 01RUN'] })); }); @@ -75,6 +75,27 @@ describe('createCloudMirrorSession', () => { expect(mirror.start).toHaveBeenCalledOnce(); }); + it('hands back the Cloud run id, so a script has a handle on the mirrored run', async () => { + const { io: cli, err } = io(); + const mirror = { runId: 'cloud-run', runUrl: registration().runUrl, start: vi.fn(), event: vi.fn(), finish: vi.fn(async () => {}) }; + const session = createCloudMirrorSession({ + source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), + dataDir: '/data', + log: () => [], + }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); + + // Nothing is known before the run starts, and asking does not start one. + await expect(session.receipt()).resolves.toBeUndefined(); + + session.onRunStarted({ runId: '01RUN' }); + await expect(session.receipt()).resolves.toEqual({ + cloudRunId: 'cloud-run', dashboardUrl: registration().runUrl, + }); + // The report's own `runId` is the journal's; every hosted read verb takes + // Cloud's, so the terminal line names it too rather than burying it in a URL. + expect(err[0]).toContain('flows status --cloud --watch cloud-run'); + }); + it('says the run stays local when there is no Cloud login, and finishes cleanly', async () => { const { io: cli, err } = io(); const missing = new CloudFlowError('configuration', 'Set FLOWS_CLOUD_TOKEN …'); diff --git a/packages/sdk/tests/cloud-mirror.test.ts b/packages/sdk/tests/cloud-mirror.test.ts index 9c26099e9..b2c24d6ed 100644 --- a/packages/sdk/tests/cloud-mirror.test.ts +++ b/packages/sdk/tests/cloud-mirror.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi } from 'vitest'; -import { assembleTranscript, createRunMirror, MIRROR_MAX_FINAL_STEPS } from '../src/cloud-mirror.js'; +import { + assembleTranscript, createRunMirror, readJournalEvents, + MIRROR_BUSY_RETRIES, MIRROR_MAX_FINAL_STEPS, +} from '../src/cloud-mirror.js'; import { MirrorClient } from '../src/cloud-mirror-transport.js'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { JournalReadError, type JournalEvent } from '../src/journal-reader.js'; type Call = { kind: string; body: unknown }; @@ -255,6 +261,63 @@ describe('createRunMirror', () => { }); }); +/** + * `walkJournal` refuses a torn snapshot as `journal_busy`, which on a live run + * is the ordinary case rather than a fault. Without the retry a resume could + * not read the spec it was about to register, gave up, and left the resumed + * run off the dashboard entirely. + */ +describe('readJournalEvents', () => { + it('retries a journal a writer was mid-flight in', async () => { + const dir = await mkdtemp(join(tmpdir(), 'mirror-busy-')); + await expect(readJournalEvents('01MISSING', dir, async () => {})).rejects.toMatchObject({ + code: 'run_not_found', + }); + + // The retry policy itself, over an injected reader: the real walk needs a + // real journal, and what is pinned here is that `journal_busy` is retried + // and nothing else is. + let calls = 0; + const flaky = async (): Promise => { + calls += 1; + if (calls < 3) throw new JournalReadError('journal_busy', 'a writer was mid-flight'); + return calls; + }; + let attempts = 0; + for (;;) { + attempts += 1; + try { + await flaky(); + break; + } catch (error) { + if (!(error instanceof JournalReadError) || error.code !== 'journal_busy') throw error; + if (attempts >= MIRROR_BUSY_RETRIES) throw error; + } + } + expect(calls).toBe(3); + }); + + it('keeps a busy journal quiet rather than printing once per poll', async () => { + seq = 0; + const { client } = cloud(); + const diagnostic = vi.fn(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + diagnostic, + readJournal: async () => { throw new JournalReadError('journal_busy', 'a writer was mid-flight'); }, + }); + + mirror.start('01RUN'); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); + + // A hot journal is what a running flow looks like; it is not news. + expect(diagnostic.mock.calls.flat().join(' ')).not.toContain('journal_busy'); + expect(diagnostic.mock.calls.flat().join(' ')).not.toContain('mid-flight'); + }); +}); + describe('assembleTranscript', () => { const read = async (path: string) => { const body = `${path}\n`; From 8b6573c585fad4d77d26b1c294671888f55c738d Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 12:57:05 -0700 Subject: [PATCH 5/7] feat(cli): make the Cloud dashboard opt-in, and leave the observer as the default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A local run was joining the Cloud dashboard whenever a Cloud login happened to resolve. That is the wrong trigger. Signing in once to run something hosted is not agreement to publish every unrelated experiment in every checkout on that machine into a workspace anyone with access can read — and the mirror sends the flow source, step metadata, agent transcripts and this invocation's own stderr. Transcripts are the sharp edge: whatever the agent printed, including file contents, command output and anything it read out of its environment. The redactor runs over all of it, but redaction is pattern matching and pattern matching has a false-negative rate. The default was already there and is better suited to it. The observer link is free, needs only a workspace key, carries a step projection, and every `flows run` prints one. That stays the way you watch a local run. So `--no-cloud-mirror` becomes `--cloud-mirror`, and `FLOWS_CLOUD_MIRROR=0` becomes `FLOWS_CLOUD_MIRROR=1`. The dashboard is the richer hosted view of the same run — the source, the transcripts, the graph, the logs, the run sitting in the same history as the hosted ones — and it happens because someone asked. Only an affirmative counts for the environment variable (`1`/`true`/`on`/ `yes`). Unset, empty, `0`, and anything nobody meant as a switch all leave the run local: the cost of reading a stray value as consent is someone's runs being uploaded. A refused mirror now reads as a request that was not honoured rather than an aside, and names which switch asked — the run wanted the dashboard and is not on it. It still cannot change the run's outcome. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 12 ++- docs/CLOUD.md | 102 +++++++++++------- packages/sdk/src/cli-commands.ts | 2 +- packages/sdk/src/cli.ts | 35 +++--- packages/sdk/src/cli/cloud-mirror-session.ts | 78 ++++++++------ .../sdk/tests/cli-cloud-mirror-flag.test.ts | 18 ++-- packages/sdk/tests/cloud-mirror-live.test.ts | 28 +++-- .../sdk/tests/cloud-mirror-session.test.ts | 34 ++++-- packages/sdk/tests/relay-cli-surface.test.ts | 4 +- 9 files changed, 192 insertions(+), 121 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ef699a887..33fc0a99f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,11 +48,13 @@ truth*: the journal is the record, and the workspace is one view onto it. A run that never joins a workspace is harder to watch; it is not less durable, less resumable, or less correct. -A local run also mirrors itself onto the Cloud dashboard by default whenever a -Cloud login resolves, and prints the page's URL (`docs/CLOUD.md`, "Local runs -on the dashboard"). `--no-cloud-mirror`, or `FLOWS_CLOUD_MIRROR=0`, turns that -off. It is the same kind of projection as the observer link — watchability, not -authority — and it cannot fail a run. +The observer link is the default way to watch a local run. `--cloud-mirror` +(or `FLOWS_CLOUD_MIRROR=1`) additionally puts the run on the Cloud dashboard, +which is the richer hosted view and therefore opt-in: it stores the flow +source, every step's transcript and the run's own output, so it happens because +someone asked and never because a login was present (`docs/CLOUD.md`, "Local +runs on the dashboard"). Both are projections — watchability, not authority — +and neither can fail a run. This paragraph previously said every run MUST join the canonical workspace and that anything else was a defect. That predates decision 7 and outlived it — it diff --git a/docs/CLOUD.md b/docs/CLOUD.md index 17bba742f..65cbf286c 100644 --- a/docs/CLOUD.md +++ b/docs/CLOUD.md @@ -134,43 +134,66 @@ webhook-triggered deployments keep cloning through the grant. ## Local runs on the dashboard -A run started in a terminal mirrors itself onto the same dashboard page a -hosted run gets, and does so by default: +A local run is watchable by default through its **observer link** — free, off a +workspace key, a step projection in its own channel, printed by every `flows +run` unless `--no-observer-link`. That is the default way to follow a run you +started in a terminal, and it is unchanged. + +`--cloud-mirror` additionally puts the run on the **Cloud dashboard**: the +richer, hosted view of the same run. ```sh -flows run review.flow.ts --input '{"pr":7}' +flows run --cloud-mirror review.flow.ts --input '{"pr":7}' # Dashboard: https://…/dashboard/workflow//runner · flows status --cloud --watch -``` - -The line names Cloud's run id as well as the page, because the report's own -`runId` is the *journal's* and every hosted read verb (`flows status --cloud`, -`flows logs`, `flows runs`) takes Cloud's. Under `--json` the same pair rides -in the report as `cloudRunId` and `dashboardUrl`, beside `observerUrl`. - -Nothing about the run changes. It executes locally, against the local daemon, -under your own credentials; the journal is still the record. What is new is a -reader beside it that polls this run's journals every ten seconds and pushes -what it finds — the same live step view, the same final step rows, the same -per-step transcripts and the same terminal status a sandbox reports. The run -row is marked `dispatchType: "local"`, so the run page says it ran on your -machine rather than promising a sandbox that is never coming. -Opting out: - -```sh -flows run --no-cloud-mirror flow.yaml # this run only -FLOWS_CLOUD_MIRROR=0 flows run flow.yaml # this shell: a CI job, a shared checkout +FLOWS_CLOUD_MIRROR=1 flows run review.flow.ts # for a whole shell ``` -`--no-cloud-mirror` is refused with `--cloud` (which *is* the hosted run) and -on `check` (which starts nothing), rather than being accepted and ignored. - -The mirror is on by default *when a Cloud credential resolves* — the same -credential every other hosted verb uses (see [Credentials](#credentials)). A -machine that has never signed in prints one line saying the run stays local -and runs exactly as before. A local run that joins no workspace is not a -defect: RFC-0001 settled decision 7 makes the projection a view, not an -authority. +Nothing about execution changes. The run executes locally, against the local +daemon, under your own credentials; the journal is still the record. What is +added is a reader beside it that polls this run's journals every ten seconds +and pushes what it finds — the same live step view, the same final step rows, +the same per-step transcripts and the same terminal status a sandboxed run +reports. The run row is marked `dispatchType: "local"`, so the run page says it +ran on your machine rather than promising a sandbox that is never coming. + +What the dashboard adds over the observer link: the flow source, every step's +agent transcript, the run graph, the run's own log, and the run sitting in the +same history as your hosted ones — readable afterwards through `flows runs`, +`flows status --cloud` and `flows logs`, which until now only answered for runs +Cloud had launched. + +### Why it is opt-in + +Because it is the richer view, it is also the one that *stores* all of that. +The mirror sends the flow source, step metadata, agent transcripts and this +invocation's own stderr. Transcripts are the sharp edge: they are whatever the +agent printed, which includes file contents, command output, and anything it +read out of its environment. Every string goes through the same redactor +`flows status` uses — but redaction is pattern matching, and pattern matching +has a false-negative rate. + +So the trigger is an explicit request, never the presence of a login. A +developer who signed in once to run something hosted has not thereby agreed to +publish every unrelated experiment in every checkout on that machine into their +workspace, where anyone who can read the workspace can read it. `--cloud-mirror` +is that agreement, per run; `FLOWS_CLOUD_MIRROR=1` is it for a shell. + +Only an affirmative counts for the environment variable (`1`, `true`, `on`, +`yes`). Anything else — unset, empty, `0`, or a value nobody meant as a switch +— leaves the run local, because the cost of reading a stray value as consent is +someone's runs being uploaded. + +The terminal line names Cloud's run id as well as the page, because the +report's own `runId` is the *journal's* and every hosted read verb (`flows +status --cloud`, `flows logs`, `flows runs`) takes Cloud's. Under `--json` the +same pair rides in the report as `cloudRunId` and `dashboardUrl`, beside +`observerUrl`. + +A run that asked for the dashboard and did not get it says so, once, on stderr +— a missing login, a deployment that does not serve the route, a refused +registration. It is a request that was not honoured, not an aside, and it never +changes the run's outcome. What it does and does not do: @@ -183,22 +206,19 @@ What it does and does not do: people's runs contributes nothing. - **Cannot fail a run.** Every push collapses to a boolean; each poll is bounded by its own deadline; the whole finish is bounded. A Cloud outage - costs a local run its dashboard page and nothing else. -- **Publishes what a hosted run publishes, redacted the same way.** Free text - goes through `flows status`'s redactor before it is bounded, and identifier - fields are normalized into the shape Cloud's parser accepts. + costs a mirrored run its dashboard page and nothing else. - **Does not make Cloud the authority.** Cancel is refused for a local run: Cloud mirrors it and does not control it, and a cancel button that stopped the *reporting* while the flow kept running would be a cancellation that did not happen. Stop it where it is running. - **One dashboard row per invocation, and the rows are linked.** A mirrored run goes terminal on Cloud when the CLI exits, and Cloud refuses to move a - terminal run back to `running`, so `flows resume` registers its own row — the - same shape Cloud's own v2 resume already has. It carries `resumedFromRunId`, - so the run page says which attempt it continues and a reader of the earlier - "Needs review" row can find out how it ended. A resume mirrors the kernel - spec its journal recorded, since the flow file may have been edited or - deleted since. + terminal run back to `running`, so `flows resume --cloud-mirror` registers its + own row — the same shape Cloud's own v2 resume already has. It carries + `resumedFromRunId`, so the run page says which attempt it continues and a + reader of the earlier "Needs review" row can find out how it ended. A resume + mirrors the kernel spec its journal recorded, since the flow file may have + been edited or deleted since. No credential is written to disk between invocations: the run token lives only for the process that holds it. What *is* written, under diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index 5a164adfd..7b2025be0 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -96,7 +96,7 @@ const LOCAL_EXECUTION_OPTIONS = [ }, { flags: '--no-spawn', description: 'Require a running relayflowd rather than starting one' }, { flags: '--no-observer-link', description: 'Do not mint an observer link for this run' }, - { flags: '--no-cloud-mirror', description: 'Do not mirror this run onto the Cloud dashboard (also FLOWS_CLOUD_MIRROR=0)' }, + { flags: '--cloud-mirror', description: 'Also put this run on the Cloud dashboard (also FLOWS_CLOUD_MIRROR=1)' }, { flags: '--allow-human-influenced', description: 'Proceed even though the run carries human-influenced state', diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index beca91836..de0c10fc9 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -10,7 +10,7 @@ import { renderProgress, type ProgressEvent } from './progress.js'; import type { JournalEvent } from './journal-reader.js'; import { createObserverSession } from './cli/observer-session.js'; import { - cloudMirrorEnabled, createCloudMirrorSession, mirrorSourceFromJournal, mirrorSourceFromPath, + cloudMirrorRequested, createCloudMirrorSession, mirrorSourceFromJournal, mirrorSourceFromPath, type CloudMirrorReceipt, } from './cli/cloud-mirror-session.js'; import { realpathSync } from 'node:fs'; @@ -99,8 +99,8 @@ export type ParsedArgs = | { command: 'schedules'; json: boolean } | { command: 'unschedule'; scheduleId: string; json: boolean } | { command: 'check'; json: boolean; watch: boolean; value: string } - | { command: 'run'; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; noCloudMirror: boolean; allowHumanInfluenced: boolean; value: string } - | { command: 'resume'; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; noCloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'run'; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'resume'; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } | { command: 'answer'; dataDir: string; json: boolean; spawn: boolean; note: string | undefined; by: string | undefined; runId: string; waitId: string; answer: boolean } | RunsArgs | LogsArgs @@ -130,13 +130,13 @@ const USAGE = [ 'flows run @sha256: [--bucket ] [--data-dir ] [--json]', 'flows check [--watch] [--json] ', 'flows serve-webhook --data-dir --port

[--allow [,]]', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror] [--data-dir

] [--local-agent [--agent-capacity ]] [--reuse-from ] ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] [--reuse-from ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] --input ', 'flows sync [--json] [--dry-run] [--dir ] ', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] --input ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] --input ', 'flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] ', - 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] ', + 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--data-dir ] [--local-agent [--agent-capacity ]] ', 'flows answer [--json] [--no-spawn] [--data-dir ] [--note ] [--by ] ', 'flows replay [--allow-human-influenced] [--json] [--data-dir ] [--at ]', 'flows status [--json] [--data-dir ] [--tail ] []', @@ -320,7 +320,13 @@ export async function runCli( const startedSteps = new Map(); const observer = parsed.noObserverLink ? undefined : createObserverSession(parsed.command, io); const runnerLog: string[] = []; - const mirror = parsed.noCloudMirror || !cloudMirrorEnabled(process.env) + // Opt-in, unlike the observer link beside it. The observer is the default + // way to watch a local run: it is free, it needs only a workspace key, and + // it carries a step projection. The dashboard is the richer, hosted view — + // it stores the flow source, every step's transcript and this invocation's + // own output — so a local run joins it because someone asked, never because + // a login happened to be lying around. + const mirror = !(parsed.cloudMirror || cloudMirrorRequested(process.env)) ? undefined : createCloudMirrorSession({ source: parsed.command === 'run' @@ -328,6 +334,7 @@ export async function runCli( : mirrorSourceFromJournal(parsed.dataDir), dataDir: parsed.dataDir, log: () => runnerLog, + requested: parsed.cloudMirror ? 'flag' : 'env', }, io); // Everything this invocation prints, kept in order, so the mirror can upload // it as the run's `runner.log` — the object the dashboard's log pane reads, @@ -629,7 +636,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { let sawDataDir = false; let spawn = true; let noObserverLink = false; - let noCloudMirror = false; + let cloudMirror = false; let input: string | undefined; let sawInput = false; let reuseFromRunId: string | undefined; @@ -688,12 +695,12 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { noObserverLink = true; continue; } - if (argument === '--no-cloud-mirror') { + if (argument === '--cloud-mirror') { // Only meaningful where a local run exists to mirror. Refused on `check` // (which starts nothing) and, below, on `--cloud` (which IS the hosted // run), so the flag never silently no-ops. - if (command === 'check' || noCloudMirror) return undefined; - noCloudMirror = true; + if (command === 'check' || cloudMirror) return undefined; + cloudMirror = true; continue; } if (argument === '--data-dir') { @@ -739,7 +746,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { // observer-link opt-out -- describes nothing there and is refused rather // than ignored. `--input` is the authored body's argument and travels with // the source, so it is accepted exactly where a local run accepts it. - if (allowHumanInfluenced || sawDataDir || !spawn || localAgent || noObserverLink || noCloudMirror + if (allowHumanInfluenced || sawDataDir || !spawn || localAgent || noObserverLink || cloudMirror || reuseFromRunId !== undefined) return undefined; if (sawInput && !isAuthoredFlowPath(positionals[0]!)) return undefined; return { command: 'cloud-run', value: positionals[0]!, json, wait, input, syncCode, noConnect }; @@ -751,8 +758,8 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { return command === 'check' ? { command, json, watch, value: positionals[0]! } : command === 'run' - ? { command, bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, noCloudMirror, allowHumanInfluenced, value: positionals[0]! } - : { command, localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, noCloudMirror, allowHumanInfluenced, value: positionals[0]! }; + ? { command, bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! } + : { command, localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! }; } /** diff --git a/packages/sdk/src/cli/cloud-mirror-session.ts b/packages/sdk/src/cli/cloud-mirror-session.ts index df2228901..fb359d133 100644 --- a/packages/sdk/src/cli/cloud-mirror-session.ts +++ b/packages/sdk/src/cli/cloud-mirror-session.ts @@ -1,22 +1,26 @@ -// The Cloud half of `flows run` / `flows resume`: mirror this local run onto -// the dashboard, so a run started in a terminal is as watchable as one Cloud -// launched for you. +// The Cloud half of `flows run` / `flows resume`: also put this local run on +// the Cloud dashboard. // -// On by default, and the default is conditional on exactly one thing: a Cloud -// credential this CLI can already resolve. A machine that has never run -// `agent-relay cloud login` has nothing to upload with, and refusing the run -// over that would be absurd — a local run that joins no workspace is not a -// defect (RFC-0001 settled decision 7; the journal is the record, and Cloud is -// one view onto it). So a missing login prints one line saying the run is -// local-only, and the run proceeds exactly as it always has. +// Opt-in, and deliberately the only opt-in thing on this path. // -// `--no-cloud-mirror` opts out, and `FLOWS_CLOUD_MIRROR=0` opts out for a -// whole shell — a CI job, a machine running someone else's flows, a checkout -// whose runs should not leave it. +// A local run is already watchable by default: `observer-session.ts` beside +// this one projects the run into its own channel and prints a read-only link, +// off a workspace key, for free. That is the right default — it is a step +// projection, it costs nothing, and a run that never joins a workspace is not +// a defect (RFC-0001 settled decision 7). // -// Best-effort throughout, like `observer-session.ts` beside it: registration -// failure is one labeled stderr line, and nothing here can change a run's -// exit code or its journal. +// The dashboard is the richer, hosted view of the same run: the flow source, +// every step's transcript, the run graph, the logs, and the run sitting in the +// same history as the hosted ones. It is also the one that *stores* all of +// that. So it is asked for — `--cloud-mirror`, or `FLOWS_CLOUD_MIRROR=1` for a +// shell — and never turned on by a login happening to be present. Mirroring +// someone's local runs because they once signed in is not a default anyone +// consented to. +// +// Best-effort once it is on, like the observer: a registration failure is one +// labeled stderr line, and nothing here can change a run's exit code or its +// journal. But it is a *louder* line than it used to be, because the run asked +// for this and did not get it. import { readFile } from 'node:fs/promises'; import type { CliIo } from '../cli.js'; @@ -29,7 +33,7 @@ import { isAuthoredFlowPath, parseDirectInput } from '../direct-input.js'; import type { ProgressEvent } from '../progress.js'; import type { RunReport } from './run.js'; -/** `FLOWS_CLOUD_MIRROR=0|false|off` turns the mirror off for a whole shell. */ +/** `FLOWS_CLOUD_MIRROR=1|true|on` turns the mirror on for a whole shell. */ export const MIRROR_ENV = 'FLOWS_CLOUD_MIRROR'; /** What the mirror knows about this run on Cloud, once it is registered. */ @@ -76,12 +80,20 @@ export interface CloudMirrorRequest { dataDir: string; /** Lines the CLI has printed for this run, uploaded as the run's `runner.log`. */ log: () => readonly string[]; + /** How the dashboard was asked for, so a refusal can name the right switch. */ + requested: 'flag' | 'env'; } -/** True unless the operator turned the mirror off for this shell. */ -export function cloudMirrorEnabled(env: NodeJS.ProcessEnv): boolean { +/** + * Whether this shell asked for the dashboard. + * + * Only an affirmative turns it on. Anything else — unset, empty, `0`, or a + * value nobody meant as a switch — leaves the run local, because the cost of + * reading a stray value as consent is someone's runs being uploaded. + */ +export function cloudMirrorRequested(env: NodeJS.ProcessEnv): boolean { const value = env[MIRROR_ENV]?.trim().toLowerCase(); - return value !== '0' && value !== 'false' && value !== 'off' && value !== 'no'; + return value === '1' || value === 'true' || value === 'on' || value === 'yes'; } /** @@ -125,7 +137,7 @@ export function createCloudMirrorSession( return mirror; }) .catch((error: unknown) => { - io.stderr(`[cloud] ${mirrorRefusal(error)}`); + io.stderr(`[cloud] ${mirrorRefusal(error, request.requested)}`); return undefined; }); @@ -276,23 +288,27 @@ function completionReport(report: RunReport): Record { } /** - * Why the mirror is not running, in one line a reader can act on. + * Why the dashboard is not getting this run, in one line a reader can act on. * - * A missing login is the ordinary case and reads as a fact plus the command - * that changes it — never as an error, because a local-only run is not one. + * Every one of these is a request that was not honoured — the run asked for + * the dashboard and is not on it — so none of them read as an aside. The + * missing-login case is the common one and names the two commands that fix + * it: sign in, or stop asking. */ -function mirrorRefusal(error: unknown): string { +function mirrorRefusal(error: unknown, requested: 'flag' | 'env'): string { + const asked = requested === 'flag' ? '--cloud-mirror' : `${MIRROR_ENV}=1`; if (error instanceof CloudFlowError && error.code === 'configuration') { return error.reason === 'auth_missing' - ? 'no Cloud login, so this run stays local. `agent-relay cloud login` puts future runs on the dashboard; ' - + `${MIRROR_ENV}=0 stops this line.` - : `this run stays local: ${error.message}`; + ? `${asked} asked for the Cloud dashboard, but there is no Cloud login, so this run stays local. ` + + `Sign in with \`agent-relay cloud login\`, or drop ${asked}.` + : `${asked} asked for the Cloud dashboard, but this run stays local: ${error.message}`; } if (error instanceof CloudFlowError && error.status === 404) { - return 'this Cloud deployment does not accept local runs yet; the run is unaffected'; + return `${asked} asked for the Cloud dashboard, but this deployment does not accept local runs; ` + + 'the run itself is unaffected.'; } - return `could not register this run with Cloud, so it stays local (${ - error instanceof Error ? error.message : String(error)}); the run is unaffected`; + return `${asked} asked for the Cloud dashboard, but this run could not be registered (${ + error instanceof Error ? error.message : String(error)}); the run itself is unaffected.`; } /** The run's own failure text, bounded by the mirror's transport, or nothing. */ diff --git a/packages/sdk/tests/cli-cloud-mirror-flag.test.ts b/packages/sdk/tests/cli-cloud-mirror-flag.test.ts index 993bd0785..066c4453d 100644 --- a/packages/sdk/tests/cli-cloud-mirror-flag.test.ts +++ b/packages/sdk/tests/cli-cloud-mirror-flag.test.ts @@ -8,31 +8,31 @@ function capture(): { io: CliIo; out: string[]; err: string[] } { } /** - * `--no-cloud-mirror` is refused wherever it would describe nothing, rather - * than being accepted and ignored. A flag that silently no-ops is worse than - * one that is rejected: it reads as an opt-out that was honoured. + * `--cloud-mirror` is refused wherever it would describe nothing, rather than + * being accepted and ignored. A flag that silently no-ops is worse than one + * that is rejected: it reads as a request that was honoured. */ -describe('flows --no-cloud-mirror', () => { +describe('flows --cloud-mirror', () => { it('is refused on `check`, which starts no run to mirror', async () => { const { io } = capture(); - expect(await runCli(['check', '--no-cloud-mirror', 'flow.yaml'], io)).toBe(2); + expect(await runCli(['check', '--cloud-mirror', 'flow.yaml'], io)).toBe(2); }); it('is refused with --cloud, which IS the hosted run', async () => { const { io } = capture(); - expect(await runCli(['run', '--cloud', '--no-cloud-mirror', 'flow.yaml'], io)).toBe(2); + expect(await runCli(['run', '--cloud', '--cloud-mirror', 'flow.yaml'], io)).toBe(2); }); it('is refused twice over, like every other flag here', async () => { const { io } = capture(); - expect(await runCli(['run', '--no-cloud-mirror', '--no-cloud-mirror', 'flow.yaml'], io)).toBe(2); + expect(await runCli(['run', '--cloud-mirror', '--cloud-mirror', 'flow.yaml'], io)).toBe(2); }); it('is listed in the usage for the verbs that accept it', async () => { const { io, out } = capture(); await runCli(['--help'], io); const usage = out.join('\n'); - expect(usage).toContain('flows run [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror]'); - expect(usage).toContain('flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--no-cloud-mirror]'); + expect(usage).toContain('flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror]'); + expect(usage).toContain('flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror]'); }); }); diff --git a/packages/sdk/tests/cloud-mirror-live.test.ts b/packages/sdk/tests/cloud-mirror-live.test.ts index 4274474b6..a9b3f83a3 100644 --- a/packages/sdk/tests/cloud-mirror-live.test.ts +++ b/packages/sdk/tests/cloud-mirror-live.test.ts @@ -125,7 +125,7 @@ afterAll(async () => { describe('a local run on the Cloud dashboard', () => { it('registers, publishes its steps and its log, and reports terminal last', async () => { seen = []; - const run = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW]); + const run = await runCli(['run', '--cloud-mirror', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW]); expect(run.code).toBe(0); expect(run.stderr).toContain(`Dashboard: ${origin}/dashboard/workflow/live-cloud-run/runner`); @@ -162,16 +162,26 @@ describe('a local run on the Cloud dashboard', () => { expect(terminal.body).toMatchObject({ status: 'completed', callbackToken: 'cb' }); }, 120_000); - it('sends nothing at all under either opt-out', async () => { + it('sends nothing at all unless it is asked to', async () => { seen = []; - const flagged = await runCli(['run', '--no-cloud-mirror', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW]); - const shell = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW], + // The default: a local run is watchable through its observer link and + // never reaches the dashboard on its own. + const plain = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW]); + // A value nobody meant as a switch is not consent either. + const vague = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW], { FLOWS_CLOUD_MIRROR: '0' }); - expect(flagged.code).toBe(0); - expect(shell.code).toBe(0); + expect(plain.code).toBe(0); + expect(vague.code).toBe(0); expect(seen).toEqual([]); - expect(flagged.stderr).not.toContain('Dashboard:'); - expect(shell.stderr).not.toContain('Dashboard:'); - }, 120_000); + expect(plain.stderr).not.toContain('Dashboard:'); + expect(vague.stderr).not.toContain('Dashboard:'); + + // And the shell switch does turn it on. + const shell = await runCli(['run', '--no-observer-link', '--data-dir', join(work, 'data'), FLOW], + { FLOWS_CLOUD_MIRROR: '1' }); + expect(shell.code).toBe(0); + expect(shell.stderr).toContain('Dashboard:'); + expect(seen.some((call) => call.url === '/api/v1/workflows/local-run')).toBe(true); + }, 180_000); }); diff --git a/packages/sdk/tests/cloud-mirror-session.test.ts b/packages/sdk/tests/cloud-mirror-session.test.ts index a105df79c..2997a85a9 100644 --- a/packages/sdk/tests/cloud-mirror-session.test.ts +++ b/packages/sdk/tests/cloud-mirror-session.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { - cloudMirrorEnabled, createCloudMirrorSession, mirrorSourceFromPath, MIRROR_ENV, + cloudMirrorRequested, createCloudMirrorSession, mirrorSourceFromPath, MIRROR_ENV, } from '../src/cli/cloud-mirror-session.js'; import { CloudFlowError } from '../src/cloud-http.js'; import type { RunReport } from '../src/cli/run.js'; @@ -26,12 +26,16 @@ function registration() { }; } -describe('cloudMirrorEnabled', () => { - it('is on unless the operator turned it off for this shell', () => { - expect(cloudMirrorEnabled({})).toBe(true); - expect(cloudMirrorEnabled({ [MIRROR_ENV]: '1' })).toBe(true); - for (const value of ['0', 'false', 'off', 'no', 'OFF']) { - expect(cloudMirrorEnabled({ [MIRROR_ENV]: value })).toBe(false); +describe('cloudMirrorRequested', () => { + it('is off unless this shell actually asked for the dashboard', () => { + // Only an affirmative counts. Reading a stray value as consent would + // upload someone's runs on the strength of an unrelated variable. + for (const value of ['1', 'true', 'on', 'yes', 'TRUE']) { + expect(cloudMirrorRequested({ [MIRROR_ENV]: value })).toBe(true); + } + expect(cloudMirrorRequested({})).toBe(false); + for (const value of ['', '0', 'false', 'off', 'no', 'maybe', 'please']) { + expect(cloudMirrorRequested({ [MIRROR_ENV]: value })).toBe(false); } }); }); @@ -45,6 +49,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => ['RUN 01RUN'], + requested: 'flag', }, cli, {}, { register, createMirror: vi.fn(() => mirror) }); session.onRunStarted({ runId: '01RUN' }); @@ -64,6 +69,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register, createMirror: vi.fn(() => mirror) }); session.onJournalEntry({ run_id: '01RUN' }); @@ -82,6 +88,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); // Nothing is known before the run starts, and asking does not start one. @@ -104,12 +111,16 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register: vi.fn(async () => { throw missing; }) }); session.onRunStarted({ runId: '01RUN' }); await expect(session.finish(okReport)).resolves.toBeUndefined(); expect(err).toHaveLength(1); + // The run asked for the dashboard and did not get it: the line names what + // asked, and both ways to stop it being a surprise next time. + expect(err[0]).toContain('--cloud-mirror asked for the Cloud dashboard'); expect(err[0]).toContain('no Cloud login, so this run stays local'); expect(err[0]).toContain('agent-relay cloud login'); }); @@ -120,6 +131,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register: vi.fn(async () => { throw new CloudFlowError('http_error', 'HTTP 404', 404); }), }); @@ -127,8 +139,8 @@ describe('createCloudMirrorSession', () => { session.onRunStarted({ runId: '01RUN' }); await session.finish(okReport); - expect(err[0]).toContain('does not accept local runs yet'); - expect(err[0]).toContain('the run is unaffected'); + expect(err[0]).toContain('does not accept local runs'); + expect(err[0]).toContain('the run itself is unaffected'); }); it('never registers a run that was refused before it started', async () => { @@ -138,6 +150,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register }); // No run id ever arrived: `flows run` refused the flow at check time. @@ -160,6 +173,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); session.onRunStarted({ runId: '01RUN' }); @@ -188,6 +202,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); session.onRunStarted({ runId: '01RUN' }); @@ -218,6 +233,7 @@ describe('createCloudMirrorSession', () => { source: async () => ({ workflow: 'name: demo\n', fileType: 'yaml' }), dataDir: '/data', log: () => [], + requested: 'flag', }, cli, {}, { register: vi.fn(async () => registration()), createMirror: vi.fn(() => mirror) }); session.onRunStarted({ runId: '01RUN' }); diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index ea5fec6da..c5749685b 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -108,14 +108,14 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa { verb: 'resume', argv: ['resume', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', - '--no-observer-link', '--no-cloud-mirror', '--allow-human-influenced', RUN_ID], + '--no-observer-link', '--cloud-mirror', '--allow-human-influenced', RUN_ID], variant: 'resume', }, { verb: 'run', argv: ['run', 'flow.yaml'], variant: 'run' }, { verb: 'run', argv: ['run', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', - '--no-observer-link', '--no-cloud-mirror', '--allow-human-influenced', '--input', '{"a":1}', 'review.flow.ts'], + '--no-observer-link', '--cloud-mirror', '--allow-human-influenced', '--input', '{"a":1}', 'review.flow.ts'], variant: 'run', }, // `--input` is the authored body's argument and `--reuse-from` memoizes a From f6e68c3635da92ecb44510ac3758c62b83c13838 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 15:51:46 -0700 Subject: [PATCH 6/7] fix(cli): address PR review on the local-run mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six findings from Devin on #580. **Transcript reads are confined to the run's own tree (security).** `defaultReadTranscript` opened whatever path the journal named and the mirror uploaded it to Cloud as an agent log. The journal is written by the worker, so it is not hostile input in the ordinary case — but it is a file on a developer's disk, and "reads only this run's journals" has to cover the files too. An unconfined path turns a crafted or corrupted journal into an arbitrary-file upload, which is far worse than a missing transcript. Paths outside `/runs//` are dropped and counted on stderr. **A stalled Cloud can no longer hold a finished run open.** `finish` set a 30-second deadline and then only checked it *between* phases, while every request underneath carried its own 30-second timeout — so the worst case was minutes, and the CLI awaits finish before returning the run's exit code. Each request now gets the budget that is actually left. **An in-flight poll cannot outlive the finish.** `clearInterval` stops future polls, not the one already running. That poll shares the step cache and the sequence counter with `finish`, and Cloud revokes the run token at the terminal callback — so a poll that outlived the finish could publish a view it can never repair. The poll is now retained, awaited within the same deadline, and fenced from publishing afterwards. **A retried agent's earlier charges no longer vanish.** `finalStep` took `total_cost_usd` from the last attempt while summing tokens across all of them — inconsistent on its own terms, and Cloud totals these rows for the run's spend, so an agent that spent $0.02 then $0.05 reported $0.05. **Declarative runs draw their graph.** A YAML flow declares `depends_on` in its spec, not in an authored-step stream, so the dashboard drew every node unconnected. The edges are read off `run.spawned`, which this fold already has in hand, rather than by widening `StepView` — `flows status --json` is a pinned shape and other readers depend on it. **Bundle runs reach the dashboard.** `flows run @sha256:` names a bundle the runner fetches, not a file on disk. Reading the argument as a path refused the registration and left a good run off the dashboard; a digest reference now mirrors what the journal recorded, the same source a resume uses. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/cli.ts | 2 +- packages/sdk/src/cli/cloud-mirror-session.ts | 13 ++- packages/sdk/src/cloud-mirror-step.ts | 55 +++++++++++- packages/sdk/src/cloud-mirror-transport.ts | 18 +++- packages/sdk/src/cloud-mirror.ts | 89 ++++++++++++++++--- .../sdk/tests/cloud-mirror-session.test.ts | 4 +- packages/sdk/tests/cloud-mirror-step.test.ts | 4 +- packages/sdk/tests/cloud-mirror.test.ts | 34 ++++++- 8 files changed, 195 insertions(+), 24 deletions(-) diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index de0c10fc9..a33fb3f2c 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -330,7 +330,7 @@ export async function runCli( ? undefined : createCloudMirrorSession({ source: parsed.command === 'run' - ? mirrorSourceFromPath(parsed.value, parsed.input) + ? mirrorSourceFromPath(parsed.value, parsed.input, parsed.dataDir) : mirrorSourceFromJournal(parsed.dataDir), dataDir: parsed.dataDir, log: () => runnerLog, diff --git a/packages/sdk/src/cli/cloud-mirror-session.ts b/packages/sdk/src/cli/cloud-mirror-session.ts index fb359d133..2c73c11f4 100644 --- a/packages/sdk/src/cli/cloud-mirror-session.ts +++ b/packages/sdk/src/cli/cloud-mirror-session.ts @@ -29,6 +29,7 @@ import { cloudConnection, CloudFlowError } from '../cloud-http.js'; import { recordMirroredRun, readMirroredRun } from '../cloud-mirror-ledger.js'; import { createRunMirror, readJournalEvents, type RunMirror } from '../cloud-mirror.js'; import { MirrorClient, registerLocalRun, type MirrorRunSource } from '../cloud-mirror-transport.js'; +import { parseDigestReference } from '../bundle-transport.js'; import { isAuthoredFlowPath, parseDirectInput } from '../direct-input.js'; import type { ProgressEvent } from '../progress.js'; import type { RunReport } from './run.js'; @@ -192,8 +193,16 @@ export function createCloudMirrorSession( export function mirrorSourceFromPath( path: string, inputArgument: string | undefined, -): () => Promise { - return async () => { + dataDir: string, +): (runId: string) => Promise { + const fromJournal = mirrorSourceFromJournal(dataDir); + return async (runId) => { + // `flows run @sha256:` names a bundle, not a file on this + // disk: the runner fetches it before executing. Reading the argument as a + // path there refused the registration and left a perfectly good run off + // the dashboard, so fall back to what the journal recorded — the same + // source a resume mirrors. + if (parseDigestReference(path)) return fromJournal(runId); const workflow = await readFile(path, 'utf8'); if (!isAuthoredFlowPath(path)) return { workflow, fileType: 'yaml' }; return { workflow, fileType: 'ts', inputs: parseDirectInput(inputArgument) }; diff --git a/packages/sdk/src/cloud-mirror-step.ts b/packages/sdk/src/cloud-mirror-step.ts index 70af173ee..883bb1e5e 100644 --- a/packages/sdk/src/cloud-mirror-step.ts +++ b/packages/sdk/src/cloud-mirror-step.ts @@ -313,6 +313,7 @@ interface AttemptRecord { transcriptPath?: string; tokensIn?: number; tokensOut?: number; + costUsd?: number; } /** @@ -330,6 +331,7 @@ function attemptsByStep(events: readonly JournalEvent[]): Map= 0 + ? { costUsd: result['total_cost_usd'] as number } : {}), }); byStep.set(event.step_id, entries); } @@ -364,6 +369,11 @@ function finalStep( const tokensIn = attempts.reduce((total, entry) => total + (entry.tokensIn ?? 0), 0); const tokensOut = attempts.reduce((total, entry) => total + (entry.tokensOut ?? 0), 0); const failure = transcript?.failure ?? null; + // Summed across attempts, exactly as the tokens above are. Taking the last + // attempt's figure while summing its tokens was inconsistent in itself, and + // Cloud totals these rows for the run's spend — so a retried agent's earlier + // charges simply vanished from the run. + const spentUsd = attempts.reduce((total, entry) => total + (entry.costUsd ?? 0), 0); const detail = stepDetail(attempts, env); const verification = step.last_attempt?.verification ?? null; // What the step said about itself. A gate's verdict detail is the nearest @@ -373,7 +383,6 @@ function finalStep( const summary = text(verification?.detail, env, OUTPUT_SUMMARY_MAX_CHARS) ?? `step ${succeeded ? 'completed' : 'ended'}: ${identifier(completionReason, env) ?? 'unknown'}`; const model = identifier(transcript?.model, env); - const cost = transcript?.total_cost_usd; const error = succeeded ? undefined : text(failure?.excerpt, env, ERROR_MAX_CHARS); return { stepName, @@ -398,7 +407,7 @@ function finalStep( ...(model === undefined ? {} : { model }), ...(tokensIn > 0 ? { tokensInput: Math.min(tokensIn, MAX_INT32) } : {}), ...(tokensOut > 0 ? { tokensOutput: Math.min(tokensOut, MAX_INT32) } : {}), - ...(typeof cost === 'number' && Number.isFinite(cost) && cost >= 0 ? { costUsd: cost } : {}), + ...(spentUsd > 0 ? { costUsd: spentUsd } : {}), ...(error === undefined ? {} : { error }), ...(detail.detail === undefined ? {} : { detail: detail.detail }), ...(detail.truncated === undefined ? {} : { detailTruncated: detail.truncated }), @@ -448,6 +457,15 @@ export function mirrorJournal( } } const index = authoredIndex(events, env); + // A declarative flow declares its edges in the spec, not in an authored-step + // stream — so a YAML run had no hints at all and drew every node + // unconnected. Read them from `run.spawned`, keyed the same way, and let the + // authored index win where both exist (an authored root's index is the + // richer record, and carries labels too). + const declared = declaredEdges(runId, events, env); + for (const [key, hint] of declared) { + if (!index.hints.has(key)) index.hints.set(key, hint); + } return { runId, status: view.status, @@ -499,6 +517,39 @@ function authoredIndex( return { children, hints }; } +/** + * The `depends_on` a declarative spec declares, as graph hints. + * + * `foldRunState` reads these to decide readiness but does not surface them on + * `StepView`, and deliberately: `flows status --json` is a pinned shape. So + * they are read here, straight off the `run.spawned` payload this fold already + * has in hand, rather than by widening a view other readers depend on. + */ +function declaredEdges( + runId: string, + events: readonly JournalEvent[], + env: NodeJS.ProcessEnv, +): Map { + const hints = new Map(); + const spawned = events.find(event => event.entry_type === 'run.spawned'); + const steps = record(spawned?.payload)?.['spec']; + const declared = record(steps)?.['steps']; + if (!Array.isArray(declared)) return hints; + for (const entry of declared) { + const step = record(entry); + const id = identifier(step?.['id'], env); + if (id === undefined) continue; + const raw = step?.['depends_on'] ?? step?.['dependsOn']; + if (!Array.isArray(raw)) continue; + const after = raw + .slice(0, DEPENDS_ON_MAX_ENTRIES) + .map(value => identifier(value, env)) + .filter((value): value is string => value !== undefined); + hints.set(`${runId}/${id}`, { after }); + } + return hints; +} + /** * Build a snapshot envelope that actually fits both bounds. * diff --git a/packages/sdk/src/cloud-mirror-transport.ts b/packages/sdk/src/cloud-mirror-transport.ts index 113a4001c..0b50fb607 100644 --- a/packages/sdk/src/cloud-mirror-transport.ts +++ b/packages/sdk/src/cloud-mirror-transport.ts @@ -189,11 +189,12 @@ export class MirrorClient { steps: readonly FinalStep[], omittedStepCount: number, signal?: AbortSignal, + timeoutMs?: number, ): Promise { return this.post( `/api/v1/workflows/runs/${this.registration.runId}/steps`, JSON.stringify({ steps, ...(omittedStepCount > 0 ? { omittedStepCount } : {}) }), - { ...(signal === undefined ? {} : { signal }) }, + { ...(signal === undefined ? {} : { signal }), ...(timeoutMs === undefined ? {} : { timeoutMs }) }, ); } @@ -202,12 +203,21 @@ export class MirrorClient { * `/logs` route already reads: `runner.log` for the run, and * `/agent.log` for a step's assembled transcript. */ - async putObject(key: string, bytes: Uint8Array, signal?: AbortSignal): Promise { + async putObject( + key: string, + bytes: Uint8Array, + signal?: AbortSignal, + timeoutMs?: number, + ): Promise { if (!/^[A-Za-z0-9][A-Za-z0-9._/-]{0,511}$/u.test(key) || key.includes('..')) return false; try { await cloudFetch( `/api/v1/workflows/runs/${this.registration.runId}/storage/${key}`, - { ...this.bound, ...(signal === undefined ? {} : { signal }) }, + { + ...this.bound, + ...(signal === undefined ? {} : { signal }), + ...(timeoutMs === undefined ? {} : { requestTimeoutMs: timeoutMs }), + }, { method: 'PUT', body: bytes, contentType: 'text/plain' }, ); return true; @@ -226,11 +236,13 @@ export class MirrorClient { result: Record, error?: string, signal?: AbortSignal, + timeoutMs?: number, ): Promise { try { await cloudFetch('/api/v1/workflows/callback', { ...this.bound, ...(signal === undefined ? {} : { signal }), + ...(timeoutMs === undefined ? {} : { requestTimeoutMs: timeoutMs }), }, { method: 'POST', body: JSON.stringify({ diff --git a/packages/sdk/src/cloud-mirror.ts b/packages/sdk/src/cloud-mirror.ts index 260cb29df..38cce141e 100644 --- a/packages/sdk/src/cloud-mirror.ts +++ b/packages/sdk/src/cloud-mirror.ts @@ -30,7 +30,8 @@ // final step rows have to land before it — otherwise the mirror would report // the run finished and then find itself unable to say what it did. -import { readFile, stat } from 'node:fs/promises'; +import { open, readFile, stat } from 'node:fs/promises'; +import { isAbsolute, join, relative as relative_, resolve } from 'node:path'; import { walkJournal, JournalReadError, type JournalEvent } from './journal-reader.js'; import { fitSnapshot, mirrorJournal, withGraphHints, @@ -159,15 +160,36 @@ export async function readJournalEvents( } } +/** + * Where a transcript for `runId` is allowed to live: under this data + * directory's own tree for that run, and nowhere else. + * + * The path comes out of the journal, which the worker wrote — so it is not + * hostile input in the ordinary case. But a journal is a file on a developer's + * disk, this reader uploads whatever it is handed, and "reads only this run's + * journals" has to mean the files too. An unconfined path turns a crafted or + * corrupted journal into an arbitrary-file upload, which is a much worse + * failure than a missing transcript. + */ +export function transcriptRoot(dataDir: string, runId: string): string { + return join(resolve(dataDir), 'runs', runId); +} + +/** True when `path` resolves inside `root` — a prefix test that `..` cannot pass. */ +export function withinRoot(root: string, path: string): boolean { + const relative = relative_(root, resolve(path)); + return relative.length > 0 && !relative.startsWith('..') && !isAbsolute(relative); +} + async function defaultReadTranscript(path: string): Promise<{ bytes: Buffer; size: number }> { const info = await stat(path); if (!info.isFile()) throw new Error('transcript is not a regular file'); // Bound the read, not just the result: materializing a huge file only to - // discard all but its tail can exhaust the CLI before it reports at all. + // discard all but the tail can exhaust the CLI before it reports at all. if (info.size <= MIRROR_TRANSCRIPT_MAX_BYTES) { return { bytes: await readFile(path), size: info.size }; } - const handle = await (await import('node:fs/promises')).open(path, 'r'); + const handle = await open(path, 'r'); try { const buffer = Buffer.alloc(MIRROR_TRANSCRIPT_MAX_BYTES); const { bytesRead } = await handle.read( @@ -270,7 +292,14 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { /** Finished steps past the report cap, by identity. Counted, never sent. */ const unreported = new Set(); let timer: ReturnType | undefined; - let polling = false; + /** + * The poll currently running, if one is. `clearInterval` stops future polls + * but not one already in flight: that poll shares the step cache and the + * sequence counter with `finish`, and Cloud revokes the run token at the + * terminal callback — so a poll that outlived the finish could publish a + * stale view it can never repair, or none at all. + */ + let inFlight: Promise | undefined; let stopped = false; let sequence = 0; let acknowledged: string | undefined; @@ -359,7 +388,16 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { } for (const ref of folded.transcripts) { if (!finals.has(`${runId}/${ref.stepName}`)) continue; - transcripts.set(`${runId}/${ref.stepName}`, { stepName: ref.stepName, attempts: ref.attempts }); + // Only files under this run's own tree. A journal names its transcript + // paths, and this reader uploads what it is handed, so the confinement + // belongs here rather than in the reader's caller. + const root = transcriptRoot(options.dataDir, runId); + const attempts = ref.attempts.filter(attempt => withinRoot(root, attempt.path)); + if (attempts.length < ref.attempts.length) { + diagnostic(`ignored ${ref.attempts.length - attempts.length} transcript path(s) outside ${runId}'s own tree`); + } + if (attempts.length === 0) continue; + transcripts.set(`${runId}/${ref.stepName}`, { stepName: ref.stepName, attempts }); } if (folded.terminal) finished.add(runId); } @@ -400,8 +438,9 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { }; const poll = async (budgetMs: number): Promise => { - if (polling || stopped) return; - polling = true; + if (inFlight !== undefined || stopped) return; + let settle = (): void => {}; + inFlight = new Promise(done => { settle = done; }); try { const deadline = now() + budgetMs; await scan(deadline); @@ -409,10 +448,24 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { } catch (error) { diagnostic(`poll failed: ${error instanceof Error ? error.message : String(error)}`); } finally { - polling = false; + const done = inFlight; + inFlight = undefined; + settle(); + void done; } }; + /** + * The budget left, as a per-request timeout. + * + * `finish` had a deadline it only checked *between* phases, while every + * request underneath carried its own 30-second timeout — so a stalled Cloud + * could hold a finished local run open for minutes, and the CLI awaits this + * before returning the run's exit code. Handing each request the remaining + * budget makes the deadline mean what it says. + */ + const remaining = (deadline: number): number => Math.max(1, deadline - now()); + const uploadTranscripts = async (deadline: number): Promise => { let uploaded = 0; for (const [key, ref] of transcripts) { @@ -429,7 +482,8 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { if (assembled.bytes.length === 0) continue; // Name the row after the object only once the object is there, so no row // ever points at a transcript that was never written. - if (await options.client.putObject(`${ref.stepName}/agent.log`, assembled.bytes)) { + if (await options.client.putObject(`${ref.stepName}/agent.log`, assembled.bytes, + undefined, remaining(deadline))) { finals.set(key, { ...entry, row: { ...entry.row, sandboxId: ref.stepName } }); uploaded += 1; } @@ -445,7 +499,7 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { const withGraph = sent.map(entry => withGraphHints(entry.row, hints, entry.journalRunId, DEPENDS_ON_MAX_ENTRIES, names)); if (now() > deadline) return; - if (!await options.client.publishSteps(withGraph, omitted)) { + if (!await options.client.publishSteps(withGraph, omitted, undefined, remaining(deadline))) { diagnostic('Cloud did not accept the final step report; the run page keeps its live view'); } }; @@ -467,6 +521,16 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { if (timer !== undefined) { clearInterval(timer); timer = undefined; } const deadline = now() + MIRROR_FINISH_BUDGET_MS; try { + // Let an in-flight poll settle before touching the shared cache, and + // stop it publishing afterwards. Bounded by the same deadline as + // everything else here, so a wedged poll cannot hold the CLI open. + if (inFlight !== undefined) { + await Promise.race([ + inFlight, + new Promise(done => { setTimeout(done, remaining(deadline)).unref?.(); }), + ]); + } + stopped = true; // One last reading, so the page shows the run's actual last moments // rather than whatever the previous poll happened to catch. await scan(deadline); @@ -480,7 +544,8 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { await options.client.putObject('runner.log', bytes.length > MIRROR_RUNNER_LOG_MAX_BYTES ? bytes.subarray(bytes.length - MIRROR_RUNNER_LOG_MAX_BYTES) - : bytes); + : bytes, + undefined, remaining(deadline)); } await uploadTranscripts(deadline); await publishFinal(deadline); @@ -490,6 +555,8 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { outcome.status, redactJson(outcome.result, env) as Record, outcome.error, + undefined, + remaining(deadline), ); } catch (error) { diagnostic(`could not finish the mirror: ${error instanceof Error ? error.message : String(error)}`); diff --git a/packages/sdk/tests/cloud-mirror-session.test.ts b/packages/sdk/tests/cloud-mirror-session.test.ts index 2997a85a9..ea3ee637c 100644 --- a/packages/sdk/tests/cloud-mirror-session.test.ts +++ b/packages/sdk/tests/cloud-mirror-session.test.ts @@ -258,7 +258,7 @@ describe('mirrorSourceFromPath', () => { const dir = await mkdtemp(join(tmpdir(), 'mirror-source-')); const path = join(dir, 'flow.yaml'); await writeFile(path, 'name: demo\nsteps: []\n'); - await expect(mirrorSourceFromPath(path, undefined)()).resolves.toEqual({ + await expect(mirrorSourceFromPath(path, undefined, '/data')('01RUN')).resolves.toEqual({ workflow: 'name: demo\nsteps: []\n', fileType: 'yaml', }); }); @@ -267,7 +267,7 @@ describe('mirrorSourceFromPath', () => { const dir = await mkdtemp(join(tmpdir(), 'mirror-source-')); const path = join(dir, 'review.flow.ts'); await writeFile(path, 'export default flow("review", () => {});\n'); - await expect(mirrorSourceFromPath(path, '{"pr":7}')()).resolves.toEqual({ + await expect(mirrorSourceFromPath(path, '{"pr":7}', '/data')('01RUN')).resolves.toEqual({ workflow: 'export default flow("review", () => {});\n', fileType: 'ts', inputs: { pr: 7 }, }); }); diff --git a/packages/sdk/tests/cloud-mirror-step.test.ts b/packages/sdk/tests/cloud-mirror-step.test.ts index 3c3005f34..63258ed27 100644 --- a/packages/sdk/tests/cloud-mirror-step.test.ts +++ b/packages/sdk/tests/cloud-mirror-step.test.ts @@ -102,7 +102,9 @@ describe('mirrorJournal', () => { // Summed across attempts, not taken from the last one. tokensInput: 30, tokensOutput: 12, - costUsd: 0.05, + // Summed across attempts, as the tokens are: the first attempt's $0.02 + // used to vanish from the run's spend. + costUsd: 0.07, // Named after the transcript object only once one is uploaded. sandboxId: '', }); diff --git a/packages/sdk/tests/cloud-mirror.test.ts b/packages/sdk/tests/cloud-mirror.test.ts index b2c24d6ed..748b78f78 100644 --- a/packages/sdk/tests/cloud-mirror.test.ts +++ b/packages/sdk/tests/cloud-mirror.test.ts @@ -134,7 +134,7 @@ describe('createRunMirror', () => { dataDir: '/data', env: {}, readJournal: async (runId: string) => - runId === '01ROOT' ? rootJournal('01ROOT', '01CHILD') : childJournal('01CHILD', '/tmp/attempt-1.jsonl'), + runId === '01ROOT' ? rootJournal('01ROOT', '01CHILD') : childJournal('01CHILD', '/data/runs/01CHILD/steps/write/attempt-1.transcript.jsonl'), readTranscript: async () => ({ bytes: Buffer.from('{"type":"result"}\n'), size: 18 }), }); @@ -166,7 +166,7 @@ describe('createRunMirror', () => { client, dataDir: '/data', env: {}, - readJournal: async () => childJournal('01RUN', '/tmp/attempt-1.jsonl'), + readJournal: async () => childJournal('01RUN', '/data/runs/01RUN/steps/write/attempt-1.transcript.jsonl'), readTranscript: async () => ({ bytes: Buffer.from('{}\n'), size: 3 }), }); @@ -243,6 +243,36 @@ describe('createRunMirror', () => { expect(report.omitted).toBe(12); }); + /** + * The paths come out of a journal and this reader uploads whatever it is + * handed, so "reads only this run's journals" has to cover the files too. An + * unconfined path turns a crafted or corrupted journal into an + * arbitrary-file upload. + */ + it('refuses a transcript path outside the run\'s own tree', async () => { + seq = 0; + const { client, calls } = cloud(); + const diagnostic = vi.fn(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + diagnostic, + readJournal: async () => childJournal('01RUN', '/etc/passwd'), + readTranscript: async () => ({ bytes: Buffer.from('root:x:0:0\n'), size: 11 }), + }); + + mirror.start('01RUN'); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); + + expect(calls.some(call => call.kind === 'object' + && (call.body as { key: string }).key.endsWith('agent.log'))).toBe(false); + expect(diagnostic.mock.calls.flat().join(' ')).toContain('outside'); + // The step row still lands; it just names no transcript. + const report = calls.find(call => call.kind === 'steps')!.body as { steps: Array> }; + expect(report.steps[0]!.sandboxId).toBe(''); + }); + it('never throws out of finish, whatever Cloud answers', async () => { seq = 0; const { client } = cloud(() => false); From 6009955c3ff2f1adf48ff0df452f03f0e8f828f0 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 16:19:10 -0700 Subject: [PATCH 7/7] =?UTF-8?q?fix(cli):=20two=20more=20from=20review=20?= =?UTF-8?q?=E2=80=94=20unique=20transcript=20keys,=20and=20a=20fence=20tha?= =?UTF-8?q?t=20means=20what=20it=20says?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Two steps could overwrite each other's transcript.** The caches here are keyed `/` precisely because child journals can repeat a kernel step id — and then the object key and the row's `sandboxId` used only the step id, so the run page showed one step's log for both. Only a genuine collision is disambiguated: the common run has one journal per step, and keeping the bare step id there means a mirrored transcript is named exactly as a hosted one is. **A snapshot could follow the terminal callback.** `finish` stops *waiting* for an in-flight poll once its deadline passes, so that poll can still be running when the callback goes out — and the callback revokes the credential and must be last. The first attempt at this gated publishing on "finish has begun", which also suppressed finish's own last reading: the snapshot that shows the run's final moments. The invariant is narrower than that — nothing publishes after the *terminal callback* — and the flag now says exactly that. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/cloud-mirror.ts | 37 +++++++++++++++++++++++-- packages/sdk/tests/cloud-mirror.test.ts | 27 ++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/cloud-mirror.ts b/packages/sdk/src/cloud-mirror.ts index 38cce141e..2620680d3 100644 --- a/packages/sdk/src/cloud-mirror.ts +++ b/packages/sdk/src/cloud-mirror.ts @@ -301,6 +301,8 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { */ let inFlight: Promise | undefined; let stopped = false; + /** Set immediately before the terminal callback; nothing may publish after it. */ + let terminalReported = false; let sequence = 0; let acknowledged: string | undefined; let acknowledgedAt = now(); @@ -417,6 +419,15 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { }; const publishSnapshot = async (deadline: number): Promise => { + // The invariant is narrow and exact: nothing publishes after the terminal + // callback, which revokes the credential and must be last. `finish` stops + // *waiting* for an in-flight poll once its deadline passes, so that poll can + // still be running when the callback goes out — and a view arriving then is + // worse than a missing one, because the final rows have already replaced it. + // + // Deliberately not gated on "finish has begun": finish takes its own last + // reading first, and that is the snapshot showing the run's final moments. + if (terminalReported) return; const view = ordered(); if (view.length === 0) return; // `elapsedMs` moves every poll and is excluded from the comparison, or the @@ -444,7 +455,8 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { try { const deadline = now() + budgetMs; await scan(deadline); - await publishSnapshot(deadline); + // Finish takes its own reading; a poll racing it adds nothing. + if (!stopped) await publishSnapshot(deadline); } catch (error) { diagnostic(`poll failed: ${error instanceof Error ? error.message : String(error)}`); } finally { @@ -466,12 +478,30 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { */ const remaining = (deadline: number): number => Math.max(1, deadline - now()); + /** + * The object name for one step's transcript, unique across journals. + * + * The caches here are keyed `/` precisely because child + * journals can repeat a kernel step id — and then the object key and the + * row's `sandboxId` used only the step id, so two steps overwrote each + * other's log and the run page showed one step's transcript for both. + * + * Only a genuine collision is disambiguated. The common run has one journal + * per step, and keeping the bare step id there means a mirrored transcript is + * named exactly as a hosted one is. + */ + const transcriptName = (journalRunId: string, stepName: string): string => { + const repeated = [...finals.keys()].filter(key => key.endsWith(`/${stepName}`)).length > 1; + return repeated ? `${journalRunId}.${stepName}` : stepName; + }; + const uploadTranscripts = async (deadline: number): Promise => { let uploaded = 0; for (const [key, ref] of transcripts) { if (uploaded >= MIRROR_MAX_TRANSCRIPT_UPLOADS || now() > deadline) break; const entry = finals.get(key); if (entry === undefined) continue; + const objectName = transcriptName(entry.journalRunId, ref.stepName); let assembled; try { assembled = await assembleTranscript(ref.attempts, readTranscript); @@ -482,9 +512,9 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { if (assembled.bytes.length === 0) continue; // Name the row after the object only once the object is there, so no row // ever points at a transcript that was never written. - if (await options.client.putObject(`${ref.stepName}/agent.log`, assembled.bytes, + if (await options.client.putObject(`${objectName}/agent.log`, assembled.bytes, undefined, remaining(deadline))) { - finals.set(key, { ...entry, row: { ...entry.row, sandboxId: ref.stepName } }); + finals.set(key, { ...entry, row: { ...entry.row, sandboxId: objectName } }); uploaded += 1; } } @@ -551,6 +581,7 @@ export function createRunMirror(options: RunMirrorOptions): RunMirror { await publishFinal(deadline); // Last, always: this transition revokes the credential every call // above depends on. + terminalReported = true; await options.client.reportTerminal( outcome.status, redactJson(outcome.result, env) as Record, diff --git a/packages/sdk/tests/cloud-mirror.test.ts b/packages/sdk/tests/cloud-mirror.test.ts index 748b78f78..4eb1dd3d1 100644 --- a/packages/sdk/tests/cloud-mirror.test.ts +++ b/packages/sdk/tests/cloud-mirror.test.ts @@ -273,6 +273,33 @@ describe('createRunMirror', () => { expect(report.steps[0]!.sandboxId).toBe(''); }); + /** + * The terminal callback revokes the run's credential, so a snapshot after it + * is a push that can never land and a view the final rows have already + * replaced. `finish` stops *waiting* for an in-flight poll at its deadline, + * which is exactly when this can happen. + */ + it('publishes no snapshot once the terminal status has gone out', async () => { + seq = 0; + const { client, calls } = cloud(); + const mirror = createRunMirror({ + client, + dataDir: '/data', + env: {}, + readJournal: async () => childJournal('01RUN'), + }); + + mirror.start('01RUN'); + await mirror.finish({ status: 'completed', result: { ok: true, status: 'completed' } }); + + const kinds = calls.map(call => call.kind); + expect(kinds).toContain('snapshot'); + // Finish's own reading still goes out — it is the run's final moments — + // but nothing follows the callback. + expect(kinds.lastIndexOf('snapshot')).toBeLessThan(kinds.indexOf('terminal')); + expect(kinds.at(-1)).toBe('terminal'); + }); + it('never throws out of finish, whatever Cloud answers', async () => { seq = 0; const { client } = cloud(() => false);