From f5dbcb22407d8e29e615752c8b978edd7d81f879 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 09:57:52 +0200 Subject: [PATCH 001/242] fix(ci): pin coordinator source revision --- .github/workflows/deploy-crabbox-coordinator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-crabbox-coordinator.yml b/.github/workflows/deploy-crabbox-coordinator.yml index e49355cd..a1b4d89b 100644 --- a/.github/workflows/deploy-crabbox-coordinator.yml +++ b/.github/workflows/deploy-crabbox-coordinator.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: openclaw/crabbox - ref: main + ref: ec073fdcf868b2d5c450d600d900cdf41f958de9 # crabbox main, 2026-07-11 - name: Set up Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 From d9213f8bb6ea79c25474dbd798ce62b10c5627ec Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 09:57:52 +0200 Subject: [PATCH 002/242] fix(image): verify crabbox release archive --- Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2e919eb9..1928ac62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ FROM docker.io/cloudflare/sandbox:0.10.1 USER root ARG CRABBOX_VERSION=0.17.1 +ARG CRABBOX_SHA256_AMD64=3c41839257e4622e28bcec8b0f0153f19d78d436fd548894a7c7d7726d922611 +ARG CRABBOX_SHA256_ARM64=4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2 RUN set -eux; \ apt-get update; \ @@ -61,10 +63,15 @@ RUN set -eux; \ RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ - case "$arch" in amd64|arm64) ;; *) echo "unsupported arch: $arch" >&2; exit 1 ;; esac; \ + case "$arch" in \ + amd64) checksum="$CRABBOX_SHA256_AMD64" ;; \ + arm64) checksum="$CRABBOX_SHA256_ARM64" ;; \ + *) echo "unsupported arch: $arch" >&2; exit 1 ;; \ + esac; \ curl -fsSL \ "https://github.com/openclaw/crabbox/releases/download/v${CRABBOX_VERSION}/crabbox_${CRABBOX_VERSION}_linux_${arch}.tar.gz" \ -o /tmp/crabbox.tar.gz; \ + echo "$checksum /tmp/crabbox.tar.gz" | sha256sum -c -; \ tar -xzf /tmp/crabbox.tar.gz -C /usr/local/bin crabbox; \ chmod +x /usr/local/bin/crabbox; \ rm -f /tmp/crabbox.tar.gz; \ From c7648904a1b523bc50b6bc7104323a363bfba756 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 09:58:59 +0200 Subject: [PATCH 003/242] fix(terminal): confirm one-shot input delivery --- internal/fleetapi/client.go | 9 ++-- internal/terminalws/client.go | 54 +++++++++++++++++-- internal/terminalws/client_test.go | 85 ++++++++++++++++++++++++++++++ src/worker/terminal-hub.ts | 27 ++++++---- tests/terminal-hub.test.ts | 11 +++- 5 files changed, 168 insertions(+), 18 deletions(-) diff --git a/internal/fleetapi/client.go b/internal/fleetapi/client.go index 5e53ed5a..6fc54af6 100644 --- a/internal/fleetapi/client.go +++ b/internal/fleetapi/client.go @@ -193,7 +193,7 @@ func (c *Client) Message( if enter { message += "\n" } - return client.SendInput(ctx, []byte(message)) + return client.SendInputConfirmed(ctx, []byte(message)) } func (c *Client) Attach( @@ -222,9 +222,10 @@ func (c *Client) terminal(ctx context.Context, id string, cols uint32, rows uint return nil, err } client, err := terminalws.Dial(ctx, endpoint, id, terminalws.Options{ - Header: headers, - Cols: cols, - Rows: rows, + HTTPClient: c.http, + Header: headers, + Cols: cols, + Rows: rows, }) if err != nil { var statusErr *terminalws.HandshakeStatusError diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index bc84fdc4..39f924ef 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -49,9 +49,10 @@ const ( ) type Options struct { - Header http.Header - Cols uint32 - Rows uint32 + HTTPClient *http.Client + Header http.Header + Cols uint32 + Rows uint32 } type HandshakeStatusError struct { @@ -122,7 +123,13 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option if sessionID == "" { return nil, errors.New("terminal session id is required") } + if options.HTTPClient != nil && options.HTTPClient.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, options.HTTPClient.Timeout) + defer cancel() + } conn, resp, err := websocket.Dial(ctx, endpoint, &websocket.DialOptions{ + HTTPClient: options.HTTPClient, HTTPHeader: options.Header, }) if err != nil { @@ -201,6 +208,47 @@ func (c *Client) SendInput(ctx context.Context, payload []byte) error { }) } +func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { + if err := c.SendInput(ctx, payload); err != nil { + return err + } + for { + current, err := c.read(ctx) + if err != nil { + return err + } + if current.sessionID != "" && current.sessionID != c.sessionID { + continue + } + switch current.messageType { + case messageOutput: + if err := c.write(ctx, frame{ + messageType: messageAck, + sessionID: c.sessionID, + payload: ackPayload(uint32(len(current.payload))), + }); err != nil { + return err + } + case messageError, messageControlRevoked: + c.canInput.Store(false) + return frameError(current, "terminal input rejected") + case messageControlGranted: + c.canInput.Store(true) + case messageEvent: + var event eventPayload + if err := json.Unmarshal(current.payload, &event); err != nil { + return fmt.Errorf("decode terminal event: %w", err) + } + switch event.Type { + case "input-accepted": + return nil + case "closed": + return errors.New("terminal closed before accepting input") + } + } + } +} + func (c *Client) Resize(ctx context.Context, size Size) error { if size.Cols == 0 || size.Rows == 0 { return nil diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index f1e04cc6..83055520 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "sync" "testing" "time" @@ -274,6 +275,90 @@ func TestClientSubscribesSendsInputAndAcknowledgesOutput(t *testing.T) { } } +func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-revoked", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + revoked, _ := json.Marshal(eventPayload{Error: "terminal control revoked"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageControlRevoked, + sessionID: "IS-revoked", + payload: revoked, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-revoked", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + err = client.SendInputConfirmed(context.Background(), []byte("blocked\n")) + if err == nil || !strings.Contains(err.Error(), "control revoked") { + t.Fatalf("error = %v", err) + } +} + +func TestDialUsesConfiguredHTTPClientAndTimeout(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + return + } + } + <-r.Context().Done() + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + httpClient := server.Client() + httpClient.Timeout = 25 * time.Millisecond + started := time.Now() + _, err = Dial(context.Background(), endpoint, "IS-timeout", Options{HTTPClient: httpClient}) + if err == nil { + t.Fatal("expected subscription timeout") + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("dial timeout took %s", elapsed) + } +} + func TestAttachClosesCloseableTerminalAfterRemoteClosure(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 88888d27..58437add 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -217,17 +217,24 @@ export class TerminalHub { if (!canInput) { return; } - if (subscription.upstream.readyState === WebSocket.OPEN) { - const inputs = await this.dependencies.inputPayloads( - subscription, - user, - frame.payload, - ); - for (const [index, input] of inputs.entries()) { - if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); - subscription.upstream.send(input); - } + if (subscription.upstream.readyState !== WebSocket.OPEN) { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: "terminal upstream is not open", + }); + return; + } + const inputs = await this.dependencies.inputPayloads( + subscription, + user, + frame.payload, + ); + for (const [index, input] of inputs.entries()) { + if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); + subscription.upstream.send(input); } + sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { + type: "input-accepted", + }); return; } if (frame.type === TerminalMessageType.Resize) { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 7ed93128..b54ed691 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -337,6 +337,9 @@ test("terminal hub routes multiplex frames and explicit output acknowledgements" }); await flushQueues(); assert.deepEqual(new Uint8Array(upstream.sent.at(-1) as Uint8Array), inputPayload); + const inputAccepted = frame(server.sent.at(-1)!); + assert.equal(inputAccepted.type, TerminalMessageType.Event); + assert.deepEqual(decodeJsonPayload(inputAccepted.payload), { type: "input-accepted" }); server.emit("message", { data: encodeTerminalFrame({ @@ -397,8 +400,14 @@ test("terminal hub publishes live controller downgrades and promotions", async ( }), }); await flushQueues(); - assert.equal(frame(server.sent.at(-1)!).type, TerminalMessageType.ControlGranted); + assert.equal( + server.sent.map((payload) => frame(payload).type).at(-2), + TerminalMessageType.ControlGranted, + ); assert.equal(new TextDecoder().decode(upstream.sent.at(-1) as Uint8Array), "allowed"); + assert.deepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "input-accepted", + }); server.emit("close"); }); From d119c8f06dbac8ad94e99ebc16564eb6646c31dc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:00:05 +0200 Subject: [PATCH 004/242] fix(runtime): reject unroutable profile ids --- src/runtime-profiles.ts | 2 +- tests/runtime-profiles.test.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/runtime-profiles.ts b/src/runtime-profiles.ts index 7181adab..b16e9bba 100644 --- a/src/runtime-profiles.ts +++ b/src/runtime-profiles.ts @@ -32,7 +32,7 @@ export type RuntimeProfileCodexSshValues = { profile: string; }; -const profileIDPattern = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,118}[A-Za-z0-9])?$/; +const profileIDPattern = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; const targetPattern = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,38}[A-Za-z0-9])?$/; const capabilityNames = ["terminal", "takeover", "vnc", "desktop", "logs", "artifacts"] as const; const capabilityNameSet = new Set(capabilityNames); diff --git a/tests/runtime-profiles.test.ts b/tests/runtime-profiles.test.ts index b4a8e15c..7a9c7f28 100644 --- a/tests/runtime-profiles.test.ts +++ b/tests/runtime-profiles.test.ts @@ -69,6 +69,10 @@ test("runtime profile catalog fails closed on malformed or ambiguous input", () '[{"id":"a","label":"A","capabilities":null}]', '[{"id":"a","label":"A","capabilities":{"unknown":true}}]', '[{"id":"a","label":"A","privateProvider":"hidden"}]', + '[{"id":"Desktop","label":"Desktop"}]', + '[{"id":"desktop.profile","label":"Desktop"}]', + '[{"id":"desktop_profile","label":"Desktop"}]', + `[{"id":"${"a".repeat(64)}","label":"Desktop"}]`, '[{"id":"a","label":"A","codexSsh":null}]', '[{"id":"a","label":"A","codexSsh":{"aliasTemplate":"box {sessionId}"}}]', '[{"id":"a","label":"A","codexSsh":{"aliasTemplate":"box-{unknown}"}}]', @@ -81,6 +85,10 @@ test("runtime profile catalog fails closed on malformed or ambiguous input", () for (const value of invalid) { assert.throws(() => parseRuntimeProfiles(value)); } + assert.equal( + parseRuntimeProfiles(JSON.stringify([{ id: "a".repeat(63), label: "Maximum" }]))[0]?.id.length, + 63, + ); assert.deepEqual(parseRuntimeProfiles(undefined), []); assert.deepEqual(parseRuntimeProfiles(""), []); }); From 80c56dcc49c7d25d9d8677c6d75b90df2d9a583d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:00:05 +0200 Subject: [PATCH 005/242] fix(terminal): send attributed commands atomically --- src/terminal-multiplayer.ts | 3 +-- tests/terminal-multiplayer.test.ts | 32 ++++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/terminal-multiplayer.ts b/src/terminal-multiplayer.ts index 3fb8a860..05346150 100644 --- a/src/terminal-multiplayer.ts +++ b/src/terminal-multiplayer.ts @@ -69,8 +69,7 @@ export function attributedTerminalInputPayloads( ): Uint8Array[] { const sender = terminalSenderTag(user); const attributed = `${sender} ${terminalSingleLineInput(submitted.text)}${submitted.eol}`; - const chunks = submitted.replaceCurrentLine ? ["\x15", ...attributed] : [...attributed]; - return chunks.map((chunk) => encoder.encode(chunk)); + return [encoder.encode(`${submitted.replaceCurrentLine ? "\x15" : ""}${attributed}`)]; } function updateTerminalInputLine(state: TerminalInputState, text: string): void { diff --git a/tests/terminal-multiplayer.test.ts b/tests/terminal-multiplayer.test.ts index f1609cd8..2c79abed 100644 --- a/tests/terminal-multiplayer.test.ts +++ b/tests/terminal-multiplayer.test.ts @@ -37,10 +37,14 @@ test("multiplayer input tracks interleaved writers on the shared session line", [encoder.encode("world")], ); - assert.equal( - text(multiplayerTerminalInputPayloadsForMode(state, secondUser, encoder.encode("\r"), true)), - '\x15 hello world\r', + const attributed = multiplayerTerminalInputPayloadsForMode( + state, + secondUser, + encoder.encode("\r"), + true, ); + assert.equal(attributed.length, 1); + assert.equal(text(attributed), '\x15 hello world\r'); }); test("multiplayer input attributes a final text fragment batched with enter", () => { @@ -51,10 +55,26 @@ test("multiplayer input attributes a final text fragment batched with enter", () [encoder.encode("hel")], ); - assert.equal( - text(multiplayerTerminalInputPayloadsForMode(state, user, encoder.encode("lo\r"), true)), - '\x15 hello\r', + const attributed = multiplayerTerminalInputPayloadsForMode( + state, + user, + encoder.encode("lo\r"), + true, ); + assert.equal(attributed.length, 1); + assert.equal(text(attributed), '\x15 hello\r'); +}); + +test("multiplayer input emits a complete attributed command atomically", () => { + const attributed = multiplayerTerminalInputPayloadsForMode( + newTerminalInputState(), + user, + encoder.encode("hello\r"), + true, + ); + + assert.equal(attributed.length, 1); + assert.equal(text(attributed), ' hello\r'); }); test("multiplayer input does not attribute text while a control sequence is pending", () => { From 192e1025ab46216de4f408e5b8f7826f057363db Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:00:05 +0200 Subject: [PATCH 006/242] fix(actions): honor live relay availability --- src/fleet-state.ts | 14 +++++++------- src/github-actions-runtime.ts | 5 ++++- tests/fleet-state.test.ts | 25 +++++++++++++++++++++++++ tests/github-actions-runtime.test.ts | 19 +++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/fleet-state.ts b/src/fleet-state.ts index 543db5a4..16e4dd6b 100644 --- a/src/fleet-state.ts +++ b/src/fleet-state.ts @@ -301,13 +301,13 @@ export function fleetSessionSummary( terminalCapable && (session.ptyAvailable === true || (session.canControl !== false && - (session.runtime === "github_actions" || - (session.ptyAvailable ?? - Boolean( - ptyRouteKind(session, { - sandboxAvailable: options.sandboxAvailable, - }), - ))))) && + session.runtime !== "github_actions" && + (session.ptyAvailable ?? + Boolean( + ptyRouteKind(session, { + sandboxAvailable: options.sandboxAvailable, + }), + )))) && ptyReadyStatuses.has(session.status), vnc: !inactiveStatuses.has(session.status) && diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 454f722d..46341ee3 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -111,7 +111,10 @@ export function forwardGitHubActionsRelayMessage( viewers: readonly GitHubActionsRelaySocket[], ): number { if (sender === "viewer" && isGitHubActionsViewerControlMessage(message)) return 0; - const targets = sender === "runner" ? viewers : runners.slice(0, 1); + const targets = + sender === "runner" + ? viewers + : runners.filter((socket) => socket.readyState === webSocketOpen).slice(0, 1); let forwarded = 0; for (const socket of targets) { if (socket.readyState !== webSocketOpen) continue; diff --git a/tests/fleet-state.test.ts b/tests/fleet-state.test.ts index b90fe3a1..a022518d 100644 --- a/tests/fleet-state.test.ts +++ b/tests/fleet-state.test.ts @@ -204,6 +204,7 @@ test("GitHub Actions sessions are attachable through the Worker relay", () => { runtime: "github_actions", leaseId: "github-actions:s1", attachUrl: null, + ptyAvailable: true, workKey: "openclaw/crabfleet:pr:42", workKind: "pr_repair", workState: "running", @@ -228,6 +229,30 @@ test("GitHub Actions sessions are attachable through the Worker relay", () => { assert.equal(fleet.sessions[0]?.workPhase, "fixing"); }); +test("GitHub Actions sessions require an available Worker relay", () => { + const fleet = buildFleetState( + [ + { + ...baseSession, + runtime: "github_actions", + leaseId: "github-actions:s1", + attachUrl: null, + ptyAvailable: false, + }, + ], + [], + { + canonicalUrl: "https://crabfleet.openclaw.ai", + defaultEgressHosts: [], + generatedAt: 100, + productUrl: "https://clawfleet.ai", + }, + ); + + assert.equal(fleet.totals.attachable, 0); + assert.equal(fleet.sessions[0]?.attachable, false); +}); + test("sandbox lease parser ignores non-sandbox leases", () => { assert.equal( sandboxIdFromLeaseId("sandbox:crabbox-s1-abcd1234:terminal-s1-abcd1234:autostart-v4"), diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 606ebe07..fce0dc5b 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -90,6 +90,25 @@ test("relay replaces the current runner and routes messages by role", () => { assert.deepEqual(runner.sent, ["input"]); }); +test("relay sends viewer input to the first open runner", () => { + const closedRunner = relaySocket(3); + const openRunner = relaySocket(); + const laterRunner = relaySocket(); + + assert.equal( + forwardGitHubActionsRelayMessage( + "viewer", + "input", + [closedRunner, openRunner, laterRunner], + [], + ), + 1, + ); + assert.deepEqual(closedRunner.sent, []); + assert.deepEqual(openRunner.sent, ["input"]); + assert.deepEqual(laterRunner.sent, []); +}); + test("relay consumes viewer resize controls without corrupting raw runner input", () => { const runner = relaySocket(); const resize = JSON.stringify({ type: "resize", cols: 120, rows: 40 }); From e14d408c3533a906dd96c73913a5eb88b052291c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:00:05 +0200 Subject: [PATCH 007/242] fix(app): reconcile browser history state --- src/app/app-navigation.js | 26 +++++++++++++++++++++++-- src/app/routing.js | 10 +++++++++- tests/app-navigation.test.ts | 37 +++++++++++++++++++++++++++++++++++- tests/app-routing.test.ts | 5 +++++ 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/app/app-navigation.js b/src/app/app-navigation.js index b64f73de..64dd3799 100644 --- a/src/app/app-navigation.js +++ b/src/app/app-navigation.js @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from "preact/hooks"; -import { appViewUrl, initialAppView, sessionRouteUrl } from "./routing.js"; +import { appViewUrl, initialAppView, parseSessionLink, sessionRouteUrl } from "./routing.js"; import { loadSessionLayout, saveSessionLayout } from "./session-layout.js"; import { disposeAllTerminals, warmGhosttyModule } from "./terminal.js"; @@ -27,6 +27,17 @@ export function sessionOpenTarget(id, currentId, sessionItemById, options = {}) }; } +export function appNavigationLocationState(locationLike = location) { + const sessionLink = parseSessionLink(locationLike); + return { + appView: initialAppView(locationLike), + drawers: sessionLink.route ? { sessions: true } : {}, + focusedSessionId: sessionLink.id, + sharedSessionId: sessionLink.id, + sharedToken: sessionLink.token, + }; +} + export function useAppNavigation({ initialSessionLink, sessionItemByIdRef }) { const [appView, setAppViewState] = useState(initialAppView); const [drawers, setDrawers] = useState(initialSessionLink.route ? { sessions: true } : {}); @@ -44,7 +55,18 @@ export function useAppNavigation({ initialSessionLink, sessionItemByIdRef }) { focusedSessionIdRef.current = focusedSessionId; useEffect(() => { - const onPopState = () => setAppViewState(initialAppView()); + const onPopState = () => { + const next = appNavigationLocationState(); + setAppViewState(next.appView); + setDrawers(next.drawers); + setActiveRunId(null); + setFocusedSessionId(next.focusedSessionId); + focusedSessionIdRef.current = next.focusedSessionId; + setSharedSessionId(next.sharedSessionId); + setSharedToken(next.sharedToken); + if (next.focusedSessionId) warmGhosttyModule(); + else disposeAllTerminals(); + }; window.addEventListener("popstate", onPopState); return () => window.removeEventListener("popstate", onPopState); }, []); diff --git a/src/app/routing.js b/src/app/routing.js index a501b22c..bf47edfe 100644 --- a/src/app/routing.js +++ b/src/app/routing.js @@ -2,9 +2,17 @@ export const loginReturnKey = "crabbox-login-return"; export function parseSessionLink(locationLike = location) { const match = locationLike.pathname.match(/^\/(?:app\/)?sessions(?:\/([^/]+))?\/?$/); + let id = null; + if (match?.[1]) { + try { + id = decodeURIComponent(match[1]); + } catch { + return { route: false, id: null, token: null }; + } + } return { route: Boolean(match), - id: match?.[1] ? decodeURIComponent(match[1]) : null, + id, token: new URLSearchParams(locationLike.search).get("token"), }; } diff --git a/tests/app-navigation.test.ts b/tests/app-navigation.test.ts index a316ab38..5ddb92d5 100644 --- a/tests/app-navigation.test.ts +++ b/tests/app-navigation.test.ts @@ -1,7 +1,12 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { normalizedAppView, sessionOpenTarget, topOpenDrawer } from "../src/app/app-navigation.js"; +import { + appNavigationLocationState, + normalizedAppView, + sessionOpenTarget, + topOpenDrawer, +} from "../src/app/app-navigation.js"; test("navigation normalizes app views and closes the topmost drawer", () => { assert.equal(normalizedAppView("board"), "board"); @@ -48,3 +53,33 @@ test("session navigation derives focus and durable route targets", () => { grid: true, }); }); + +test("browser history locations reconcile view, drawers, and session focus", () => { + assert.deepEqual( + appNavigationLocationState({ + pathname: "/sessions/IS-2", + search: "?token=shared", + }), + { + appView: "fleet", + drawers: { sessions: true }, + focusedSessionId: "IS-2", + sharedSessionId: "IS-2", + sharedToken: "shared", + }, + ); + assert.deepEqual(appNavigationLocationState({ pathname: "/sessions", search: "" }), { + appView: "fleet", + drawers: { sessions: true }, + focusedSessionId: null, + sharedSessionId: null, + sharedToken: null, + }); + assert.deepEqual(appNavigationLocationState({ pathname: "/app/board", search: "" }), { + appView: "board", + drawers: {}, + focusedSessionId: null, + sharedSessionId: null, + sharedToken: null, + }); +}); diff --git a/tests/app-routing.test.ts b/tests/app-routing.test.ts index 524e7dfd..a0b110db 100644 --- a/tests/app-routing.test.ts +++ b/tests/app-routing.test.ts @@ -26,6 +26,11 @@ test("app routing parses board and shared session locations", () => { id: null, token: null, }); + assert.deepEqual(parseSessionLink({ pathname: "/sessions/%", search: "?token=ignored" }), { + route: false, + id: null, + token: null, + }); }); test("app and session route builders preserve only owned URL state", () => { From 4e02e97abf626078203296de799cd200606434d6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:01:05 +0200 Subject: [PATCH 008/242] fix(vnc): harden authentication and input keysyms --- .../RoyalVNCKit/Encryption/BigNum.swift | 4 ++ .../ARDDiffieHellmanKeyAgreement.swift | 1 + .../UltraVNCMSLogonII/UltraVNCBigNum.swift | 56 ++++++++------- .../SDK/Input/VNCKeyCode+ObjC.swift | 2 +- .../RoyalVNCKit/SDK/Input/VNCKeyCode.swift | 8 ++- .../SecurityAndInputTests.swift | 68 +++++++++++++++++++ 6 files changed, 110 insertions(+), 29 deletions(-) create mode 100644 macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift index 56ced73e..a0dfc797 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift @@ -20,6 +20,10 @@ final class BigNum { } extension BigNum { + var isGreaterThanOne: Bool { + self.bigInt > 1 + } + var isZero: Bool { let isIt = self.bigInt == 0 diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift index 0a23e98b..8ad964e3 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift @@ -59,6 +59,7 @@ private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { let bigPubKey = BigNum() guard let bigPrime = BigNum(data: prime), + bigPrime.isGreaterThanOne, let bigGenerator = BigNum(data: generator) else { return nil } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCBigNum.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCBigNum.swift index ac31c275..89c024d7 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCBigNum.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCBigNum.swift @@ -40,57 +40,61 @@ extension VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAgreement static func addM64(x: UInt64, y: UInt64, m: UInt64) -> UInt64 { - let part = Int64(x + y < x - ? (-1 % .init(m) + 1) % .init(m) - : 0) + guard m != 0 else { return 0 } - let partU: UInt64 = numericCast(part) + let reducedX = x % m + let reducedY = y % m + let distanceToModulus = m - reducedY - let result: UInt64 = (x + y) % m + partU + if reducedX >= distanceToModulus { + return reducedX - distanceToModulus + } - return result + return reducedX + reducedY } /// (x * y) % m static func mulM64(x: UInt64, y: UInt64, m: UInt64) -> UInt64 { - var y = y - var r = UInt64(0) - var x = UInt64(0) + guard m != 0 else { return 0 } - repeat { - x>>=1 + var multiplicand = x % m + var multiplier = y % m + var result = UInt64(0) - if x & 1 != 0 { - r = addM64(x: r, y: y, m: m) + while multiplier > 0 { + if multiplier & 1 != 0 { + result = addM64(x: result, y: multiplicand, m: m) } - y = addM64(x: y, y: y, m: m) - } while x > 0 + multiplier >>= 1 + multiplicand = addM64(x: multiplicand, y: multiplicand, m: m) + } - return r + return result } /// (x ^ y) % m static func powM64(b: UInt64, e: UInt64, m: UInt64) -> UInt64 { - var b = b - var r = UInt64(0) - var e = UInt64(0) + guard m != 0 else { return 0 } - repeat { - e>>=1 + var base = b % m + var exponent = e + var result = UInt64(1) % m - if e & 1 != 0 { - r = mulM64(x: r, y: b, m: m) + while exponent > 0 { + if exponent & 1 != 0 { + result = mulM64(x: result, y: base, m: m) } - b = mulM64(x: b, y: b, m: m) - } while e > 0 + exponent >>= 1 + base = mulM64(x: base, y: base, m: m) + } - return r + return result } } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode+ObjC.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode+ObjC.swift index 630d69a8..4c86e3fb 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode+ObjC.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode+ObjC.swift @@ -86,7 +86,7 @@ public final class _ObjC_VNCKeyCode: NSObject { @objc public static let ansiKeypadEnter = X11KeySymbols.XK_KP_Enter @objc - public static let ansiKeypadDecimal = X11KeySymbols.XK_KP_Separator + public static let ansiKeypadDecimal = X11KeySymbols.XK_KP_Decimal @objc public static let f1 = X11KeySymbols.XK_F1 diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift index 7cffc973..357b3adb 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift @@ -48,7 +48,7 @@ public struct VNCKeyCode: Equatable { public static let ansiKeypadMinus = VNCKeyCode(X11KeySymbols.XK_KP_Subtract) public static let ansiKeypadPlus = VNCKeyCode(X11KeySymbols.XK_KP_Add) public static let ansiKeypadEnter = VNCKeyCode(X11KeySymbols.XK_KP_Enter) - public static let ansiKeypadDecimal = VNCKeyCode(X11KeySymbols.XK_KP_Separator) + public static let ansiKeypadDecimal = VNCKeyCode(X11KeySymbols.XK_KP_Decimal) public static let f1 = VNCKeyCode(X11KeySymbols.XK_F1) public static let f2 = VNCKeyCode(X11KeySymbols.XK_F2) @@ -161,8 +161,12 @@ public extension VNCKeyCode { for scalar in character.unicodeScalars { let unicodeValue = scalar.value + let keySym = + (0x00a0...0x00ff).contains(unicodeValue) + ? unicodeValue + : 0x0100_0000 | unicodeValue - codes.append(.init(unicodeValue)) + codes.append(.init(keySym)) } return codes diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift new file mode 100644 index 00000000..3be4f6cf --- /dev/null +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -0,0 +1,68 @@ +import Foundation +import Testing + +@testable import RoyalVNCKit + +struct SecurityAndInputTests { + typealias ARDKeyAgreement = VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement + typealias UltraVNCBigNum = + VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAgreement.UltraVNCBigNum + + @Test + func rejectsZeroAndOneAppleRemoteDesktopModuli() { + for prime in [Data([0]), Data([1]), Data([0, 1])] { + let agreement = ARDKeyAgreement( + prime: prime, + generator: Data([2]), + peerKey: Data([2]), + keyLength: prime.count + ) + + #expect(agreement.map { _ in true } == nil) + } + } + + @Test + func computesUltraVNCModularArithmeticKnownAnswers() { + #expect( + UltraVNCBigNum.addM64( + x: 0xffff_ffff_ffff_fffe, + y: 0xffff_ffff_ffff_fffd, + m: 0x61 + ) == 0x14 + ) + #expect( + UltraVNCBigNum.mulM64( + x: 0xffff_ffff_ffff_ffc5, + y: 0xffff_ffff_ffff_ffa3, + m: 0xffff_ffff_ffff_ff61 + ) == 0x19c8 + ) + #expect(UltraVNCBigNum.powM64(b: 4, e: 13, m: 497) == 445) + #expect( + UltraVNCBigNum.powM64( + b: 0xffff_ffff_ffff_ffc5, + e: 0x1_2345, + m: 0xffff_ffff_ffff_ff61 + ) == 0x34be_28a2_05bf_50b9 + ) + } + + @Test + func encodesCharactersAsX11KeySyms() { + #expect(VNCKeyCode.withCharacter("A").map(\.rawValue) == [0x41]) + #expect(VNCKeyCode.withCharacter("é").map(\.rawValue) == [0xe9]) + #expect(VNCKeyCode.withCharacter("α").map(\.rawValue) == [0x0100_03b1]) + #expect(VNCKeyCode.withCharacter("🦀").map(\.rawValue) == [0x0101_f980]) + } + + @Test + func mapsKeypadDecimalToTheDecimalKeysym() { + #expect(VNCKeyCode.ansiKeypadDecimal.rawValue == X11KeySymbols.XK_KP_Decimal) + #expect(VNCKeyCode.ansiKeypadDecimal.rawValue != X11KeySymbols.XK_KP_Separator) + + #if canImport(ObjectiveC) + #expect(_ObjC_VNCKeyCode.ansiKeypadDecimal == X11KeySymbols.XK_KP_Decimal) + #endif + } +} From 0be23dd65e7001c77d352ace89db6359434b5561 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:04:41 +0200 Subject: [PATCH 009/242] fix(cards): persist run claims atomically --- src/worker/card-repository.ts | 67 +++++++++++++++++++++-------------- tests/card-repository.test.ts | 51 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 27 deletions(-) diff --git a/src/worker/card-repository.ts b/src/worker/card-repository.ts index bf06472c..0e0232da 100644 --- a/src/worker/card-repository.ts +++ b/src/worker/card-repository.ts @@ -337,7 +337,7 @@ export class CardRepository implements CardLifecycleStore { async claimRun(input: CardRunClaimInput): Promise<"claimed" | "capacity" | "active"> { const db = database(this.env); - const transition = await sql` + const transition = sql` UPDATE cards SET lane = 'Running', active_run_id = ${input.runId}, @@ -351,8 +351,42 @@ export class CardRepository implements CardLifecycleStore { AND (lane = 'Running' OR ( SELECT count(*) FROM cards WHERE lane = 'Running' AND id <> ${input.card.id} ) < ${input.cap}) - `.execute(db); - if ((transition.numAffectedRows ?? 0n) === 0n) { + AND NOT EXISTS ( + SELECT 1 + FROM run_attempts + WHERE id = ${input.runId} + OR (card_id = ${input.card.id} AND attempt = ${input.attempt}) + ) + `; + const insert = sql` + INSERT INTO run_attempts ( + id, card_id, attempt, runtime, status, control_intent, lease_id, attach_url, vnc_url, + selection_reason, capabilities_json, operator, last_heartbeat_at, started_at, ended_at, + created_at, updated_at, error + ) + SELECT + ${input.runId}, ${input.card.id}, ${input.attempt}, ${input.descriptor.runtime}, 'queued', + NULL, NULL, NULL, NULL, ${input.descriptor.reason}, + ${JSON.stringify(input.descriptor.capabilities)}, NULL, ${input.now}, ${input.now}, NULL, + ${input.now}, ${input.now}, NULL + FROM cards + WHERE id = ${input.card.id} + AND active_run_id = ${input.runId} + AND updated_at = ${input.now} + AND NOT EXISTS ( + SELECT 1 + FROM run_attempts + WHERE id = ${input.runId} + OR (card_id = ${input.card.id} AND attempt = ${input.attempt}) + ) + `; + const results = await this.env.DB.batch( + [transition, insert].map((query) => { + const compiled = query.compile(db); + return this.env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), + ); + if ((results[0]?.meta.changes ?? 0) === 0) { const activeCount = await db .selectFrom("cards") .select(sql`count(*)`.as("count")) @@ -360,30 +394,9 @@ export class CardRepository implements CardLifecycleStore { .executeTakeFirst(); return Number(activeCount?.count ?? 0) >= input.cap ? "capacity" : "active"; } - await db - .insertInto("run_attempts") - .values({ - id: input.runId, - card_id: input.card.id, - attempt: input.attempt, - runtime: input.descriptor.runtime, - status: "queued", - control_intent: null, - lease_id: null, - attach_url: null, - vnc_url: null, - selection_reason: input.descriptor.reason, - capabilities_json: JSON.stringify(input.descriptor.capabilities), - operator: null, - last_heartbeat_at: input.now, - started_at: input.now, - ended_at: null, - created_at: input.now, - updated_at: input.now, - error: null, - }) - .onConflict((conflict) => conflict.doNothing()) - .execute(); + if ((results[1]?.meta.changes ?? 0) !== 1) { + throw new Error("card run claim did not persist its run attempt"); + } return "claimed"; } diff --git a/tests/card-repository.test.ts b/tests/card-repository.test.ts index eea0b7d8..d6050c89 100644 --- a/tests/card-repository.test.ts +++ b/tests/card-repository.test.ts @@ -2,8 +2,10 @@ import assert from "node:assert/strict"; import test from "node:test"; import { CardRepository } from "../src/worker/card-repository.ts"; +import type { CardRunClaimInput } from "../src/worker/card-lifecycle-service.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import type { User } from "../src/worker/models.ts"; +import { containerCapabilities } from "../src/worker/session-model.ts"; test("private card reads require the stable owner subject", async () => { const current: User = { @@ -110,3 +112,52 @@ test("card list batches related D1 reads below the bind-parameter limit", async assert.equal(executions.filter(({ sql }) => /from "run_attempts"/i.test(sql)).length, 3); assert.equal(executions.filter(({ sql }) => /from events/i.test(sql)).length, 3); }); + +test("card run claims batch the card transition with the run-attempt insert", async () => { + const batches: Array> = []; + const env = { + DB: { + prepare(sql: string) { + return { + bind(...parameters: unknown[]) { + return { + sql, + parameters, + async all() { + return { results: [], meta: { changes: 0 } }; + }, + async run() { + return { meta: { changes: 0 } }; + }, + }; + }, + }; + }, + async batch(statements: Array<{ sql: string; parameters: unknown[] }>) { + batches.push(statements); + return [{ meta: { changes: 1 } }, { meta: { changes: 1 } }]; + }, + } as unknown as D1Database, + } as RuntimeEnv; + const input = { + card: { id: "CY-101" }, + runId: "CY-101-R1", + attempt: 1, + cap: 2, + descriptor: { + runtime: "container", + reason: "repo default", + capabilities: containerCapabilities, + }, + now: 500, + } as CardRunClaimInput; + + assert.equal(await new CardRepository(env).claimRun(input), "claimed"); + assert.equal(batches.length, 1); + assert.equal(batches[0]?.length, 2); + assert.match(batches[0]?.[0]?.sql ?? "", /^\s*update cards/i); + assert.match(batches[0]?.[0]?.sql ?? "", /not exists/i); + assert.match(batches[0]?.[1]?.sql ?? "", /^\s*insert into run_attempts/i); + assert.ok(batches[0]?.[0]?.parameters.includes("CY-101-R1")); + assert.ok(batches[0]?.[1]?.parameters.includes("CY-101-R1")); +}); From ed40ea52a25a5e743229f79683289f7b6f56c3c5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:04:44 +0200 Subject: [PATCH 010/242] fix(actions): fence concurrent session updates --- src/worker/github-actions-repository.ts | 41 +++++++++++++++++++++++-- tests/github-actions-repository.test.ts | 24 +++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/worker/github-actions-repository.ts b/src/worker/github-actions-repository.ts index 931c69d7..1bf72ad2 100644 --- a/src/worker/github-actions-repository.ts +++ b/src/worker/github-actions-repository.ts @@ -6,12 +6,16 @@ import type { RuntimeEnv } from "./env.ts"; import type { GitHubActionsRunnerConnectionUpdate } from "./github-actions-runner-connection.ts"; import type { GitHubActionsSessionRegistrationUpdate } from "./github-actions-session-registration.ts"; import type { GitHubActionsWorkStateUpdate } from "./github-actions-session-work-state.ts"; +import { conflict } from "./http.ts"; type GitHubActionsSessionUpdate = | GitHubActionsSessionRegistrationUpdate | GitHubActionsWorkStateUpdate | GitHubActionsRunnerConnectionUpdate; +const terminalWorkStates = ["completed", "failed", "canceled"]; +const terminalSessionStatuses = ["stopped", "expired", "failed"] as const; + export class GitHubActionsRepository { private readonly env: RuntimeEnv; @@ -44,10 +48,43 @@ export class GitHubActionsRepository { } async updateSession(id: string, values: GitHubActionsSessionUpdate): Promise { - await database(this.env) + let update = database(this.env) .updateTable("interactive_sessions") .set(values) .where("id", "=", id) - .execute(); + .where("runtime", "=", "github_actions") + .where("updated_at", "<=", values.updated_at); + + if (isRegistrationUpdate(values)) { + update = update.where("owner_subject", "=", values.owner_subject); + } else if (isWorkStateUpdate(values) && terminalWorkStates.includes(values.work_state)) { + update = update.where((expressions) => + expressions.or([ + expressions("work_state", "not in", terminalWorkStates), + expressions("work_state", "=", values.work_state), + ]), + ); + } else { + update = update + .where("work_state", "not in", terminalWorkStates) + .where("status", "not in", terminalSessionStatuses); + } + + const result = await update.executeTakeFirst(); + if ((result.numUpdatedRows ?? 0n) !== 1n) { + throw conflict("GitHub Actions session changed; retry"); + } } } + +function isRegistrationUpdate( + values: GitHubActionsSessionUpdate, +): values is GitHubActionsSessionRegistrationUpdate { + return "agent_token_hash" in values; +} + +function isWorkStateUpdate( + values: GitHubActionsSessionUpdate, +): values is GitHubActionsWorkStateUpdate { + return "stopped_at" in values && "codex_thread_id" in values; +} diff --git a/tests/github-actions-repository.test.ts b/tests/github-actions-repository.test.ts index 2f491f86..35dd5209 100644 --- a/tests/github-actions-repository.test.ts +++ b/tests/github-actions-repository.test.ts @@ -17,7 +17,7 @@ type Execution = { kind: "all" | "run"; }; -function runtimeEnv(executions: Execution[]): RuntimeEnv { +function runtimeEnv(executions: Execution[], mutationChanges = 1): RuntimeEnv { const row = sessionRow({ id: "IS-101", runtime: "github_actions", @@ -35,7 +35,7 @@ function runtimeEnv(executions: Execution[]): RuntimeEnv { }, async run() { executions.push({ sql, parameters, kind: "run" }); - return { meta: { changes: 1 } }; + return { meta: { changes: mutationChanges } }; }, }; }, @@ -86,8 +86,28 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = for (const execution of executions.slice(3)) { assert.match(execution.sql, /update "interactive_sessions"/i); assert.match(execution.sql, /where "id" = \?/i); + assert.match(execution.sql, /"runtime" = \?/i); + assert.match(execution.sql, /"updated_at" <= \?/i); assert.ok(execution.parameters.includes("IS-101")); } + assert.match(executions[3].sql, /"owner_subject" = \?/i); + assert.doesNotMatch(executions[3].sql, /"work_state" not in/i); + assert.match(executions[4].sql, /"work_state" not in/i); + assert.match(executions[5].sql, /"status" not in/i); +}); + +test("GitHub Actions repository rejects stale or invalid state transitions", async () => { + const executions: Execution[] = []; + const repository = new GitHubActionsRepository(runtimeEnv(executions, 0)); + + await assert.rejects(repository.updateSession("IS-101", runnerConnectionUpdate), (error) => { + assert.equal( + typeof error === "object" && error && "status" in error ? error.status : undefined, + 409, + ); + return true; + }); + assert.equal(executions.length, 1); }); const registrationUpdate: GitHubActionsSessionRegistrationUpdate = { From ae7afc8137480e1054e82950455e0d568784016b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:04:47 +0200 Subject: [PATCH 011/242] fix(events): reject lossy JSON values --- src/worker/http.ts | 28 +++++++++++++++++-- src/worker/session-events.ts | 23 ++++++++++++++- tests/http.test.ts | 22 +++++++++++++++ tests/session-events.test.ts | 54 ++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 3 deletions(-) diff --git a/src/worker/http.ts b/src/worker/http.ts index 724097b4..7de0a82b 100644 --- a/src/worker/http.ts +++ b/src/worker/http.ts @@ -54,11 +54,14 @@ export function wantsMarkdown(request: Request): boolean { } export async function readJson(request: Request): Promise { + let parsed: unknown; try { - return (await request.json()) as T; + parsed = JSON.parse(await request.text()) as unknown; } catch { throw badRequest("invalid json"); } + assertRoundTrippableJsonIntegers(parsed); + return parsed as T; } export async function readBoundedJson(request: Request, maximumBytes: number): Promise { @@ -97,11 +100,14 @@ export async function readBoundedJson(request: Request, maximumBytes: number) bytes.set(chunk, offset); offset += chunk.byteLength; } + let parsed: unknown; try { - return JSON.parse(new TextDecoder().decode(bytes)) as T; + parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; } catch { throw badRequest("invalid json"); } + assertRoundTrippableJsonIntegers(parsed); + return parsed as T; } export function bearerToken(request: Request): string { @@ -170,3 +176,21 @@ function clean(value: unknown, maximum: number): string { .trim() .slice(0, maximum); } + +function assertRoundTrippableJsonIntegers(value: unknown): void { + if (typeof value === "number") { + if ( + !Number.isFinite(value) || + (Number.isInteger(value) && (!Number.isSafeInteger(value) || Object.is(value, -0))) + ) { + throw badRequest("json integers must be safe and round-trippable"); + } + return; + } + if (!value || typeof value !== "object") return; + if (Array.isArray(value)) { + for (const item of value) assertRoundTrippableJsonIntegers(item); + return; + } + for (const item of Object.values(value)) assertRoundTrippableJsonIntegers(item); +} diff --git a/src/worker/session-events.ts b/src/worker/session-events.ts index 4bc81a48..e07c1f1e 100644 --- a/src/worker/session-events.ts +++ b/src/worker/session-events.ts @@ -307,7 +307,7 @@ function structuredPayloadJson(value: unknown): string { if ( !Object.hasOwn(record, "version") || typeof version !== "number" || - !Number.isInteger(version) || + !Number.isSafeInteger(version) || version < 1 ) { throw badRequest("payload.version must be a positive integer"); @@ -432,6 +432,9 @@ function canonicalJsonValue( } if (typeof value === "number") { if (!Number.isFinite(value)) throw badRequest("payload must contain valid JSON values"); + if (Number.isInteger(value) && (!Number.isSafeInteger(value) || Object.is(value, -0))) { + throw badRequest("payload integers must be safe and round-trippable"); + } return value; } if (typeof value !== "object") { @@ -466,6 +469,9 @@ function consumePayloadMembers(budget: { members: number }, count: number): void } function assertPayloadStringSize(value: string): void { + if (hasLoneSurrogate(value)) { + throw badRequest("payload strings must contain valid Unicode"); + } if (encoder.encode(value).byteLength > structuredEventPayloadMaxStringBytes) { throw badRequest( `payload strings must be at most ${structuredEventPayloadMaxStringBytes} UTF-8 bytes`, @@ -477,12 +483,27 @@ function requiredString(value: unknown, name: string, maximum: number): string { if (typeof value !== "string") throw badRequest(`${name} must be a string`); const normalized = value.trim(); if (!normalized) throw badRequest(`${name} is required`); + if (hasLoneSurrogate(normalized)) throw badRequest(`${name} must contain valid Unicode`); if (normalized.length > maximum) { throw badRequest(`${name} must be at most ${maximum} characters`); } return normalized; } +function hasLoneSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +} + function clean(value: unknown, maximum: number): string { return String(value ?? "") .trim() diff --git a/tests/http.test.ts b/tests/http.test.ts index 8540ecc0..f550595e 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -113,6 +113,28 @@ test("bounded JSON parsing rejects declared and streamed bodies before unbounded } }); +test("JSON parsing rejects integers that cannot round-trip exactly", async () => { + for (const body of ['{"value":9007199254740993}', '{"value":-0}', '{"nested":[1e400]}']) { + for (const parse of [ + () => readJson(new Request("https://fleet.example", { method: "POST", body })), + () => + readBoundedJson( + new Request("https://fleet.example", { method: "POST", body }), + body.length + 1, + ), + ]) { + await assert.rejects(parse(), (error: unknown) => { + assert.equal( + typeof error === "object" && error && "status" in error ? error.status : undefined, + 400, + ); + assert.match(error instanceof Error ? error.message : "", /round-trippable/); + return true; + }); + } + } +}); + test("bearer and cookie helpers normalize only their owned protocol surface", () => { assert.equal( bearerToken( diff --git a/tests/session-events.test.ts b/tests/session-events.test.ts index 777864f5..48edce28 100644 --- a/tests/session-events.test.ts +++ b/tests/session-events.test.ts @@ -382,6 +382,26 @@ test("structured session events require bounded identifiers and a versioned obje { eventKey: "key", type: "action", message: "message", payload: [] }, { eventKey: "key", type: "action", message: "message", payload: {} }, { eventKey: "key", type: "action", message: "message", payload: { version: 0 } }, + { eventKey: "\ud800", type: "action", message: "message", payload: { version: 1 } }, + { eventKey: "key", type: "action", message: "\udfff", payload: { version: 1 } }, + { + eventKey: "key", + type: "action", + message: "message", + payload: { version: 1, value: "\ud800" }, + }, + { + eventKey: "key", + type: "action", + message: "message", + payload: { version: 1, value: Number.MAX_SAFE_INTEGER + 1 }, + }, + { + eventKey: "key", + type: "action", + message: "message", + payload: { version: 1, value: -0 }, + }, ]; for (const input of cases) { await assert.rejects( @@ -403,6 +423,40 @@ test("structured session events require bounded identifiers and a versioned obje assert.equal(persisted, false); }); +test("structured session events retain valid supplementary Unicode", async () => { + let payloadJson = ""; + const service = new InteractiveSessionEventLedgerService({ + async persistAndInvalidate(event) { + payloadJson = event.payloadJson; + return { + inserted: true, + row: { + id: 1, + session_id: event.sessionId, + actor: event.actor, + event_key: event.eventKey, + event_type: event.type, + message: event.message, + payload_json: event.payloadJson, + created_at: event.now, + }, + }; + }, + async archive() {}, + }); + + await service.append({ + sessionId: "IS-1", + actor: "operator", + eventKey: "run:\u{1f980}", + type: "action", + message: "valid \u{1f980}", + payload: { version: 1, value: "\u{1f980}" }, + now: 123, + }); + assert.equal(payloadJson, '{"value":"\u{1f980}","version":1}'); +}); + test("structured session event payload budgets fail with controlled client errors", async () => { let persisted = false; const service = new InteractiveSessionEventLedgerService({ From 3d21dc13a0ad50ef91540c28d9ad2ab970b905bc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:04:51 +0200 Subject: [PATCH 012/242] fix(routes): reject malformed session ids --- src/worker/routes/service-sessions.ts | 8 ++++++-- tests/service-session-routes.test.ts | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/worker/routes/service-sessions.ts b/src/worker/routes/service-sessions.ts index 9271fdd6..db69f995 100644 --- a/src/worker/routes/service-sessions.ts +++ b/src/worker/routes/service-sessions.ts @@ -1,4 +1,4 @@ -import { json } from "../http.ts"; +import { badRequest, json } from "../http.ts"; import type { User } from "../models.ts"; import type { InteractiveSession } from "../session-model.ts"; import type { InteractiveSessionSummaryInput } from "../session-metadata.ts"; @@ -92,5 +92,9 @@ export async function handleServiceSessionRoute( } function decoded(value: string | undefined): string { - return decodeURIComponent(value ?? ""); + try { + return decodeURIComponent(value ?? ""); + } catch { + throw badRequest("invalid session id"); + } } diff --git a/tests/service-session-routes.test.ts b/tests/service-session-routes.test.ts index a86bbe9e..d28efb8a 100644 --- a/tests/service-session-routes.test.ts +++ b/tests/service-session-routes.test.ts @@ -304,3 +304,17 @@ test("service-session routes report missing reads and fall through on inexact re } assert.deepEqual(calls, []); }); + +test("service-session routes reject malformed encoded session ids with a client error", async () => { + const calls: string[] = []; + const value = request("POST", "/api/agent/interactive-sessions/%/events", {}); + + await assert.rejects(dispatch(value, calls), (error) => { + assert.equal( + typeof error === "object" && error && "status" in error ? error.status : undefined, + 400, + ); + return true; + }); + assert.deepEqual(calls, []); +}); From c3c7cf2862122cb5408f507de10a21742e29ee37 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:04:58 +0200 Subject: [PATCH 013/242] fix(terminal): retain shared input state --- src/worker/interactive-terminal-service.ts | 73 +++++++++++++++++----- tests/interactive-terminal-service.test.ts | 14 +++++ 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/worker/interactive-terminal-service.ts b/src/worker/interactive-terminal-service.ts index 266a2a58..57b231a2 100644 --- a/src/worker/interactive-terminal-service.ts +++ b/src/worker/interactive-terminal-service.ts @@ -1,4 +1,3 @@ -import { getSandbox } from "@cloudflare/sandbox"; import { terminalOutputAcknowledgements } from "@openclaw/libterminal/worker"; import { @@ -19,7 +18,6 @@ import { isOpenClawEmbedSessionToken, terminalInputAuthorization, } from "./openclaw-embed-access.ts"; -import { SandboxLifecycleService } from "./provisioning/sandbox-lifecycle.ts"; import { isSandboxLeaseOwnerReconnectError } from "./provisioning/sandbox.ts"; import { readTerminalClipboardBytes, @@ -28,7 +26,6 @@ import { terminalClipboardMaxBytes, } from "./interactive-terminal.ts"; import { InteractiveTerminalRepository } from "./interactive-terminal-repository.ts"; -import { reconcileSandboxCredentialPolicyCleanupBatch } from "./sandbox-credential-policy-cleanup-service.ts"; import { stageTerminalCredentialPolicyCleanupById } from "./sandbox-credential-policy-cleanup.ts"; import { isSandboxInteractiveSession, sandboxLeaseInfo } from "./sandbox-lease.ts"; import { openSandboxTerminalResponse, sandboxWorkdir } from "./sandbox-runtime.ts"; @@ -56,7 +53,38 @@ import { import { interactiveTerminalFetch } from "./runtime-adapter-transport.ts"; import { tenancyMode, tenantSubject } from "./tenancy.ts"; -const terminalInputStates = new Map(); +export class TerminalInputStateRegistry { + private readonly entries = new Map(); + + retain(sessionId: string): void { + const entry = this.entry(sessionId); + entry.subscribers += 1; + } + + release(sessionId: string): void { + const entry = this.entries.get(sessionId); + if (!entry || entry.subscribers <= 1) { + this.entries.delete(sessionId); + return; + } + entry.subscribers -= 1; + } + + state(sessionId: string): TerminalInputState { + return this.entry(sessionId).state; + } + + private entry(sessionId: string): { state: TerminalInputState; subscribers: number } { + let entry = this.entries.get(sessionId); + if (!entry) { + entry = { state: newTerminalInputState(), subscribers: 0 }; + this.entries.set(sessionId, entry); + } + return entry; + } +} + +const terminalInputStates = new TerminalInputStateRegistry(); export type InteractiveTerminalServiceDependencies = { readSession(sessionId: string): Promise; @@ -125,12 +153,14 @@ export class InteractiveTerminalService { const routeKind = interactivePtyRouteKind(this.env, session); if (routeKind === "sandbox" && this.env.SANDBOX) { const runtimeSession = await this.dependencies.resolveSandboxSession(request, user, session); + const { SandboxLifecycleService } = await import("./provisioning/sandbox-lifecycle.ts"); const sandboxSession = await new SandboxLifecycleService(this.env).ensureCurrentLease( request, user, runtimeSession, ); const lease = sandboxLeaseInfo(sandboxSession); + const { getSandbox } = await import("@cloudflare/sandbox"); const sandbox = getSandbox(this.env.SANDBOX, lease.sandboxId); const upstreamResponse = await openSandboxTerminalResponse( request, @@ -212,6 +242,7 @@ export class InteractiveTerminalService { } private terminalHub(): TerminalHub { + const retainedInputSessions = new Set(); return new TerminalHub({ createSocketPair: () => { const pair = new WebSocketPair(); @@ -241,11 +272,25 @@ export class InteractiveTerminalService { terminalViewGrant(request, this.env, this.repository, user, session), reconcileSubscription: (sessionId) => terminalSubscriptionReconciler(this.dependencies, sessionId), - openUpstream: (request, user, session, cols, rows) => - this.openUpstream(request, user, session, cols, rows), + openUpstream: async (request, user, session, cols, rows) => { + const upstream = await this.openUpstream(request, user, session, cols, rows); + return { + ...upstream, + markConnected: async () => { + if (!retainedInputSessions.has(session.id)) { + retainedInputSessions.add(session.id); + terminalInputStates.retain(session.id); + } + await upstream.markConnected(); + }, + }; + }, inputPayloads: (subscription, user, payload) => multiplayerTerminalInputPayloads(this.repository, subscription, user, payload), - releaseInputState: releaseTerminalInputState, + releaseInputState: (sessionId) => { + if (!retainedInputSessions.delete(sessionId)) return; + terminalInputStates.release(sessionId); + }, markConnectionFailure: async (user, session, message, error) => { if (isSandboxLeaseOwnerReconnectError(error)) return; const markTerminal = @@ -285,6 +330,7 @@ async function writeTerminalClipboardFile( const mediaType = clean(rawMediaType || "application/octet-stream", 120); const name = terminalClipboardFilename(rawName, mediaType); const lease = sandboxLeaseInfo(session); + const { getSandbox } = await import("@cloudflare/sandbox"); const sandbox = getSandbox(env.SANDBOX, lease.sandboxId); const directory = `${sandboxWorkdir(session.id)}/.crabbox/clipboard`; const path = `${directory}/${Date.now()}-${crypto.randomUUID().slice(0, 8)}-${name}`; @@ -357,6 +403,8 @@ async function markInteractiveTerminalUnavailable( ); if (!staged) return; await appendTerminalLog(env, sessionId, user, message, now); + const { reconcileSandboxCredentialPolicyCleanupBatch } = + await import("./sandbox-credential-policy-cleanup-service.ts"); await reconcileSandboxCredentialPolicyCleanupBatch(env, now, sessionId); return; } @@ -544,16 +592,7 @@ async function multiplayerTerminalInputPayloads( } function terminalInputState(sessionId: string): TerminalInputState { - let state = terminalInputStates.get(sessionId); - if (!state) { - state = newTerminalInputState(); - terminalInputStates.set(sessionId, state); - } - return state; -} - -function releaseTerminalInputState(sessionId: string): void { - terminalInputStates.delete(sessionId); + return terminalInputStates.state(sessionId); } async function readInteractiveSessionMultiplayerMode( diff --git a/tests/interactive-terminal-service.test.ts b/tests/interactive-terminal-service.test.ts index 753f31b8..43a6d4ad 100644 --- a/tests/interactive-terminal-service.test.ts +++ b/tests/interactive-terminal-service.test.ts @@ -5,6 +5,7 @@ import { readTerminalClipboardBytes, terminalClipboardFilename, } from "../src/worker/interactive-terminal.ts"; +import { TerminalInputStateRegistry } from "../src/worker/interactive-terminal-service.ts"; test("terminal clipboard filenames are bounded, sanitized, and typed", () => { assert.equal(terminalClipboardFilename("screen shot", "image/png"), "screen-shot.png"); @@ -40,3 +41,16 @@ test("terminal clipboard upload rejects empty and declared oversized bodies", as new Uint8Array([97, 98, 99]), ); }); + +test("terminal input state survives until the final subscriber releases it", () => { + const states = new TerminalInputStateRegistry(); + states.retain("IS-1"); + states.retain("IS-1"); + states.state("IS-1").line = "shared input"; + + states.release("IS-1"); + assert.equal(states.state("IS-1").line, "shared input"); + + states.release("IS-1"); + assert.equal(states.state("IS-1").line, ""); +}); From d419a869b8f1c68c41a6bf262083e33b68c5fbd8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:05:01 +0200 Subject: [PATCH 014/242] fix(auth): revoke session grants atomically --- src/worker/session-grant-repository.ts | 38 ++++++++++++++++---------- tests/session-grant-repository.test.ts | 26 +++++++++--------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/worker/session-grant-repository.ts b/src/worker/session-grant-repository.ts index 3fba3f2d..24649bf3 100644 --- a/src/worker/session-grant-repository.ts +++ b/src/worker/session-grant-repository.ts @@ -2,7 +2,7 @@ import { sql } from "kysely"; import { trustedProxyConfigured } from "../trusted-proxy-auth.ts"; import { authorize, trustedProxyAutomaticRole } from "./auth.ts"; -import { database, executeBatch, type InteractiveSessionGrantRow } from "./database.ts"; +import { database, type InteractiveSessionGrantRow } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import type { User } from "./models.ts"; import type { @@ -152,12 +152,12 @@ export class InteractiveSessionGrantRepository { async revoke(sessionId: string, subject: string, now = Date.now()): Promise { const db = database(this.env); - const deleted = await db - .deleteFrom("interactive_session_grants") - .where("session_id", "=", sessionId) - .where("subject", "=", subject) - .executeTakeFirst(); - if (deleted.numDeletedRows < 1n) return false; + const grantExists = sql`EXISTS ( + SELECT 1 + FROM interactive_session_grants + WHERE session_id = ${sessionId} + AND subject = ${subject} + )`; const clearPendingControl = db .updateTable("interactive_sessions") .set({ @@ -166,6 +166,7 @@ export class InteractiveSessionGrantRepository { control_requested_at: null, }) .where("id", "=", sessionId) + .where(grantExists) .where("control_requested_by_subject", "=", subject); const clearDelegatedControl = db .updateTable("interactive_sessions") @@ -176,17 +177,26 @@ export class InteractiveSessionGrantRepository { control_expires_at: null, }) .where("id", "=", sessionId) + .where(grantExists) .where("controller_subject", "=", subject); const advanceSessionRevision = db .updateTable("interactive_sessions") .set({ updated_at: sql`MAX(updated_at + 1, ${now})` }) - .where("id", "=", sessionId); - await executeBatch(this.env, [ - clearPendingControl, - clearDelegatedControl, - advanceSessionRevision, - ]); - return true; + .where("id", "=", sessionId) + .where(grantExists); + const deleteGrant = db + .deleteFrom("interactive_session_grants") + .where("session_id", "=", sessionId) + .where("subject", "=", subject); + const results = await this.env.DB.batch( + [clearPendingControl, clearDelegatedControl, advanceSessionRevision, deleteGrant].map( + (query) => { + const compiled = query.compile(); + return this.env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }, + ), + ); + return (results[3]?.meta.changes ?? 0) > 0; } } diff --git a/tests/session-grant-repository.test.ts b/tests/session-grant-repository.test.ts index a50f796a..0e3d8028 100644 --- a/tests/session-grant-repository.test.ts +++ b/tests/session-grant-repository.test.ts @@ -17,6 +17,7 @@ function runtimeEnv(options: { batches?: PreparedStatement[][]; mutations?: PreparedStatement[]; mutationChanges?: number; + batchChanges?: number[]; env?: Partial; }): RuntimeEnv { return { @@ -52,7 +53,9 @@ function runtimeEnv(options: { }, async batch(statements: unknown[]) { options.batches?.push(statements as PreparedStatement[]); - return []; + return statements.map((_, index) => ({ + meta: { changes: options.batchChanges?.[index] ?? 0 }, + })); }, } as unknown as D1Database, } as RuntimeEnv; @@ -206,23 +209,18 @@ test("grant upsert is atomically fenced to the live session revision", async () test("grant revocation atomically removes access and delegated control", async () => { const batches: PreparedStatement[][] = []; - const mutations: PreparedStatement[] = []; const repository = new InteractiveSessionGrantRepository( runtimeEnv({ - mutations, - mutationChanges: 1, batches, + batchChanges: [1, 1, 1, 1], }), ); assert.equal(await repository.revoke("IS-42", "proxy:collaborator@example.test", 2_000), true); - assert.equal(mutations.length, 1); - assert.match(mutations[0]?.sql ?? "", /^delete from "interactive_session_grants"/i); - assert.ok(mutations[0]?.parameters.includes("IS-42")); - assert.ok(mutations[0]?.parameters.includes("proxy:collaborator@example.test")); assert.equal(batches.length, 1); - assert.equal(batches[0]?.length, 3); + assert.equal(batches[0]?.length, 4); assert.match(batches[0]?.[0]?.sql ?? "", /^update "interactive_sessions"/i); + assert.match(batches[0]?.[0]?.sql ?? "", /exists/i); assert.match(batches[0]?.[0]?.sql ?? "", /"control_requested_by_subject" = \?/i); assert.doesNotMatch(batches[0]?.[0]?.sql ?? "", /"control_requested_by" in/i); assert.doesNotMatch(batches[0]?.[0]?.sql ?? "", /where[^]*"controller_subject" = \?/i); @@ -231,19 +229,21 @@ test("grant revocation atomically removes access and delegated control", async ( assert.doesNotMatch(batches[0]?.[1]?.sql ?? "", /"controller" in/i); assert.doesNotMatch(batches[0]?.[1]?.sql ?? "", /where[^]*"control_requested_by_subject" = \?/i); assert.match(batches[0]?.[2]?.sql ?? "", /"updated_at" = MAX\(updated_at \+ 1, \?\)/i); + assert.match(batches[0]?.[3]?.sql ?? "", /^delete from "interactive_session_grants"/i); assert.ok(batches[0]?.[0]?.parameters.includes("proxy:collaborator@example.test")); assert.ok(batches[0]?.[1]?.parameters.includes("proxy:collaborator@example.test")); assert.ok(batches[0]?.[2]?.parameters.includes(2_000)); + assert.ok(batches[0]?.[3]?.parameters.includes("IS-42")); + assert.ok(batches[0]?.[3]?.parameters.includes("proxy:collaborator@example.test")); }); test("grant revocation leaves sessions untouched when no grant exists", async () => { const batches: PreparedStatement[][] = []; - const mutations: PreparedStatement[] = []; const repository = new InteractiveSessionGrantRepository( - runtimeEnv({ mutations, mutationChanges: 0, batches }), + runtimeEnv({ batches, batchChanges: [0, 0, 0, 0] }), ); assert.equal(await repository.revoke("IS-42", "proxy:missing@example.test"), false); - assert.equal(mutations.length, 1); - assert.deepEqual(batches, []); + assert.equal(batches.length, 1); + assert.equal(batches[0]?.length, 4); }); From 48944ce9f271456840890227cbb27043d69adc1e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:07:06 +0200 Subject: [PATCH 015/242] fix(macos): harden private desktop sharing --- .../CrabfleetMac/CrabboxVNCBridge.swift | 7 +- .../CrabfleetDesktopRegistration.swift | 54 +++- .../CrabfleetMac/CrabfleetMacApp.swift | 21 +- .../Sources/CrabfleetMac/FleetRootView.swift | 3 +- .../Sources/CrabfleetMac/FleetStore.swift | 14 +- .../Sources/CrabfleetMac/HostClipboard.swift | 16 +- .../Sources/CrabfleetMac/MacRemoteInput.swift | 24 +- .../CrabfleetMac/NativeAPIClient.swift | 6 +- .../PrivateMacShareController.swift | 108 +++++-- .../CrabfleetMac/SubprocessEnvironment.swift | 33 +++ .../CrabfleetMac/TailnetIdentity.swift | 247 +++++++++++----- .../CrabfleetMac/TailnetRFBServer.swift | 53 +++- .../CrabfleetMacTests/FleetModelsTests.swift | 31 ++ .../HostShareProtocolTests.swift | 61 +++- .../NativeConnectionTests.swift | 73 ++++- .../PrivateMacShareTests.swift | 276 +++++++++++++++++- macos/CrabfleetMac/scripts/build-app.sh | 1 + 17 files changed, 856 insertions(+), 172 deletions(-) create mode 100644 macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift index 082046f5..73fadf94 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift @@ -129,6 +129,7 @@ final class CrabboxVNCBridge: @unchecked Sendable { process.standardInput = stdin process.standardOutput = stdout process.standardError = stderr + process.environment = commandEnvironment(from: ProcessInfo.processInfo.environment) do { try process.run() @@ -240,6 +241,10 @@ final class CrabboxVNCBridge: @unchecked Sendable { } } + static func commandEnvironment(from source: [String: String]) -> [String: String] { + SubprocessEnvironment.minimal(from: source, includeSSHAgent: true) + } + private static func drain(_ pipe: Pipe) { pipe.fileHandleForReading.readabilityHandler = { handle in _ = handle.availableData @@ -256,7 +261,7 @@ final class CrabboxVNCBridge: @unchecked Sendable { private static func validGrant(_ grant: NativeVNCGrant) -> Bool { let ticketPrefix = "native_vnc_" let ticketSuffix = grant.ticket.dropFirst(ticketPrefix.count) - let secureBroker = grant.brokerURL.scheme == "https" + let secureBroker = (grant.brokerURL.scheme == "https" && grant.brokerURL.host?.isEmpty == false) || (grant.brokerURL.scheme == "http" && ["localhost", "127.0.0.1", "::1"].contains(grant.brokerURL.host ?? "")) return secureBroker diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index f0347263..08a8d1f6 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -2,9 +2,10 @@ import Foundation protocol DesktopHostRegistering: Sendable { func register(identity: TailnetIdentity, port: UInt16) async throws + func unregister(identity: TailnetIdentity) async throws } -struct CrabfleetDesktopRegistration: DesktopHostRegistering, Sendable { +struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable { private struct RegistrationBody: Encodable { let name: String let address: String @@ -13,11 +14,11 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, Sendable { private let baseURL: URL private let sessionCookie: String - private let session: URLSession + private let transport: any HTTPDataTransport init?( environment: [String: String] = ProcessInfo.processInfo.environment, - session: URLSession = .shared + transport: any HTTPDataTransport = RejectingRedirectURLSessionTransport() ) { guard let rawURL = environment["CRABFLEET_API_URL"], @@ -37,17 +38,33 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, Sendable { guard normalizedURL.path.isEmpty || normalizedURL.path == "/" else { return nil } self.baseURL = normalizedURL self.sessionCookie = cookie - self.session = session + self.transport = transport } func register(identity: TailnetIdentity, port: UInt16) async throws { let request = try registrationRequest(identity: identity, port: port) - let (_, response) = try await session.data(for: request) - guard let http = response as? HTTPURLResponse else { - throw DesktopHostRegistrationError.invalidResponse + let (_, http) = try await transport.data(for: request) + try validate(response: http, for: request, acceptingNotFound: false) + } + + func unregister(identity: TailnetIdentity) async throws { + let request = removalRequest(identity: identity) + let (_, http) = try await transport.data(for: request) + try validate(response: http, for: request, acceptingNotFound: true) + } + + private func validate( + response: HTTPURLResponse, + for request: URLRequest, + acceptingNotFound: Bool + ) throws { + guard response.url == request.url else { + throw DesktopHostRegistrationError.redirectRejected } - guard (200..<300).contains(http.statusCode) else { - throw DesktopHostRegistrationError.httpStatus(http.statusCode) + guard (200..<300).contains(response.statusCode) + || (acceptingNotFound && response.statusCode == 404) + else { + throw DesktopHostRegistrationError.httpStatus(response.statusCode) } } @@ -73,6 +90,20 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, Sendable { return request } + func removalRequest(identity: TailnetIdentity) -> URLRequest { + let url = + baseURL + .appending(path: "api") + .appending(path: "desktop-hosts") + .appending(path: Self.hostID(identity: identity)) + var request = URLRequest(url: url) + request.httpMethod = "DELETE" + request.timeoutInterval = 15 + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") + return request + } + static func hostID(identity: TailnetIdentity) -> String { let dnsLabel = identity.dnsName.split(separator: ".").first.map(String.init) ?? "" let normalized = dnsLabel.lowercased().filter { @@ -97,14 +128,17 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, Sendable { } } -enum DesktopHostRegistrationError: LocalizedError { +enum DesktopHostRegistrationError: LocalizedError, Equatable { case invalidResponse + case redirectRejected case httpStatus(Int) var errorDescription: String? { switch self { case .invalidResponse: "Crabfleet returned an invalid registration response." + case .redirectRejected: + "Crabfleet redirected the desktop registration request." case .httpStatus(let status): "Crabfleet registration returned HTTP \(status)." } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift index 25cf1939..0e5d6a89 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift @@ -47,18 +47,26 @@ enum VNCConnectionLaunchMode { @MainActor final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { - private var shareController: PrivateMacShareController? + let shareController: PrivateMacShareController private var autoShareTask: Task? + override init() { + shareController = PrivateMacShareController() + super.init() + } + + init(shareController: PrivateMacShareController) { + self.shareController = shareController + super.init() + } + func applicationDidFinishLaunching(_ notification: Notification) { guard PrivateMacShareLaunchMode.isRequested() else { return } NSApp.activate(ignoringOtherApps: true) - let controller = PrivateMacShareController() - shareController = controller autoShareTask = Task { [weak self] in guard let self else { return } try? await Task.sleep(for: .milliseconds(500)) - await self.startPrivateShare(controller) + await self.startPrivateShare(shareController) } } @@ -119,6 +127,7 @@ struct CrabfleetMacApp: App { fleetStore: fleetStore, connectionLibrary: connectionLibrary, sessionPool: sessionPool, + privateShare: appDelegate.shareController, launchConnection: launchConnection ) } @@ -142,6 +151,7 @@ private struct CrabfleetAppRoot: View { @ObservedObject var fleetStore: FleetStore @ObservedObject var connectionLibrary: ConnectionLibrary @ObservedObject var sessionPool: VNCSessionPool + @ObservedObject var privateShare: PrivateMacShareController let launchConnection: VNCAddress? @Environment(\.scenePhase) private var scenePhase @@ -151,11 +161,13 @@ private struct CrabfleetAppRoot: View { fleetStore: FleetStore, connectionLibrary: ConnectionLibrary, sessionPool: VNCSessionPool, + privateShare: PrivateMacShareController, launchConnection: VNCAddress? ) { self.fleetStore = fleetStore self.connectionLibrary = connectionLibrary self.sessionPool = sessionPool + self.privateShare = privateShare self.launchConnection = launchConnection _localOnly = State( initialValue: ProcessInfo.processInfo.environment["CRABFLEET_LOCAL_ONLY"] == "1" @@ -170,6 +182,7 @@ private struct CrabfleetAppRoot: View { store: fleetStore, connections: connectionLibrary, sessions: sessionPool, + privateShare: privateShare, launchConnection: launchConnection, deploymentLabel: fleetStore.isConnected ? fleetStore.deploymentLabel : "Local VNC", accountLabel: fleetStore.isConnected ? fleetStore.accountLabel : NSUserName(), diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/FleetRootView.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/FleetRootView.swift index 96171fe3..1194db77 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/FleetRootView.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/FleetRootView.swift @@ -5,13 +5,12 @@ struct FleetRootView: View { @ObservedObject var store: FleetStore @ObservedObject var connections: ConnectionLibrary @ObservedObject var sessions: VNCSessionPool + @ObservedObject var privateShare: PrivateMacShareController let launchConnection: VNCAddress? let deploymentLabel: String let accountLabel: String let disconnectLabel: String let disconnectDeployment: () -> Void - @StateObject private var privateShare = PrivateMacShareController() - @Namespace private var desktopTransition @Environment(\.accessibilityReduceMotion) private var reduceMotion diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/FleetStore.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/FleetStore.swift index 9ad5fdc9..c3a50b5b 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/FleetStore.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/FleetStore.swift @@ -498,7 +498,12 @@ final class FleetStore: ObservableObject { throw CancellationError() } do { - return (try await operation(token), token) + let value = try await operation(token) + try Task.checkCancellation() + guard isCurrent(generation), client === api, connectedOrigin == origin else { + throw CancellationError() + } + return (value, token) } catch NativeAPIError.unauthorized { try Task.checkCancellation() guard isCurrent(generation), client === api, connectedOrigin == origin else { @@ -518,7 +523,12 @@ final class FleetStore: ObservableObject { credential = .init(origin: origin, token: refreshed) adopted = true } - return (try await operation(refreshed), refreshed) + let value = try await operation(refreshed) + try Task.checkCancellation() + guard isCurrent(generation), client === api, connectedOrigin == origin else { + throw CancellationError() + } + return (value, refreshed) } catch { if !adopted, preserveRotatedCredentialOnTransientFailure, diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift index 0ace3b80..02fb4246 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift @@ -27,7 +27,6 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { private var lastObservedChangeCount: Int? private var suppressedChangeCount: Int? private var lastKnownText: String? - private var lastAppliedClientText: String? init( pasteboard: NSPasteboard = .general, @@ -67,7 +66,6 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { func detach() { withLock { pusher = nil - lastAppliedClientText = nil suppressedChangeCount = nil } DispatchQueue.main.async { [weak self] in @@ -81,16 +79,12 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { DispatchQueue.main.async { [weak self] in guard let self else { return } let alreadyCurrent = self.withLock { self.lastKnownText == text } - if alreadyCurrent { - self.withLock { self.lastAppliedClientText = text } - return - } + if alreadyCurrent { return } self.pasteboard.clearContents() guard self.pasteboard.setString(text, forType: .string) else { return } self.withLock { self.suppressedChangeCount = self.pasteboard.changeCount self.lastObservedChangeCount = self.pasteboard.changeCount - self.lastAppliedClientText = text self.lastKnownText = text } } @@ -119,13 +113,11 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { suppressedChangeCount = nil return } - guard let text, !text.isEmpty, - text != lastAppliedClientText, - text.utf8.count <= RFBWire.maximumClipboardBytes - else { + let outboundText = text ?? "" + guard outboundText.utf8.count <= RFBWire.maximumClipboardBytes else { return } - textToPush = text + textToPush = outboundText pushHandler = pusher } if let textToPush, let pushHandler { diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index 3244940e..db6efac9 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -118,19 +118,23 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable func releaseAllInput() { eventQueue.async { [self] in - guard Self.isAccessibilityGranted else { return } - for keysym in pressedKeysyms { - postKeyEvent(down: false, keysym: keysym) + let canPostEvents = Self.isAccessibilityGranted + if canPostEvents { + for keysym in pressedKeysyms { + postKeyEvent(down: false, keysym: keysym) + } } pressedKeysyms.removeAll() - for button in Self.mouseButtons where previousButtonMask & button.mask != 0 { - CGEvent( - mouseEventSource: eventSource(), - mouseType: button.upType, - mouseCursorPosition: previousPointerLocation, - mouseButton: button.button - )?.post(tap: .cghidEventTap) + if canPostEvents { + for button in Self.mouseButtons where previousButtonMask & button.mask != 0 { + CGEvent( + mouseEventSource: eventSource(), + mouseType: button.upType, + mouseCursorPosition: previousPointerLocation, + mouseButton: button.button + )?.post(tap: .cghidEventTap) + } } previousButtonMask = 0 } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/NativeAPIClient.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/NativeAPIClient.swift index 7712e5dc..7501a240 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/NativeAPIClient.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/NativeAPIClient.swift @@ -647,8 +647,10 @@ final class NativeAPIClient: NativeAPIClientProtocol { guard url.user == nil, url.password == nil, url.query == nil, url.fragment == nil else { return false } - if url.scheme == "https" { return true } - return url.scheme == "http" && ["localhost", "127.0.0.1", "::1"].contains(url.host ?? "") + let scheme = url.scheme?.lowercased() + let host = url.host ?? "" + if scheme == "https" { return !host.isEmpty } + return scheme == "http" && ["localhost", "127.0.0.1", "::1"].contains(host) } private func validOpaqueNativeVNCValue(_ value: String, maximumBytes: Int) -> Bool { diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 3decba85..8b99a097 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -16,6 +16,7 @@ enum PrivateMacSharePermissionPolicy { final class PrivateMacShareController: ObservableObject { enum RegistryPhase: Equatable { case notConfigured + case notPublished case registering case registered case failed(String) @@ -23,6 +24,7 @@ final class PrivateMacShareController: ObservableObject { var detail: String { switch self { case .notConfigured: "Not configured" + case .notPublished: "Not published" case .registering: "Registering" case .registered: "Published" case .failed(let message): message @@ -103,7 +105,9 @@ final class PrivateMacShareController: ObservableObject { private var capture: MacScreenCapture? private var server: TailnetRFBServer? private var clipboardBridge: HostClipboardBridge? - private var serverGeneration: UUID? + private var activeIdentity: TailnetIdentity? + private var lifecycleGeneration: UInt64 = 0 + private var serverGeneration: UInt64? private var registrationTask: Task? init( @@ -113,7 +117,7 @@ final class PrivateMacShareController: ObservableObject { ) { self.desktopRegistration = desktopRegistration self.defaults = defaults - registryPhase = desktopRegistration == nil ? .notConfigured : .registering + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished let savedDisplayID = defaults.object(forKey: Self.selectedDisplayDefaultsKey) as? Int selectedDisplayID = savedDisplayID.map(CGDirectDisplayID.init) ?? CGMainDisplayID() clipboardSyncEnabled = @@ -139,7 +143,7 @@ final class PrivateMacShareController: ObservableObject { } var canStart: Bool { - phase == .idle + phase == .idle && !isRefreshing && PrivateMacSharePermissionPolicy.canStart( identityAvailable: identity != nil, screenRecordingGranted: screenRecordingGranted @@ -150,7 +154,12 @@ final class PrivateMacShareController: ObservableObject { guard !isRefreshing, phase != .starting, phase != .stopping else { return } isRefreshing = true notice = nil - await loadIdentity() + do { + identity = try await fetchIdentity() + } catch { + identity = nil + notice = error.localizedDescription + } refreshPermissions() await refreshDisplays() launchAtLoginEnabled = SMAppService.mainApp.status == .enabled @@ -207,13 +216,24 @@ final class PrivateMacShareController: ObservableObject { } func start() async { - guard phase == .idle else { return } + guard phase == .idle, !isRefreshing else { return } + let generation = beginLifecycleTransition() phase = .starting notice = nil connectedPeer = nil streamStats = nil registryPhase = desktopRegistration == nil ? .notConfigured : .registering - await loadIdentity() + do { + let loadedIdentity = try await fetchIdentity() + guard isCurrent(generation), phase == .starting else { return } + identity = loadedIdentity + } catch { + guard isCurrent(generation), phase == .starting else { return } + identity = nil + phase = .failed + notice = error.localizedDescription + return + } refreshPermissions() guard let identity else { @@ -223,6 +243,7 @@ final class PrivateMacShareController: ObservableObject { } guard screenRecordingGranted else { phase = .idle + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished notice = PrivateMacShareError.screenRecordingDenied.localizedDescription return } @@ -237,9 +258,12 @@ final class PrivateMacShareController: ObservableObject { let capture = MacScreenCapture() do { let descriptor = try await capture.start(displayID: selectedDisplayID) + guard isCurrent(generation), phase == .starting else { + await capture.stop() + return + } let input = MacRemoteInputController(descriptor: descriptor) let bridge = clipboardSyncEnabled ? HostClipboardBridge() : nil - let generation = UUID() serverGeneration = generation let server = TailnetRFBServer( identity: identity, @@ -258,7 +282,12 @@ final class PrivateMacShareController: ObservableObject { self.capture = capture self.server = server self.clipboardBridge = bridge + activeIdentity = identity } catch { + guard isCurrent(generation) else { + await capture.stop() + return + } serverGeneration = nil await capture.stop() phase = .failed @@ -268,11 +297,13 @@ final class PrivateMacShareController: ObservableObject { func stop() async { guard phase.isRunning || phase == .failed else { return } + let generation = beginLifecycleTransition() phase = .stopping connectedPeer = nil streamStats = nil + let registrationTask = self.registrationTask + self.registrationTask = nil registrationTask?.cancel() - registrationTask = nil serverGeneration = nil server?.stop() server = nil @@ -281,7 +312,23 @@ final class PrivateMacShareController: ObservableObject { let capture = capture self.capture = nil await capture?.stop() - phase = .idle + await registrationTask?.value + var removedRegistryEntry = true + if let desktopRegistration, let activeIdentity { + do { + try await desktopRegistration.unregister(identity: activeIdentity) + registryPhase = .notPublished + } catch { + removedRegistryEntry = false + registryPhase = .failed(error.localizedDescription) + notice = error.localizedDescription + } + } else { + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished + } + if removedRegistryEntry { self.activeIdentity = nil } + guard isCurrent(generation) else { return } + phase = removedRegistryEntry ? .idle : .failed } func openPrivacySettings(_ pane: PrivacyPane) { @@ -305,25 +352,16 @@ final class PrivateMacShareController: ObservableObject { case accessibility } - private func loadIdentity() async { + private func fetchIdentity() async throws -> TailnetIdentity { guard let runner else { - identity = nil - notice = - (runnerInitializationError ?? PrivateMacShareError.tailscaleNotInstalled) - .localizedDescription - return - } - do { - let result = try await runner.run(arguments: ["status", "--json"]) - let document = try JSONDecoder().decode( - TailscaleStatusDocument.self, - from: Data(result.standardOutput.utf8) - ) - identity = try TailnetIdentityPolicy.identity(from: document) - } catch { - identity = nil - notice = error.localizedDescription + throw runnerInitializationError ?? PrivateMacShareError.tailscaleNotInstalled } + let result = try await runner.run(arguments: ["status", "--json"]) + let document = try JSONDecoder().decode( + TailscaleStatusDocument.self, + from: Data(result.standardOutput.utf8) + ) + return try TailnetIdentityPolicy.identity(from: document) } private func refreshPermissions() { @@ -344,7 +382,7 @@ final class PrivateMacShareController: ObservableObject { } } - private func handle(_ event: TailnetRFBServerEvent, generation: UUID) { + private func handle(_ event: TailnetRFBServerEvent, generation: UInt64) { guard serverGeneration == generation else { return } switch event { case .listening: @@ -390,10 +428,10 @@ final class PrivateMacShareController: ObservableObject { } } - private func registerDesktopHost(generation: UUID) { + private func registerDesktopHost(generation: UInt64) { registrationTask?.cancel() - guard let desktopRegistration, let identity else { - registryPhase = .notConfigured + guard let desktopRegistration, let identity = activeIdentity else { + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } registryPhase = .registering @@ -410,4 +448,14 @@ final class PrivateMacShareController: ObservableObject { } } } + + @discardableResult + private func beginLifecycleTransition() -> UInt64 { + lifecycleGeneration &+= 1 + return lifecycleGeneration + } + + private func isCurrent(_ generation: UInt64) -> Bool { + lifecycleGeneration == generation + } } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift new file mode 100644 index 00000000..c5c85c1d --- /dev/null +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift @@ -0,0 +1,33 @@ +import Foundation + +enum SubprocessEnvironment { + private static let inheritedKeys = [ + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + ] + + static let safePath = "/usr/bin:/bin:/usr/sbin:/sbin" + + static func minimal( + from source: [String: String], + includeSSHAgent: Bool = false, + overrides: [String: String] = [:] + ) -> [String: String] { + var environment = Dictionary( + uniqueKeysWithValues: inheritedKeys.compactMap { key in + source[key].map { (key, $0) } + } + ) + environment["PATH"] = safePath + if includeSSHAgent, let socket = source["SSH_AUTH_SOCK"], !socket.isEmpty { + environment["SSH_AUTH_SOCK"] = socket + } + for (key, value) in overrides { + environment[key] = value + } + return environment + } +} diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift index 77103a02..dbc87a68 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift @@ -11,13 +11,18 @@ protocol TailscaleCommandRunning: Sendable { struct SystemTailscaleCommandRunner: TailscaleCommandRunning { private static let maximumOutputBytes = 4 * 1_024 * 1_024 + private static let defaultTimeout: TimeInterval = 15 static let executableCandidates = [ "/Applications/Tailscale.app/Contents/MacOS/Tailscale", ] let executableURL: URL + let timeout: TimeInterval - init(fileManager: FileManager = .default) throws { + init( + fileManager: FileManager = .default, + timeout: TimeInterval = Self.defaultTimeout + ) throws { guard let path = Self.executableCandidates.first(where: { Self.isTrustedExecutable(atPath: $0, fileManager: fileManager) @@ -26,89 +31,40 @@ struct SystemTailscaleCommandRunner: TailscaleCommandRunning { throw PrivateMacShareError.tailscaleNotInstalled } executableURL = URL(fileURLWithPath: path) + self.timeout = timeout } - init(executableURL: URL) { + init(executableURL: URL, timeout: TimeInterval = Self.defaultTimeout) { self.executableURL = executableURL + self.timeout = timeout } func run(arguments: [String]) async throws -> TailscaleCommandResult { - try await withCheckedThrowingContinuation { continuation in - DispatchQueue.global(qos: .userInitiated).async { - let process = Process() - let outputPipe = Pipe() - let errorPipe = Pipe() - let readGroup = DispatchGroup() - let capture = CommandCapture() - - process.executableURL = executableURL - process.arguments = arguments - process.standardOutput = outputPipe - process.standardError = errorPipe - process.qualityOfService = .userInitiated - - process.environment = Self.commandEnvironment( - from: ProcessInfo.processInfo.environment - ) - - readGroup.enter() - DispatchQueue.global(qos: .userInitiated).async { - capture.setStandardOutput(outputPipe.fileHandleForReading.readDataToEndOfFile()) - readGroup.leave() - } - - readGroup.enter() + let execution = TailscaleCommandExecution( + executableURL: executableURL, + arguments: arguments, + environment: Self.commandEnvironment(from: ProcessInfo.processInfo.environment), + timeout: timeout, + maximumOutputBytes: Self.maximumOutputBytes + ) + return try await withTaskCancellationHandler { + let result = try await withCheckedThrowingContinuation { continuation in DispatchQueue.global(qos: .userInitiated).async { - capture.setStandardError(errorPipe.fileHandleForReading.readDataToEndOfFile()) - readGroup.leave() - } - - do { - try process.run() - process.waitUntilExit() - readGroup.wait() - - let (outputData, errorData) = capture.values() - guard - outputData.count <= Self.maximumOutputBytes, - errorData.count <= Self.maximumOutputBytes - else { - continuation.resume(throwing: PrivateMacShareError.commandOutputTooLarge) - return - } - - let result = TailscaleCommandResult( - standardOutput: String(decoding: outputData, as: UTF8.self), - standardError: String(decoding: errorData, as: UTF8.self) - ) - guard process.terminationStatus == 0 else { - let message = result.standardError.trimmingCharacters(in: .whitespacesAndNewlines) - continuation.resume( - throwing: PrivateMacShareError.commandFailed( - status: process.terminationStatus, - message: String(message.prefix(500)) - )) - return - } - continuation.resume(returning: result) - } catch { - outputPipe.fileHandleForWriting.closeFile() - errorPipe.fileHandleForWriting.closeFile() - readGroup.wait() - continuation.resume(throwing: error) + continuation.resume(with: Result { try execution.run() }) } } + try Task.checkCancellation() + return result + } onCancel: { + execution.cancel() } } static func commandEnvironment(from source: [String: String]) -> [String: String] { - var environment = source - for key in environment.keys - where key.hasPrefix("TS_") || key.hasPrefix("TAILSCALE_") { - environment.removeValue(forKey: key) - } - environment["TAILSCALE_BE_CLI"] = "1" - return environment + SubprocessEnvironment.minimal( + from: source, + overrides: ["TAILSCALE_BE_CLI": "1"] + ) } static func isTrustedExecutable( @@ -131,27 +87,152 @@ struct SystemTailscaleCommandRunner: TailscaleCommandRunning { } } -private final class CommandCapture: @unchecked Sendable { +private final class TailscaleCommandExecution: @unchecked Sendable { + private enum StopReason { + case cancelled + case outputTooLarge + case timedOut + } + private let lock = NSLock() + private let process = Process() + private let outputPipe = Pipe() + private let errorPipe = Pipe() + private let readGroup = DispatchGroup() + private let timeout: TimeInterval + private let maximumOutputBytes: Int + private var stopReason: StopReason? private var standardOutput = Data() private var standardError = Data() - func setStandardOutput(_ data: Data) { + init( + executableURL: URL, + arguments: [String], + environment: [String: String], + timeout: TimeInterval, + maximumOutputBytes: Int + ) { + self.timeout = max(0.1, timeout) + self.maximumOutputBytes = maximumOutputBytes + process.executableURL = executableURL + process.arguments = arguments + process.environment = environment + process.standardOutput = outputPipe + process.standardError = errorPipe + process.qualityOfService = .userInitiated + } + + func run() throws -> TailscaleCommandResult { + if currentStopReason() != nil { throw CancellationError() } + startCapture(pipe: outputPipe, isStandardOutput: true) + startCapture(pipe: errorPipe, isStandardOutput: false) + + do { + try process.run() + } catch { + outputPipe.fileHandleForWriting.closeFile() + errorPipe.fileHandleForWriting.closeFile() + readGroup.wait() + throw error + } + + if currentStopReason() != nil { terminate() } + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning { + if currentStopReason() != nil { + terminate() + } else if Date() >= deadline { + stop(.timedOut) + } + Thread.sleep(forTimeInterval: 0.01) + } + process.waitUntilExit() + readGroup.wait() + + switch currentStopReason() { + case .cancelled: + throw CancellationError() + case .outputTooLarge: + throw PrivateMacShareError.commandOutputTooLarge + case .timedOut: + throw PrivateMacShareError.commandTimedOut + case nil: + break + } + + let result = values() + guard process.terminationStatus == 0 else { + let message = result.standardError.trimmingCharacters(in: .whitespacesAndNewlines) + throw PrivateMacShareError.commandFailed( + status: process.terminationStatus, + message: String(message.prefix(500)) + ) + } + return result + } + + func cancel() { + stop(.cancelled) + } + + private func startCapture(pipe: Pipe, isStandardOutput: Bool) { + readGroup.enter() + DispatchQueue.global(qos: .userInitiated).async { [self] in + defer { readGroup.leave() } + var data = Data() + while true { + let chunk = pipe.fileHandleForReading.readData(ofLength: 64 * 1_024) + if chunk.isEmpty { break } + guard chunk.count <= maximumOutputBytes - data.count else { + stop(.outputTooLarge) + break + } + data.append(chunk) + } + setCaptured(data, isStandardOutput: isStandardOutput) + } + } + + private func setCaptured(_ data: Data, isStandardOutput: Bool) { lock.lock() - standardOutput = data + if isStandardOutput { + standardOutput = data + } else { + standardError = data + } lock.unlock() } - func setStandardError(_ data: Data) { + private func values() -> TailscaleCommandResult { lock.lock() - standardError = data - lock.unlock() + defer { lock.unlock() } + return TailscaleCommandResult( + standardOutput: String(decoding: standardOutput, as: UTF8.self), + standardError: String(decoding: standardError, as: UTF8.self) + ) } - func values() -> (Data, Data) { + private func currentStopReason() -> StopReason? { lock.lock() defer { lock.unlock() } - return (standardOutput, standardError) + return stopReason + } + + private func stop(_ reason: StopReason) { + lock.lock() + if stopReason == nil { stopReason = reason } + lock.unlock() + terminate() + } + + private func terminate() { + guard process.isRunning else { return } + process.terminate() + let pid = process.processIdentifier + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 0.5) { [process] in + guard process.isRunning else { return } + _ = Darwin.kill(pid, SIGKILL) + } } } @@ -317,7 +398,10 @@ struct TailnetPeerAuthorizer: TailnetPeerAuthorizing, Sendable { let expectedIdentity: TailnetIdentity func authorize(remoteAddress: String) async -> Bool { - guard TailnetIdentityPolicy.isTailscaleIPv4(remoteAddress) else { return false } + guard + TailnetIdentityPolicy.isTailscaleIPv4(remoteAddress), + remoteAddress != expectedIdentity.ipv4Address + else { return false } do { let result = try await runner.run(arguments: ["whois", "--json", remoteAddress]) let document = try JSONDecoder().decode( @@ -347,6 +431,7 @@ enum PrivateMacShareError: LocalizedError, Equatable { case accessibilityDenied case commandFailed(status: Int32, message: String) case commandOutputTooLarge + case commandTimedOut case captureUnavailable case listenerFailed(String) case protocolError(String) @@ -375,6 +460,8 @@ enum PrivateMacShareError: LocalizedError, Equatable { : "Tailscale exited with status \(status): \(message)" case .commandOutputTooLarge: "Tailscale returned more status data than Crabfleet will accept." + case .commandTimedOut: + "Tailscale did not respond before the command deadline." case .captureUnavailable: "Crabfleet could not capture the main display." case .listenerFailed(let message): diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift index e3b00699..a7ac0e15 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift @@ -31,6 +31,7 @@ final class TailnetRFBServer: @unchecked Sendable { private let clipboard: (any HostClipboardSyncing)? private let peerAuthorizer: (any TailnetPeerAuthorizing)? private let port: UInt16 + private let handshakeTimeout: Duration private let queue = DispatchQueue(label: "org.openclaw.crabfleet.rfb-listener") private let lock = NSLock() private let eventHandler: EventHandler @@ -47,6 +48,7 @@ final class TailnetRFBServer: @unchecked Sendable { clipboard: (any HostClipboardSyncing)? = nil, peerAuthorizer: (any TailnetPeerAuthorizing)? = nil, port: UInt16, + handshakeTimeout: Duration = .seconds(10), eventHandler: @escaping EventHandler ) { self.identity = identity @@ -57,6 +59,7 @@ final class TailnetRFBServer: @unchecked Sendable { self.clipboard = clipboard self.peerAuthorizer = peerAuthorizer self.port = port + self.handshakeTimeout = handshakeTimeout self.eventHandler = eventHandler } @@ -134,6 +137,7 @@ final class TailnetRFBServer: @unchecked Sendable { clipboard: clipboard, requiredLocalAddress: identity.ipv4Address, desktopName: "Crabfleet — \(identity.hostName)", + handshakeTimeout: handshakeTimeout, viewOnly: false, didAuthorize: { [weak capture] in capture?.setConsumerActive(true) }, eventHandler: eventHandler, @@ -173,6 +177,7 @@ private final class RFBHostSession: @unchecked Sendable { private let clipboard: (any HostClipboardSyncing)? private let requiredLocalAddress: String private let desktopName: String + private let handshakeTimeout: Duration private let didAuthorize: @Sendable () -> Void private let queue = DispatchQueue(label: "org.openclaw.crabfleet.rfb-session") private let eventHandler: TailnetRFBServer.EventHandler @@ -180,6 +185,8 @@ private final class RFBHostSession: @unchecked Sendable { private let lock = NSLock() private var started = false private var finished = false + private var handshakeFinished = false + private var handshakeTimedOut = false private var task: Task? private var pushIO: RFBConnectionIO? @@ -213,6 +220,7 @@ private final class RFBHostSession: @unchecked Sendable { clipboard: (any HostClipboardSyncing)?, requiredLocalAddress: String, desktopName: String, + handshakeTimeout: Duration, viewOnly: Bool, didAuthorize: @escaping @Sendable () -> Void, eventHandler: @escaping TailnetRFBServer.EventHandler, @@ -226,6 +234,7 @@ private final class RFBHostSession: @unchecked Sendable { self.clipboard = clipboard self.requiredLocalAddress = requiredLocalAddress self.desktopName = desktopName + self.handshakeTimeout = handshakeTimeout self.viewOnly = viewOnly self.didAuthorize = didAuthorize self.eventHandler = eventHandler @@ -310,7 +319,7 @@ private final class RFBHostSession: @unchecked Sendable { didAuthorize() let io = RFBConnectionIO(connection: connection) - try await handshake(io: io) + try await handshakeBeforeDeadline(io: io) withLock { pushIO = io } attachClipboard() eventHandler(.connected(remoteAddress)) @@ -351,6 +360,45 @@ private final class RFBHostSession: @unchecked Sendable { )) } + private func handshakeBeforeDeadline(io: RFBConnectionIO) async throws { + let deadlineTask = Task { [weak self, handshakeTimeout] in + do { + try await Task.sleep(for: handshakeTimeout) + } catch { + return + } + self?.expireHandshake() + } + do { + try await handshake(io: io) + deadlineTask.cancel() + let timedOut = withLock { () -> Bool in + handshakeFinished = true + return handshakeTimedOut + } + guard !timedOut else { + throw PrivateMacShareError.protocolError("RFB handshake timed out") + } + } catch { + deadlineTask.cancel() + if withLock({ handshakeTimedOut }) { + throw PrivateMacShareError.protocolError("RFB handshake timed out") + } + throw error + } + } + + private func expireHandshake() { + let shouldCancel = withLock { () -> Bool in + guard !finished, !handshakeFinished else { return false } + handshakeTimedOut = true + return true + } + if shouldCancel { + finish(event: .sessionFailed("RFB handshake timed out")) + } + } + private func messageLoop(io: RFBConnectionIO) async throws { var hasSentJPEGFrame = false var lastSentJPEGSequence: UInt64 = 0 @@ -599,7 +647,7 @@ private final class RFBHostSession: @unchecked Sendable { case .notify: payload = VNCExtendedClipboard.frame( messageType: 3, - body: VNCExtendedClipboard.encodeNotify(hasText: true) + body: VNCExtendedClipboard.encodeNotify(hasText: !text.isEmpty) ) case .legacy: payload = RFBWire.legacyServerCutText(text: text) @@ -1005,6 +1053,7 @@ private final class RFBHostSession: @unchecked Sendable { finishPixelMailbox() let encoder = replaceVideoEncoder(with: nil) encoder?.invalidate() + input.releaseAllInput() clipboard?.detach() connection.cancel() guard encoder != nil else { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift index a2bad09c..74962fd8 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift @@ -16,6 +16,23 @@ private func nativeVNCGrant(leaseID: String = "cbx_native123") -> NativeVNCGrant } struct FleetModelsTests { + @Test + func crabboxReceivesOnlyItsMinimalSubprocessEnvironment() { + let environment = CrabboxVNCBridge.commandEnvironment( + from: [ + "HOME": "/Users/tester", + "PATH": "/tmp/untrusted", + "SSH_AUTH_SOCK": "/tmp/agent.sock", + "CRABFLEET_SESSION_COOKIE": "secret", + ] + ) + + #expect(environment["HOME"] == "/Users/tester") + #expect(environment["PATH"] == SubprocessEnvironment.safePath) + #expect(environment["SSH_AUTH_SOCK"] == "/tmp/agent.sock") + #expect(environment["CRABFLEET_SESSION_COOKIE"] == nil) + } + @Test func sizesRemoteDesktopToEvenViewportPixelsWithinPerformanceCap() { #expect( @@ -159,6 +176,20 @@ struct FleetModelsTests { } } + @Test + func rejectsHTTPSNativeGrantWithoutAHost() async { + let grant = NativeVNCGrant( + brokerURL: URL(string: "https:///native-vnc")!, + leaseID: "cbx_native123", + ticket: nativeVNCTicket, + expiresAt: Date().addingTimeInterval(60) + ) + + await #expect(throws: CrabboxVNCBridgeError.invalidHandoff) { + _ = try await CrabboxVNCBridge.start(grant: grant) + } + } + @Test func parsesGenericVNCAddresses() throws { let direct = try VNCAddress.parse("workstation.example:5907") diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift index 943d8a77..f6145541 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift @@ -15,20 +15,21 @@ struct HostShareWireTests { ) #expect( - update == Data([ - 0, 0, // FramebufferUpdate + padding - 0, 1, // one rectangle - 0, 1, // x = reason (client-requested) - 0, 0, // y = status (no error) - 0x05, 0x00, // width 1280 - 0x02, 0xD0, // height 720 - 0xFF, 0xFF, 0xFE, 0xCC, // ExtendedDesktopSize (-308) - 1, 0, 0, 0, // one screen + padding - 0, 0, 0, 1, // screen id - 0, 0, 0, 0, // position - 0x05, 0x00, 0x02, 0xD0, // screen size - 0, 0, 0, 0, // flags - ]) + update + == Data([ + 0, 0, // FramebufferUpdate + padding + 0, 1, // one rectangle + 0, 1, // x = reason (client-requested) + 0, 0, // y = status (no error) + 0x05, 0x00, // width 1280 + 0x02, 0xD0, // height 720 + 0xFF, 0xFF, 0xFE, 0xCC, // ExtendedDesktopSize (-308) + 1, 0, 0, 0, // one screen + padding + 0, 0, 0, 1, // screen id + 0, 0, 0, 0, // position + 0x05, 0x00, 0x02, 0xD0, // screen size + 0, 0, 0, 0, // flags + ]) ) } @@ -146,6 +147,38 @@ struct HostClipboardBridgeTests { bridge.detach() } + @Test + func forwardsClipboardClearsAndLocallyReusedClientValues() async throws { + let pasteboard = NSPasteboard(name: .init("CrabfleetMacTests.\(UUID().uuidString)")) + pasteboard.clearContents() + pasteboard.setString("initial", forType: .string) + + let recorder = PushRecorder() + let bridge = HostClipboardBridge(pasteboard: pasteboard, pollingInterval: 0.01) + bridge.attach { recorder.append($0) } + try await Task.sleep(for: .milliseconds(30)) + + bridge.receiveClientText("reused") + try await waitUntil { pasteboard.string(forType: .string) == "reused" } + bridge.poll() + #expect(recorder.values.isEmpty) + + pasteboard.clearContents() + pasteboard.setString("other", forType: .string) + bridge.poll() + #expect(recorder.values == ["other"]) + + pasteboard.clearContents() + bridge.poll() + #expect(recorder.values == ["other", ""]) + + pasteboard.clearContents() + pasteboard.setString("reused", forType: .string) + bridge.poll() + #expect(recorder.values == ["other", "", "reused"]) + bridge.detach() + } + private func waitUntil( timeout: Duration = .seconds(1), condition: @escaping @MainActor () -> Bool diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/NativeConnectionTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/NativeConnectionTests.swift index b6d9ce1d..6d2bf33e 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/NativeConnectionTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/NativeConnectionTests.swift @@ -123,6 +123,26 @@ struct NativeConnectionTests { #expect(grant.leaseID == "cbx_native123") #expect(grant.ticket == "native_vnc_0123456789abcdef0123456789abcdef") + let missingHostTransport = RecordingHTTPTransport { request in + let body = Data( + """ + { + "grant": { + "brokerUrl": "https:///native-vnc", + "leaseId": "cbx_native123", + "ticket": "native_vnc_0123456789abcdef0123456789abcdef", + "expiresAt": "\(expiryFormatter.string(from: expiresAt))" + } + } + """.utf8 + ) + return (body, httpResponse(url: request.url!, status: 200)) + } + await #expect(throws: NativeAPIError.invalidResponse) { + try await NativeAPIClient(origin: origin, transport: missingHostTransport) + .nativeVNCGrant(sessionID: "IS-257", accessToken: "access-token") + } + await #expect(throws: NativeAPIError.invalidResponse) { try await NativeAPIClient(origin: origin, transport: transport) .nativeVNCGrant(sessionID: "IS-0", accessToken: "access-token") @@ -693,6 +713,44 @@ struct NativeConnectionTests { #expect(api.fleetTokens == ["saved-token"]) } + @Test + func disconnectDiscardsAnInFlightNativeVNCGrant() async throws { + let origin = try DeploymentOrigin("https://fleet.example.test") + let origins = MemoryOriginStore(value: origin.displayValue) + let tokens = MemoryTokenStore(values: [origin.displayValue: "saved-token"]) + let api = StubNativeAPIClient(origin: origin) + api.sessionResult = .success(testSession()) + api.fleetResult = .success(testFleet()) + var grantStarted = false + var grantContinuation: CheckedContinuation? + api.nativeVNCGrantHandler = { _, _ in + grantStarted = true + return try await withCheckedThrowingContinuation { continuation in + grantContinuation = continuation + } + } + let store = FleetStore( + environment: [:], + originStore: origins, + tokenStore: tokens, + clientFactory: { _ in api }, + openURL: { _ in false } + ) + await store.restore() + + let grantTask = Task { + try await store.nativeVNCGrant(sessionID: "IS-257") + } + try await waitUntil { grantStarted } + store.disconnect() + let continuation = try #require(grantContinuation) + continuation.resume(returning: testNativeVNCGrant()) + + await #expect(throws: CancellationError.self) { + try await grantTask.value + } + } + @Test func unauthorizedOAuthReadRotatesAndPersistsCredentialBeforeRetry() async throws { let origin = try DeploymentOrigin("https://fleet.example.test") @@ -1725,6 +1783,7 @@ private final class StubNativeAPIClient: NativeAPIClientProtocol { var fleetResult: Result = .failure(NativeAPIError.invalidResponse) var nativeVNCGrantResult: Result = .failure( NativeAPIError.invalidResponse) + var nativeVNCGrantHandler: ((String, String) async throws -> NativeVNCGrant)? var fleetHandler: (() async throws -> NativeAPIFleet)? var sessionTokens: [String] = [] var fleetTokens: [String] = [] @@ -1773,7 +1832,10 @@ private final class StubNativeAPIClient: NativeAPIClientProtocol { } func nativeVNCGrant(sessionID: String, accessToken: String) async throws -> NativeVNCGrant { - try nativeVNCGrantResult.get() + if let nativeVNCGrantHandler { + return try await nativeVNCGrantHandler(sessionID, accessToken) + } + return try nativeVNCGrantResult.get() } func refreshCredential(accessToken: String) async throws -> String? { @@ -1890,6 +1952,15 @@ private func testSession() -> NativeAPISession { ) } +private func testNativeVNCGrant() -> NativeVNCGrant { + .init( + brokerURL: URL(string: "https://crabbox.example.test")!, + leaseID: "cbx_native123", + ticket: "native_vnc_0123456789abcdef0123456789abcdef", + expiresAt: Date().addingTimeInterval(60) + ) +} + private func testLease(id: String = "IS-live") -> CrabboxLease { .init( id: id, diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index f33a57b5..5bc0d8e4 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -1,5 +1,6 @@ import AppKit import Foundation +import Network import Testing @testable import CrabfleetMac @@ -22,12 +23,16 @@ struct PrivateMacShareTests { let environment = SystemTailscaleCommandRunner.commandEnvironment( from: [ - "PATH": "/usr/bin:/bin", + "HOME": "/Users/tester", + "PATH": "/tmp/untrusted", + "SECRET_TOKEN": "do-not-forward", "TS_DEBUG": "unsafe", "TAILSCALE_SOCKET": "/tmp/unsafe.sock", ] ) - #expect(environment["PATH"] == "/usr/bin:/bin") + #expect(environment["HOME"] == "/Users/tester") + #expect(environment["PATH"] == SubprocessEnvironment.safePath) + #expect(environment["SECRET_TOKEN"] == nil) #expect(environment["TS_DEBUG"] == nil) #expect(environment["TAILSCALE_SOCKET"] == nil) #expect(environment["TAILSCALE_BE_CLI"] == "1") @@ -55,6 +60,85 @@ struct PrivateMacShareTests { )) } + @Test + func tailscaleCommandTimesOutAndRespondsToCancellation() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("CrabfleetMacTests.\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let executable = directory.appendingPathComponent("tailscale") + let pidFile = directory.appendingPathComponent("pid") + try Data( + """ + #!/bin/sh + printf '%s' "$$" > '\(pidFile.path)' + exec sleep 30 + """.utf8 + ).write(to: executable) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: executable.path) + + let timedRunner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 0.1) + await #expect(throws: PrivateMacShareError.commandTimedOut) { + _ = try await timedRunner.run(arguments: ["status"]) + } + + try? FileManager.default.removeItem(at: pidFile) + let cancellableRunner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 30) + let task = Task { + try await cancellableRunner.run(arguments: ["status"]) + } + let launched = await waitUntilAsync { + FileManager.default.fileExists(atPath: pidFile.path) + } + #expect(launched) + let cancelledPID = try #require(Int(String(contentsOf: pidFile, encoding: .utf8))) + task.cancel() + await #expect(throws: CancellationError.self) { + try await task.value + } + #expect(await waitUntilAsync { Darwin.kill(Int32(cancelledPID), 0) != 0 }) + } + + @Test @MainActor + func stopInvalidatesAnInFlightPrivateShareStart() async throws { + let runner = SuspendedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: nil, + defaults: defaults + ) + let startTask = Task { await controller.start() } + let started = await waitUntilAsync { await runner.hasStarted } + #expect(started) + + await controller.stop() + await runner.resume( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + await startTask.value + + #expect(controller.phase == .idle) + #expect(controller.identity == nil) + } + + @Test @MainActor + func applicationDelegateOwnsTheShareControllerUsedByTheApp() throws { + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: StaticTailscaleRunner(output: statusJSON()), + desktopRegistration: nil, + defaults: defaults + ) + let delegate = CrabfleetApplicationDelegate(shareController: controller) + + #expect(delegate.shareController === controller) + } + @Test func privateShareCanStartViewOnlyWithoutAccessibility() { #expect( @@ -201,6 +285,42 @@ struct PrivateMacShareTests { #expect(json["name"] as? String == "Workstation") #expect(json["address"] as? String == "100.64.12.34") #expect(json["port"] as? Int == 5901) + + let removal = registration.removalRequest(identity: identity) + #expect(removal.url == request.url) + #expect(removal.httpMethod == "DELETE") + #expect(removal.value(forHTTPHeaderField: "Cookie") == "crabbox_session=secret") + #expect(removal.httpBody == nil) + } + + @Test + func desktopRegistrationRejectsRedirectedResponses() async throws { + let redirectedURL = try #require(URL(string: "https://login.example.test/desktop-host")) + let transport = DesktopRegistrationTransport { _ in + ( + Data(), + try #require( + HTTPURLResponse( + url: redirectedURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + await #expect(throws: DesktopHostRegistrationError.redirectRejected) { + try await registration.register(identity: identity, port: 5_901) + } } @Test @@ -290,6 +410,62 @@ struct PrivateMacShareTests { #expect(!(await otherAddress.authorize(remoteAddress: "100.100.10.20"))) #expect(!(await unauthorizedNode.authorize(remoteAddress: "100.100.10.20"))) #expect(!(await accepted.authorize(remoteAddress: "192.168.1.4"))) + #expect(!(await accepted.authorize(remoteAddress: identity.ipv4Address))) + } + + @Test @MainActor + func expiresIncompleteRFBHandshakeAndReleasesInput() async throws { + let identity = TailnetIdentity( + tailnetName: "example.com", + loginName: "tester@example.com", + dnsName: "workstation.example.ts.net.", + hostName: "Workstation", + ipv4Address: "127.0.0.1", + userID: 42 + ) + let capture = MacScreenCapture() + let input = RemoteInputRecorder() + let events = RFBEventRecorder() + let port: UInt16 = 5_923 + let server = TailnetRFBServer( + identity: identity, + runner: StaticTailscaleRunner(output: ""), + capture: capture, + descriptor: .init( + displayID: 0, + displayBounds: CGRect(x: 0, y: 0, width: 64, height: 64), + frameWidth: 64, + frameHeight: 64, + sourcePixelWidth: 64, + sourcePixelHeight: 64 + ), + input: input, + peerAuthorizer: LoopbackPeerAuthorizer(), + port: port, + handshakeTimeout: .milliseconds(100), + eventHandler: { events.append($0) } + ) + try server.start() + defer { server.stop() } + try await Task.sleep(for: .milliseconds(100)) + + let connection = NWConnection( + host: "127.0.0.1", + port: try #require(NWEndpoint.Port(rawValue: port)), + using: .tcp + ) + connection.start(queue: .global(qos: .userInitiated)) + defer { connection.cancel() } + + try await waitFor { + events.values.contains { + if case .sessionFailed(let message) = $0 { + return message.contains("handshake timed out") + } + return false + } + } + #expect(input.releaseCount == 1) } @Test @@ -583,6 +759,21 @@ struct PrivateMacShareTests { ) return bitmap.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) } + + @Test + func buildScriptRecreatesTheAppBundleBeforeAssembly() throws { + let testFile = URL(fileURLWithPath: #filePath) + let script = + testFile + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("scripts/build-app.sh") + let contents = try String(contentsOf: script, encoding: .utf8) + let removal = try #require(contents.range(of: "rm -rf \"$app_dir\"")) + let assembly = try #require(contents.range(of: "mkdir -p \"$macos_dir\" \"$resources_dir\"")) + #expect(removal.lowerBound < assembly.lowerBound) + } } private struct StaticTailscaleRunner: TailscaleCommandRunning { @@ -603,3 +794,84 @@ private struct LoopbackPeerAuthorizer: TailnetPeerAuthorizing { remoteAddress == "127.0.0.1" } } + +private actor SuspendedTailscaleRunner: TailscaleCommandRunning { + private var continuation: CheckedContinuation? + private(set) var hasStarted = false + + func run(arguments: [String]) async throws -> TailscaleCommandResult { + hasStarted = true + return try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + } + } + + func resume(_ result: Result) { + continuation?.resume(with: result) + continuation = nil + } +} + +private final class RemoteInputRecorder: RemoteInputForwarding, @unchecked Sendable { + private let lock = NSLock() + private var releases = 0 + + var releaseCount: Int { + lock.lock() + defer { lock.unlock() } + return releases + } + + func keyEvent(down: Bool, keysym: UInt32) {} + func pointerEvent(buttonMask: UInt8, x: UInt16, y: UInt16) {} + + func releaseAllInput() { + lock.lock() + releases += 1 + lock.unlock() + } +} + +private final class RFBEventRecorder: @unchecked Sendable { + private let lock = NSLock() + private var storage: [TailnetRFBServerEvent] = [] + + var values: [TailnetRFBServerEvent] { + lock.lock() + defer { lock.unlock() } + return storage + } + + func append(_ event: TailnetRFBServerEvent) { + lock.lock() + storage.append(event) + lock.unlock() + } +} + +private final class DesktopRegistrationTransport: HTTPDataTransport { + private let handler: (URLRequest) throws -> (Data, HTTPURLResponse) + + init(handler: @escaping (URLRequest) throws -> (Data, HTTPURLResponse)) { + self.handler = handler + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + try handler(request) + } + + func close() {} +} + +private func waitUntilAsync( + timeout: Duration = .seconds(2), + condition: @escaping () async -> Bool +) async -> Bool { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + while clock.now < deadline { + if await condition() { return true } + try? await Task.sleep(for: .milliseconds(10)) + } + return await condition() +} diff --git a/macos/CrabfleetMac/scripts/build-app.sh b/macos/CrabfleetMac/scripts/build-app.sh index 9347e28c..b6456f24 100644 --- a/macos/CrabfleetMac/scripts/build-app.sh +++ b/macos/CrabfleetMac/scripts/build-app.sh @@ -35,6 +35,7 @@ contents_dir="$app_dir/Contents" macos_dir="$contents_dir/MacOS" resources_dir="$contents_dir/Resources" +rm -rf "$app_dir" mkdir -p "$macos_dir" "$resources_dir" install -m 755 "$build_dir/CrabfleetMac" "$macos_dir/CrabfleetMac" install -m 755 "$build_dir/libRoyalVNCKit.dylib" "$macos_dir/libRoyalVNCKit.dylib" From 15cc13a02f140846d87af381de1befb0118ac022 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:07:34 +0200 Subject: [PATCH 016/242] fix(vnc): harden protocol state transitions --- .../RoyalVNCKit/Compression/ZlibStream.swift | 123 ++++--- .../Framebuffer/VNCFramebuffer.swift | 30 +- .../NWConnection+NetworkConnection.swift | 59 ++-- .../Encodings/Frame/TightEncoding.swift | 32 +- .../Encodings/Frame/ZRLEEncoding.swift | 40 ++- .../SDK/Connection/VNCConnection+API.swift | 118 ++++++- .../Connection/VNCConnection+Delegate.swift | 25 +- .../SDK/Connection/VNCConnection.swift | 64 +++- .../RoyalVNCKit/SDK/Cursor/VNCCursor.swift | 6 +- .../RoyalVNCKitTests/AuditFindingsTests.swift | 307 ++++++++++++++++++ 10 files changed, 697 insertions(+), 107 deletions(-) create mode 100644 macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift index a0797cbd..8fb21617 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift @@ -38,6 +38,12 @@ extension ZlibStream { func decompressedData(compressedData: Data, maximumOutputSize: Int) throws -> Data { + guard maximumOutputSize >= 0 else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataOverflow + )) + } + let stream = self.stream let flush = ZlibFlush.noFlush @@ -64,47 +70,53 @@ extension ZlibStream { stream.availIn = .init(compressedSize) while true { -// print("AVAILIN: \(stream.availIn)") - - if stream.availIn <= 0 { - break - } - - var isDone = false - stream.nextOut = buffer stream.availOut = .init(bufferSize) -// print("AVAIL OUT BEFORE INFLATE: \(stream.availOut)") + let inputBefore = stream.availIn + let outputBefore = stream.totalOut + let isDone: Bool - // Inflate another chunk. do { isDone = try stream.inflate(flush: flush) - - if stream.availOut >= 0 { - let availOut: UInt = .init(stream.availOut) - - // print("AVAIL OUT AFTER INFLATE: \(availOut)") - - let actualOut = bufferSize - availOut - - if actualOut > 0 { - guard decompressedData.count <= maximumOutputSize - Int(actualOut) else { - throw VNCError.protocol(.zlibDecompress( - underlyingError: ZlibStreamError.decompressedDataOverflow - )) - } - decompressedData.append(buffer, - count: .init(actualOut)) - } + } catch let error as ZlibError { + if case .bufferError = error, inputBefore == 0 { + break } + throw VNCError.protocol(.zlibDecompress(underlyingError: error)) } catch { throw VNCError.protocol(.zlibDecompress(underlyingError: error)) } + let actualOut = bufferSize - UInt(stream.availOut) + + if actualOut > 0 { + guard decompressedData.count <= maximumOutputSize - Int(actualOut) else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataOverflow + )) + } + decompressedData.append(buffer, count: Int(actualOut)) + } + if isDone { + guard stream.availIn == 0 else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataLengthMismatch + )) + } + break + } + + if stream.availIn == 0, stream.availOut > 0 { break } + + guard stream.availIn < inputBefore || stream.totalOut > outputBefore else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataLengthMismatch + )) + } } } @@ -135,6 +147,8 @@ extension ZlibStream { throw VNCError.protocol(.zlibDecompress(underlyingError: nil)) } + var overflowByte: UInt8 = 0 + while true { let doneBytes = stream.totalOut let remainingBytes = uncompressedSize - doneBytes @@ -143,26 +157,59 @@ extension ZlibStream { throw VNCError.protocol(.zlibDecompress(underlyingError: ZlibStreamError.decompressedDataOverflow)) } - stream.nextOut = decompressedDataBytes.advanced(by: .init(doneBytes)) - stream.availOut = .init(remainingBytes) + let inputBefore = stream.availIn + let outputBefore = stream.totalOut + let isDone: Bool - if remainingBytes <= 0 { - break - } - - // Inflate another chunk. do { - let isDone = try stream.inflate(flush: flush) - - if isDone { + if remainingBytes > 0 { + stream.nextOut = decompressedDataBytes.advanced(by: .init(doneBytes)) + stream.availOut = .init(remainingBytes) + isDone = try stream.inflate(flush: flush) + } else { + isDone = try withUnsafeMutablePointer(to: &overflowByte) { overflowPtr in + stream.nextOut = overflowPtr + stream.availOut = 1 + return try stream.inflate(flush: flush) + } + } + } catch let error as ZlibError { + if case .bufferError = error, inputBefore == 0 { break } + throw VNCError.protocol(.zlibDecompress(underlyingError: error)) } catch { throw VNCError.protocol(.zlibDecompress(underlyingError: error)) } + + guard stream.totalOut <= uncompressedSize else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataOverflow + )) + } + + if isDone { + guard stream.availIn == 0 else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataLengthMismatch + )) + } + break + } + + if stream.availIn == 0, stream.availOut > 0 { + break + } + + guard stream.availIn < inputBefore || stream.totalOut > outputBefore else { + throw VNCError.protocol(.zlibDecompress( + underlyingError: ZlibStreamError.decompressedDataLengthMismatch + )) + } } - guard stream.totalOut == uncompressedSize else { + guard stream.totalOut == uncompressedSize, + stream.availIn == 0 else { throw VNCError.protocol(.zlibDecompress(underlyingError: ZlibStreamError.decompressedDataLengthMismatch)) } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Framebuffer/VNCFramebuffer.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Framebuffer/VNCFramebuffer.swift index 63c0689b..086a12dc 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Framebuffer/VNCFramebuffer.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Framebuffer/VNCFramebuffer.swift @@ -731,10 +731,34 @@ private extension VNCFramebuffer { return } - var data = bufferData(ofRegion: sourceRegion) + let data = bufferData(ofRegion: sourceRegion) + let bytesPerPixel = destinationProperties.bytesPerPixel + let rowByteCount = Int(destinationRegion.width) * bytesPerPixel + let destinationX = Int(destinationRegion.x) + let destinationY = Int(destinationRegion.y) + + guard data.count == rowByteCount * Int(destinationRegion.height) else { + logger.logError("Invalid internal framebuffer data length for CopyRect") + return + } + + data.withUnsafeBytes { sourceBytes in + guard let sourceBase = sourceBytes.baseAddress else { return } + + for row in 0.. Data { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Network/NWConnection+NetworkConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Network/NWConnection+NetworkConnection.swift index 9ad2b099..1e54c744 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Network/NWConnection+NetworkConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Network/NWConnection+NetworkConnection.swift @@ -76,36 +76,45 @@ extension NWConnection: NetworkConnectionReading { maximumLength: Int) async throws -> Data { return try await withCheckedThrowingContinuation { continuation in receive(minimumIncompleteLength: minimumLength, maximumLength: maximumLength) { content, _, isComplete, error in - guard !isComplete else { - continuation.resume(throwing: VNCError.connection(.closed)) - - return - } - - guard error == nil else { - continuation.resume(throwing: error!) - - return - } - - guard let content else { - continuation.resume(throwing: VNCError.protocol(.noData)) - - return + do { + continuation.resume(returning: try Self.validateReadContent( + content, + isComplete: isComplete, + error: error, + minimumLength: minimumLength, + maximumLength: maximumLength + )) + } catch { + continuation.resume(throwing: error) } + } + } + } - let receivedLength = content.count - - guard receivedLength >= minimumLength, - receivedLength <= maximumLength else { - continuation.resume(throwing: VNCError.protocol(.invalidData)) - - return - } + static func validateReadContent( + _ content: Data?, + isComplete: Bool, + error: Error?, + minimumLength: Int, + maximumLength: Int + ) throws -> Data { + if let error { + throw error + } - continuation.resume(returning: content) + if let content, !content.isEmpty { + guard content.count >= minimumLength, + content.count <= maximumLength else { + throw VNCError.protocol(.invalidData) } + return content + } + + if isComplete { + throw VNCError.connection(.closed) } + + throw VNCError.protocol(.noData) } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/TightEncoding.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/TightEncoding.swift index 4fa2c14b..e98645b4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/TightEncoding.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/TightEncoding.swift @@ -333,7 +333,7 @@ private extension VNCProtocol.TightEncoding { case gradient = 2 } - func resetZStreamsIfNeeded(control: UInt8, + func resetZStreamsIfNeeded(control: UInt8, logger: VNCLogger) { for idx in 0..<4 { let mask = UInt8(1 << idx) @@ -349,28 +349,30 @@ private extension VNCProtocol.TightEncoding { } } } +} +extension VNCProtocol.TightEncoding { func readCompactLength(connection: NetworkConnectionReading, logger: VNCLogger) async throws -> Int { - var length = 0 - var shift = 0 - - for _ in 0..<3 { -// logger.logDebug("Reading Tight Compact Length") - - let byte = try await connection.readUInt8() - length |= Int(byte & 0x7F) << shift - - if (byte & 0x80) == 0 { - return length - } + let first = try await connection.readUInt8() + var length = Int(first & 0x7F) + guard (first & 0x80) != 0 else { + return length + } - shift += 7 + let second = try await connection.readUInt8() + length |= Int(second & 0x7F) << 7 + guard (second & 0x80) != 0 else { + return length } - throw VNCError.protocol(.invalidData) + let third = try await connection.readUInt8() + length |= Int(third) << 14 + return length } +} +private extension VNCProtocol.TightEncoding { func readBuffered(connection: NetworkConnectionReading, length: Int, logger: VNCLogger) async throws -> Data { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/ZRLEEncoding.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/ZRLEEncoding.swift index f291d7ed..1fd80e43 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/ZRLEEncoding.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Encodings/Frame/ZRLEEncoding.swift @@ -68,7 +68,10 @@ extension VNCProtocol.ZRLEEncoding { let decompressedData = try zStream.decompressedData( compressedData: compressedData, - maximumOutputSize: VNCProtocolLimits.maximumFramebufferBytes + maximumOutputSize: Self.maximumInflatedSize( + width: Int(rectangle.width), + height: Int(rectangle.height) + ) ) let stream = DataStream(data: decompressedData) @@ -156,6 +159,35 @@ extension VNCProtocol.ZRLEEncoding { framebuffer.didUpdate(region: rectangle.region) } + + static func maximumInflatedSize(width: Int, height: Int) -> Int { + guard width > 0, height > 0 else { return 0 } + + let tileSize = Int(Self.tileSize) + var maximumSize = 0 + + for tileY in stride(from: 0, to: height, by: tileSize) { + let tileHeight = min(tileSize, height - tileY) + + for tileX in stride(from: 0, to: width, by: tileSize) { + let tileWidth = min(tileSize, width - tileX) + let pixelCount = tileWidth * tileHeight + let packedPaletteBytes = 16 * 3 + ((tileWidth * 4 + 7) / 8) * tileHeight + let rlePaletteBytes = 127 * 3 + pixelCount * 2 + let tilePayloadBytes = max( + pixelCount * 3, + 3, + packedPaletteBytes, + pixelCount * 4, + rlePaletteBytes + ) + + maximumSize += 1 + tilePayloadBytes + } + } + + return maximumSize + } } private extension VNCProtocol.ZRLEEncoding { @@ -234,6 +266,12 @@ private extension VNCProtocol.ZRLEEncoding { } let indexInPalette = (Int(encoded) >> shift) & mask + guard indexInPalette < Int(paletteSize) else { + throw VNCError.protocol(.zrlePaletteIndexOverflow( + paletteIndex: indexInPalette, + paletteSize: paletteSize + )) + } let sourceStartIndex = indexInPalette * 4 diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 22d3be7f..8deab894 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -4,6 +4,19 @@ import FoundationEssentials import Foundation #endif +private struct PixelFormatTransitionMessage: VNCSendableMessage { + let pixelFormatMessage: VNCProtocol.SetPixelFormat + let didSend: () -> Void + + var messageType: UInt8 { pixelFormatMessage.messageType } + var data: Data { pixelFormatMessage.data } + + func send(connection: NetworkConnectionWriting) async throws { + try await pixelFormatMessage.send(connection: connection) + didSend() + } +} + // MARK: - Connect/Disconnect public extension VNCConnection { #if canImport(ObjectiveC) @@ -26,19 +39,15 @@ public extension VNCConnection { @objc #endif func updateColorDepth(_ colorDepth: Settings.ColorDepth) { - guard let framebuffer = framebuffer else { return } - - let newPixelFormat = VNCProtocol.PixelFormat(depth: colorDepth.rawValue) - - state.pixelFormat = newPixelFormat - - let sendPixelFormatMessage = VNCProtocol.SetPixelFormat(pixelFormat: newPixelFormat) - - enqueueClientToServerMessage(sendPixelFormatMessage) + withLifecycleLock { + guard connectionState.status == .connected, + framebuffer != nil else { + return + } - recreateFramebuffer(size: framebuffer.size, - screens: framebuffer.screens, - pixelFormat: newPixelFormat) + let newPixelFormat = VNCProtocol.PixelFormat(depth: colorDepth.rawValue) + requestPixelFormatTransition(newPixelFormat) + } } /// Requests a single-screen desktop matching the viewer viewport. @@ -73,6 +82,74 @@ public extension VNCConnection { } } +private extension VNCConnection { + func requestPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { + framebufferRequestLock.lock() + pendingPixelFormatTransition = pixelFormat + framebufferRequestGeneration &+= 1 + framebufferPacingTask?.cancel() + framebufferPacingTask = nil + let transition = takePendingPixelFormatTransitionLocked() + framebufferRequestLock.unlock() + + if let transition { + enqueuePixelFormatTransition(transition) + } + } + + func takePendingPixelFormatTransitionLocked() -> VNCProtocol.PixelFormat? { + guard !framebufferUpdateRequestOutstanding, + !isPixelFormatTransitionInFlight, + let pixelFormat = pendingPixelFormatTransition else { + return nil + } + + pendingPixelFormatTransition = nil + isPixelFormatTransitionInFlight = true + return pixelFormat + } + + func enqueuePixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { + let message = PixelFormatTransitionMessage( + pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat) + ) { [weak self] in + self?.applyPixelFormatTransition(pixelFormat) + } + + enqueueClientToServerMessage(message) + } + + func applyPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { + var didApply = false + + withLifecycleLock { + guard connectionState.status == .connected, + let framebuffer = framebuffer else { + return + } + + state.pixelFormat = pixelFormat + recreateFramebuffer(size: framebuffer.size, + screens: framebuffer.screens, + pixelFormat: pixelFormat) + didApply = connectionState.status == .connected + } + + guard didApply else { return } + + framebufferRequestLock.lock() + isPixelFormatTransitionInFlight = false + let nextTransition = takePendingPixelFormatTransitionLocked() + framebufferRequestLock.unlock() + + if let nextTransition { + enqueuePixelFormatTransition(nextTransition) + } else { + scheduleNextFramebufferUpdate() + } + } +} + // MARK: - Mouse Input public extension VNCConnection { #if canImport(ObjectiveC) @@ -240,7 +317,11 @@ extension VNCConnection { func reserveFramebufferUpdateRequest() -> Bool { framebufferRequestLock.lock() defer { framebufferRequestLock.unlock() } - guard !framebufferUpdateRequestOutstanding else { return false } + guard !framebufferUpdateRequestOutstanding, + pendingPixelFormatTransition == nil, + !isPixelFormatTransitionInFlight else { + return false + } framebufferUpdateRequestOutstanding = true return true } @@ -248,14 +329,21 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() framebufferUpdateRequestOutstanding = false + let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() + + if let transition { + enqueuePixelFormatTransition(transition) + } } func scheduleNextFramebufferUpdate() { framebufferRequestLock.lock() framebufferPacingTask?.cancel() - guard !framebufferUpdateRequestOutstanding else { + guard !framebufferUpdateRequestOutstanding, + pendingPixelFormatTransition == nil, + !isPixelFormatTransitionInFlight else { framebufferPacingTask = nil framebufferRequestLock.unlock() return @@ -300,6 +388,8 @@ extension VNCConnection { framebufferPacingTask?.cancel() framebufferPacingTask = nil framebufferUpdateRequestOutstanding = false + pendingPixelFormatTransition = nil + isPixelFormatTransitionInFlight = false framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift index 533fa606..32b8322f 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift @@ -136,17 +136,32 @@ extension VNCConnection { private extension VNCConnection { func askDelegateForCredential(authenticationType: VNCAuthenticationType) async throws -> VNCCredential { - let credential: VNCCredential? = await withCheckedContinuation { continuation in - DispatchQueue.main.async { [weak self] in - guard let self, let delegate = self.delegate else { + let requestID = UUID() + let credential: VNCCredential? = await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + guard registerCredentialContinuation(continuation, id: requestID) else { continuation.resume(returning: nil) return } - delegate.connection(self, credentialFor: authenticationType) { credential in - continuation.resume(returning: credential) + if Task.isCancelled { + cancelPendingCredentialRequest(id: requestID) + return + } + + DispatchQueue.main.async { [weak self] in + guard let self, let delegate = self.delegate else { + self?.resolveCredentialRequest(id: requestID, credential: nil) + return + } + + delegate.connection(self, credentialFor: authenticationType) { [weak self] credential in + self?.resolveCredentialRequest(id: requestID, credential: credential) + } } } + } onCancel: { + cancelPendingCredentialRequest(id: requestID) } guard let credential else { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index cfa15ec8..9634393d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -110,9 +110,15 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var framebufferRequestGeneration: UInt64 = 0 var framebufferUpdateRequestOutstanding = false var framebufferPacingTask: Task? + var pendingPixelFormatTransition: VNCProtocol.PixelFormat? + var isPixelFormatTransitionInFlight = false private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() + private let credentialContinuationLock = NSLock() + private var pendingCredentialContinuations = [ + UUID: CheckedContinuation + ]() private let sharedZStream: ZlibStream private let sharedZRLEZStream: ZlibStream @@ -370,6 +376,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { deinit { let _self = self + _self.cancelPendingCredentialRequests() _self.clipboardMonitor.delegate = nil _self.clipboardDelegate = nil @@ -379,14 +386,19 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { // MARK: - Internal Connection State API extension VNCConnection { - func beginConnecting() { + @discardableResult + func beginConnecting() -> Bool { lifecycleLock.lock() defer { lifecycleLock.unlock() } - guard !state.disconnectRequested else { return } + guard !state.disconnectRequested, + connectionState.status == .disconnected else { + return false + } updateConnectionState(.connecting) connection.start(queue: queue) + return true } func beginDisconnecting(error: Error? = nil) { @@ -397,6 +409,7 @@ extension VNCConnection { updateConnectionState(.disconnecting) handshakeTask?.cancel() handshakeTask = nil + cancelPendingCredentialRequests() receiveTask?.cancel() receiveTask = nil sendTask?.cancel() @@ -418,6 +431,53 @@ extension VNCConnection { beginDisconnecting(error: error) } + func withLifecycleLock(_ operation: () -> T) -> T { + lifecycleLock.lock() + defer { lifecycleLock.unlock() } + return operation() + } + + func registerCredentialContinuation( + _ continuation: CheckedContinuation, + id: UUID + ) -> Bool { + lifecycleLock.lock() + defer { lifecycleLock.unlock() } + + guard !state.disconnectRequested else { + return false + } + + credentialContinuationLock.lock() + defer { credentialContinuationLock.unlock() } + + pendingCredentialContinuations[id] = continuation + return true + } + + func resolveCredentialRequest(id: UUID, credential: VNCCredential?) { + credentialContinuationLock.lock() + let continuation = pendingCredentialContinuations.removeValue(forKey: id) + credentialContinuationLock.unlock() + + continuation?.resume(returning: credential) + } + + func cancelPendingCredentialRequest(id: UUID) { + resolveCredentialRequest(id: id, credential: nil) + } + + func cancelPendingCredentialRequests() { + credentialContinuationLock.lock() + let continuations = Array(pendingCredentialContinuations.values) + pendingCredentialContinuations.removeAll() + credentialContinuationLock.unlock() + + for continuation in continuations { + continuation.resume(returning: nil) + } + } + func updateConnectionState(_ newConnectionState: ConnectionState) { self.connectionState = newConnectionState diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Cursor/VNCCursor.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Cursor/VNCCursor.swift index 1f4a1366..6df75d2d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Cursor/VNCCursor.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Cursor/VNCCursor.swift @@ -96,10 +96,8 @@ public extension VNCCursor { return } - // TODO: This assumes BGRA32 which might not be the case. - GraphicsUtils.copyBGRAtoRGBA(srcBuffer: ptrAddr, - dstBuffer: destinationPixelBuffer, - byteCount: byteCount) + destinationPixelBuffer.copyMemory(from: ptrAddr, + byteCount: byteCount) } } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift new file mode 100644 index 00000000..5fb68c30 --- /dev/null +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -0,0 +1,307 @@ +import Foundation +import Testing + +#if canImport(Network) +import Network +#endif + +@testable import RoyalVNCKit + +struct AuditFindingsTests { + @Test + func rejectsOutOfRangeZRLEPackedPaletteIndex() async throws { + let framebuffer = try makeFramebuffer(width: 1, height: 1, depth: 24) + let encoding = VNCProtocol.ZRLEEncoding(zStream: ZlibStream()) + let rectangle = VNCProtocol.Rectangle( + xPosition: 0, + yPosition: 0, + width: 1, + height: 1, + encodingType: Int32(VNCFrameEncodingType.zrle.rawValue.rawValue) + ) + + var payload = Data([3]) + payload.append(contentsOf: [ + 0, 0, 0, + 64, 64, 64, + 128, 128, 128, + 0xC0, + ]) + let compressed = try ZlibOneShot.deflate(payload) + var compressedLength = UInt32(compressed.count).bigEndian + var wire = withUnsafeBytes(of: &compressedLength) { Data($0) } + wire.append(compressed) + + await #expect(throws: (any Error).self) { + try await encoding.decodeRectangle( + rectangle, + framebuffer: framebuffer, + connection: AuditBufferConnection(wire), + logger: VNCPrintLogger() + ) + } + } + + @Test + func derivesZRLEInflationLimitFromRectangleGeometry() { + #expect(VNCProtocol.ZRLEEncoding.maximumInflatedSize(width: 1, height: 1) == 384) + #expect(VNCProtocol.ZRLEEncoding.maximumInflatedSize(width: 64, height: 64) == 16_385) + #expect(VNCProtocol.ZRLEEncoding.maximumInflatedSize(width: 65, height: 1) == 894) + } + + @Test + func drainsPendingZlibOutputAfterConsumingAllInput() throws { + let expected = Data(repeating: 0xA5, count: 204_800) + let compressed = try ZlibOneShot.deflate(expected) + + let actual = try ZlibStream().decompressedData( + compressedData: compressed, + maximumOutputSize: expected.count + ) + + #expect(actual == expected) + } + + @Test + func consumesSyncFlushBytesAfterFixedSizeOutputIsFull() throws { + let firstCompressed = Data([ + 0x78, 0x9C, 0x72, 0x74, 0x1C, 0x05, 0xA3, 0x60, 0x14, + 0x0C, 0x77, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0xFF, 0xFF, + ]) + let secondCompressed = Data([ + 0x72, 0x1A, 0x05, 0xA3, 0x60, 0x14, 0x0C, + 0x7B, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, + ]) + let stream = ZlibStream() + + let first = try stream.decompressedData( + compressedData: firstCompressed, + uncompressedSize: 1_000 + ) + let second = try stream.decompressedData( + compressedData: secondCompressed, + uncompressedSize: 1_000 + ) + + #expect(first == Data(repeating: 0x41, count: 1_000)) + #expect(second == Data(repeating: 0x42, count: 1_000)) + } + + @Test + func readsAllEightBitsOfThirdTightLengthByte() async throws { + let encoding = VNCProtocol.TightEncoding() + + let length = try await encoding.readCompactLength( + connection: AuditBufferConnection(Data([0x80, 0x80, 0xFF])), + logger: VNCPrintLogger() + ) + + #expect(length == 0xFF << 14) + } + + @Test + func serializesPixelFormatAndFramebufferTransitionAtSendBoundary() async throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + connection.framebufferUpdateRequestOutstanding = true + + connection.updateColorDepth(.depth8Bit) + #expect(connection.state.pixelFormat?.depth == 24) + #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + + connection.completeFramebufferUpdateRequest() + + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + let writer = AuditWritingConnection() + try await queued.message.send(connection: writer) + + #expect(writer.data.count == 20) + #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) + } + + @Test + func copyRectPreservesInternalFramebufferPixels() throws { + let framebuffer = try makeFramebuffer(width: 2, height: 1, depth: 16) + var redPixel = Data([0x00, 0x7C]) + framebuffer.update( + region: VNCRegion(x: 0, y: 0, width: 1, height: 1), + data: &redPixel + ) + + framebuffer.copy( + region: VNCRegion(x: 0, y: 0, width: 1, height: 1), + to: VNCRegion(x: 1, y: 0, width: 1, height: 1) + ) + + let pixels = Data(bytes: framebuffer.surfaceAddress, count: framebuffer.surfaceByteCount) + #expect(pixels[0..<4] == pixels[4..<8]) + } + +#if canImport(Network) + @Test + func returnsFinalNetworkContentBeforeReportingEOF() throws { + let finalContent = Data([1, 2, 3]) + + let received = try NWConnection.validateReadContent( + finalContent, + isComplete: true, + error: nil, + minimumLength: 1, + maximumLength: 3 + ) + + #expect(received == finalContent) + } +#endif + + @Test + func rejectsRepeatConnectAttempts() { + let connection = VNCConnection(settings: makeSettings()) + connection.connectionState = .connecting + + #expect(connection.beginConnecting() == false) + } + + @Test @MainActor + func disconnectCancelsPendingCredentialContinuation() async { + let connection = VNCConnection(settings: makeSettings()) + let delegate = PendingCredentialDelegate() + connection.delegate = delegate + + let credentialTask = Task { + try await connection.askDelegateForPasswordCredential(authenticationType: .vnc) + } + + while delegate.completion == nil { + await Task.yield() + } + + connection.disconnect() + + do { + _ = try await credentialTask.value + Issue.record("Expected disconnect to cancel the pending credential request") + } catch { + #expect(connection.connectionState.status == .disconnected) + } + + delegate.completion?(VNCPasswordCredential(password: "late")) + } + + @Test + func preservesAlreadyRGBAFormattedCursorChannels() { + let cursor = VNCCursor( + imageData: Data([0x11, 0x22, 0x33, 0x44]), + size: VNCSize(width: 1, height: 1), + hotspot: .zero, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerPixel: 4 + ) + let destination = UnsafeMutableRawPointer.allocate(byteCount: 4, alignment: 1) + defer { destination.deallocate() } + + cursor.copyPixelDataToRGBA32(destinationPixelBuffer: destination) + + #expect(Data(bytes: destination, count: 4) == Data([0x11, 0x22, 0x33, 0x44])) + } + + private func makeFramebuffer(width: UInt16, height: UInt16, depth: UInt8) throws + -> VNCFramebuffer + { + try VNCFramebuffer( + logger: VNCPrintLogger(), + size: VNCSize(width: width, height: height), + screens: [], + pixelFormat: VNCProtocol.PixelFormat(depth: depth), + allocator: VNCFramebufferMallocAllocator() + ) + } + + private func makeSettings() -> VNCConnection.Settings { + VNCConnection.Settings( + isDebugLoggingEnabled: false, + hostname: "127.0.0.1", + port: 5900, + isShared: true, + isScalingEnabled: true, + useDisplayLink: false, + inputMode: .none, + isClipboardRedirectionEnabled: false, + colorDepth: .depth24Bit, + frameEncodings: [.raw] + ) + } +} + +private final class AuditBufferConnection: NetworkConnectionReading { + private let data: Data + private var offset = 0 + + init(_ data: Data) { + self.data = data + } + + func read(minimumLength: Int, maximumLength: Int) async throws -> Data { + let remaining = data.count - offset + guard minimumLength > 0, + maximumLength >= minimumLength, + remaining >= minimumLength else { + throw VNCError.protocol(.noData) + } + + let count = min(maximumLength, remaining) + defer { offset += count } + return data.subdata(in: offset..<(offset + count)) + } +} + +private final class AuditWritingConnection: NetworkConnectionWriting { + var data = Data() + + func write(data: Data) async throws { + self.data.append(data) + } +} + +@MainActor +private final class PendingCredentialDelegate: VNCConnectionDelegate { + var completion: ((VNCCredential?) -> Void)? + + func connection( + _ connection: VNCConnection, + stateDidChange connectionState: VNCConnection.ConnectionState + ) {} + + func connection( + _ connection: VNCConnection, + credentialFor authenticationType: VNCAuthenticationType, + completion: @escaping (VNCCredential?) -> Void + ) { + self.completion = completion + } + + func connection(_ connection: VNCConnection, didCreateFramebuffer framebuffer: VNCFramebuffer) {} + func connection(_ connection: VNCConnection, didResizeFramebuffer framebuffer: VNCFramebuffer) {} + + func connection( + _ connection: VNCConnection, + didUpdateFramebuffer framebuffer: VNCFramebuffer, + x: UInt16, + y: UInt16, + width: UInt16, + height: UInt16 + ) {} + + func connection(_ connection: VNCConnection, didUpdateCursor cursor: VNCCursor) {} +} From eb5df98489a336a7bc0843b6a3959fb817bdb9d2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:07:46 +0200 Subject: [PATCH 017/242] fix(sandbox): preserve refreshed session ownership --- src/worker/provisioning/sandbox-repository.ts | 6 +- .../sandbox-credential-policy-repository.ts | 12 +--- ...ndbox-credential-policy-repository.test.ts | 62 +++++++++++++++++++ tests/sandbox-provisioning.test.ts | 6 +- 4 files changed, 72 insertions(+), 14 deletions(-) diff --git a/src/worker/provisioning/sandbox-repository.ts b/src/worker/provisioning/sandbox-repository.ts index 6761afd8..b4f62eeb 100644 --- a/src/worker/provisioning/sandbox-repository.ts +++ b/src/worker/provisioning/sandbox-repository.ts @@ -237,7 +237,11 @@ export async function commitManagedSandboxLeaseRefresh( db .updateTable("interactive_sessions") .set({ - status: provisioned.status, + status: sql`CASE + WHEN ${provisioned.status} = 'ready' AND status IN ('attached', 'detached') + THEN status + ELSE ${provisioned.status} + END`, lease_id: provisioned.leaseId, attach_url: provisioned.attachUrl, vnc_url: provisioned.vncUrl, diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 6378ed76..a14afffa 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -500,18 +500,8 @@ export async function beginSandboxCredentialPolicyRegistration( ): Promise { const db = database(env); const lookupIds = sandboxLookupIds(env, sandboxId); - const existing = await db - .selectFrom("interactive_session_credential_policies") - .select("registration_generation") - .distinct() - .where("session_id", "=", sessionId) - .where("sandbox_id", "=", sandboxId) - .execute(); - const existingGeneration = currentSandboxCredentialPolicyGeneration( - existing.map((row) => row.registration_generation), - ); const registration = { - generation: existingGeneration ?? newSandboxCredentialPolicyGeneration(), + generation: newSandboxCredentialPolicyGeneration(), claim: `registration:${crypto.randomUUID()}`, lookupIds, }; diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index de2948dc..32a6d97e 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -3,6 +3,7 @@ import test from "node:test"; import { activeSandboxCredentialPolicyGeneration, + beginSandboxCredentialPolicyRegistration, currentSandboxCredentialPolicyGeneration, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyRegistrationQueries, @@ -155,6 +156,67 @@ test("credential-policy registration SQL proves every supported ownership fence" assert.ok(standalone.parameters.includes("standalone-1")); }); +test("credential-policy rotation always claims a fresh generation", async () => { + let generation = ""; + let claim = ""; + let registrationExpiresAt = 0; + let statements: PreparedStatement[] = []; + const env = runtimeEnv( + (sql, _parameters, kind) => { + if (kind === "all" && /select .*lookup_id/i.test(sql)) { + return { + results: [ + { + lookup_id: "sandbox-1", + state: "registering", + registration_generation: generation, + registration_claim: claim, + registration_claim_expires_at: registrationExpiresAt, + }, + ], + }; + } + if (kind === "all" && /select .*registration_generation/i.test(sql)) { + return { + results: [{ registration_generation: "generation:existing" }], + }; + } + return {}; + }, + (prepared) => { + statements = prepared; + const parameters = prepared.flatMap((statement) => statement.parameters); + generation = String( + parameters.find( + (parameter) => + typeof parameter === "string" && + parameter.startsWith("generation:") && + parameter !== "generation:existing", + ), + ); + claim = String( + parameters.find( + (parameter) => typeof parameter === "string" && parameter.startsWith("registration:"), + ), + ); + registrationExpiresAt = Math.max( + ...parameters.filter((parameter): parameter is number => typeof parameter === "number"), + ); + return prepared.map(() => ({ results: [], meta: { changes: 1 } })); + }, + ); + + const rotated = await beginSandboxCredentialPolicyRegistration(env, "IS-42", "sandbox-1", { + leaseId: "sandbox:sandbox-1:terminal-1:autostart-v4", + sandboxId: "sandbox-1", + }); + + assert.equal(statements.length, 1); + assert.match(rotated.generation, /^generation:/); + assert.notEqual(rotated.generation, "generation:existing"); + assert.equal(rotated.generation, generation); +}); + test("active credential-policy generation requires every exact lookup row", async () => { const rows = [ { diff --git a/tests/sandbox-provisioning.test.ts b/tests/sandbox-provisioning.test.ts index ebeceb3e..6ad951b9 100644 --- a/tests/sandbox-provisioning.test.ts +++ b/tests/sandbox-provisioning.test.ts @@ -754,7 +754,7 @@ test("managed Sandbox commit fences activation and previous-policy cleanup", asy assert.ok(parameters.includes(claim.previousSandboxId)); }); -test("managed Sandbox refresh commit clears the claim before prior-policy cleanup", async () => { +test("managed Sandbox refresh commit preserves detached state before prior-policy cleanup", async () => { let statements: PreparedStatement[] = []; const expectedLeaseId = sandboxLeaseId(claim.lease); const env = runtimeEnv( @@ -764,7 +764,7 @@ test("managed Sandbox refresh commit clears the claim before prior-policy cleanu results: [ { lease_id: expectedLeaseId, - status: "ready", + status: "detached", credential_cleanup_terminal_status: null, agent_token_hash: claim.agentTokenHash, }, @@ -796,6 +796,8 @@ test("managed Sandbox refresh commit clears the claim before prior-policy cleanu assert.match(batchSql, /sandbox_refresh_claim_expires_at/); assert.match(batchSql, /agent_token_hash/); assert.match(batchSql, /interactive_session_credential_policies/); + assert.match(batchSql, /when \? = 'ready' and status in \('attached', 'detached'\)/i); + assert.match(batchSql, /then status/i); const parameters = statements.flatMap((statement) => statement.parameters); assert.ok(parameters.includes("cleanup_pending")); assert.ok(parameters.includes(claim.fence.claim)); From 3c2d6ef362fd748665c0e4574f66857a73ed9d97 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:07:50 +0200 Subject: [PATCH 018/242] fix(runtime): retain superseded workspace cleanup --- src/worker/interactive-session-application.ts | 3 +- .../runtime-adapter-release-service.ts | 17 ++- .../runtime-adapter-repository.ts | 7 +- src/worker/runtime-adapter-workspaces.ts | 23 +++- src/worker/runtime-application.ts | 12 +- src/worker/session-creation.ts | 15 ++- src/worker/session-reconciliation.ts | 8 ++ tests/runtime-adapter-release-service.test.ts | 22 +++- tests/runtime-adapter-workspaces.test.ts | 39 ++++++ tests/session-creation.test.ts | 121 +++++++++++++++++- tests/session-reconciliation.test.ts | 10 +- 11 files changed, 248 insertions(+), 29 deletions(-) diff --git a/src/worker/interactive-session-application.ts b/src/worker/interactive-session-application.ts index 4739ae7a..9d73e929 100644 --- a/src/worker/interactive-session-application.ts +++ b/src/worker/interactive-session-application.ts @@ -504,10 +504,11 @@ export class InteractiveSessionApplication { finalizeTerminal: (sessionId, status, now) => finalizeTerminalInteractiveSession(this.env, sessionId, status, now), readSession: (sessionId) => readInteractiveSessionRecord(this.env, sessionId), - stopSupersededAdapter: (sessionId, adapterWorkspaceId, createPending, now) => + stopSupersededAdapter: (sessionId, adapterWorkspaceId, registration, createPending, now) => this.runtime.release().stopSuperseded({ sessionId, adapterWorkspaceId, + registration, createPending, now, }), diff --git a/src/worker/provisioning/runtime-adapter-release-service.ts b/src/worker/provisioning/runtime-adapter-release-service.ts index cba571be..b769720d 100644 --- a/src/worker/provisioning/runtime-adapter-release-service.ts +++ b/src/worker/provisioning/runtime-adapter-release-service.ts @@ -1,10 +1,17 @@ import type { RuntimeAdapterWorkspaceStopResult } from "../session-runtime-adapter-stop.ts"; +export type RuntimeAdapterWorkspaceRegistration = { + profile: string; + controlPlane: string; +}; + export type RuntimeAdapterReleaseServiceDependencies = { clearCreatePending(sessionId: string, adapterWorkspaceId: string): Promise; stopWorkspace( sessionId: string, adapterWorkspaceId: string, + registration: RuntimeAdapterWorkspaceRegistration | null, + createPending: boolean, ): Promise; confirmRelease( sessionId: string, @@ -32,15 +39,21 @@ export class RuntimeAdapterReleaseService { async stopSuperseded(input: { sessionId: string; adapterWorkspaceId: string; + registration: RuntimeAdapterWorkspaceRegistration | null; createPending: boolean; now: number; }): Promise { - const { sessionId, adapterWorkspaceId, createPending, now } = input; + const { sessionId, adapterWorkspaceId, registration, createPending, now } = input; if (!createPending) { await this.dependencies.clearCreatePending(sessionId, adapterWorkspaceId); } try { - const release = await this.dependencies.stopWorkspace(sessionId, adapterWorkspaceId); + const release = await this.dependencies.stopWorkspace( + sessionId, + adapterWorkspaceId, + registration, + createPending, + ); if (release.status === "stopped") { await this.dependencies.confirmRelease(sessionId, adapterWorkspaceId, now, release.message); return; diff --git a/src/worker/provisioning/runtime-adapter-repository.ts b/src/worker/provisioning/runtime-adapter-repository.ts index b32ad377..6774b711 100644 --- a/src/worker/provisioning/runtime-adapter-repository.ts +++ b/src/worker/provisioning/runtime-adapter-repository.ts @@ -362,13 +362,18 @@ export async function clearRuntimeAdapterCreatePending( sessionId: string, adapterWorkspaceId: string, ): Promise { + const now = Date.now(); await database(env) .updateTable("interactive_sessions") - .set({ adapter_create_pending: 0 }) + .set({ + adapter_create_pending: 0, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) .where("id", "=", sessionId) .where("adapter", "=", runtimeAdapterName) .where("adapter_workspace_id", "=", adapterWorkspaceId) .where("status", "=", "stopping") + .where("adapter_create_pending", "=", 1) .execute(); } diff --git a/src/worker/runtime-adapter-workspaces.ts b/src/worker/runtime-adapter-workspaces.ts index 1e911275..6ef12771 100644 --- a/src/worker/runtime-adapter-workspaces.ts +++ b/src/worker/runtime-adapter-workspaces.ts @@ -26,6 +26,7 @@ import { type RuntimeAdapterCreateAttemptFence, type RuntimeAdapterWorkspaceConflictInput, } from "./provisioning/runtime-adapter.ts"; +import type { RuntimeAdapterWorkspaceRegistration } from "./provisioning/runtime-adapter-release-service.ts"; import { safeProviderError } from "./provisioning/result.ts"; import type { InteractiveProvisionResult } from "./provisioning/types.ts"; import { @@ -186,14 +187,22 @@ export class RuntimeAdapterWorkspaceLifecycle { async stopForSession( sessionId: string, adapterWorkspaceId: string, + retainedRegistration?: RuntimeAdapterWorkspaceRegistration | null, + retainedCreatePending?: boolean, ): Promise { - const registration = await database(this.env) - .selectFrom("interactive_sessions") - .select(["adapter_control_plane", "adapter_create_pending", "profile"]) - .where("id", "=", sessionId) - .where("adapter", "=", runtimeAdapterName) - .where("adapter_workspace_id", "=", adapterWorkspaceId) - .executeTakeFirst(); + const registration = retainedRegistration + ? { + adapter_control_plane: retainedRegistration.controlPlane, + adapter_create_pending: retainedCreatePending ? 1 : 0, + profile: retainedRegistration.profile, + } + : await database(this.env) + .selectFrom("interactive_sessions") + .select(["adapter_control_plane", "adapter_create_pending", "profile"]) + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .executeTakeFirst(); const controlPlane = requireRegisteredRuntimeAdapterControlPlane( this.env, registration?.profile ?? "", diff --git a/src/worker/runtime-application.ts b/src/worker/runtime-application.ts index 5551dc3a..5870ce10 100644 --- a/src/worker/runtime-application.ts +++ b/src/worker/runtime-application.ts @@ -188,8 +188,13 @@ export class RuntimeApplication { new RuntimeAdapterReleaseService({ clearCreatePending: (sessionId, adapterWorkspaceId) => clearRuntimeAdapterCreatePending(this.env, sessionId, adapterWorkspaceId), - stopWorkspace: (sessionId, adapterWorkspaceId) => - this.workspaceLifecycle().stopForSession(sessionId, adapterWorkspaceId), + stopWorkspace: (sessionId, adapterWorkspaceId, registration, createPending) => + this.workspaceLifecycle().stopForSession( + sessionId, + adapterWorkspaceId, + registration, + createPending, + ), confirmRelease: (sessionId, adapterWorkspaceId, now, message) => confirmRuntimeAdapterRelease(this.env, sessionId, adapterWorkspaceId, now, message), persistStopEvidence: (sessionId, adapterWorkspaceId, message, now, reconcileError) => @@ -263,10 +268,11 @@ export class RuntimeApplication { runtimeAdapterName, ), readSession: (sessionId) => readInteractiveSessionRecord(this.env, sessionId), - stopSuperseded: (sessionId, adapterWorkspaceId, createPending, now) => + stopSuperseded: (sessionId, adapterWorkspaceId, registration, createPending, now) => this.release().stopSuperseded({ sessionId, adapterWorkspaceId, + registration, createPending, now, }), diff --git a/src/worker/session-creation.ts b/src/worker/session-creation.ts index 6d7ea8fa..79fe9fcc 100644 --- a/src/worker/session-creation.ts +++ b/src/worker/session-creation.ts @@ -16,6 +16,7 @@ import type { InteractiveProvisionResult, SandboxProvisionOwnership, } from "./provisioning/types.ts"; +import type { RuntimeAdapterWorkspaceRegistration } from "./provisioning/runtime-adapter-release-service.ts"; export type InteractiveSessionCreateOptions = { createdBy?: string; @@ -45,6 +46,7 @@ export type InteractiveSessionCreationReservation = { export type InteractiveSessionProvisionRecoveryInput = { sessionId: string; adapterName: string; + adapterRegistration: RuntimeAdapterWorkspaceRegistration | null; sandboxLeasePrefix: string; now: number; }; @@ -123,6 +125,7 @@ export type InteractiveSessionCreationStore = { stopSupersededAdapter( sessionId: string, adapterWorkspaceId: string, + registration: RuntimeAdapterWorkspaceRegistration | null, createPending: boolean, now: number, ): Promise; @@ -166,7 +169,8 @@ export class InteractiveSessionCreationService { request.createdBy, lineage, ); - const preparationReservation = Boolean(options.afterReserve || supervisedRootSessionId); + // Keep the insert removable until request evidence is durable. + const preparationReservation = true; const now = this.store.now(); for (let attempt = 0; attempt < this.configuration.maximumAttempts; attempt += 1) { @@ -240,6 +244,12 @@ export class InteractiveSessionCreationService { { sessionId: id, adapterName: this.configuration.adapterName, + adapterRegistration: context.adapterControlPlane + ? { + profile: request.profile, + controlPlane: context.adapterControlPlane, + } + : null, sandboxLeasePrefix: this.configuration.sandboxLeasePrefix, now: this.store.now(), }, @@ -279,6 +289,7 @@ export class InteractiveSessionCreationService { } try { await prepare?.(); + await this.store.recordRequest(reservation.id, reservation.insertedAt); } catch (error) { await this.store.rollbackReservation(reservation.id, reservation.insertedAt); throw error; @@ -290,7 +301,6 @@ export class InteractiveSessionCreationService { reservation.adapterWorkspaceId, ); } - await this.store.recordRequest(reservation.id, reservation.insertedAt); return provision(); } @@ -372,6 +382,7 @@ export class InteractiveSessionCreationService { await this.store.stopSupersededAdapter( input.sessionId, result.adapterWorkspaceId, + input.adapterRegistration, result.createPending === true, input.now, ); diff --git a/src/worker/session-reconciliation.ts b/src/worker/session-reconciliation.ts index 93961870..ee64ece4 100644 --- a/src/worker/session-reconciliation.ts +++ b/src/worker/session-reconciliation.ts @@ -5,6 +5,7 @@ import { database, type CompilableQuery, type InteractiveSessionRow } from "./da import type { RuntimeEnv } from "./env.ts"; import type { InteractiveSessionStatus } from "./models.ts"; import type { InteractiveProvisionResult } from "./provisioning/types.ts"; +import type { RuntimeAdapterWorkspaceRegistration } from "./provisioning/runtime-adapter-release-service.ts"; import type { InteractiveSession } from "./session-model.ts"; export type RuntimeAdapterReconciliationTransition = { @@ -37,6 +38,7 @@ export type InteractiveSessionReconciliationStore = { stopSuperseded( sessionId: string, adapterWorkspaceId: string, + registration: RuntimeAdapterWorkspaceRegistration | null, createPending: boolean, now: number, ): Promise; @@ -120,6 +122,12 @@ export class InteractiveSessionReconciliationService { await this.store.stopSuperseded( row.id, inspection.adapterWorkspaceId, + row.adapter_control_plane + ? { + profile: row.profile, + controlPlane: row.adapter_control_plane, + } + : null, inspection.createPending === true, this.store.now(), ); diff --git a/tests/runtime-adapter-release-service.test.ts b/tests/runtime-adapter-release-service.test.ts index 2b7fefa8..bc4fb143 100644 --- a/tests/runtime-adapter-release-service.test.ts +++ b/tests/runtime-adapter-release-service.test.ts @@ -10,8 +10,14 @@ import { import { RuntimeAdapterReleaseService, type RuntimeAdapterReleaseServiceDependencies, + type RuntimeAdapterWorkspaceRegistration, } from "../src/worker/provisioning/runtime-adapter-release-service.ts"; +const registration: RuntimeAdapterWorkspaceRegistration = { + profile: "default", + controlPlane: "https://adapter.example.test/", +}; + type PreparedStatement = { sql: string; parameters: unknown[]; @@ -75,8 +81,10 @@ test("superseded release clears the create marker before stopping and confirming async clearCreatePending(sessionId, adapterWorkspaceId) { calls.push(`clear:${sessionId}:${adapterWorkspaceId}`); }, - async stopWorkspace(sessionId, adapterWorkspaceId) { - calls.push(`stop:${sessionId}:${adapterWorkspaceId}`); + async stopWorkspace(sessionId, adapterWorkspaceId, retained, createPending) { + calls.push( + `stop:${sessionId}:${adapterWorkspaceId}:${retained?.profile}:${retained?.controlPlane}:${createPending}`, + ); return { status: "stopped", message: "runtime workspace released" }; }, async confirmRelease(sessionId, adapterWorkspaceId, now, message) { @@ -89,13 +97,14 @@ test("superseded release clears the create marker before stopping and confirming await service.stopSuperseded({ sessionId: "IS-101", adapterWorkspaceId: "fleet-a-is-101", + registration, createPending: false, now: 200, }); assert.deepEqual(calls, [ "clear:IS-101:fleet-a-is-101", - "stop:IS-101:fleet-a-is-101", + "stop:IS-101:fleet-a-is-101:default:https://adapter.example.test/:false", "confirm:IS-101:fleet-a-is-101:200:runtime workspace released", ]); }); @@ -116,6 +125,7 @@ test("superseded release preserves pending stop evidence", async () => { await service.stopSuperseded({ sessionId: "IS-101", adapterWorkspaceId: "fleet-a-is-101", + registration, createPending: true, now: 200, }); @@ -144,6 +154,7 @@ test("superseded release records redacted provider failures for retry", async () await service.stopSuperseded({ sessionId: "IS-101", adapterWorkspaceId: "fleet-a-is-101", + registration, createPending: true, now: 200, }); @@ -280,7 +291,7 @@ test("confirmed stopped release persists provider evidence before finalization", assert.ok(statements[0].parameters.includes("runtime workspace released")); }); -test("create-pending clearing is fenced to the registered stopping workspace", async () => { +test("create-pending clearing fences the prior marker and advances its revision", async () => { const executions: Array<{ sql: string; parameters: unknown[] }> = []; const env = runtimeEnv((sql, parameters, kind) => { assert.equal(kind, "run"); @@ -296,9 +307,12 @@ test("create-pending clearing is fenced to the registered stopping workspace", a assert.match(executions[0].sql, /"adapter" = \?/i); assert.match(executions[0].sql, /"adapter_workspace_id" = \?/i); assert.match(executions[0].sql, /"status" = \?/i); + assert.match(executions[0].sql, /max\(updated_at \+ 1, \?\)/i); + assert.match(executions[0].sql, /where[\s\S]*"adapter_create_pending" = \?/i); assert.ok(executions[0].parameters.includes("IS-101")); assert.ok(executions[0].parameters.includes("fleet-a-is-101")); assert.ok(executions[0].parameters.includes("stopping")); + assert.ok(executions[0].parameters.includes(1)); }); function releaseEffects(calls: string[]): RuntimeAdapterReleaseEffects { diff --git a/tests/runtime-adapter-workspaces.test.ts b/tests/runtime-adapter-workspaces.test.ts index 010d822c..c72bcd8b 100644 --- a/tests/runtime-adapter-workspaces.test.ts +++ b/tests/runtime-adapter-workspaces.test.ts @@ -362,6 +362,45 @@ test("session-bound stop parses DELETE evidence and preserves the registered pat }); }); +test("superseded stop uses retained registration after the session row moves on", async () => { + let databaseReads = 0; + const requests: Array<{ url: string; method: string | undefined }> = []; + const service = new RuntimeAdapterWorkspaceLifecycle( + runtimeEnv(() => { + databaseReads += 1; + return []; + }), + dependencies({ + async fetch(input, init) { + requests.push({ url: input, method: init.method }); + return new Response(null, { status: 204 }); + }, + }), + ); + + const result = await service.stopForSession( + "IS-42", + "workspace-superseded", + { + profile: "default", + controlPlane: "https://adapter.example.test/", + }, + false, + ); + + assert.equal(databaseReads, 0); + assert.deepEqual(requests, [ + { + url: "https://adapter.example.test/v1/workspaces/workspace-superseded", + method: "DELETE", + }, + ]); + assert.deepEqual(result, { + status: "stopped", + message: "runtime adapter workspace released", + }); +}); + test("session-bound stop redacts provider credentials from failures", async () => { const env = runtimeEnv(() => [ { diff --git a/tests/session-creation.test.ts b/tests/session-creation.test.ts index 95b67d7d..ca73e802 100644 --- a/tests/session-creation.test.ts +++ b/tests/session-creation.test.ts @@ -229,8 +229,8 @@ test("session creation owns normalization through decorated durable result", asy "insert", "supervise:IS-root:IS-2:100", "prepare", - "activate:IS-2:100:workspace-2", "request:IS-2:100", + "activate:IS-2:100:workspace-2", "provision:agent-token:unowned", "persist", "event", @@ -355,7 +355,7 @@ test("session creation returns a durable replay after a request reservation race assert.equal(provisioned, false); }); -test("session creation orders supervision, preparation, activation, evidence, and provisioning", async () => { +test("session creation records request evidence before activation and provisioning", async () => { const calls: string[] = []; const service = new InteractiveSessionCreationService( creationStore({ @@ -387,8 +387,8 @@ test("session creation orders supervision, preparation, activation, evidence, an assert.deepEqual(calls, [ "supervise:IS-1:IS-2:100", "prepare", - "activate:IS-2:100:workspace-2", "record:IS-2:100", + "activate:IS-2:100:workspace-2", "provision", ]); }); @@ -430,6 +430,65 @@ test("session creation rolls back failed preparation before returning the error" assert.deepEqual(calls, ["supervise", "prepare", "rollback:IS-2:100"]); }); +test("session creation rolls back when request recording fails", async () => { + const calls: string[] = []; + const failure = new Error("request event write failed"); + const service = new InteractiveSessionCreationService( + creationStore({ + enforceSupervision: async () => { + calls.push("supervise"); + }, + recordRequest: async () => { + calls.push("record"); + throw failure; + }, + rollbackReservation: async (sessionId, insertedAt) => { + calls.push(`rollback:${sessionId}:${insertedAt}`); + }, + activateReservation: async () => { + calls.push("activate"); + }, + }), + creationConfiguration, + ); + + await assert.rejects( + service.provision( + reservation, + async () => { + calls.push("prepare"); + }, + async () => { + calls.push("provision"); + }, + ), + failure, + ); + assert.deepEqual(calls, ["supervise", "prepare", "record", "rollback:IS-2:100"]); +}); + +test("session creation keeps ordinary reservations rollbackable through request recording", async () => { + const current = session({ id: "IS-2", status: "ready" }); + let preparationReservation: boolean | null = null; + let activated = false; + const service = new InteractiveSessionCreationService( + creationStore({ + insertReservation: async (input) => { + preparationReservation = input.preparationReservation; + }, + activateReservation: async () => { + activated = true; + }, + readSession: async () => current, + }), + creationConfiguration, + ); + + assert.equal(await service.create({ repo: "openclaw/crabfleet" }), current); + assert.equal(preparationReservation, true); + assert.equal(activated, true); +}); + test("session creation skips optional supervision, preparation, and activation", async () => { const calls: string[] = []; const service = new InteractiveSessionCreationService( @@ -623,6 +682,10 @@ test("session creation preserves the currently owned adapter provision", async ( { sessionId: "IS-2", adapterName: "runtime-v1", + adapterRegistration: { + profile: "default", + controlPlane: "https://controller.example", + }, sandboxLeasePrefix: "sandbox:", now: 150, }, @@ -652,8 +715,10 @@ test("session creation stops superseded adapter workspaces", async () => { const service = new InteractiveSessionCreationService( creationStore({ readSession: async () => current, - stopSupersededAdapter: async (sessionId, workspaceId, createPending, now) => { - calls.push(`stop:${sessionId}:${workspaceId}:${createPending}:${now}`); + stopSupersededAdapter: async (sessionId, workspaceId, registration, createPending, now) => { + calls.push( + `stop:${sessionId}:${workspaceId}:${registration?.profile}:${registration?.controlPlane}:${createPending}:${now}`, + ); }, }), creationConfiguration, @@ -664,6 +729,10 @@ test("session creation stops superseded adapter workspaces", async () => { { sessionId: "IS-2", adapterName: "runtime-v1", + adapterRegistration: { + profile: "default", + controlPlane: "https://controller.example", + }, sandboxLeasePrefix: "sandbox:", now: 150, }, @@ -680,7 +749,45 @@ test("session creation stops superseded adapter workspaces", async () => { ), current, ); - assert.deepEqual(calls, ["stop:IS-2:workspace-late:true:150"]); + assert.deepEqual(calls, ["stop:IS-2:workspace-late:default:https://controller.example:true:150"]); +}); + +test("session creation retains adapter registration when late provisioning loses ownership", async () => { + const current = session({ + id: "IS-2", + status: "ready", + adapter: "runtime-v1", + adapter_workspace_id: "workspace-current", + }); + const releases: unknown[][] = []; + const service = new InteractiveSessionCreationService( + creationStore({ + persistProvisionResult: async () => ({ + updated: false, + terminalStatus: null, + terminalAt: 101, + }), + readSession: async () => current, + stopSupersededAdapter: async (...args) => { + releases.push(args); + }, + }), + creationConfiguration, + ); + + assert.equal(await service.create({ repo: "openclaw/crabfleet" }), current); + assert.deepEqual(releases, [ + [ + "IS-2", + "workspace-2", + { + profile: "default", + controlPlane: "https://controller.example", + }, + false, + 100, + ], + ]); }); test("session creation cleans superseded sandbox ownership and rereads durability", async () => { @@ -706,6 +813,7 @@ test("session creation cleans superseded sandbox ownership and rereads durabilit { sessionId: "IS-2", adapterName: "runtime-v1", + adapterRegistration: null, sandboxLeasePrefix: "sandbox:", now: 150, }, @@ -730,6 +838,7 @@ test("session creation fails explicitly when durable ownership disappears", asyn { sessionId: "IS-2", adapterName: "runtime-v1", + adapterRegistration: null, sandboxLeasePrefix: "sandbox:", now: 150, }, diff --git a/tests/session-reconciliation.test.ts b/tests/session-reconciliation.test.ts index a2182d0b..28cb8e92 100644 --- a/tests/session-reconciliation.test.ts +++ b/tests/session-reconciliation.test.ts @@ -333,14 +333,18 @@ test("lost reconciliation ownership finalizes terminal rereads or releases super [], ); }, - async stopSuperseded(sessionId, workspaceId, createPending, now) { - released.push(`${sessionId}:${workspaceId}:${createPending}:${now}`); + async stopSuperseded(sessionId, workspaceId, registration, createPending, now) { + released.push( + `${sessionId}:${workspaceId}:${registration?.profile}:${registration?.controlPlane}:${createPending}:${now}`, + ); }, }), "runtime-adapter", ); await supersededService.reconcile(row, 150); - assert.deepEqual(released, [`${row.id}:workspace-1:false:200`]); + assert.deepEqual(released, [ + `${row.id}:workspace-1:cloudflare-sandbox:https://adapter.example:false:200`, + ]); }); test("reconciliation failures retain the claimed lifecycle fence", async () => { From e5b7fbbbfbb3c951e2eb543e7345014ddcd5e403 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:08:02 +0200 Subject: [PATCH 019/242] docs(changelog): record audit hardening --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31e76755..7f074008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- Harden session lifecycle concurrency with atomic card claims and grant revocation, revision-fenced GitHub Actions updates, rollback-safe creation evidence, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, acknowledge one-shot CLI input only after the Worker accepts it, and send attributed commands atomically to prevent interleaving. +- Reject unroutable runtime profile identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations, stuck remote input, and lost clipboard clears or reused values. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication, Unicode keysyms, Tight and ZRLE parsing, bounded zlib streams, color-depth transitions, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. - Add persisted send-only and receive-only clipboard directions to the native viewer's focus toolbar; automatic sync respects the direction while the explicit Send and Get actions keep working. From 6bd3205087d664a72fe75614c44250b58392a142 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:08:30 +0200 Subject: [PATCH 020/242] style(terminal): format input dispatch --- src/worker/terminal-hub.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 58437add..1ae1e4d6 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -223,11 +223,7 @@ export class TerminalHub { }); return; } - const inputs = await this.dependencies.inputPayloads( - subscription, - user, - frame.payload, - ); + const inputs = await this.dependencies.inputPayloads(subscription, user, frame.payload); for (const [index, input] of inputs.entries()) { if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); subscription.upstream.send(input); From c6641c9a2d21efa5b9f85d40f48df9e09cc74bc4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:10:13 +0200 Subject: [PATCH 021/242] test(macos): use explicit token placeholder --- .../Tests/CrabfleetMacTests/PrivateMacShareTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 5bc0d8e4..5b99801b 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -25,7 +25,7 @@ struct PrivateMacShareTests { from: [ "HOME": "/Users/tester", "PATH": "/tmp/untrusted", - "SECRET_TOKEN": "do-not-forward", + "SECRET_TOKEN": "test-token-placeholder", "TS_DEBUG": "unsafe", "TAILSCALE_SOCKET": "/tmp/unsafe.sock", ] From 54193ada19d5d6bd5ef0460a5d5e510137b8d212 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:16:06 +0200 Subject: [PATCH 022/242] fix(macos): normalize native broker schemes --- .../Sources/CrabfleetMac/CrabboxVNCBridge.swift | 7 ++++--- .../Tests/CrabfleetMacTests/FleetModelsTests.swift | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift index 73fadf94..dd35cf21 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift @@ -258,11 +258,12 @@ final class CrabboxVNCBridge: @unchecked Sendable { value.unicodeScalars.allSatisfy { !CharacterSet.controlCharacters.contains($0) } } - private static func validGrant(_ grant: NativeVNCGrant) -> Bool { + static func validGrant(_ grant: NativeVNCGrant) -> Bool { let ticketPrefix = "native_vnc_" let ticketSuffix = grant.ticket.dropFirst(ticketPrefix.count) - let secureBroker = (grant.brokerURL.scheme == "https" && grant.brokerURL.host?.isEmpty == false) - || (grant.brokerURL.scheme == "http" + let scheme = grant.brokerURL.scheme?.lowercased() + let secureBroker = (scheme == "https" && grant.brokerURL.host?.isEmpty == false) + || (scheme == "http" && ["localhost", "127.0.0.1", "::1"].contains(grant.brokerURL.host ?? "")) return secureBroker && grant.brokerURL.user == nil diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift index 74962fd8..43f3dc62 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift @@ -190,6 +190,18 @@ struct FleetModelsTests { } } + @Test + func acceptsCaseInsensitiveNativeGrantSchemes() { + let grant = NativeVNCGrant( + brokerURL: URL(string: "HTTPS://crabbox.example.test/native-vnc")!, + leaseID: "cbx_native123", + ticket: nativeVNCTicket, + expiresAt: Date().addingTimeInterval(60) + ) + + #expect(CrabboxVNCBridge.validGrant(grant)) + } + @Test func parsesGenericVNCAddresses() throws { let direct = try VNCAddress.parse("workstation.example:5907") From 40eb46b1415716676ab78f2998ada9f2618e10f8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:16:55 +0200 Subject: [PATCH 023/242] fix(terminal): negotiate input acknowledgements --- internal/terminalws/client.go | 22 +++++++-- internal/terminalws/client_test.go | 73 ++++++++++++++++++++++++++++++ src/worker/terminal-hub.ts | 2 + tests/terminal-hub.test.ts | 1 + 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 39f924ef..42a958ce 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -74,11 +74,12 @@ type Size struct { } type Client struct { - conn *websocket.Conn - sessionID string - canInput atomic.Bool - lastSize atomic.Uint64 - writeMu sync.Mutex + conn *websocket.Conn + sessionID string + supportsInputAcknowledgement bool + canInput atomic.Bool + lastSize atomic.Uint64 + writeMu sync.Mutex } type frame struct { @@ -94,6 +95,10 @@ type eventPayload struct { CanInput bool `json:"canInput"` } +type welcomePayload struct { + InputAcknowledgements bool `json:"inputAcknowledgements"` +} + type readCanceler interface { CancelRead() error } @@ -171,6 +176,10 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option } switch current.messageType { case messageWelcome: + var welcome welcomePayload + if json.Unmarshal(current.payload, &welcome) == nil { + client.supportsInputAcknowledgement = welcome.InputAcknowledgements + } continue case messageError, messageControlRevoked: return closeWithError(frameError(current, "terminal subscription failed")) @@ -209,6 +218,9 @@ func (c *Client) SendInput(ctx context.Context, payload []byte) error { } func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { + if !c.supportsInputAcknowledgement { + return c.SendInput(ctx, payload) + } if err := c.SendInput(ctx, payload); err != nil { return err } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 83055520..ac43e0f3 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -289,6 +289,14 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { return } } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ messageType: messageEvent, @@ -326,6 +334,71 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { } } +func TestSendInputConfirmedFallsBackWithoutServerCapability(t *testing.T) { + receivedInput := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-legacy", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + current, err := decodeFrame(payload) + if err != nil { + t.Error(err) + return + } + receivedInput <- append([]byte(nil), current.payload...) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-legacy", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := client.SendInputConfirmed(ctx, []byte("legacy\n")); err != nil { + t.Fatal(err) + } + if input := <-receivedInput; string(input) != "legacy\n" { + t.Fatalf("input = %q", input) + } +} + func TestDialUsesConfiguredHTTPClientAndTimeout(t *testing.T) { server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 1ae1e4d6..a3059074 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -119,6 +119,7 @@ export class TerminalHub { ok: true, version: TERMINAL_WS_VERSION, multiplex: true, + inputAcknowledgements: true, }); const closeSubscription = (id: string, code = 1000, reason = "unsubscribed") => { @@ -154,6 +155,7 @@ export class TerminalHub { ok: true, version: TERMINAL_WS_VERSION, multiplex: true, + inputAcknowledgements: true, }); return; } diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index b54ed691..383eb9f5 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -276,6 +276,7 @@ test("terminal hub routes multiplex frames and explicit output acknowledgements" ok: true, version: 2, multiplex: true, + inputAcknowledgements: true, }); server.emit("message", { From 7317e0fdd5fb8c5012c537730c6804484b73637c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:17:06 +0200 Subject: [PATCH 024/242] fix(actions): fence blocked work states --- src/worker/github-actions-repository.ts | 2 +- tests/github-actions-repository.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/worker/github-actions-repository.ts b/src/worker/github-actions-repository.ts index 1bf72ad2..f0be6f3c 100644 --- a/src/worker/github-actions-repository.ts +++ b/src/worker/github-actions-repository.ts @@ -13,7 +13,7 @@ type GitHubActionsSessionUpdate = | GitHubActionsWorkStateUpdate | GitHubActionsRunnerConnectionUpdate; -const terminalWorkStates = ["completed", "failed", "canceled"]; +const terminalWorkStates = ["completed", "failed", "canceled", "blocked"]; const terminalSessionStatuses = ["stopped", "expired", "failed"] as const; export class GitHubActionsRepository { diff --git a/tests/github-actions-repository.test.ts b/tests/github-actions-repository.test.ts index 35dd5209..5de33ae1 100644 --- a/tests/github-actions-repository.test.ts +++ b/tests/github-actions-repository.test.ts @@ -93,7 +93,9 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = assert.match(executions[3].sql, /"owner_subject" = \?/i); assert.doesNotMatch(executions[3].sql, /"work_state" not in/i); assert.match(executions[4].sql, /"work_state" not in/i); + assert.ok(executions[4].parameters.includes("blocked")); assert.match(executions[5].sql, /"status" not in/i); + assert.ok(executions[5].parameters.includes("blocked")); }); test("GitHub Actions repository rejects stale or invalid state transitions", async () => { From 0a81c8130c6ec0a063f536ec84de1eca8a476790 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:17:15 +0200 Subject: [PATCH 025/242] fix(http): validate deeply nested JSON iteratively --- src/worker/http.ts | 30 +++++++++++++++++------------- tests/http.test.ts | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/worker/http.ts b/src/worker/http.ts index 7de0a82b..26463928 100644 --- a/src/worker/http.ts +++ b/src/worker/http.ts @@ -178,19 +178,23 @@ function clean(value: unknown, maximum: number): string { } function assertRoundTrippableJsonIntegers(value: unknown): void { - if (typeof value === "number") { - if ( - !Number.isFinite(value) || - (Number.isInteger(value) && (!Number.isSafeInteger(value) || Object.is(value, -0))) - ) { - throw badRequest("json integers must be safe and round-trippable"); + const pending = [value]; + while (pending.length > 0) { + const current = pending.pop(); + if (typeof current === "number") { + if ( + !Number.isFinite(current) || + (Number.isInteger(current) && (!Number.isSafeInteger(current) || Object.is(current, -0))) + ) { + throw badRequest("json integers must be safe and round-trippable"); + } + continue; } - return; - } - if (!value || typeof value !== "object") return; - if (Array.isArray(value)) { - for (const item of value) assertRoundTrippableJsonIntegers(item); - return; + if (!current || typeof current !== "object") continue; + if (Array.isArray(current)) { + for (const item of current) pending.push(item); + continue; + } + for (const item of Object.values(current)) pending.push(item); } - for (const item of Object.values(value)) assertRoundTrippableJsonIntegers(item); } diff --git a/tests/http.test.ts b/tests/http.test.ts index f550595e..4b480d50 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -135,6 +135,20 @@ test("JSON parsing rejects integers that cannot round-trip exactly", async () => } }); +test("JSON parsing handles deeply nested bounded payloads without exhausting the call stack", async () => { + const depth = 20_000; + const body = `${"[".repeat(depth)}0${"]".repeat(depth)}`; + let current = await readBoundedJson( + new Request("https://fleet.example", { method: "POST", body }), + body.length, + ); + for (let index = 0; index < depth; index += 1) { + assert.ok(Array.isArray(current)); + current = current[0]; + } + assert.equal(current, 0); +}); + test("bearer and cookie helpers normalize only their owned protocol surface", () => { assert.equal( bearerToken( From a897983afbf753f5bf2dc9d448a306b56a300fb3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:20:27 +0200 Subject: [PATCH 026/242] fix(vnc): preserve ASCII keysyms in composed input --- .../Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift | 13 +++++++------ .../RoyalVNCKitTests/SecurityAndInputTests.swift | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift index 357b3adb..ef9659c4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Input/VNCKeyCode.swift @@ -159,12 +159,13 @@ public extension VNCKeyCode { var codes = [VNCKeyCode]() - for scalar in character.unicodeScalars { - let unicodeValue = scalar.value - let keySym = - (0x00a0...0x00ff).contains(unicodeValue) - ? unicodeValue - : 0x0100_0000 | unicodeValue + for scalar in character.unicodeScalars { + let unicodeValue = scalar.value + let keySym = + (0x0020...0x007e).contains(unicodeValue) + || (0x00a0...0x00ff).contains(unicodeValue) + ? unicodeValue + : 0x0100_0000 | unicodeValue codes.append(.init(keySym)) } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift index 3be4f6cf..a665605c 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -54,6 +54,9 @@ struct SecurityAndInputTests { #expect(VNCKeyCode.withCharacter("é").map(\.rawValue) == [0xe9]) #expect(VNCKeyCode.withCharacter("α").map(\.rawValue) == [0x0100_03b1]) #expect(VNCKeyCode.withCharacter("🦀").map(\.rawValue) == [0x0101_f980]) + #expect( + VNCKeyCode.withCharacter("e\u{301}").map(\.rawValue) == [0x65, 0x0100_0301] + ) } @Test From aca2537a648074387320a290cc1d2ddebe7be590 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:20:39 +0200 Subject: [PATCH 027/242] fix(runtime): preserve opaque fixed profile ids --- src/runtime-profiles.ts | 2 +- tests/runtime-profiles.test.ts | 51 ++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/runtime-profiles.ts b/src/runtime-profiles.ts index b16e9bba..7181adab 100644 --- a/src/runtime-profiles.ts +++ b/src/runtime-profiles.ts @@ -32,7 +32,7 @@ export type RuntimeProfileCodexSshValues = { profile: string; }; -const profileIDPattern = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; +const profileIDPattern = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,118}[A-Za-z0-9])?$/; const targetPattern = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,38}[A-Za-z0-9])?$/; const capabilityNames = ["terminal", "takeover", "vnc", "desktop", "logs", "artifacts"] as const; const capabilityNameSet = new Set(capabilityNames); diff --git a/tests/runtime-profiles.test.ts b/tests/runtime-profiles.test.ts index 7a9c7f28..110a8ce6 100644 --- a/tests/runtime-profiles.test.ts +++ b/tests/runtime-profiles.test.ts @@ -12,6 +12,7 @@ import { runtimeProfileByID, runtimeProfileCapabilities, } from "../src/runtime-profiles.ts"; +import { runtimeAdapterControlPlaneForProfile } from "../src/runtime-adapter.ts"; test("runtime profile catalog preserves generic labels, targets, and capabilities", () => { const profiles = parseRuntimeProfiles( @@ -69,10 +70,10 @@ test("runtime profile catalog fails closed on malformed or ambiguous input", () '[{"id":"a","label":"A","capabilities":null}]', '[{"id":"a","label":"A","capabilities":{"unknown":true}}]', '[{"id":"a","label":"A","privateProvider":"hidden"}]', - '[{"id":"Desktop","label":"Desktop"}]', - '[{"id":"desktop.profile","label":"Desktop"}]', - '[{"id":"desktop_profile","label":"Desktop"}]', - `[{"id":"${"a".repeat(64)}","label":"Desktop"}]`, + '[{"id":"profile/escape","label":"Desktop"}]', + '[{"id":"-profile","label":"Desktop"}]', + '[{"id":"profile.","label":"Desktop"}]', + `[{"id":"${"a".repeat(121)}","label":"Desktop"}]`, '[{"id":"a","label":"A","codexSsh":null}]', '[{"id":"a","label":"A","codexSsh":{"aliasTemplate":"box {sessionId}"}}]', '[{"id":"a","label":"A","codexSsh":{"aliasTemplate":"box-{unknown}"}}]', @@ -86,13 +87,51 @@ test("runtime profile catalog fails closed on malformed or ambiguous input", () assert.throws(() => parseRuntimeProfiles(value)); } assert.equal( - parseRuntimeProfiles(JSON.stringify([{ id: "a".repeat(63), label: "Maximum" }]))[0]?.id.length, - 63, + parseRuntimeProfiles(JSON.stringify([{ id: `A${"_".repeat(118)}Z`, label: "Maximum" }]))[0]?.id + .length, + 120, ); assert.deepEqual(parseRuntimeProfiles(undefined), []); assert.deepEqual(parseRuntimeProfiles(""), []); }); +test("fixed adapters accept opaque profile ids without weakening profile-routed URLs", () => { + const profiles = parseRuntimeProfiles( + JSON.stringify([ + { id: "Desktop.PROFILE_2026", label: "Desktop" }, + { id: "desktop_profile", label: "Terminal" }, + ]), + ); + + for (const profile of profiles) { + assert.equal( + runtimeAdapterControlPlaneForProfile( + "https://adapter.example.test/base", + undefined, + profile.id, + ), + "https://adapter.example.test/base", + ); + assert.equal( + runtimeAdapterControlPlaneForProfile( + undefined, + "https://controller.example.test/adapters/{profile}", + profile.id, + ), + null, + ); + } + + assert.equal( + runtimeAdapterControlPlaneForProfile( + undefined, + "https://controller.example.test/adapters/{profile}", + "desktop-profile", + ), + "https://controller.example.test/adapters/desktop-profile", + ); +}); + test("runtime profiles resolve bounded Codex SSH handoff data", () => { const [profile] = parseRuntimeProfiles( JSON.stringify([ From 74e04a2bec23ea1df6364e184cf816d693e32f5d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:21:57 +0200 Subject: [PATCH 028/242] fix(runtime): clean reservation archives before rollback --- src/worker/openclaw-repository.ts | 48 +++++++++-- tests/openclaw-repository.test.ts | 133 ++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 6 deletions(-) diff --git a/src/worker/openclaw-repository.ts b/src/worker/openclaw-repository.ts index 11214318..035df816 100644 --- a/src/worker/openclaw-repository.ts +++ b/src/worker/openclaw-repository.ts @@ -3,6 +3,7 @@ import { sql } from "kysely"; import { database, executeBatch, type InteractiveSessionRow } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import { interactiveSession, type InteractiveSession } from "./session-model.ts"; +import { cleanupSessionLogArchiveObjects } from "./session-log-archive.ts"; export type OpenClawRoomSessions = { sessions: InteractiveSession[]; @@ -172,35 +173,70 @@ export async function removeInteractiveSessionReservation( insertedAt: number, ): Promise { const db = database(env); - const ownsReservation = sql`EXISTS ( + const rollbackClaim = `reservation-rollback:${insertedAt}`; + const rollbackClaimedAt = insertedAt + 1; + await db + .updateTable("interactive_sessions") + .set({ + reconcile_error: rollbackClaim, + updated_at: rollbackClaimedAt, + }) + .where("id", "=", insertedSessionId) + .where("status", "=", "provisioning") + .where("preparation_pending", "=", 1) + .where("created_at", "=", insertedAt) + .where("updated_at", "=", insertedAt) + .execute(); + const claimed = await db + .selectFrom("interactive_sessions") + .select("id") + .where("id", "=", insertedSessionId) + .where("status", "=", "provisioning") + .where("preparation_pending", "=", 1) + .where("created_at", "=", insertedAt) + .where("updated_at", "=", rollbackClaimedAt) + .where("reconcile_error", "=", rollbackClaim) + .executeTakeFirst(); + if (!claimed) return false; + + const archive = await db + .selectFrom("interactive_session_log_archives") + .select(["events_key", "transcript_key", "summary_key"]) + .where("session_id", "=", insertedSessionId) + .executeTakeFirst(); + await cleanupSessionLogArchiveObjects(env, archive); + + const ownsRollbackClaim = sql`EXISTS ( SELECT 1 FROM interactive_sessions WHERE id = ${insertedSessionId} AND status = 'provisioning' AND preparation_pending = 1 AND created_at = ${insertedAt} - AND updated_at = ${insertedAt} + AND updated_at = ${rollbackClaimedAt} + AND reconcile_error = ${rollbackClaim} )`; await executeBatch(env, [ db .deleteFrom("openclaw_request_replays") .where("session_id", "=", insertedSessionId) - .where(ownsReservation), + .where(ownsRollbackClaim), db .deleteFrom("interactive_session_events") .where("session_id", "=", insertedSessionId) - .where(ownsReservation), + .where(ownsRollbackClaim), db .deleteFrom("interactive_session_log_archives") .where("session_id", "=", insertedSessionId) - .where(ownsReservation), + .where(ownsRollbackClaim), db .deleteFrom("interactive_sessions") .where("id", "=", insertedSessionId) .where("status", "=", "provisioning") .where("preparation_pending", "=", 1) .where("created_at", "=", insertedAt) - .where("updated_at", "=", insertedAt), + .where("updated_at", "=", rollbackClaimedAt) + .where("reconcile_error", "=", rollbackClaim), ]); const current = await db .selectFrom("interactive_sessions") diff --git a/tests/openclaw-repository.test.ts b/tests/openclaw-repository.test.ts index 7ed39176..700c25d9 100644 --- a/tests/openclaw-repository.test.ts +++ b/tests/openclaw-repository.test.ts @@ -30,6 +30,7 @@ type PreparedStatement = { function runtimeEnv( handler: D1Handler, batchHandler: (statements: PreparedStatement[]) => void = () => undefined, + sessionLogs?: Pick, ): RuntimeEnv { return { DB: { @@ -57,6 +58,7 @@ function runtimeEnv( return []; }, } as unknown as D1Database, + ...(sessionLogs ? { SESSION_LOGS: sessionLogs as R2Bucket } : {}), } as RuntimeEnv; } @@ -268,23 +270,68 @@ test("OpenClaw stale reservation reads are bounded and map persistence names", a test("OpenClaw reservation rollback deletes all owned records in one batch", async () => { let batch: PreparedStatement[] = []; + const deletedKeys: string[] = []; const removed = await removeInteractiveSessionReservation( runtimeEnv( (sql, parameters, kind) => { + if (/^update "interactive_sessions"/i.test(sql)) { + assert.equal(kind, "run"); + assert.match(sql, /set "reconcile_error" = .+, "updated_at" =/i); + assert.match(sql, /"updated_at" =/i); + assert.deepEqual(parameters, [ + "reservation-rollback:100", + 101, + "IS-2", + "provisioning", + 1, + 100, + 100, + ]); + return { changes: 1 }; + } assert.equal(kind, "all"); + if (/from "interactive_session_log_archives"/i.test(sql)) { + assert.deepEqual(parameters, ["IS-2"]); + return { + results: [ + { + events_key: "events", + transcript_key: "transcript", + summary_key: "summary", + }, + ], + }; + } assert.match(sql, /^select "id" from "interactive_sessions"/i); + if (parameters.length > 1) { + assert.deepEqual(parameters, [ + "IS-2", + "provisioning", + 1, + 100, + 101, + "reservation-rollback:100", + ]); + return { results: [{ id: "IS-2" }] }; + } assert.deepEqual(parameters, ["IS-2"]); return { results: [] }; }, (statements) => { batch = statements; }, + { + async delete(key) { + deletedKeys.push(key); + }, + }, ), "IS-2", 100, ); assert.equal(removed, true); + assert.deepEqual(deletedKeys.sort(), ["events", "summary", "transcript"]); assert.equal(batch.length, 4); assert.match(batch[0]?.sql ?? "", /^delete from "openclaw_request_replays"/i); assert.match(batch[1]?.sql ?? "", /^delete from "interactive_session_events"/i); @@ -292,6 +339,92 @@ test("OpenClaw reservation rollback deletes all owned records in one batch", asy assert.match(batch[3]?.sql ?? "", /^delete from "interactive_sessions"/i); assert.ok(batch.every((statement) => statement.parameters.includes("IS-2"))); assert.ok(batch.every((statement) => statement.parameters.includes(100))); + assert.ok(batch.every((statement) => statement.parameters.includes(101))); + assert.ok(batch.every((statement) => statement.parameters.includes("reservation-rollback:100"))); +}); + +test("OpenClaw reservation rollback retains its durable claim when archive cleanup fails", async () => { + let batched = false; + await assert.rejects( + removeInteractiveSessionReservation( + runtimeEnv( + (sql, _parameters, kind) => { + if (/^update "interactive_sessions"/i.test(sql)) { + assert.equal(kind, "run"); + return { changes: 1 }; + } + assert.equal(kind, "all"); + if (/from "interactive_session_log_archives"/i.test(sql)) { + return { + results: [ + { + events_key: "events", + transcript_key: "transcript", + summary_key: "summary", + }, + ], + }; + } + return { results: [{ id: "IS-2" }] }; + }, + () => { + batched = true; + }, + { + async delete() { + throw new Error("R2 unavailable"); + }, + }, + ), + "IS-2", + 100, + ), + /R2 unavailable/, + ); + assert.equal(batched, false); +}); + +test("OpenClaw reservation rollback resumes an existing archive cleanup claim", async () => { + let batched = false; + const deletedKeys: string[] = []; + const removed = await removeInteractiveSessionReservation( + runtimeEnv( + (sql, _parameters, kind) => { + if (/^update "interactive_sessions"/i.test(sql)) { + assert.equal(kind, "run"); + return { changes: 0 }; + } + assert.equal(kind, "all"); + if (/from "interactive_session_log_archives"/i.test(sql)) { + return { + results: [ + { + events_key: "events", + transcript_key: "transcript", + summary_key: "summary", + }, + ], + }; + } + if (/reconcile_error/i.test(sql)) return { results: [{ id: "IS-2" }] }; + return { results: [] }; + }, + () => { + batched = true; + }, + { + async delete(key) { + deletedKeys.push(key); + }, + }, + ), + "IS-2", + 100, + ); + + assert.equal(removed, true); + assert.equal(batched, true); + assert.deepEqual(deletedKeys.sort(), ["events", "summary", "transcript"]); }); test("OpenClaw reservation activation reports the fenced compare-and-set result", async () => { From f461439cdd30a99d4a780807e2f466bb81176f6a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:22:04 +0200 Subject: [PATCH 029/242] fix(macos): preserve network and clipboard semantics --- .../CrabfleetMac/CrabboxVNCBridge.swift | 24 ++++++++++++++++- .../Sources/CrabfleetMac/HostClipboard.swift | 7 +++-- .../CrabfleetMac/SubprocessEnvironment.swift | 4 +++ .../CrabfleetMacTests/FleetModelsTests.swift | 10 +++++++ .../HostShareProtocolTests.swift | 27 +++++++++++++++++++ 5 files changed, 69 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift index dd35cf21..d47a25f2 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift @@ -36,6 +36,24 @@ private struct CrabboxVNCHandoff: Decodable { } final class CrabboxVNCBridge: @unchecked Sendable { + private static let networkEnvironmentKeys = [ + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + "no_proxy", + "AWS_CA_BUNDLE", + "CURL_CA_BUNDLE", + "GIT_SSL_CAINFO", + "NODE_EXTRA_CA_CERTS", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + ] + let request: VNCConnectionRequest private let process: Process @@ -242,7 +260,11 @@ final class CrabboxVNCBridge: @unchecked Sendable { } static func commandEnvironment(from source: [String: String]) -> [String: String] { - SubprocessEnvironment.minimal(from: source, includeSSHAgent: true) + SubprocessEnvironment.minimal( + from: source, + includeSSHAgent: true, + additionalInheritedKeys: networkEnvironmentKeys + ) } private static func drain(_ pipe: Pipe) { diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift index 02fb4246..fe6cf044 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift @@ -100,6 +100,7 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { let previous = withLock { lastObservedChangeCount } guard changeCount != previous else { return } + let types = pasteboard.types ?? [] let text = pasteboard.string(forType: .string) guard pasteboard.changeCount == changeCount else { return } @@ -107,13 +108,15 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { var pushHandler: (@Sendable (String) -> Void)? withLock { lastObservedChangeCount = changeCount - lastKnownText = text if suppressedChangeCount == changeCount { suppressedChangeCount = nil return } - let outboundText = text ?? "" + guard let outboundText = text ?? (types.isEmpty ? "" : nil) else { + return + } + lastKnownText = text guard outboundText.utf8.count <= RFBWire.maximumClipboardBytes else { return } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift index c5c85c1d..645810ca 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift @@ -14,6 +14,7 @@ enum SubprocessEnvironment { static func minimal( from source: [String: String], includeSSHAgent: Bool = false, + additionalInheritedKeys: [String] = [], overrides: [String: String] = [:] ) -> [String: String] { var environment = Dictionary( @@ -21,6 +22,9 @@ enum SubprocessEnvironment { source[key].map { (key, $0) } } ) + for key in additionalInheritedKeys { + environment[key] = source[key] + } environment["PATH"] = safePath if includeSSHAgent, let socket = source["SSH_AUTH_SOCK"], !socket.isEmpty { environment["SSH_AUTH_SOCK"] = socket diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift index 43f3dc62..c35a7539 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift @@ -23,14 +23,24 @@ struct FleetModelsTests { "HOME": "/Users/tester", "PATH": "/tmp/untrusted", "SSH_AUTH_SOCK": "/tmp/agent.sock", + "HTTPS_PROXY": "http://proxy.example.test:8443", + "NO_PROXY": "localhost,.example.test", + "SSL_CERT_FILE": "/etc/ssl/custom-ca.pem", + "SSL_CERT_DIR": "/etc/ssl/custom-certs", "CRABFLEET_SESSION_COOKIE": "secret", + "NODE_TLS_REJECT_UNAUTHORIZED": "0", ] ) #expect(environment["HOME"] == "/Users/tester") #expect(environment["PATH"] == SubprocessEnvironment.safePath) #expect(environment["SSH_AUTH_SOCK"] == "/tmp/agent.sock") + #expect(environment["HTTPS_PROXY"] == "http://proxy.example.test:8443") + #expect(environment["NO_PROXY"] == "localhost,.example.test") + #expect(environment["SSL_CERT_FILE"] == "/etc/ssl/custom-ca.pem") + #expect(environment["SSL_CERT_DIR"] == "/etc/ssl/custom-certs") #expect(environment["CRABFLEET_SESSION_COOKIE"] == nil) + #expect(environment["NODE_TLS_REJECT_UNAUTHORIZED"] == nil) } @Test diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift index f6145541..54f797ed 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift @@ -179,6 +179,33 @@ struct HostClipboardBridgeTests { bridge.detach() } + @Test + func ignoresNonTextChangesButForwardsEmptyText() async throws { + let pasteboard = NSPasteboard(name: .init("CrabfleetMacTests.\(UUID().uuidString)")) + pasteboard.clearContents() + pasteboard.setString("initial", forType: .string) + + let recorder = PushRecorder() + let bridge = HostClipboardBridge(pasteboard: pasteboard, pollingInterval: 0.01) + bridge.attach { recorder.append($0) } + try await Task.sleep(for: .milliseconds(30)) + + let item = NSPasteboardItem() + item.setData(Data([0x00, 0x01]), forType: .init("com.example.crabfleet.binary")) + pasteboard.clearContents() + pasteboard.writeObjects([item]) + bridge.poll() + #expect(recorder.values.isEmpty) + #expect(bridge.currentText() == "initial") + + pasteboard.clearContents() + pasteboard.setString("", forType: .string) + bridge.poll() + #expect(recorder.values == [""]) + #expect(bridge.currentText() == "") + bridge.detach() + } + private func waitUntil( timeout: Duration = .seconds(1), condition: @escaping @MainActor () -> Bool From ceabb9b44da201a65f01e35eb69e444bef83c4e5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:23:05 +0200 Subject: [PATCH 030/242] fix(terminal): keep established sockets alive --- internal/terminalws/client.go | 48 +++++++++++++++++++++--- internal/terminalws/client_test.go | 60 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 42a958ce..4ba19248 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/coder/websocket" ) @@ -77,6 +78,7 @@ type Client struct { conn *websocket.Conn sessionID string supportsInputAcknowledgement bool + cancel context.CancelFunc canInput atomic.Bool lastSize atomic.Uint64 writeMu sync.Mutex @@ -128,16 +130,33 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option if sessionID == "" { return nil, errors.New("terminal session id is required") } + httpClient := options.HTTPClient + var setupCancel context.CancelFunc + var setupTimer *time.Timer + var setupFinished atomic.Bool if options.HTTPClient != nil && options.HTTPClient.Timeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, options.HTTPClient.Timeout) - defer cancel() + cloned := *options.HTTPClient + cloned.Timeout = 0 + httpClient = &cloned + ctx, setupCancel = context.WithCancel(ctx) + setupTimer = time.AfterFunc(options.HTTPClient.Timeout, func() { + if setupFinished.CompareAndSwap(false, true) { + setupCancel() + } + }) } conn, resp, err := websocket.Dial(ctx, endpoint, &websocket.DialOptions{ - HTTPClient: options.HTTPClient, + HTTPClient: httpClient, HTTPHeader: options.Header, }) if err != nil { + setupFinished.Store(true) + if setupTimer != nil { + setupTimer.Stop() + } + if setupCancel != nil { + setupCancel() + } if resp != nil { body := "" if resp.Body != nil { @@ -150,9 +169,16 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option return nil, err } conn.SetReadLimit(maxFrameBytes) - client := &Client{conn: conn, sessionID: sessionID} + client := &Client{conn: conn, sessionID: sessionID, cancel: setupCancel} client.rememberSize(Size{Cols: options.Cols, Rows: options.Rows}) closeWithError := func(err error) (*Client, error) { + setupFinished.Store(true) + if setupTimer != nil { + setupTimer.Stop() + } + if setupCancel != nil { + setupCancel() + } _ = conn.Close(websocket.StatusInternalError, "terminal setup failed") return nil, err } @@ -189,7 +215,13 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option return closeWithError(fmt.Errorf("decode terminal event: %w", err)) } if event.Type == "subscribed" { + if setupTimer != nil && !setupFinished.CompareAndSwap(false, true) { + return closeWithError(errors.New("terminal subscription timed out")) + } client.canInput.Store(event.CanInput) + if setupTimer != nil { + setupTimer.Stop() + } return client, nil } if event.Type == "closed" { @@ -200,7 +232,11 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option } func (c *Client) Close() error { - return c.conn.Close(websocket.StatusNormalClosure, "") + err := c.conn.Close(websocket.StatusNormalClosure, "") + if c.cancel != nil { + c.cancel() + } + return err } func (c *Client) SendInput(ctx context.Context, payload []byte) error { diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index ac43e0f3..a583f0a7 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -432,6 +432,66 @@ func TestDialUsesConfiguredHTTPClientAndTimeout(t *testing.T) { } } +func TestDialDoesNotApplyHTTPClientTimeoutToEstablishedConnection(t *testing.T) { + receivedInput := make(chan []byte, 1) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-established", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + current, err := decodeFrame(payload) + if err != nil { + t.Error(err) + return + } + receivedInput <- append([]byte(nil), current.payload...) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + httpClient := server.Client() + httpClient.Timeout = 25 * time.Millisecond + client, err := Dial(context.Background(), endpoint, "IS-established", Options{ + HTTPClient: httpClient, + }) + if err != nil { + t.Fatal(err) + } + defer client.Close() + time.Sleep(2 * httpClient.Timeout) + if err := client.SendInput(context.Background(), []byte("still-open\n")); err != nil { + t.Fatal(err) + } + if input := <-receivedInput; string(input) != "still-open\n" { + t.Fatalf("input = %q", input) + } +} + func TestAttachClosesCloseableTerminalAfterRemoteClosure(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) From fc4862d6843dc2106f2d67f8f1f48aadd22999d2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:23:24 +0200 Subject: [PATCH 031/242] fix(macos): serialize private share lifecycle --- .../CrabfleetDesktopRegistration.swift | 39 ++++++ .../PrivateMacShareController.swift | 44 ++++-- .../PrivateMacShareTests.swift | 128 ++++++++++++++++++ 3 files changed, 202 insertions(+), 9 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index 08a8d1f6..1cd410aa 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -5,6 +5,45 @@ protocol DesktopHostRegistering: Sendable { func unregister(identity: TailnetIdentity) async throws } +actor DesktopHostRegistrationCoordinator { + private let registration: any DesktopHostRegistering + private var pendingOperation: Task? + + init(registration: any DesktopHostRegistering) { + self.registration = registration + } + + func register(identity: TailnetIdentity, port: UInt16) async throws { + let registration = self.registration + let operation = enqueue { + try await registration.register(identity: identity, port: port) + } + try await operation.value + } + + func unregister(identity: TailnetIdentity) async throws { + let registration = self.registration + let operation = enqueue { + try await registration.unregister(identity: identity) + } + try await operation.value + } + + private func enqueue( + _ operation: @escaping @Sendable () async throws -> Void + ) -> Task { + let previous = pendingOperation + let task = Task { + await previous?.value + try await operation() + } + pendingOperation = Task { + _ = try? await task.value + } + return task + } +} + struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable { private struct RegistrationBody: Encodable { let name: String diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 8b99a097..25648047 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -100,6 +100,7 @@ final class PrivateMacShareController: ObservableObject { private let runner: (any TailscaleCommandRunning)? private let desktopRegistration: (any DesktopHostRegistering)? + private let desktopRegistrationCoordinator: DesktopHostRegistrationCoordinator? private let runnerInitializationError: Error? private let defaults: UserDefaults private var capture: MacScreenCapture? @@ -109,6 +110,7 @@ final class PrivateMacShareController: ObservableObject { private var lifecycleGeneration: UInt64 = 0 private var serverGeneration: UInt64? private var registrationTask: Task? + private var refreshWaiters: [CheckedContinuation] = [] init( runner: (any TailscaleCommandRunning)? = nil, @@ -116,6 +118,9 @@ final class PrivateMacShareController: ObservableObject { defaults: UserDefaults = .standard ) { self.desktopRegistration = desktopRegistration + desktopRegistrationCoordinator = desktopRegistration.map { + DesktopHostRegistrationCoordinator(registration: $0) + } self.defaults = defaults registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished let savedDisplayID = defaults.object(forKey: Self.selectedDisplayDefaultsKey) as? Int @@ -151,8 +156,13 @@ final class PrivateMacShareController: ObservableObject { } func refresh() async { - guard !isRefreshing, phase != .starting, phase != .stopping else { return } + if isRefreshing { + await waitForRefreshCompletion() + return + } + guard phase != .starting, phase != .stopping else { return } isRefreshing = true + defer { finishRefresh() } notice = nil do { identity = try await fetchIdentity() @@ -163,7 +173,6 @@ final class PrivateMacShareController: ObservableObject { refreshPermissions() await refreshDisplays() launchAtLoginEnabled = SMAppService.mainApp.status == .enabled - isRefreshing = false } /// Registers or removes the login item and remembers to auto-start the @@ -216,7 +225,9 @@ final class PrivateMacShareController: ObservableObject { } func start() async { - guard phase == .idle, !isRefreshing else { return } + guard phase == .idle else { return } + await waitForRefreshCompletion() + guard phase == .idle, !Task.isCancelled else { return } let generation = beginLifecycleTransition() phase = .starting notice = nil @@ -303,7 +314,6 @@ final class PrivateMacShareController: ObservableObject { streamStats = nil let registrationTask = self.registrationTask self.registrationTask = nil - registrationTask?.cancel() serverGeneration = nil server?.stop() server = nil @@ -314,9 +324,9 @@ final class PrivateMacShareController: ObservableObject { await capture?.stop() await registrationTask?.value var removedRegistryEntry = true - if let desktopRegistration, let activeIdentity { + if let desktopRegistrationCoordinator, let activeIdentity { do { - try await desktopRegistration.unregister(identity: activeIdentity) + try await desktopRegistrationCoordinator.unregister(identity: activeIdentity) registryPhase = .notPublished } catch { removedRegistryEntry = false @@ -382,6 +392,22 @@ final class PrivateMacShareController: ObservableObject { } } + private func waitForRefreshCompletion() async { + guard isRefreshing else { return } + await withCheckedContinuation { continuation in + refreshWaiters.append(continuation) + } + } + + private func finishRefresh() { + isRefreshing = false + let waiters = refreshWaiters + refreshWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + } + private func handle(_ event: TailnetRFBServerEvent, generation: UInt64) { guard serverGeneration == generation else { return } switch event { @@ -429,15 +455,15 @@ final class PrivateMacShareController: ObservableObject { } private func registerDesktopHost(generation: UInt64) { - registrationTask?.cancel() - guard let desktopRegistration, let identity = activeIdentity else { + guard registrationTask == nil else { return } + guard let desktopRegistrationCoordinator, let identity = activeIdentity else { registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } registryPhase = .registering registrationTask = Task { [weak self] in do { - try await desktopRegistration.register(identity: identity, port: Self.port) + try await desktopRegistrationCoordinator.register(identity: identity, port: Self.port) guard !Task.isCancelled, self?.serverGeneration == generation else { return } self?.registryPhase = .registered } catch is CancellationError { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 5b99801b..e9cba8ce 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -124,6 +124,72 @@ struct PrivateMacShareTests { #expect(controller.identity == nil) } + @Test @MainActor + func startWaitsForAnInFlightRefresh() async throws { + let runner = SequencedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: nil, + defaults: defaults + ) + + let refreshTask = Task { await controller.refresh() } + #expect(await waitUntilAsync { await runner.callCount == 1 }) + #expect(controller.isRefreshing) + + let startState = AsyncInvocationState() + let startTask = Task { + await startState.markStarted() + await controller.start() + await startState.markFinished() + } + #expect(await waitUntilAsync { await startState.started }) + try await Task.sleep(for: .milliseconds(20)) + #expect(!(await startState.finished)) + + await runner.resumeNext( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + await refreshTask.value + #expect(await waitUntilAsync { await runner.callCount == 2 }) + await runner.resumeNext(.failure(PrivateMacShareError.tailscaleOffline)) + await startTask.value + + #expect(controller.phase == .failed) + #expect(await runner.callCount == 2) + } + + @Test + func desktopRemovalWaitsForACommittedRegistrationAfterCancellation() async throws { + let registration = SuspendedDesktopRegistration() + let coordinator = DesktopHostRegistrationCoordinator(registration: registration) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + let publish = Task { + try await coordinator.register(identity: identity, port: 5_901) + } + #expect(await waitUntilAsync { await registration.hasStartedRegistration }) + publish.cancel() + + let remove = Task { + try await coordinator.unregister(identity: identity) + } + try await Task.sleep(for: .milliseconds(20)) + #expect(await registration.events == [.registerStarted]) + + await registration.finishRegistration() + try await publish.value + try await remove.value + + #expect( + await registration.events + == [.registerStarted, .registerFinished, .unregisterStarted] + ) + } + @Test @MainActor func applicationDelegateOwnsTheShareControllerUsedByTheApp() throws { let defaults = try #require( @@ -812,6 +878,68 @@ private actor SuspendedTailscaleRunner: TailscaleCommandRunning { } } +private actor SequencedTailscaleRunner: TailscaleCommandRunning { + private var continuations: [CheckedContinuation] = [] + private(set) var callCount = 0 + + func run(arguments: [String]) async throws -> TailscaleCommandResult { + callCount += 1 + return try await withCheckedThrowingContinuation { continuation in + continuations.append(continuation) + } + } + + func resumeNext(_ result: Result) { + guard !continuations.isEmpty else { return } + continuations.removeFirst().resume(with: result) + } +} + +private actor AsyncInvocationState { + private(set) var started = false + private(set) var finished = false + + func markStarted() { + started = true + } + + func markFinished() { + finished = true + } +} + +private actor SuspendedDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case registerStarted + case registerFinished + case unregisterStarted + } + + private var registrationContinuation: CheckedContinuation? + private(set) var events: [Event] = [] + + var hasStartedRegistration: Bool { + registrationContinuation != nil + } + + func register(identity: TailnetIdentity, port: UInt16) async throws { + events.append(.registerStarted) + await withCheckedContinuation { continuation in + registrationContinuation = continuation + } + events.append(.registerFinished) + } + + func unregister(identity: TailnetIdentity) async throws { + events.append(.unregisterStarted) + } + + func finishRegistration() { + registrationContinuation?.resume() + registrationContinuation = nil + } +} + private final class RemoteInputRecorder: RemoteInputForwarding, @unchecked Sendable { private let lock = NSLock() private var releases = 0 From da26a144a252da3c90eb06cea21f6b2ccb2e4770 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:23:35 +0200 Subject: [PATCH 032/242] docs(changelog): expand audit compatibility notes --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f074008..f8b2da9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and grant revocation, revision-fenced GitHub Actions updates, rollback-safe creation evidence, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, acknowledge one-shot CLI input only after the Worker accepts it, and send attributed commands atomically to prevent interleaving. -- Reject unroutable runtime profile identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations, stuck remote input, and lost clipboard clears or reused values. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication, Unicode keysyms, Tight and ZRLE parsing, bounded zlib streams, color-depth transitions, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden session lifecycle concurrency with atomic card claims and grant revocation, revision-fenced GitHub Actions updates, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, negotiate one-shot input acknowledgements across rolling upgrades, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. +- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations, dropped auto-starts, stuck remote input, and destructive non-text clipboard changes while preserving proxy and custom-CA networking. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, color-depth transitions, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From d9bf41fe99e8b729a94babdcbceab767f3298ed7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:28:00 +0200 Subject: [PATCH 033/242] fix(macos): refresh clipboard state before dedupe --- .../Sources/CrabfleetMac/HostClipboard.swift | 12 ++++++++++-- .../CrabfleetMacTests/HostShareProtocolTests.swift | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift index fe6cf044..162ca617 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift @@ -78,8 +78,16 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { guard text.utf8.count <= RFBWire.maximumClipboardBytes else { return } DispatchQueue.main.async { [weak self] in guard let self else { return } - let alreadyCurrent = self.withLock { self.lastKnownText == text } - if alreadyCurrent { return } + let changeCount = self.pasteboard.changeCount + let currentText = self.pasteboard.string(forType: .string) + guard self.pasteboard.changeCount == changeCount else { return } + if currentText == text { + self.withLock { + self.lastObservedChangeCount = changeCount + self.lastKnownText = text + } + return + } self.pasteboard.clearContents() guard self.pasteboard.setString(text, forType: .string) else { return } self.withLock { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift index 54f797ed..10274849 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift @@ -198,6 +198,10 @@ struct HostClipboardBridgeTests { #expect(recorder.values.isEmpty) #expect(bridge.currentText() == "initial") + bridge.receiveClientText("initial") + try await waitUntil { pasteboard.string(forType: .string) == "initial" } + #expect(bridge.currentText() == "initial") + pasteboard.clearContents() pasteboard.setString("", forType: .string) bridge.poll() From 39e0ee4e75fc75a0e3b244bf57c9619ad32ecf1c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:30:07 +0200 Subject: [PATCH 034/242] fix(vnc): validate authentication and format transitions --- ...NCMSLogonIIDiffieHellmanKeyAgreement.swift | 21 +++++++++++----- .../SDK/Connection/VNCConnection+API.swift | 18 +++++++------- .../RoyalVNCKitTests/AuditFindingsTests.swift | 11 ++++++++- .../SecurityAndInputTests.swift | 24 +++++++++++++++++++ 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift index e068c791..8fa45cd4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift @@ -44,13 +44,15 @@ private extension VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAg static func generateKeyPair(generator: Data, modulus: Data) -> KeyPair? { let generatorNum = UltraVNCBigNum.dataToBigNum(generator) - guard generatorNum < maxNum else { return nil } - let modulusNum = UltraVNCBigNum.dataToBigNum(modulus) - guard modulusNum < maxNum else { return nil } + guard modulusNum > 3, + modulusNum < maxNum, + generatorNum > 1, + generatorNum < modulusNum else { + return nil + } - let privNum = UltraVNCBigNum.randomBigNum(max: .init(maxNum)) - guard privNum < maxNum else { return nil } + let privNum = UInt64.random(in: 2..<(modulusNum - 1)) let privData = UltraVNCBigNum.bigNumToData(privNum) @@ -73,7 +75,14 @@ private extension VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAg let modulusNum = UltraVNCBigNum.dataToBigNum(modulus) let respNum = UltraVNCBigNum.dataToBigNum(resp) - guard respNum < maxNum else { return nil } + guard modulusNum > 3, + modulusNum < maxNum, + privNum > 1, + privNum < modulusNum, + respNum > 1, + respNum < modulusNum else { + return nil + } let keyNum = UltraVNCBigNum.powM64(b: .init(respNum), e: .init(privNum), diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 8deab894..86a1ba0a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -6,12 +6,14 @@ import Foundation private struct PixelFormatTransitionMessage: VNCSendableMessage { let pixelFormatMessage: VNCProtocol.SetPixelFormat + let willSend: () -> Void let didSend: () -> Void var messageType: UInt8 { pixelFormatMessage.messageType } var data: Data { pixelFormatMessage.data } func send(connection: NetworkConnectionWriting) async throws { + willSend() try await pixelFormatMessage.send(connection: connection) didSend() } @@ -111,17 +113,18 @@ private extension VNCConnection { func enqueuePixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { let message = PixelFormatTransitionMessage( - pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat) + pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat), + willSend: { [weak self] in + self?.beginPixelFormatTransition(pixelFormat) + } ) { [weak self] in - self?.applyPixelFormatTransition(pixelFormat) + self?.completePixelFormatTransition() } enqueueClientToServerMessage(message) } - func applyPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { - var didApply = false - + func beginPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { withLifecycleLock { guard connectionState.status == .connected, let framebuffer = framebuffer else { @@ -132,11 +135,10 @@ private extension VNCConnection { recreateFramebuffer(size: framebuffer.size, screens: framebuffer.screens, pixelFormat: pixelFormat) - didApply = connectionState.status == .connected } + } - guard didApply else { return } - + func completePixelFormatTransition() { framebufferRequestLock.lock() isPixelFormatTransitionInFlight = false let nextTransition = takePendingPixelFormatTransitionLocked() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 5fb68c30..15d67288 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -121,7 +121,10 @@ struct AuditFindingsTests { connection.completeFramebufferUpdateRequest() let queued = try #require(connection.clientToServerMessageQueue.dequeue()) - let writer = AuditWritingConnection() + let writer = AuditWritingConnection { + #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) + } try await queued.message.send(connection: writer) #expect(writer.data.count == 20) @@ -268,8 +271,14 @@ private final class AuditBufferConnection: NetworkConnectionReading { private final class AuditWritingConnection: NetworkConnectionWriting { var data = Data() + private let onWrite: () -> Void + + init(onWrite: @escaping () -> Void = {}) { + self.onWrite = onWrite + } func write(data: Data) async throws { + onWrite() self.data.append(data) } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift index a665605c..32e60f9c 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -48,6 +48,30 @@ struct SecurityAndInputTests { ) } + @Test + func rejectsDegenerateUltraVNCKeyAgreementParameters() { + let eightBytes: (UInt64) -> Data = { value in + withUnsafeBytes(of: value.bigEndian) { Data($0) } + } + + for (generator, modulus, response) in [ + (2, 0, 3), + (2, 1, 3), + (1, 17, 3), + (17, 17, 3), + (2, 17, 1), + (2, 17, 17), + ] { + #expect( + VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAgreement( + generator: eightBytes(UInt64(generator)), + modulus: eightBytes(UInt64(modulus)), + resp: eightBytes(UInt64(response)) + ) == nil + ) + } + } + @Test func encodesCharactersAsX11KeySyms() { #expect(VNCKeyCode.withCharacter("A").map(\.rawValue) == [0x41]) From 01ab3ddd4a50366a8f8f5eab47a1cedcdda369bb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:32:33 +0200 Subject: [PATCH 035/242] fix(actions): fence registration against read state --- src/worker/github-actions-application.ts | 2 +- src/worker/github-actions-repository.ts | 39 +++++++--- .../github-actions-session-registration.ts | 74 ++++++++++++------- tests/github-actions-repository.test.ts | 16 +++- ...ithub-actions-session-registration.test.ts | 17 ++++- 5 files changed, 103 insertions(+), 45 deletions(-) diff --git a/src/worker/github-actions-application.ts b/src/worker/github-actions-application.ts index 066682d4..60ca46a0 100644 --- a/src/worker/github-actions-application.ts +++ b/src/worker/github-actions-application.ts @@ -84,7 +84,7 @@ export class GitHubActionsApplication { nextSessionId: () => nextInteractiveSessionId(this.env), insertSession: (values) => repository.insertSession(values), readById: (id) => repository.readById(id), - updateSession: (id, values) => repository.updateSession(id, values), + updateSession: (id, values, expected) => repository.updateSession(id, values, expected), isConstraintError, disconnectRunner: (id) => this.disconnectRunner(id), appendEvent: (id, message, now) => this.appendMessageEvent(id, user, message, now), diff --git a/src/worker/github-actions-repository.ts b/src/worker/github-actions-repository.ts index f0be6f3c..0a598b8b 100644 --- a/src/worker/github-actions-repository.ts +++ b/src/worker/github-actions-repository.ts @@ -4,7 +4,10 @@ import type { InteractiveSessionRow, InteractiveSessionTable } from "./database. import { database } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import type { GitHubActionsRunnerConnectionUpdate } from "./github-actions-runner-connection.ts"; -import type { GitHubActionsSessionRegistrationUpdate } from "./github-actions-session-registration.ts"; +import type { + GitHubActionsSessionRegistrationExpectation, + GitHubActionsSessionRegistrationUpdate, +} from "./github-actions-session-registration.ts"; import type { GitHubActionsWorkStateUpdate } from "./github-actions-session-work-state.ts"; import { conflict } from "./http.ts"; @@ -47,25 +50,39 @@ export class GitHubActionsRepository { await database(this.env).insertInto("interactive_sessions").values(values).execute(); } - async updateSession(id: string, values: GitHubActionsSessionUpdate): Promise { + async updateSession( + id: string, + values: GitHubActionsSessionUpdate, + expectedRegistration?: GitHubActionsSessionRegistrationExpectation, + ): Promise { let update = database(this.env) .updateTable("interactive_sessions") .set(values) .where("id", "=", id) - .where("runtime", "=", "github_actions") - .where("updated_at", "<=", values.updated_at); + .where("runtime", "=", "github_actions"); if (isRegistrationUpdate(values)) { - update = update.where("owner_subject", "=", values.owner_subject); + if (!expectedRegistration) { + throw new Error("GitHub Actions registration update requires expected state"); + } + update = update + .where("updated_at", "=", expectedRegistration.updated_at) + .where("status", "=", expectedRegistration.status) + .where("work_state", "=", expectedRegistration.work_state) + .where("work_phase", "=", expectedRegistration.work_phase) + .where("owner_subject", "=", values.owner_subject); } else if (isWorkStateUpdate(values) && terminalWorkStates.includes(values.work_state)) { - update = update.where((expressions) => - expressions.or([ - expressions("work_state", "not in", terminalWorkStates), - expressions("work_state", "=", values.work_state), - ]), - ); + update = update + .where("updated_at", "<=", values.updated_at) + .where((expressions) => + expressions.or([ + expressions("work_state", "not in", terminalWorkStates), + expressions("work_state", "=", values.work_state), + ]), + ); } else { update = update + .where("updated_at", "<=", values.updated_at) .where("work_state", "not in", terminalWorkStates) .where("status", "not in", terminalSessionStatuses); } diff --git a/src/worker/github-actions-session-registration.ts b/src/worker/github-actions-session-registration.ts index cf6b640d..7f024959 100644 --- a/src/worker/github-actions-session-registration.ts +++ b/src/worker/github-actions-session-registration.ts @@ -47,6 +47,11 @@ export type GitHubActionsSessionRegistrationUpdate = { completion_reason: null; }; +export type GitHubActionsSessionRegistrationExpectation = Pick< + InteractiveSessionRow, + "updated_at" | "status" | "work_state" | "work_phase" +>; + export type GitHubActionsSessionRegistrationStore = { now(): number; newAgentToken(): string; @@ -57,7 +62,11 @@ export type GitHubActionsSessionRegistrationStore = { nextSessionId(): Promise; insertSession(values: Insertable): Promise; readById(id: string): Promise; - updateSession(id: string, values: GitHubActionsSessionRegistrationUpdate): Promise; + updateSession( + id: string, + values: GitHubActionsSessionRegistrationUpdate, + expected: GitHubActionsSessionRegistrationExpectation, + ): Promise; isConstraintError(error: unknown): boolean; disconnectRunner(id: string): Promise; appendEvent(id: string, message: string, now: number): Promise; @@ -148,33 +157,42 @@ export class GitHubActionsSessionRegistrationService { const resumed = existing.work_state !== "registered" || existing.status !== "ready"; const message = resumed ? "GitHub Actions work resumed" : "GitHub Actions work registered"; - await this.store.updateSession(existing.id, { - owner, - owner_subject: ownerSubject, - repo, - branch, - purpose, - summary, - prompt: purpose, - status: "ready", - lease_id: null, - stopped_at: null, - terminal_status: null, - terminal_failure_reason: null, - terminal_finalize_pending: 0, - credential_cleanup_terminal_status: null, - updated_at: now, - last_seen_at: now, - last_event: message, - agent_token_hash: agentTokenHash, - work_kind: workKind, - work_state: "registered", - work_phase: "waiting_for_runner", - source_url: input.sourceUrl === undefined ? existing.source_url : sourceUrl, - github_run_url: input.runUrl === undefined ? existing.github_run_url : runUrl, - last_heartbeat_at: null, - completion_reason: null, - }); + await this.store.updateSession( + existing.id, + { + owner, + owner_subject: ownerSubject, + repo, + branch, + purpose, + summary, + prompt: purpose, + status: "ready", + lease_id: null, + stopped_at: null, + terminal_status: null, + terminal_failure_reason: null, + terminal_finalize_pending: 0, + credential_cleanup_terminal_status: null, + updated_at: now, + last_seen_at: now, + last_event: message, + agent_token_hash: agentTokenHash, + work_kind: workKind, + work_state: "registered", + work_phase: "waiting_for_runner", + source_url: input.sourceUrl === undefined ? existing.source_url : sourceUrl, + github_run_url: input.runUrl === undefined ? existing.github_run_url : runUrl, + last_heartbeat_at: null, + completion_reason: null, + }, + { + updated_at: existing.updated_at, + status: existing.status, + work_state: existing.work_state, + work_phase: existing.work_phase, + }, + ); await this.store.disconnectRunner(existing.id).catch(() => undefined); await this.store.appendEvent(existing.id, message, now); await this.store.audit( diff --git a/tests/github-actions-repository.test.ts b/tests/github-actions-repository.test.ts index 5de33ae1..f21290e2 100644 --- a/tests/github-actions-repository.test.ts +++ b/tests/github-actions-repository.test.ts @@ -68,7 +68,7 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = now: 100, }), ); - await repository.updateSession("IS-101", registrationUpdate); + await repository.updateSession("IS-101", registrationUpdate, registrationExpectation); await repository.updateSession("IS-101", workStateUpdate); await repository.updateSession("IS-101", runnerConnectionUpdate); @@ -87,9 +87,14 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = assert.match(execution.sql, /update "interactive_sessions"/i); assert.match(execution.sql, /where "id" = \?/i); assert.match(execution.sql, /"runtime" = \?/i); - assert.match(execution.sql, /"updated_at" <= \?/i); assert.ok(execution.parameters.includes("IS-101")); } + assert.match(executions[3].sql, /"updated_at" = \?/i); + assert.match(executions[3].sql, /"status" = \?/i); + assert.match(executions[3].sql, /"work_state" = \?/i); + assert.match(executions[3].sql, /"work_phase" = \?/i); + assert.match(executions[4].sql, /"updated_at" <= \?/i); + assert.match(executions[5].sql, /"updated_at" <= \?/i); assert.match(executions[3].sql, /"owner_subject" = \?/i); assert.doesNotMatch(executions[3].sql, /"work_state" not in/i); assert.match(executions[4].sql, /"work_state" not in/i); @@ -140,6 +145,13 @@ const registrationUpdate: GitHubActionsSessionRegistrationUpdate = { completion_reason: null, }; +const registrationExpectation = { + updated_at: 90, + status: "stopped", + work_state: "completed", + work_phase: "finished", +} as const; + const workStateUpdate: GitHubActionsWorkStateUpdate = { status: "attached", summary: "working", diff --git a/tests/github-actions-session-registration.test.ts b/tests/github-actions-session-registration.test.ts index ce181abc..de65b393 100644 --- a/tests/github-actions-session-registration.test.ts +++ b/tests/github-actions-session-registration.test.ts @@ -8,6 +8,7 @@ import { actionWorkIdentifier, buildGitHubActionsSessionValues, optionalHttpUrl, + type GitHubActionsSessionRegistrationExpectation, type GitHubActionsSessionRegistrationStore, type GitHubActionsSessionRegistrationUpdate, } from "../src/worker/github-actions-session-registration.ts"; @@ -18,7 +19,11 @@ type StoreState = { rows: Map; workKeyReads: number; inserted: InteractiveSessionTable[]; - updates: Array<{ id: string; values: GitHubActionsSessionRegistrationUpdate }>; + updates: Array<{ + id: string; + values: GitHubActionsSessionRegistrationUpdate; + expected: GitHubActionsSessionRegistrationExpectation; + }>; events: string[]; audits: string[]; operations: string[]; @@ -73,9 +78,9 @@ function registrationStore(initialRows: InteractiveSessionRow[] = []): { state.rows.set(row.id, row); }, readById: async (id) => state.rows.get(id) ?? null, - updateSession: async (id, values) => { + updateSession: async (id, values, expected) => { state.operations.push("update"); - state.updates.push({ id, values }); + state.updates.push({ id, values, expected }); const row = state.rows.get(id); if (row) state.rows.set(id, { ...row, ...values }); }, @@ -260,6 +265,12 @@ test("GitHub Actions work keys can be resumed by the matching owner", async () = assert.equal(state.updates[0]?.values.owner, "operator"); assert.equal(state.updates[0]?.values.owner_subject, "github:42"); assert.equal(state.updates[0]?.values.agent_token_hash, "agent-token-hash"); + assert.deepEqual(state.updates[0]?.expected, { + updated_at: existing.updated_at, + status: existing.status, + work_state: existing.work_state, + work_phase: existing.work_phase, + }); }); test("GitHub Actions rejects work keys without a stable owner", async () => { From f1c145ffc14bb8493bdf7e0ebd9ba7abb7072964 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:33:45 +0200 Subject: [PATCH 036/242] fix(macos): decode X11 Unicode keysyms --- .../Sources/CrabfleetMac/MacRemoteInput.swift | 10 +++++++++- .../Tests/CrabfleetMacTests/PrivateMacShareTests.swift | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index db6efac9..fddc85f7 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -144,7 +144,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable let event: CGEvent? if let keyCode = Self.keyCode(for: keysym) { event = CGEvent(keyboardEventSource: eventSource(), virtualKey: keyCode, keyDown: down) - } else if let scalar = UnicodeScalar(keysym) { + } else if let scalar = Self.unicodeScalar(for: keysym) { let candidate = CGEvent(keyboardEventSource: eventSource(), virtualKey: 0, keyDown: down) var codeUnits = Array(String(scalar).utf16) candidate?.keyboardSetUnicodeString( @@ -229,6 +229,14 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable } } + static func unicodeScalar(for keysym: UInt32) -> UnicodeScalar? { + let value = + keysym & 0xFF00_0000 == 0x0100_0000 + ? keysym & 0x00FF_FFFF + : keysym + return UnicodeScalar(value) + } + private static let asciiKeyCodes: [Character: CGKeyCode] = [ "a": CGKeyCode(kVK_ANSI_A), "b": CGKeyCode(kVK_ANSI_B), "c": CGKeyCode(kVK_ANSI_C), "d": CGKeyCode(kVK_ANSI_D), diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index e9cba8ce..201cdc79 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -589,6 +589,13 @@ struct PrivateMacShareTests { #expect(MacRemoteInputController.keyCode(for: 0x1F980) == nil) } + @Test + func decodesX11UnicodeKeysymsForMacInput() { + #expect(MacRemoteInputController.unicodeScalar(for: 0x0100_03BB) == "λ") + #expect(MacRemoteInputController.unicodeScalar(for: 0x0101_F980) == "🦀") + #expect(MacRemoteInputController.unicodeScalar(for: 0x0111_0000) == nil) + } + @Test @MainActor func servesRoyalVNCKitOverTheCurrentTailnet() async throws { guard ProcessInfo.processInfo.environment["CRABFLEET_TAILNET_RFB_SMOKE"] == "1" else { From a2b312c7a1fb133edd2a1b6e64625ff8a5803efd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:33:47 +0200 Subject: [PATCH 037/242] fix(macos): preserve empty clipboard notifications --- .../CrabfleetMac/TailnetRFBServer.swift | 54 +++++++++++-------- .../HostShareProtocolTests.swift | 16 ++++++ 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift index a7ac0e15..70b291fa 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift @@ -636,27 +636,11 @@ private final class RFBHostSession: @unchecked Sendable { } guard let io else { return } - let payload: Data? - if extendedNegotiated, let caps, caps.supportsText { - let wireByteCount = VNCExtendedClipboard.wireTextByteCount(text) - switch VNCExtendedClipboard.textRoute(wireByteCount: wireByteCount, caps: caps) { - case .provide: - payload = (try? VNCExtendedClipboard.encodeProvide(text: text)).map { - VNCExtendedClipboard.frame(messageType: 3, body: $0) - } - case .notify: - payload = VNCExtendedClipboard.frame( - messageType: 3, - body: VNCExtendedClipboard.encodeNotify(hasText: !text.isEmpty) - ) - case .legacy: - payload = RFBWire.legacyServerCutText(text: text) - } - } else { - // Legacy path: silently skip text that cannot survive Latin-1. - payload = RFBWire.legacyServerCutText(text: text) - } - + let payload = RFBWire.hostClipboardPayload( + text: text, + extendedNegotiated: extendedNegotiated, + caps: caps + ) guard let payload else { return } Task { try? await io.send(payload) @@ -1084,6 +1068,34 @@ private final class RFBHostSession: @unchecked Sendable { } } +extension RFBWire { + static func hostClipboardPayload( + text: String, + extendedNegotiated: Bool, + caps: VNCExtendedClipboardCaps? + ) -> Data? { + guard extendedNegotiated, let caps, caps.supportsText else { + // Legacy path: silently skip text that cannot survive Latin-1. + return legacyServerCutText(text: text) + } + + let wireByteCount = VNCExtendedClipboard.wireTextByteCount(text) + switch VNCExtendedClipboard.textRoute(wireByteCount: wireByteCount, caps: caps) { + case .provide: + return (try? VNCExtendedClipboard.encodeProvide(text: text)).map { + VNCExtendedClipboard.frame(messageType: 3, body: $0) + } + case .notify: + return VNCExtendedClipboard.frame( + messageType: 3, + body: VNCExtendedClipboard.encodeNotify(hasText: true) + ) + case .legacy: + return legacyServerCutText(text: text) + } + } +} + private struct RFBConnectionIO: Sendable { let connection: NWConnection diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift index 10274849..8c6199d5 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift @@ -1,5 +1,6 @@ import AppKit import Foundation +import RoyalVNCKit import Testing @testable import CrabfleetMac @@ -53,6 +54,21 @@ struct HostShareWireTests { #expect(RFBWire.legacyServerCutText(text: "emoji 🦀") == nil) } + @Test + func emptyExtendedClipboardTextRemainsRequestable() throws { + let caps = VNCExtendedClipboardCaps( + supportsText: true, + maximumUnsolicitedTextBytes: 0, + actions: VNCExtendedClipboard.notifyAction + ) + let packet = try #require( + RFBWire.hostClipboardPayload(text: "", extendedNegotiated: true, caps: caps) + ) + + #expect(packet[0] == 3) + #expect(try VNCExtendedClipboard.decode(body: packet.subdata(in: 8.. Date: Sun, 12 Jul 2026 10:33:51 +0200 Subject: [PATCH 038/242] fix(cards): classify duplicate claims before capacity --- src/worker/card-repository.ts | 24 ++++++++++++++++++ tests/card-repository.test.ts | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/src/worker/card-repository.ts b/src/worker/card-repository.ts index 0e0232da..5b1fde5f 100644 --- a/src/worker/card-repository.ts +++ b/src/worker/card-repository.ts @@ -387,6 +387,30 @@ export class CardRepository implements CardLifecycleStore { }), ); if ((results[0]?.meta.changes ?? 0) === 0) { + const duplicateAttempt = await db + .selectFrom("run_attempts") + .select("id") + .where((expressions) => + expressions.or([ + expressions("id", "=", input.runId), + expressions.and([ + expressions("card_id", "=", input.card.id), + expressions("attempt", "=", input.attempt), + ]), + ]), + ) + .executeTakeFirst(); + if (duplicateAttempt) return "active"; + + const activeCardRun = await db + .selectFrom("cards") + .innerJoin("run_attempts", "run_attempts.id", "cards.active_run_id") + .select("cards.id") + .where("cards.id", "=", input.card.id) + .where("run_attempts.status", "in", activeRunStatuses) + .executeTakeFirst(); + if (activeCardRun) return "active"; + const activeCount = await db .selectFrom("cards") .select(sql`count(*)`.as("count")) diff --git a/tests/card-repository.test.ts b/tests/card-repository.test.ts index d6050c89..924d4ea3 100644 --- a/tests/card-repository.test.ts +++ b/tests/card-repository.test.ts @@ -161,3 +161,50 @@ test("card run claims batch the card transition with the run-attempt insert", as assert.ok(batches[0]?.[0]?.parameters.includes("CY-101-R1")); assert.ok(batches[0]?.[1]?.parameters.includes("CY-101-R1")); }); + +test("duplicate card run claims report active before global capacity", async () => { + const queries: string[] = []; + const env = { + DB: { + prepare(sql: string) { + queries.push(sql); + return { + bind() { + return { + async all() { + if (/from "run_attempts"/i.test(sql)) { + return { results: [{ id: "CY-101-R1" }], meta: { changes: 0 } }; + } + return { results: [{ count: 2 }], meta: { changes: 0 } }; + }, + async run() { + return { meta: { changes: 0 } }; + }, + }; + }, + }; + }, + async batch() { + return [{ meta: { changes: 0 } }, { meta: { changes: 0 } }]; + }, + } as unknown as D1Database, + } as RuntimeEnv; + const input = { + card: { id: "CY-101" }, + runId: "CY-101-R1", + attempt: 1, + cap: 2, + descriptor: { + runtime: "container", + reason: "repo default", + capabilities: containerCapabilities, + }, + now: 500, + } as CardRunClaimInput; + + assert.equal(await new CardRepository(env).claimRun(input), "active"); + const diagnosticQueries = queries.filter((query) => /^\s*select/i.test(query)); + assert.equal(diagnosticQueries.length, 1); + assert.match(diagnosticQueries[0] ?? "", /from "run_attempts"/i); + assert.doesNotMatch(diagnosticQueries[0] ?? "", /count\(\*\)/i); +}); From b2bdeaf090770b63d1502e4fbeacc50b8e76bc0a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:34:00 +0200 Subject: [PATCH 039/242] docs(changelog): complete audit hardening notes --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b2da9f..8cff1393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and grant revocation, revision-fenced GitHub Actions updates, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, read-state-fenced GitHub Actions registration, revision-fenced lifecycle updates and grant revocation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate one-shot input acknowledgements across rolling upgrades, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations, dropped auto-starts, stuck remote input, and destructive non-text clipboard changes while preserving proxy and custom-CA networking. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, color-depth transitions, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, synchronized color-depth transitions, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From a018a30a0c8dbfe6065d9a69b2f7350211be6be6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:40:40 +0200 Subject: [PATCH 040/242] fix(terminal): return after empty confirmed input --- internal/terminalws/client.go | 3 +++ internal/terminalws/client_test.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 4ba19248..489ba1ef 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -254,6 +254,9 @@ func (c *Client) SendInput(ctx context.Context, payload []byte) error { } func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { + if len(payload) == 0 { + return nil + } if !c.supportsInputAcknowledgement { return c.SendInput(ctx, payload) } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index a583f0a7..7974b90b 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -334,6 +334,13 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { } } +func TestSendInputConfirmedReturnsImmediatelyForEmptyInput(t *testing.T) { + client := &Client{supportsInputAcknowledgement: true} + if err := client.SendInputConfirmed(context.Background(), nil); err != nil { + t.Fatal(err) + } +} + func TestSendInputConfirmedFallsBackWithoutServerCapability(t *testing.T) { receivedInput := make(chan []byte, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 654f0e4424319f040cc1282b56011317674aef9a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:40:40 +0200 Subject: [PATCH 041/242] fix(macos): remove hosts after listener failure --- .../PrivateMacShareController.swift | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 25648047..adaa19bd 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -432,8 +432,8 @@ final class PrivateMacShareController: ObservableObject { connectedPeer = nil streamStats = nil case .listenerFailed(let message): - registrationTask?.cancel() - registrationTask = nil + let pendingRegistration = registrationTask + pendingRegistration?.cancel() phase = .failed connectedPeer = nil streamStats = nil @@ -446,6 +446,7 @@ final class PrivateMacShareController: ObservableObject { let failedCapture = capture capture = nil Task { await failedCapture?.stop() } + removeDesktopHost(after: pendingRegistration) case .sessionFailed(let message): phase = .sharing connectedPeer = nil @@ -475,6 +476,23 @@ final class PrivateMacShareController: ObservableObject { } } + private func removeDesktopHost(after pendingRegistration: Task?) { + guard let desktopRegistrationCoordinator, let activeIdentity else { + registrationTask = nil + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished + return + } + registrationTask = Task { [weak self] in + await pendingRegistration?.value + do { + try await desktopRegistrationCoordinator.unregister(identity: activeIdentity) + self?.registryPhase = .notPublished + } catch { + self?.registryPhase = .failed(error.localizedDescription) + } + } + } + @discardableResult private func beginLifecycleTransition() -> UInt64 { lifecycleGeneration &+= 1 From bc6648ffd5cbef261f102413d0ee959288975eae Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:40:40 +0200 Subject: [PATCH 042/242] fix(vnc): force idle pixel format boundaries --- .../SDK/Connection/VNCConnection+API.swift | 23 +++++++++++++++++++ .../SDK/Connection/VNCConnection.swift | 1 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 7 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 86a1ba0a..1ea82f0d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -92,13 +92,35 @@ private extension VNCConnection { framebufferPacingTask?.cancel() framebufferPacingTask = nil let transition = takePendingPixelFormatTransitionLocked() + let probe = takePixelFormatTransitionProbeLocked() framebufferRequestLock.unlock() if let transition { enqueuePixelFormatTransition(transition) + } else if let probe { + enqueueClientToServerMessage(probe) } } + func takePixelFormatTransitionProbeLocked() -> VNCProtocol.FramebufferUpdateRequest? { + guard framebufferUpdateRequestOutstanding, + !isPixelFormatTransitionProbeQueued, + !isPixelFormatTransitionInFlight, + pendingPixelFormatTransition != nil, + let framebuffer else { + return nil + } + + isPixelFormatTransitionProbeQueued = true + return VNCProtocol.FramebufferUpdateRequest( + incremental: false, + xPosition: 0, + yPosition: 0, + width: framebuffer.size.width, + height: framebuffer.size.height + ) + } + func takePendingPixelFormatTransitionLocked() -> VNCProtocol.PixelFormat? { guard !framebufferUpdateRequestOutstanding, !isPixelFormatTransitionInFlight, @@ -331,6 +353,7 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() framebufferUpdateRequestOutstanding = false + isPixelFormatTransitionProbeQueued = false let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 9634393d..5f92c52b 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -112,6 +112,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var framebufferPacingTask: Task? var pendingPixelFormatTransition: VNCProtocol.PixelFormat? var isPixelFormatTransitionInFlight = false + var isPixelFormatTransitionProbeQueued = false private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 15d67288..7f6108f5 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -116,7 +116,12 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - #expect(connection.clientToServerMessageQueue.dequeue() == nil) + let probe = try #require(connection.clientToServerMessageQueue.dequeue()) + let probeWriter = AuditWritingConnection() + try await probe.message.send(connection: probeWriter) + #expect(probeWriter.data.count == 10) + #expect(probeWriter.data[0] == 3) + #expect(probeWriter.data[1] == 0) connection.completeFramebufferUpdateRequest() From 5f9911eb9eb1618ab69e32b479b78a4f5c655e12 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:40:40 +0200 Subject: [PATCH 043/242] docs(changelog): record final audit fixes --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cff1393..3f3ad9cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,10 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, read-state-fenced GitHub Actions registration, revision-fenced lifecycle updates and grant revocation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, negotiate one-shot input acknowledgements across rolling upgrades, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. +- Make terminal input delivery durable across multiplex subscribers, negotiate one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, synchronized color-depth transitions, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, synchronized color-depth transitions that force idle update boundaries, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From 1c3f71a91e0d7b2b477d124e091346814ce0b07a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:46:50 +0200 Subject: [PATCH 044/242] fix(sandbox): rotate credential generations monotonically --- src/credential-policy-fence.ts | 8 ++++---- tests/credential-policy-fence.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/credential-policy-fence.ts b/src/credential-policy-fence.ts index 2697e8f4..17072808 100644 --- a/src/credential-policy-fence.ts +++ b/src/credential-policy-fence.ts @@ -31,12 +31,12 @@ export function credentialPolicyRegistrationAccepted current.registrationExpiresAt; + } if (current.registrationClaim === incoming.registrationClaim) { return incoming.registrationExpiresAt >= current.registrationExpiresAt; } diff --git a/tests/credential-policy-fence.test.ts b/tests/credential-policy-fence.test.ts index b00b40ce..616f666f 100644 --- a/tests/credential-policy-fence.test.ts +++ b/tests/credential-policy-fence.test.ts @@ -67,6 +67,32 @@ test("generation fences isolate new policies from stale cleanup", () => { ); }); +test("newer generations rotate policy for the same session only", () => { + const current = registration("generation-1", "claim-current", 300); + + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-2", "claim-replacement", 301), + 100, + ), + true, + ); + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + { + ...registration("generation-2", "claim-replacement", 301), + policy: { sessionId: "IS-102", value: "claim-replacement" }, + }, + 100, + ), + false, + ); +}); + test("same-generation registration claims advance monotonically", () => { const current = registration("generation-1", "claim-current", 300); From 2299f9c50c3df43de472d0a382e8c35a0bf5c0b6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:46:50 +0200 Subject: [PATCH 045/242] fix(terminal): bound input confirmation waits --- internal/fleetapi/client.go | 6 +++++- internal/fleetapi/client_test.go | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/fleetapi/client.go b/internal/fleetapi/client.go index 6fc54af6..bd12cd2d 100644 --- a/internal/fleetapi/client.go +++ b/internal/fleetapi/client.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/openclaw/crabfleet/internal/terminalws" ) @@ -18,6 +19,7 @@ type TerminalSize = terminalws.Size const maxResponseBytes = 4 * 1024 * 1024 const maxErrorBytes = 512 +const terminalInputConfirmationTimeout = 15 * time.Second var ErrMissingAuth = errors.New("API mode requires SSH gateway token + fingerprint or agent token + session ID") @@ -193,7 +195,9 @@ func (c *Client) Message( if enter { message += "\n" } - return client.SendInputConfirmed(ctx, []byte(message)) + confirmationContext, cancel := context.WithTimeout(ctx, terminalInputConfirmationTimeout) + defer cancel() + return client.SendInputConfirmed(confirmationContext, []byte(message)) } func (c *Client) Attach( diff --git a/internal/fleetapi/client_test.go b/internal/fleetapi/client_test.go index 7bb54775..95ba5e6b 100644 --- a/internal/fleetapi/client_test.go +++ b/internal/fleetapi/client_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func TestClientUsesSSHAuthentication(t *testing.T) { @@ -66,6 +67,12 @@ func TestClientRejectsIncompleteAuthentication(t *testing.T) { } } +func TestTerminalInputConfirmationTimeoutIsBounded(t *testing.T) { + if terminalInputConfirmationTimeout <= 0 || terminalInputConfirmationTimeout > 30*time.Second { + t.Fatalf("terminal input confirmation timeout = %s", terminalInputConfirmationTimeout) + } +} + func TestClientSanitizesStatusErrorBody(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) From 2f27fc06a97d032df8ed499d1db3c074c46dfe0c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:46:50 +0200 Subject: [PATCH 046/242] fix(vnc): drain pixel format probe responses --- .../RoyalVNCKit/SDK/Connection/VNCConnection+API.swift | 10 ++++++++++ .../RoyalVNCKit/SDK/Connection/VNCConnection.swift | 1 + .../Tests/RoyalVNCKitTests/AuditFindingsTests.swift | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 1ea82f0d..55ee3dbc 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -112,6 +112,7 @@ private extension VNCConnection { } isPixelFormatTransitionProbeQueued = true + pixelFormatTransitionResponsesRemaining = 2 return VNCProtocol.FramebufferUpdateRequest( incremental: false, xPosition: 0, @@ -352,6 +353,13 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() + if pixelFormatTransitionResponsesRemaining > 0 { + pixelFormatTransitionResponsesRemaining -= 1 + if pixelFormatTransitionResponsesRemaining > 0 { + framebufferRequestLock.unlock() + return + } + } framebufferUpdateRequestOutstanding = false isPixelFormatTransitionProbeQueued = false let transition = takePendingPixelFormatTransitionLocked() @@ -415,6 +423,8 @@ extension VNCConnection { framebufferUpdateRequestOutstanding = false pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = false + isPixelFormatTransitionProbeQueued = false + pixelFormatTransitionResponsesRemaining = 0 framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 5f92c52b..cfba0f6d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -113,6 +113,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pendingPixelFormatTransition: VNCProtocol.PixelFormat? var isPixelFormatTransitionInFlight = false var isPixelFormatTransitionProbeQueued = false + var pixelFormatTransitionResponsesRemaining = 0 private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 7f6108f5..fb8188a2 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -123,6 +123,10 @@ struct AuditFindingsTests { #expect(probeWriter.data[0] == 3) #expect(probeWriter.data[1] == 0) + connection.completeFramebufferUpdateRequest() + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + #expect(connection.state.pixelFormat?.depth == 24) + connection.completeFramebufferUpdateRequest() let queued = try #require(connection.clientToServerMessageQueue.dequeue()) From f1298a16cdc61f7da7da990ba00bec20196db71b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:46:50 +0200 Subject: [PATCH 047/242] docs(changelog): complete audit lifecycle notes --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f3ad9cd..ae8170db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, read-state-fenced GitHub Actions registration, revision-fenced lifecycle updates and grant revocation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, negotiate one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, read-state-fenced GitHub Actions registration, revision-fenced lifecycle updates and grant revocation, monotonic Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, synchronized color-depth transitions that force idle update boundaries, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, synchronized color-depth transitions that drain forced idle update boundaries, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From 62b5aebdca5b8a9af5d322922824bffce90cdeb6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:55:05 +0200 Subject: [PATCH 048/242] fix(vnc): synchronize pixel formats with RFB fences --- .../Messages/ClientToServer/ClientFence.swift | 45 +++++++ .../Messages/ServerToClient/ServerFence.swift | 28 ++++ .../RoyalVNCKit/SDK/Connection/State.swift | 6 + .../SDK/Connection/VNCConnection+API.swift | 125 +++++++++++------- .../Connection/VNCConnection+Receive.swift | 27 ++++ .../SDK/Connection/VNCConnection.swift | 6 +- .../SDK/VNCPseudoEncodingType.swift | 3 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 83 ++++++++++-- 8 files changed, 261 insertions(+), 62 deletions(-) create mode 100644 macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ClientToServer/ClientFence.swift create mode 100644 macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ServerToClient/ServerFence.swift diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ClientToServer/ClientFence.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ClientToServer/ClientFence.swift new file mode 100644 index 00000000..4e158f5c --- /dev/null +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ClientToServer/ClientFence.swift @@ -0,0 +1,45 @@ +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import Foundation +#endif + +extension VNCProtocol { + struct FenceFlags: OptionSet { + let rawValue: UInt32 + + static let blockBefore = Self(rawValue: 1 << 0) + static let blockAfter = Self(rawValue: 1 << 1) + static let syncNext = Self(rawValue: 1 << 2) + static let request = Self(rawValue: 1 << 31) + + static let supported: Self = [.blockBefore, .blockAfter, .syncNext, .request] + } + + struct ClientFence: VNCSendableMessage { + static let maximumPayloadLength = 64 + static let messageType: UInt8 = 248 + + var messageType: UInt8 { Self.messageType } + let flags: FenceFlags + let payload: Data + } +} + +extension VNCProtocol.ClientFence { + var data: Data { + precondition(payload.count <= Self.maximumPayloadLength) + + var data = Data(capacity: 9 + payload.count) + data.append(messageType) + data.appendPadding(length: 3) + data.append(flags.rawValue, bigEndian: true) + data.append(UInt8(payload.count)) + data.append(payload) + return data + } + + func send(connection: NetworkConnectionWriting) async throws { + try await connection.write(data: data) + } +} diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ServerToClient/ServerFence.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ServerToClient/ServerFence.swift new file mode 100644 index 00000000..e369d93b --- /dev/null +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/Messages/ServerToClient/ServerFence.swift @@ -0,0 +1,28 @@ +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import Foundation +#endif + +extension VNCProtocol { + struct ServerFence: VNCReceivableMessage { + static let messageType: UInt8 = 248 + + let messageType: UInt8 + let flags: FenceFlags + let payload: Data + } +} + +extension VNCProtocol.ServerFence { + static func receive(connection: NetworkConnectionReading) async throws -> Self { + try await connection.readPadding(length: 3) + let flags = VNCProtocol.FenceFlags(rawValue: try await connection.readUInt32()) + let payloadLength = Int(try await connection.readUInt8()) + guard payloadLength <= VNCProtocol.ClientFence.maximumPayloadLength else { + throw VNCError.protocol(.invalidData) + } + let payload = try await connection.read(length: payloadLength) + return .init(messageType: messageType, flags: flags, payload: payload) + } +} diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift index 1b08e904..c828705a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift @@ -18,6 +18,7 @@ extension VNCConnection { private var _pixelFormat: VNCProtocol.PixelFormat? private var _desktopName: String? private var _incrementalUpdatesEnabled = false + private var _areFencesSupported = false private var _areContinuousUpdatesSupported = false private var _areContinuousUpdatesEnabled = false private var _extendedClipboardServerCaps: VNCExtendedClipboardCaps? @@ -70,6 +71,11 @@ extension VNCConnection { set { withLock { _incrementalUpdatesEnabled = newValue } } } + var areFencesSupported: Bool { + get { withLock { _areFencesSupported } } + set { withLock { _areFencesSupported = newValue } } + } + var areContinuousUpdatesSupported: Bool { get { withLock { _areContinuousUpdatesSupported } } set { withLock { _areContinuousUpdatesSupported = newValue } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 55ee3dbc..3c93a9e2 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -5,20 +5,32 @@ import Foundation #endif private struct PixelFormatTransitionMessage: VNCSendableMessage { + let fenceMessage: VNCProtocol.ClientFence? let pixelFormatMessage: VNCProtocol.SetPixelFormat let willSend: () -> Void let didSend: () -> Void - var messageType: UInt8 { pixelFormatMessage.messageType } - var data: Data { pixelFormatMessage.data } + var messageType: UInt8 { fenceMessage?.messageType ?? pixelFormatMessage.messageType } + var data: Data { + (fenceMessage?.data ?? Data()) + pixelFormatMessage.data + } func send(connection: NetworkConnectionWriting) async throws { - willSend() - try await pixelFormatMessage.send(connection: connection) - didSend() + if fenceMessage == nil { + willSend() + } + try await connection.write(data: data) + if fenceMessage == nil { + didSend() + } } } +private struct PixelFormatTransition { + let pixelFormat: VNCProtocol.PixelFormat + let fencePayload: Data? +} + // MARK: - Connect/Disconnect public extension VNCConnection { #if canImport(ObjectiveC) @@ -84,61 +96,52 @@ public extension VNCConnection { } } -private extension VNCConnection { - func requestPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { +extension VNCConnection { + private func requestPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { framebufferRequestLock.lock() pendingPixelFormatTransition = pixelFormat framebufferRequestGeneration &+= 1 framebufferPacingTask?.cancel() framebufferPacingTask = nil let transition = takePendingPixelFormatTransitionLocked() - let probe = takePixelFormatTransitionProbeLocked() framebufferRequestLock.unlock() if let transition { enqueuePixelFormatTransition(transition) - } else if let probe { - enqueueClientToServerMessage(probe) } } - func takePixelFormatTransitionProbeLocked() -> VNCProtocol.FramebufferUpdateRequest? { - guard framebufferUpdateRequestOutstanding, - !isPixelFormatTransitionProbeQueued, - !isPixelFormatTransitionInFlight, - pendingPixelFormatTransition != nil, - let framebuffer else { - return nil - } - - isPixelFormatTransitionProbeQueued = true - pixelFormatTransitionResponsesRemaining = 2 - return VNCProtocol.FramebufferUpdateRequest( - incremental: false, - xPosition: 0, - yPosition: 0, - width: framebuffer.size.width, - height: framebuffer.size.height - ) - } - - func takePendingPixelFormatTransitionLocked() -> VNCProtocol.PixelFormat? { - guard !framebufferUpdateRequestOutstanding, - !isPixelFormatTransitionInFlight, + private func takePendingPixelFormatTransitionLocked() -> PixelFormatTransition? { + guard !isPixelFormatTransitionInFlight, + (!framebufferUpdateRequestOutstanding || state.areFencesSupported), let pixelFormat = pendingPixelFormatTransition else { return nil } pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = true - return pixelFormat + pixelFormatTransitionInFlight = pixelFormat + let fencePayload: Data? + if state.areFencesSupported { + pixelFormatTransitionFenceSequence &+= 1 + var sequence = pixelFormatTransitionFenceSequence.bigEndian + fencePayload = withUnsafeBytes(of: &sequence) { Data($0) } + pixelFormatTransitionFencePayload = fencePayload + } else { + fencePayload = nil + } + return PixelFormatTransition(pixelFormat: pixelFormat, fencePayload: fencePayload) } - func enqueuePixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { + private func enqueuePixelFormatTransition(_ transition: PixelFormatTransition) { + let fenceMessage = transition.fencePayload.map { + VNCProtocol.ClientFence(flags: [.request, .syncNext], payload: $0) + } let message = PixelFormatTransitionMessage( - pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat), + fenceMessage: fenceMessage, + pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: transition.pixelFormat), willSend: { [weak self] in - self?.beginPixelFormatTransition(pixelFormat) + self?.beginPixelFormatTransition(transition.pixelFormat) } ) { [weak self] in self?.completePixelFormatTransition() @@ -147,7 +150,38 @@ private extension VNCConnection { enqueueClientToServerMessage(message) } - func beginPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { + func didLearnFenceSupport() { + framebufferRequestLock.lock() + let transition = takePendingPixelFormatTransitionLocked() + framebufferRequestLock.unlock() + + if let transition { + enqueuePixelFormatTransition(transition) + } + } + + func completePixelFormatFence(_ fence: VNCProtocol.ServerFence) throws { + framebufferRequestLock.lock() + guard fence.payload == pixelFormatTransitionFencePayload else { + framebufferRequestLock.unlock() + return + } + guard fence.flags.contains(.syncNext) else { + framebufferRequestLock.unlock() + throw VNCError.protocol(.invalidData) + } + guard let pixelFormat = pixelFormatTransitionInFlight else { + framebufferRequestLock.unlock() + throw VNCError.protocol(.invalidData) + } + pixelFormatTransitionFencePayload = nil + framebufferRequestLock.unlock() + + beginPixelFormatTransition(pixelFormat) + completePixelFormatTransition() + } + + private func beginPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { withLifecycleLock { guard connectionState.status == .connected, let framebuffer = framebuffer else { @@ -161,9 +195,10 @@ private extension VNCConnection { } } - func completePixelFormatTransition() { + private func completePixelFormatTransition() { framebufferRequestLock.lock() isPixelFormatTransitionInFlight = false + pixelFormatTransitionInFlight = nil let nextTransition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() @@ -353,15 +388,7 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() - if pixelFormatTransitionResponsesRemaining > 0 { - pixelFormatTransitionResponsesRemaining -= 1 - if pixelFormatTransitionResponsesRemaining > 0 { - framebufferRequestLock.unlock() - return - } - } framebufferUpdateRequestOutstanding = false - isPixelFormatTransitionProbeQueued = false let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() @@ -423,8 +450,8 @@ extension VNCConnection { framebufferUpdateRequestOutstanding = false pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = false - isPixelFormatTransitionProbeQueued = false - pixelFormatTransitionResponsesRemaining = 0 + pixelFormatTransitionInFlight = nil + pixelFormatTransitionFencePayload = nil framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift index 70cdc958..3807feba 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift @@ -55,6 +55,9 @@ private extension VNCConnection { case VNCProtocol.EndOfContinuousUpdates.messageType: try await handleEndOfContinuousUpdatesMessage() + case VNCProtocol.ServerFence.messageType: + try await handleServerFenceMessage() + default: throw VNCError.protocol(.unsupportedServerToClientMessage(messageType: messageType)) } @@ -195,9 +198,33 @@ private extension VNCConnection { func handleEndOfContinuousUpdatesMessage() async throws { didReceiveEndOfContinuousUpdates() } + + func handleServerFenceMessage() async throws { + let fence = try await VNCProtocol.ServerFence.receive(connection: connection) + try handleServerFence(fence) + } } extension VNCConnection { + func handleServerFence(_ fence: VNCProtocol.ServerFence) throws { + let first = !state.areFencesSupported + state.areFencesSupported = true + + if fence.flags.contains(.request) { + let responseFlags = fence.flags.intersection([.blockBefore, .blockAfter]) + enqueueClientToServerMessage( + VNCProtocol.ClientFence(flags: responseFlags, payload: fence.payload) + ) + } else { + try completePixelFormatFence(fence) + } + + if first { + logger.logDebug("Fence supported (server sent ServerFence)") + didLearnFenceSupport() + } + } + func didReceiveEndOfContinuousUpdates() { let first = !state.areContinuousUpdatesSupported diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index cfba0f6d..b3428d30 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -112,8 +112,9 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var framebufferPacingTask: Task? var pendingPixelFormatTransition: VNCProtocol.PixelFormat? var isPixelFormatTransitionInFlight = false - var isPixelFormatTransitionProbeQueued = false - var pixelFormatTransitionResponsesRemaining = 0 + var pixelFormatTransitionInFlight: VNCProtocol.PixelFormat? + var pixelFormatTransitionFenceSequence: UInt64 = 0 + var pixelFormatTransitionFencePayload: Data? private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() @@ -269,6 +270,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { // Pseudo Encodings encs.append(contentsOf: [ VNCPseudoEncodingType.lastRect.rawValue, + VNCPseudoEncodingType.fence.rawValue, VNCPseudoEncodingType.continuousUpdates.rawValue, VNCPseudoEncodingType.extendedDesktopSize.rawValue, VNCPseudoEncodingType.desktopSize.rawValue, diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/VNCPseudoEncodingType.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/VNCPseudoEncodingType.swift index d0300acb..97b6620d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/VNCPseudoEncodingType.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/VNCPseudoEncodingType.swift @@ -8,6 +8,7 @@ public enum VNCPseudoEncodingType: VNCEncodingType { case lastRect = -224 case cursor = -239 case desktopName = -307 + case fence = -312 case continuousUpdates = -313 case desktopSize = -223 case extendedDesktopSize = -308 @@ -51,6 +52,8 @@ extension VNCPseudoEncodingType: CustomStringConvertible { "Cursor" case .desktopName: "Desktop Name" + case .fence: + "Fence" case .continuousUpdates: "Continuous Updates" case .desktopSize: diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index fb8188a2..0f2187b4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -101,7 +101,7 @@ struct AuditFindingsTests { } @Test - func serializesPixelFormatAndFramebufferTransitionAtSendBoundary() async throws { + func waitsForOutstandingFramebufferBeforeUnfencedPixelFormatTransition() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -116,19 +116,9 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - let probe = try #require(connection.clientToServerMessageQueue.dequeue()) - let probeWriter = AuditWritingConnection() - try await probe.message.send(connection: probeWriter) - #expect(probeWriter.data.count == 10) - #expect(probeWriter.data[0] == 3) - #expect(probeWriter.data[1] == 0) - - connection.completeFramebufferUpdateRequest() #expect(connection.clientToServerMessageQueue.dequeue() == nil) - #expect(connection.state.pixelFormat?.depth == 24) connection.completeFramebufferUpdateRequest() - let queued = try #require(connection.clientToServerMessageQueue.dequeue()) let writer = AuditWritingConnection { #expect(connection.state.pixelFormat?.depth == 8) @@ -141,6 +131,77 @@ struct AuditFindingsTests { #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } + @Test + func synchronizesPixelFormatTransitionWithFenceResponse() async throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + connection.framebufferUpdateRequestOutstanding = true + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .syncNext], + payload: Data("support".utf8) + ) + ) + let supportResponse = try #require(connection.clientToServerMessageQueue.dequeue()) + let supportWriter = AuditWritingConnection() + try await supportResponse.message.send(connection: supportWriter) + #expect(supportWriter.data[0] == VNCProtocol.ClientFence.messageType) + #expect(supportWriter.data[8] == 7) + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + let writer = AuditWritingConnection { + #expect(connection.state.pixelFormat?.depth == 24) + } + try await queued.message.send(connection: writer) + + #expect(writer.data.count == 37) + #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) + #expect(writer.data[4..<8] == Data([0x80, 0, 0, 4])) + #expect(writer.data[8] == 8) + #expect(writer.data[17] == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType) + #expect(connection.state.pixelFormat?.depth == 24) + + let payload = Data(writer.data[9..<17]) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.syncNext], + payload: payload + ) + ) + + #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) + } + + @Test + func advertisesAndDecodesFenceExtension() async throws { + let connection = VNCConnection(settings: makeSettings()) + #expect(try connection.orderedEncodingTypes().contains(VNCPseudoEncodingType.fence.rawValue)) + + var body = Data([0, 0, 0]) + body.append(UInt32(0x8000_0004), bigEndian: true) + body.append(UInt8(3)) + body.append(Data([1, 2, 3])) + + let fence = try await VNCProtocol.ServerFence.receive( + connection: AuditBufferConnection(body) + ) + + #expect(fence.flags == [.request, .syncNext]) + #expect(fence.payload == Data([1, 2, 3])) + } + @Test func copyRectPreservesInternalFramebufferPixels() throws { let framebuffer = try makeFramebuffer(width: 2, height: 1, depth: 16) From 1919e62b5ceda84fd690af983006d950b92bd7bd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 10:55:11 +0200 Subject: [PATCH 049/242] docs(changelog): note fenced VNC transitions --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae8170db..a14f4a5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, synchronized color-depth transitions that drain forced idle update boundaries, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with a conservative legacy fallback, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From a79bffda6ada10f07a78298a45d045eb2f81863d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:00:44 +0200 Subject: [PATCH 050/242] fix(vnc): negotiate pixel format fences safely --- .../RoyalVNCKit/SDK/Connection/State.swift | 6 ++ .../SDK/Connection/VNCConnection+API.swift | 80 +++++++++++++++++-- .../Connection/VNCConnection+Receive.swift | 4 +- .../SDK/Connection/VNCConnection.swift | 2 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 70 +++++++++++++++- 5 files changed, 151 insertions(+), 11 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift index c828705a..65017b6a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift @@ -19,6 +19,7 @@ extension VNCConnection { private var _desktopName: String? private var _incrementalUpdatesEnabled = false private var _areFencesSupported = false + private var _areSyncNextFencesSupported = false private var _areContinuousUpdatesSupported = false private var _areContinuousUpdatesEnabled = false private var _extendedClipboardServerCaps: VNCExtendedClipboardCaps? @@ -76,6 +77,11 @@ extension VNCConnection { set { withLock { _areFencesSupported = newValue } } } + var areSyncNextFencesSupported: Bool { + get { withLock { _areSyncNextFencesSupported } } + set { withLock { _areSyncNextFencesSupported = newValue } } + } + var areContinuousUpdatesSupported: Bool { get { withLock { _areContinuousUpdatesSupported } } set { withLock { _areContinuousUpdatesSupported = newValue } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 3c93a9e2..6a0d2801 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -31,6 +31,18 @@ private struct PixelFormatTransition { let fencePayload: Data? } +private struct FenceCapabilityProbeMessage: VNCSendableMessage { + let fenceMessage: VNCProtocol.ClientFence + let pixelFormatMessage: VNCProtocol.SetPixelFormat + + var messageType: UInt8 { fenceMessage.messageType } + var data: Data { fenceMessage.data + pixelFormatMessage.data } + + func send(connection: NetworkConnectionWriting) async throws { + try await connection.write(data: data) + } +} + // MARK: - Connect/Disconnect public extension VNCConnection { #if canImport(ObjectiveC) @@ -104,16 +116,20 @@ extension VNCConnection { framebufferPacingTask?.cancel() framebufferPacingTask = nil let transition = takePendingPixelFormatTransitionLocked() + let probe = takePixelFormatTransitionProbeLocked() framebufferRequestLock.unlock() if let transition { enqueuePixelFormatTransition(transition) + } else if let probe { + enqueueClientToServerMessage(probe) } } private func takePendingPixelFormatTransitionLocked() -> PixelFormatTransition? { guard !isPixelFormatTransitionInFlight, - (!framebufferUpdateRequestOutstanding || state.areFencesSupported), + pixelFormatFenceCapabilityProbePayload == nil, + (!framebufferUpdateRequestOutstanding || state.areSyncNextFencesSupported), let pixelFormat = pendingPixelFormatTransition else { return nil } @@ -122,7 +138,7 @@ extension VNCConnection { isPixelFormatTransitionInFlight = true pixelFormatTransitionInFlight = pixelFormat let fencePayload: Data? - if state.areFencesSupported { + if state.areSyncNextFencesSupported { pixelFormatTransitionFenceSequence &+= 1 var sequence = pixelFormatTransitionFenceSequence.bigEndian fencePayload = withUnsafeBytes(of: &sequence) { Data($0) } @@ -133,6 +149,27 @@ extension VNCConnection { return PixelFormatTransition(pixelFormat: pixelFormat, fencePayload: fencePayload) } + private func takePixelFormatTransitionProbeLocked() -> VNCProtocol.FramebufferUpdateRequest? { + guard framebufferUpdateRequestOutstanding, + !isPixelFormatTransitionProbeQueued, + !isPixelFormatTransitionInFlight, + pixelFormatFenceCapabilityProbePayload == nil, + !state.areSyncNextFencesSupported, + pendingPixelFormatTransition != nil, + let framebuffer else { + return nil + } + + isPixelFormatTransitionProbeQueued = true + return VNCProtocol.FramebufferUpdateRequest( + incremental: false, + xPosition: 0, + yPosition: 0, + width: framebuffer.size.width, + height: framebuffer.size.height + ) + } + private func enqueuePixelFormatTransition(_ transition: PixelFormatTransition) { let fenceMessage = transition.fencePayload.map { VNCProtocol.ClientFence(flags: [.request, .syncNext], payload: $0) @@ -150,18 +187,44 @@ extension VNCConnection { enqueueClientToServerMessage(message) } - func didLearnFenceSupport() { + func probeSyncNextFenceSupport() { framebufferRequestLock.lock() - let transition = takePendingPixelFormatTransitionLocked() + guard pixelFormatFenceCapabilityProbePayload == nil, + let pixelFormat = state.pixelFormat else { + framebufferRequestLock.unlock() + return + } + let payload = Data("royalvnc-sync-next".utf8) + pixelFormatFenceCapabilityProbePayload = payload framebufferRequestLock.unlock() - if let transition { - enqueuePixelFormatTransition(transition) - } + enqueueClientToServerMessage( + FenceCapabilityProbeMessage( + fenceMessage: VNCProtocol.ClientFence( + flags: [.request, .syncNext], + payload: payload + ), + pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat) + ) + ) } func completePixelFormatFence(_ fence: VNCProtocol.ServerFence) throws { framebufferRequestLock.lock() + if fence.payload == pixelFormatFenceCapabilityProbePayload { + pixelFormatFenceCapabilityProbePayload = nil + state.areSyncNextFencesSupported = fence.flags.contains(.syncNext) + let transition = takePendingPixelFormatTransitionLocked() + let probe = takePixelFormatTransitionProbeLocked() + framebufferRequestLock.unlock() + + if let transition { + enqueuePixelFormatTransition(transition) + } else if let probe { + enqueueClientToServerMessage(probe) + } + return + } guard fence.payload == pixelFormatTransitionFencePayload else { framebufferRequestLock.unlock() return @@ -389,6 +452,7 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() framebufferUpdateRequestOutstanding = false + isPixelFormatTransitionProbeQueued = false let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() @@ -451,7 +515,9 @@ extension VNCConnection { pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = false pixelFormatTransitionInFlight = nil + isPixelFormatTransitionProbeQueued = false pixelFormatTransitionFencePayload = nil + pixelFormatFenceCapabilityProbePayload = nil framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift index 3807feba..bf28410a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift @@ -211,7 +211,7 @@ extension VNCConnection { state.areFencesSupported = true if fence.flags.contains(.request) { - let responseFlags = fence.flags.intersection([.blockBefore, .blockAfter]) + let responseFlags = fence.flags.intersection(.blockBefore) enqueueClientToServerMessage( VNCProtocol.ClientFence(flags: responseFlags, payload: fence.payload) ) @@ -221,7 +221,7 @@ extension VNCConnection { if first { logger.logDebug("Fence supported (server sent ServerFence)") - didLearnFenceSupport() + probeSyncNextFenceSupport() } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index b3428d30..06a76453 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -113,8 +113,10 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pendingPixelFormatTransition: VNCProtocol.PixelFormat? var isPixelFormatTransitionInFlight = false var pixelFormatTransitionInFlight: VNCProtocol.PixelFormat? + var isPixelFormatTransitionProbeQueued = false var pixelFormatTransitionFenceSequence: UInt64 = 0 var pixelFormatTransitionFencePayload: Data? + var pixelFormatFenceCapabilityProbePayload: Data? private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 0f2187b4..d7a88fe4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -116,7 +116,12 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - #expect(connection.clientToServerMessageQueue.dequeue() == nil) + let probe = try #require(connection.clientToServerMessageQueue.dequeue()) + let probeWriter = AuditWritingConnection() + try await probe.message.send(connection: probeWriter) + #expect(probeWriter.data.count == 10) + #expect(probeWriter.data[0] == 3) + #expect(probeWriter.data[1] == 0) connection.completeFramebufferUpdateRequest() let queued = try #require(connection.clientToServerMessageQueue.dequeue()) @@ -147,7 +152,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [.request, .syncNext], + flags: [.request, .blockAfter, .syncNext], payload: Data("support".utf8) ) ) @@ -155,8 +160,24 @@ struct AuditFindingsTests { let supportWriter = AuditWritingConnection() try await supportResponse.message.send(connection: supportWriter) #expect(supportWriter.data[0] == VNCProtocol.ClientFence.messageType) + #expect(supportWriter.data[4..<8] == Data([0, 0, 0, 0])) #expect(supportWriter.data[8] == 7) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityWriter = AuditWritingConnection() + try await capabilityProbe.message.send(connection: capabilityWriter) + let capabilityLength = Int(capabilityWriter.data[8]) + let capabilityPayload = Data(capabilityWriter.data[9..<(9 + capabilityLength)]) + #expect(capabilityWriter.data[0] == VNCProtocol.ClientFence.messageType) + #expect(capabilityWriter.data[(9 + capabilityLength)] == 0) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.syncNext], + payload: capabilityPayload + ) + ) + connection.updateColorDepth(.depth8Bit) let queued = try #require(connection.clientToServerMessageQueue.dequeue()) let writer = AuditWritingConnection { @@ -184,6 +205,51 @@ struct AuditFindingsTests { #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } + @Test + func fallsBackToForcedUpdateWhenSyncNextIsUnsupported() async throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + connection.framebufferUpdateRequestOutstanding = true + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request], + payload: Data() + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityWriter = AuditWritingConnection() + try await capabilityProbe.message.send(connection: capabilityWriter) + let capabilityLength = Int(capabilityWriter.data[8]) + let capabilityPayload = Data(capabilityWriter.data[9..<(9 + capabilityLength)]) + + connection.updateColorDepth(.depth8Bit) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [], + payload: capabilityPayload + ) + ) + + let probe = try #require(connection.clientToServerMessageQueue.dequeue()) + let probeWriter = AuditWritingConnection() + try await probe.message.send(connection: probeWriter) + #expect(probeWriter.data[0] == 3) + #expect(probeWriter.data[1] == 0) + } + @Test func advertisesAndDecodesFenceExtension() async throws { let connection = VNCConnection(settings: makeSettings()) From 6ebb42d9eed731545b1df639d2428e5f2d1cf13b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:07:43 +0200 Subject: [PATCH 051/242] fix(vnc): correlate fallback format boundaries --- .../RoyalVNCKit/SDK/Connection/State.swift | 8 +-- .../SDK/Connection/VNCConnection+API.swift | 65 ++++++++++++++++--- .../Connection/VNCConnection+Receive.swift | 2 +- .../SDK/Connection/VNCConnection.swift | 2 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 46 +++++++++---- 5 files changed, 97 insertions(+), 26 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift index 65017b6a..423c11bf 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/State.swift @@ -19,7 +19,7 @@ extension VNCConnection { private var _desktopName: String? private var _incrementalUpdatesEnabled = false private var _areFencesSupported = false - private var _areSyncNextFencesSupported = false + private var _pixelFormatTransitionFenceFlags: VNCProtocol.FenceFlags = [] private var _areContinuousUpdatesSupported = false private var _areContinuousUpdatesEnabled = false private var _extendedClipboardServerCaps: VNCExtendedClipboardCaps? @@ -77,9 +77,9 @@ extension VNCConnection { set { withLock { _areFencesSupported = newValue } } } - var areSyncNextFencesSupported: Bool { - get { withLock { _areSyncNextFencesSupported } } - set { withLock { _areSyncNextFencesSupported = newValue } } + var pixelFormatTransitionFenceFlags: VNCProtocol.FenceFlags { + get { withLock { _pixelFormatTransitionFenceFlags } } + set { withLock { _pixelFormatTransitionFenceFlags = newValue } } } var areContinuousUpdatesSupported: Bool { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 6a0d2801..bb2ca6cc 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -28,6 +28,7 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { private struct PixelFormatTransition { let pixelFormat: VNCProtocol.PixelFormat + let fenceFlags: VNCProtocol.FenceFlags let fencePayload: Data? } @@ -129,24 +130,58 @@ extension VNCConnection { private func takePendingPixelFormatTransitionLocked() -> PixelFormatTransition? { guard !isPixelFormatTransitionInFlight, pixelFormatFenceCapabilityProbePayload == nil, - (!framebufferUpdateRequestOutstanding || state.areSyncNextFencesSupported), + !isPixelFormatTransitionProbeQueued, let pixelFormat = pendingPixelFormatTransition else { return nil } + let supportedFenceFlags = state.pixelFormatTransitionFenceFlags + let requiresFence = framebufferUpdateRequestOutstanding + || pixelFormatTransitionRequiresFence + || state.areContinuousUpdatesEnabled + var fenceFlags: VNCProtocol.FenceFlags = [] + if requiresFence { + if framebufferUpdateRequestOutstanding { + if supportedFenceFlags.contains(.syncNext) { + fenceFlags = [.request, .syncNext] + if supportedFenceFlags.contains(.blockAfter) { + fenceFlags.insert(.blockAfter) + } + } else { + return nil + } + } else if supportedFenceFlags.contains(.blockAfter) { + fenceFlags = [.request, .blockAfter] + if supportedFenceFlags.contains(.blockBefore) { + fenceFlags.insert(.blockBefore) + } + } else { + return nil + } + } + pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = true pixelFormatTransitionInFlight = pixelFormat let fencePayload: Data? - if state.areSyncNextFencesSupported { + if !fenceFlags.isEmpty { pixelFormatTransitionFenceSequence &+= 1 var sequence = pixelFormatTransitionFenceSequence.bigEndian fencePayload = withUnsafeBytes(of: &sequence) { Data($0) } pixelFormatTransitionFencePayload = fencePayload + pixelFormatTransitionRequiredFenceFlags = fenceFlags.contains(.syncNext) + ? [.syncNext] + : [.blockAfter] } else { fencePayload = nil + pixelFormatTransitionRequiredFenceFlags = [] } - return PixelFormatTransition(pixelFormat: pixelFormat, fencePayload: fencePayload) + pixelFormatTransitionRequiresFence = false + return PixelFormatTransition( + pixelFormat: pixelFormat, + fenceFlags: fenceFlags, + fencePayload: fencePayload + ) } private func takePixelFormatTransitionProbeLocked() -> VNCProtocol.FramebufferUpdateRequest? { @@ -154,13 +189,15 @@ extension VNCConnection { !isPixelFormatTransitionProbeQueued, !isPixelFormatTransitionInFlight, pixelFormatFenceCapabilityProbePayload == nil, - !state.areSyncNextFencesSupported, + !state.pixelFormatTransitionFenceFlags.contains(.syncNext), + state.pixelFormatTransitionFenceFlags.contains(.blockAfter), pendingPixelFormatTransition != nil, let framebuffer else { return nil } isPixelFormatTransitionProbeQueued = true + pixelFormatTransitionRequiresFence = true return VNCProtocol.FramebufferUpdateRequest( incremental: false, xPosition: 0, @@ -172,7 +209,7 @@ extension VNCConnection { private func enqueuePixelFormatTransition(_ transition: PixelFormatTransition) { let fenceMessage = transition.fencePayload.map { - VNCProtocol.ClientFence(flags: [.request, .syncNext], payload: $0) + VNCProtocol.ClientFence(flags: transition.fenceFlags, payload: $0) } let message = PixelFormatTransitionMessage( fenceMessage: fenceMessage, @@ -187,21 +224,21 @@ extension VNCConnection { enqueueClientToServerMessage(message) } - func probeSyncNextFenceSupport() { + func probePixelFormatFenceSupport() { framebufferRequestLock.lock() guard pixelFormatFenceCapabilityProbePayload == nil, let pixelFormat = state.pixelFormat else { framebufferRequestLock.unlock() return } - let payload = Data("royalvnc-sync-next".utf8) + let payload = Data("royalvnc-pixel-format".utf8) pixelFormatFenceCapabilityProbePayload = payload framebufferRequestLock.unlock() enqueueClientToServerMessage( FenceCapabilityProbeMessage( fenceMessage: VNCProtocol.ClientFence( - flags: [.request, .syncNext], + flags: [.request, .blockBefore, .blockAfter, .syncNext], payload: payload ), pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat) @@ -213,7 +250,11 @@ extension VNCConnection { framebufferRequestLock.lock() if fence.payload == pixelFormatFenceCapabilityProbePayload { pixelFormatFenceCapabilityProbePayload = nil - state.areSyncNextFencesSupported = fence.flags.contains(.syncNext) + state.pixelFormatTransitionFenceFlags = fence.flags.intersection([ + .blockBefore, + .blockAfter, + .syncNext + ]) let transition = takePendingPixelFormatTransitionLocked() let probe = takePixelFormatTransitionProbeLocked() framebufferRequestLock.unlock() @@ -229,7 +270,8 @@ extension VNCConnection { framebufferRequestLock.unlock() return } - guard fence.flags.contains(.syncNext) else { + let requiredFlags = pixelFormatTransitionRequiredFenceFlags + guard fence.flags.intersection(requiredFlags) == requiredFlags else { framebufferRequestLock.unlock() throw VNCError.protocol(.invalidData) } @@ -238,6 +280,7 @@ extension VNCConnection { throw VNCError.protocol(.invalidData) } pixelFormatTransitionFencePayload = nil + pixelFormatTransitionRequiredFenceFlags = [] framebufferRequestLock.unlock() beginPixelFormatTransition(pixelFormat) @@ -516,7 +559,9 @@ extension VNCConnection { isPixelFormatTransitionInFlight = false pixelFormatTransitionInFlight = nil isPixelFormatTransitionProbeQueued = false + pixelFormatTransitionRequiresFence = false pixelFormatTransitionFencePayload = nil + pixelFormatTransitionRequiredFenceFlags = [] pixelFormatFenceCapabilityProbePayload = nil framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift index bf28410a..5bab385b 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift @@ -221,7 +221,7 @@ extension VNCConnection { if first { logger.logDebug("Fence supported (server sent ServerFence)") - probeSyncNextFenceSupport() + probePixelFormatFenceSupport() } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 06a76453..f77775d1 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -114,8 +114,10 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var isPixelFormatTransitionInFlight = false var pixelFormatTransitionInFlight: VNCProtocol.PixelFormat? var isPixelFormatTransitionProbeQueued = false + var pixelFormatTransitionRequiresFence = false var pixelFormatTransitionFenceSequence: UInt64 = 0 var pixelFormatTransitionFencePayload: Data? + var pixelFormatTransitionRequiredFenceFlags: VNCProtocol.FenceFlags = [] var pixelFormatFenceCapabilityProbePayload: Data? private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index d7a88fe4..0181c7df 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -116,12 +116,7 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - let probe = try #require(connection.clientToServerMessageQueue.dequeue()) - let probeWriter = AuditWritingConnection() - try await probe.message.send(connection: probeWriter) - #expect(probeWriter.data.count == 10) - #expect(probeWriter.data[0] == 3) - #expect(probeWriter.data[1] == 0) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) connection.completeFramebufferUpdateRequest() let queued = try #require(connection.clientToServerMessageQueue.dequeue()) @@ -173,7 +168,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [.syncNext], + flags: [.blockAfter, .syncNext], payload: capabilityPayload ) ) @@ -187,7 +182,7 @@ struct AuditFindingsTests { #expect(writer.data.count == 37) #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) - #expect(writer.data[4..<8] == Data([0x80, 0, 0, 4])) + #expect(writer.data[4..<8] == Data([0x80, 0, 0, 6])) #expect(writer.data[8] == 8) #expect(writer.data[17] == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType) #expect(connection.state.pixelFormat?.depth == 24) @@ -196,7 +191,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [.syncNext], + flags: [.blockAfter, .syncNext], payload: payload ) ) @@ -206,7 +201,7 @@ struct AuditFindingsTests { } @Test - func fallsBackToForcedUpdateWhenSyncNextIsUnsupported() async throws { + func fencesForcedUpdateWhenSyncNextIsUnsupported() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -238,7 +233,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [], + flags: [.blockBefore, .blockAfter], payload: capabilityPayload ) ) @@ -248,6 +243,35 @@ struct AuditFindingsTests { try await probe.message.send(connection: probeWriter) #expect(probeWriter.data[0] == 3) #expect(probeWriter.data[1] == 0) + + connection.completeFramebufferUpdateRequest() + let transition = try #require(connection.clientToServerMessageQueue.dequeue()) + let transitionWriter = AuditWritingConnection { + #expect(connection.state.pixelFormat?.depth == 24) + } + try await transition.message.send(connection: transitionWriter) + #expect(transitionWriter.data.count == 37) + #expect(transitionWriter.data[0] == VNCProtocol.ClientFence.messageType) + #expect(transitionWriter.data[4..<8] == Data([0x80, 0, 0, 3])) + #expect(transitionWriter.data[8] == 8) + #expect(transitionWriter.data[17] == 0) + #expect(connection.state.pixelFormat?.depth == 24) + + connection.completeFramebufferUpdateRequest() + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + #expect(connection.state.pixelFormat?.depth == 24) + + let transitionPayload = Data(transitionWriter.data[9..<17]) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .blockAfter], + payload: transitionPayload + ) + ) + + #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } @Test From fd52c2b29e956668015492262d4052c83b914bb9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:14:23 +0200 Subject: [PATCH 052/242] fix(vnc): fail closed on partial fence support --- .../SDK/Connection/VNCConnection+API.swift | 38 +++++++++++++++---- .../RoyalVNCKitTests/AuditFindingsTests.swift | 31 ++++++++++++++- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index bb2ca6cc..8dfdd479 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -136,6 +136,9 @@ extension VNCConnection { } let supportedFenceFlags = state.pixelFormatTransitionFenceFlags + let supportsFallbackBoundary = + supportedFenceFlags.contains(.blockBefore) + && supportedFenceFlags.contains(.blockAfter) let requiresFence = framebufferUpdateRequestOutstanding || pixelFormatTransitionRequiresFence || state.areContinuousUpdatesEnabled @@ -147,15 +150,30 @@ extension VNCConnection { if supportedFenceFlags.contains(.blockAfter) { fenceFlags.insert(.blockAfter) } + } else if state.areFencesSupported && !supportsFallbackBoundary { + pendingPixelFormatTransition = nil + pixelFormatTransitionRequiresFence = false + return nil } else { return nil } - } else if supportedFenceFlags.contains(.blockAfter) { - fenceFlags = [.request, .blockAfter] - if supportedFenceFlags.contains(.blockBefore) { - fenceFlags.insert(.blockBefore) + } else if pixelFormatTransitionRequiresFence { + guard supportsFallbackBoundary else { + pendingPixelFormatTransition = nil + pixelFormatTransitionRequiresFence = false + return nil + } + fenceFlags = [.request, .blockBefore, .blockAfter] + } else if supportedFenceFlags.contains(.syncNext) { + fenceFlags = [.request, .syncNext] + if supportedFenceFlags.contains(.blockAfter) { + fenceFlags.insert(.blockAfter) } + } else if supportsFallbackBoundary { + fenceFlags = [.request, .blockBefore, .blockAfter] } else { + pendingPixelFormatTransition = nil + pixelFormatTransitionRequiresFence = false return nil } } @@ -171,7 +189,7 @@ extension VNCConnection { pixelFormatTransitionFencePayload = fencePayload pixelFormatTransitionRequiredFenceFlags = fenceFlags.contains(.syncNext) ? [.syncNext] - : [.blockAfter] + : [.blockBefore, .blockAfter] } else { fencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] @@ -185,19 +203,23 @@ extension VNCConnection { } private func takePixelFormatTransitionProbeLocked() -> VNCProtocol.FramebufferUpdateRequest? { + let supportedFenceFlags = state.pixelFormatTransitionFenceFlags + let supportsFallbackBoundary = + supportedFenceFlags.contains(.blockBefore) + && supportedFenceFlags.contains(.blockAfter) guard framebufferUpdateRequestOutstanding, !isPixelFormatTransitionProbeQueued, !isPixelFormatTransitionInFlight, pixelFormatFenceCapabilityProbePayload == nil, - !state.pixelFormatTransitionFenceFlags.contains(.syncNext), - state.pixelFormatTransitionFenceFlags.contains(.blockAfter), + !supportedFenceFlags.contains(.syncNext), + (!state.areFencesSupported || supportsFallbackBoundary), pendingPixelFormatTransition != nil, let framebuffer else { return nil } isPixelFormatTransitionProbeQueued = true - pixelFormatTransitionRequiresFence = true + pixelFormatTransitionRequiresFence = supportsFallbackBoundary return VNCProtocol.FramebufferUpdateRequest( incremental: false, xPosition: 0, diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 0181c7df..f62a0f22 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -101,7 +101,7 @@ struct AuditFindingsTests { } @Test - func waitsForOutstandingFramebufferBeforeUnfencedPixelFormatTransition() async throws { + func forcesLegacyFramebufferBeforeUnfencedPixelFormatTransition() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -116,7 +116,12 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - #expect(connection.clientToServerMessageQueue.dequeue() == nil) + let probe = try #require(connection.clientToServerMessageQueue.dequeue()) + let probeWriter = AuditWritingConnection() + try await probe.message.send(connection: probeWriter) + #expect(probeWriter.data.count == 10) + #expect(probeWriter.data[0] == 3) + #expect(probeWriter.data[1] == 0) connection.completeFramebufferUpdateRequest() let queued = try #require(connection.clientToServerMessageQueue.dequeue()) @@ -131,6 +136,28 @@ struct AuditFindingsTests { #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } + @Test + func rejectsPartialFallbackFenceBoundaries() throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.state.areFencesSupported = true + connection.state.pixelFormatTransitionFenceFlags = [.blockAfter] + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + connection.framebufferUpdateRequestOutstanding = true + + connection.updateColorDepth(.depth8Bit) + + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + #expect(connection.pendingPixelFormatTransition == nil) + #expect(connection.state.pixelFormat?.depth == 24) + } + @Test func synchronizesPixelFormatTransitionWithFenceResponse() async throws { let connection = VNCConnection( From 7df941d112d97612ca305a427443c0a8fd15fb16 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:19:29 +0200 Subject: [PATCH 053/242] fix(vnc): reject unsafe legacy format switches --- .../SDK/Connection/VNCConnection+API.swift | 5 ++-- .../RoyalVNCKitTests/AuditFindingsTests.swift | 23 ++++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 8dfdd479..4c42fdc1 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -150,7 +150,7 @@ extension VNCConnection { if supportedFenceFlags.contains(.blockAfter) { fenceFlags.insert(.blockAfter) } - } else if state.areFencesSupported && !supportsFallbackBoundary { + } else if !state.areFencesSupported || !supportsFallbackBoundary { pendingPixelFormatTransition = nil pixelFormatTransitionRequiresFence = false return nil @@ -212,7 +212,8 @@ extension VNCConnection { !isPixelFormatTransitionInFlight, pixelFormatFenceCapabilityProbePayload == nil, !supportedFenceFlags.contains(.syncNext), - (!state.areFencesSupported || supportsFallbackBoundary), + state.areFencesSupported, + supportsFallbackBoundary, pendingPixelFormatTransition != nil, let framebuffer else { return nil diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index f62a0f22..e345bff7 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -101,7 +101,7 @@ struct AuditFindingsTests { } @Test - func forcesLegacyFramebufferBeforeUnfencedPixelFormatTransition() async throws { + func rejectsUnsafeLegacyPixelFormatTransition() throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -114,26 +114,11 @@ struct AuditFindingsTests { connection.framebufferUpdateRequestOutstanding = true connection.updateColorDepth(.depth8Bit) - #expect(connection.state.pixelFormat?.depth == 24) - #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - let probe = try #require(connection.clientToServerMessageQueue.dequeue()) - let probeWriter = AuditWritingConnection() - try await probe.message.send(connection: probeWriter) - #expect(probeWriter.data.count == 10) - #expect(probeWriter.data[0] == 3) - #expect(probeWriter.data[1] == 0) - connection.completeFramebufferUpdateRequest() - let queued = try #require(connection.clientToServerMessageQueue.dequeue()) - let writer = AuditWritingConnection { - #expect(connection.state.pixelFormat?.depth == 8) - #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) - } - try await queued.message.send(connection: writer) - - #expect(writer.data.count == 20) - #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + #expect(connection.pendingPixelFormatTransition == nil) } @Test From eedc4699a4b42c01387c5951ad972260853cc430 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:19:29 +0200 Subject: [PATCH 054/242] docs(changelog): clarify legacy VNC handling --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a14f4a5c..a817b4ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with a conservative legacy fallback, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From b361ac7fdf1c87f37ca71250c5f296c4061025f0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:20:50 +0200 Subject: [PATCH 055/242] fix(macos): clear stale non-text clipboard cache --- macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift | 2 +- .../Tests/CrabfleetMacTests/HostShareProtocolTests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift index 162ca617..b5ac18c9 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/HostClipboard.swift @@ -121,10 +121,10 @@ final class HostClipboardBridge: HostClipboardSyncing, @unchecked Sendable { suppressedChangeCount = nil return } + lastKnownText = text guard let outboundText = text ?? (types.isEmpty ? "" : nil) else { return } - lastKnownText = text guard outboundText.utf8.count <= RFBWire.maximumClipboardBytes else { return } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift index 8c6199d5..f5eee6cd 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/HostShareProtocolTests.swift @@ -212,7 +212,7 @@ struct HostClipboardBridgeTests { pasteboard.writeObjects([item]) bridge.poll() #expect(recorder.values.isEmpty) - #expect(bridge.currentText() == "initial") + #expect(bridge.currentText() == nil) bridge.receiveClientText("initial") try await waitUntil { pasteboard.string(forType: .string) == "initial" } From e75959a9812e6c70e8c2f7f025f1c5c70158e56b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:26:44 +0200 Subject: [PATCH 056/242] fix(vnc): await initial fence negotiation --- .../SDK/Connection/VNCConnection+API.swift | 43 ++++++++++++++++++- .../SDK/Connection/VNCConnection.swift | 1 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 38 +++++++++++++++- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 4c42fdc1..409d9427 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -150,7 +150,10 @@ extension VNCConnection { if supportedFenceFlags.contains(.blockAfter) { fenceFlags.insert(.blockAfter) } - } else if !state.areFencesSupported || !supportsFallbackBoundary { + } else if !state.areFencesSupported { + schedulePixelFormatFenceNegotiationTimeoutLocked() + return nil + } else if !supportsFallbackBoundary { pendingPixelFormatTransition = nil pixelFormatTransitionRequiresFence = false return nil @@ -178,6 +181,7 @@ extension VNCConnection { } } + cancelPixelFormatFenceNegotiationTimeoutLocked() pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = true pixelFormatTransitionInFlight = pixelFormat @@ -249,6 +253,7 @@ extension VNCConnection { func probePixelFormatFenceSupport() { framebufferRequestLock.lock() + cancelPixelFormatFenceNegotiationTimeoutLocked() guard pixelFormatFenceCapabilityProbePayload == nil, let pixelFormat = state.pixelFormat else { framebufferRequestLock.unlock() @@ -269,6 +274,41 @@ extension VNCConnection { ) } + private func schedulePixelFormatFenceNegotiationTimeoutLocked() { + guard pixelFormatFenceNegotiationTask == nil else { return } + + pixelFormatFenceNegotiationTask = Task { [weak self] in + do { + try await Task.sleep(nanoseconds: 1_000_000_000) + } catch { + return + } + self?.expirePixelFormatFenceNegotiation() + } + } + + private func cancelPixelFormatFenceNegotiationTimeoutLocked() { + pixelFormatFenceNegotiationTask?.cancel() + pixelFormatFenceNegotiationTask = nil + } + + func expirePixelFormatFenceNegotiation() { + framebufferRequestLock.lock() + cancelPixelFormatFenceNegotiationTimeoutLocked() + guard !state.areFencesSupported, + framebufferUpdateRequestOutstanding, + pendingPixelFormatTransition != nil, + !isPixelFormatTransitionInFlight else { + framebufferRequestLock.unlock() + return + } + pendingPixelFormatTransition = nil + pixelFormatTransitionRequiresFence = false + framebufferRequestLock.unlock() + + logger.logDebug("Rejecting pixel format transition because Fence support was not negotiated") + } + func completePixelFormatFence(_ fence: VNCProtocol.ServerFence) throws { framebufferRequestLock.lock() if fence.payload == pixelFormatFenceCapabilityProbePayload { @@ -586,6 +626,7 @@ extension VNCConnection { pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] pixelFormatFenceCapabilityProbePayload = nil + cancelPixelFormatFenceNegotiationTimeoutLocked() framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index f77775d1..6d10139b 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -119,6 +119,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pixelFormatTransitionFencePayload: Data? var pixelFormatTransitionRequiredFenceFlags: VNCProtocol.FenceFlags = [] var pixelFormatFenceCapabilityProbePayload: Data? + var pixelFormatFenceNegotiationTask: Task? private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index e345bff7..8467a4d8 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -101,7 +101,7 @@ struct AuditFindingsTests { } @Test - func rejectsUnsafeLegacyPixelFormatTransition() throws { + func rejectsUnsafeLegacyPixelFormatTransitionAfterNegotiationTimeout() throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -115,12 +115,48 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) + #expect(connection.pendingPixelFormatTransition?.depth == 8) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + + connection.expirePixelFormatFenceNegotiation() + #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) #expect(connection.clientToServerMessageQueue.dequeue() == nil) #expect(connection.pendingPixelFormatTransition == nil) } + @Test + func preservesEarlyPixelFormatTransitionUntilFenceSupportArrives() throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + connection.framebufferUpdateRequestOutstanding = true + + connection.updateColorDepth(.depth8Bit) + + #expect(connection.pendingPixelFormatTransition?.depth == 8) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockAfter, .syncNext], + payload: Data("support".utf8) + ) + ) + + #expect(connection.pendingPixelFormatTransition?.depth == 8) + #expect(connection.clientToServerMessageQueue.dequeue() != nil) + #expect(connection.clientToServerMessageQueue.dequeue() != nil) + } + @Test func rejectsPartialFallbackFenceBoundaries() throws { let connection = VNCConnection( From 4cbb4f7f214d8615f7931cab12683c41cb4a075b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:30:51 +0200 Subject: [PATCH 057/242] test(events): follow ledger store contract --- tests/session-events.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/session-events.test.ts b/tests/session-events.test.ts index 48edce28..30d95bc5 100644 --- a/tests/session-events.test.ts +++ b/tests/session-events.test.ts @@ -426,10 +426,11 @@ test("structured session events require bounded identifiers and a versioned obje test("structured session events retain valid supplementary Unicode", async () => { let payloadJson = ""; const service = new InteractiveSessionEventLedgerService({ - async persistAndInvalidate(event) { + async persist(event) { payloadJson = event.payloadJson; return { inserted: true, + refreshArchive: false, row: { id: 1, session_id: event.sessionId, From 6c6af3f2257b74f3a5a88f1d04478e293ba0ebc0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:40:39 +0200 Subject: [PATCH 058/242] fix(events): defer request archive publication --- src/worker/interactive-session-application.ts | 7 +++++-- src/worker/session-events.ts | 9 ++++++++- tests/session-events.test.ts | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/worker/interactive-session-application.ts b/src/worker/interactive-session-application.ts index 9d73e929..c2708924 100644 --- a/src/worker/interactive-session-application.ts +++ b/src/worker/interactive-session-application.ts @@ -24,7 +24,10 @@ import { sandboxLeasePrefix, } from "./sandbox-lease.ts"; import { ServiceRegistry } from "./service-registry.ts"; -import { appendInteractiveSessionEventRecord } from "./session-events.ts"; +import { + appendInteractiveSessionEventRecord, + persistInteractiveSessionEventRecord, +} from "./session-events.ts"; import { canChangeInteractiveSessionMultiplayer, canControlInteractiveSession, @@ -480,7 +483,7 @@ export class InteractiveSessionApplication { activateReservation: (insertedSessionId, insertedAt, adapterWorkspaceId) => supervision.requireReservationActivation(insertedSessionId, insertedAt, adapterWorkspaceId), recordRequest: (insertedSessionId, insertedAt) => - appendInteractiveSessionEventRecord(this.env, { + persistInteractiveSessionEventRecord(this.env, { sessionId: insertedSessionId, actor: actor(user), message: "interactive workspace requested", diff --git a/src/worker/session-events.ts b/src/worker/session-events.ts index e07c1f1e..c46a95e1 100644 --- a/src/worker/session-events.ts +++ b/src/worker/session-events.ts @@ -106,6 +106,14 @@ export async function appendInteractiveSessionEventRecord( input: AppendInteractiveSessionEventInput, archive: InteractiveSessionEventArchive = (sessionId, now) => archiveInteractiveSessionLogs(env, sessionId, now), +): Promise { + await persistInteractiveSessionEventRecord(env, input); + await archive(input.sessionId, input.now).catch(() => undefined); +} + +export async function persistInteractiveSessionEventRecord( + env: RuntimeEnv, + input: AppendInteractiveSessionEventInput, ): Promise { const db = database(env); await executeBatch(env, [ @@ -117,7 +125,6 @@ export async function appendInteractiveSessionEventRecord( }), terminalFinalizationPendingQuery(db, input.sessionId), ]); - await archive(input.sessionId, input.now).catch(() => undefined); } export async function appendStructuredInteractiveSessionEventRecord( diff --git a/tests/session-events.test.ts b/tests/session-events.test.ts index 30d95bc5..f53325fe 100644 --- a/tests/session-events.test.ts +++ b/tests/session-events.test.ts @@ -6,6 +6,7 @@ import { appendInteractiveSessionEventRecord, appendStructuredInteractiveSessionEventRecord, InteractiveSessionEventLedgerService, + persistInteractiveSessionEventRecord, structuredEventLedgerMaxBytes, structuredEventLedgerMaxCount, structuredEventPayloadMaxBytes, @@ -93,6 +94,24 @@ test("session event archive refresh remains best effort after durable persistenc assert.equal(persisted, true); }); +test("session event persistence can defer archive publication", async () => { + let statements: PreparedStatement[] = []; + const env = runtimeEnv((batch) => { + statements = batch; + }); + + await persistInteractiveSessionEventRecord(env, { + sessionId: "IS-1", + actor: "operator", + message: "interactive workspace requested", + now: 123, + }); + + assert.equal(statements.length, 2); + assert.match(statements[0]?.sql ?? "", /insert into "interactive_session_events"/i); + assert.match(statements[1]?.sql ?? "", /update "interactive_sessions"/i); +}); + test("structured session events canonicalize additive payloads and replay idempotently", async () => { let row: InteractiveSessionEventRow | undefined; const persistedPayloads: string[] = []; From 98d44715336f33848ed1811be761a841b8fbaa8d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:40:59 +0200 Subject: [PATCH 059/242] fix(macos): preserve unowned desktop registrations --- .../Sources/CrabfleetMac/PrivateMacShareController.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index adaa19bd..1763f3c2 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -477,13 +477,18 @@ final class PrivateMacShareController: ObservableObject { } private func removeDesktopHost(after pendingRegistration: Task?) { + guard let pendingRegistration else { + registrationTask = nil + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished + return + } guard let desktopRegistrationCoordinator, let activeIdentity else { registrationTask = nil registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } registrationTask = Task { [weak self] in - await pendingRegistration?.value + await pendingRegistration.value do { try await desktopRegistrationCoordinator.unregister(identity: activeIdentity) self?.registryPhase = .notPublished From 86e124da5d26d8117948d6c22625e2f5cccc3ae1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:42:54 +0200 Subject: [PATCH 060/242] fix(vnc): require synchronized format boundaries --- .../SDK/Connection/VNCConnection+API.swift | 93 +++---------------- .../SDK/Connection/VNCConnection.swift | 2 - .../RoyalVNCKitTests/AuditFindingsTests.swift | 46 +++------ 3 files changed, 28 insertions(+), 113 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 409d9427..95ddd364 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -117,66 +117,42 @@ extension VNCConnection { framebufferPacingTask?.cancel() framebufferPacingTask = nil let transition = takePendingPixelFormatTransitionLocked() - let probe = takePixelFormatTransitionProbeLocked() framebufferRequestLock.unlock() if let transition { enqueuePixelFormatTransition(transition) - } else if let probe { - enqueueClientToServerMessage(probe) } } private func takePendingPixelFormatTransitionLocked() -> PixelFormatTransition? { guard !isPixelFormatTransitionInFlight, pixelFormatFenceCapabilityProbePayload == nil, - !isPixelFormatTransitionProbeQueued, let pixelFormat = pendingPixelFormatTransition else { return nil } let supportedFenceFlags = state.pixelFormatTransitionFenceFlags - let supportsFallbackBoundary = + let supportsSynchronizedBoundary = supportedFenceFlags.contains(.blockBefore) - && supportedFenceFlags.contains(.blockAfter) - let requiresFence = framebufferUpdateRequestOutstanding - || pixelFormatTransitionRequiresFence - || state.areContinuousUpdatesEnabled + && supportedFenceFlags.contains(.syncNext) var fenceFlags: VNCProtocol.FenceFlags = [] - if requiresFence { - if framebufferUpdateRequestOutstanding { - if supportedFenceFlags.contains(.syncNext) { - fenceFlags = [.request, .syncNext] - if supportedFenceFlags.contains(.blockAfter) { - fenceFlags.insert(.blockAfter) - } - } else if !state.areFencesSupported { + if state.areContinuousUpdatesEnabled { + guard supportsSynchronizedBoundary else { + if !state.areFencesSupported { schedulePixelFormatFenceNegotiationTimeoutLocked() - return nil - } else if !supportsFallbackBoundary { - pendingPixelFormatTransition = nil - pixelFormatTransitionRequiresFence = false - return nil } else { - return nil - } - } else if pixelFormatTransitionRequiresFence { - guard supportsFallbackBoundary else { pendingPixelFormatTransition = nil - pixelFormatTransitionRequiresFence = false - return nil - } - fenceFlags = [.request, .blockBefore, .blockAfter] - } else if supportedFenceFlags.contains(.syncNext) { - fenceFlags = [.request, .syncNext] - if supportedFenceFlags.contains(.blockAfter) { - fenceFlags.insert(.blockAfter) } - } else if supportsFallbackBoundary { - fenceFlags = [.request, .blockBefore, .blockAfter] + return nil + } + fenceFlags = [.request, .blockBefore, .syncNext] + } else if framebufferUpdateRequestOutstanding { + if supportsSynchronizedBoundary { + fenceFlags = [.request, .blockBefore, .syncNext] } else { - pendingPixelFormatTransition = nil - pixelFormatTransitionRequiresFence = false + if !state.areFencesSupported { + schedulePixelFormatFenceNegotiationTimeoutLocked() + } return nil } } @@ -191,14 +167,11 @@ extension VNCConnection { var sequence = pixelFormatTransitionFenceSequence.bigEndian fencePayload = withUnsafeBytes(of: &sequence) { Data($0) } pixelFormatTransitionFencePayload = fencePayload - pixelFormatTransitionRequiredFenceFlags = fenceFlags.contains(.syncNext) - ? [.syncNext] - : [.blockBefore, .blockAfter] + pixelFormatTransitionRequiredFenceFlags = [.blockBefore, .syncNext] } else { fencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] } - pixelFormatTransitionRequiresFence = false return PixelFormatTransition( pixelFormat: pixelFormat, fenceFlags: fenceFlags, @@ -206,34 +179,6 @@ extension VNCConnection { ) } - private func takePixelFormatTransitionProbeLocked() -> VNCProtocol.FramebufferUpdateRequest? { - let supportedFenceFlags = state.pixelFormatTransitionFenceFlags - let supportsFallbackBoundary = - supportedFenceFlags.contains(.blockBefore) - && supportedFenceFlags.contains(.blockAfter) - guard framebufferUpdateRequestOutstanding, - !isPixelFormatTransitionProbeQueued, - !isPixelFormatTransitionInFlight, - pixelFormatFenceCapabilityProbePayload == nil, - !supportedFenceFlags.contains(.syncNext), - state.areFencesSupported, - supportsFallbackBoundary, - pendingPixelFormatTransition != nil, - let framebuffer else { - return nil - } - - isPixelFormatTransitionProbeQueued = true - pixelFormatTransitionRequiresFence = supportsFallbackBoundary - return VNCProtocol.FramebufferUpdateRequest( - incremental: false, - xPosition: 0, - yPosition: 0, - width: framebuffer.size.width, - height: framebuffer.size.height - ) - } - private func enqueuePixelFormatTransition(_ transition: PixelFormatTransition) { let fenceMessage = transition.fencePayload.map { VNCProtocol.ClientFence(flags: transition.fenceFlags, payload: $0) @@ -296,14 +241,12 @@ extension VNCConnection { framebufferRequestLock.lock() cancelPixelFormatFenceNegotiationTimeoutLocked() guard !state.areFencesSupported, - framebufferUpdateRequestOutstanding, pendingPixelFormatTransition != nil, !isPixelFormatTransitionInFlight else { framebufferRequestLock.unlock() return } pendingPixelFormatTransition = nil - pixelFormatTransitionRequiresFence = false framebufferRequestLock.unlock() logger.logDebug("Rejecting pixel format transition because Fence support was not negotiated") @@ -319,13 +262,10 @@ extension VNCConnection { .syncNext ]) let transition = takePendingPixelFormatTransitionLocked() - let probe = takePixelFormatTransitionProbeLocked() framebufferRequestLock.unlock() if let transition { enqueuePixelFormatTransition(transition) - } else if let probe { - enqueueClientToServerMessage(probe) } return } @@ -558,7 +498,6 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() framebufferUpdateRequestOutstanding = false - isPixelFormatTransitionProbeQueued = false let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() @@ -621,8 +560,6 @@ extension VNCConnection { pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = false pixelFormatTransitionInFlight = nil - isPixelFormatTransitionProbeQueued = false - pixelFormatTransitionRequiresFence = false pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] pixelFormatFenceCapabilityProbePayload = nil diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 6d10139b..f98b901c 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -113,8 +113,6 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pendingPixelFormatTransition: VNCProtocol.PixelFormat? var isPixelFormatTransitionInFlight = false var pixelFormatTransitionInFlight: VNCProtocol.PixelFormat? - var isPixelFormatTransitionProbeQueued = false - var pixelFormatTransitionRequiresFence = false var pixelFormatTransitionFenceSequence: UInt64 = 0 var pixelFormatTransitionFencePayload: Data? var pixelFormatTransitionRequiredFenceFlags: VNCProtocol.FenceFlags = [] diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 8467a4d8..b3db87d0 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -158,7 +158,7 @@ struct AuditFindingsTests { } @Test - func rejectsPartialFallbackFenceBoundaries() throws { + func rejectsPartialFenceBoundariesDuringContinuousUpdates() throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -168,6 +168,7 @@ struct AuditFindingsTests { connection.state.pixelFormat = framebuffer.sourcePixelFormat connection.state.areFencesSupported = true connection.state.pixelFormatTransitionFenceFlags = [.blockAfter] + connection.state.areContinuousUpdatesEnabled = true connection.connectionState = .connected connection._framebufferUpdatePolicy = .paused connection.framebufferUpdateRequestOutstanding = true @@ -195,7 +196,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [.request, .blockAfter, .syncNext], + flags: [.request, .blockBefore, .syncNext], payload: Data("support".utf8) ) ) @@ -203,7 +204,7 @@ struct AuditFindingsTests { let supportWriter = AuditWritingConnection() try await supportResponse.message.send(connection: supportWriter) #expect(supportWriter.data[0] == VNCProtocol.ClientFence.messageType) - #expect(supportWriter.data[4..<8] == Data([0, 0, 0, 0])) + #expect(supportWriter.data[4..<8] == Data([0, 0, 0, 1])) #expect(supportWriter.data[8] == 7) let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) @@ -216,7 +217,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [.blockAfter, .syncNext], + flags: [.blockBefore, .syncNext], payload: capabilityPayload ) ) @@ -230,7 +231,7 @@ struct AuditFindingsTests { #expect(writer.data.count == 37) #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) - #expect(writer.data[4..<8] == Data([0x80, 0, 0, 6])) + #expect(writer.data[4..<8] == Data([0x80, 0, 0, 5])) #expect(writer.data[8] == 8) #expect(writer.data[17] == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType) #expect(connection.state.pixelFormat?.depth == 24) @@ -239,7 +240,7 @@ struct AuditFindingsTests { try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, - flags: [.blockAfter, .syncNext], + flags: [.blockBefore, .syncNext], payload: payload ) ) @@ -249,7 +250,7 @@ struct AuditFindingsTests { } @Test - func fencesForcedUpdateWhenSyncNextIsUnsupported() async throws { + func waitsForOutstandingUpdateWhenSyncNextIsUnsupported() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -286,38 +287,17 @@ struct AuditFindingsTests { ) ) - let probe = try #require(connection.clientToServerMessageQueue.dequeue()) - let probeWriter = AuditWritingConnection() - try await probe.message.send(connection: probeWriter) - #expect(probeWriter.data[0] == 3) - #expect(probeWriter.data[1] == 0) - connection.completeFramebufferUpdateRequest() let transition = try #require(connection.clientToServerMessageQueue.dequeue()) let transitionWriter = AuditWritingConnection { - #expect(connection.state.pixelFormat?.depth == 24) + #expect(connection.state.pixelFormat?.depth == 8) } try await transition.message.send(connection: transitionWriter) - #expect(transitionWriter.data.count == 37) - #expect(transitionWriter.data[0] == VNCProtocol.ClientFence.messageType) - #expect(transitionWriter.data[4..<8] == Data([0x80, 0, 0, 3])) - #expect(transitionWriter.data[8] == 8) - #expect(transitionWriter.data[17] == 0) - #expect(connection.state.pixelFormat?.depth == 24) - - connection.completeFramebufferUpdateRequest() - #expect(connection.clientToServerMessageQueue.dequeue() == nil) - #expect(connection.state.pixelFormat?.depth == 24) - - let transitionPayload = Data(transitionWriter.data[9..<17]) - try connection.handleServerFence( - VNCProtocol.ServerFence( - messageType: VNCProtocol.ServerFence.messageType, - flags: [.blockBefore, .blockAfter], - payload: transitionPayload - ) + #expect(transitionWriter.data.count == 20) + #expect( + transitionWriter.data[0] + == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType ) - #expect(connection.state.pixelFormat?.depth == 8) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } From 90b8f22e018560737b331945eb13d9d85e076cb5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:51:21 +0200 Subject: [PATCH 061/242] fix(terminal): revalidate upstream before input ack --- src/worker/terminal-hub.ts | 18 +++++++++++- tests/terminal-hub.test.ts | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index a3059074..6d83ce3d 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -228,7 +228,23 @@ export class TerminalHub { const inputs = await this.dependencies.inputPayloads(subscription, user, frame.payload); for (const [index, input] of inputs.entries()) { if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); - subscription.upstream.send(input); + if ( + subscriptions.get(frame.sessionId) !== subscription || + subscription.upstream.readyState !== WebSocket.OPEN + ) { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: "terminal upstream is not open", + }); + return; + } + try { + subscription.upstream.send(input); + } catch { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: "terminal upstream send failed", + }); + return; + } } sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { type: "input-accepted", diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 383eb9f5..5587c68a 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -412,6 +412,65 @@ test("terminal hub publishes live controller downgrades and promotions", async ( server.emit("close"); }); +test("terminal hub never acknowledges input after its upstream closes", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let resolvePayloads: ((payloads: Uint8Array[]) => void) | undefined; + const payloads = new Promise((resolve) => { + resolvePayloads = resolve; + }); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async inputPayloads() { + return payloads; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: session.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: session.id, + payload: new TextEncoder().encode("dropped"), + }), + }); + await flushQueues(); + upstream.close(1011, "upstream failed"); + resolvePayloads?.([new TextEncoder().encode("dropped")]); + await flushQueues(); + + assert.deepEqual(upstream.sent, []); + const messages = server.sent.map((payload) => frame(payload)); + assert.equal( + messages.some( + (message) => + message.type === TerminalMessageType.Event && + (decodeJsonPayload(message.payload) as { type?: string }).type === "input-accepted", + ), + false, + ); + assert.deepEqual(decodeJsonPayload(messages.at(-1)!.payload), { + error: "terminal upstream is not open", + }); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From fc9b82a8ca355f9d85049a65f83f649aed665260 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:51:51 +0200 Subject: [PATCH 062/242] fix(vnc): bound fence capability probes --- .../SDK/Connection/VNCConnection+API.swift | 28 ++++++++++++--- .../RoyalVNCKitTests/AuditFindingsTests.swift | 34 +++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 95ddd364..24528ad8 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -206,6 +206,7 @@ extension VNCConnection { } let payload = Data("royalvnc-pixel-format".utf8) pixelFormatFenceCapabilityProbePayload = payload + schedulePixelFormatFenceNegotiationTimeoutLocked() framebufferRequestLock.unlock() enqueueClientToServerMessage( @@ -240,16 +241,33 @@ extension VNCConnection { func expirePixelFormatFenceNegotiation() { framebufferRequestLock.lock() cancelPixelFormatFenceNegotiationTimeoutLocked() - guard !state.areFencesSupported, - pendingPixelFormatTransition != nil, - !isPixelFormatTransitionInFlight else { + let probeTimedOut = pixelFormatFenceCapabilityProbePayload != nil + let negotiationTimedOut = + !state.areFencesSupported + && pendingPixelFormatTransition != nil + && !isPixelFormatTransitionInFlight + guard probeTimedOut || negotiationTimedOut else { framebufferRequestLock.unlock() return } - pendingPixelFormatTransition = nil + pixelFormatFenceCapabilityProbePayload = nil + if negotiationTimedOut { + pendingPixelFormatTransition = nil + } + let transition = probeTimedOut ? takePendingPixelFormatTransitionLocked() : nil + let shouldResumeUpdates = + transition == nil + && pendingPixelFormatTransition == nil + && !framebufferUpdateRequestOutstanding + && !isPixelFormatTransitionInFlight framebufferRequestLock.unlock() - logger.logDebug("Rejecting pixel format transition because Fence support was not negotiated") + if let transition { + enqueuePixelFormatTransition(transition) + } else if shouldResumeUpdates { + scheduleNextFramebufferUpdate() + } + logger.logDebug("Fence capability negotiation timed out") } func completePixelFormatFence(_ fence: VNCProtocol.ServerFence) throws { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index b3db87d0..5f0e9d43 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -157,6 +157,40 @@ struct AuditFindingsTests { #expect(connection.clientToServerMessageQueue.dequeue() != nil) } + @Test + func expiresUnansweredFenceCapabilityProbeWithoutStallingUpdates() async throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + + connection.updateColorDepth(.depth8Bit) + #expect(connection.pendingPixelFormatTransition?.depth == 8) + + connection.expirePixelFormatFenceNegotiation() + + #expect(connection.pixelFormatFenceCapabilityProbePayload == nil) + let transition = try #require(connection.clientToServerMessageQueue.dequeue()) + try await transition.message.send(connection: AuditWritingConnection()) + #expect(connection.pendingPixelFormatTransition == nil) + #expect(connection.state.pixelFormat?.depth == 8) + } + @Test func rejectsPartialFenceBoundariesDuringContinuousUpdates() throws { let connection = VNCConnection( From 532bf52358cd2fea0247b9381c8bb833c13572a7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:52:20 +0200 Subject: [PATCH 063/242] fix(macos): decouple stop from registry cleanup --- .../Sources/CrabfleetMac/PrivateMacShareController.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 1763f3c2..b8152147 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -338,7 +338,7 @@ final class PrivateMacShareController: ObservableObject { } if removedRegistryEntry { self.activeIdentity = nil } guard isCurrent(generation) else { return } - phase = removedRegistryEntry ? .idle : .failed + phase = .idle } func openPrivacySettings(_ pane: PrivacyPane) { From c4d8f3db04c5672a60a6d5a16c2085d50bf09e5f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:52:20 +0200 Subject: [PATCH 064/242] fix(macos): use monotonic command deadlines --- .../CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift index dbc87a68..666023f4 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift @@ -137,11 +137,12 @@ private final class TailscaleCommandExecution: @unchecked Sendable { } if currentStopReason() != nil { terminate() } - let deadline = Date().addingTimeInterval(timeout) + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: .seconds(timeout)) while process.isRunning { if currentStopReason() != nil { terminate() - } else if Date() >= deadline { + } else if clock.now >= deadline { stop(.timedOut) } Thread.sleep(forTimeInterval: 0.01) From 6c5aa559f1597a3e42100a59ba603d71140c22b9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:59:54 +0200 Subject: [PATCH 065/242] fix(vnc): preserve timeout synchronization boundaries --- .../SDK/Connection/VNCConnection+API.swift | 21 ++++++- .../RoyalVNCKitTests/AuditFindingsTests.swift | 58 ++++++++++++++++++- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 24528ad8..498c13a4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -35,12 +35,14 @@ private struct PixelFormatTransition { private struct FenceCapabilityProbeMessage: VNCSendableMessage { let fenceMessage: VNCProtocol.ClientFence let pixelFormatMessage: VNCProtocol.SetPixelFormat + let didSend: () -> Void var messageType: UInt8 { fenceMessage.messageType } var data: Data { fenceMessage.data + pixelFormatMessage.data } func send(connection: NetworkConnectionWriting) async throws { try await connection.write(data: data) + didSend() } } @@ -206,7 +208,6 @@ extension VNCConnection { } let payload = Data("royalvnc-pixel-format".utf8) pixelFormatFenceCapabilityProbePayload = payload - schedulePixelFormatFenceNegotiationTimeoutLocked() framebufferRequestLock.unlock() enqueueClientToServerMessage( @@ -215,11 +216,21 @@ extension VNCConnection { flags: [.request, .blockBefore, .blockAfter, .syncNext], payload: payload ), - pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat) + pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat), + didSend: { [weak self] in + self?.didSendPixelFormatFenceCapabilityProbe(payload: payload) + } ) ) } + private func didSendPixelFormatFenceCapabilityProbe(payload: Data) { + framebufferRequestLock.lock() + defer { framebufferRequestLock.unlock() } + guard pixelFormatFenceCapabilityProbePayload == payload else { return } + schedulePixelFormatFenceNegotiationTimeoutLocked() + } + private func schedulePixelFormatFenceNegotiationTimeoutLocked() { guard pixelFormatFenceNegotiationTask == nil else { return } @@ -251,7 +262,11 @@ extension VNCConnection { return } pixelFormatFenceCapabilityProbePayload = nil - if negotiationTimedOut { + let isWaitingForLegacyFramebufferBoundary = + negotiationTimedOut + && !state.areContinuousUpdatesEnabled + && framebufferUpdateRequestOutstanding + if negotiationTimedOut && !isWaitingForLegacyFramebufferBoundary { pendingPixelFormatTransition = nil } let transition = probeTimedOut ? takePendingPixelFormatTransitionLocked() : nil diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 5f0e9d43..b91d83cb 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -101,7 +101,7 @@ struct AuditFindingsTests { } @Test - func rejectsUnsafeLegacyPixelFormatTransitionAfterNegotiationTimeout() throws { + func retainsLegacyPixelFormatTransitionUntilSlowFramebufferUpdateCompletes() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -117,13 +117,26 @@ struct AuditFindingsTests { #expect(connection.pendingPixelFormatTransition?.depth == 8) #expect(connection.clientToServerMessageQueue.dequeue() == nil) + #expect(connection.pixelFormatFenceNegotiationTask != nil) - connection.expirePixelFormatFenceNegotiation() + for _ in 0..<100 { + guard connection.pixelFormatFenceNegotiationTask != nil else { break } + try await Task.sleep(nanoseconds: 20_000_000) + } #expect(connection.state.pixelFormat?.depth == 24) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) #expect(connection.clientToServerMessageQueue.dequeue() == nil) + #expect(connection.pixelFormatFenceNegotiationTask == nil) + #expect(connection.pendingPixelFormatTransition?.depth == 8) + + connection.completeFramebufferUpdateRequest() + + let transition = try #require(connection.clientToServerMessageQueue.dequeue()) + try await transition.message.send(connection: AuditWritingConnection()) #expect(connection.pendingPixelFormatTransition == nil) + #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } @Test @@ -157,6 +170,40 @@ struct AuditFindingsTests { #expect(connection.clientToServerMessageQueue.dequeue() != nil) } + @Test + func startsFenceCapabilityTimeoutAfterDelayedProbeSend() async throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + + #expect(connection.pixelFormatFenceCapabilityProbePayload != nil) + #expect(connection.pixelFormatFenceNegotiationTask == nil) + + try await capabilityProbe.message.send( + connection: AuditWritingConnection(delayNanoseconds: 1_100_000_000) + ) + + #expect(connection.pixelFormatFenceCapabilityProbePayload != nil) + #expect(connection.pixelFormatFenceNegotiationTask != nil) + connection.expirePixelFormatFenceNegotiation() + } + @Test func expiresUnansweredFenceCapabilityProbeWithoutStallingUpdates() async throws { let connection = VNCConnection( @@ -493,13 +540,18 @@ private final class AuditBufferConnection: NetworkConnectionReading { private final class AuditWritingConnection: NetworkConnectionWriting { var data = Data() + private let delayNanoseconds: UInt64 private let onWrite: () -> Void - init(onWrite: @escaping () -> Void = {}) { + init(delayNanoseconds: UInt64 = 0, onWrite: @escaping () -> Void = {}) { + self.delayNanoseconds = delayNanoseconds self.onWrite = onWrite } func write(data: Data) async throws { + if delayNanoseconds > 0 { + try await Task.sleep(nanoseconds: delayNanoseconds) + } onWrite() self.data.append(data) } From 0876c2affa48c1a46511ea4b173549aacfb38107 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:00:17 +0200 Subject: [PATCH 066/242] fix(macos): preserve private share registration ownership --- .../PrivateMacShareController.swift | 65 ++++++++--- .../PrivateMacShareTests.swift | 104 ++++++++++++++++++ 2 files changed, 156 insertions(+), 13 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index b8152147..f3a667b9 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -12,6 +12,49 @@ enum PrivateMacSharePermissionPolicy { } } +@MainActor +final class DesktopHostRegistrationLifecycle { + private let coordinator: DesktopHostRegistrationCoordinator + private var publishedIdentity: TailnetIdentity? + private var pendingRemovalIdentities: [TailnetIdentity] = [] + + init(registration: any DesktopHostRegistering) { + coordinator = DesktopHostRegistrationCoordinator(registration: registration) + } + + func publish(identity: TailnetIdentity, port: UInt16) async throws { + try await coordinator.register(identity: identity, port: port) + if let publishedIdentity, publishedIdentity != identity, + !pendingRemovalIdentities.contains(publishedIdentity) + { + pendingRemovalIdentities.append(publishedIdentity) + } + pendingRemovalIdentities.removeAll { $0 == identity } + publishedIdentity = identity + } + + func removePublishedIdentities() async throws { + if let publishedIdentity { + if !pendingRemovalIdentities.contains(publishedIdentity) { + pendingRemovalIdentities.append(publishedIdentity) + } + self.publishedIdentity = nil + } + + var firstError: Error? + let identities = pendingRemovalIdentities + for identity in identities { + do { + try await coordinator.unregister(identity: identity) + pendingRemovalIdentities.removeAll { $0 == identity } + } catch { + firstError = firstError ?? error + } + } + if let firstError { throw firstError } + } +} + @MainActor final class PrivateMacShareController: ObservableObject { enum RegistryPhase: Equatable { @@ -100,7 +143,7 @@ final class PrivateMacShareController: ObservableObject { private let runner: (any TailscaleCommandRunning)? private let desktopRegistration: (any DesktopHostRegistering)? - private let desktopRegistrationCoordinator: DesktopHostRegistrationCoordinator? + private let desktopRegistrationLifecycle: DesktopHostRegistrationLifecycle? private let runnerInitializationError: Error? private let defaults: UserDefaults private var capture: MacScreenCapture? @@ -118,9 +161,7 @@ final class PrivateMacShareController: ObservableObject { defaults: UserDefaults = .standard ) { self.desktopRegistration = desktopRegistration - desktopRegistrationCoordinator = desktopRegistration.map { - DesktopHostRegistrationCoordinator(registration: $0) - } + desktopRegistrationLifecycle = desktopRegistration.map(DesktopHostRegistrationLifecycle.init) self.defaults = defaults registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished let savedDisplayID = defaults.object(forKey: Self.selectedDisplayDefaultsKey) as? Int @@ -323,20 +364,18 @@ final class PrivateMacShareController: ObservableObject { self.capture = nil await capture?.stop() await registrationTask?.value - var removedRegistryEntry = true - if let desktopRegistrationCoordinator, let activeIdentity { + activeIdentity = nil + if let desktopRegistrationLifecycle { do { - try await desktopRegistrationCoordinator.unregister(identity: activeIdentity) + try await desktopRegistrationLifecycle.removePublishedIdentities() registryPhase = .notPublished } catch { - removedRegistryEntry = false registryPhase = .failed(error.localizedDescription) notice = error.localizedDescription } } else { registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished } - if removedRegistryEntry { self.activeIdentity = nil } guard isCurrent(generation) else { return } phase = .idle } @@ -457,14 +496,14 @@ final class PrivateMacShareController: ObservableObject { private func registerDesktopHost(generation: UInt64) { guard registrationTask == nil else { return } - guard let desktopRegistrationCoordinator, let identity = activeIdentity else { + guard let desktopRegistrationLifecycle, let identity = activeIdentity else { registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } registryPhase = .registering registrationTask = Task { [weak self] in do { - try await desktopRegistrationCoordinator.register(identity: identity, port: Self.port) + try await desktopRegistrationLifecycle.publish(identity: identity, port: Self.port) guard !Task.isCancelled, self?.serverGeneration == generation else { return } self?.registryPhase = .registered } catch is CancellationError { @@ -482,7 +521,7 @@ final class PrivateMacShareController: ObservableObject { registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } - guard let desktopRegistrationCoordinator, let activeIdentity else { + guard let desktopRegistrationLifecycle else { registrationTask = nil registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return @@ -490,7 +529,7 @@ final class PrivateMacShareController: ObservableObject { registrationTask = Task { [weak self] in await pendingRegistration.value do { - try await desktopRegistrationCoordinator.unregister(identity: activeIdentity) + try await desktopRegistrationLifecycle.removePublishedIdentities() self?.registryPhase = .notPublished } catch { self?.registryPhase = .failed(error.localizedDescription) diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 201cdc79..2487ace4 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -190,6 +190,53 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func failedDesktopPublicationIsNotUnregistered() async throws { + let identity = desktopIdentity(name: "failed-publish", address: "100.64.12.40") + let registration = RecordingDesktopRegistration(registerFailures: [identity.dnsName: 1]) + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + + await #expect(throws: DesktopRegistrationTestError.failed) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + try await lifecycle.removePublishedIdentities() + + #expect(await registration.events == [.register(identity.dnsName)]) + } + + @Test @MainActor + func failedDesktopRemovalSurvivesLaterIdentityChanges() async throws { + let first = desktopIdentity(name: "first-host", address: "100.64.12.41") + let second = desktopIdentity(name: "second-host", address: "100.64.12.42") + let registration = RecordingDesktopRegistration( + unregisterFailures: [first.dnsName: 2] + ) + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + + try await lifecycle.publish(identity: first, port: 5_901) + await #expect(throws: DesktopRegistrationTestError.failed) { + try await lifecycle.removePublishedIdentities() + } + + try await lifecycle.publish(identity: second, port: 5_901) + await #expect(throws: DesktopRegistrationTestError.failed) { + try await lifecycle.removePublishedIdentities() + } + try await lifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(first.dnsName), + .unregister(first.dnsName), + .register(second.dnsName), + .unregister(first.dnsName), + .unregister(second.dnsName), + .unregister(first.dnsName), + ] + ) + } + @Test @MainActor func applicationDelegateOwnsTheShareControllerUsedByTheApp() throws { let defaults = try #require( @@ -767,6 +814,17 @@ struct PrivateMacShareTests { try JSONDecoder().decode(TailscaleStatusDocument.self, from: Data(statusJSON().utf8)) } + private func desktopIdentity(name: String, address: String) -> TailnetIdentity { + TailnetIdentity( + tailnetName: "example.com", + loginName: "operator@example.com", + dnsName: "\(name).example.ts.net", + hostName: name, + ipv4Address: address, + userID: 42 + ) + } + private func statusJSON() -> String { """ { @@ -947,6 +1005,52 @@ private actor SuspendedDesktopRegistration: DesktopHostRegistering { } } +private enum DesktopRegistrationTestError: Error { + case failed +} + +private actor RecordingDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register(String) + case unregister(String) + } + + private var registerFailures: [String: Int] + private var unregisterFailures: [String: Int] + private(set) var events: [Event] = [] + + init( + registerFailures: [String: Int] = [:], + unregisterFailures: [String: Int] = [:] + ) { + self.registerFailures = registerFailures + self.unregisterFailures = unregisterFailures + } + + func register(identity: TailnetIdentity, port: UInt16) async throws { + events.append(.register(identity.dnsName)) + if consumeFailure(for: identity.dnsName, from: ®isterFailures) { + throw DesktopRegistrationTestError.failed + } + } + + func unregister(identity: TailnetIdentity) async throws { + events.append(.unregister(identity.dnsName)) + if consumeFailure(for: identity.dnsName, from: &unregisterFailures) { + throw DesktopRegistrationTestError.failed + } + } + + private func consumeFailure( + for identity: String, + from failures: inout [String: Int] + ) -> Bool { + guard let remaining = failures[identity], remaining > 0 else { return false } + failures[identity] = remaining - 1 + return true + } +} + private final class RemoteInputRecorder: RemoteInputForwarding, @unchecked Sendable { private let lock = NSLock() private var releases = 0 From 1054a76f23c988b826d7ff4ec3efeb961cfc9d0b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:01:10 +0200 Subject: [PATCH 067/242] fix(actions): acknowledge runner input delivery --- src/github-actions-runtime.ts | 71 +++++++++++++- src/worker/session-control-do.ts | 5 +- src/worker/terminal-hub.ts | 85 ++++++++++++++++ tests/github-actions-runtime.test.ts | 70 ++++++++++++- tests/terminal-hub.test.ts | 141 +++++++++++++++++++++++++++ 5 files changed, 366 insertions(+), 6 deletions(-) diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 46341ee3..1572c7f3 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -16,6 +16,11 @@ export type GitHubActionsRelaySocket = { close(code?: number, reason?: string): void; }; +export type GitHubActionsRelayInputAcknowledgement = { + accepted: boolean; + error?: string; +}; + export const githubActionsCapabilities = { terminal: true, takeover: true, @@ -42,6 +47,8 @@ const terminalWorkStates = new Set([ ]); const webSocketOpen = 1; +const relayInputAcknowledgementType = "github_actions_input_ack"; +const relayInputRejectedError = "GitHub Actions runner did not accept terminal input"; export function githubActionsRuntimeLabel(runtime: unknown): string { return runtime === githubActionsRuntime ? "GitHub Actions" : ""; @@ -118,12 +125,72 @@ export function forwardGitHubActionsRelayMessage( let forwarded = 0; for (const socket of targets) { if (socket.readyState !== webSocketOpen) continue; - socket.send(message); - forwarded += 1; + try { + socket.send(message); + forwarded += 1; + } catch { + // The caller uses the forwarded count to reject undelivered viewer input. + } } return forwarded; } +export function relayGitHubActionsWebSocketMessage( + sender: GitHubActionsRelayRole, + senderSocket: GitHubActionsRelaySocket, + message: string | ArrayBuffer, + runners: readonly GitHubActionsRelaySocket[], + viewers: readonly GitHubActionsRelaySocket[], +): number { + const viewerInput = sender === "viewer" && !isGitHubActionsViewerControlMessage(message); + const forwarded = forwardGitHubActionsRelayMessage(sender, message, runners, viewers); + if (viewerInput) { + sendGitHubActionsRelayInputAcknowledgement(senderSocket, forwarded === 1); + } + return forwarded; +} + +export function sendGitHubActionsRelayInputAcknowledgement( + viewer: GitHubActionsRelaySocket, + accepted: boolean, +): boolean { + if (viewer.readyState !== webSocketOpen) return false; + try { + viewer.send( + JSON.stringify({ + type: relayInputAcknowledgementType, + accepted, + ...(accepted ? {} : { error: relayInputRejectedError }), + }), + ); + return true; + } catch { + return false; + } +} + +export function parseGitHubActionsRelayInputAcknowledgement( + message: string | ArrayBuffer, +): GitHubActionsRelayInputAcknowledgement | null { + if (typeof message !== "string") return null; + try { + const parsed = JSON.parse(message) as Record; + if (parsed.type !== relayInputAcknowledgementType || typeof parsed.accepted !== "boolean") { + return null; + } + if (parsed.accepted) return { accepted: true }; + return { + accepted: false, + error: + typeof parsed.error === "string" && parsed.error.trim() + ? parsed.error.trim() + : relayInputRejectedError, + }; + } catch { + return null; + } +} + export function isGitHubActionsViewerControlMessage(message: string | ArrayBuffer): boolean { if (typeof message !== "string") return false; try { diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index 1c951f36..8b9f54ae 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -7,9 +7,9 @@ import { } from "../credential-policy-fence.ts"; import type { FleetSandboxPolicySummary } from "../fleet-state.ts"; import { - forwardGitHubActionsRelayMessage, githubActionsRelayRole, notifyGitHubActionsViewers, + relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, } from "../github-actions-runtime.ts"; import type { RuntimeEnv } from "./env.ts"; @@ -200,8 +200,9 @@ export class SessionControlDO extends DurableObject { socket.close(1008, "unknown relay peer"); return; } - forwardGitHubActionsRelayMessage( + relayGitHubActionsWebSocketMessage( role, + socket, message, this.ctx.getWebSockets("github-actions-runner"), this.ctx.getWebSockets("github-actions-viewer"), diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 6d83ce3d..762c2986 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -13,6 +13,11 @@ import { normalizeWebSocketMessageData, sendOutputAcknowledgement, } from "@openclaw/libterminal/worker"; +import { + githubActionsRuntime, + parseGitHubActionsRelayInputAcknowledgement, + type GitHubActionsRelayInputAcknowledgement, +} from "../github-actions-runtime.ts"; import { redactedAdapterMessage } from "../runtime-adapter.ts"; import { badRequest, unauthorized } from "./http.ts"; import type { User } from "./models.ts"; @@ -20,6 +25,12 @@ import type { InteractiveSession } from "./session-model.ts"; const encoder = new TextEncoder(); const terminalFrameLimits = { maxFrameBytes: 16 * 1024 * 1024 }; +const terminalInputAcknowledgementTimeoutMs = 5_000; + +type PendingTerminalInputAcknowledgement = { + resolve(result: GitHubActionsRelayInputAcknowledgement): void; + timeout: ReturnType; +}; export type TerminalUpstream = { socket: WebSocket; @@ -37,6 +48,8 @@ export type TerminalHubSubscription = { viewCheck: ReturnType | null; cols: number; rows: number; + inputAcknowledgements: boolean; + pendingInputAcknowledgement: PendingTerminalInputAcknowledgement | null; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; }; @@ -237,14 +250,27 @@ export class TerminalHub { }); return; } + const acknowledgement = subscription.inputAcknowledgements + ? beginTerminalInputAcknowledgement(subscription) + : null; try { subscription.upstream.send(input); } catch { + cancelTerminalInputAcknowledgement(subscription); sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { error: "terminal upstream send failed", }); return; } + if (acknowledgement) { + const result = await acknowledgement; + if (!result.accepted) { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: result.error ?? "terminal input was not accepted", + }); + return; + } + } } sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { type: "input-accepted", @@ -421,6 +447,8 @@ export class TerminalHub { viewCheck, cols, rows, + inputAcknowledgements: session.runtime === githubActionsRuntime, + pendingInputAcknowledgement: null, outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; @@ -464,6 +492,11 @@ export class TerminalHub { const data = await normalizeWebSocketMessageData(raw); if (client.readyState !== WebSocket.OPEN || !viewGranted) return; if (typeof data === "string") { + const inputAcknowledgement = parseGitHubActionsRelayInputAcknowledgement(data); + if (inputAcknowledgement) { + completeTerminalInputAcknowledgement(activeSubscription, inputAcknowledgement); + return; + } const parsed = parseTerminalControlMessage(data); if (parsed) { sendTerminalJson(client, TerminalMessageType.Event, id, parsed); @@ -488,6 +521,10 @@ export class TerminalHub { }); }); upstream.addEventListener("close", (event) => { + completeTerminalInputAcknowledgement(activeSubscription, { + accepted: false, + error: "terminal upstream closed before accepting input", + }); const closeReason = consumeCloseReason(); const safeUpstreamReason = event.reason ? redactedAdapterMessage( @@ -512,6 +549,10 @@ export class TerminalHub { } }); upstream.addEventListener("error", () => { + completeTerminalInputAcknowledgement(activeSubscription, { + accepted: false, + error: "terminal upstream failed before accepting input", + }); const closeReason = closingReason; if (subscriptions.delete(id)) this.dependencies.releaseInputState(id); if (viewCheck !== null) clearInterval(viewCheck); @@ -540,6 +581,50 @@ export class TerminalHub { } } +function beginTerminalInputAcknowledgement( + subscription: TerminalHubSubscription, +): Promise { + cancelTerminalInputAcknowledgement(subscription); + return new Promise((resolve) => { + const pending: PendingTerminalInputAcknowledgement = { + resolve, + timeout: setTimeout(() => { + if ( + !completeTerminalInputAcknowledgement(subscription, { + accepted: false, + error: "terminal input delivery was not acknowledged", + }) + ) { + return; + } + if (subscription.upstream.readyState === WebSocket.OPEN) { + subscription.upstream.close(1011, "input acknowledgement timed out"); + } + }, terminalInputAcknowledgementTimeoutMs), + }; + subscription.pendingInputAcknowledgement = pending; + }); +} + +function completeTerminalInputAcknowledgement( + subscription: TerminalHubSubscription, + result: GitHubActionsRelayInputAcknowledgement, +): boolean { + const pending = subscription.pendingInputAcknowledgement; + if (!pending) return false; + subscription.pendingInputAcknowledgement = null; + clearTimeout(pending.timeout); + pending.resolve(result); + return true; +} + +function cancelTerminalInputAcknowledgement(subscription: TerminalHubSubscription): void { + const pending = subscription.pendingInputAcknowledgement; + if (!pending) return; + subscription.pendingInputAcknowledgement = null; + clearTimeout(pending.timeout); +} + function updateTerminalInputCapability( socket: WebSocket, subscription: TerminalHubSubscription, diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index fce0dc5b..3fd74032 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -10,8 +10,11 @@ import { isGitHubActionsViewerControlMessage, isTerminalGitHubActionsWorkState, notifyGitHubActionsViewers, + parseGitHubActionsRelayInputAcknowledgement, parseGitHubActionsWorkState, + relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, + sendGitHubActionsRelayInputAcknowledgement, type GitHubActionsRelaySocket, } from "../src/github-actions-runtime.ts"; @@ -109,17 +112,80 @@ test("relay sends viewer input to the first open runner", () => { assert.deepEqual(laterRunner.sent, []); }); +test("relay reports runner delivery failures instead of claiming forwarded input", () => { + const runner = relaySocket(); + runner.send = () => { + throw new Error("runner disconnected"); + }; + + assert.equal(forwardGitHubActionsRelayMessage("viewer", "input", [runner], []), 0); +}); + +test("relay input acknowledgements distinguish accepted and rejected delivery", () => { + const viewer = relaySocket(); + + assert.equal(sendGitHubActionsRelayInputAcknowledgement(viewer, true), true); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + accepted: true, + }); + + assert.equal(sendGitHubActionsRelayInputAcknowledgement(viewer, false), true); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[1]!), { + accepted: false, + error: "GitHub Actions runner did not accept terminal input", + }); + assert.equal(parseGitHubActionsRelayInputAcknowledgement('{"type":"runner_waiting"}'), null); + assert.equal(sendGitHubActionsRelayInputAcknowledgement(relaySocket(3), false), false); +}); + +test("viewer relay acknowledges only input delivered to an open runner", () => { + const runner = relaySocket(); + const viewer = relaySocket(); + + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, "input", [runner], []), 1); + assert.deepEqual(runner.sent, ["input"]); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + accepted: true, + }); + + const waitingViewer = relaySocket(); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", waitingViewer, "input", [], []), 0); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(waitingViewer.sent[0]!), { + accepted: false, + error: "GitHub Actions runner did not accept terminal input", + }); + + const failedRunner = relaySocket(); + failedRunner.send = () => { + throw new Error("runner disconnected"); + }; + const failedViewer = relaySocket(); + assert.equal( + relayGitHubActionsWebSocketMessage("viewer", failedViewer, "input", [failedRunner], []), + 0, + ); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(failedViewer.sent[0]!), { + accepted: false, + error: "GitHub Actions runner did not accept terminal input", + }); +}); + test("relay consumes viewer resize controls without corrupting raw runner input", () => { const runner = relaySocket(); + const viewer = relaySocket(); const resize = JSON.stringify({ type: "resize", cols: 120, rows: 40 }); const typedJson = new TextEncoder().encode(resize).buffer; assert.equal(isGitHubActionsViewerControlMessage(resize), true); assert.equal(isGitHubActionsViewerControlMessage(typedJson), false); - assert.equal(forwardGitHubActionsRelayMessage("viewer", resize, [runner], []), 0); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, resize, [runner], []), 0); assert.deepEqual(runner.sent, []); - assert.equal(forwardGitHubActionsRelayMessage("viewer", typedJson, [runner], []), 1); + assert.deepEqual(viewer.sent, []); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, typedJson, [runner], []), 1); assert.deepEqual(runner.sent, [typedJson]); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + accepted: true, + }); }); test("relay tags and runner lifecycle notifications stay explicit", () => { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 5587c68a..d9661904 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -10,6 +10,7 @@ import { encodeSubscribePayload, encodeTerminalFrame, } from "@openclaw/libterminal/protocol"; +import { sendGitHubActionsRelayInputAcknowledgement } from "../src/github-actions-runtime.ts"; import type { User } from "../src/worker/models.ts"; import { containerCapabilities, interactiveSession } from "../src/worker/session-model.ts"; import { TerminalHub, type TerminalHubDependencies } from "../src/worker/terminal-hub.ts"; @@ -84,6 +85,16 @@ const session = interactiveSession( }), ); +const githubActionsSession = interactiveSession( + sessionRow({ + adapter: null, + adapter_workspace_id: null, + capabilities_json: JSON.stringify(containerCapabilities), + runtime: "github_actions", + status: "ready", + }), +); + function dependencies( client: WebSocket, server: WebSocket, @@ -471,6 +482,136 @@ test("terminal hub never acknowledges input after its upstream closes", async () server.emit("close"); }); +test("GitHub Actions input waits for relay delivery before acknowledgement", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("steer\r"), + }), + }); + await flushQueues(); + + assert.equal(new TextDecoder().decode(upstream.sent.at(-1) as Uint8Array), "steer\r"); + assert.equal( + server.sent.some( + (payload) => + frame(payload).type === TerminalMessageType.Event && + (decodeJsonPayload(frame(payload).payload) as { type?: string }).type === "input-accepted", + ), + false, + ); + + sendGitHubActionsRelayInputAcknowledgement( + { + readyState: WebSocket.OPEN, + send(message) { + upstream.emit("message", { data: message }); + }, + close() {}, + }, + true, + ); + await flushQueues(); + await flushQueues(); + + const accepted = frame(server.sent.at(-1)!); + assert.equal(accepted.type, TerminalMessageType.Event); + assert.deepEqual(decodeJsonPayload(accepted.payload), { type: "input-accepted" }); + server.emit("close"); +}); + +test("GitHub Actions relay rejection becomes a terminal input error", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("dropped"), + }), + }); + await flushQueues(); + + sendGitHubActionsRelayInputAcknowledgement( + { + readyState: WebSocket.OPEN, + send(message) { + upstream.emit("message", { data: message }); + }, + close() {}, + }, + false, + ); + await flushQueues(); + await flushQueues(); + + const messages = server.sent.map((payload) => frame(payload)); + assert.equal( + messages.some( + (message) => + message.type === TerminalMessageType.Event && + (decodeJsonPayload(message.payload) as { type?: string }).type === "input-accepted", + ), + false, + ); + const rejected = messages.at(-1)!; + assert.equal(rejected.type, TerminalMessageType.Error); + assert.deepEqual(decodeJsonPayload(rejected.payload), { + error: "GitHub Actions runner did not accept terminal input", + }); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From 9232a35ed05340cd1b36c2eb5fc181926bbada7d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:03:06 +0200 Subject: [PATCH 068/242] fix(terminal): queue input acknowledgements --- src/worker/terminal-hub.ts | 116 ++++++++++++-------- tests/terminal-hub.test.ts | 212 +++++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+), 43 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 762c2986..f3c14562 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -28,6 +28,7 @@ const terminalFrameLimits = { maxFrameBytes: 16 * 1024 * 1024 }; const terminalInputAcknowledgementTimeoutMs = 5_000; type PendingTerminalInputAcknowledgement = { + promise: Promise; resolve(result: GitHubActionsRelayInputAcknowledgement): void; timeout: ReturnType; }; @@ -49,7 +50,7 @@ export type TerminalHubSubscription = { cols: number; rows: number; inputAcknowledgements: boolean; - pendingInputAcknowledgement: PendingTerminalInputAcknowledgement | null; + pendingInputAcknowledgements: PendingTerminalInputAcknowledgement[]; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; }; @@ -239,6 +240,7 @@ export class TerminalHub { return; } const inputs = await this.dependencies.inputPayloads(subscription, user, frame.payload); + const acknowledgements: PendingTerminalInputAcknowledgement[] = []; for (const [index, input] of inputs.entries()) { if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); if ( @@ -253,25 +255,36 @@ export class TerminalHub { const acknowledgement = subscription.inputAcknowledgements ? beginTerminalInputAcknowledgement(subscription) : null; + if (acknowledgement) acknowledgements.push(acknowledgement); try { subscription.upstream.send(input); } catch { - cancelTerminalInputAcknowledgement(subscription); - sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: "terminal upstream send failed", - }); - return; - } - if (acknowledgement) { - const result = await acknowledgement; - if (!result.accepted) { + if (acknowledgement) { + completeTerminalInputAcknowledgement(subscription, acknowledgement, { + accepted: false, + error: "terminal upstream send failed", + }); + break; + } else { sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: result.error ?? "terminal input was not accepted", + error: "terminal upstream send failed", }); return; } } } + if (acknowledgements.length > 0) { + const results = await Promise.all( + acknowledgements.map((acknowledgement) => acknowledgement.promise), + ); + const rejection = results.find((result) => !result.accepted); + if (rejection) { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: rejection.error ?? "terminal input was not accepted", + }); + return; + } + } sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { type: "input-accepted", }); @@ -448,7 +461,7 @@ export class TerminalHub { cols, rows, inputAcknowledgements: session.runtime === githubActionsRuntime, - pendingInputAcknowledgement: null, + pendingInputAcknowledgements: [], outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; @@ -494,7 +507,7 @@ export class TerminalHub { if (typeof data === "string") { const inputAcknowledgement = parseGitHubActionsRelayInputAcknowledgement(data); if (inputAcknowledgement) { - completeTerminalInputAcknowledgement(activeSubscription, inputAcknowledgement); + completeNextTerminalInputAcknowledgement(activeSubscription, inputAcknowledgement); return; } const parsed = parseTerminalControlMessage(data); @@ -521,7 +534,7 @@ export class TerminalHub { }); }); upstream.addEventListener("close", (event) => { - completeTerminalInputAcknowledgement(activeSubscription, { + completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, error: "terminal upstream closed before accepting input", }); @@ -549,7 +562,7 @@ export class TerminalHub { } }); upstream.addEventListener("error", () => { - completeTerminalInputAcknowledgement(activeSubscription, { + completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, error: "terminal upstream failed before accepting input", }); @@ -583,46 +596,63 @@ export class TerminalHub { function beginTerminalInputAcknowledgement( subscription: TerminalHubSubscription, -): Promise { - cancelTerminalInputAcknowledgement(subscription); - return new Promise((resolve) => { - const pending: PendingTerminalInputAcknowledgement = { - resolve, - timeout: setTimeout(() => { - if ( - !completeTerminalInputAcknowledgement(subscription, { - accepted: false, - error: "terminal input delivery was not acknowledged", - }) - ) { - return; - } - if (subscription.upstream.readyState === WebSocket.OPEN) { - subscription.upstream.close(1011, "input acknowledgement timed out"); - } - }, terminalInputAcknowledgementTimeoutMs), - }; - subscription.pendingInputAcknowledgement = pending; +): PendingTerminalInputAcknowledgement { + let resolve!: (result: GitHubActionsRelayInputAcknowledgement) => void; + const promise = new Promise((complete) => { + resolve = complete; }); + const pending: PendingTerminalInputAcknowledgement = { + promise, + resolve, + timeout: setTimeout(() => { + if ( + completeAllTerminalInputAcknowledgements(subscription, { + accepted: false, + error: "terminal input delivery was not acknowledged", + }) === 0 + ) { + return; + } + if (subscription.upstream.readyState === WebSocket.OPEN) { + subscription.upstream.close(1011, "input acknowledgement timed out"); + } + }, terminalInputAcknowledgementTimeoutMs), + }; + subscription.pendingInputAcknowledgements.push(pending); + return pending; } function completeTerminalInputAcknowledgement( subscription: TerminalHubSubscription, + pending: PendingTerminalInputAcknowledgement, result: GitHubActionsRelayInputAcknowledgement, ): boolean { - const pending = subscription.pendingInputAcknowledgement; - if (!pending) return false; - subscription.pendingInputAcknowledgement = null; + const index = subscription.pendingInputAcknowledgements.indexOf(pending); + if (index < 0) return false; + subscription.pendingInputAcknowledgements.splice(index, 1); clearTimeout(pending.timeout); pending.resolve(result); return true; } -function cancelTerminalInputAcknowledgement(subscription: TerminalHubSubscription): void { - const pending = subscription.pendingInputAcknowledgement; - if (!pending) return; - subscription.pendingInputAcknowledgement = null; - clearTimeout(pending.timeout); +function completeNextTerminalInputAcknowledgement( + subscription: TerminalHubSubscription, + result: GitHubActionsRelayInputAcknowledgement, +): boolean { + const pending = subscription.pendingInputAcknowledgements[0]; + return pending ? completeTerminalInputAcknowledgement(subscription, pending, result) : false; +} + +function completeAllTerminalInputAcknowledgements( + subscription: TerminalHubSubscription, + result: GitHubActionsRelayInputAcknowledgement, +): number { + const pending = subscription.pendingInputAcknowledgements.splice(0); + for (const acknowledgement of pending) { + clearTimeout(acknowledgement.timeout); + acknowledgement.resolve(result); + } + return pending.length; } function updateTerminalInputCapability( diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index d9661904..79687a11 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -66,6 +66,10 @@ async function flushQueues(): Promise { await new Promise((resolve) => setImmediate(resolve)); } +async function waitForInputPayloads(): Promise { + await new Promise((resolve) => setTimeout(resolve, 100)); +} + const user: User = { subject: "github:42", login: "operator", @@ -612,6 +616,214 @@ test("GitHub Actions relay rejection becomes a terminal input error", async () = server.emit("close"); }); +test("GitHub Actions input acknowledgements resolve overlapping payloads in FIFO order", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async inputPayloads() { + return [new TextEncoder().encode("first"), new TextEncoder().encode("second")]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("input"), + }), + }); + await waitForInputPayloads(); + + assert.deepEqual( + upstream.sent.map((payload) => new TextDecoder().decode(payload as Uint8Array)), + ["first", "second"], + ); + sendGitHubActionsRelayInputAcknowledgement( + { + readyState: WebSocket.OPEN, + send(message) { + upstream.emit("message", { data: message }); + }, + close() {}, + }, + true, + ); + await flushQueues(); + assert.notDeepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "input-accepted", + }); + + sendGitHubActionsRelayInputAcknowledgement( + { + readyState: WebSocket.OPEN, + send(message) { + upstream.emit("message", { data: message }); + }, + close() {}, + }, + true, + ); + await flushQueues(); + await flushQueues(); + + assert.deepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "input-accepted", + }); + server.emit("close"); +}); + +test("GitHub Actions send failure removes only its own acknowledgement waiter", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const send = upstream.send.bind(upstream); + let sendCount = 0; + upstream.send = (payload) => { + sendCount += 1; + if (sendCount === 2) throw new Error("runner disconnected"); + send(payload); + }; + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async inputPayloads() { + return [new TextEncoder().encode("delivered"), new TextEncoder().encode("failed")]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("input"), + }), + }); + await waitForInputPayloads(); + assert.deepEqual( + upstream.sent.map((payload) => new TextDecoder().decode(payload as Uint8Array)), + ["delivered"], + ); + + sendGitHubActionsRelayInputAcknowledgement( + { + readyState: WebSocket.OPEN, + send(message) { + upstream.emit("message", { data: message }); + }, + close() {}, + }, + true, + ); + await flushQueues(); + await flushQueues(); + + const rejected = frame(server.sent.at(-1)!); + assert.equal(rejected.type, TerminalMessageType.Error); + assert.deepEqual(decodeJsonPayload(rejected.payload), { + error: "terminal upstream send failed", + }); + server.emit("close"); +}); + +test("GitHub Actions close rejects every pending input acknowledgement", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async inputPayloads() { + return [new TextEncoder().encode("first"), new TextEncoder().encode("second")]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("input"), + }), + }); + await waitForInputPayloads(); + upstream.emit("close", { code: 1011, reason: "runner disconnected" }); + await flushQueues(); + await flushQueues(); + + const messages = server.sent.map((payload) => frame(payload)); + assert.equal( + messages.some( + (message) => + message.type === TerminalMessageType.Event && + (decodeJsonPayload(message.payload) as { type?: string }).type === "input-accepted", + ), + false, + ); + assert.equal( + messages.some( + (message) => + message.type === TerminalMessageType.Error && + (decodeJsonPayload(message.payload) as { error?: string }).error === + "terminal upstream closed before accepting input", + ), + true, + ); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From f7a82cd3a5728ff6eec7498845ce04a7b6e6c7ee Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:13:51 +0200 Subject: [PATCH 069/242] fix(vnc): bound pixel format transition waits --- .../SDK/Connection/VNCConnection+API.swift | 53 ++++++++++++- .../SDK/Connection/VNCConnection.swift | 1 + .../RoyalVNCKit/SDK/Error/ProtocolError.swift | 3 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 76 +++++++++++++++++++ 4 files changed, 130 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 498c13a4..b0ced27b 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -9,6 +9,7 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { let pixelFormatMessage: VNCProtocol.SetPixelFormat let willSend: () -> Void let didSend: () -> Void + let didSendFence: () -> Void var messageType: UInt8 { fenceMessage?.messageType ?? pixelFormatMessage.messageType } var data: Data { @@ -22,6 +23,8 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { try await connection.write(data: data) if fenceMessage == nil { didSend() + } else { + didSendFence() } } } @@ -190,14 +193,56 @@ extension VNCConnection { pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: transition.pixelFormat), willSend: { [weak self] in self?.beginPixelFormatTransition(transition.pixelFormat) + }, + didSend: { [weak self] in + self?.completePixelFormatTransition() + }, + didSendFence: { [weak self] in + guard let payload = transition.fencePayload else { return } + self?.schedulePixelFormatTransitionDeadline(payload: payload) } - ) { [weak self] in - self?.completePixelFormatTransition() - } + ) enqueueClientToServerMessage(message) } + private func schedulePixelFormatTransitionDeadline(payload: Data) { + framebufferRequestLock.lock() + defer { framebufferRequestLock.unlock() } + guard isPixelFormatTransitionInFlight, + pixelFormatTransitionFencePayload == payload else { + return + } + + cancelPixelFormatTransitionDeadlineLocked() + pixelFormatTransitionDeadlineTask = Task { [weak self] in + do { + try await Task.sleep(nanoseconds: 5_000_000_000) + } catch { + return + } + self?.expirePixelFormatTransitionDeadline(payload: payload) + } + } + + private func cancelPixelFormatTransitionDeadlineLocked() { + pixelFormatTransitionDeadlineTask?.cancel() + pixelFormatTransitionDeadlineTask = nil + } + + func expirePixelFormatTransitionDeadline(payload: Data) { + framebufferRequestLock.lock() + guard isPixelFormatTransitionInFlight, + pixelFormatTransitionFencePayload == payload else { + framebufferRequestLock.unlock() + return + } + pixelFormatTransitionDeadlineTask = nil + framebufferRequestLock.unlock() + + handleBreakingError(VNCError.protocol(.pixelFormatTransitionTimedOut)) + } + func probePixelFormatFenceSupport() { framebufferRequestLock.lock() cancelPixelFormatFenceNegotiationTimeoutLocked() @@ -315,6 +360,7 @@ extension VNCConnection { framebufferRequestLock.unlock() throw VNCError.protocol(.invalidData) } + cancelPixelFormatTransitionDeadlineLocked() pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] framebufferRequestLock.unlock() @@ -595,6 +641,7 @@ extension VNCConnection { pixelFormatTransitionInFlight = nil pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] + cancelPixelFormatTransitionDeadlineLocked() pixelFormatFenceCapabilityProbePayload = nil cancelPixelFormatFenceNegotiationTimeoutLocked() framebufferRequestLock.unlock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index f98b901c..d81024ba 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -116,6 +116,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pixelFormatTransitionFenceSequence: UInt64 = 0 var pixelFormatTransitionFencePayload: Data? var pixelFormatTransitionRequiredFenceFlags: VNCProtocol.FenceFlags = [] + var pixelFormatTransitionDeadlineTask: Task? var pixelFormatFenceCapabilityProbePayload: Data? var pixelFormatFenceNegotiationTask: Task? private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Error/ProtocolError.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Error/ProtocolError.swift index 1448c415..8004e272 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Error/ProtocolError.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Error/ProtocolError.swift @@ -16,6 +16,7 @@ public extension VNCError { case framebufferUpdateReceivedWithoutFramebuffer case framebufferFailedToCreateIOSurface case setColourMapEntriesReceivedWithoutFramebuffer + case pixelFormatTransitionTimedOut case frameDecode(encodingType: VNCEncodingType, underlyingError: Error?) case zlibDecompress(underlyingError: Error?) case zrleInvalidSubencoding(subencoding: UInt8) @@ -50,6 +51,8 @@ public extension VNCError { return "Failed to create IOSurface for Framebuffer." case .setColourMapEntriesReceivedWithoutFramebuffer: return "A Set Colour Map Entries request has been retrieved but no Framebuffer has been created yet." + case .pixelFormatTransitionTimedOut: + return "The server did not acknowledge the pixel format transition." case .frameDecode(let encodingType, let underlyingError): return VNCError.combinedErrorDescription("An error occurred while decoding a Framebuffer Update Message (Encoding Type: \(encodingType)).", underlyingError: underlyingError) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index b91d83cb..9fa7d446 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -310,6 +310,7 @@ struct AuditFindingsTests { } try await queued.message.send(connection: writer) + #expect(connection.pixelFormatTransitionDeadlineTask != nil) #expect(writer.data.count == 37) #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) #expect(writer.data[4..<8] == Data([0x80, 0, 0, 5])) @@ -328,6 +329,45 @@ struct AuditFindingsTests { #expect(connection.state.pixelFormat?.depth == 8) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) + #expect(connection.connectionState.status == .connected) + } + + @Test + func disconnectsWhenPixelFormatTransitionFenceIsMissing() async throws { + let connection = try await makeFenceCapableConnection() + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + try await queued.message.send(connection: AuditWritingConnection()) + let payload = try #require(connection.pixelFormatTransitionFencePayload) + + #expect(connection.pixelFormatTransitionDeadlineTask != nil) + #expect(connection.isPixelFormatTransitionInFlight) + + connection.expirePixelFormatTransitionDeadline(payload: payload) + + #expect(connection.connectionState.status == .disconnected) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) + #expect(!connection.isPixelFormatTransitionInFlight) + } + + @Test + func cancellingFramebufferSchedulingInvalidatesPixelFormatTransitionDeadline() async throws { + let connection = try await makeFenceCapableConnection() + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + try await queued.message.send(connection: AuditWritingConnection()) + let payload = try #require(connection.pixelFormatTransitionFencePayload) + + #expect(connection.pixelFormatTransitionDeadlineTask != nil) + + connection.cancelFramebufferUpdateScheduling() + connection.expirePixelFormatTransitionDeadline(payload: payload) + + #expect(connection.pixelFormatTransitionDeadlineTask == nil) + #expect(connection.connectionState.status == .connected) } @Test @@ -374,6 +414,7 @@ struct AuditFindingsTests { #expect(connection.state.pixelFormat?.depth == 8) } try await transition.message.send(connection: transitionWriter) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) #expect(transitionWriter.data.count == 20) #expect( transitionWriter.data[0] @@ -514,6 +555,41 @@ struct AuditFindingsTests { frameEncodings: [.raw] ) } + + private func makeFenceCapableConnection() async throws -> VNCConnection { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + connection.framebufferUpdateRequestOutstanding = true + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityWriter = AuditWritingConnection() + try await capabilityProbe.message.send(connection: capabilityWriter) + let capabilityLength = Int(capabilityWriter.data[8]) + let capabilityPayload = Data(capabilityWriter.data[9..<(9 + capabilityLength)]) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: capabilityPayload + ) + ) + return connection + } } private final class AuditBufferConnection: NetworkConnectionReading { From 307304cd8e75756aa9f41d88ecf73cece5e1c99d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:16:07 +0200 Subject: [PATCH 070/242] fix(actions): correlate runner input acknowledgements --- src/github-actions-runner.ts | 27 ++++ src/github-actions-runtime.ts | 204 +++++++++++++++++++++++---- src/worker/session-control-do.ts | 2 +- src/worker/terminal-hub.ts | 72 ++++++---- tests/github-actions-runner.test.ts | 67 +++++++++ tests/github-actions-runtime.test.ts | 122 +++++++++------- tests/terminal-hub.test.ts | 140 ++++++++++-------- 7 files changed, 465 insertions(+), 169 deletions(-) create mode 100644 src/github-actions-runner.ts create mode 100644 tests/github-actions-runner.test.ts diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts new file mode 100644 index 00000000..95c5717d --- /dev/null +++ b/src/github-actions-runner.ts @@ -0,0 +1,27 @@ +import { + parseGitHubActionsRelayInput, + sendGitHubActionsRelayInputAcknowledgement, + type GitHubActionsRelaySocket, +} from "./github-actions-runtime.ts"; + +export async function acceptGitHubActionsRunnerInput( + socket: GitHubActionsRelaySocket, + message: string | ArrayBuffer, + writeToPty: (payload: ArrayBuffer) => void | Promise, +): Promise { + const input = parseGitHubActionsRelayInput(message); + if (!input) return false; + try { + await writeToPty(input.payload); + sendGitHubActionsRelayInputAcknowledgement(socket, { + inputId: input.inputId, + accepted: true, + }); + } catch { + sendGitHubActionsRelayInputAcknowledgement(socket, { + inputId: input.inputId, + accepted: false, + }); + } + return true; +} diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 1572c7f3..26f6d8d0 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -19,6 +19,12 @@ export type GitHubActionsRelaySocket = { export type GitHubActionsRelayInputAcknowledgement = { accepted: boolean; error?: string; + inputId: string; +}; + +export type GitHubActionsRelayInput = { + inputId: string; + payload: ArrayBuffer; }; export const githubActionsCapabilities = { @@ -47,8 +53,27 @@ const terminalWorkStates = new Set([ ]); const webSocketOpen = 1; -const relayInputAcknowledgementType = "github_actions_input_ack"; const relayInputRejectedError = "GitHub Actions runner did not accept terminal input"; +const relayFrameMagic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); +const relayFrameHeaderBytes = relayFrameMagic.byteLength + 2; +const relayInputFrameType = 1; +const relayInputAcknowledgementFrameType = 2; +const relayEventFrameType = 3; +const relayInputIdMaximumBytes = 80; +const relayInputIdPattern = /^[A-Za-z0-9_-]+$/; +const relayEventCodes = { + runner_connected: 1, + runner_disconnected: 2, + runner_waiting: 3, +} as const; +const relayEvents = new Map( + Object.entries(relayEventCodes).map(([event, code]) => [ + code, + event as keyof typeof relayEventCodes, + ]), +); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); export function githubActionsRuntimeLabel(runtime: unknown): string { return runtime === githubActionsRuntime ? "GitHub Actions" : ""; @@ -117,7 +142,6 @@ export function forwardGitHubActionsRelayMessage( runners: readonly GitHubActionsRelaySocket[], viewers: readonly GitHubActionsRelaySocket[], ): number { - if (sender === "viewer" && isGitHubActionsViewerControlMessage(message)) return 0; const targets = sender === "runner" ? viewers @@ -142,27 +166,30 @@ export function relayGitHubActionsWebSocketMessage( runners: readonly GitHubActionsRelaySocket[], viewers: readonly GitHubActionsRelaySocket[], ): number { - const viewerInput = sender === "viewer" && !isGitHubActionsViewerControlMessage(message); - const forwarded = forwardGitHubActionsRelayMessage(sender, message, runners, viewers); - if (viewerInput) { - sendGitHubActionsRelayInputAcknowledgement(senderSocket, forwarded === 1); + if (sender === "viewer") { + if (isGitHubActionsViewerControlMessage(message)) return 0; + const input = parseGitHubActionsRelayInput(message); + if (!input) return 0; + const forwarded = forwardGitHubActionsRelayMessage(sender, message, runners, viewers); + if (forwarded !== 1) { + sendGitHubActionsRelayInputAcknowledgement(senderSocket, { + inputId: input.inputId, + accepted: false, + }); + } + return forwarded; } - return forwarded; + + return forwardGitHubActionsRelayMessage(sender, message, runners, viewers); } export function sendGitHubActionsRelayInputAcknowledgement( viewer: GitHubActionsRelaySocket, - accepted: boolean, + acknowledgement: GitHubActionsRelayInputAcknowledgement, ): boolean { if (viewer.readyState !== webSocketOpen) return false; try { - viewer.send( - JSON.stringify({ - type: relayInputAcknowledgementType, - accepted, - ...(accepted ? {} : { error: relayInputRejectedError }), - }), - ); + viewer.send(encodeGitHubActionsRelayInputAcknowledgement(acknowledgement)); return true; } catch { return false; @@ -172,23 +199,73 @@ export function sendGitHubActionsRelayInputAcknowledgement( export function parseGitHubActionsRelayInputAcknowledgement( message: string | ArrayBuffer, ): GitHubActionsRelayInputAcknowledgement | null { - if (typeof message !== "string") return null; - try { - const parsed = JSON.parse(message) as Record; - if (parsed.type !== relayInputAcknowledgementType || typeof parsed.accepted !== "boolean") { - return null; - } - if (parsed.accepted) return { accepted: true }; + const frame = decodeGitHubActionsRelayFrame(message, relayInputAcknowledgementFrameType); + if (!frame?.inputId || frame.payload.byteLength < 1) return null; + const acceptedByte = frame.payload[0]; + if (acceptedByte !== 0 && acceptedByte !== 1) return null; + const accepted = acceptedByte === 1; + if (accepted) { return { - accepted: false, - error: - typeof parsed.error === "string" && parsed.error.trim() - ? parsed.error.trim() - : relayInputRejectedError, + inputId: frame.inputId, + accepted: true, }; - } catch { - return null; } + const error = decoder.decode(frame.payload.subarray(1)).trim(); + return { + inputId: frame.inputId, + accepted: false, + error: error || relayInputRejectedError, + }; +} + +export function createGitHubActionsRelayInputId(): string { + return crypto.randomUUID().replaceAll("-", ""); +} + +export function encodeGitHubActionsRelayInput( + inputId: string, + payload: string | ArrayBuffer | ArrayBufferView, +): ArrayBuffer { + requireGitHubActionsRelayInputId(inputId); + return encodeGitHubActionsRelayFrame(relayInputFrameType, inputId, messageBytes(payload)); +} + +export function parseGitHubActionsRelayInput( + message: string | ArrayBuffer, +): GitHubActionsRelayInput | null { + const frame = decodeGitHubActionsRelayFrame(message, relayInputFrameType); + if (!frame?.inputId) return null; + return { + inputId: frame.inputId, + payload: Uint8Array.from(frame.payload).buffer, + }; +} + +export function encodeGitHubActionsRelayInputAcknowledgement( + acknowledgement: GitHubActionsRelayInputAcknowledgement, +): ArrayBuffer { + requireGitHubActionsRelayInputId(acknowledgement.inputId); + const error = + acknowledgement.accepted || !acknowledgement.error + ? new Uint8Array() + : encoder.encode(acknowledgement.error.trim()); + const payload = new Uint8Array(1 + error.byteLength); + payload[0] = acknowledgement.accepted ? 1 : 0; + payload.set(error, 1); + return encodeGitHubActionsRelayFrame( + relayInputAcknowledgementFrameType, + acknowledgement.inputId, + payload, + ); +} + +export function parseGitHubActionsRelayEvent( + message: string | ArrayBuffer, +): { type: keyof typeof relayEventCodes } | null { + const frame = decodeGitHubActionsRelayFrame(message, relayEventFrameType); + if (!frame || frame.inputId || frame.payload.byteLength !== 1) return null; + const type = relayEvents.get(frame.payload[0] ?? 0); + return type ? { type } : null; } export function isGitHubActionsViewerControlMessage(message: string | ArrayBuffer): boolean { @@ -207,7 +284,11 @@ export function notifyGitHubActionsViewers( viewers: readonly GitHubActionsRelaySocket[], type: "runner_connected" | "runner_disconnected" | "runner_waiting", ): number { - const payload = JSON.stringify({ type }); + const payload = encodeGitHubActionsRelayFrame( + relayEventFrameType, + "", + new Uint8Array([relayEventCodes[type]]), + ); let notified = 0; for (const socket of viewers) { if (socket.readyState !== webSocketOpen) continue; @@ -216,3 +297,66 @@ export function notifyGitHubActionsViewers( } return notified; } + +function encodeGitHubActionsRelayFrame( + type: number, + inputId: string, + payload: Uint8Array, +): ArrayBuffer { + const inputIdBytes = encoder.encode(inputId); + if ( + inputIdBytes.byteLength > relayInputIdMaximumBytes || + (inputId && !relayInputIdPattern.test(inputId)) + ) { + throw new Error("invalid GitHub Actions relay input id"); + } + const frame = new Uint8Array( + relayFrameHeaderBytes + inputIdBytes.byteLength + payload.byteLength, + ); + frame.set(relayFrameMagic, 0); + frame[relayFrameMagic.byteLength] = type; + frame[relayFrameMagic.byteLength + 1] = inputIdBytes.byteLength; + frame.set(inputIdBytes, relayFrameHeaderBytes); + frame.set(payload, relayFrameHeaderBytes + inputIdBytes.byteLength); + return frame.buffer; +} + +function decodeGitHubActionsRelayFrame( + message: string | ArrayBuffer, + expectedType: number, +): { inputId: string; payload: Uint8Array } | null { + if (typeof message === "string") return null; + const frame = new Uint8Array(message); + if (frame.byteLength < relayFrameHeaderBytes) return null; + for (const [index, value] of relayFrameMagic.entries()) { + if (frame[index] !== value) return null; + } + if (frame[relayFrameMagic.byteLength] !== expectedType) return null; + const inputIdBytes = frame[relayFrameMagic.byteLength + 1] ?? 0; + if ( + inputIdBytes > relayInputIdMaximumBytes || + relayFrameHeaderBytes + inputIdBytes > frame.byteLength + ) { + return null; + } + const inputId = decoder.decode( + frame.subarray(relayFrameHeaderBytes, relayFrameHeaderBytes + inputIdBytes), + ); + if (inputId && !relayInputIdPattern.test(inputId)) return null; + return { + inputId, + payload: frame.subarray(relayFrameHeaderBytes + inputIdBytes), + }; +} + +function messageBytes(message: string | ArrayBuffer | ArrayBufferView): Uint8Array { + if (typeof message === "string") return encoder.encode(message); + if (ArrayBuffer.isView(message)) { + return new Uint8Array(message.buffer, message.byteOffset, message.byteLength); + } + return new Uint8Array(message); +} + +function requireGitHubActionsRelayInputId(inputId: string): void { + if (!inputId) throw new Error("invalid GitHub Actions relay input id"); +} diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index 8b9f54ae..3795dd03 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -237,7 +237,7 @@ export class SessionControlDO extends DurableObject { } else { this.ctx.acceptWebSocket(server, ["github-actions-viewer"]); if (this.ctx.getWebSockets("github-actions-runner").length === 0) { - server.send(JSON.stringify({ type: "runner_waiting" })); + notifyGitHubActionsViewers([server], "runner_waiting"); } } return new Response(null, { status: 101, webSocket: client }); diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index f3c14562..6557962f 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -14,8 +14,11 @@ import { sendOutputAcknowledgement, } from "@openclaw/libterminal/worker"; import { + createGitHubActionsRelayInputId, + encodeGitHubActionsRelayInput, githubActionsRuntime, parseGitHubActionsRelayInputAcknowledgement, + parseGitHubActionsRelayEvent, type GitHubActionsRelayInputAcknowledgement, } from "../github-actions-runtime.ts"; import { redactedAdapterMessage } from "../runtime-adapter.ts"; @@ -28,6 +31,7 @@ const terminalFrameLimits = { maxFrameBytes: 16 * 1024 * 1024 }; const terminalInputAcknowledgementTimeoutMs = 5_000; type PendingTerminalInputAcknowledgement = { + inputId: string; promise: Promise; resolve(result: GitHubActionsRelayInputAcknowledgement): void; timeout: ReturnType; @@ -50,7 +54,7 @@ export type TerminalHubSubscription = { cols: number; rows: number; inputAcknowledgements: boolean; - pendingInputAcknowledgements: PendingTerminalInputAcknowledgement[]; + pendingInputAcknowledgements: Map; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; }; @@ -252,15 +256,21 @@ export class TerminalHub { }); return; } - const acknowledgement = subscription.inputAcknowledgements - ? beginTerminalInputAcknowledgement(subscription) + const inputId = subscription.inputAcknowledgements + ? createGitHubActionsRelayInputId() + : null; + const acknowledgement = inputId + ? beginTerminalInputAcknowledgement(subscription, inputId) : null; if (acknowledgement) acknowledgements.push(acknowledgement); try { - subscription.upstream.send(input); + subscription.upstream.send( + inputId ? encodeGitHubActionsRelayInput(inputId, input) : input, + ); } catch { if (acknowledgement) { - completeTerminalInputAcknowledgement(subscription, acknowledgement, { + completeTerminalInputAcknowledgement(subscription, acknowledgement.inputId, { + inputId: acknowledgement.inputId, accepted: false, error: "terminal upstream send failed", }); @@ -461,7 +471,7 @@ export class TerminalHub { cols, rows, inputAcknowledgements: session.runtime === githubActionsRuntime, - pendingInputAcknowledgements: [], + pendingInputAcknowledgements: new Map(), outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; @@ -504,17 +514,30 @@ export class TerminalHub { .then(async () => { const data = await normalizeWebSocketMessageData(raw); if (client.readyState !== WebSocket.OPEN || !viewGranted) return; - if (typeof data === "string") { + if (activeSubscription.inputAcknowledgements && typeof data !== "string") { const inputAcknowledgement = parseGitHubActionsRelayInputAcknowledgement(data); if (inputAcknowledgement) { - completeNextTerminalInputAcknowledgement(activeSubscription, inputAcknowledgement); + completeTerminalInputAcknowledgement( + activeSubscription, + inputAcknowledgement.inputId, + inputAcknowledgement, + ); return; } - const parsed = parseTerminalControlMessage(data); - if (parsed) { - sendTerminalJson(client, TerminalMessageType.Event, id, parsed); + const relayEvent = parseGitHubActionsRelayEvent(data); + if (relayEvent) { + sendTerminalJson(client, TerminalMessageType.Event, id, relayEvent); return; } + } + if (typeof data === "string") { + if (!activeSubscription.inputAcknowledgements) { + const parsed = parseTerminalControlMessage(data); + if (parsed) { + sendTerminalJson(client, TerminalMessageType.Event, id, parsed); + return; + } + } const output = encoder.encode(data); sendTerminalFrame(client, TerminalMessageType.Output, id, output); if (activeSubscription.outputAcknowledgements) { @@ -596,12 +619,14 @@ export class TerminalHub { function beginTerminalInputAcknowledgement( subscription: TerminalHubSubscription, + inputId: string, ): PendingTerminalInputAcknowledgement { let resolve!: (result: GitHubActionsRelayInputAcknowledgement) => void; const promise = new Promise((complete) => { resolve = complete; }); const pending: PendingTerminalInputAcknowledgement = { + inputId, promise, resolve, timeout: setTimeout(() => { @@ -618,39 +643,32 @@ function beginTerminalInputAcknowledgement( } }, terminalInputAcknowledgementTimeoutMs), }; - subscription.pendingInputAcknowledgements.push(pending); + subscription.pendingInputAcknowledgements.set(inputId, pending); return pending; } function completeTerminalInputAcknowledgement( subscription: TerminalHubSubscription, - pending: PendingTerminalInputAcknowledgement, + inputId: string, result: GitHubActionsRelayInputAcknowledgement, ): boolean { - const index = subscription.pendingInputAcknowledgements.indexOf(pending); - if (index < 0) return false; - subscription.pendingInputAcknowledgements.splice(index, 1); + const pending = subscription.pendingInputAcknowledgements.get(inputId); + if (!pending) return false; + subscription.pendingInputAcknowledgements.delete(inputId); clearTimeout(pending.timeout); pending.resolve(result); return true; } -function completeNextTerminalInputAcknowledgement( - subscription: TerminalHubSubscription, - result: GitHubActionsRelayInputAcknowledgement, -): boolean { - const pending = subscription.pendingInputAcknowledgements[0]; - return pending ? completeTerminalInputAcknowledgement(subscription, pending, result) : false; -} - function completeAllTerminalInputAcknowledgements( subscription: TerminalHubSubscription, - result: GitHubActionsRelayInputAcknowledgement, + result: Omit, ): number { - const pending = subscription.pendingInputAcknowledgements.splice(0); + const pending = [...subscription.pendingInputAcknowledgements.values()]; + subscription.pendingInputAcknowledgements.clear(); for (const acknowledgement of pending) { clearTimeout(acknowledgement.timeout); - acknowledgement.resolve(result); + acknowledgement.resolve({ inputId: acknowledgement.inputId, ...result }); } return pending.length; } diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts new file mode 100644 index 00000000..d214d593 --- /dev/null +++ b/tests/github-actions-runner.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { acceptGitHubActionsRunnerInput } from "../src/github-actions-runner.ts"; +import { + encodeGitHubActionsRelayInput, + parseGitHubActionsRelayInputAcknowledgement, + type GitHubActionsRelaySocket, +} from "../src/github-actions-runtime.ts"; + +function relaySocket(): GitHubActionsRelaySocket & { sent: Array } { + return { + readyState: WebSocket.OPEN, + sent: [], + send(message) { + this.sent.push(message); + }, + close() {}, + }; +} + +test("runner acknowledges input only after the PTY write completes", async () => { + const socket = relaySocket(); + let completeWrite!: () => void; + const write = new Promise((resolve) => { + completeWrite = resolve; + }); + const handled = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-one", "steer"), + async (payload) => { + assert.equal(new TextDecoder().decode(payload), "steer"); + await write; + }, + ); + + await Promise.resolve(); + assert.deepEqual(socket.sent, []); + completeWrite(); + assert.equal(await handled, true); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(socket.sent[0]!), { + inputId: "input-one", + accepted: true, + }); +}); + +test("runner rejects failed writes and ignores unframed terminal data", async () => { + const socket = relaySocket(); + + assert.equal( + await acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-two", "steer"), + async () => { + throw new Error("PTY closed"); + }, + ), + true, + ); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(socket.sent[0]!), { + inputId: "input-two", + accepted: false, + error: "GitHub Actions runner did not accept terminal input", + }); + assert.equal(await acceptGitHubActionsRunnerInput(socket, "raw output", async () => {}), false); + assert.equal(socket.sent.length, 1); +}); diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 3fd74032..13f6a692 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -2,6 +2,8 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { buildGitHubActionsRunnerPtyUrl, + encodeGitHubActionsRelayInput, + encodeGitHubActionsRelayInputAcknowledgement, forwardGitHubActionsRelayMessage, gitHubActionsSessionStatus, githubActionsCapabilities, @@ -10,6 +12,8 @@ import { isGitHubActionsViewerControlMessage, isTerminalGitHubActionsWorkState, notifyGitHubActionsViewers, + parseGitHubActionsRelayEvent, + parseGitHubActionsRelayInput, parseGitHubActionsRelayInputAcknowledgement, parseGitHubActionsWorkState, relayGitHubActionsWebSocketMessage, @@ -67,7 +71,7 @@ test("work states preserve running phases and map terminal outcomes", () => { assert.equal(gitHubActionsSessionStatus("failed"), "failed"); }); -test("relay replaces the current runner and routes messages by role", () => { +test("relay replaces the current runner and fans out raw runner output", () => { const oldRunner = relaySocket(); const runner = relaySocket(); const viewerOne = relaySocket(); @@ -85,92 +89,109 @@ test("relay replaces the current runner and routes messages by role", () => { ); assert.deepEqual(viewerOne.sent, ["output"]); assert.deepEqual(viewerTwo.sent, ["output"]); - - assert.equal( - forwardGitHubActionsRelayMessage("viewer", "input", [runner], [viewerOne, viewerTwo]), - 1, - ); - assert.deepEqual(runner.sent, ["input"]); }); -test("relay sends viewer input to the first open runner", () => { +test("relay sends framed viewer input to the first open runner without acknowledging queueing", () => { const closedRunner = relaySocket(3); const openRunner = relaySocket(); const laterRunner = relaySocket(); + const viewer = relaySocket(); + const input = encodeGitHubActionsRelayInput("input-one", "steer"); assert.equal( - forwardGitHubActionsRelayMessage( + relayGitHubActionsWebSocketMessage( "viewer", - "input", + viewer, + input, [closedRunner, openRunner, laterRunner], [], ), 1, ); assert.deepEqual(closedRunner.sent, []); - assert.deepEqual(openRunner.sent, ["input"]); + assert.deepEqual(openRunner.sent, [input]); assert.deepEqual(laterRunner.sent, []); + assert.deepEqual(viewer.sent, []); + assert.deepEqual(parseGitHubActionsRelayInput(openRunner.sent[0]!), { + inputId: "input-one", + payload: new TextEncoder().encode("steer").buffer, + }); }); -test("relay reports runner delivery failures instead of claiming forwarded input", () => { +test("relay rejects framed input only when no runner accepts the frame", () => { const runner = relaySocket(); runner.send = () => { throw new Error("runner disconnected"); }; - - assert.equal(forwardGitHubActionsRelayMessage("viewer", "input", [runner], []), 0); -}); - -test("relay input acknowledgements distinguish accepted and rejected delivery", () => { const viewer = relaySocket(); + const input = encodeGitHubActionsRelayInput("input-failed", "steer"); - assert.equal(sendGitHubActionsRelayInputAcknowledgement(viewer, true), true); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, input, [runner], []), 0); assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { - accepted: true, + inputId: "input-failed", + accepted: false, + error: "GitHub Actions runner did not accept terminal input", }); - assert.equal(sendGitHubActionsRelayInputAcknowledgement(viewer, false), true); - assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[1]!), { + const waitingViewer = relaySocket(); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", waitingViewer, input, [], []), 0); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(waitingViewer.sent[0]!), { + inputId: "input-failed", accepted: false, error: "GitHub Actions runner did not accept terminal input", }); - assert.equal(parseGitHubActionsRelayInputAcknowledgement('{"type":"runner_waiting"}'), null); - assert.equal(sendGitHubActionsRelayInputAcknowledgement(relaySocket(3), false), false); }); -test("viewer relay acknowledges only input delivered to an open runner", () => { +test("runner acknowledgements retain correlation and fan out to viewers", () => { const runner = relaySocket(); - const viewer = relaySocket(); - - assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, "input", [runner], []), 1); - assert.deepEqual(runner.sent, ["input"]); - assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + const viewerOne = relaySocket(); + const viewerTwo = relaySocket(); + const acknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "input-two", accepted: true, }); - const waitingViewer = relaySocket(); - assert.equal(relayGitHubActionsWebSocketMessage("viewer", waitingViewer, "input", [], []), 0); - assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(waitingViewer.sent[0]!), { - accepted: false, - error: "GitHub Actions runner did not accept terminal input", + assert.equal( + relayGitHubActionsWebSocketMessage( + "runner", + runner, + acknowledgement, + [runner], + [viewerOne, viewerTwo], + ), + 2, + ); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewerOne.sent[0]!), { + inputId: "input-two", + accepted: true, }); + assert.deepEqual(viewerTwo.sent, [acknowledgement]); +}); - const failedRunner = relaySocket(); - failedRunner.send = () => { - throw new Error("runner disconnected"); - }; - const failedViewer = relaySocket(); +test("typed acknowledgements reject malformed ids and preserve collision-shaped terminal text", () => { + const viewer = relaySocket(); + const runner = relaySocket(); + const collision = '{"type":"github_actions_input_ack","inputId":"input-three","accepted":true}'; + + assert.equal(parseGitHubActionsRelayInputAcknowledgement(collision), null); assert.equal( - relayGitHubActionsWebSocketMessage("viewer", failedViewer, "input", [failedRunner], []), - 0, + relayGitHubActionsWebSocketMessage("runner", runner, collision, [runner], [viewer]), + 1, ); - assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(failedViewer.sent[0]!), { - accepted: false, - error: "GitHub Actions runner did not accept terminal input", + assert.deepEqual(viewer.sent, [collision]); + assert.throws(() => encodeGitHubActionsRelayInput("bad id", "input"), { + message: "invalid GitHub Actions relay input id", }); + assert.equal( + sendGitHubActionsRelayInputAcknowledgement(relaySocket(3), { + inputId: "input-three", + accepted: false, + }), + false, + ); }); -test("relay consumes viewer resize controls without corrupting raw runner input", () => { +test("relay consumes viewer resize controls and rejects unframed input", () => { const runner = relaySocket(); const viewer = relaySocket(); const resize = JSON.stringify({ type: "resize", cols: 120, rows: 40 }); @@ -181,11 +202,8 @@ test("relay consumes viewer resize controls without corrupting raw runner input" assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, resize, [runner], []), 0); assert.deepEqual(runner.sent, []); assert.deepEqual(viewer.sent, []); - assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, typedJson, [runner], []), 1); - assert.deepEqual(runner.sent, [typedJson]); - assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { - accepted: true, - }); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, typedJson, [runner], []), 0); + assert.deepEqual(runner.sent, []); }); test("relay tags and runner lifecycle notifications stay explicit", () => { @@ -194,5 +212,7 @@ test("relay tags and runner lifecycle notifications stay explicit", () => { assert.equal(githubActionsRelayRole(["github-actions-viewer"]), "viewer"); assert.equal(githubActionsRelayRole([]), null); assert.equal(notifyGitHubActionsViewers([viewer], "runner_waiting"), 1); - assert.deepEqual(viewer.sent, ['{"type":"runner_waiting"}']); + assert.deepEqual(parseGitHubActionsRelayEvent(viewer.sent[0]!), { + type: "runner_waiting", + }); }); diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 79687a11..682afd02 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -10,7 +10,10 @@ import { encodeSubscribePayload, encodeTerminalFrame, } from "@openclaw/libterminal/protocol"; -import { sendGitHubActionsRelayInputAcknowledgement } from "../src/github-actions-runtime.ts"; +import { + encodeGitHubActionsRelayInputAcknowledgement, + parseGitHubActionsRelayInput, +} from "../src/github-actions-runtime.ts"; import type { User } from "../src/worker/models.ts"; import { containerCapabilities, interactiveSession } from "../src/worker/session-model.ts"; import { TerminalHub, type TerminalHubDependencies } from "../src/worker/terminal-hub.ts"; @@ -62,6 +65,22 @@ function frame(value: string | ArrayBuffer | ArrayBufferView | Blob) { return decoded; } +function relayInput(value: string | ArrayBuffer | ArrayBufferView | Blob) { + assert.ok(value instanceof ArrayBuffer); + const input = parseGitHubActionsRelayInput(value); + assert.ok(input); + return { + inputId: input.inputId, + text: new TextDecoder().decode(input.payload), + }; +} + +function emitRelayAcknowledgement(upstream: TestSocket, inputId: string, accepted: boolean): void { + upstream.emit("message", { + data: encodeGitHubActionsRelayInputAcknowledgement({ inputId, accepted }), + }); +} + async function flushQueues(): Promise { await new Promise((resolve) => setImmediate(resolve)); } @@ -486,7 +505,7 @@ test("terminal hub never acknowledges input after its upstream closes", async () server.emit("close"); }); -test("GitHub Actions input waits for relay delivery before acknowledgement", async () => { +test("GitHub Actions input waits for the correlated runner acknowledgement", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -522,7 +541,8 @@ test("GitHub Actions input waits for relay delivery before acknowledgement", asy }); await flushQueues(); - assert.equal(new TextDecoder().decode(upstream.sent.at(-1) as Uint8Array), "steer\r"); + const input = relayInput(upstream.sent.at(-1)!); + assert.equal(input.text, "steer\r"); assert.equal( server.sent.some( (payload) => @@ -532,16 +552,7 @@ test("GitHub Actions input waits for relay delivery before acknowledgement", asy false, ); - sendGitHubActionsRelayInputAcknowledgement( - { - readyState: WebSocket.OPEN, - send(message) { - upstream.emit("message", { data: message }); - }, - close() {}, - }, - true, - ); + emitRelayAcknowledgement(upstream, input.inputId, true); await flushQueues(); await flushQueues(); @@ -586,16 +597,8 @@ test("GitHub Actions relay rejection becomes a terminal input error", async () = }); await flushQueues(); - sendGitHubActionsRelayInputAcknowledgement( - { - readyState: WebSocket.OPEN, - send(message) { - upstream.emit("message", { data: message }); - }, - close() {}, - }, - false, - ); + const input = relayInput(upstream.sent.at(-1)!); + emitRelayAcknowledgement(upstream, input.inputId, false); await flushQueues(); await flushQueues(); @@ -616,7 +619,7 @@ test("GitHub Actions relay rejection becomes a terminal input error", async () = server.emit("close"); }); -test("GitHub Actions input acknowledgements resolve overlapping payloads in FIFO order", async () => { +test("GitHub Actions input acknowledgements correlate overlapping payloads out of order", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -654,35 +657,27 @@ test("GitHub Actions input acknowledgements resolve overlapping payloads in FIFO }); await waitForInputPayloads(); + const inputs = upstream.sent.map(relayInput); assert.deepEqual( - upstream.sent.map((payload) => new TextDecoder().decode(payload as Uint8Array)), + inputs.map((input) => input.text), ["first", "second"], ); - sendGitHubActionsRelayInputAcknowledgement( - { - readyState: WebSocket.OPEN, - send(message) { - upstream.emit("message", { data: message }); - }, - close() {}, - }, - true, - ); + assert.notEqual(inputs[0]!.inputId, inputs[1]!.inputId); + + emitRelayAcknowledgement(upstream, inputs[1]!.inputId, true); + emitRelayAcknowledgement(upstream, "stale-input-id", true); await flushQueues(); + assert.equal( + server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Output).length, + 0, + ); assert.notDeepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { type: "input-accepted", }); - sendGitHubActionsRelayInputAcknowledgement( - { - readyState: WebSocket.OPEN, - send(message) { - upstream.emit("message", { data: message }); - }, - close() {}, - }, - true, - ); + emitRelayAcknowledgement(upstream, inputs[0]!.inputId, true); await flushQueues(); await flushQueues(); @@ -692,6 +687,43 @@ test("GitHub Actions input acknowledgements resolve overlapping payloads in FIFO server.emit("close"); }); +test("GitHub Actions collision-shaped terminal text remains raw output", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + const collision = '{"type":"runner_waiting","inputId":"stale-input-id","accepted":true}'; + upstream.emit("message", { data: collision }); + await flushQueues(); + + const output = frame(server.sent.at(-1)!); + assert.equal(output.type, TerminalMessageType.Output); + assert.equal(new TextDecoder().decode(output.payload), collision); + server.emit("close"); +}); + test("GitHub Actions send failure removes only its own acknowledgement waiter", async () => { const client = socket(); const server = socket(); @@ -736,21 +768,9 @@ test("GitHub Actions send failure removes only its own acknowledgement waiter", }), }); await waitForInputPayloads(); - assert.deepEqual( - upstream.sent.map((payload) => new TextDecoder().decode(payload as Uint8Array)), - ["delivered"], - ); - - sendGitHubActionsRelayInputAcknowledgement( - { - readyState: WebSocket.OPEN, - send(message) { - upstream.emit("message", { data: message }); - }, - close() {}, - }, - true, - ); + const input = relayInput(upstream.sent[0]!); + assert.equal(input.text, "delivered"); + emitRelayAcknowledgement(upstream, input.inputId, true); await flushQueues(); await flushQueues(); From b1410b287c6a13470f18abbe0e891bffe398418f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:16:24 +0200 Subject: [PATCH 071/242] fix(worker): fence desktop host cleanup ownership --- CHANGELOG.md | 1 + migrations/0033_desktop_host_ownership.sql | 2 + src/worker/database.ts | 1 + src/worker/desktop-host-repository.ts | 10 +++- src/worker/desktop-host-service.ts | 55 ++++++++++++++++++++-- src/worker/routes/control-plane.ts | 24 +++++++--- src/worker/worker-application.ts | 3 +- tests/control-plane-routes.test.ts | 37 ++++++++++----- tests/desktop-host-migration.test.ts | 11 +++++ tests/desktop-host-repository.test.ts | 8 +++- tests/desktop-host-service.test.ts | 51 +++++++++++++++++--- 11 files changed, 168 insertions(+), 35 deletions(-) create mode 100644 migrations/0033_desktop_host_ownership.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index a817b4ed..e759c4bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. +- Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. diff --git a/migrations/0033_desktop_host_ownership.sql b/migrations/0033_desktop_host_ownership.sql new file mode 100644 index 00000000..f9aa13c2 --- /dev/null +++ b/migrations/0033_desktop_host_ownership.sql @@ -0,0 +1,2 @@ +ALTER TABLE desktop_hosts + ADD COLUMN ownership_token TEXT NOT NULL DEFAULT ''; diff --git a/src/worker/database.ts b/src/worker/database.ts index 2bdd96e9..64eba7ba 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -69,6 +69,7 @@ export type DesktopHostTable = { name: string; address: string; port: number; + ownership_token: string; created_at: number; updated_at: number; }; diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 63147aae..46cd7f1a 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -8,6 +8,7 @@ export type DesktopHostRow = { name: string; address: string; port: number; + ownershipToken: string; createdAt: number; updatedAt: number; }; @@ -17,7 +18,7 @@ export type DesktopHostWrite = DesktopHostRow; export interface DesktopHostStore { list(ownerSubject: string): Promise; upsert(host: DesktopHostWrite): Promise; - remove(ownerSubject: string, id: string): Promise; + remove(ownerSubject: string, id: string, ownershipToken: string): Promise; } export class DesktopHostRepository implements DesktopHostStore { @@ -42,6 +43,7 @@ export class DesktopHostRepository implements DesktopHostStore { name: row.name, address: row.address, port: row.port, + ownershipToken: row.ownership_token, createdAt: row.created_at, updatedAt: row.updated_at, })); @@ -57,6 +59,7 @@ export class DesktopHostRepository implements DesktopHostStore { name: host.name, address: host.address, port: host.port, + ownership_token: host.ownershipToken, created_at: host.createdAt, updated_at: host.updatedAt, }) @@ -66,6 +69,7 @@ export class DesktopHostRepository implements DesktopHostStore { name: host.name, address: host.address, port: host.port, + ownership_token: host.ownershipToken, updated_at: host.updatedAt, }), ) @@ -83,16 +87,18 @@ export class DesktopHostRepository implements DesktopHostStore { name: row.name, address: row.address, port: row.port, + ownershipToken: row.ownership_token, createdAt: row.created_at, updatedAt: row.updated_at, }; } - async remove(ownerSubject: string, id: string): Promise { + async remove(ownerSubject: string, id: string, ownershipToken: string): Promise { await database(this.env) .deleteFrom("desktop_hosts") .where("owner_subject", "=", ownerSubject) .where("id", "=", id) + .where("ownership_token", "=", ownershipToken) .execute(); } } diff --git a/src/worker/desktop-host-service.ts b/src/worker/desktop-host-service.ts index f3931419..24262fe5 100644 --- a/src/worker/desktop-host-service.ts +++ b/src/worker/desktop-host-service.ts @@ -19,13 +19,26 @@ export type DesktopHost = { updatedAt: number; }; +export type DesktopHostRegistration = { + host: DesktopHost; + ownershipToken: string; +}; + +export const desktopHostOwnershipHeader = "x-crabfleet-ownership-token"; + export class DesktopHostService { private readonly store: DesktopHostStore; private readonly now: () => number; + private readonly createOwnershipToken: () => string; - constructor(store: DesktopHostStore, now: () => number = Date.now) { + constructor( + store: DesktopHostStore, + now: () => number = Date.now, + createOwnershipToken: () => string = randomOwnershipToken, + ) { this.store = store; this.now = now; + this.createOwnershipToken = createOwnershipToken; } async list(user: User): Promise { @@ -33,12 +46,17 @@ export class DesktopHostService { return rows.map(presentDesktopHost); } - async register(user: User, rawID: string, input: DesktopHostInput): Promise { + async register( + user: User, + rawID: string, + input: DesktopHostInput, + ): Promise { const id = desktopHostID(rawID); const name = boundedText(input.name, "name", 100); const address = tailscaleIPv4(input.address); const port = desktopHostPort(input.port); const now = this.now(); + const ownershipToken = this.createOwnershipToken(); const host: DesktopHostRow = { ownerSubject: tenantSubject(user), id, @@ -46,17 +64,29 @@ export class DesktopHostService { name, address, port, + ownershipToken, createdAt: now, updatedAt: now, }; - return presentDesktopHost(await this.store.upsert(host)); + return { + host: presentDesktopHost(await this.store.upsert(host)), + ownershipToken, + }; } - async remove(user: User, rawID: string): Promise { - await this.store.remove(tenantSubject(user), desktopHostID(rawID)); + async remove(user: User, rawID: string, rawOwnershipToken: unknown): Promise { + await this.store.remove( + tenantSubject(user), + desktopHostID(rawID), + desktopHostOwnershipToken(rawOwnershipToken), + ); } } +function randomOwnershipToken(): string { + return crypto.randomUUID() + crypto.randomUUID(); +} + function presentDesktopHost(row: DesktopHostRow): DesktopHost { return { id: row.id, @@ -121,3 +151,18 @@ function desktopHostPort(value: unknown): number { } return value; } + +function desktopHostOwnershipToken(value: unknown): string { + if ( + typeof value !== "string" || + value.length === 0 || + new TextEncoder().encode(value).byteLength > 200 || + [...value].some((character) => { + const codePoint = character.codePointAt(0) ?? 0; + return codePoint <= 0x20 || codePoint === 0x7f; + }) + ) { + throw badRequest("desktop host ownership token is required"); + } + return value; +} diff --git a/src/worker/routes/control-plane.ts b/src/worker/routes/control-plane.ts index 59eaa8ea..039ab43f 100644 --- a/src/worker/routes/control-plane.ts +++ b/src/worker/routes/control-plane.ts @@ -5,15 +5,23 @@ import type { AdminRepoInput, AdminWorkflowInput, } from "../admin-service.ts"; -import type { DesktopHost, DesktopHostInput } from "../desktop-host-service.ts"; -import { json, notFound, readJson } from "../http.ts"; +import { + desktopHostOwnershipHeader, + type DesktopHostInput, + type DesktopHostRegistration, +} from "../desktop-host-service.ts"; +import { badRequest, json, notFound, readJson } from "../http.ts"; import type { User } from "../models.ts"; export type ControlPlaneRouteDependencies = { readState(request: Request, user: User): Promise; readFleet(user: User): Promise; - registerDesktopHost(user: User, id: string, input: DesktopHostInput): Promise; - removeDesktopHost(user: User, id: string): Promise; + registerDesktopHost( + user: User, + id: string, + input: DesktopHostInput, + ): Promise; + removeDesktopHost(user: User, id: string, ownershipToken: string): Promise; searchGitHubRefs(number: unknown): Promise; createCard(request: Request, user: User): Promise; readCardRuns(user: User, cardId: string): Promise; @@ -44,16 +52,18 @@ export async function handleControlPlaneRoute( const desktopHostMatch = url.pathname.match(/^\/api\/desktop-hosts\/([^/]+)$/); if (request.method === "PUT" && desktopHostMatch) { requireRole(user, "viewer"); - const host = await dependencies.registerDesktopHost( + const registration = await dependencies.registerDesktopHost( user, decoded(desktopHostMatch[1]), await readJson(request), ); - return json({ host }); + return json(registration); } if (request.method === "DELETE" && desktopHostMatch) { requireRole(user, "viewer"); - await dependencies.removeDesktopHost(user, decoded(desktopHostMatch[1])); + const ownershipToken = request.headers.get(desktopHostOwnershipHeader); + if (!ownershipToken) throw badRequest("desktop host ownership token is required"); + await dependencies.removeDesktopHost(user, decoded(desktopHostMatch[1]), ownershipToken); return json({ ok: true }); } if (request.method === "GET" && url.pathname === "/api/github/refs") { diff --git a/src/worker/worker-application.ts b/src/worker/worker-application.ts index 81ede5c1..8fc1b451 100644 --- a/src/worker/worker-application.ts +++ b/src/worker/worker-application.ts @@ -163,7 +163,8 @@ export class WorkerApplication { readState: (request, user) => this.readState(request, user, context), readFleet: (user) => this.readFleetState(user, undefined, context), registerDesktopHost: (user, id, input) => this.desktopHosts().register(user, id, input), - removeDesktopHost: (user, id) => this.desktopHosts().remove(user, id), + removeDesktopHost: (user, id, ownershipToken) => + this.desktopHosts().remove(user, id, ownershipToken), searchGitHubRefs: (number) => this.githubReferenceService().search(number), createCard: async (request, user) => this.cardLifecycleService().create(await readJson(request), user), diff --git a/tests/control-plane-routes.test.ts b/tests/control-plane-routes.test.ts index 4ce1a405..ab15b8e1 100644 --- a/tests/control-plane-routes.test.ts +++ b/tests/control-plane-routes.test.ts @@ -44,17 +44,20 @@ function dependencies(calls: string[]): ControlPlaneRouteDependencies { async registerDesktopHost(user, id, input) { calls.push(`desktop-host:register:${user.login}:${id}:${input.name}`); return { - id, - owner: user.login ?? user.subject, - name: String(input.name), - address: String(input.address), - port: Number(input.port), - createdAt: 1, - updatedAt: 1, + host: { + id, + owner: user.login ?? user.subject, + name: String(input.name), + address: String(input.address), + port: Number(input.port), + createdAt: 1, + updatedAt: 1, + }, + ownershipToken: "ownership-token", }; }, - async removeDesktopHost(user, id) { - calls.push(`desktop-host:remove:${user.login}:${id}`); + async removeDesktopHost(user, id, ownershipToken) { + calls.push(`desktop-host:remove:${user.login}:${id}:${ownershipToken}`); }, async searchGitHubRefs(number) { calls.push(`github-refs:${number}`); @@ -170,18 +173,30 @@ test("desktop host routes register and remove only the authenticated user's host createdAt: 1, updatedAt: 1, }, + ownershipToken: "ownership-token", }); const removed = await dispatch( - request("DELETE", "/api/desktop-hosts/mac%2Dstudio"), + new Request("https://fleet.example/api/desktop-hosts/mac%2Dstudio", { + method: "DELETE", + headers: { "x-crabfleet-ownership-token": "ownership-token" }, + }), viewer, calls, ); assert.equal(removed?.status, 200); assert.deepEqual(calls, [ "desktop-host:register:viewer:mac-studio:Mac Studio", - "desktop-host:remove:viewer:mac-studio", + "desktop-host:remove:viewer:mac-studio:ownership-token", ]); + + await assert.rejects( + dispatch(request("DELETE", "/api/desktop-hosts/mac%2Dstudio"), viewer, []), + (error) => { + assert.equal(status(error), 400); + return true; + }, + ); }); test("card actions derive viewer or maintainer authorization from the action", async () => { diff --git a/tests/desktop-host-migration.test.ts b/tests/desktop-host-migration.test.ts index b24723e8..c9d82cd8 100644 --- a/tests/desktop-host-migration.test.ts +++ b/tests/desktop-host-migration.test.ts @@ -9,8 +9,13 @@ test("desktop host migration creates an owner-scoped registry with bounded ports new URL("../migrations/0030_desktop_hosts.sql", import.meta.url), "utf8", ); + const ownershipMigration = readFileSync( + new URL("../migrations/0033_desktop_host_ownership.sql", import.meta.url), + "utf8", + ); database.exec(migration); database.exec(migration); + database.exec(ownershipMigration); const insert = database.prepare(` INSERT INTO desktop_hosts @@ -25,6 +30,12 @@ test("desktop host migration creates an owner-scoped registry with bounded ports ?.count, 2, ); + assert.equal( + database + .prepare("SELECT ownership_token FROM desktop_hosts WHERE owner_subject = 'github:1'") + .get()?.ownership_token, + "", + ); assert.throws( () => insert.run("github:3", "bad", "bad", "Bad", "100.64.1.4", 0, 1, 1), /constraint/i, diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index 6404b87d..b2d8f114 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -13,6 +13,7 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec name: "Studio", address: "100.64.1.2", port: 5901, + ownership_token: "ownership-token", created_at: 1, updated_at: 2, }; @@ -45,6 +46,7 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec name: "Studio", address: "100.64.1.2", port: 5901, + ownershipToken: "ownership-token", createdAt: 1, updatedAt: 2, }, @@ -59,6 +61,7 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec name: "Studio", address: "100.64.1.2", port: 5901, + ownershipToken: "ownership-token", createdAt: 1, updatedAt: 2, }); @@ -67,7 +70,8 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec assert.match(executions[2]?.sql ?? "", /where "owner_subject" = \? and "id" = \?/i); assert.deepEqual(executions[2]?.parameters, ["github:1", "studio"]); - await repository.remove("github:1", "studio"); + await repository.remove("github:1", "studio", "ownership-token"); assert.match(executions[3]?.sql ?? "", /^delete from "desktop_hosts"/i); - assert.deepEqual(executions[3]?.parameters, ["github:1", "studio"]); + assert.match(executions[3]?.sql ?? "", /"ownership_token" = \?/i); + assert.deepEqual(executions[3]?.parameters, ["github:1", "studio", "ownership-token"]); }); diff --git a/tests/desktop-host-service.test.ts b/tests/desktop-host-service.test.ts index dddf93c0..01fc09cd 100644 --- a/tests/desktop-host-service.test.ts +++ b/tests/desktop-host-service.test.ts @@ -36,20 +36,29 @@ class MemoryDesktopHostStore implements DesktopHostStore { return stored; } - async remove(ownerSubject: string, id: string): Promise { - this.rows.delete(`${ownerSubject}:${id}`); + async remove(ownerSubject: string, id: string, ownershipToken: string): Promise { + const key = `${ownerSubject}:${id}`; + if (this.rows.get(key)?.ownershipToken === ownershipToken) { + this.rows.delete(key); + } } } test("desktop hosts are canonicalized and isolated to their stable owner", async () => { const store = new MemoryDesktopHostStore(); let now = 42; - const service = new DesktopHostService(store, () => now); - const host = await service.register(alice, " Studio.ONE ", { + const tokens = ["ownership-1", "ownership-2"]; + const service = new DesktopHostService( + store, + () => now, + () => tokens.shift() ?? "unexpected-token", + ); + const registration = await service.register(alice, " Studio.ONE ", { name: " Peter's Mac Studio ", address: "100.68.201.40", port: 5901, }); + const host = registration.host; assert.deepEqual(host, { id: "studio.one", @@ -60,22 +69,48 @@ test("desktop hosts are canonicalized and isolated to their stable owner", async createdAt: 42, updatedAt: 42, }); + assert.equal(registration.ownershipToken, "ownership-1"); assert.deepEqual(await service.list(alice), [host]); assert.deepEqual(await service.list(bob), []); now = 84; - const updated = await service.register(alice, host.id, { + const updatedRegistration = await service.register(alice, host.id, { name: "Renamed Studio", address: host.address, port: host.port, }); + const updated = updatedRegistration.host; assert.equal(updated.createdAt, 42); assert.equal(updated.updatedAt, 84); assert.equal(updated.name, "Renamed Studio"); + assert.equal(updatedRegistration.ownershipToken, "ownership-2"); - await service.remove(bob, host.id); + await service.remove(bob, host.id, updatedRegistration.ownershipToken); assert.deepEqual(await service.list(alice), [updated]); - await service.remove(alice, host.id); + await service.remove(alice, host.id, updatedRegistration.ownershipToken); + assert.deepEqual(await service.list(alice), []); +}); + +test("stale desktop host cleanup cannot remove a newer registration", async () => { + const store = new MemoryDesktopHostStore(); + const tokens = ["old-process-token", "new-process-token"]; + const service = new DesktopHostService( + store, + () => 42, + () => tokens.shift() ?? "unexpected-token", + ); + const input = { name: "Studio", address: "100.64.1.2", port: 5901 }; + + const oldRegistration = await service.register(alice, "studio", input); + const newRegistration = await service.register(alice, "studio", { + ...input, + name: "New Studio Process", + }); + + await service.remove(alice, "studio", oldRegistration.ownershipToken); + assert.deepEqual(await service.list(alice), [newRegistration.host]); + + await service.remove(alice, "studio", newRegistration.ownershipToken); assert.deepEqual(await service.list(alice), []); }); @@ -98,4 +133,6 @@ test("desktop hosts accept only bounded metadata and Tailscale IPv4 endpoints", } await assert.rejects(service.register(alice, "studio", { ...valid, name: "bad\nname" }), /name/); await assert.rejects(service.register(alice, "studio", { ...valid, port: 0 }), /port/); + await assert.rejects(service.remove(alice, "studio", null), /ownership token/); + await assert.rejects(service.remove(alice, "studio", "bad token"), /ownership token/); }); From fe7051374ef92593969bb46f741a23df68547309 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:16:47 +0200 Subject: [PATCH 072/242] fix(macos): harden private share teardown lifecycle --- .../CrabfleetDesktopRegistration.swift | 53 ++++-- .../PrivateMacShareController.swift | 117 ++++++++----- .../PrivateMacShareTests.swift | 160 ++++++++++++++++-- 3 files changed, 259 insertions(+), 71 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index 1cd410aa..d087fafa 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -1,8 +1,8 @@ import Foundation protocol DesktopHostRegistering: Sendable { - func register(identity: TailnetIdentity, port: UInt16) async throws - func unregister(identity: TailnetIdentity) async throws + func register(identity: TailnetIdentity, port: UInt16) async throws -> String + func unregister(identity: TailnetIdentity, ownershipToken: String) async throws } actor DesktopHostRegistrationCoordinator { @@ -13,29 +13,29 @@ actor DesktopHostRegistrationCoordinator { self.registration = registration } - func register(identity: TailnetIdentity, port: UInt16) async throws { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String { let registration = self.registration let operation = enqueue { try await registration.register(identity: identity, port: port) } - try await operation.value + return try await operation.value } - func unregister(identity: TailnetIdentity) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { let registration = self.registration let operation = enqueue { - try await registration.unregister(identity: identity) + try await registration.unregister(identity: identity, ownershipToken: ownershipToken) } try await operation.value } - private func enqueue( - _ operation: @escaping @Sendable () async throws -> Void - ) -> Task { + private func enqueue( + _ operation: @escaping @Sendable () async throws -> Value + ) -> Task { let previous = pendingOperation let task = Task { await previous?.value - try await operation() + return try await operation() } pendingOperation = Task { _ = try? await task.value @@ -45,6 +45,10 @@ actor DesktopHostRegistrationCoordinator { } struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable { + private struct RegistrationResponse: Decodable { + let ownershipToken: String + } + private struct RegistrationBody: Encodable { let name: String let address: String @@ -80,14 +84,21 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable self.transport = transport } - func register(identity: TailnetIdentity, port: UInt16) async throws { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String { let request = try registrationRequest(identity: identity, port: port) - let (_, http) = try await transport.data(for: request) + let (data, http) = try await transport.data(for: request) try validate(response: http, for: request, acceptingNotFound: false) + guard + let response = try? JSONDecoder().decode(RegistrationResponse.self, from: data), + Self.isValidOwnershipToken(response.ownershipToken) + else { + throw DesktopHostRegistrationError.invalidResponse + } + return response.ownershipToken } - func unregister(identity: TailnetIdentity) async throws { - let request = removalRequest(identity: identity) + func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + let request = try removalRequest(identity: identity, ownershipToken: ownershipToken) let (_, http) = try await transport.data(for: request) try validate(response: http, for: request, acceptingNotFound: true) } @@ -129,7 +140,10 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable return request } - func removalRequest(identity: TailnetIdentity) -> URLRequest { + func removalRequest(identity: TailnetIdentity, ownershipToken: String) throws -> URLRequest { + guard Self.isValidOwnershipToken(ownershipToken) else { + throw DesktopHostRegistrationError.invalidResponse + } let url = baseURL .appending(path: "api") @@ -140,6 +154,7 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable request.timeoutInterval = 15 request.setValue("application/json", forHTTPHeaderField: "Accept") request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") + request.setValue(ownershipToken, forHTTPHeaderField: "X-Crabfleet-Ownership-Token") return request } @@ -165,6 +180,14 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable if scheme == "https" { return true } return scheme == "http" && (host == "127.0.0.1" || host == "::1") } + + private static func isValidOwnershipToken(_ value: String) -> Bool { + !value.isEmpty && value.utf8.count <= 200 + && !value.unicodeScalars.contains { + CharacterSet.whitespacesAndNewlines.contains($0) + || CharacterSet.controlCharacters.contains($0) + } + } } enum DesktopHostRegistrationError: LocalizedError, Equatable { diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index f3a667b9..9f95c195 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -14,39 +14,50 @@ enum PrivateMacSharePermissionPolicy { @MainActor final class DesktopHostRegistrationLifecycle { + private struct PublishedRegistration: Equatable { + let identity: TailnetIdentity + let ownershipToken: String + } + private let coordinator: DesktopHostRegistrationCoordinator - private var publishedIdentity: TailnetIdentity? - private var pendingRemovalIdentities: [TailnetIdentity] = [] + private var publishedRegistration: PublishedRegistration? + private var pendingRemovals: [PublishedRegistration] = [] init(registration: any DesktopHostRegistering) { coordinator = DesktopHostRegistrationCoordinator(registration: registration) } func publish(identity: TailnetIdentity, port: UInt16) async throws { - try await coordinator.register(identity: identity, port: port) - if let publishedIdentity, publishedIdentity != identity, - !pendingRemovalIdentities.contains(publishedIdentity) + let ownershipToken = try await coordinator.register(identity: identity, port: port) + if let publishedRegistration, publishedRegistration.identity != identity, + !pendingRemovals.contains(publishedRegistration) { - pendingRemovalIdentities.append(publishedIdentity) + pendingRemovals.append(publishedRegistration) } - pendingRemovalIdentities.removeAll { $0 == identity } - publishedIdentity = identity + pendingRemovals.removeAll { $0.identity == identity } + publishedRegistration = PublishedRegistration( + identity: identity, + ownershipToken: ownershipToken + ) } func removePublishedIdentities() async throws { - if let publishedIdentity { - if !pendingRemovalIdentities.contains(publishedIdentity) { - pendingRemovalIdentities.append(publishedIdentity) + if let publishedRegistration { + if !pendingRemovals.contains(publishedRegistration) { + pendingRemovals.append(publishedRegistration) } - self.publishedIdentity = nil + self.publishedRegistration = nil } var firstError: Error? - let identities = pendingRemovalIdentities - for identity in identities { + let removals = pendingRemovals + for removal in removals { do { - try await coordinator.unregister(identity: identity) - pendingRemovalIdentities.removeAll { $0 == identity } + try await coordinator.unregister( + identity: removal.identity, + ownershipToken: removal.ownershipToken + ) + pendingRemovals.removeAll { $0 == removal } } catch { firstError = firstError ?? error } @@ -153,15 +164,19 @@ final class PrivateMacShareController: ObservableObject { private var lifecycleGeneration: UInt64 = 0 private var serverGeneration: UInt64? private var registrationTask: Task? + private var registryOperationGeneration: UInt64 = 0 + private var publishingServerGeneration: UInt64? private var refreshWaiters: [CheckedContinuation] = [] init( runner: (any TailscaleCommandRunning)? = nil, desktopRegistration: (any DesktopHostRegistering)? = CrabfleetDesktopRegistration(), + registrationLifecycle: DesktopHostRegistrationLifecycle? = nil, defaults: UserDefaults = .standard ) { self.desktopRegistration = desktopRegistration - desktopRegistrationLifecycle = desktopRegistration.map(DesktopHostRegistrationLifecycle.init) + desktopRegistrationLifecycle = + registrationLifecycle ?? desktopRegistration.map(DesktopHostRegistrationLifecycle.init) self.defaults = defaults registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished let savedDisplayID = defaults.object(forKey: Self.selectedDisplayDefaultsKey) as? Int @@ -267,14 +282,14 @@ final class PrivateMacShareController: ObservableObject { func start() async { guard phase == .idle else { return } - await waitForRefreshCompletion() - guard phase == .idle, !Task.isCancelled else { return } let generation = beginLifecycleTransition() phase = .starting notice = nil connectedPeer = nil streamStats = nil registryPhase = desktopRegistration == nil ? .notConfigured : .registering + await waitForRefreshCompletion() + guard isCurrent(generation), phase == .starting, !Task.isCancelled else { return } do { let loadedIdentity = try await fetchIdentity() guard isCurrent(generation), phase == .starting else { return } @@ -363,19 +378,8 @@ final class PrivateMacShareController: ObservableObject { let capture = capture self.capture = nil await capture?.stop() - await registrationTask?.value activeIdentity = nil - if let desktopRegistrationLifecycle { - do { - try await desktopRegistrationLifecycle.removePublishedIdentities() - registryPhase = .notPublished - } catch { - registryPhase = .failed(error.localizedDescription) - notice = error.localizedDescription - } - } else { - registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished - } + removeDesktopHost(after: registrationTask) guard isCurrent(generation) else { return } phase = .idle } @@ -472,7 +476,6 @@ final class PrivateMacShareController: ObservableObject { streamStats = nil case .listenerFailed(let message): let pendingRegistration = registrationTask - pendingRegistration?.cancel() phase = .failed connectedPeer = nil streamStats = nil @@ -495,45 +498,58 @@ final class PrivateMacShareController: ObservableObject { } private func registerDesktopHost(generation: UInt64) { - guard registrationTask == nil else { return } + guard publishingServerGeneration != generation else { return } guard let desktopRegistrationLifecycle, let identity = activeIdentity else { registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } + let pendingOperation = registrationTask + let operationGeneration = beginRegistryOperation() + publishingServerGeneration = generation registryPhase = .registering registrationTask = Task { [weak self] in do { + await pendingOperation?.value try await desktopRegistrationLifecycle.publish(identity: identity, port: Self.port) - guard !Task.isCancelled, self?.serverGeneration == generation else { return } + guard + self?.isCurrentRegistryOperation(operationGeneration) == true, + self?.serverGeneration == generation + else { return } self?.registryPhase = .registered - } catch is CancellationError { - return } catch { - guard self?.serverGeneration == generation else { return } + guard + self?.isCurrentRegistryOperation(operationGeneration) == true, + self?.serverGeneration == generation + else { return } self?.registryPhase = .failed(error.localizedDescription) } + if self?.publishingServerGeneration == generation { + self?.publishingServerGeneration = nil + } + self?.finishRegistryOperation(operationGeneration) } } private func removeDesktopHost(after pendingRegistration: Task?) { - guard let pendingRegistration else { - registrationTask = nil - registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished - return - } guard let desktopRegistrationLifecycle else { registrationTask = nil registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished return } + publishingServerGeneration = nil + let operationGeneration = beginRegistryOperation() registrationTask = Task { [weak self] in - await pendingRegistration.value + await pendingRegistration?.value do { try await desktopRegistrationLifecycle.removePublishedIdentities() + guard self?.isCurrentRegistryOperation(operationGeneration) == true else { return } self?.registryPhase = .notPublished } catch { + guard self?.isCurrentRegistryOperation(operationGeneration) == true else { return } self?.registryPhase = .failed(error.localizedDescription) + self?.notice = error.localizedDescription } + self?.finishRegistryOperation(operationGeneration) } } @@ -546,4 +562,19 @@ final class PrivateMacShareController: ObservableObject { private func isCurrent(_ generation: UInt64) -> Bool { lifecycleGeneration == generation } + + @discardableResult + private func beginRegistryOperation() -> UInt64 { + registryOperationGeneration &+= 1 + return registryOperationGeneration + } + + private func isCurrentRegistryOperation(_ generation: UInt64) -> Bool { + registryOperationGeneration == generation + } + + private func finishRegistryOperation(_ generation: UInt64) { + guard isCurrentRegistryOperation(generation) else { return } + registrationTask = nil + } } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 2487ace4..d99d3ab8 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -162,6 +162,37 @@ struct PrivateMacShareTests { #expect(await runner.callCount == 2) } + @Test @MainActor + func stopInvalidatesAStartWaitingForRefreshCompletion() async throws { + let runner = SequencedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: nil, + defaults: defaults + ) + + let refreshTask = Task { await controller.refresh() } + #expect(await waitUntilAsync { await runner.callCount == 1 }) + + let startTask = Task { await controller.start() } + #expect(await waitUntilAsync { controller.phase == .starting }) + + await controller.stop() + #expect(controller.phase == .idle) + + await runner.resumeNext( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + await refreshTask.value + await startTask.value + + #expect(await runner.callCount == 1) + #expect(controller.phase == .idle) + } + @Test func desktopRemovalWaitsForACommittedRegistrationAfterCancellation() async throws { let registration = SuspendedDesktopRegistration() @@ -169,13 +200,13 @@ struct PrivateMacShareTests { let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) let publish = Task { - try await coordinator.register(identity: identity, port: 5_901) + _ = try await coordinator.register(identity: identity, port: 5_901) } #expect(await waitUntilAsync { await registration.hasStartedRegistration }) publish.cancel() let remove = Task { - try await coordinator.unregister(identity: identity) + try await coordinator.unregister(identity: identity, ownershipToken: "registration-token") } try await Task.sleep(for: .milliseconds(20)) #expect(await registration.events == [.registerStarted]) @@ -190,6 +221,44 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func stopReturnsBeforeSlowDesktopRegistryCleanup() async throws { + let registration = SuspendedDesktopCleanupRegistration() + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + let identity = desktopIdentity(name: "slow-cleanup", address: "100.64.12.43") + try await lifecycle.publish(identity: identity, port: 5_901) + + let runner = SuspendedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: registration, + registrationLifecycle: lifecycle, + defaults: defaults + ) + let startTask = Task { await controller.start() } + #expect(await waitUntilAsync { await runner.hasStarted }) + + let stopState = AsyncInvocationState() + let stopTask = Task { + await controller.stop() + await stopState.markFinished() + } + #expect(await waitUntilAsync { await stopState.finished }) + #expect(controller.phase == .idle) + #expect(await waitUntilAsync { await registration.hasStartedUnregistration }) + + await runner.resume( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + await startTask.value + await registration.finishUnregistration() + await stopTask.value + #expect(await waitUntilAsync { controller.registryPhase == .notPublished }) + } + @Test @MainActor func failedDesktopPublicationIsNotUnregistered() async throws { let identity = desktopIdentity(name: "failed-publish", address: "100.64.12.40") @@ -228,11 +297,11 @@ struct PrivateMacShareTests { await registration.events == [ .register(first.dnsName), - .unregister(first.dnsName), + .unregister(first.dnsName, "token:\(first.dnsName)"), .register(second.dnsName), - .unregister(first.dnsName), - .unregister(second.dnsName), - .unregister(first.dnsName), + .unregister(first.dnsName, "token:\(first.dnsName)"), + .unregister(second.dnsName, "token:\(second.dnsName)"), + .unregister(first.dnsName, "token:\(first.dnsName)"), ] ) } @@ -399,13 +468,51 @@ struct PrivateMacShareTests { #expect(json["address"] as? String == "100.64.12.34") #expect(json["port"] as? Int == 5901) - let removal = registration.removalRequest(identity: identity) + let removal = try registration.removalRequest( + identity: identity, + ownershipToken: "desktop-ownership-token" + ) #expect(removal.url == request.url) #expect(removal.httpMethod == "DELETE") #expect(removal.value(forHTTPHeaderField: "Cookie") == "crabbox_session=secret") + #expect( + removal.value(forHTTPHeaderField: "X-Crabfleet-Ownership-Token") + == "desktop-ownership-token" + ) #expect(removal.httpBody == nil) } + @Test + func desktopRegistrationReturnsTheServerOwnershipToken() async throws { + let transport = DesktopRegistrationTransport { request in + let responseURL = try #require(request.url) + return ( + Data(#"{"ownershipToken":"server-ownership-token"}"#.utf8), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + #expect( + try await registration.register(identity: identity, port: 5_901) + == "server-ownership-token" + ) + } + @Test func desktopRegistrationRejectsRedirectedResponses() async throws { let redirectedURL = try #require(URL(string: "https://login.example.test/desktop-host")) @@ -987,15 +1094,17 @@ private actor SuspendedDesktopRegistration: DesktopHostRegistering { registrationContinuation != nil } - func register(identity: TailnetIdentity, port: UInt16) async throws { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String { events.append(.registerStarted) await withCheckedContinuation { continuation in registrationContinuation = continuation } events.append(.registerFinished) + return "registration-token" } - func unregister(identity: TailnetIdentity) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + #expect(ownershipToken == "registration-token") events.append(.unregisterStarted) } @@ -1012,7 +1121,7 @@ private enum DesktopRegistrationTestError: Error { private actor RecordingDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String) - case unregister(String) + case unregister(String, String) } private var registerFailures: [String: Int] @@ -1027,15 +1136,16 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { self.unregisterFailures = unregisterFailures } - func register(identity: TailnetIdentity, port: UInt16) async throws { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String { events.append(.register(identity.dnsName)) if consumeFailure(for: identity.dnsName, from: ®isterFailures) { throw DesktopRegistrationTestError.failed } + return "token:\(identity.dnsName)" } - func unregister(identity: TailnetIdentity) async throws { - events.append(.unregister(identity.dnsName)) + func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + events.append(.unregister(identity.dnsName, ownershipToken)) if consumeFailure(for: identity.dnsName, from: &unregisterFailures) { throw DesktopRegistrationTestError.failed } @@ -1051,6 +1161,30 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { } } +private actor SuspendedDesktopCleanupRegistration: DesktopHostRegistering { + private var unregistrationContinuation: CheckedContinuation? + + var hasStartedUnregistration: Bool { + unregistrationContinuation != nil + } + + func register(identity: TailnetIdentity, port: UInt16) async throws -> String { + "slow-cleanup-token" + } + + func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + #expect(ownershipToken == "slow-cleanup-token") + await withCheckedContinuation { continuation in + unregistrationContinuation = continuation + } + } + + func finishUnregistration() { + unregistrationContinuation?.resume() + unregistrationContinuation = nil + } +} + private final class RemoteInputRecorder: RemoteInputForwarding, @unchecked Sendable { private let lock = NSLock() private var releases = 0 From 51260eefb9ab588a67610c9479d054e21fb864e5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:17:12 +0200 Subject: [PATCH 073/242] fix(credentials): stage policy generation rotation --- ...credential_policy_registration_staging.sql | 70 ++++ src/worker/database.ts | 17 + ...ndbox-credential-policy-cleanup-service.ts | 149 ++++++++- .../sandbox-credential-policy-cleanup.ts | 13 + .../sandbox-credential-policy-repository.ts | 199 ++++++++--- .../sandbox-credential-policy-scanner.ts | 97 +++++- tests/runtime-adapter.test.ts | 29 +- .../sandbox-credential-policy-cleanup.test.ts | 3 +- ...ndbox-credential-policy-repository.test.ts | 310 +++++++++++++++++- 9 files changed, 816 insertions(+), 71 deletions(-) create mode 100644 migrations/0034_credential_policy_registration_staging.sql diff --git a/migrations/0034_credential_policy_registration_staging.sql b/migrations/0034_credential_policy_registration_staging.sql new file mode 100644 index 00000000..e0587917 --- /dev/null +++ b/migrations/0034_credential_policy_registration_staging.sql @@ -0,0 +1,70 @@ +CREATE TABLE IF NOT EXISTS interactive_session_credential_policy_registrations ( + session_id TEXT NOT NULL, + sandbox_id TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('registering', 'cleanup_pending')), + registration_generation TEXT NOT NULL, + registration_claim TEXT, + registration_claim_expires_at INTEGER, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + last_error TEXT, + cleanup_claim TEXT, + cleanup_claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, sandbox_id), + CHECK ( + ( + state = 'registering' + AND registration_claim IS NOT NULL + AND registration_claim_expires_at IS NOT NULL + ) + OR ( + state = 'cleanup_pending' + AND registration_claim IS NULL + AND registration_claim_expires_at IS NULL + ) + ) +); + +CREATE INDEX IF NOT EXISTS idx_credential_policy_registration_cleanup + ON interactive_session_credential_policy_registrations( + state, + cleanup_claim_expires_at, + last_attempt_at + ); + +CREATE INDEX IF NOT EXISTS idx_credential_policy_registration_expiry + ON interactive_session_credential_policy_registrations( + state, + registration_claim_expires_at, + updated_at + ); + +INSERT OR IGNORE INTO interactive_session_credential_policy_registrations ( + session_id, + sandbox_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at +) +SELECT + session_id, + sandbox_id, + 'registering', + MIN(registration_generation), + MIN(registration_claim), + MIN(registration_claim_expires_at), + MIN(created_at), + MAX(updated_at) +FROM interactive_session_credential_policies +WHERE state = 'registering' + AND registration_claim IS NOT NULL + AND registration_claim_expires_at IS NOT NULL +GROUP BY session_id, sandbox_id +HAVING count(DISTINCT registration_generation) = 1 + AND count(DISTINCT registration_claim) = 1 + AND count(DISTINCT registration_claim_expires_at) = 1; diff --git a/src/worker/database.ts b/src/worker/database.ts index 64eba7ba..83e5fb19 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -301,6 +301,22 @@ export type InteractiveSessionCredentialPolicyTable = { updated_at: number; }; +export type InteractiveSessionCredentialPolicyRegistrationTable = { + session_id: string; + sandbox_id: string; + state: "registering" | "cleanup_pending"; + registration_generation: string; + registration_claim: string | null; + registration_claim_expires_at: number | null; + attempt_count: Generated; + last_attempt_at: number | null; + last_error: string | null; + cleanup_claim: string | null; + cleanup_claim_expires_at: number | null; + created_at: number; + updated_at: number; +}; + export type CredentialPolicyReconcileStateTable = { id: number; last_rowid: number; @@ -381,6 +397,7 @@ export type Database = { interactive_session_events: InteractiveSessionEventTable; interactive_session_log_archives: InteractiveSessionLogArchiveTable; interactive_session_credential_policies: InteractiveSessionCredentialPolicyTable; + interactive_session_credential_policy_registrations: InteractiveSessionCredentialPolicyRegistrationTable; credential_policy_reconcile_state: CredentialPolicyReconcileStateTable; standalone_sandbox_provisions: StandaloneSandboxProvisionTable; repo_workflows: RepoWorkflowTable; diff --git a/src/worker/sandbox-credential-policy-cleanup-service.ts b/src/worker/sandbox-credential-policy-cleanup-service.ts index a2eeea41..94ddeb95 100644 --- a/src/worker/sandbox-credential-policy-cleanup-service.ts +++ b/src/worker/sandbox-credential-policy-cleanup-service.ts @@ -7,6 +7,7 @@ import { database, type Database, type InteractiveSessionCredentialPolicyTable, + type InteractiveSessionCredentialPolicyRegistrationTable, } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import { serviceUnavailable } from "./http.ts"; @@ -75,6 +76,98 @@ async function unregisterSandboxCredentialPolicyLookup( } } +async function reconcileStagedCredentialPolicyCleanup( + env: RuntimeEnv, + now: number, + sessionId?: string, +): Promise { + let query = database(env) + .selectFrom("interactive_session_credential_policy_registrations") + .selectAll() + .where("state", "=", "cleanup_pending") + .where((expression) => + expression.or([ + expression("cleanup_claim", "is", null), + expression("cleanup_claim_expires_at", "<", now), + ]), + ) + .orderBy(sql`COALESCE(last_attempt_at, created_at)`, "asc") + .limit(credentialPolicyCleanupLimit); + if (sessionId) query = query.where("session_id", "=", sessionId); + await mapWithConcurrency(await query.execute(), 3, async (registration) => { + await reconcileStagedCredentialPolicyRegistration(env, registration, now); + }); +} + +async function reconcileStagedCredentialPolicyRegistration( + env: RuntimeEnv, + registration: Selectable, + now: number, +): Promise { + const claim = crypto.randomUUID(); + const claimed = await database(env) + .updateTable("interactive_session_credential_policy_registrations") + .set({ + cleanup_claim: claim, + cleanup_claim_expires_at: now + credentialPolicyCleanupClaimMs, + attempt_count: sql`attempt_count + 1`, + last_attempt_at: now, + updated_at: now, + }) + .where("session_id", "=", registration.session_id) + .where("sandbox_id", "=", registration.sandbox_id) + .where("state", "=", "cleanup_pending") + .where("registration_generation", "=", registration.registration_generation) + .where((expression) => + expression.or([ + expression("cleanup_claim", "is", null), + expression("cleanup_claim_expires_at", "<", now), + ]), + ) + .executeTakeFirst(); + if ((claimed.numUpdatedRows ?? 0n) === 0n) return; + try { + await Promise.all( + sandboxLookupIds(env, registration.sandbox_id).map((lookupId) => + unregisterSandboxCredentialPolicyLookup( + env, + lookupId, + registration.registration_generation, + registration.session_id, + ), + ), + ); + } catch (error) { + await database(env) + .updateTable("interactive_session_credential_policy_registrations") + .set({ + last_error: clean(error instanceof Error ? error.message : String(error), 500), + cleanup_claim: null, + cleanup_claim_expires_at: null, + updated_at: Date.now(), + }) + .where("session_id", "=", registration.session_id) + .where("sandbox_id", "=", registration.sandbox_id) + .where("registration_generation", "=", registration.registration_generation) + .where("cleanup_claim", "=", claim) + .execute(); + return; + } + await database(env) + .deleteFrom("interactive_session_credential_policy_registrations") + .where("session_id", "=", registration.session_id) + .where("sandbox_id", "=", registration.sandbox_id) + .where("registration_generation", "=", registration.registration_generation) + .where("cleanup_claim", "=", claim) + .execute(); + await completeCredentialPolicyCleanupSession(env, registration.session_id, Date.now()); + await completeStandaloneSandboxProvisionCleanupSafely( + env, + registration.session_id, + registration.sandbox_id, + ); +} + async function normalizeCredentialPolicyCleanupGroups( env: RuntimeEnv, now: number, @@ -202,6 +295,9 @@ export async function reconcileSandboxCredentialPolicyCleanupBatch( console.error("credential policy cleanup scan failed", error); }, ); + await reconcileStagedCredentialPolicyCleanup(env, now, sessionId).catch((error) => { + console.error("staged credential policy cleanup failed", error); + }); await normalizeCredentialPolicyCleanupGroups(env, now, sessionId).catch((error) => { console.error("credential policy cleanup group normalization failed", error); }); @@ -225,6 +321,16 @@ export async function reconcileSandboxCredentialPolicyCleanupBatch( AND registration.registration_claim_expires_at > ${now} ) `) + .where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS registration + WHERE registration.session_id = interactive_session_credential_policies.session_id + AND registration.sandbox_id = interactive_session_credential_policies.sandbox_id + AND registration.state = 'registering' + AND registration.registration_claim_expires_at > ${now} + ) + `) .orderBy(sql`COALESCE(last_attempt_at, created_at)`, "asc") .orderBy("session_id", "asc") .orderBy("sandbox_id", "asc") @@ -246,6 +352,11 @@ export async function reconcileSandboxCredentialPolicyCleanupBatch( FROM interactive_session_credential_policies AS policy WHERE policy.session_id = interactive_sessions.id ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS registration + WHERE registration.session_id = interactive_sessions.id + ) `) .orderBy("stopped_at", "asc") .orderBy("id", "asc") @@ -298,6 +409,14 @@ async function reconcileCredentialPolicyCleanup( AND registration.registration_claim IS NOT NULL AND registration.registration_claim_expires_at > ${now} ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS registration + WHERE registration.session_id = interactive_session_credential_policies.session_id + AND registration.sandbox_id = interactive_session_credential_policies.sandbox_id + AND registration.state = 'registering' + AND registration.registration_claim_expires_at > ${now} + ) `.execute(database(env)); if ((claimed.numAffectedRows ?? 0n) === 0n) return; try { @@ -411,6 +530,12 @@ async function completeStandaloneSandboxProvisionCleanup( WHERE policy.session_id = ${provisionId} AND policy.sandbox_id = ${sandboxId} ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS registration + WHERE registration.session_id = ${provisionId} + AND registration.sandbox_id = ${sandboxId} + ) `) .execute(); } @@ -421,12 +546,19 @@ async function completeCredentialPolicyCleanupSession( now: number, ): Promise { const db = database(env); - const remaining = await db - .selectFrom("interactive_session_credential_policies") - .select(({ fn }) => fn.countAll().as("count")) - .where("session_id", "=", sessionId) - .executeTakeFirst(); - if (Number(remaining?.count ?? 0) > 0) return; + const remaining = await sql<{ count: number }>` + SELECT + ( + SELECT count(*) + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + ) + ( + SELECT count(*) + FROM interactive_session_credential_policy_registrations + WHERE session_id = ${sessionId} + ) AS count + `.execute(db); + if (Number(remaining.rows[0]?.count ?? 0) > 0) return; const session = await db .selectFrom("interactive_sessions") .select([ @@ -479,6 +611,11 @@ async function completeCredentialPolicyCleanupSession( FROM interactive_session_credential_policies WHERE session_id = ${sessionId} ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations + WHERE session_id = ${sessionId} + ) `) .executeTakeFirst(); if ((updated.numUpdatedRows ?? 0n) === 0n) return; diff --git a/src/worker/sandbox-credential-policy-cleanup.ts b/src/worker/sandbox-credential-policy-cleanup.ts index 31823781..cd884cac 100644 --- a/src/worker/sandbox-credential-policy-cleanup.ts +++ b/src/worker/sandbox-credential-policy-cleanup.ts @@ -208,6 +208,19 @@ export async function stageTerminalCredentialPolicyCleanup( .where( sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), ), + db + .updateTable("interactive_session_credential_policy_registrations") + .set({ + state: "cleanup_pending", + registration_claim: null, + registration_claim_expires_at: null, + updated_at: stageRevision, + }) + .where("session_id", "=", session.id) + .where("sandbox_id", "=", sandboxId) + .where( + sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), + ), ]); await executeBatch(env, [sessionTransition, ...policyTransitions]); const staged = await db diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index a14afffa..1cc0c8c1 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -439,12 +439,11 @@ export function sandboxCredentialPolicyRegistrationQueries( now: number, ownershipFence: SandboxCredentialPolicyOwnershipFence, ): CompilableQuery[] { - return registration.lookupIds.map( - (lookupId) => sql` - INSERT INTO interactive_session_credential_policies ( + return [ + sql` + INSERT INTO interactive_session_credential_policy_registrations ( session_id, sandbox_id, - lookup_id, state, registration_generation, registration_claim, @@ -460,7 +459,6 @@ export function sandboxCredentialPolicyRegistrationQueries( SELECT ${sessionId}, ${sandboxId}, - ${lookupId}, 'registering', ${registration.generation}, ${registration.claim}, @@ -473,7 +471,14 @@ export function sandboxCredentialPolicyRegistrationQueries( ${now}, ${now} WHERE ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} - ON CONFLICT(session_id, sandbox_id, lookup_id) DO UPDATE SET + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'cleanup_pending' + ) + ON CONFLICT(session_id, sandbox_id) DO UPDATE SET state = 'registering', registration_generation = excluded.registration_generation, registration_claim = excluded.registration_claim, @@ -482,14 +487,21 @@ export function sandboxCredentialPolicyRegistrationQueries( cleanup_claim = NULL, cleanup_claim_expires_at = NULL, updated_at = excluded.updated_at - WHERE interactive_session_credential_policies.state != 'cleanup_pending' + WHERE interactive_session_credential_policy_registrations.state != 'cleanup_pending' AND ( - interactive_session_credential_policies.registration_claim IS NULL - OR interactive_session_credential_policies.registration_claim_expires_at <= ${now} + interactive_session_credential_policy_registrations.registration_claim IS NULL + OR interactive_session_credential_policy_registrations.registration_claim_expires_at <= ${now} ) AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'cleanup_pending' + ) `, - ); + ]; } export async function beginSandboxCredentialPolicyRegistration( @@ -519,9 +531,8 @@ export async function beginSandboxCredentialPolicyRegistration( ), ); const claimed = await db - .selectFrom("interactive_session_credential_policies") + .selectFrom("interactive_session_credential_policy_registrations") .select([ - "lookup_id", "state", "registration_generation", "registration_claim", @@ -529,17 +540,12 @@ export async function beginSandboxCredentialPolicyRegistration( ]) .where("session_id", "=", sessionId) .where("sandbox_id", "=", sandboxId) - .where("lookup_id", "in", lookupIds) - .execute(); + .executeTakeFirst(); if ( - claimed.length !== lookupIds.length || - claimed.some( - (row) => - row.state !== "registering" || - row.registration_generation !== registration.generation || - row.registration_claim !== registration.claim || - row.registration_claim_expires_at !== registrationExpiresAt, - ) + claimed?.state !== "registering" || + claimed.registration_generation !== registration.generation || + claimed.registration_claim !== registration.claim || + claimed.registration_claim_expires_at !== registrationExpiresAt ) { await abandonSandboxCredentialPolicyRegistration( env, @@ -563,22 +569,19 @@ export async function renewSandboxCredentialPolicyRegistration( const now = Date.now(); const registrationExpiresAt = now + credentialPolicyRegistrationClaimMs; const renewed = await database(env) - .updateTable("interactive_session_credential_policies") + .updateTable("interactive_session_credential_policy_registrations") .set({ registration_claim_expires_at: registrationExpiresAt, updated_at: now, }) .where("session_id", "=", sessionId) .where("sandbox_id", "=", sandboxId) - .where("lookup_id", "in", registration.lookupIds) .where("state", "=", "registering") .where("registration_generation", "=", registration.generation) .where("registration_claim", "=", registration.claim) .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) .executeTakeFirst(); - return Number(renewed.numUpdatedRows ?? 0n) === registration.lookupIds.length - ? registrationExpiresAt - : null; + return Number(renewed.numUpdatedRows ?? 0n) === 1 ? registrationExpiresAt : null; } export async function finishSandboxCredentialPolicyRegistration( @@ -590,22 +593,17 @@ export async function finishSandboxCredentialPolicyRegistration( ): Promise { const now = Date.now(); const db = database(env); - await db - .updateTable("interactive_session_credential_policies") - .set({ - state: "active", - registration_claim: null, - registration_claim_expires_at: null, - updated_at: now, - }) - .where("session_id", "=", sessionId) - .where("sandbox_id", "=", sandboxId) - .where("lookup_id", "in", registration.lookupIds) - .where("state", "=", "registering") - .where("registration_generation", "=", registration.generation) - .where("registration_claim", "=", registration.claim) - .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) - .execute(); + await executeBatch( + env, + sandboxCredentialPolicyPromotionQueries( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + now, + ), + ); const active = await db .selectFrom("interactive_session_credential_policies") .select(["lookup_id", "state", "registration_generation", "registration_claim"]) @@ -613,7 +611,14 @@ export async function finishSandboxCredentialPolicyRegistration( .where("sandbox_id", "=", sandboxId) .where("lookup_id", "in", registration.lookupIds) .execute(); + const staged = await db + .selectFrom("interactive_session_credential_policy_registrations") + .select("registration_generation") + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .executeTakeFirst(); return ( + !staged && active.length === registration.lookupIds.length && active.every( (row) => @@ -633,13 +638,9 @@ export async function abandonSandboxCredentialPolicyRegistration( ): Promise { const now = Date.now(); await database(env) - .updateTable("interactive_session_credential_policies") + .updateTable("interactive_session_credential_policy_registrations") .set({ - state: sql<"registering" | "cleanup_pending">`CASE - WHEN ${sandboxCredentialPolicyCleanupAuthorizedCondition(sessionId, sandboxId, now)} - THEN 'cleanup_pending' - ELSE 'registering' - END`, + state: "cleanup_pending", registration_claim: null, registration_claim_expires_at: null, last_error: reason, @@ -652,6 +653,106 @@ export async function abandonSandboxCredentialPolicyRegistration( .execute(); } +export function sandboxCredentialPolicyPromotionQueries( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + now: number, +): CompilableQuery[] { + const promotionAuthorized = sql` + EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'registering' + AND registration_generation = ${registration.generation} + AND registration_claim = ${registration.claim} + ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'cleanup_pending' + ) + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + `; + const promotions = registration.lookupIds.map( + (lookupId) => sql` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + attempt_count, + last_attempt_at, + last_error, + cleanup_claim, + cleanup_claim_expires_at, + created_at, + updated_at + ) + SELECT + ${sessionId}, + ${sandboxId}, + ${lookupId}, + 'active', + ${registration.generation}, + NULL, + NULL, + 0, + NULL, + NULL, + NULL, + NULL, + ${now}, + ${now} + WHERE ${promotionAuthorized} + ON CONFLICT(session_id, sandbox_id, lookup_id) DO UPDATE SET + state = 'active', + registration_generation = excluded.registration_generation, + registration_claim = NULL, + registration_claim_expires_at = NULL, + last_error = NULL, + cleanup_claim = NULL, + cleanup_claim_expires_at = NULL, + updated_at = excluded.updated_at + WHERE interactive_session_credential_policies.state != 'cleanup_pending' + AND ${promotionAuthorized} + `, + ); + const promotionComplete = sql` + ( + SELECT count(DISTINCT lookup_id) + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND lookup_id IN (${sql.join(registration.lookupIds)}) + AND state = 'active' + AND registration_generation = ${registration.generation} + AND registration_claim IS NULL + ) = ${registration.lookupIds.length} + `; + return [ + ...promotions, + sql` + DELETE FROM interactive_session_credential_policy_registrations + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'registering' + AND registration_generation = ${registration.generation} + AND registration_claim = ${registration.claim} + AND ${promotionComplete} + `, + ]; +} + export async function standaloneSandboxPolicyExpiresAt( env: RuntimeEnv, sessionId: string, diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index 63956fda..587e4a08 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -9,11 +9,15 @@ import { database, executeBatch, type Database } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import type { InteractiveSessionStatus } from "./models.ts"; import { + abandonSandboxCredentialPolicyRegistration, + finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyCleanupAuthorizedCondition, + sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; import { sandboxLeaseInfo, sandboxLeasePrefix } from "./sandbox-lease.ts"; +import type { SandboxCredentialPolicyRegistration } from "./session-control-policy.ts"; const credentialPolicyScanLimit = 32; export const credentialPolicyProvisioningStaleMs = 15 * 60_000; @@ -45,6 +49,27 @@ export type CredentialPolicyScanRow = { standalone_updated_at: number | null; }; +type CredentialPolicyOwnershipRow = Pick< + CredentialPolicyScanRow, + | "session_id" + | "sandbox_id" + | "matched_session_id" + | "session_adapter" + | "session_lease_id" + | "session_sandbox_refresh_sandbox_id" + | "session_sandbox_refresh_claim" + | "session_sandbox_refresh_claim_expires_at" + | "matched_standalone_id" + | "standalone_state" + | "standalone_claim" + | "standalone_claim_expires_at" +>; + +type StagedCredentialPolicyScanRow = CredentialPolicyOwnershipRow & { + registration_generation: string; + registration_claim: string; +}; + export type SandboxCredentialPolicyExists = ( env: RuntimeEnv, sandboxId: string, @@ -58,6 +83,7 @@ export async function scanCredentialPolicyCleanupPage( sessionId?: string, ): Promise { const db = database(env); + await scanStagedCredentialPolicyRegistrations(env, db, now, policyExists, sessionId); const state = sessionId ? null : await db @@ -250,6 +276,75 @@ export async function scanCredentialPolicyCleanupPage( } } +async function scanStagedCredentialPolicyRegistrations( + env: RuntimeEnv, + db: Kysely, + now: number, + policyExists: SandboxCredentialPolicyExists, + sessionId?: string, +): Promise { + const sessionFilter = sessionId ? sql`AND registration.session_id = ${sessionId}` : sql``; + const result = await sql` + SELECT + registration.session_id, + registration.sandbox_id, + registration.registration_generation, + registration.registration_claim, + session.id AS matched_session_id, + session.adapter AS session_adapter, + session.lease_id AS session_lease_id, + session.sandbox_refresh_sandbox_id AS session_sandbox_refresh_sandbox_id, + session.sandbox_refresh_claim AS session_sandbox_refresh_claim, + session.sandbox_refresh_claim_expires_at AS session_sandbox_refresh_claim_expires_at, + standalone.id AS matched_standalone_id, + standalone.state AS standalone_state, + standalone.ownership_claim AS standalone_claim, + standalone.ownership_claim_expires_at AS standalone_claim_expires_at + FROM interactive_session_credential_policy_registrations AS registration + LEFT JOIN interactive_sessions AS session ON session.id = registration.session_id + LEFT JOIN standalone_sandbox_provisions AS standalone + ON standalone.id = registration.session_id + AND standalone.sandbox_id = registration.sandbox_id + WHERE registration.state = 'registering' + AND registration.registration_claim_expires_at <= ${now} + ${sessionFilter} + ORDER BY registration.updated_at ASC + LIMIT ${credentialPolicyScanLimit} + `.execute(db); + for (const row of result.rows) { + const registration: SandboxCredentialPolicyRegistration = { + generation: row.registration_generation, + claim: row.registration_claim, + lookupIds: sandboxLookupIds(env, row.sandbox_id), + }; + try { + const ownershipFence = credentialPolicyScanOwnershipFence(row, now); + if ( + ownershipFence && + (await policyExists(env, row.sandbox_id, row.registration_generation)) && + (await finishSandboxCredentialPolicyRegistration( + env, + row.session_id, + row.sandbox_id, + registration, + ownershipFence, + )) + ) { + continue; + } + await abandonSandboxCredentialPolicyRegistration( + env, + row.session_id, + row.sandbox_id, + registration, + "sandbox credential policy registration did not complete", + ); + } catch (error) { + console.error("staged sandbox credential policy recovery failed", error); + } + } +} + async function readCredentialPolicyScanPage( db: Kysely, cursor: number, @@ -341,7 +436,7 @@ async function repairActiveSandboxCredentialPolicyRegistration( } export function credentialPolicyScanOwnershipFence( - row: CredentialPolicyScanRow, + row: CredentialPolicyOwnershipRow, now: number, ): SandboxCredentialPolicyOwnershipFence | null { if ( diff --git a/tests/runtime-adapter.test.ts b/tests/runtime-adapter.test.ts index 5d854924..e59f1ed4 100644 --- a/tests/runtime-adapter.test.ts +++ b/tests/runtime-adapter.test.ts @@ -1037,6 +1037,10 @@ test("sandbox credential cleanup is durably staged and retried", async () => { new URL("../migrations/0022_credential_policy_cleanup.sql", import.meta.url), "utf8", ); + const registrationStagingMigration = await readFile( + new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), + "utf8", + ); const scanStart = scannerSource.indexOf("type CredentialPolicyScanRow"); const scanSource = scannerSource.slice(scanStart); const batchStart = cleanupServiceSource.indexOf( @@ -1207,14 +1211,20 @@ test("sandbox credential cleanup is durably staged and retried", async () => { registerSource.indexOf('stub.fetch("https://crabfleet.internal/api/session-control/register"') < registerSource.indexOf("finishSandboxCredentialPolicyRegistration"), ); - assert.doesNotMatch(finishSource, /INSERT INTO|insertInto/); - assert.match(finishSource, /state: "active"/); - assert.match(finishSource, /where\(sandboxCredentialPolicyOwnerCondition/); - assert.doesNotMatch(finishSource, /cleanup_pending/); + assert.match(finishSource, /executeBatch/); + assert.match(finishSource, /sandboxCredentialPolicyPromotionQueries/); + assert.match(finishSource, /row\.state === "active"/); + assert.match(registrationLifecycleSource, /INSERT INTO interactive_session_credential_policies/); + assert.match( + registrationLifecycleSource, + /DELETE FROM interactive_session_credential_policy_registrations/, + ); assert.match(registerSource, /abandonSandboxCredentialPolicyRegistration/); - assert.match(abandonSource, /sandboxCredentialPolicyCleanupAuthorizedCondition/); - assert.match(abandonSource, /THEN 'cleanup_pending'/); - assert.match(abandonSource, /ELSE 'registering'/); + assert.match( + abandonSource, + /updateTable\("interactive_session_credential_policy_registrations"\)/, + ); + assert.match(abandonSource, /state: "cleanup_pending"/); assert.ok( scanDecisionSource.indexOf("sandboxExpected") < scanDecisionSource.indexOf("if (registrationAbandoned) return true"), @@ -1229,6 +1239,11 @@ test("sandbox credential cleanup is durably staged and retried", async () => { assert.match(migration, /state IN \('registering', 'active', 'cleanup_pending'\)/); assert.match(migration, /registration_generation TEXT NOT NULL/); assert.match(migration, /registration_claim_expires_at INTEGER/); + assert.match( + registrationStagingMigration, + /CREATE TABLE IF NOT EXISTS interactive_session_credential_policy_registrations/, + ); + assert.match(registrationStagingMigration, /state IN \('registering', 'cleanup_pending'\)/); assert.match(migration, /CREATE TABLE IF NOT EXISTS credential_policy_reconcile_state/); assert.match(migration, /scan_max_rowid INTEGER NOT NULL/); assert.match(migration, /group_max_session_id TEXT NOT NULL/); diff --git a/tests/sandbox-credential-policy-cleanup.test.ts b/tests/sandbox-credential-policy-cleanup.test.ts index 2f6cd82c..5089da29 100644 --- a/tests/sandbox-credential-policy-cleanup.test.ts +++ b/tests/sandbox-credential-policy-cleanup.test.ts @@ -195,7 +195,7 @@ test("terminal cleanup atomically stages the session and credential-policy refs" true, ); - assert.equal(batch.length, 3); + assert.equal(batch.length, 4); const sql = batch.map((statement) => statement.sql).join("\n"); const parameters = batch.flatMap((statement) => statement.parameters); assert.match(sql, /update "interactive_sessions"/i); @@ -205,6 +205,7 @@ test("terminal cleanup atomically stages the session and credential-policy refs" assert.match(sql, /"lease_id" = \?/i); assert.match(sql, /on conflict\s*\(session_id, sandbox_id, lookup_id\)/i); assert.match(sql, /update "interactive_session_credential_policies"/i); + assert.match(sql, /update "interactive_session_credential_policy_registrations"/i); assert.match(sql, /not exists/i); assert.ok(parameters.includes("failed")); assert.ok(parameters.includes("generation:test-1")); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 32a6d97e..508bac38 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -1,10 +1,13 @@ import assert from "node:assert/strict"; +import { DatabaseSync } from "node:sqlite"; import test from "node:test"; import { activeSandboxCredentialPolicyGeneration, + abandonSandboxCredentialPolicyRegistration, beginSandboxCredentialPolicyRegistration, currentSandboxCredentialPolicyGeneration, + finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyRegistrationQueries, sandboxLookupIds, @@ -21,6 +24,19 @@ type PreparedStatement = { run(): Promise; }; +type SqliteStatement = { + all(...parameters: unknown[]): Record[]; + run(...parameters: unknown[]): { changes: number | bigint; lastInsertRowid: number | bigint }; +}; + +type BoundStatement = PreparedStatement & { + execute(): { + results: Record[]; + success: true; + meta: { changes: number; last_row_id?: number }; + }; +}; + function runtimeEnv( handler: ( sql: string, @@ -69,6 +85,168 @@ function runtimeEnv( } as RuntimeEnv; } +function credentialPolicyDatabase(): DatabaseSync { + const db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE interactive_sessions ( + id TEXT PRIMARY KEY, + adapter TEXT, + status TEXT NOT NULL, + credential_cleanup_terminal_status TEXT, + agent_token_hash TEXT, + lease_id TEXT, + sandbox_refresh_sandbox_id TEXT, + sandbox_refresh_claim TEXT, + sandbox_refresh_claim_expires_at INTEGER + ); + CREATE TABLE interactive_session_credential_policies ( + session_id TEXT NOT NULL, + sandbox_id TEXT NOT NULL, + lookup_id TEXT NOT NULL, + state TEXT NOT NULL, + registration_generation TEXT NOT NULL, + registration_claim TEXT, + registration_claim_expires_at INTEGER, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + last_error TEXT, + cleanup_claim TEXT, + cleanup_claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, sandbox_id, lookup_id) + ); + CREATE TABLE interactive_session_credential_policy_registrations ( + session_id TEXT NOT NULL, + sandbox_id TEXT NOT NULL, + state TEXT NOT NULL, + registration_generation TEXT NOT NULL, + registration_claim TEXT, + registration_claim_expires_at INTEGER, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + last_error TEXT, + cleanup_claim TEXT, + cleanup_claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, sandbox_id) + ); + INSERT INTO interactive_sessions ( + id, + adapter, + status, + credential_cleanup_terminal_status, + agent_token_hash, + lease_id + ) VALUES ( + 'IS-42', + NULL, + 'ready', + NULL, + 'agent-token', + 'sandbox:sandbox-1:terminal-1:autostart-v4' + ); + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at + ) VALUES + ('IS-42', 'sandbox-1', 'sandbox-1', 'active', 'generation:existing', NULL, NULL, 1, 1), + ('IS-42', 'sandbox-1', 'do-1', 'active', 'generation:existing', NULL, NULL, 1, 1); + `); + return db; +} + +function sqliteRuntimeEnv( + sqlite: DatabaseSync, + options: { interruptAfterStatement?: number } = {}, +): RuntimeEnv { + function execute(sql: string, parameters: unknown[]) { + const statement = sqlite.prepare(sql) as unknown as SqliteStatement; + if (/^\s*(?:select|pragma|with)\b|\breturning\b/i.test(sql)) { + const results = statement.all(...parameters).map((row) => ({ ...row })); + const changes = Number(sqlite.prepare("SELECT changes() AS changes").get()?.changes ?? 0); + return { results, success: true as const, meta: { changes } }; + } + const result = statement.run(...parameters); + return { + results: [], + success: true as const, + meta: { + changes: Number(result.changes), + last_row_id: Number(result.lastInsertRowid), + }, + }; + } + return { + DB: { + prepare(sql: string) { + return { + bind(...parameters: unknown[]) { + const bound = { + sql, + parameters, + execute: () => execute(sql, parameters), + async all() { + return bound.execute(); + }, + async run() { + return bound.execute(); + }, + }; + return bound; + }, + }; + }, + async batch(statements: D1PreparedStatement[]) { + sqlite.exec("BEGIN IMMEDIATE"); + try { + const results = []; + for (const [index, statement] of statements.entries()) { + results.push((statement as unknown as BoundStatement).execute()); + if (options.interruptAfterStatement === index + 1) { + throw new Error("simulated batch interruption"); + } + } + sqlite.exec("COMMIT"); + return results; + } catch (error) { + sqlite.exec("ROLLBACK"); + throw error; + } + }, + } as unknown as D1Database, + SANDBOX: { + idFromName() { + return { toString: () => "do-1" }; + }, + } as unknown as DurableObjectNamespace, + } as RuntimeEnv; +} + +const ownershipFence: SandboxCredentialPolicyOwnershipFence = { + leaseId: "sandbox:sandbox-1:terminal-1:autostart-v4", + sandboxId: "sandbox-1", +}; + +function activeCredentialPolicyRows(db: DatabaseSync): Record[] { + return db + .prepare(` + SELECT lookup_id, state, registration_generation, registration_claim + FROM interactive_session_credential_policies + ORDER BY lookup_id + `) + .all() + .map((row) => ({ ...row })); +} + const registration: SandboxCredentialPolicyRegistration = { generation: "generation:test-1", claim: "registration-1", @@ -124,6 +302,7 @@ test("credential-policy registration SQL proves every supported ownership fence" sandboxId: "sandbox-1", }); assert.match(current.sql, /from interactive_sessions/i); + assert.match(current.sql, /interactive_session_credential_policy_registrations/i); assert.match(current.sql, /adapter is null|adapter !=/i); assert.match(current.sql, /agent_token_hash is not null/i); assert.match(current.sql, /lease_id =/i); @@ -163,11 +342,14 @@ test("credential-policy rotation always claims a fresh generation", async () => let statements: PreparedStatement[] = []; const env = runtimeEnv( (sql, _parameters, kind) => { - if (kind === "all" && /select .*lookup_id/i.test(sql)) { + if ( + kind === "all" && + /select .*state/i.test(sql) && + /interactive_session_credential_policy_registrations/i.test(sql) + ) { return { results: [ { - lookup_id: "sandbox-1", state: "registering", registration_generation: generation, registration_claim: claim, @@ -176,11 +358,6 @@ test("credential-policy rotation always claims a fresh generation", async () => ], }; } - if (kind === "all" && /select .*registration_generation/i.test(sql)) { - return { - results: [{ registration_generation: "generation:existing" }], - }; - } return {}; }, (prepared) => { @@ -217,6 +394,125 @@ test("credential-policy rotation always claims a fresh generation", async () => assert.equal(rotated.generation, generation); }); +test("partial credential-policy rotation failure preserves the prior active generation", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => row.registration_generation), + ["generation:existing", "generation:existing"], + ); + + await abandonSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + "simulated Durable Object registration failure", + ); + + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => row.registration_generation), + ["generation:existing", "generation:existing"], + ); + assert.deepEqual( + { + ...sqlite + .prepare(` + SELECT state, registration_generation, registration_claim, last_error + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(), + }, + { + state: "cleanup_pending", + registration_generation: staged.generation, + registration_claim: null, + last_error: "simulated Durable Object registration failure", + }, + ); +}); + +test("completed credential-policy rotation atomically promotes every active lookup", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + + assert.equal( + await finishSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + true, + ); + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => ({ + generation: row.registration_generation, + state: row.state, + })), + [ + { generation: staged.generation, state: "active" }, + { generation: staged.generation, state: "active" }, + ], + ); + assert.equal( + sqlite + .prepare("SELECT count(*) AS count FROM interactive_session_credential_policy_registrations") + .get()?.count, + 0, + ); +}); + +test("interrupted credential-policy promotion rolls back every active lookup", async () => { + const sqlite = credentialPolicyDatabase(); + const staged = await beginSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite), + "IS-42", + "sandbox-1", + ownershipFence, + ); + + await assert.rejects( + finishSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite, { interruptAfterStatement: 1 }), + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + /simulated batch interruption/, + ); + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => row.registration_generation), + ["generation:existing", "generation:existing"], + ); + assert.equal( + sqlite + .prepare(` + SELECT registration_generation + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get()?.registration_generation, + staged.generation, + ); +}); + test("active credential-policy generation requires every exact lookup row", async () => { const rows = [ { From 82ad6d7129c1109b132de96c6e9511daad6ba63c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:19:44 +0200 Subject: [PATCH 074/242] docs(actions): document framed runner protocol --- CHANGELOG.md | 1 + README.md | 24 +++++- docs/api.md | 40 ++++++++- docs/architecture.md | 4 +- docs/github-actions-sessions.md | 145 +++++++++++++++++++++++++++----- docs/runs.md | 5 +- docs/spec.md | 7 +- 7 files changed, 196 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e759c4bb..48c0ae2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, read-state-fenced GitHub Actions registration, revision-fenced lifecycle updates and grant revocation, monotonic Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. +- Change the GitHub Actions runner PTY contract to correlated `CFR1` binary input and acknowledgement frames, emit success only after the runner accepts the input into its PTY, preserve unframed runner output, and reject legacy raw-input clients. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. - Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host. diff --git a/README.md b/README.md index d4c228b9..1cc362ea 100644 --- a/README.md +++ b/README.md @@ -94,14 +94,32 @@ Content-Type: application/json {"workKey":"openclaw/crabfleet:pr:42","workKind":"pr_repair","repo":"openclaw/crabfleet","branch":"fix/pr-42","owner":"operator@example.test","sourceUrl":"https://github.com/openclaw/crabfleet/pull/42","runUrl":"https://github.com/openclaw/crabfleet/actions/runs/123","purpose":"repair PR 42","summary":"starting repair"} ``` -The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and works directly with Node's global `WebSocket`: +The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and can be opened with Node's global `WebSocket` without custom headers, but it is not a raw duplex byte stream. The abbreviated runner handler is: ```js const terminal = new WebSocket(runnerPtyUrl); -terminal.onmessage = (event) => process.stdout.write(Buffer.from(event.data)); -process.stdin.on("data", (chunk) => terminal.send(chunk)); +terminal.binaryType = "arraybuffer"; + +pty.onData((output) => terminal.send(output)); // Runner output stays raw. +terminal.onmessage = ({ data }) => { + const input = decodeCfr1Input(data); + if (!input) return; + try { + pty.write(new TextDecoder().decode(input.payload)); + terminal.send(encodeCfr1Ack(input.inputId, true)); + } catch { + terminal.send(encodeCfr1Ack(input.inputId, false)); + } +}; ``` +Crabfleet sends viewer input in correlated binary `CFR1` frames. The runner must +return the matching binary acknowledgement only after its PTY accepts the +input. Runner terminal output remains unframed and raw. Legacy clients that +expect raw viewer input are incompatible; the complete encoder, decoder, and +Node runner example are in +[`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). + The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. The full registration, relay, resumption, steering, heartbeat, completion, diff --git a/docs/api.md b/docs/api.md index f35f50d2..acbdea93 100644 --- a/docs/api.md +++ b/docs/api.md @@ -598,11 +598,47 @@ Response: } ``` -Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` is directly usable with Node's global `WebSocket`; no custom headers are required. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. +Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers, but the runner must implement the framed input and acknowledgement protocol below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. ### GET /api/agent/interactive-sessions/:id/runner-pty -WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses the scoped `agentToken` query parameter embedded in `runnerPtyUrl`. The runner sends raw terminal output bytes and receives raw viewer input bytes. One runner is current; a reconnect replaces the previous runner while browser viewers remain attached. +WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses the scoped `agentToken` query parameter embedded in `runnerPtyUrl`. One runner is current; a reconnect replaces the previous runner while browser viewers remain attached. + +Runner output remains unframed: text and binary WebSocket messages are fanned +out as raw terminal output. Viewer input and relay control traffic use binary +`CFR1` frames so terminal text cannot be mistaken for an acknowledgement. +Legacy runners that expect raw viewer input are incompatible and their +unframed input is rejected. + +Each `CFR1` frame occupies one binary WebSocket message and starts with: + +| Offset | Size | Value | +| ------ | -------- | ---------------------------------------- | +| 0 | 4 | ASCII `CFR1` (`43 46 52 31` hexadecimal) | +| 4 | 1 | frame type | +| 5 | 1 | input ID byte length | +| 6 | variable | input ID, then type-specific payload | + +Input IDs are nonempty ASCII `[A-Za-z0-9_-]` values of at most 80 bytes. + +| Type | Direction | Payload | +| ---------------------- | ---------------- | ----------------------------------------------------------------------------- | +| `0x01` input | relay to runner | raw terminal input bytes | +| `0x02` acknowledgement | runner to relay | one byte: `1` accepted or `0` rejected, followed by optional UTF-8 error text | +| `0x03` lifecycle event | relay to viewers | empty input ID and one event-code byte | + +Lifecycle event codes are `0x01` runner connected, `0x02` runner disconnected, +and `0x03` runner waiting. + +The runner must copy the input frame's ID into its acknowledgement. It must send +an accepted acknowledgement only after its PTY write API has accepted the +payload. Queueing the frame in `WebSocket.send()` is not acceptance. Crabfleet +generates a rejected acknowledgement only when no current runner is available +to receive the input frame or the relay send fails. Stale or mismatched +acknowledgement IDs do not complete another pending input. + +See [GitHub Actions Sessions](/github-actions-sessions/#runner-pty) for a +complete Node runner integration. ### POST /api/agent/interactive-sessions/:id/work-state diff --git a/docs/architecture.md b/docs/architecture.md index 0a35ed29..fd383434 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ D1 is canonical for product metadata: ### Durable Objects - `Sandbox` runs first-party Cloudflare Sandbox workspaces. -- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. +- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Runner output remains raw; viewer input, runner acknowledgements, and lifecycle events use correlated binary `CFR1` frames so control traffic cannot consume terminal text. There is no `BoardDO` or `RunDO`. General Board/Fleet state is D1 plus REST polling. @@ -137,7 +137,7 @@ Interactive sessions are the live execution plane. Supported paths: - **Built-in Sandbox:** Worker provisions a Cloudflare Sandbox, prepares the repo, starts a Codex-capable shell, and proxies PTY traffic. - **Versioned runtime adapter:** Worker durably registers a tenant-namespaced workspace ID, creates and reconciles the provider workspace, proxies PTY access, mints transient desktop links, and confirms provider release before terminal state. -- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, and receives browser steering. +- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, receives correlated `CFR1` browser input, writes it to its PTY, and acknowledges the matching ID before Crabfleet reports input acceptance. Sessions can carry a stable tenant owner, parent/root lineage, purpose, summary, named grants, public share state, delegated control, multiplayer mode, archive metadata, and runtime-specific capability state. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index bc095a93..a6be4d98 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -41,7 +41,8 @@ flowchart LR E[GitHub Actions runner] -->|outbound WebSocket| D F[Browser Ghostty viewer] -->|terminal hub| D F -->|input| D - D -->|raw input bytes| E + D -->|CFR1 input frame| E + E -->|CFR1 acknowledgement| D E -->|Codex turn/steer| G[Codex app-server] E -->|heartbeat and work state| B B --> H[(R2 event archives)] @@ -261,35 +262,139 @@ returns only the sanitized event. ## Runner PTY -The Action connects outbound to the returned `runnerPtyUrl`: +The Action connects outbound to the returned `runnerPtyUrl`. Node's global +`WebSocket` can open the URL without custom headers, but runner input is a +framed protocol rather than raw WebSocket bytes. + +Runner output remains unframed and raw. Viewer input arrives in a binary `CFR1` +frame carrying a correlation ID. The runner returns a binary acknowledgement +with the same ID only after its PTY accepts the input write. Legacy runners that +expect raw viewer input are incompatible. + +Complete Node runner integration: + +```sh +npm install @lydell/node-pty +``` ```js +import { spawn } from "@lydell/node-pty"; + +const runnerPtyUrl = process.env.CRABFLEET_RUNNER_PTY_URL; +if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); + +const magic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); // CFR1 +const decoder = new TextDecoder(); +const encoder = new TextEncoder(); const terminal = new WebSocket(runnerPtyUrl); terminal.binaryType = "arraybuffer"; -terminal.onmessage = (event) => { - // Browser input bytes for the active runner. -}; +await new Promise((resolve, reject) => { + terminal.addEventListener("open", resolve, { once: true }); + terminal.addEventListener("error", reject, { once: true }); +}); + +const pty = spawn(process.env.SHELL || "/bin/bash", [], { + cwd: process.cwd(), + env: process.env, +}); + +pty.onData((output) => { + terminal.send(output); // Terminal output stays raw and unframed. +}); + +terminal.addEventListener("message", (event) => { + acceptInput(event.data); +}); + +function acceptInput(data) { + const input = decodeInput(data); + if (!input) return; + try { + // A successful node-pty write is this adapter's PTY acceptance point. + pty.write(decoder.decode(input.payload)); + terminal.send(encodeAck(input.inputId, true)); + } catch { + terminal.send(encodeAck(input.inputId, false)); + } +} + +function decodeInput(data) { + if (!(data instanceof ArrayBuffer)) return null; + const frame = new Uint8Array(data); + if (frame.byteLength < 7 || !magic.every((value, index) => frame[index] === value)) { + return null; + } + if (frame[4] !== 0x01) return null; + const inputIdBytes = frame[5]; + if (!inputIdBytes || inputIdBytes > 80 || 6 + inputIdBytes > frame.byteLength) { + return null; + } + const inputId = decoder.decode(frame.subarray(6, 6 + inputIdBytes)); + if (!/^[A-Za-z0-9_-]+$/.test(inputId)) return null; + return { + inputId, + payload: frame.slice(6 + inputIdBytes), + }; +} -function writeTerminal(bytes) { - terminal.send(bytes); +function encodeAck(inputId, accepted) { + const inputIdBytes = encoder.encode(inputId); + const frame = new Uint8Array(7 + inputIdBytes.byteLength); + frame.set(magic); + frame[4] = 0x02; + frame[5] = inputIdBytes.byteLength; + frame.set(inputIdBytes, 6); + frame[6 + inputIdBytes.byteLength] = accepted ? 1 : 0; + return frame; } + +terminal.addEventListener("close", () => { + pty.kill(); +}); + +terminal.addEventListener("error", () => { + pty.kill(); +}); ``` +Set `CRABFLEET_RUNNER_PTY_URL` to the `runnerPtyUrl` returned by registration. +For a PTY API with an asynchronous write callback or promise, await that +acceptance signal before sending `encodeAck(..., true)`. Do not acknowledge when +the WebSocket merely queues the input frame. + +Each `CFR1` frame occupies one binary WebSocket message: + +| Offset | Size | Value | +| ------ | -------- | --------------------------------------------------------------- | +| 0 | 4 | ASCII `CFR1` | +| 4 | 1 | `0x01` input, `0x02` acknowledgement, or `0x03` lifecycle event | +| 5 | 1 | input ID byte length | +| 6 | variable | input ID followed by the type-specific payload | + +Input payloads are raw terminal bytes. An acknowledgement payload starts with +`1` for accepted or `0` for rejected and may include UTF-8 error text after the +status byte. Lifecycle events use an empty input ID and event code `0x01` for +runner connected, `0x02` for runner disconnected, or `0x03` for runner waiting. +The full wire contract is also specified in +[API](/api/#get-api-agent-interactive-sessions-id-runner-pty). + Properties: -- The URL is directly usable by Node's global `WebSocket`. - Authentication is the session-scoped `agentToken` query value. - Only one runner is current. - A new runner connection replaces the previous runner. - Multiple browser viewers may remain connected. -- Runner output is fanned out to viewers. -- Writable viewer input is sent to the current runner only. -- Runner lifecycle events are visible to viewers even while no runner is +- Unframed runner output is fanned out to viewers unchanged. +- Writable viewer input is framed and sent to the current runner only. +- A viewer sees `input-accepted` only after the correlated runner + acknowledgement. +- Runner lifecycle events are typed binary frames even while no runner is connected. +- Unframed viewer input is rejected. -The relay transports raw terminal bytes. It does not interpret Codex JSON-RPC. -The runner-side integration decides how terminal input maps to model steering. +The relay does not interpret Codex JSON-RPC. The runner-side integration decides +how accepted terminal input maps to model steering. ## Browser Attach @@ -325,13 +430,15 @@ instead of inventing a local shell. ## Steering Semantics -Crabfleet itself forwards terminal input bytes. In the ClawSweeper integration, -the runner: +Crabfleet forwards terminal input inside correlated `CFR1` frames. In the +ClawSweeper integration, the runner: -1. Collects printable input until Enter. -2. Echoes `[steer] ` to the terminal. -3. Calls Codex `turn/steer` with the active thread and expected turn ID. -4. Reports rejection or no-active-turn conditions in the terminal. +1. Accepts the framed bytes into its input handler and acknowledges that input + ID. +2. Collects printable input until Enter. +3. Echoes `[steer] ` to the terminal as raw output. +4. Calls Codex `turn/steer` with the active thread and expected turn ID. +5. Reports rejection or no-active-turn conditions in the terminal. `Ctrl-C` maps to `turn/interrupt`. diff --git a/docs/runs.md b/docs/runs.md index 75cede75..31ee84da 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -120,8 +120,9 @@ Terminal contract: GitHub Actions PTY contract: - OpenClaw registers or resumes work through `POST /api/openclaw/action-sessions`. -- The returned `runnerPtyUrl` is a `wss:` URL with a rotated session-scoped query credential, directly usable by Node's global `WebSocket`. -- The Actions process connects outbound and sends raw terminal output bytes. Raw Ghostty input bytes are returned on the same socket. +- The returned `runnerPtyUrl` is a `wss:` URL with a rotated session-scoped query credential. Node's global `WebSocket` can open it without custom headers, but the runner must implement the `CFR1` protocol. +- The Actions process sends unframed raw terminal output. Viewer input arrives in correlated binary `CFR1` frames, and the runner returns the matching acknowledgement only after its PTY accepts the write. +- Legacy runners that expect raw viewer input are incompatible; unframed viewer input is rejected. - `SessionControlDO` allows one current runner and multiple viewers. A new runner replaces the previous runner; viewers remain connected and receive runner lifecycle events. - Authorized browser viewers attach through the existing `/api/terminal/ws` hub. Service and agent credentials are never included in viewer responses. - The runner updates `state`, `phase`, `summary`, Codex thread/turn IDs, and heartbeat through the agent work-state endpoint. `completed`, `blocked`, `failed`, and `canceled` are terminal. diff --git a/docs/spec.md b/docs/spec.md index f03415f1..67c5c3bd 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -178,12 +178,15 @@ Crabfleet owns: - session identity and metadata; - rotating scoped agent token; -- outbound runner relay through `SessionControlDO`; +- outbound runner relay through `SessionControlDO`, with raw runner output and correlated binary `CFR1` input, acknowledgement, and lifecycle frames; - browser terminal steering; - work-state heartbeats; - event and transcript finalization. -The Action remains the execution host and mutation authority. Ending the Crabfleet session does not cancel the workflow run. +The Action remains the execution host and mutation authority. It acknowledges +viewer input only after its PTY accepts the correlated write; relay queueing is +not acceptance. Legacy raw-input runners are incompatible. Ending the Crabfleet +session does not cancel the workflow run. ## Session Lifecycle From 35afbf278b956a27b7674d28ed43cb33bd600125 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:30:31 +0200 Subject: [PATCH 075/242] fix(actions): fence registration and terminal races --- src/worker/github-actions-application.ts | 3 +- src/worker/github-actions-repository.ts | 7 ++ .../github-actions-session-registration.ts | 3 +- .../github-actions-session-work-state.ts | 38 ++++++----- tests/github-actions-repository.test.ts | 24 ++++++- ...ithub-actions-session-registration.test.ts | 66 +++++++++++++++++++ .../github-actions-session-work-state.test.ts | 19 +++++- 7 files changed, 141 insertions(+), 19 deletions(-) diff --git a/src/worker/github-actions-application.ts b/src/worker/github-actions-application.ts index 60ca46a0..3fd54b4b 100644 --- a/src/worker/github-actions-application.ts +++ b/src/worker/github-actions-application.ts @@ -104,7 +104,8 @@ export class GitHubActionsApplication { const store: GitHubActionsWorkStateStore = { now: () => Date.now(), readRow: (sessionId) => repository.readById(sessionId), - persist: (sessionId, values) => repository.updateSession(sessionId, values), + persist: (sessionId, values, expectedTerminalStatus) => + repository.updateSession(sessionId, values, undefined, expectedTerminalStatus), appendEvent: (sessionId, message, now) => this.appendMessageEvent(sessionId, user, message, now), disconnectRunner: (sessionId) => this.disconnectRunner(sessionId), diff --git a/src/worker/github-actions-repository.ts b/src/worker/github-actions-repository.ts index 0a598b8b..6708bc11 100644 --- a/src/worker/github-actions-repository.ts +++ b/src/worker/github-actions-repository.ts @@ -10,6 +10,7 @@ import type { } from "./github-actions-session-registration.ts"; import type { GitHubActionsWorkStateUpdate } from "./github-actions-session-work-state.ts"; import { conflict } from "./http.ts"; +import type { InteractiveSessionStatus } from "./models.ts"; type GitHubActionsSessionUpdate = | GitHubActionsSessionRegistrationUpdate @@ -54,6 +55,7 @@ export class GitHubActionsRepository { id: string, values: GitHubActionsSessionUpdate, expectedRegistration?: GitHubActionsSessionRegistrationExpectation, + expectedTerminalStatus?: InteractiveSessionStatus, ): Promise { let update = database(this.env) .updateTable("interactive_sessions") @@ -72,8 +74,13 @@ export class GitHubActionsRepository { .where("work_phase", "=", expectedRegistration.work_phase) .where("owner_subject", "=", values.owner_subject); } else if (isWorkStateUpdate(values) && terminalWorkStates.includes(values.work_state)) { + if (!expectedTerminalStatus) { + throw new Error("terminal GitHub Actions update requires expected session status"); + } update = update .where("updated_at", "<=", values.updated_at) + .where("status", "=", expectedTerminalStatus) + .where("status", "not in", terminalSessionStatuses) .where((expressions) => expressions.or([ expressions("work_state", "not in", terminalWorkStates), diff --git a/src/worker/github-actions-session-registration.ts b/src/worker/github-actions-session-registration.ts index 7f024959..26780fdc 100644 --- a/src/worker/github-actions-session-registration.ts +++ b/src/worker/github-actions-session-registration.ts @@ -157,6 +157,7 @@ export class GitHubActionsSessionRegistrationService { const resumed = existing.work_state !== "registered" || existing.status !== "ready"; const message = resumed ? "GitHub Actions work resumed" : "GitHub Actions work registered"; + const registrationRevision = Math.max(now, existing.updated_at + 1); await this.store.updateSession( existing.id, { @@ -174,7 +175,7 @@ export class GitHubActionsSessionRegistrationService { terminal_failure_reason: null, terminal_finalize_pending: 0, credential_cleanup_terminal_status: null, - updated_at: now, + updated_at: registrationRevision, last_seen_at: now, last_event: message, agent_token_hash: agentTokenHash, diff --git a/src/worker/github-actions-session-work-state.ts b/src/worker/github-actions-session-work-state.ts index a19f0a10..1ffe7fc4 100644 --- a/src/worker/github-actions-session-work-state.ts +++ b/src/worker/github-actions-session-work-state.ts @@ -38,7 +38,11 @@ export type GitHubActionsWorkStateUpdate = { export type GitHubActionsWorkStateStore = { now(): number; readRow(id: string): Promise; - persist(id: string, values: GitHubActionsWorkStateUpdate): Promise; + persist( + id: string, + values: GitHubActionsWorkStateUpdate, + expectedTerminalStatus?: InteractiveSessionStatus, + ): Promise; appendEvent(id: string, message: string, now: number): Promise; disconnectRunner(id: string): Promise; readSession(id: string): Promise; @@ -91,20 +95,24 @@ export class GitHubActionsWorkStateService { row.completion_reason !== completionReason; const now = this.store.now(); - await this.store.persist(session.id, { - status, - summary, - work_state: state, - work_phase: phase, - codex_thread_id: codexThreadId, - codex_turn_id: codexTurnId, - last_heartbeat_at: now, - completion_reason: completionReason, - last_event: lastEvent, - last_seen_at: now, - updated_at: now, - stopped_at: terminal ? now : null, - }); + await this.store.persist( + session.id, + { + status, + summary, + work_state: state, + work_phase: phase, + codex_thread_id: codexThreadId, + codex_turn_id: codexTurnId, + last_heartbeat_at: now, + completion_reason: completionReason, + last_event: lastEvent, + last_seen_at: now, + updated_at: now, + stopped_at: terminal ? now : null, + }, + terminal ? row.status : undefined, + ); if (changed) { await this.store.appendEvent(session.id, lastEvent, now); } diff --git a/tests/github-actions-repository.test.ts b/tests/github-actions-repository.test.ts index f21290e2..8a7855f7 100644 --- a/tests/github-actions-repository.test.ts +++ b/tests/github-actions-repository.test.ts @@ -69,7 +69,7 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = }), ); await repository.updateSession("IS-101", registrationUpdate, registrationExpectation); - await repository.updateSession("IS-101", workStateUpdate); + await repository.updateSession("IS-101", workStateUpdate, undefined, "attached"); await repository.updateSession("IS-101", runnerConnectionUpdate); assert.equal(executions.length, 6); @@ -94,6 +94,10 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = assert.match(executions[3].sql, /"work_state" = \?/i); assert.match(executions[3].sql, /"work_phase" = \?/i); assert.match(executions[4].sql, /"updated_at" <= \?/i); + assert.match(executions[4].sql, /"status" = \?/i); + assert.match(executions[4].sql, /"status" not in/i); + assert.ok(executions[4].parameters.includes("attached")); + assert.ok(executions[4].parameters.includes("expired")); assert.match(executions[5].sql, /"updated_at" <= \?/i); assert.match(executions[3].sql, /"owner_subject" = \?/i); assert.doesNotMatch(executions[3].sql, /"work_state" not in/i); @@ -117,6 +121,24 @@ test("GitHub Actions repository rejects stale or invalid state transitions", asy assert.equal(executions.length, 1); }); +test("terminal work-state updates require an observed non-terminal status", async () => { + const executions: Execution[] = []; + const repository = new GitHubActionsRepository(runtimeEnv(executions)); + + await assert.rejects( + repository.updateSession("IS-101", { + ...workStateUpdate, + status: "stopped", + work_state: "completed", + stopped_at: 200, + }), + { + message: "terminal GitHub Actions update requires expected session status", + }, + ); + assert.equal(executions.length, 0); +}); + const registrationUpdate: GitHubActionsSessionRegistrationUpdate = { owner: "operator@example.test", owner_subject: "github:42", diff --git a/tests/github-actions-session-registration.test.ts b/tests/github-actions-session-registration.test.ts index de65b393..c14d39c5 100644 --- a/tests/github-actions-session-registration.test.ts +++ b/tests/github-actions-session-registration.test.ts @@ -363,6 +363,72 @@ test("registration adopts a concurrently inserted work key", async () => { assert.equal(result.session.id, "IS-concurrent"); assert.equal(state.workKeyReads, 2); assert.equal(state.updates[0]?.id, "IS-concurrent"); + assert.equal( + state.updates[0]?.values.updated_at, + Math.max(100, state.concurrentRow.updated_at + 1), + ); +}); + +test("concurrent registration adoption rotates exactly one usable token", async () => { + const existing = sessionRow({ + id: "IS-concurrent", + runtime: "github_actions", + work_key: "issue:race-cas", + owner: "operator", + owner_subject: "github:42", + updated_at: 100, + }); + const { store, state } = registrationStore([existing]); + let tokenSequence = 0; + let arrivals = 0; + let releaseUpdates!: () => void; + const updatesReady = new Promise((resolve) => { + releaseUpdates = resolve; + }); + store.newAgentToken = () => `agent-token-${++tokenSequence}`; + store.hashToken = async (token) => `${token}-hash`; + store.updateSession = async (id, values, expected) => { + arrivals += 1; + if (arrivals === 2) releaseUpdates(); + await updatesReady; + const current = state.rows.get(id); + if ( + !current || + current.updated_at !== expected.updated_at || + current.status !== expected.status || + current.work_state !== expected.work_state || + current.work_phase !== expected.work_phase + ) { + throw new Error("GitHub Actions session changed; retry"); + } + state.rows.set(id, { ...current, ...values }); + }; + + const input = { + workKey: "issue:race-cas", + workKind: "issue", + repo: "openclaw/crabfleet", + owner: "operator@example.test", + }; + const results = await Promise.allSettled([ + new GitHubActionsSessionRegistrationService(store).register(input), + new GitHubActionsSessionRegistrationService(store).register(input), + ]); + + const fulfilled = results.filter( + ( + result, + ): result is PromiseFulfilledResult< + Awaited> + > => result.status === "fulfilled", + ); + assert.equal(fulfilled.length, 1); + assert.equal(results.filter((result) => result.status === "rejected").length, 1); + assert.equal( + state.rows.get(existing.id)?.agent_token_hash, + `${fulfilled[0]?.value.agentToken}-hash`, + ); + assert.equal(state.rows.get(existing.id)?.updated_at, existing.updated_at + 1); }); test("registration rejects invalid input and work keys owned by another runtime", async () => { diff --git a/tests/github-actions-session-work-state.test.ts b/tests/github-actions-session-work-state.test.ts index 11a69d80..7be6baa4 100644 --- a/tests/github-actions-session-work-state.test.ts +++ b/tests/github-actions-session-work-state.test.ts @@ -13,6 +13,7 @@ import { sessionRow } from "./helpers/session-row.ts"; type WorkStateStoreState = { row: InteractiveSessionRow | null; update: GitHubActionsWorkStateUpdate | null; + expectedTerminalStatus: InteractiveSessionRow["status"] | undefined; events: string[]; operations: string[]; disconnectError: unknown; @@ -48,6 +49,7 @@ function workStateStore(values: Partial = {}): { ...values, }), update: null, + expectedTerminalStatus: undefined, events: [], operations: [], disconnectError: null, @@ -55,9 +57,10 @@ function workStateStore(values: Partial = {}): { const store: GitHubActionsWorkStateStore = { now: () => 500, readRow: async () => state.row, - persist: async (_id, update) => { + persist: async (_id, update, expectedTerminalStatus) => { state.operations.push("persist"); state.update = update; + state.expectedTerminalStatus = expectedTerminalStatus; if (state.row) state.row = { ...state.row, ...update }; }, appendEvent: async (_id, message) => { @@ -139,10 +142,24 @@ test("terminal work-state updates stop the session and disconnect the runner", a assert.equal(result.status, "failed"); assert.equal(state.update?.completion_reason, "existing reason"); assert.equal(state.update?.stopped_at, 500); + assert.equal(state.expectedTerminalStatus, "ready"); assert.deepEqual(state.events, ["failed: tests"]); assert.deepEqual(state.operations, ["persist", "event", "disconnect", "read"]); }); +test("terminal work-state updates carry the status observed before persistence", async () => { + const { store, state } = workStateStore({ + status: "attached", + work_state: "running", + }); + + await new GitHubActionsWorkStateService(store).update(workSession(), { + state: "completed", + }); + + assert.equal(state.expectedTerminalStatus, "attached"); +}); + test("terminal runner disconnect races remain best effort", async () => { const { store, state } = workStateStore(); state.disconnectError = new Error("runner already disconnected"); From e3fec6f350779aa42a48a22a2714b271efd7b960 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:30:54 +0200 Subject: [PATCH 076/242] fix(vnc): defer transition timeout until framebuffer boundary --- .../SDK/Connection/VNCConnection+API.swift | 22 +++++++++++-- .../SDK/Connection/VNCConnection.swift | 1 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 31 +++++++++++++++++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index b0ced27b..e1f33819 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -173,9 +173,11 @@ extension VNCConnection { fencePayload = withUnsafeBytes(of: &sequence) { Data($0) } pixelFormatTransitionFencePayload = fencePayload pixelFormatTransitionRequiredFenceFlags = [.blockBefore, .syncNext] + pixelFormatTransitionFenceWasSent = false } else { fencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] + pixelFormatTransitionFenceWasSent = false } return PixelFormatTransition( pixelFormat: pixelFormat, @@ -214,7 +216,17 @@ extension VNCConnection { return } - cancelPixelFormatTransitionDeadlineLocked() + pixelFormatTransitionFenceWasSent = true + schedulePixelFormatTransitionDeadlineIfReadyLocked(payload: payload) + } + + private func schedulePixelFormatTransitionDeadlineIfReadyLocked(payload: Data) { + guard pixelFormatTransitionFenceWasSent, + !framebufferUpdateRequestOutstanding, + pixelFormatTransitionDeadlineTask == nil else { + return + } + pixelFormatTransitionDeadlineTask = Task { [weak self] in do { try await Task.sleep(nanoseconds: 5_000_000_000) @@ -233,7 +245,8 @@ extension VNCConnection { func expirePixelFormatTransitionDeadline(payload: Data) { framebufferRequestLock.lock() guard isPixelFormatTransitionInFlight, - pixelFormatTransitionFencePayload == payload else { + pixelFormatTransitionFencePayload == payload, + pixelFormatTransitionDeadlineTask != nil else { framebufferRequestLock.unlock() return } @@ -361,6 +374,7 @@ extension VNCConnection { throw VNCError.protocol(.invalidData) } cancelPixelFormatTransitionDeadlineLocked() + pixelFormatTransitionFenceWasSent = false pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] framebufferRequestLock.unlock() @@ -577,6 +591,9 @@ extension VNCConnection { func completeFramebufferUpdateRequest() { framebufferRequestLock.lock() framebufferUpdateRequestOutstanding = false + if let payload = pixelFormatTransitionFencePayload { + schedulePixelFormatTransitionDeadlineIfReadyLocked(payload: payload) + } let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() @@ -641,6 +658,7 @@ extension VNCConnection { pixelFormatTransitionInFlight = nil pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] + pixelFormatTransitionFenceWasSent = false cancelPixelFormatTransitionDeadlineLocked() pixelFormatFenceCapabilityProbePayload = nil cancelPixelFormatFenceNegotiationTimeoutLocked() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index d81024ba..ecb1112a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -116,6 +116,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pixelFormatTransitionFenceSequence: UInt64 = 0 var pixelFormatTransitionFencePayload: Data? var pixelFormatTransitionRequiredFenceFlags: VNCProtocol.FenceFlags = [] + var pixelFormatTransitionFenceWasSent = false var pixelFormatTransitionDeadlineTask: Task? var pixelFormatFenceCapabilityProbePayload: Data? var pixelFormatFenceNegotiationTask: Task? diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 9fa7d446..529abf60 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -310,7 +310,7 @@ struct AuditFindingsTests { } try await queued.message.send(connection: writer) - #expect(connection.pixelFormatTransitionDeadlineTask != nil) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) #expect(writer.data.count == 37) #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) #expect(writer.data[4..<8] == Data([0x80, 0, 0, 5])) @@ -319,6 +319,8 @@ struct AuditFindingsTests { #expect(connection.state.pixelFormat?.depth == 24) let payload = Data(writer.data[9..<17]) + connection.completeFramebufferUpdateRequest() + #expect(connection.pixelFormatTransitionDeadlineTask != nil) try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, @@ -333,6 +335,24 @@ struct AuditFindingsTests { #expect(connection.connectionState.status == .connected) } + @Test + func waitsForSlowFramebufferBoundaryBeforeArmingTransitionDeadline() async throws { + let connection = try await makeFenceCapableConnection() + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + try await queued.message.send(connection: AuditWritingConnection()) + + try await Task.sleep(nanoseconds: 100_000_000) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) + #expect(connection.connectionState.status == .connected) + + connection.completeFramebufferUpdateRequest() + #expect(connection.pixelFormatTransitionDeadlineTask != nil) + + connection.cancelFramebufferUpdateScheduling() + } + @Test func disconnectsWhenPixelFormatTransitionFenceIsMissing() async throws { let connection = try await makeFenceCapableConnection() @@ -342,9 +362,14 @@ struct AuditFindingsTests { try await queued.message.send(connection: AuditWritingConnection()) let payload = try #require(connection.pixelFormatTransitionFencePayload) - #expect(connection.pixelFormatTransitionDeadlineTask != nil) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) #expect(connection.isPixelFormatTransitionInFlight) + connection.expirePixelFormatTransitionDeadline(payload: payload) + #expect(connection.connectionState.status == .connected) + + connection.completeFramebufferUpdateRequest() + #expect(connection.pixelFormatTransitionDeadlineTask != nil) connection.expirePixelFormatTransitionDeadline(payload: payload) #expect(connection.connectionState.status == .disconnected) @@ -361,6 +386,8 @@ struct AuditFindingsTests { try await queued.message.send(connection: AuditWritingConnection()) let payload = try #require(connection.pixelFormatTransitionFencePayload) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) + connection.completeFramebufferUpdateRequest() #expect(connection.pixelFormatTransitionDeadlineTask != nil) connection.cancelFramebufferUpdateScheduling() From 8fcce0f969516c792d0c4ea1c7ea62325a5f4c9a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:32:00 +0200 Subject: [PATCH 077/242] fix(vnc): release unresolved credential requests --- .../Connection/VNCConnection+Delegate.swift | 55 ++++----- .../SDK/Connection/VNCConnection.swift | 104 ++++++++++++++---- .../RoyalVNCKitTests/AuditFindingsTests.swift | 53 +++++++++ 3 files changed, 161 insertions(+), 51 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift index 32b8322f..ee629746 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Delegate.swift @@ -136,33 +136,8 @@ extension VNCConnection { private extension VNCConnection { func askDelegateForCredential(authenticationType: VNCAuthenticationType) async throws -> VNCCredential { - let requestID = UUID() - let credential: VNCCredential? = await withTaskCancellationHandler { - await withCheckedContinuation { continuation in - guard registerCredentialContinuation(continuation, id: requestID) else { - continuation.resume(returning: nil) - return - } - - if Task.isCancelled { - cancelPendingCredentialRequest(id: requestID) - return - } - - DispatchQueue.main.async { [weak self] in - guard let self, let delegate = self.delegate else { - self?.resolveCredentialRequest(id: requestID, credential: nil) - return - } - - delegate.connection(self, credentialFor: authenticationType) { [weak self] credential in - self?.resolveCredentialRequest(id: requestID, credential: credential) - } - } - } - } onCancel: { - cancelPendingCredentialRequest(id: requestID) - } + let request = beginCredentialRequest(authenticationType: authenticationType) + let credential = await request.value() guard let credential else { throw VNCError.authentication(.noAuthenticationDataProvided) @@ -171,3 +146,29 @@ private extension VNCConnection { return credential } } + +extension VNCConnection { + func beginCredentialRequest(authenticationType: VNCAuthenticationType) -> PendingCredentialRequest { + let requestID = UUID() + let request = PendingCredentialRequest { [weak self] in + self?.removeCredentialRequest(id: requestID) + } + guard registerCredentialRequest(request, id: requestID) else { + request.resolve(with: nil) + return request + } + + DispatchQueue.main.async { [weak self, request] in + guard request.pending else { return } + guard let self, let delegate = self.delegate else { + request.resolve(with: nil) + return + } + + delegate.connection(self, credentialFor: authenticationType) { [request] credential in + request.resolve(with: credential) + } + } + return request + } +} diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index ecb1112a..a138ddbb 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -123,9 +123,9 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() - private let credentialContinuationLock = NSLock() - private var pendingCredentialContinuations = [ - UUID: CheckedContinuation + private let credentialRequestLock = NSLock() + private var pendingCredentialRequests = [ + UUID: WeakCredentialRequest ]() private let sharedZStream: ZlibStream @@ -446,8 +446,8 @@ extension VNCConnection { return operation() } - func registerCredentialContinuation( - _ continuation: CheckedContinuation, + func registerCredentialRequest( + _ request: PendingCredentialRequest, id: UUID ) -> Bool { lifecycleLock.lock() @@ -457,33 +457,27 @@ extension VNCConnection { return false } - credentialContinuationLock.lock() - defer { credentialContinuationLock.unlock() } + credentialRequestLock.lock() + defer { credentialRequestLock.unlock() } - pendingCredentialContinuations[id] = continuation + pendingCredentialRequests[id] = WeakCredentialRequest(request) return true } - func resolveCredentialRequest(id: UUID, credential: VNCCredential?) { - credentialContinuationLock.lock() - let continuation = pendingCredentialContinuations.removeValue(forKey: id) - credentialContinuationLock.unlock() - - continuation?.resume(returning: credential) - } - - func cancelPendingCredentialRequest(id: UUID) { - resolveCredentialRequest(id: id, credential: nil) + func removeCredentialRequest(id: UUID) { + credentialRequestLock.lock() + pendingCredentialRequests.removeValue(forKey: id) + credentialRequestLock.unlock() } func cancelPendingCredentialRequests() { - credentialContinuationLock.lock() - let continuations = Array(pendingCredentialContinuations.values) - pendingCredentialContinuations.removeAll() - credentialContinuationLock.unlock() + credentialRequestLock.lock() + let requests = pendingCredentialRequests.values.compactMap(\.request) + pendingCredentialRequests.removeAll() + credentialRequestLock.unlock() - for continuation in continuations { - continuation.resume(returning: nil) + for request in requests { + request.resolve(with: nil) } } @@ -508,6 +502,68 @@ extension VNCConnection { } } +final class PendingCredentialRequest: @unchecked Sendable { + private let lock = NSLock() + private var continuation: CheckedContinuation? + private var isResolved = false + private var resolvedCredential: VNCCredential? + private var onResolution: (() -> Void)? + + init(onResolution: @escaping () -> Void) { + self.onResolution = onResolution + } + + var pending: Bool { + lock.lock() + defer { lock.unlock() } + return !isResolved + } + + func value() async -> VNCCredential? { + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + lock.lock() + if isResolved { + let credential = resolvedCredential + lock.unlock() + continuation.resume(returning: credential) + } else { + self.continuation = continuation + lock.unlock() + } + } + } onCancel: { + resolve(with: nil) + } + } + + func resolve(with credential: VNCCredential?) { + lock.lock() + guard !isResolved else { + lock.unlock() + return + } + isResolved = true + resolvedCredential = credential + let continuation = self.continuation + self.continuation = nil + let onResolution = self.onResolution + self.onResolution = nil + lock.unlock() + + continuation?.resume(returning: credential) + onResolution?() + } +} + +private final class WeakCredentialRequest { + weak var request: PendingCredentialRequest? + + init(_ request: PendingCredentialRequest) { + self.request = request + } +} + // MARK: - Connection State Change Handling private extension VNCConnection { func connectionStatusDidChange(_ newState: NetworkConnectionStatus) { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 529abf60..2bad9414 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -538,6 +538,59 @@ struct AuditFindingsTests { delegate.completion?(VNCPasswordCredential(password: "late")) } + @Test @MainActor + func unresolvedCredentialDelegateRequestDoesNotRetainConnection() async { + var connection: VNCConnection? = VNCConnection(settings: makeSettings()) + weak var weakConnection = connection + let delegate = PendingCredentialDelegate() + connection?.delegate = delegate + + let request = connection!.beginCredentialRequest(authenticationType: .vnc) + let credentialTask = Task { + await request.value() + } + + while delegate.completion == nil { + await Task.yield() + } + + connection = nil + for _ in 0..<100 where weakConnection != nil { + await Task.yield() + } + + #expect(weakConnection == nil) + #expect(await credentialTask.value == nil) + + delegate.completion?(VNCPasswordCredential(password: "late")) + } + + @Test @MainActor + func taskCancellationResolvesPendingCredentialRequest() async { + let connection = VNCConnection(settings: makeSettings()) + let delegate = PendingCredentialDelegate() + connection.delegate = delegate + + let credentialTask = Task { + try await connection.askDelegateForPasswordCredential(authenticationType: .vnc) + } + + while delegate.completion == nil { + await Task.yield() + } + + credentialTask.cancel() + + do { + _ = try await credentialTask.value + Issue.record("Expected task cancellation to resolve the pending credential request") + } catch { + #expect(credentialTask.isCancelled) + } + + delegate.completion?(VNCPasswordCredential(password: "late")) + } + @Test func preservesAlreadyRGBAFormattedCursorChannels() { let cursor = VNCCursor( From b018ec397d1f30b931b4e4a501a76d25e2070bb0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:33:55 +0200 Subject: [PATCH 078/242] fix(credentials): restore failed policy rotations --- CHANGELOG.md | 2 +- ...redential_policy_registration_rollback.sql | 2 + src/credential-policy-fence.ts | 30 ++++ src/worker/database.ts | 1 + ...ndbox-credential-policy-cleanup-service.ts | 23 ++- ...-credential-policy-registration-service.ts | 63 ++++++++- .../sandbox-credential-policy-repository.ts | 52 +++++++ .../sandbox-credential-policy-rollback.ts | 133 ++++++++++++++++++ .../sandbox-credential-policy-scanner.ts | 31 +++- tests/credential-policy-fence.test.ts | 6 + tests/runtime-adapter.test.ts | 22 +++ ...ndbox-credential-policy-repository.test.ts | 32 ++++- ...sandbox-credential-policy-rollback.test.ts | 121 ++++++++++++++++ 13 files changed, 510 insertions(+), 8 deletions(-) create mode 100644 migrations/0035_credential_policy_registration_rollback.sql create mode 100644 src/worker/sandbox-credential-policy-rollback.ts create mode 100644 tests/sandbox-credential-policy-rollback.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c0ae2d..b6be40c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, read-state-fenced GitHub Actions registration, revision-fenced lifecycle updates and grant revocation, monotonic Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, rollback-restored Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Change the GitHub Actions runner PTY contract to correlated `CFR1` binary input and acknowledgement frames, emit success only after the runner accepts the input into its PTY, preserve unframed runner output, and reject legacy raw-input clients. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. diff --git a/migrations/0035_credential_policy_registration_rollback.sql b/migrations/0035_credential_policy_registration_rollback.sql new file mode 100644 index 00000000..8af5f195 --- /dev/null +++ b/migrations/0035_credential_policy_registration_rollback.sql @@ -0,0 +1,2 @@ +ALTER TABLE interactive_session_credential_policy_registrations + ADD COLUMN rollback_policies_json TEXT; diff --git a/src/credential-policy-fence.ts b/src/credential-policy-fence.ts index 17072808..f70ee05c 100644 --- a/src/credential-policy-fence.ts +++ b/src/credential-policy-fence.ts @@ -11,6 +11,11 @@ export type CredentialPolicyGenerationTombstone = { tombstonedAt: number; }; +export type CredentialPolicyRollbackRecord = { + generation: string; + policy: T; +}; + export function isCurrentCredentialPolicyGeneration(value: unknown): value is string { return ( typeof value === "string" && @@ -20,6 +25,31 @@ export function isCurrentCredentialPolicyGeneration(value: unknown): value is st ); } +export function credentialPolicyRollbackRecord( + value: unknown, +): CredentialPolicyRollbackRecord | undefined { + if (!value || typeof value !== "object") return undefined; + const record = value as Partial>; + if ( + !isCurrentCredentialPolicyGeneration(record.generation) || + !record.policy || + typeof record.policy !== "object" || + typeof record.policy.sessionId !== "string" || + !record.policy.sessionId + ) { + return undefined; + } + return record as CredentialPolicyRollbackRecord; +} + +export function credentialPolicyRollbackExpiresAt( + replacedRegistrationExpiresAt: number, + now: number, + claimTtlMs: number, +): number { + return Math.max(replacedRegistrationExpiresAt + 1, now + claimTtlMs); +} + export function credentialPolicyRegistrationAccepted( current: CredentialPolicyGenerationRecord | undefined, tombstone: CredentialPolicyGenerationTombstone | undefined, diff --git a/src/worker/database.ts b/src/worker/database.ts index 83e5fb19..98a432f3 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -313,6 +313,7 @@ export type InteractiveSessionCredentialPolicyRegistrationTable = { last_error: string | null; cleanup_claim: string | null; cleanup_claim_expires_at: number | null; + rollback_policies_json: Generated; created_at: number; updated_at: number; }; diff --git a/src/worker/sandbox-credential-policy-cleanup-service.ts b/src/worker/sandbox-credential-policy-cleanup-service.ts index 94ddeb95..96dc18dd 100644 --- a/src/worker/sandbox-credential-policy-cleanup-service.ts +++ b/src/worker/sandbox-credential-policy-cleanup-service.ts @@ -18,6 +18,7 @@ import { sandboxCredentialPolicyCleanupAuthorizedCondition, sandboxLookupIds, } from "./sandbox-credential-policy-repository.ts"; +import { restoreSandboxCredentialPolicyRollback } from "./sandbox-credential-policy-rollback.ts"; import { scanCredentialPolicyCleanupPage } from "./sandbox-credential-policy-scanner.ts"; import { isCurrentSandboxLease, sandboxLeaseInfo } from "./sandbox-lease.ts"; import { isSandboxSessionAlreadyGone } from "./sandbox-session-errors.ts"; @@ -290,11 +291,25 @@ export async function reconcileSandboxCredentialPolicyCleanupBatch( await expireStandaloneSandboxProvisions(env, now, sessionId).catch((error) => { console.error("standalone Sandbox expiry failed", error); }); - await scanCredentialPolicyCleanupPage(env, now, sandboxCredentialPolicyExists, sessionId).catch( - (error) => { - console.error("credential policy cleanup scan failed", error); + await scanCredentialPolicyCleanupPage( + env, + now, + sandboxCredentialPolicyExists, + sessionId, + async ({ registration, registrationExpiresAt, rollbackJson, sessionId: rollbackSessionId }) => { + const stub = sandboxControlStub(env); + if (!stub) throw serviceUnavailable("sandbox credential policy rollback is unavailable"); + await restoreSandboxCredentialPolicyRollback( + stub, + registration, + registrationExpiresAt, + rollbackJson, + rollbackSessionId, + ); }, - ); + ).catch((error) => { + console.error("credential policy cleanup scan failed", error); + }); await reconcileStagedCredentialPolicyCleanup(env, now, sessionId).catch((error) => { console.error("staged credential policy cleanup failed", error); }); diff --git a/src/worker/sandbox-credential-policy-registration-service.ts b/src/worker/sandbox-credential-policy-registration-service.ts index 44b2c5ed..0cc215dd 100644 --- a/src/worker/sandbox-credential-policy-registration-service.ts +++ b/src/worker/sandbox-credential-policy-registration-service.ts @@ -2,15 +2,22 @@ import { fetchGithubRepoNodeId } from "./github.ts"; import { sealSecret } from "./crypto.ts"; import type { RuntimeEnv } from "./env.ts"; import { + activeSandboxCredentialPolicyGeneration, abandonSandboxCredentialPolicyRegistration, beginSandboxCredentialPolicyRegistration, + deferSandboxCredentialPolicyRollback, existingSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, + recordSandboxCredentialPolicyRollback, renewSandboxCredentialPolicyRegistration, standaloneSandboxPolicyExpiresAt, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; +import { + captureSandboxCredentialPolicyRollback, + restoreSandboxCredentialPolicyRollback, +} from "./sandbox-credential-policy-rollback.ts"; import { sandboxCredentialPolicyExists } from "./sandbox-credential-policy-cleanup-service.ts"; import { sandboxLeaseInfo, @@ -46,7 +53,34 @@ export async function registerSandboxCredentialPolicy( sandboxId, ownershipFence, ); + let latestRegistrationExpiresAt = 0; + let rollbackJson: string | null = null; + let registrationWriteStarted = false; try { + const activeGeneration = await activeSandboxCredentialPolicyGeneration( + env, + session.id, + sandboxId, + ); + const rollback = await captureSandboxCredentialPolicyRollback( + stub, + registration.lookupIds, + activeGeneration, + session.id, + ); + if ( + !(await recordSandboxCredentialPolicyRollback( + env, + session.id, + sandboxId, + registration, + rollback, + ownershipFence, + )) + ) { + throw new Error("sandbox credential policy rollback snapshot was not recorded"); + } + rollbackJson = JSON.stringify(rollback); const githubToken = "githubToken" in session ? session.githubToken : undefined; const githubTokenCiphertext = githubToken ? await sealSecret(env, githubToken) : null; if (githubToken && !githubTokenCiphertext) { @@ -87,6 +121,8 @@ export async function registerSandboxCredentialPolicy( if (!registrationExpiresAt) { throw new Error("sandbox credential policy registration claim was revoked"); } + latestRegistrationExpiresAt = registrationExpiresAt; + registrationWriteStarted = true; const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { method: "POST", body: JSON.stringify({ @@ -113,12 +149,37 @@ export async function registerSandboxCredentialPolicy( throw new Error("sandbox credential policy cleanup became pending during registration"); } } catch (error) { + const message = clean(error instanceof Error ? error.message : String(error), 500); + if (registrationWriteStarted && rollbackJson) { + try { + await restoreSandboxCredentialPolicyRollback( + stub, + registration, + latestRegistrationExpiresAt, + rollbackJson, + session.id, + ); + } catch (rollbackError) { + const rollbackMessage = clean( + rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + 500, + ); + await deferSandboxCredentialPolicyRollback( + env, + session.id, + sandboxId, + registration, + `${message}; ${rollbackMessage}`, + ).catch(() => undefined); + throw new Error("sandbox credential policy rollback restore is pending", { cause: error }); + } + } await abandonSandboxCredentialPolicyRegistration( env, session.id, sandboxId, registration, - clean(error instanceof Error ? error.message : String(error), 500), + message, ).catch(() => undefined); throw error; } diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 1cc0c8c1..7a61e071 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -6,6 +6,7 @@ import { type SandboxCredentialPolicy, type SandboxCredentialPolicyRegistration, } from "./session-control-policy.ts"; +import type { SandboxCredentialPolicyRollbackRecord } from "./sandbox-credential-policy-rollback.ts"; import { database, executeBatch, type CompilableQuery } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import { @@ -483,6 +484,7 @@ export function sandboxCredentialPolicyRegistrationQueries( registration_generation = excluded.registration_generation, registration_claim = excluded.registration_claim, registration_claim_expires_at = excluded.registration_claim_expires_at, + rollback_policies_json = NULL, last_error = NULL, cleanup_claim = NULL, cleanup_claim_expires_at = NULL, @@ -584,6 +586,56 @@ export async function renewSandboxCredentialPolicyRegistration( return Number(renewed.numUpdatedRows ?? 0n) === 1 ? registrationExpiresAt : null; } +export async function recordSandboxCredentialPolicyRollback( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + rollback: readonly SandboxCredentialPolicyRollbackRecord[], + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const now = Date.now(); + const recorded = await database(env) + .updateTable("interactive_session_credential_policy_registrations") + .set({ + rollback_policies_json: JSON.stringify(rollback), + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("state", "=", "registering") + .where("registration_generation", "=", registration.generation) + .where("registration_claim", "=", registration.claim) + .where("registration_claim_expires_at", ">", now) + .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .executeTakeFirst(); + return Number(recorded.numUpdatedRows ?? 0n) === 1; +} + +export async function deferSandboxCredentialPolicyRollback( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + reason: string, +): Promise { + const now = Date.now(); + await database(env) + .updateTable("interactive_session_credential_policy_registrations") + .set({ + registration_claim_expires_at: now, + last_error: reason, + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("state", "=", "registering") + .where("registration_generation", "=", registration.generation) + .where("registration_claim", "=", registration.claim) + .where("rollback_policies_json", "is not", null) + .execute(); +} + export async function finishSandboxCredentialPolicyRegistration( env: RuntimeEnv, sessionId: string, diff --git a/src/worker/sandbox-credential-policy-rollback.ts b/src/worker/sandbox-credential-policy-rollback.ts new file mode 100644 index 00000000..edd0ba13 --- /dev/null +++ b/src/worker/sandbox-credential-policy-rollback.ts @@ -0,0 +1,133 @@ +import { + credentialPolicyRollbackExpiresAt, + credentialPolicyRollbackRecord, + isCurrentCredentialPolicyGeneration, + type CredentialPolicyRollbackRecord, +} from "../credential-policy-fence.ts"; +import type { + SandboxCredentialPolicy, + SandboxCredentialPolicyRegistration, + StoredSandboxCredentialPolicy, +} from "./session-control-policy.ts"; + +const credentialPolicyRollbackClaimMs = 60_000; + +export type SandboxCredentialPolicyRollbackRecord = + CredentialPolicyRollbackRecord; + +type SessionControlStub = Pick; + +export async function captureSandboxCredentialPolicyRollback( + stub: SessionControlStub, + lookupIds: readonly string[], + expectedGeneration: string | null, + sessionId: string, +): Promise { + const records = await Promise.all( + lookupIds.map(async (lookupId): Promise => { + const response = await stub.fetch( + `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, + ); + if (response.status === 404) return null; + if (!response.ok) throw new Error("sandbox credential policy rollback snapshot failed"); + const generation = response.headers.get("x-crabfleet-policy-generation"); + const policy = (await response.json()) as SandboxCredentialPolicy; + if ( + !isCurrentCredentialPolicyGeneration(generation) || + policy.sessionId !== sessionId || + policy.sandboxId !== lookupId + ) { + throw new Error("sandbox credential policy rollback snapshot is inconsistent"); + } + return { generation, policy }; + }), + ); + const present = records.filter((record): record is SandboxCredentialPolicyRollbackRecord => + Boolean(record), + ); + if (!expectedGeneration) { + if (present.length > 0) { + throw new Error("sandbox credential policy has no durable rollback owner"); + } + return []; + } + if ( + present.length !== lookupIds.length || + present.some((record) => record.generation !== expectedGeneration) + ) { + throw new Error("sandbox credential policy rollback generation is incomplete"); + } + return present; +} + +export function parseSandboxCredentialPolicyRollback( + value: string, + lookupIds: readonly string[], + sessionId: string, +): SandboxCredentialPolicyRollbackRecord[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error("sandbox credential policy rollback snapshot is invalid"); + } + if (!Array.isArray(parsed)) { + throw new Error("sandbox credential policy rollback snapshot is invalid"); + } + const records = parsed.map((item) => + credentialPolicyRollbackRecord(item), + ); + if (records.some((record) => !record)) { + throw new Error("sandbox credential policy rollback snapshot is invalid"); + } + const rollback = records as SandboxCredentialPolicyRollbackRecord[]; + const expectedLookups = new Set(lookupIds); + const actualLookups = new Set(rollback.map((record) => record.policy.sandboxId)); + const generations = new Set(rollback.map((record) => record.generation)); + if ( + rollback.some( + (record) => + record.policy.sessionId !== sessionId || !expectedLookups.has(record.policy.sandboxId), + ) || + actualLookups.size !== rollback.length || + (rollback.length > 0 && (rollback.length !== expectedLookups.size || generations.size !== 1)) + ) { + throw new Error("sandbox credential policy rollback snapshot is inconsistent"); + } + return rollback; +} + +export async function restoreSandboxCredentialPolicyRollback( + stub: SessionControlStub, + registration: SandboxCredentialPolicyRegistration, + registrationExpiresAt: number, + rollbackJson: string, + sessionId: string, +): Promise { + const rollback = parseSandboxCredentialPolicyRollback( + rollbackJson, + registration.lookupIds, + sessionId, + ); + const now = Date.now(); + const rollbackExpiresAt = credentialPolicyRollbackExpiresAt( + registrationExpiresAt, + now, + credentialPolicyRollbackClaimMs, + ); + for (const record of rollback) { + const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: record.generation, + registrationClaim: `rollback:${registration.claim}`, + registrationExpiresAt: rollbackExpiresAt, + policy: record.policy, + } satisfies StoredSandboxCredentialPolicy), + headers: { "content-type": "application/json" }, + }); + if (!response.ok) { + throw new Error("sandbox credential policy rollback restore failed"); + } + } +} diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index 587e4a08..f126a721 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -68,6 +68,8 @@ type CredentialPolicyOwnershipRow = Pick< type StagedCredentialPolicyScanRow = CredentialPolicyOwnershipRow & { registration_generation: string; registration_claim: string; + registration_claim_expires_at: number; + rollback_policies_json: string | null; }; export type SandboxCredentialPolicyExists = ( @@ -76,14 +78,29 @@ export type SandboxCredentialPolicyExists = ( generation: string, ) => Promise; +export type RestoreSandboxCredentialPolicyRollback = (input: { + registration: SandboxCredentialPolicyRegistration; + registrationExpiresAt: number; + rollbackJson: string; + sessionId: string; +}) => Promise; + export async function scanCredentialPolicyCleanupPage( env: RuntimeEnv, now: number, policyExists: SandboxCredentialPolicyExists, sessionId?: string, + restoreRollback?: RestoreSandboxCredentialPolicyRollback, ): Promise { const db = database(env); - await scanStagedCredentialPolicyRegistrations(env, db, now, policyExists, sessionId); + await scanStagedCredentialPolicyRegistrations( + env, + db, + now, + policyExists, + sessionId, + restoreRollback, + ); const state = sessionId ? null : await db @@ -282,6 +299,7 @@ async function scanStagedCredentialPolicyRegistrations( now: number, policyExists: SandboxCredentialPolicyExists, sessionId?: string, + restoreRollback?: RestoreSandboxCredentialPolicyRollback, ): Promise { const sessionFilter = sessionId ? sql`AND registration.session_id = ${sessionId}` : sql``; const result = await sql` @@ -290,6 +308,8 @@ async function scanStagedCredentialPolicyRegistrations( registration.sandbox_id, registration.registration_generation, registration.registration_claim, + registration.registration_claim_expires_at, + registration.rollback_policies_json, session.id AS matched_session_id, session.adapter AS session_adapter, session.lease_id AS session_lease_id, @@ -332,6 +352,15 @@ async function scanStagedCredentialPolicyRegistrations( ) { continue; } + if (row.rollback_policies_json !== null) { + if (!restoreRollback) throw new Error("sandbox credential policy rollback is unavailable"); + await restoreRollback({ + registration, + registrationExpiresAt: row.registration_claim_expires_at, + rollbackJson: row.rollback_policies_json, + sessionId: row.session_id, + }); + } await abandonSandboxCredentialPolicyRegistration( env, row.session_id, diff --git a/tests/credential-policy-fence.test.ts b/tests/credential-policy-fence.test.ts index 616f666f..f44f857e 100644 --- a/tests/credential-policy-fence.test.ts +++ b/tests/credential-policy-fence.test.ts @@ -3,6 +3,7 @@ import test from "node:test"; import { credentialPolicyCleanupMatches, + credentialPolicyRollbackExpiresAt, credentialPolicyRegistrationAccepted, credentialPolicySandboxIsExpected, type CredentialPolicyGenerationRecord, @@ -150,6 +151,11 @@ test("delayed abandoned registration cannot replace a newer claim", () => { assert.equal(credentialPolicyRegistrationAccepted(newer, undefined, abandoned, 200), false); }); +test("rollback claims advance beyond the generation they replace", () => { + assert.equal(credentialPolicyRollbackExpiresAt(500, 100, 200), 501); + assert.equal(credentialPolicyRollbackExpiresAt(200, 100, 200), 300); +}); + test("live lease refresh fences both current and expected sandbox policies", () => { assert.equal( credentialPolicySandboxIsExpected("sandbox-old", "sandbox-old", null, null, null, 100), diff --git a/tests/runtime-adapter.test.ts b/tests/runtime-adapter.test.ts index e59f1ed4..49f6423e 100644 --- a/tests/runtime-adapter.test.ts +++ b/tests/runtime-adapter.test.ts @@ -1041,6 +1041,10 @@ test("sandbox credential cleanup is durably staged and retried", async () => { new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), "utf8", ); + const registrationRollbackMigration = await readFile( + new URL("../migrations/0035_credential_policy_registration_rollback.sql", import.meta.url), + "utf8", + ); const scanStart = scannerSource.indexOf("type CredentialPolicyScanRow"); const scanSource = scannerSource.slice(scanStart); const batchStart = cleanupServiceSource.indexOf( @@ -1201,6 +1205,18 @@ test("sandbox credential cleanup is durably staged and retried", async () => { 'stub.fetch("https://crabfleet.internal/api/session-control/register"', ), ); + assert.ok( + registerSource.indexOf("captureSandboxCredentialPolicyRollback") < + registerSource.indexOf( + 'stub.fetch("https://crabfleet.internal/api/session-control/register"', + ), + ); + assert.ok( + registerSource.indexOf("recordSandboxCredentialPolicyRollback") < + registerSource.indexOf( + 'stub.fetch("https://crabfleet.internal/api/session-control/register"', + ), + ); assert.ok( registerSource.indexOf("renewSandboxCredentialPolicyRegistration") < registerSource.indexOf( @@ -1220,6 +1236,11 @@ test("sandbox credential cleanup is durably staged and retried", async () => { /DELETE FROM interactive_session_credential_policy_registrations/, ); assert.match(registerSource, /abandonSandboxCredentialPolicyRegistration/); + assert.match(registerSource, /restoreSandboxCredentialPolicyRollback/); + assert.ok( + scanSource.indexOf("restoreRollback") < + scanSource.indexOf("abandonSandboxCredentialPolicyRegistration"), + ); assert.match( abandonSource, /updateTable\("interactive_session_credential_policy_registrations"\)/, @@ -1244,6 +1265,7 @@ test("sandbox credential cleanup is durably staged and retried", async () => { /CREATE TABLE IF NOT EXISTS interactive_session_credential_policy_registrations/, ); assert.match(registrationStagingMigration, /state IN \('registering', 'cleanup_pending'\)/); + assert.match(registrationRollbackMigration, /ADD COLUMN rollback_policies_json TEXT/); assert.match(migration, /CREATE TABLE IF NOT EXISTS credential_policy_reconcile_state/); assert.match(migration, /scan_max_rowid INTEGER NOT NULL/); assert.match(migration, /group_max_session_id TEXT NOT NULL/); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 508bac38..5b68790e 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -9,6 +9,7 @@ import { currentSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, + recordSandboxCredentialPolicyRollback, sandboxCredentialPolicyRegistrationQueries, sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, @@ -128,6 +129,7 @@ function credentialPolicyDatabase(): DatabaseSync { last_error TEXT, cleanup_claim TEXT, cleanup_claim_expires_at INTEGER, + rollback_policies_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (session_id, sandbox_id) @@ -403,6 +405,28 @@ test("partial credential-policy rotation failure preserves the prior active gene "sandbox-1", ownershipFence, ); + const rollback = ["sandbox-1", "do-1"].map((lookupId) => ({ + generation: "generation:existing", + policy: { + allowedHosts: [], + githubCredentialSource: "none" as const, + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: lookupId, + sessionId: "IS-42", + }, + })); + assert.equal( + await recordSandboxCredentialPolicyRollback( + env, + "IS-42", + "sandbox-1", + staged, + rollback, + ownershipFence, + ), + true, + ); assert.deepEqual( activeCredentialPolicyRows(sqlite).map((row) => row.registration_generation), @@ -425,7 +449,12 @@ test("partial credential-policy rotation failure preserves the prior active gene { ...sqlite .prepare(` - SELECT state, registration_generation, registration_claim, last_error + SELECT + state, + registration_generation, + registration_claim, + rollback_policies_json, + last_error FROM interactive_session_credential_policy_registrations WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' `) @@ -435,6 +464,7 @@ test("partial credential-policy rotation failure preserves the prior active gene state: "cleanup_pending", registration_generation: staged.generation, registration_claim: null, + rollback_policies_json: JSON.stringify(rollback), last_error: "simulated Durable Object registration failure", }, ); diff --git a/tests/sandbox-credential-policy-rollback.test.ts b/tests/sandbox-credential-policy-rollback.test.ts new file mode 100644 index 00000000..2b68ec6a --- /dev/null +++ b/tests/sandbox-credential-policy-rollback.test.ts @@ -0,0 +1,121 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { credentialPolicyRegistrationAccepted } from "../src/credential-policy-fence.ts"; +import { + captureSandboxCredentialPolicyRollback, + restoreSandboxCredentialPolicyRollback, +} from "../src/worker/sandbox-credential-policy-rollback.ts"; +import type { + SandboxCredentialPolicyRegistration, + StoredSandboxCredentialPolicy, +} from "../src/worker/session-control-policy.ts"; + +function storedPolicy( + lookupId: string, + generation: string, + claim: string, + registrationExpiresAt: number, +): StoredSandboxCredentialPolicy { + return { + generation, + registrationClaim: claim, + registrationExpiresAt, + policy: { + allowedHosts: [], + githubCredentialSource: "none", + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: lookupId, + sessionId: "IS-42", + }, + }; +} + +function policyStub(policies: Map) { + return { + async fetch(input: RequestInfo | URL, init?: RequestInit): Promise { + const url = new URL(String(input)); + const egress = url.pathname.match(/^\/api\/session-control\/egress\/([^/]+)$/); + if (egress && (!init?.method || init.method === "GET")) { + const current = policies.get(decodeURIComponent(egress[1] ?? "")); + return current + ? Response.json(current.policy, { + headers: { "x-crabfleet-policy-generation": current.generation }, + }) + : Response.json({ error: "not found" }, { status: 404 }); + } + if (url.pathname === "/api/session-control/register" && init?.method === "POST") { + const incoming = JSON.parse(String(init.body)) as StoredSandboxCredentialPolicy; + const current = policies.get(incoming.policy.sandboxId); + if (!credentialPolicyRegistrationAccepted(current, undefined, incoming, Date.now())) { + return Response.json({ error: "conflict" }, { status: 409 }); + } + policies.set(incoming.policy.sandboxId, incoming); + return Response.json({ ok: true }); + } + return Response.json({ error: "not found" }, { status: 404 }); + }, + }; +} + +test("partial policy generation writes restore every prior live lookup", async () => { + const now = Date.now(); + const lookupIds = ["sandbox-1", "do-1"]; + const policies = new Map( + lookupIds.map((lookupId) => [ + lookupId, + storedPolicy(lookupId, "generation:prior", "registration:prior", now + 1_000), + ]), + ); + const stub = policyStub(policies); + const rollback = await captureSandboxCredentialPolicyRollback( + stub, + lookupIds, + "generation:prior", + "IS-42", + ); + const registration: SandboxCredentialPolicyRegistration = { + generation: "generation:replacement", + claim: "registration:replacement", + lookupIds, + }; + const replacementExpiresAt = now + 60_000; + + policies.set( + "sandbox-1", + storedPolicy("sandbox-1", registration.generation, registration.claim, replacementExpiresAt), + ); + await restoreSandboxCredentialPolicyRollback( + stub, + registration, + replacementExpiresAt, + JSON.stringify(rollback), + "IS-42", + ); + + for (const current of policies.values()) { + assert.equal(current.generation, "generation:prior"); + assert.match(current.registrationClaim, /^rollback:registration:replacement$/); + assert.ok(current.registrationExpiresAt > replacementExpiresAt); + } +}); + +test("rollback snapshots reject incomplete prior generations before replacement", async () => { + const policies = new Map([ + [ + "sandbox-1", + storedPolicy("sandbox-1", "generation:prior", "registration:prior", Date.now() + 1_000), + ], + ]); + + await assert.rejects( + captureSandboxCredentialPolicyRollback( + policyStub(policies), + ["sandbox-1", "do-1"], + "generation:prior", + "IS-42", + ), + /rollback generation is incomplete/, + ); +}); From 4a3f93ee4b6291b4db7dec6751d37b4572e8aa39 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:46:10 +0200 Subject: [PATCH 079/242] fix(actions): negotiate framed runner io --- CHANGELOG.md | 2 +- README.md | 20 +++-- docs/api.md | 19 +++-- docs/architecture.md | 4 +- docs/github-actions-sessions.md | 63 +++++++++----- docs/spec.md | 11 +-- src/github-actions-runner.ts | 13 +++ src/github-actions-runtime.ts | 123 ++++++++++++++++++++++++++- src/worker/terminal-hub.ts | 57 +++++++++---- tests/github-actions-runner.test.ts | 22 ++++- tests/github-actions-runtime.test.ts | 98 +++++++++++++++++++-- tests/terminal-hub.test.ts | 116 ++++++++++++++++++++++++- 12 files changed, 477 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6be40c1..ac75f1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, rollback-restored Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. -- Change the GitHub Actions runner PTY contract to correlated `CFR1` binary input and acknowledgement frames, emit success only after the runner accepts the input into its PTY, preserve unframed runner output, and reject legacy raw-input clients. +- Add capability-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. - Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host. diff --git a/README.md b/README.md index 1cc362ea..bf3fabbe 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,15 @@ The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New reg const terminal = new WebSocket(runnerPtyUrl); terminal.binaryType = "arraybuffer"; -pty.onData((output) => terminal.send(output)); // Runner output stays raw. +terminal.onopen = () => { + terminal.send( + JSON.stringify({ + type: "crabfleet_runner_capabilities", + capabilities: ["cfr1-framed-io-v1"], + }), + ); +}; +pty.onData((output) => terminal.send(encodeCfr1Output(output))); terminal.onmessage = ({ data }) => { const input = decodeCfr1Input(data); if (!input) return; @@ -113,11 +121,11 @@ terminal.onmessage = ({ data }) => { }; ``` -Crabfleet sends viewer input in correlated binary `CFR1` frames. The runner must -return the matching binary acknowledgement only after its PTY accepts the -input. Runner terminal output remains unframed and raw. Legacy clients that -expect raw viewer input are incompatible; the complete encoder, decoder, and -Node runner example are in +Existing runners retain raw input and output. A runner opts into correlated +binary `CFR1` input, output, and acknowledgement frames by advertising the +`cfr1-framed-io-v1` capability immediately after connecting. Negotiated runners +acknowledge only after their PTY accepts the input; the complete encoder, +decoder, and Node runner example are in [`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. diff --git a/docs/api.md b/docs/api.md index acbdea93..1173d8fd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -598,17 +598,19 @@ Response: } ``` -Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers, but the runner must implement the framed input and acknowledgement protocol below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. +Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output; runners advertise `cfr1-framed-io-v1` to negotiate the framed contract below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. ### GET /api/agent/interactive-sessions/:id/runner-pty WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses the scoped `agentToken` query parameter embedded in `runnerPtyUrl`. One runner is current; a reconnect replaces the previous runner while browser viewers remain attached. -Runner output remains unframed: text and binary WebSocket messages are fanned -out as raw terminal output. Viewer input and relay control traffic use binary -`CFR1` frames so terminal text cannot be mistaken for an acknowledgement. -Legacy runners that expect raw viewer input are incompatible and their -unframed input is rejected. +Runner sockets begin in legacy mode with raw input and output. A runner +negotiates framed I/O by sending +`{"type":"crabfleet_runner_capabilities","capabilities":["cfr1-framed-io-v1"]}`. +The relay responds with the accepted capability. Negotiated input, output, +acknowledgements, and relay control traffic use binary `CFR1` frames. The relay +wraps legacy output before forwarding it to viewers, so arbitrary raw PTY bytes +cannot be consumed as control traffic. Each `CFR1` frame occupies one binary WebSocket message and starts with: @@ -626,6 +628,7 @@ Input IDs are nonempty ASCII `[A-Za-z0-9_-]` values of at most 80 bytes. | `0x01` input | relay to runner | raw terminal input bytes | | `0x02` acknowledgement | runner to relay | one byte: `1` accepted or `0` rejected, followed by optional UTF-8 error text | | `0x03` lifecycle event | relay to viewers | empty input ID and one event-code byte | +| `0x04` output | runner to relay | empty input ID followed by raw terminal output bytes | Lifecycle event codes are `0x01` runner connected, `0x02` runner disconnected, and `0x03` runner waiting. @@ -637,6 +640,10 @@ generates a rejected acknowledgement only when no current runner is available to receive the input frame or the relay send fails. Stale or mismatched acknowledgement IDs do not complete another pending input. +Before negotiation, the relay unwraps viewer input to raw bytes and reports +acceptance once the runner socket accepts the send. This preserves existing +runner integrations while negotiated runners provide PTY-level completion. + See [GitHub Actions Sessions](/github-actions-sessions/#runner-pty) for a complete Node runner integration. diff --git a/docs/architecture.md b/docs/architecture.md index fd383434..dcafcf4f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ D1 is canonical for product metadata: ### Durable Objects - `Sandbox` runs first-party Cloudflare Sandbox workspaces. -- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Runner output remains raw; viewer input, runner acknowledgements, and lifecycle events use correlated binary `CFR1` frames so control traffic cannot consume terminal text. +- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; capability-negotiated runners use correlated binary `CFR1` input, output, and acknowledgement frames. Viewer-bound output is always framed so terminal bytes cannot collide with control traffic. There is no `BoardDO` or `RunDO`. General Board/Fleet state is D1 plus REST polling. @@ -137,7 +137,7 @@ Interactive sessions are the live execution plane. Supported paths: - **Built-in Sandbox:** Worker provisions a Cloudflare Sandbox, prepares the repo, starts a Codex-capable shell, and proxies PTY traffic. - **Versioned runtime adapter:** Worker durably registers a tenant-namespaced workspace ID, creates and reconciles the provider workspace, proxies PTY access, mints transient desktop links, and confirms provider release before terminal state. -- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, receives correlated `CFR1` browser input, writes it to its PTY, and acknowledges the matching ID before Crabfleet reports input acceptance. +- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, and either retains legacy raw terminal traffic or negotiates correlated `CFR1` browser input/output. Negotiated runners acknowledge each PTY write before Crabfleet reports input acceptance. Sessions can carry a stable tenant owner, parent/root lineage, purpose, summary, named grants, public share state, delegated control, multiplayer mode, archive metadata, and runtime-specific capability state. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index a6be4d98..61baebe4 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -263,13 +263,14 @@ returns only the sanitized event. ## Runner PTY The Action connects outbound to the returned `runnerPtyUrl`. Node's global -`WebSocket` can open the URL without custom headers, but runner input is a -framed protocol rather than raw WebSocket bytes. +`WebSocket` can open the URL without custom headers. -Runner output remains unframed and raw. Viewer input arrives in a binary `CFR1` -frame carrying a correlation ID. The runner returns a binary acknowledgement -with the same ID only after its PTY accepts the input write. Legacy runners that -expect raw viewer input are incompatible. +Runner sockets begin in legacy mode with raw input and output. A runner opts +into collision-free framed I/O by advertising `cfr1-framed-io-v1` immediately +after connecting. Negotiated viewer input arrives in a binary `CFR1` frame +carrying a correlation ID; runner output uses a distinct `CFR1` output frame. +The runner returns a correlated acknowledgement only after its PTY accepts the +input write. Complete Node runner integration: @@ -293,6 +294,12 @@ await new Promise((resolve, reject) => { terminal.addEventListener("open", resolve, { once: true }); terminal.addEventListener("error", reject, { once: true }); }); +terminal.send( + JSON.stringify({ + type: "crabfleet_runner_capabilities", + capabilities: ["cfr1-framed-io-v1"], + }), +); const pty = spawn(process.env.SHELL || "/bin/bash", [], { cwd: process.cwd(), @@ -300,7 +307,7 @@ const pty = spawn(process.env.SHELL || "/bin/bash", [], { }); pty.onData((output) => { - terminal.send(output); // Terminal output stays raw and unframed. + terminal.send(encodeOutput(output)); }); terminal.addEventListener("message", (event) => { @@ -349,6 +356,16 @@ function encodeAck(inputId, accepted) { return frame; } +function encodeOutput(output) { + const payload = encoder.encode(output); + const frame = new Uint8Array(6 + payload.byteLength); + frame.set(magic); + frame[4] = 0x04; + frame[5] = 0; + frame.set(payload, 6); + return frame; +} + terminal.addEventListener("close", () => { pty.kill(); }); @@ -363,19 +380,23 @@ For a PTY API with an asynchronous write callback or promise, await that acceptance signal before sending `encodeAck(..., true)`. Do not acknowledge when the WebSocket merely queues the input frame. -Each `CFR1` frame occupies one binary WebSocket message: +The relay answers the advertisement with +`{"type":"crabfleet_runner_capabilities","accepted":["cfr1-framed-io-v1"]}`. +WebSocket ordering applies the negotiated mode before subsequent runner +messages. Each `CFR1` frame occupies one binary WebSocket message: -| Offset | Size | Value | -| ------ | -------- | --------------------------------------------------------------- | -| 0 | 4 | ASCII `CFR1` | -| 4 | 1 | `0x01` input, `0x02` acknowledgement, or `0x03` lifecycle event | -| 5 | 1 | input ID byte length | -| 6 | variable | input ID followed by the type-specific payload | +| Offset | Size | Value | +| ------ | -------- | ------------------------------------------------------------------------------ | +| 0 | 4 | ASCII `CFR1` | +| 4 | 1 | `0x01` input, `0x02` acknowledgement, `0x03` lifecycle event, or `0x04` output | +| 5 | 1 | input ID byte length | +| 6 | variable | input ID followed by the type-specific payload | Input payloads are raw terminal bytes. An acknowledgement payload starts with `1` for accepted or `0` for rejected and may include UTF-8 error text after the status byte. Lifecycle events use an empty input ID and event code `0x01` for runner connected, `0x02` for runner disconnected, or `0x03` for runner waiting. +Output uses an empty input ID followed by raw terminal bytes. The full wire contract is also specified in [API](/api/#get-api-agent-interactive-sessions-id-runner-pty). @@ -385,13 +406,15 @@ Properties: - Only one runner is current. - A new runner connection replaces the previous runner. - Multiple browser viewers may remain connected. -- Unframed runner output is fanned out to viewers unchanged. -- Writable viewer input is framed and sent to the current runner only. -- A viewer sees `input-accepted` only after the correlated runner - acknowledgement. +- Legacy runners receive raw viewer input and send raw output. +- Negotiated runners receive framed input and must wrap every output payload in + a `0x04` frame. +- Negotiated input produces `input-accepted` only after the correlated runner + acknowledgement. Legacy input reports acceptance after relay delivery. - Runner lifecycle events are typed binary frames even while no runner is connected. -- Unframed viewer input is rejected. +- The relay frames legacy output internally, so raw bytes beginning with + `CFR1` cannot collide with acknowledgements or lifecycle events. The relay does not interpret Codex JSON-RPC. The runner-side integration decides how accepted terminal input maps to model steering. @@ -430,7 +453,7 @@ instead of inventing a local shell. ## Steering Semantics -Crabfleet forwards terminal input inside correlated `CFR1` frames. In the +Crabfleet forwards negotiated terminal input inside correlated `CFR1` frames. In the ClawSweeper integration, the runner: 1. Accepts the framed bytes into its input handler and acknowledges that input diff --git a/docs/spec.md b/docs/spec.md index 67c5c3bd..ab6ab2db 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -178,15 +178,16 @@ Crabfleet owns: - session identity and metadata; - rotating scoped agent token; -- outbound runner relay through `SessionControlDO`, with raw runner output and correlated binary `CFR1` input, acknowledgement, and lifecycle frames; +- outbound runner relay through `SessionControlDO`, preserving legacy raw runner traffic while capability-negotiated runners use correlated binary `CFR1` input, output, acknowledgement, and lifecycle frames; - browser terminal steering; - work-state heartbeats; - event and transcript finalization. -The Action remains the execution host and mutation authority. It acknowledges -viewer input only after its PTY accepts the correlated write; relay queueing is -not acceptance. Legacy raw-input runners are incompatible. Ending the Crabfleet -session does not cancel the workflow run. +The Action remains the execution host and mutation authority. A negotiated +runner acknowledges viewer input only after its PTY accepts the correlated +write; relay queueing is not acceptance. Legacy runners keep raw input/output +and relay-level delivery reporting. Ending the Crabfleet session does not +cancel the workflow run. ## Session Lifecycle diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 95c5717d..c46e7c0d 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -1,9 +1,22 @@ import { + encodeGitHubActionsRelayOutput, + encodeGitHubActionsRunnerCapabilities, parseGitHubActionsRelayInput, sendGitHubActionsRelayInputAcknowledgement, type GitHubActionsRelaySocket, } from "./github-actions-runtime.ts"; +export function negotiateGitHubActionsRunnerProtocol(socket: GitHubActionsRelaySocket): void { + socket.send(encodeGitHubActionsRunnerCapabilities()); +} + +export function sendGitHubActionsRunnerOutput( + socket: GitHubActionsRelaySocket, + output: string | ArrayBuffer | ArrayBufferView, +): void { + socket.send(encodeGitHubActionsRelayOutput(output)); +} + export async function acceptGitHubActionsRunnerInput( socket: GitHubActionsRelaySocket, message: string | ArrayBuffer, diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 26f6d8d0..db0a9b72 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -14,6 +14,8 @@ export type GitHubActionsRelaySocket = { readyState: number; send(message: string | ArrayBuffer): void; close(code?: number, reason?: string): void; + serializeAttachment?(attachment: unknown): void; + deserializeAttachment?(): unknown; }; export type GitHubActionsRelayInputAcknowledgement = { @@ -27,6 +29,8 @@ export type GitHubActionsRelayInput = { payload: ArrayBuffer; }; +export const githubActionsFramedRunnerCapability = "cfr1-framed-io-v1"; + export const githubActionsCapabilities = { terminal: true, takeover: true, @@ -59,6 +63,7 @@ const relayFrameHeaderBytes = relayFrameMagic.byteLength + 2; const relayInputFrameType = 1; const relayInputAcknowledgementFrameType = 2; const relayEventFrameType = 3; +const relayOutputFrameType = 4; const relayInputIdMaximumBytes = 80; const relayInputIdPattern = /^[A-Za-z0-9_-]+$/; const relayEventCodes = { @@ -74,6 +79,11 @@ const relayEvents = new Map( ); const encoder = new TextEncoder(); const decoder = new TextDecoder(); +const runnerCapabilitiesMessageType = "crabfleet_runner_capabilities"; + +type GitHubActionsRunnerAttachment = { + protocol?: typeof githubActionsFramedRunnerCapability; +}; export function githubActionsRuntimeLabel(runtime: unknown): string { return runtime === githubActionsRuntime ? "GitHub Actions" : ""; @@ -170,17 +180,65 @@ export function relayGitHubActionsWebSocketMessage( if (isGitHubActionsViewerControlMessage(message)) return 0; const input = parseGitHubActionsRelayInput(message); if (!input) return 0; - const forwarded = forwardGitHubActionsRelayMessage(sender, message, runners, viewers); - if (forwarded !== 1) { + const runner = runners.find((socket) => socket.readyState === webSocketOpen); + if (!runner) { + sendGitHubActionsRelayInputAcknowledgement(senderSocket, { + inputId: input.inputId, + accepted: false, + }); + return 0; + } + const framed = gitHubActionsRunnerUsesFramedProtocol(runner); + try { + runner.send(framed ? message : input.payload); + if (!framed) { + sendGitHubActionsRelayInputAcknowledgement(senderSocket, { + inputId: input.inputId, + accepted: true, + }); + } + return 1; + } catch { sendGitHubActionsRelayInputAcknowledgement(senderSocket, { inputId: input.inputId, accepted: false, }); + return 0; } - return forwarded; } - return forwardGitHubActionsRelayMessage(sender, message, runners, viewers); + const capabilities = parseGitHubActionsRunnerCapabilities(message); + if (capabilities) { + if (capabilities.includes(githubActionsFramedRunnerCapability)) { + senderSocket.serializeAttachment?.({ + protocol: githubActionsFramedRunnerCapability, + } satisfies GitHubActionsRunnerAttachment); + sendGitHubActionsRunnerCapabilitiesAccepted(senderSocket, [ + githubActionsFramedRunnerCapability, + ]); + } else { + senderSocket.serializeAttachment?.({} satisfies GitHubActionsRunnerAttachment); + sendGitHubActionsRunnerCapabilitiesAccepted(senderSocket, []); + } + return 0; + } + + if (gitHubActionsRunnerUsesFramedProtocol(senderSocket)) { + if ( + !parseGitHubActionsRelayInputAcknowledgement(message) && + !parseGitHubActionsRelayOutput(message) + ) { + return 0; + } + return forwardGitHubActionsRelayMessage(sender, message, runners, viewers); + } + + return forwardGitHubActionsRelayMessage( + sender, + encodeGitHubActionsRelayOutput(message), + runners, + viewers, + ); } export function sendGitHubActionsRelayInputAcknowledgement( @@ -230,6 +288,18 @@ export function encodeGitHubActionsRelayInput( return encodeGitHubActionsRelayFrame(relayInputFrameType, inputId, messageBytes(payload)); } +export function encodeGitHubActionsRelayOutput( + payload: string | ArrayBuffer | ArrayBufferView, +): ArrayBuffer { + return encodeGitHubActionsRelayFrame(relayOutputFrameType, "", messageBytes(payload)); +} + +export function parseGitHubActionsRelayOutput(message: string | ArrayBuffer): ArrayBuffer | null { + const frame = decodeGitHubActionsRelayFrame(message, relayOutputFrameType); + if (!frame || frame.inputId) return null; + return Uint8Array.from(frame.payload).buffer; +} + export function parseGitHubActionsRelayInput( message: string | ArrayBuffer, ): GitHubActionsRelayInput | null { @@ -241,6 +311,30 @@ export function parseGitHubActionsRelayInput( }; } +export function encodeGitHubActionsRunnerCapabilities(): string { + return JSON.stringify({ + type: runnerCapabilitiesMessageType, + capabilities: [githubActionsFramedRunnerCapability], + }); +} + +export function parseGitHubActionsRunnerCapabilities( + message: string | ArrayBuffer, +): string[] | null { + if (typeof message !== "string") return null; + try { + const parsed = JSON.parse(message) as Record; + if (parsed.type !== runnerCapabilitiesMessageType || !Array.isArray(parsed.capabilities)) { + return null; + } + return parsed.capabilities.filter((capability): capability is string => { + return typeof capability === "string"; + }); + } catch { + return null; + } +} + export function encodeGitHubActionsRelayInputAcknowledgement( acknowledgement: GitHubActionsRelayInputAcknowledgement, ): ArrayBuffer { @@ -360,3 +454,24 @@ function messageBytes(message: string | ArrayBuffer | ArrayBufferView): Uint8Arr function requireGitHubActionsRelayInputId(inputId: string): void { if (!inputId) throw new Error("invalid GitHub Actions relay input id"); } + +function gitHubActionsRunnerUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { + const attachment = socket.deserializeAttachment?.(); + if (!attachment || typeof attachment !== "object") return false; + return ( + (attachment as GitHubActionsRunnerAttachment).protocol === githubActionsFramedRunnerCapability + ); +} + +function sendGitHubActionsRunnerCapabilitiesAccepted( + socket: GitHubActionsRelaySocket, + capabilities: string[], +): void { + if (socket.readyState !== webSocketOpen) return; + socket.send( + JSON.stringify({ + type: runnerCapabilitiesMessageType, + accepted: capabilities, + }), + ); +} diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 6557962f..048ee982 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -18,6 +18,7 @@ import { encodeGitHubActionsRelayInput, githubActionsRuntime, parseGitHubActionsRelayInputAcknowledgement, + parseGitHubActionsRelayOutput, parseGitHubActionsRelayEvent, type GitHubActionsRelayInputAcknowledgement, } from "../github-actions-runtime.ts"; @@ -283,21 +284,11 @@ export class TerminalHub { } } } - if (acknowledgements.length > 0) { - const results = await Promise.all( - acknowledgements.map((acknowledgement) => acknowledgement.promise), - ); - const rejection = results.find((result) => !result.accepted); - if (rejection) { - sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: rejection.error ?? "terminal input was not accepted", - }); - return; - } - } - sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { - type: "input-accepted", - }); + reportTerminalInputCompletion( + server, + frame.sessionId, + acknowledgements.map((acknowledgement) => acknowledgement.promise), + ); return; } if (frame.type === TerminalMessageType.Resize) { @@ -529,6 +520,16 @@ export class TerminalHub { sendTerminalJson(client, TerminalMessageType.Event, id, relayEvent); return; } + const relayOutput = parseGitHubActionsRelayOutput(data); + if (!relayOutput) return; + const output = new Uint8Array(relayOutput); + sendTerminalFrame(client, TerminalMessageType.Output, id, output); + if (activeSubscription.outputAcknowledgements) { + activeSubscription.outputAcknowledgementBytes += output.byteLength; + } else if (upstreamConnection.outputAcknowledgements) { + sendOutputAcknowledgement(upstream, output.byteLength); + } + return; } if (typeof data === "string") { if (!activeSubscription.inputAcknowledgements) { @@ -673,6 +674,32 @@ function completeAllTerminalInputAcknowledgements( return pending.length; } +function reportTerminalInputCompletion( + socket: WebSocket, + sessionId: string, + acknowledgements: Promise[], +): void { + if (acknowledgements.length === 0) { + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-accepted", + }); + return; + } + void Promise.all(acknowledgements).then((results) => { + if (socket.readyState !== WebSocket.OPEN) return; + const rejection = results.find((result) => !result.accepted); + if (rejection) { + sendTerminalJson(socket, TerminalMessageType.Error, sessionId, { + error: rejection.error ?? "terminal input was not accepted", + }); + return; + } + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-accepted", + }); + }); +} + function updateTerminalInputCapability( socket: WebSocket, subscription: TerminalHubSubscription, diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index d214d593..439bb5b5 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -1,9 +1,15 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { acceptGitHubActionsRunnerInput } from "../src/github-actions-runner.ts"; +import { + acceptGitHubActionsRunnerInput, + negotiateGitHubActionsRunnerProtocol, + sendGitHubActionsRunnerOutput, +} from "../src/github-actions-runner.ts"; import { encodeGitHubActionsRelayInput, + parseGitHubActionsRelayOutput, + parseGitHubActionsRunnerCapabilities, parseGitHubActionsRelayInputAcknowledgement, type GitHubActionsRelaySocket, } from "../src/github-actions-runtime.ts"; @@ -65,3 +71,17 @@ test("runner rejects failed writes and ignores unframed terminal data", async () assert.equal(await acceptGitHubActionsRunnerInput(socket, "raw output", async () => {}), false); assert.equal(socket.sent.length, 1); }); + +test("runner helpers negotiate framed IO and envelope PTY output", () => { + const socket = relaySocket(); + + negotiateGitHubActionsRunnerProtocol(socket); + assert.deepEqual(parseGitHubActionsRunnerCapabilities(socket.sent[0]!), ["cfr1-framed-io-v1"]); + + const collision = encodeGitHubActionsRelayInput("looks-like-input", "terminal bytes"); + sendGitHubActionsRunnerOutput(socket, collision); + assert.deepEqual( + new Uint8Array(parseGitHubActionsRelayOutput(socket.sent[1]!)!), + new Uint8Array(collision), + ); +}); diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 13f6a692..5107b2e8 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -4,7 +4,8 @@ import { buildGitHubActionsRunnerPtyUrl, encodeGitHubActionsRelayInput, encodeGitHubActionsRelayInputAcknowledgement, - forwardGitHubActionsRelayMessage, + encodeGitHubActionsRelayOutput, + encodeGitHubActionsRunnerCapabilities, gitHubActionsSessionStatus, githubActionsCapabilities, githubActionsRelayRole, @@ -15,6 +16,7 @@ import { parseGitHubActionsRelayEvent, parseGitHubActionsRelayInput, parseGitHubActionsRelayInputAcknowledgement, + parseGitHubActionsRelayOutput, parseGitHubActionsWorkState, relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, @@ -23,11 +25,13 @@ import { } from "../src/github-actions-runtime.ts"; function relaySocket(readyState = 1): GitHubActionsRelaySocket & { + attachment: unknown; closed: Array<[number | undefined, string | undefined]>; sent: Array; } { return { readyState, + attachment: null, sent: [], closed: [], send(message) { @@ -37,6 +41,12 @@ function relaySocket(readyState = 1): GitHubActionsRelaySocket & { this.closed.push([code, reason]); this.readyState = 3; }, + serializeAttachment(attachment) { + this.attachment = attachment; + }, + deserializeAttachment() { + return this.attachment; + }, }; } @@ -71,7 +81,7 @@ test("work states preserve running phases and map terminal outcomes", () => { assert.equal(gitHubActionsSessionStatus("failed"), "failed"); }); -test("relay replaces the current runner and fans out raw runner output", () => { +test("relay replaces the current runner and frames legacy raw runner output", () => { const oldRunner = relaySocket(); const runner = relaySocket(); const viewerOne = relaySocket(); @@ -84,20 +94,56 @@ test("relay replaces the current runner and fans out raw runner output", () => { assert.deepEqual(stoppedRunner.closed, [[1000, "runner disconnected"]]); assert.equal( - forwardGitHubActionsRelayMessage("runner", "output", [runner], [viewerOne, viewerTwo]), + relayGitHubActionsWebSocketMessage( + "runner", + runner, + "output", + [runner], + [viewerOne, viewerTwo], + ), 2, ); - assert.deepEqual(viewerOne.sent, ["output"]); - assert.deepEqual(viewerTwo.sent, ["output"]); + assert.equal( + new TextDecoder().decode(parseGitHubActionsRelayOutput(viewerOne.sent[0]!)!), + "output", + ); + assert.equal( + new TextDecoder().decode(parseGitHubActionsRelayOutput(viewerTwo.sent[0]!)!), + "output", + ); }); -test("relay sends framed viewer input to the first open runner without acknowledging queueing", () => { +test("legacy runners receive raw input and the relay acknowledges delivery", () => { + const runner = relaySocket(); + const viewer = relaySocket(); + const input = encodeGitHubActionsRelayInput("input-legacy", "steer"); + + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, input, [runner], []), 1); + assert.equal(new TextDecoder().decode(runner.sent[0] as ArrayBuffer), "steer"); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + inputId: "input-legacy", + accepted: true, + }); +}); + +test("negotiated runners receive framed input without an early acknowledgement", () => { const closedRunner = relaySocket(3); const openRunner = relaySocket(); const laterRunner = relaySocket(); const viewer = relaySocket(); const input = encodeGitHubActionsRelayInput("input-one", "steer"); + assert.equal( + relayGitHubActionsWebSocketMessage( + "runner", + openRunner, + encodeGitHubActionsRunnerCapabilities(), + [openRunner], + [], + ), + 0, + ); + openRunner.sent.length = 0; assert.equal( relayGitHubActionsWebSocketMessage( "viewer", @@ -150,6 +196,14 @@ test("runner acknowledgements retain correlation and fan out to viewers", () => inputId: "input-two", accepted: true, }); + relayGitHubActionsWebSocketMessage( + "runner", + runner, + encodeGitHubActionsRunnerCapabilities(), + [runner], + [], + ); + runner.sent.length = 0; assert.equal( relayGitHubActionsWebSocketMessage( @@ -168,6 +222,33 @@ test("runner acknowledgements retain correlation and fan out to viewers", () => assert.deepEqual(viewerTwo.sent, [acknowledgement]); }); +test("negotiated runners frame output so control-shaped terminal bytes stay output", () => { + const runner = relaySocket(); + const viewer = relaySocket(); + const controlShapedOutput = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "collision", + accepted: true, + }); + + assert.equal( + relayGitHubActionsWebSocketMessage( + "runner", + runner, + encodeGitHubActionsRunnerCapabilities(), + [runner], + [viewer], + ), + 0, + ); + assert.equal(typeof runner.sent[0], "string"); + const output = encodeGitHubActionsRelayOutput(controlShapedOutput); + assert.equal(relayGitHubActionsWebSocketMessage("runner", runner, output, [runner], [viewer]), 1); + assert.deepEqual( + new Uint8Array(parseGitHubActionsRelayOutput(viewer.sent[0]!)!), + new Uint8Array(controlShapedOutput), + ); +}); + test("typed acknowledgements reject malformed ids and preserve collision-shaped terminal text", () => { const viewer = relaySocket(); const runner = relaySocket(); @@ -178,7 +259,10 @@ test("typed acknowledgements reject malformed ids and preserve collision-shaped relayGitHubActionsWebSocketMessage("runner", runner, collision, [runner], [viewer]), 1, ); - assert.deepEqual(viewer.sent, [collision]); + assert.equal( + new TextDecoder().decode(parseGitHubActionsRelayOutput(viewer.sent[0]!)!), + collision, + ); assert.throws(() => encodeGitHubActionsRelayInput("bad id", "input"), { message: "invalid GitHub Actions relay input id", }); diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 682afd02..5965842a 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -12,6 +12,7 @@ import { } from "@openclaw/libterminal/protocol"; import { encodeGitHubActionsRelayInputAcknowledgement, + encodeGitHubActionsRelayOutput, parseGitHubActionsRelayInput, } from "../src/github-actions-runtime.ts"; import type { User } from "../src/worker/models.ts"; @@ -687,7 +688,7 @@ test("GitHub Actions input acknowledgements correlate overlapping payloads out o server.emit("close"); }); -test("GitHub Actions collision-shaped terminal text remains raw output", async () => { +test("GitHub Actions framed output preserves control-shaped terminal bytes", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -714,13 +715,120 @@ test("GitHub Actions collision-shaped terminal text remains raw output", async ( await flushQueues(); await flushQueues(); - const collision = '{"type":"runner_waiting","inputId":"stale-input-id","accepted":true}'; - upstream.emit("message", { data: collision }); + const collision = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "stale-input-id", + accepted: true, + }); + upstream.emit("message", { data: encodeGitHubActionsRelayOutput(collision) }); await flushQueues(); const output = frame(server.sent.at(-1)!); assert.equal(output.type, TerminalMessageType.Output); - assert.equal(new TextDecoder().decode(output.payload), collision); + assert.deepEqual(output.payload, new Uint8Array(collision)); + server.emit("close"); +}); + +test("a stalled GitHub Actions acknowledgement does not block other sessions or ping", async () => { + const client = socket(); + const server = socket(); + const firstUpstream = socket(); + const secondUpstream = socket(); + const secondSession = interactiveSession( + sessionRow({ + id: "IS-actions-second", + adapter: null, + adapter_workspace_id: null, + capabilities_json: JSON.stringify(containerCapabilities), + runtime: "github_actions", + status: "ready", + }), + ); + const hub = new TerminalHub( + dependencies(client, server, firstUpstream, { + async readSession(_request, _user, id) { + return id === secondSession.id ? secondSession : githubActionsSession; + }, + async openUpstream(_request, _user, selectedSession) { + return { + socket: selectedSession.id === secondSession.id ? secondUpstream : firstUpstream, + outputAcknowledgements: true, + async markConnected() {}, + }; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + for (const selectedSession of [githubActionsSession, secondSession]) { + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: selectedSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + } + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("stalled"), + }), + }); + await flushQueues(); + assert.equal(relayInput(firstUpstream.sent[0]!).text, "stalled"); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Ping, + sessionId: "", + payload: new TextEncoder().encode("still-live"), + }), + }); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: secondSession.id, + payload: new TextEncoder().encode("independent"), + }), + }); + await flushQueues(); + await flushQueues(); + + assert.equal(relayInput(secondUpstream.sent[0]!).text, "independent"); + assert.equal( + server.sent.some((payload) => { + const message = frame(payload); + return ( + message.type === TerminalMessageType.Pong && + new TextDecoder().decode(message.payload) === "still-live" + ); + }), + true, + ); + + const secondInput = relayInput(secondUpstream.sent[0]!); + emitRelayAcknowledgement(secondUpstream, secondInput.inputId, true); + await flushQueues(); + assert.equal( + server.sent.some((payload) => { + const message = frame(payload); + return ( + message.type === TerminalMessageType.Event && + message.sessionId === secondSession.id && + (decodeJsonPayload(message.payload) as { type?: string }).type === "input-accepted" + ); + }), + true, + ); + server.emit("close"); }); From 535c8fed323edd7289d24e9d91f53a301e3d0c62 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:47:50 +0200 Subject: [PATCH 080/242] fix(actions): fence runner protocol negotiation --- README.md | 21 ++++++++++---- docs/api.md | 4 ++- docs/github-actions-sessions.md | 45 +++++++++++++++++++++++------ docs/spec.md | 4 ++- src/github-actions-runner.ts | 32 ++++++++++++++++++-- src/github-actions-runtime.ts | 17 +++++++++++ tests/github-actions-runner.test.ts | 31 +++++++++++++++++--- 7 files changed, 131 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index bf3fabbe..51664ec5 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New reg ```js const terminal = new WebSocket(runnerPtyUrl); terminal.binaryType = "arraybuffer"; +let framed = false; terminal.onopen = () => { terminal.send( @@ -108,10 +109,17 @@ terminal.onopen = () => { }), ); }; -pty.onData((output) => terminal.send(encodeCfr1Output(output))); +pty.onData((output) => terminal.send(framed ? encodeCfr1Output(output) : output)); terminal.onmessage = ({ data }) => { + if (isAcceptedCapabilities(data)) { + framed = true; + return; + } const input = decodeCfr1Input(data); - if (!input) return; + if (!input) { + if (!framed) pty.write(typeof data === "string" ? data : new TextDecoder().decode(data)); + return; + } try { pty.write(new TextDecoder().decode(input.payload)); terminal.send(encodeCfr1Ack(input.inputId, true)); @@ -121,11 +129,12 @@ terminal.onmessage = ({ data }) => { }; ``` -Existing runners retain raw input and output. A runner opts into correlated +Existing runners retain raw input and output. A runner requests correlated binary `CFR1` input, output, and acknowledgement frames by advertising the -`cfr1-framed-io-v1` capability immediately after connecting. Negotiated runners -acknowledge only after their PTY accepts the input; the complete encoder, -decoder, and Node runner example are in +`cfr1-framed-io-v1` capability immediately after connecting. It keeps accepting +and sending raw traffic until the relay confirms that capability. Negotiated +runners acknowledge only after their PTY accepts the input; the complete +encoder, decoder, and Node runner example are in [`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. diff --git a/docs/api.md b/docs/api.md index 1173d8fd..34ffdbff 100644 --- a/docs/api.md +++ b/docs/api.md @@ -607,7 +607,9 @@ WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses t Runner sockets begin in legacy mode with raw input and output. A runner negotiates framed I/O by sending `{"type":"crabfleet_runner_capabilities","capabilities":["cfr1-framed-io-v1"]}`. -The relay responds with the accepted capability. Negotiated input, output, +The relay responds with the accepted capability. The runner must keep accepting +raw input and sending raw output until that response arrives because viewer +input can race the advertisement on another socket. Negotiated input, output, acknowledgements, and relay control traffic use binary `CFR1` frames. The relay wraps legacy output before forwarding it to viewers, so arbitrary raw PTY bytes cannot be consumed as control traffic. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 61baebe4..be6f71e6 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -265,9 +265,10 @@ returns only the sanitized event. The Action connects outbound to the returned `runnerPtyUrl`. Node's global `WebSocket` can open the URL without custom headers. -Runner sockets begin in legacy mode with raw input and output. A runner opts -into collision-free framed I/O by advertising `cfr1-framed-io-v1` immediately -after connecting. Negotiated viewer input arrives in a binary `CFR1` frame +Runner sockets begin in legacy mode with raw input and output. A runner requests +collision-free framed I/O by advertising `cfr1-framed-io-v1` immediately after +connecting, but it remains in legacy mode until the relay accepts the +capability. Negotiated viewer input arrives in a binary `CFR1` frame carrying a correlation ID; runner output uses a distinct `CFR1` output frame. The runner returns a correlated acknowledgement only after its PTY accepts the input write. @@ -289,6 +290,7 @@ const decoder = new TextDecoder(); const encoder = new TextEncoder(); const terminal = new WebSocket(runnerPtyUrl); terminal.binaryType = "arraybuffer"; +let framed = false; await new Promise((resolve, reject) => { terminal.addEventListener("open", resolve, { once: true }); @@ -307,16 +309,25 @@ const pty = spawn(process.env.SHELL || "/bin/bash", [], { }); pty.onData((output) => { - terminal.send(encodeOutput(output)); + terminal.send(framed ? encodeOutput(output) : output); }); terminal.addEventListener("message", (event) => { + if (acceptCapabilities(event.data)) { + framed = true; + return; + } acceptInput(event.data); }); function acceptInput(data) { const input = decodeInput(data); - if (!input) return; + if (!input) { + if (!framed) { + pty.write(typeof data === "string" ? data : decoder.decode(data)); + } + return; + } try { // A successful node-pty write is this adapter's PTY acceptance point. pty.write(decoder.decode(input.payload)); @@ -326,6 +337,20 @@ function acceptInput(data) { } } +function acceptCapabilities(data) { + if (typeof data !== "string") return false; + try { + const message = JSON.parse(data); + return ( + message.type === "crabfleet_runner_capabilities" && + Array.isArray(message.accepted) && + message.accepted.includes("cfr1-framed-io-v1") + ); + } catch { + return false; + } +} + function decodeInput(data) { if (!(data instanceof ArrayBuffer)) return null; const frame = new Uint8Array(data); @@ -382,8 +407,10 @@ the WebSocket merely queues the input frame. The relay answers the advertisement with `{"type":"crabfleet_runner_capabilities","accepted":["cfr1-framed-io-v1"]}`. -WebSocket ordering applies the negotiated mode before subsequent runner -messages. Each `CFR1` frame occupies one binary WebSocket message: +The runner must continue accepting raw input and sending raw output until that +acceptance arrives. Viewer input can race the advertisement because it comes +from another socket; messages already sent in legacy mode arrive before the +acceptance response. Each `CFR1` frame occupies one binary WebSocket message: | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------ | @@ -407,8 +434,8 @@ Properties: - A new runner connection replaces the previous runner. - Multiple browser viewers may remain connected. - Legacy runners receive raw viewer input and send raw output. -- Negotiated runners receive framed input and must wrap every output payload in - a `0x04` frame. +- Runners switch to framed input and `0x04` output only after receiving the + capability acceptance response. - Negotiated input produces `input-accepted` only after the correlated runner acknowledgement. Legacy input reports acceptance after relay delivery. - Runner lifecycle events are typed binary frames even while no runner is diff --git a/docs/spec.md b/docs/spec.md index ab6ab2db..651e02d6 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -186,7 +186,9 @@ Crabfleet owns: The Action remains the execution host and mutation authority. A negotiated runner acknowledges viewer input only after its PTY accepts the correlated write; relay queueing is not acceptance. Legacy runners keep raw input/output -and relay-level delivery reporting. Ending the Crabfleet session does not +and relay-level delivery reporting. A runner remains in that legacy mode until +the relay confirms its framed-I/O capability, so input racing negotiation is +not dropped. Ending the Crabfleet session does not cancel the workflow run. ## Session Lifecycle diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index c46e7c0d..9ae8d6cb 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -1,6 +1,7 @@ import { encodeGitHubActionsRelayOutput, encodeGitHubActionsRunnerCapabilities, + parseGitHubActionsRunnerCapabilitiesAccepted, parseGitHubActionsRelayInput, sendGitHubActionsRelayInputAcknowledgement, type GitHubActionsRelaySocket, @@ -13,8 +14,23 @@ export function negotiateGitHubActionsRunnerProtocol(socket: GitHubActionsRelayS export function sendGitHubActionsRunnerOutput( socket: GitHubActionsRelaySocket, output: string | ArrayBuffer | ArrayBufferView, + framed = true, ): void { - socket.send(encodeGitHubActionsRelayOutput(output)); + if (framed) { + socket.send(encodeGitHubActionsRelayOutput(output)); + return; + } + socket.send( + typeof output === "string" || output instanceof ArrayBuffer + ? output + : Uint8Array.from(new Uint8Array(output.buffer, output.byteOffset, output.byteLength)).buffer, + ); +} + +export function gitHubActionsRunnerProtocolAccepted(message: string | ArrayBuffer): boolean { + return ( + parseGitHubActionsRunnerCapabilitiesAccepted(message)?.includes("cfr1-framed-io-v1") ?? false + ); } export async function acceptGitHubActionsRunnerInput( @@ -23,7 +39,19 @@ export async function acceptGitHubActionsRunnerInput( writeToPty: (payload: ArrayBuffer) => void | Promise, ): Promise { const input = parseGitHubActionsRelayInput(message); - if (!input) return false; + if (!input) { + if (parseGitHubActionsRunnerCapabilitiesAccepted(message)) return false; + try { + const payload = + typeof message === "string" + ? Uint8Array.from(new TextEncoder().encode(message)).buffer + : message; + await writeToPty(payload); + } catch { + // Legacy delivery is acknowledged by the relay after the WebSocket send. + } + return true; + } try { await writeToPty(input.payload); sendGitHubActionsRelayInputAcknowledgement(socket, { diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index db0a9b72..04baa0fb 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -335,6 +335,23 @@ export function parseGitHubActionsRunnerCapabilities( } } +export function parseGitHubActionsRunnerCapabilitiesAccepted( + message: string | ArrayBuffer, +): string[] | null { + if (typeof message !== "string") return null; + try { + const parsed = JSON.parse(message) as Record; + if (parsed.type !== runnerCapabilitiesMessageType || !Array.isArray(parsed.accepted)) { + return null; + } + return parsed.accepted.filter((capability): capability is string => { + return typeof capability === "string"; + }); + } catch { + return null; + } +} + export function encodeGitHubActionsRelayInputAcknowledgement( acknowledgement: GitHubActionsRelayInputAcknowledgement, ): ArrayBuffer { diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 439bb5b5..3ac5b605 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -3,6 +3,7 @@ import { test } from "node:test"; import { acceptGitHubActionsRunnerInput, + gitHubActionsRunnerProtocolAccepted, negotiateGitHubActionsRunnerProtocol, sendGitHubActionsRunnerOutput, } from "../src/github-actions-runner.ts"; @@ -50,7 +51,7 @@ test("runner acknowledges input only after the PTY write completes", async () => }); }); -test("runner rejects failed writes and ignores unframed terminal data", async () => { +test("runner rejects failed framed writes and accepts legacy input during negotiation", async () => { const socket = relaySocket(); assert.equal( @@ -68,20 +69,42 @@ test("runner rejects failed writes and ignores unframed terminal data", async () accepted: false, error: "GitHub Actions runner did not accept terminal input", }); - assert.equal(await acceptGitHubActionsRunnerInput(socket, "raw output", async () => {}), false); + let legacyInput = ""; + assert.equal( + await acceptGitHubActionsRunnerInput(socket, "raw input", async (payload) => { + legacyInput = new TextDecoder().decode(payload); + }), + true, + ); + assert.equal(legacyInput, "raw input"); assert.equal(socket.sent.length, 1); }); -test("runner helpers negotiate framed IO and envelope PTY output", () => { +test("runner helpers stay raw until negotiation is accepted, then envelope PTY output", async () => { const socket = relaySocket(); negotiateGitHubActionsRunnerProtocol(socket); assert.deepEqual(parseGitHubActionsRunnerCapabilities(socket.sent[0]!), ["cfr1-framed-io-v1"]); + sendGitHubActionsRunnerOutput(socket, "early", false); + assert.equal(socket.sent[1], "early"); + + const accepted = JSON.stringify({ + type: "crabfleet_runner_capabilities", + accepted: ["cfr1-framed-io-v1"], + }); + assert.equal(gitHubActionsRunnerProtocolAccepted(accepted), true); + assert.equal( + await acceptGitHubActionsRunnerInput(socket, accepted, async () => { + assert.fail("capability acceptance must not reach the PTY"); + }), + false, + ); + const collision = encodeGitHubActionsRelayInput("looks-like-input", "terminal bytes"); sendGitHubActionsRunnerOutput(socket, collision); assert.deepEqual( - new Uint8Array(parseGitHubActionsRelayOutput(socket.sent[1]!)!), + new Uint8Array(parseGitHubActionsRelayOutput(socket.sent[2]!)!), new Uint8Array(collision), ); }); From af57733eb1122dc6e8923e263de44a58cc08f278 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:53:22 +0200 Subject: [PATCH 081/242] fix(vnc): accept late fence capability responses --- .../SDK/Connection/VNCConnection+API.swift | 11 ++++- .../SDK/Connection/VNCConnection.swift | 1 + .../RoyalVNCKitTests/AuditFindingsTests.swift | 46 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index e1f33819..d602886b 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -319,7 +319,10 @@ extension VNCConnection { framebufferRequestLock.unlock() return } - pixelFormatFenceCapabilityProbePayload = nil + if probeTimedOut { + expiredPixelFormatFenceCapabilityProbePayload = pixelFormatFenceCapabilityProbePayload + pixelFormatFenceCapabilityProbePayload = nil + } let isWaitingForLegacyFramebufferBoundary = negotiationTimedOut && !state.areContinuousUpdatesEnabled @@ -345,8 +348,11 @@ extension VNCConnection { func completePixelFormatFence(_ fence: VNCProtocol.ServerFence) throws { framebufferRequestLock.lock() - if fence.payload == pixelFormatFenceCapabilityProbePayload { + if fence.payload == pixelFormatFenceCapabilityProbePayload + || fence.payload == expiredPixelFormatFenceCapabilityProbePayload { + cancelPixelFormatFenceNegotiationTimeoutLocked() pixelFormatFenceCapabilityProbePayload = nil + expiredPixelFormatFenceCapabilityProbePayload = nil state.pixelFormatTransitionFenceFlags = fence.flags.intersection([ .blockBefore, .blockAfter, @@ -661,6 +667,7 @@ extension VNCConnection { pixelFormatTransitionFenceWasSent = false cancelPixelFormatTransitionDeadlineLocked() pixelFormatFenceCapabilityProbePayload = nil + expiredPixelFormatFenceCapabilityProbePayload = nil cancelPixelFormatFenceNegotiationTimeoutLocked() framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index a138ddbb..257dec31 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -119,6 +119,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pixelFormatTransitionFenceWasSent = false var pixelFormatTransitionDeadlineTask: Task? var pixelFormatFenceCapabilityProbePayload: Data? + var expiredPixelFormatFenceCapabilityProbePayload: Data? var pixelFormatFenceNegotiationTask: Task? private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 2bad9414..4391eb10 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -232,12 +232,58 @@ struct AuditFindingsTests { connection.expirePixelFormatFenceNegotiation() #expect(connection.pixelFormatFenceCapabilityProbePayload == nil) + #expect(connection.expiredPixelFormatFenceCapabilityProbePayload != nil) let transition = try #require(connection.clientToServerMessageQueue.dequeue()) try await transition.message.send(connection: AuditWritingConnection()) #expect(connection.pendingPixelFormatTransition == nil) #expect(connection.state.pixelFormat?.depth == 8) } + @Test + func acceptsLateFenceCapabilityResponseAfterProbeTimeout() async throws { + let connection = VNCConnection( + settings: makeSettings(), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let probePayload = try #require(connection.pixelFormatFenceCapabilityProbePayload) + + connection.updateColorDepth(.depth8Bit) + connection.expirePixelFormatFenceNegotiation() + let fallback = try #require(connection.clientToServerMessageQueue.dequeue()) + try await fallback.message.send(connection: AuditWritingConnection()) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: probePayload + ) + ) + + #expect(connection.expiredPixelFormatFenceCapabilityProbePayload == nil) + #expect(connection.state.pixelFormatTransitionFenceFlags.contains(.blockBefore)) + #expect(connection.state.pixelFormatTransitionFenceFlags.contains(.syncNext)) + + connection.state.areContinuousUpdatesEnabled = true + connection.updateColorDepth(.depth16Bit) + #expect(connection.clientToServerMessageQueue.dequeue() != nil) + } + @Test func rejectsPartialFenceBoundariesDuringContinuousUpdates() throws { let connection = VNCConnection( From 9246841fa1104511fda0555d298da4a896bded4e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:55:33 +0200 Subject: [PATCH 082/242] fix(desktop): preserve legacy host cleanup --- CHANGELOG.md | 2 +- docs/api.md | 6 ++++- src/worker/desktop-host-repository.ts | 6 ++--- src/worker/desktop-host-service.ts | 3 ++- src/worker/routes/control-plane.ts | 5 ++-- tests/control-plane-routes.test.ts | 15 ++++++----- tests/desktop-host-repository.test.ts | 5 ++++ tests/desktop-host-service.test.ts | 37 ++++++++++++++++++++++++--- 8 files changed, 60 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac75f1a0..7fac2256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Add capability-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. -- Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host. +- Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. diff --git a/docs/api.md b/docs/api.md index 34ffdbff..b1d076aa 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1119,7 +1119,11 @@ address, port, and timestamp while preserving its creation time. ### DELETE /api/desktop-hosts/:id Removes one registered desktop owned by the signed-in viewer. The route cannot -remove another user's record with the same ID. +remove another user's record with the same ID. Registrations created by current +clients require the exact `X-Crabfleet-Ownership-Token` returned by `PUT`. +Older clients may omit the header only to remove a migrated legacy registration +whose stored ownership token is empty; omission never removes a tokenized +registration. ## Static Routes diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 46cd7f1a..1c5d03a4 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -18,7 +18,7 @@ export type DesktopHostWrite = DesktopHostRow; export interface DesktopHostStore { list(ownerSubject: string): Promise; upsert(host: DesktopHostWrite): Promise; - remove(ownerSubject: string, id: string, ownershipToken: string): Promise; + remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise; } export class DesktopHostRepository implements DesktopHostStore { @@ -93,12 +93,12 @@ export class DesktopHostRepository implements DesktopHostStore { }; } - async remove(ownerSubject: string, id: string, ownershipToken: string): Promise { + async remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise { await database(this.env) .deleteFrom("desktop_hosts") .where("owner_subject", "=", ownerSubject) .where("id", "=", id) - .where("ownership_token", "=", ownershipToken) + .where("ownership_token", "=", ownershipToken ?? "") .execute(); } } diff --git a/src/worker/desktop-host-service.ts b/src/worker/desktop-host-service.ts index 24262fe5..b688aae4 100644 --- a/src/worker/desktop-host-service.ts +++ b/src/worker/desktop-host-service.ts @@ -152,7 +152,8 @@ function desktopHostPort(value: unknown): number { return value; } -function desktopHostOwnershipToken(value: unknown): string { +function desktopHostOwnershipToken(value: unknown): string | null { + if (value === null || value === undefined) return null; if ( typeof value !== "string" || value.length === 0 || diff --git a/src/worker/routes/control-plane.ts b/src/worker/routes/control-plane.ts index 039ab43f..5f409b65 100644 --- a/src/worker/routes/control-plane.ts +++ b/src/worker/routes/control-plane.ts @@ -10,7 +10,7 @@ import { type DesktopHostInput, type DesktopHostRegistration, } from "../desktop-host-service.ts"; -import { badRequest, json, notFound, readJson } from "../http.ts"; +import { json, notFound, readJson } from "../http.ts"; import type { User } from "../models.ts"; export type ControlPlaneRouteDependencies = { @@ -21,7 +21,7 @@ export type ControlPlaneRouteDependencies = { id: string, input: DesktopHostInput, ): Promise; - removeDesktopHost(user: User, id: string, ownershipToken: string): Promise; + removeDesktopHost(user: User, id: string, ownershipToken: string | null): Promise; searchGitHubRefs(number: unknown): Promise; createCard(request: Request, user: User): Promise; readCardRuns(user: User, cardId: string): Promise; @@ -62,7 +62,6 @@ export async function handleControlPlaneRoute( if (request.method === "DELETE" && desktopHostMatch) { requireRole(user, "viewer"); const ownershipToken = request.headers.get(desktopHostOwnershipHeader); - if (!ownershipToken) throw badRequest("desktop host ownership token is required"); await dependencies.removeDesktopHost(user, decoded(desktopHostMatch[1]), ownershipToken); return json({ ok: true }); } diff --git a/tests/control-plane-routes.test.ts b/tests/control-plane-routes.test.ts index ab15b8e1..10f99c1b 100644 --- a/tests/control-plane-routes.test.ts +++ b/tests/control-plane-routes.test.ts @@ -57,7 +57,7 @@ function dependencies(calls: string[]): ControlPlaneRouteDependencies { }; }, async removeDesktopHost(user, id, ownershipToken) { - calls.push(`desktop-host:remove:${user.login}:${id}:${ownershipToken}`); + calls.push(`desktop-host:remove:${user.login}:${id}:${ownershipToken ?? "legacy"}`); }, async searchGitHubRefs(number) { calls.push(`github-refs:${number}`); @@ -190,13 +190,14 @@ test("desktop host routes register and remove only the authenticated user's host "desktop-host:remove:viewer:mac-studio:ownership-token", ]); - await assert.rejects( - dispatch(request("DELETE", "/api/desktop-hosts/mac%2Dstudio"), viewer, []), - (error) => { - assert.equal(status(error), 400); - return true; - }, + const legacyCalls: string[] = []; + const legacyRemoved = await dispatch( + request("DELETE", "/api/desktop-hosts/legacy%2Dstudio"), + viewer, + legacyCalls, ); + assert.equal(legacyRemoved?.status, 200); + assert.deepEqual(legacyCalls, ["desktop-host:remove:viewer:legacy-studio:legacy"]); }); test("card actions derive viewer or maintainer authorization from the action", async () => { diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index b2d8f114..d48b56b5 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -74,4 +74,9 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec assert.match(executions[3]?.sql ?? "", /^delete from "desktop_hosts"/i); assert.match(executions[3]?.sql ?? "", /"ownership_token" = \?/i); assert.deepEqual(executions[3]?.parameters, ["github:1", "studio", "ownership-token"]); + + await repository.remove("github:1", "legacy-studio", null); + assert.match(executions[4]?.sql ?? "", /^delete from "desktop_hosts"/i); + assert.match(executions[4]?.sql ?? "", /"ownership_token" = \?/i); + assert.deepEqual(executions[4]?.parameters, ["github:1", "legacy-studio", ""]); }); diff --git a/tests/desktop-host-service.test.ts b/tests/desktop-host-service.test.ts index 01fc09cd..f338dccb 100644 --- a/tests/desktop-host-service.test.ts +++ b/tests/desktop-host-service.test.ts @@ -36,9 +36,9 @@ class MemoryDesktopHostStore implements DesktopHostStore { return stored; } - async remove(ownerSubject: string, id: string, ownershipToken: string): Promise { + async remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise { const key = `${ownerSubject}:${id}`; - if (this.rows.get(key)?.ownershipToken === ownershipToken) { + if (this.rows.get(key)?.ownershipToken === (ownershipToken ?? "")) { this.rows.delete(key); } } @@ -114,6 +114,37 @@ test("stale desktop host cleanup cannot remove a newer registration", async () = assert.deepEqual(await service.list(alice), []); }); +test("tokenless cleanup removes only migrated legacy desktop hosts", async () => { + const store = new MemoryDesktopHostStore(); + const service = new DesktopHostService( + store, + () => 42, + () => "new-process-token", + ); + const legacy: DesktopHostRow = { + ownerSubject: alice.subject, + id: "legacy-studio", + owner: "alice", + name: "Legacy Studio", + address: "100.64.1.2", + port: 5901, + ownershipToken: "", + createdAt: 1, + updatedAt: 1, + }; + store.rows.set(`${alice.subject}:${legacy.id}`, legacy); + const registration = await service.register(alice, "new-studio", { + name: "New Studio", + address: "100.64.1.3", + port: 5901, + }); + + await service.remove(alice, legacy.id, null); + await service.remove(alice, registration.host.id, null); + + assert.deepEqual(await service.list(alice), [registration.host]); +}); + test("desktop hosts accept only bounded metadata and Tailscale IPv4 endpoints", async () => { const service = new DesktopHostService(new MemoryDesktopHostStore()); const valid = { name: "Studio", address: "100.127.255.254", port: 65_535 }; @@ -133,6 +164,6 @@ test("desktop hosts accept only bounded metadata and Tailscale IPv4 endpoints", } await assert.rejects(service.register(alice, "studio", { ...valid, name: "bad\nname" }), /name/); await assert.rejects(service.register(alice, "studio", { ...valid, port: 0 }), /port/); - await assert.rejects(service.remove(alice, "studio", null), /ownership token/); + await assert.rejects(service.remove(alice, "studio", ""), /ownership token/); await assert.rejects(service.remove(alice, "studio", "bad token"), /ownership token/); }); From 3d819a6f79c055132256c9e00958a5c26e560f64 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:56:06 +0200 Subject: [PATCH 083/242] fix(worker): fence policy rollback and registration CAS --- src/worker/github-actions-repository.ts | 4 +++ .../github-actions-session-registration.ts | 6 ++-- .../sandbox-credential-policy-scanner.ts | 31 +++++++++++++++++-- tests/github-actions-repository.test.ts | 1 + ...ithub-actions-session-registration.test.ts | 31 ++++++++++++++++--- .../sandbox-credential-policy-scanner.test.ts | 20 ++++++++++++ 6 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/worker/github-actions-repository.ts b/src/worker/github-actions-repository.ts index 6708bc11..65036d64 100644 --- a/src/worker/github-actions-repository.ts +++ b/src/worker/github-actions-repository.ts @@ -73,6 +73,10 @@ export class GitHubActionsRepository { .where("work_state", "=", expectedRegistration.work_state) .where("work_phase", "=", expectedRegistration.work_phase) .where("owner_subject", "=", values.owner_subject); + update = + expectedRegistration.agent_token_hash === null + ? update.where("agent_token_hash", "is", null) + : update.where("agent_token_hash", "=", expectedRegistration.agent_token_hash); } else if (isWorkStateUpdate(values) && terminalWorkStates.includes(values.work_state)) { if (!expectedTerminalStatus) { throw new Error("terminal GitHub Actions update requires expected session status"); diff --git a/src/worker/github-actions-session-registration.ts b/src/worker/github-actions-session-registration.ts index 26780fdc..07d4c32c 100644 --- a/src/worker/github-actions-session-registration.ts +++ b/src/worker/github-actions-session-registration.ts @@ -49,7 +49,7 @@ export type GitHubActionsSessionRegistrationUpdate = { export type GitHubActionsSessionRegistrationExpectation = Pick< InteractiveSessionRow, - "updated_at" | "status" | "work_state" | "work_phase" + "agent_token_hash" | "updated_at" | "status" | "work_state" | "work_phase" >; export type GitHubActionsSessionRegistrationStore = { @@ -157,7 +157,6 @@ export class GitHubActionsSessionRegistrationService { const resumed = existing.work_state !== "registered" || existing.status !== "ready"; const message = resumed ? "GitHub Actions work resumed" : "GitHub Actions work registered"; - const registrationRevision = Math.max(now, existing.updated_at + 1); await this.store.updateSession( existing.id, { @@ -175,7 +174,7 @@ export class GitHubActionsSessionRegistrationService { terminal_failure_reason: null, terminal_finalize_pending: 0, credential_cleanup_terminal_status: null, - updated_at: registrationRevision, + updated_at: now, last_seen_at: now, last_event: message, agent_token_hash: agentTokenHash, @@ -188,6 +187,7 @@ export class GitHubActionsSessionRegistrationService { completion_reason: null, }, { + agent_token_hash: existing.agent_token_hash, updated_at: existing.updated_at, status: existing.status, work_state: existing.work_state, diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index f126a721..be2ebb2b 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -12,6 +12,7 @@ import { abandonSandboxCredentialPolicyRegistration, finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, + renewSandboxCredentialPolicyRegistration, sandboxCredentialPolicyCleanupAuthorizedCondition, sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, @@ -339,8 +340,17 @@ async function scanStagedCredentialPolicyRegistrations( }; try { const ownershipFence = credentialPolicyScanOwnershipFence(row, now); + if (!ownershipFence) { + await abandonSandboxCredentialPolicyRegistration( + env, + row.session_id, + row.sandbox_id, + registration, + "sandbox credential policy owner is no longer current", + ); + continue; + } if ( - ownershipFence && (await policyExists(env, row.sandbox_id, row.registration_generation)) && (await finishSandboxCredentialPolicyRegistration( env, @@ -354,9 +364,26 @@ async function scanStagedCredentialPolicyRegistrations( } if (row.rollback_policies_json !== null) { if (!restoreRollback) throw new Error("sandbox credential policy rollback is unavailable"); + const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + row.session_id, + row.sandbox_id, + registration, + ownershipFence, + ); + if (!registrationExpiresAt) { + await abandonSandboxCredentialPolicyRegistration( + env, + row.session_id, + row.sandbox_id, + registration, + "sandbox credential policy owner changed before rollback", + ); + continue; + } await restoreRollback({ registration, - registrationExpiresAt: row.registration_claim_expires_at, + registrationExpiresAt, rollbackJson: row.rollback_policies_json, sessionId: row.session_id, }); diff --git a/tests/github-actions-repository.test.ts b/tests/github-actions-repository.test.ts index 8a7855f7..19bd15e5 100644 --- a/tests/github-actions-repository.test.ts +++ b/tests/github-actions-repository.test.ts @@ -168,6 +168,7 @@ const registrationUpdate: GitHubActionsSessionRegistrationUpdate = { }; const registrationExpectation = { + agent_token_hash: "prior-agent-hash", updated_at: 90, status: "stopped", work_state: "completed", diff --git a/tests/github-actions-session-registration.test.ts b/tests/github-actions-session-registration.test.ts index c14d39c5..809d96d8 100644 --- a/tests/github-actions-session-registration.test.ts +++ b/tests/github-actions-session-registration.test.ts @@ -266,6 +266,7 @@ test("GitHub Actions work keys can be resumed by the matching owner", async () = assert.equal(state.updates[0]?.values.owner_subject, "github:42"); assert.equal(state.updates[0]?.values.agent_token_hash, "agent-token-hash"); assert.deepEqual(state.updates[0]?.expected, { + agent_token_hash: existing.agent_token_hash, updated_at: existing.updated_at, status: existing.status, work_state: existing.work_state, @@ -363,10 +364,8 @@ test("registration adopts a concurrently inserted work key", async () => { assert.equal(result.session.id, "IS-concurrent"); assert.equal(state.workKeyReads, 2); assert.equal(state.updates[0]?.id, "IS-concurrent"); - assert.equal( - state.updates[0]?.values.updated_at, - Math.max(100, state.concurrentRow.updated_at + 1), - ); + assert.equal(state.updates[0]?.values.updated_at, 100); + assert.equal(state.updates[0]?.expected.agent_token_hash, state.concurrentRow.agent_token_hash); }); test("concurrent registration adoption rotates exactly one usable token", async () => { @@ -395,6 +394,7 @@ test("concurrent registration adoption rotates exactly one usable token", async if ( !current || current.updated_at !== expected.updated_at || + current.agent_token_hash !== expected.agent_token_hash || current.status !== expected.status || current.work_state !== expected.work_state || current.work_phase !== expected.work_phase @@ -428,7 +428,28 @@ test("concurrent registration adoption rotates exactly one usable token", async state.rows.get(existing.id)?.agent_token_hash, `${fulfilled[0]?.value.agentToken}-hash`, ); - assert.equal(state.rows.get(existing.id)?.updated_at, existing.updated_at + 1); + assert.equal(state.rows.get(existing.id)?.updated_at, 100); +}); + +test("registration repairs a future timestamp without blocking immediate writers", async () => { + const existing = sessionRow({ + id: "IS-future-revision", + runtime: "github_actions", + work_key: "issue:future-revision", + owner: "operator", + owner_subject: "github:42", + updated_at: 500, + }); + const { store, state } = registrationStore([existing]); + + await new GitHubActionsSessionRegistrationService(store).register({ + workKey: "issue:future-revision", + workKind: "issue", + repo: "openclaw/crabfleet", + owner: "operator@example.test", + }); + + assert.equal(state.rows.get(existing.id)?.updated_at, 100); }); test("registration rejects invalid input and work keys owned by another runtime", async () => { diff --git a/tests/sandbox-credential-policy-scanner.test.ts b/tests/sandbox-credential-policy-scanner.test.ts index 09e667c9..efa5911b 100644 --- a/tests/sandbox-credential-policy-scanner.test.ts +++ b/tests/sandbox-credential-policy-scanner.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; import test from "node:test"; import { @@ -122,6 +123,25 @@ test("credential-policy scan rejects incomplete, expired, and mismatched ownersh ); }); +test("staged rollback reclaims current ownership before restoring policy", async () => { + const source = await readFile( + new URL("../src/worker/sandbox-credential-policy-scanner.ts", import.meta.url), + "utf8", + ); + const start = source.indexOf("async function scanStagedCredentialPolicyRegistrations"); + const end = source.indexOf("async function readCredentialPolicyScanPage", start); + const stagedRecovery = source.slice(start, end); + + assert.ok( + stagedRecovery.indexOf("if (!ownershipFence)") < stagedRecovery.indexOf("restoreRollback({"), + ); + assert.ok( + stagedRecovery.indexOf("renewSandboxCredentialPolicyRegistration(") < + stagedRecovery.indexOf("restoreRollback({"), + ); + assert.match(stagedRecovery, /if \(!registrationExpiresAt\)/); +}); + test("credential-policy scan preserves live standalone and managed policies", () => { assert.equal( credentialPolicyScanRequiresCleanup( From b63139b1ef2e4e22b4fea36a0ab4e6018b4fc0d8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:56:11 +0200 Subject: [PATCH 084/242] fix(macos): bound tailnet pipe draining --- .../CrabfleetMac/TailnetIdentity.swift | 17 +++++++- .../PrivateMacShareTests.swift | 39 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift index 666023f4..def7f07a 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift @@ -88,6 +88,8 @@ struct SystemTailscaleCommandRunner: TailscaleCommandRunning { } private final class TailscaleCommandExecution: @unchecked Sendable { + private static let stoppedProcessDrainTimeout: DispatchTimeInterval = .milliseconds(250) + private enum StopReason { case cancelled case outputTooLarge @@ -148,7 +150,7 @@ private final class TailscaleCommandExecution: @unchecked Sendable { Thread.sleep(forTimeInterval: 0.01) } process.waitUntilExit() - readGroup.wait() + finishCapture() switch currentStopReason() { case .cancelled: @@ -235,6 +237,19 @@ private final class TailscaleCommandExecution: @unchecked Sendable { _ = Darwin.kill(pid, SIGKILL) } } + + private func finishCapture() { + guard currentStopReason() != nil else { + readGroup.wait() + return + } + guard readGroup.wait(timeout: .now() + Self.stoppedProcessDrainTimeout) == .timedOut else { + return + } + try? outputPipe.fileHandleForReading.close() + try? errorPipe.fileHandleForReading.close() + _ = readGroup.wait(timeout: .now() + Self.stoppedProcessDrainTimeout) + } } struct TailscaleStatusDocument: Decodable, Sendable { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index d99d3ab8..1b4d9ff1 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -99,6 +99,45 @@ struct PrivateMacShareTests { #expect(await waitUntilAsync { Darwin.kill(Int32(cancelledPID), 0) != 0 }) } + @Test + func tailscaleCommandTimeoutDoesNotWaitForDescendantPipeEOF() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("CrabfleetMacTests.\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let executable = directory.appendingPathComponent("tailscale") + let descendantPIDFile = directory.appendingPathComponent("descendant-pid") + try Data( + """ + #!/bin/sh + ( + trap '' HUP TERM + exec sleep 30 + ) & + printf '%s' "$!" > '\(descendantPIDFile.path)' + exec sleep 30 + """.utf8 + ).write(to: executable) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: executable.path) + + let runner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 0.5) + let clock = ContinuousClock() + let startedAt = clock.now + await #expect(throws: PrivateMacShareError.commandTimedOut) { + _ = try await runner.run(arguments: ["status"]) + } + let elapsed = startedAt.duration(to: clock.now) + + let descendantPID = try #require( + Int32(String(contentsOf: descendantPIDFile, encoding: .utf8)) + ) + defer { + _ = Darwin.kill(descendantPID, SIGKILL) + } + #expect(Darwin.kill(descendantPID, 0) == 0) + #expect(elapsed < .seconds(2)) + } + @Test @MainActor func stopInvalidatesAnInFlightPrivateShareStart() async throws { let runner = SuspendedTailscaleRunner() From 58ae4240cf91791e59f5eb4a9119aa6f2387c43e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:57:28 +0200 Subject: [PATCH 085/242] fix(actions): negotiate runner protocol at connect --- CHANGELOG.md | 2 +- README.md | 39 ++++------- docs/api.md | 23 +++---- docs/architecture.md | 4 +- docs/github-actions-sessions.md | 68 ++++++------------- docs/spec.md | 12 ++-- src/github-actions-runner.ts | 37 +---------- src/github-actions-runtime.ts | 84 +++++------------------- src/worker/github-actions-application.ts | 15 ++++- src/worker/session-control-do.ts | 15 ++++- tests/application-architecture.test.ts | 16 +++++ tests/github-actions-event-auth.test.ts | 18 +++++ tests/github-actions-runner.test.ts | 36 ++-------- tests/github-actions-runtime.test.ts | 67 ++++++++++--------- 14 files changed, 168 insertions(+), 268 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fac2256..b303b5bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, rollback-restored Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. -- Add capability-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. - Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. diff --git a/README.md b/README.md index 51664ec5..6f295633 100644 --- a/README.md +++ b/README.md @@ -97,29 +97,15 @@ Content-Type: application/json The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and can be opened with Node's global `WebSocket` without custom headers, but it is not a raw duplex byte stream. The abbreviated runner handler is: ```js -const terminal = new WebSocket(runnerPtyUrl); +const framedRunnerPtyUrl = new URL(runnerPtyUrl); +framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); +const terminal = new WebSocket(framedRunnerPtyUrl); terminal.binaryType = "arraybuffer"; -let framed = false; - -terminal.onopen = () => { - terminal.send( - JSON.stringify({ - type: "crabfleet_runner_capabilities", - capabilities: ["cfr1-framed-io-v1"], - }), - ); -}; -pty.onData((output) => terminal.send(framed ? encodeCfr1Output(output) : output)); + +pty.onData((output) => terminal.send(encodeCfr1Output(output))); terminal.onmessage = ({ data }) => { - if (isAcceptedCapabilities(data)) { - framed = true; - return; - } const input = decodeCfr1Input(data); - if (!input) { - if (!framed) pty.write(typeof data === "string" ? data : new TextDecoder().decode(data)); - return; - } + if (!input) return; try { pty.write(new TextDecoder().decode(input.payload)); terminal.send(encodeCfr1Ack(input.inputId, true)); @@ -129,12 +115,13 @@ terminal.onmessage = ({ data }) => { }; ``` -Existing runners retain raw input and output. A runner requests correlated -binary `CFR1` input, output, and acknowledgement frames by advertising the -`cfr1-framed-io-v1` capability immediately after connecting. It keeps accepting -and sending raw traffic until the relay confirms that capability. Negotiated -runners acknowledge only after their PTY accepts the input; the complete -encoder, decoder, and Node runner example are in +Existing runners retain raw input and output by opening the returned URL +unchanged. A new runner opts into correlated binary `CFR1` input, output, and +acknowledgement frames by adding the exact +`runnerProtocol=cfr1-framed-io-v1` query before opening the socket. The relay +selects that mode before accepting the connection, so there is no pending +handshake. Framed runners acknowledge only after their PTY accepts the input; +the complete encoder, decoder, and Node runner example are in [`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. diff --git a/docs/api.md b/docs/api.md index b1d076aa..194bcb6e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -598,21 +598,18 @@ Response: } ``` -Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output; runners advertise `cfr1-framed-io-v1` to negotiate the framed contract below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. +Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output by opening it unchanged; new runners add the exact `runnerProtocol=cfr1-framed-io-v1` query to opt into the framed contract below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. ### GET /api/agent/interactive-sessions/:id/runner-pty WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses the scoped `agentToken` query parameter embedded in `runnerPtyUrl`. One runner is current; a reconnect replaces the previous runner while browser viewers remain attached. -Runner sockets begin in legacy mode with raw input and output. A runner -negotiates framed I/O by sending -`{"type":"crabfleet_runner_capabilities","capabilities":["cfr1-framed-io-v1"]}`. -The relay responds with the accepted capability. The runner must keep accepting -raw input and sending raw output until that response arrives because viewer -input can race the advertisement on another socket. Negotiated input, output, -acknowledgements, and relay control traffic use binary `CFR1` frames. The relay -wraps legacy output before forwarding it to viewers, so arbitrary raw PTY bytes -cannot be consumed as control traffic. +Opening the returned URL unchanged selects legacy raw input and output. Adding +the exact `runnerProtocol=cfr1-framed-io-v1` query selects framed input, output, +acknowledgements, and relay control traffic. The application propagates only +that exact value to `SessionControlDO`, which stores the mode on the server +socket before accepting it. The relay wraps legacy output before forwarding it +to viewers, so arbitrary raw PTY bytes cannot be consumed as control traffic. Each `CFR1` frame occupies one binary WebSocket message and starts with: @@ -642,9 +639,9 @@ generates a rejected acknowledgement only when no current runner is available to receive the input frame or the relay send fails. Stale or mismatched acknowledgement IDs do not complete another pending input. -Before negotiation, the relay unwraps viewer input to raw bytes and reports -acceptance once the runner socket accepts the send. This preserves existing -runner integrations while negotiated runners provide PTY-level completion. +For legacy connections, the relay unwraps viewer input to raw bytes and reports +acceptance once the runner socket accepts the send. Framed connections provide +PTY-level completion through correlated acknowledgements. See [GitHub Actions Sessions](/github-actions-sessions/#runner-pty) for a complete Node runner integration. diff --git a/docs/architecture.md b/docs/architecture.md index dcafcf4f..0b1d3f68 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ D1 is canonical for product metadata: ### Durable Objects - `Sandbox` runs first-party Cloudflare Sandbox workspaces. -- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; capability-negotiated runners use correlated binary `CFR1` input, output, and acknowledgement frames. Viewer-bound output is always framed so terminal bytes cannot collide with control traffic. +- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; exact connection-query opt-in selects correlated binary `CFR1` input, output, and acknowledgement frames before socket acceptance. Viewer-bound output is always framed so terminal bytes cannot collide with control traffic. There is no `BoardDO` or `RunDO`. General Board/Fleet state is D1 plus REST polling. @@ -137,7 +137,7 @@ Interactive sessions are the live execution plane. Supported paths: - **Built-in Sandbox:** Worker provisions a Cloudflare Sandbox, prepares the repo, starts a Codex-capable shell, and proxies PTY traffic. - **Versioned runtime adapter:** Worker durably registers a tenant-namespaced workspace ID, creates and reconciles the provider workspace, proxies PTY access, mints transient desktop links, and confirms provider release before terminal state. -- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, and either retains legacy raw terminal traffic or negotiates correlated `CFR1` browser input/output. Negotiated runners acknowledge each PTY write before Crabfleet reports input acceptance. +- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, and either retains legacy raw terminal traffic or opts into correlated `CFR1` browser input/output through the connection URL. Framed runners acknowledge each PTY write before Crabfleet reports input acceptance. Sessions can carry a stable tenant owner, parent/root lineage, purpose, summary, named grants, public share state, delegated control, multiplayer mode, archive metadata, and runtime-specific capability state. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index be6f71e6..8514d4a8 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -265,13 +265,13 @@ returns only the sanitized event. The Action connects outbound to the returned `runnerPtyUrl`. Node's global `WebSocket` can open the URL without custom headers. -Runner sockets begin in legacy mode with raw input and output. A runner requests -collision-free framed I/O by advertising `cfr1-framed-io-v1` immediately after -connecting, but it remains in legacy mode until the relay accepts the -capability. Negotiated viewer input arrives in a binary `CFR1` frame -carrying a correlation ID; runner output uses a distinct `CFR1` output frame. -The runner returns a correlated acknowledgement only after its PTY accepts the -input write. +The returned URL opens a legacy raw-input/raw-output socket. A runner opts into +collision-free framed I/O by adding the exact +`runnerProtocol=cfr1-framed-io-v1` query before opening the socket. The relay +records that mode before accepting the connection. Viewer input then arrives in +a binary `CFR1` frame carrying a correlation ID, and runner output uses a +distinct `CFR1` output frame. The runner returns a correlated acknowledgement +only after its PTY accepts the input write. Complete Node runner integration: @@ -288,20 +288,15 @@ if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); const magic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); // CFR1 const decoder = new TextDecoder(); const encoder = new TextEncoder(); -const terminal = new WebSocket(runnerPtyUrl); +const framedRunnerPtyUrl = new URL(runnerPtyUrl); +framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); +const terminal = new WebSocket(framedRunnerPtyUrl); terminal.binaryType = "arraybuffer"; -let framed = false; await new Promise((resolve, reject) => { terminal.addEventListener("open", resolve, { once: true }); terminal.addEventListener("error", reject, { once: true }); }); -terminal.send( - JSON.stringify({ - type: "crabfleet_runner_capabilities", - capabilities: ["cfr1-framed-io-v1"], - }), -); const pty = spawn(process.env.SHELL || "/bin/bash", [], { cwd: process.cwd(), @@ -309,25 +304,16 @@ const pty = spawn(process.env.SHELL || "/bin/bash", [], { }); pty.onData((output) => { - terminal.send(framed ? encodeOutput(output) : output); + terminal.send(encodeOutput(output)); }); terminal.addEventListener("message", (event) => { - if (acceptCapabilities(event.data)) { - framed = true; - return; - } acceptInput(event.data); }); function acceptInput(data) { const input = decodeInput(data); - if (!input) { - if (!framed) { - pty.write(typeof data === "string" ? data : decoder.decode(data)); - } - return; - } + if (!input) return; try { // A successful node-pty write is this adapter's PTY acceptance point. pty.write(decoder.decode(input.payload)); @@ -337,20 +323,6 @@ function acceptInput(data) { } } -function acceptCapabilities(data) { - if (typeof data !== "string") return false; - try { - const message = JSON.parse(data); - return ( - message.type === "crabfleet_runner_capabilities" && - Array.isArray(message.accepted) && - message.accepted.includes("cfr1-framed-io-v1") - ); - } catch { - return false; - } -} - function decodeInput(data) { if (!(data instanceof ArrayBuffer)) return null; const frame = new Uint8Array(data); @@ -405,12 +377,9 @@ For a PTY API with an asynchronous write callback or promise, await that acceptance signal before sending `encodeAck(..., true)`. Do not acknowledge when the WebSocket merely queues the input frame. -The relay answers the advertisement with -`{"type":"crabfleet_runner_capabilities","accepted":["cfr1-framed-io-v1"]}`. -The runner must continue accepting raw input and sending raw output until that -acceptance arrives. Viewer input can race the advertisement because it comes -from another socket; messages already sent in legacy mode arrive before the -acceptance response. Each `CFR1` frame occupies one binary WebSocket message: +The protocol query is consumed during connection setup and is not forwarded as +terminal data. There is no capability message or mode transition after the +socket opens. Each `CFR1` frame occupies one binary WebSocket message: | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------ | @@ -433,9 +402,10 @@ Properties: - Only one runner is current. - A new runner connection replaces the previous runner. - Multiple browser viewers may remain connected. -- Legacy runners receive raw viewer input and send raw output. -- Runners switch to framed input and `0x04` output only after receiving the - capability acceptance response. +- Legacy runners open the returned URL unchanged, receive raw viewer input, and + send raw output. +- Framed runners add the exact protocol query before connecting, receive framed + input immediately, and wrap every output payload in a `0x04` frame. - Negotiated input produces `input-accepted` only after the correlated runner acknowledgement. Legacy input reports acceptance after relay delivery. - Runner lifecycle events are typed binary frames even while no runner is diff --git a/docs/spec.md b/docs/spec.md index 651e02d6..22873c9e 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -178,18 +178,18 @@ Crabfleet owns: - session identity and metadata; - rotating scoped agent token; -- outbound runner relay through `SessionControlDO`, preserving legacy raw runner traffic while capability-negotiated runners use correlated binary `CFR1` input, output, acknowledgement, and lifecycle frames; +- outbound runner relay through `SessionControlDO`, preserving legacy raw runner traffic while exact connection-query opt-in selects correlated binary `CFR1` input, output, acknowledgement, and lifecycle frames; - browser terminal steering; - work-state heartbeats; - event and transcript finalization. -The Action remains the execution host and mutation authority. A negotiated +The Action remains the execution host and mutation authority. A framed runner acknowledges viewer input only after its PTY accepts the correlated write; relay queueing is not acceptance. Legacy runners keep raw input/output -and relay-level delivery reporting. A runner remains in that legacy mode until -the relay confirms its framed-I/O capability, so input racing negotiation is -not dropped. Ending the Crabfleet session does not -cancel the workflow run. +and relay-level delivery reporting. The exact protocol query selects the mode +before the runner socket is accepted, so there is no in-band handshake or +mode-transition race. Ending the Crabfleet session does not cancel the workflow +run. ## Session Lifecycle diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 9ae8d6cb..788927b1 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -1,36 +1,15 @@ import { encodeGitHubActionsRelayOutput, - encodeGitHubActionsRunnerCapabilities, - parseGitHubActionsRunnerCapabilitiesAccepted, parseGitHubActionsRelayInput, sendGitHubActionsRelayInputAcknowledgement, type GitHubActionsRelaySocket, } from "./github-actions-runtime.ts"; -export function negotiateGitHubActionsRunnerProtocol(socket: GitHubActionsRelaySocket): void { - socket.send(encodeGitHubActionsRunnerCapabilities()); -} - export function sendGitHubActionsRunnerOutput( socket: GitHubActionsRelaySocket, output: string | ArrayBuffer | ArrayBufferView, - framed = true, ): void { - if (framed) { - socket.send(encodeGitHubActionsRelayOutput(output)); - return; - } - socket.send( - typeof output === "string" || output instanceof ArrayBuffer - ? output - : Uint8Array.from(new Uint8Array(output.buffer, output.byteOffset, output.byteLength)).buffer, - ); -} - -export function gitHubActionsRunnerProtocolAccepted(message: string | ArrayBuffer): boolean { - return ( - parseGitHubActionsRunnerCapabilitiesAccepted(message)?.includes("cfr1-framed-io-v1") ?? false - ); + socket.send(encodeGitHubActionsRelayOutput(output)); } export async function acceptGitHubActionsRunnerInput( @@ -39,19 +18,7 @@ export async function acceptGitHubActionsRunnerInput( writeToPty: (payload: ArrayBuffer) => void | Promise, ): Promise { const input = parseGitHubActionsRelayInput(message); - if (!input) { - if (parseGitHubActionsRunnerCapabilitiesAccepted(message)) return false; - try { - const payload = - typeof message === "string" - ? Uint8Array.from(new TextEncoder().encode(message)).buffer - : message; - await writeToPty(payload); - } catch { - // Legacy delivery is acknowledged by the relay after the WebSocket send. - } - return true; - } + if (!input) return false; try { await writeToPty(input.payload); sendGitHubActionsRelayInputAcknowledgement(socket, { diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 04baa0fb..b8597ec9 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -30,6 +30,8 @@ export type GitHubActionsRelayInput = { }; export const githubActionsFramedRunnerCapability = "cfr1-framed-io-v1"; +export const githubActionsRunnerProtocolQuery = "runnerProtocol"; +export type GitHubActionsRunnerProtocol = typeof githubActionsFramedRunnerCapability; export const githubActionsCapabilities = { terminal: true, @@ -79,10 +81,9 @@ const relayEvents = new Map( ); const encoder = new TextEncoder(); const decoder = new TextDecoder(); -const runnerCapabilitiesMessageType = "crabfleet_runner_capabilities"; type GitHubActionsRunnerAttachment = { - protocol?: typeof githubActionsFramedRunnerCapability; + protocol?: GitHubActionsRunnerProtocol; }; export function githubActionsRuntimeLabel(runtime: unknown): string { @@ -207,22 +208,6 @@ export function relayGitHubActionsWebSocketMessage( } } - const capabilities = parseGitHubActionsRunnerCapabilities(message); - if (capabilities) { - if (capabilities.includes(githubActionsFramedRunnerCapability)) { - senderSocket.serializeAttachment?.({ - protocol: githubActionsFramedRunnerCapability, - } satisfies GitHubActionsRunnerAttachment); - sendGitHubActionsRunnerCapabilitiesAccepted(senderSocket, [ - githubActionsFramedRunnerCapability, - ]); - } else { - senderSocket.serializeAttachment?.({} satisfies GitHubActionsRunnerAttachment); - sendGitHubActionsRunnerCapabilitiesAccepted(senderSocket, []); - } - return 0; - } - if (gitHubActionsRunnerUsesFramedProtocol(senderSocket)) { if ( !parseGitHubActionsRelayInputAcknowledgement(message) && @@ -311,45 +296,19 @@ export function parseGitHubActionsRelayInput( }; } -export function encodeGitHubActionsRunnerCapabilities(): string { - return JSON.stringify({ - type: runnerCapabilitiesMessageType, - capabilities: [githubActionsFramedRunnerCapability], - }); -} - -export function parseGitHubActionsRunnerCapabilities( - message: string | ArrayBuffer, -): string[] | null { - if (typeof message !== "string") return null; - try { - const parsed = JSON.parse(message) as Record; - if (parsed.type !== runnerCapabilitiesMessageType || !Array.isArray(parsed.capabilities)) { - return null; - } - return parsed.capabilities.filter((capability): capability is string => { - return typeof capability === "string"; - }); - } catch { - return null; - } +export function parseGitHubActionsRunnerProtocol( + value: string | null, +): GitHubActionsRunnerProtocol | null { + return value === githubActionsFramedRunnerCapability ? githubActionsFramedRunnerCapability : null; } -export function parseGitHubActionsRunnerCapabilitiesAccepted( - message: string | ArrayBuffer, -): string[] | null { - if (typeof message !== "string") return null; - try { - const parsed = JSON.parse(message) as Record; - if (parsed.type !== runnerCapabilitiesMessageType || !Array.isArray(parsed.accepted)) { - return null; - } - return parsed.accepted.filter((capability): capability is string => { - return typeof capability === "string"; - }); - } catch { - return null; - } +export function attachGitHubActionsRunnerProtocol( + socket: GitHubActionsRelaySocket, + protocol: GitHubActionsRunnerProtocol | null, +): void { + socket.serializeAttachment?.( + protocol ? ({ protocol } satisfies GitHubActionsRunnerAttachment) : {}, + ); } export function encodeGitHubActionsRelayInputAcknowledgement( @@ -472,23 +431,10 @@ function requireGitHubActionsRelayInputId(inputId: string): void { if (!inputId) throw new Error("invalid GitHub Actions relay input id"); } -function gitHubActionsRunnerUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { +export function gitHubActionsRunnerUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { const attachment = socket.deserializeAttachment?.(); if (!attachment || typeof attachment !== "object") return false; return ( (attachment as GitHubActionsRunnerAttachment).protocol === githubActionsFramedRunnerCapability ); } - -function sendGitHubActionsRunnerCapabilitiesAccepted( - socket: GitHubActionsRelaySocket, - capabilities: string[], -): void { - if (socket.readyState !== webSocketOpen) return; - socket.send( - JSON.stringify({ - type: runnerCapabilitiesMessageType, - accepted: capabilities, - }), - ); -} diff --git a/src/worker/github-actions-application.ts b/src/worker/github-actions-application.ts index 3fd54b4b..09d65feb 100644 --- a/src/worker/github-actions-application.ts +++ b/src/worker/github-actions-application.ts @@ -1,4 +1,8 @@ import type { GitHubActionsSessionRegistrationInput } from "./github-actions-session-registration.ts"; +import { + githubActionsRunnerProtocolQuery, + parseGitHubActionsRunnerProtocol, +} from "../github-actions-runtime.ts"; import { AdminRepository } from "./admin-repository.ts"; import { GitHubActionsSessionRegistrationService, @@ -157,7 +161,7 @@ export class GitHubActionsApplication { this.appendMessageEvent(sessionId, user, message, now), }; await new GitHubActionsRunnerConnectionService(store).connect(session); - return stub.fetch("https://crabfleet.internal/api/session-control/github-actions/runner", { + return stub.fetch(gitHubActionsRelayRunnerUrl(request), { headers: { upgrade: "websocket" }, }); } @@ -201,6 +205,15 @@ export class GitHubActionsApplication { } } +export function gitHubActionsRelayRunnerUrl(request: Request): string { + const relayUrl = new URL("https://crabfleet.internal/api/session-control/github-actions/runner"); + const protocol = parseGitHubActionsRunnerProtocol( + new URL(request.url).searchParams.get(githubActionsRunnerProtocolQuery), + ); + if (protocol) relayUrl.searchParams.set(githubActionsRunnerProtocolQuery, protocol); + return relayUrl.toString(); +} + function isConstraintError(error: unknown): boolean { return error instanceof Error && /constraint|unique/i.test(error.message); } diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index 3795dd03..e7c004b3 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -7,10 +7,14 @@ import { } from "../credential-policy-fence.ts"; import type { FleetSandboxPolicySummary } from "../fleet-state.ts"; import { + attachGitHubActionsRunnerProtocol, githubActionsRelayRole, + githubActionsRunnerProtocolQuery, notifyGitHubActionsViewers, + parseGitHubActionsRunnerProtocol, relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, + type GitHubActionsRunnerProtocol, } from "../github-actions-runtime.ts"; import type { RuntimeEnv } from "./env.ts"; import { json } from "./http.ts"; @@ -51,7 +55,10 @@ export class SessionControlDO extends DurableObject { request.method === "GET" && url.pathname === "/api/session-control/github-actions/runner" ) { - return this.openGitHubActionsRelay("runner"); + return this.openGitHubActionsRelay( + "runner", + parseGitHubActionsRunnerProtocol(url.searchParams.get(githubActionsRunnerProtocolQuery)), + ); } if ( @@ -223,12 +230,16 @@ export class SessionControlDO extends DurableObject { socket.close(1011, "relay peer error"); } - private openGitHubActionsRelay(role: "runner" | "viewer"): Response { + private openGitHubActionsRelay( + role: "runner" | "viewer", + runnerProtocol: GitHubActionsRunnerProtocol | null = null, + ): Response { const pair = new WebSocketPair(); const client = pair[0]; const server = pair[1]; if (role === "runner") { replaceGitHubActionsRunner(this.ctx.getWebSockets("github-actions-runner")); + attachGitHubActionsRunnerProtocol(server, runnerProtocol); this.ctx.acceptWebSocket(server, ["github-actions-runner"]); notifyGitHubActionsViewers( this.ctx.getWebSockets("github-actions-viewer"), diff --git a/tests/application-architecture.test.ts b/tests/application-architecture.test.ts index dec08288..4b546b29 100644 --- a/tests/application-architecture.test.ts +++ b/tests/application-architecture.test.ts @@ -59,6 +59,22 @@ test("worker entrypoint delegates OpenClaw and GitHub Actions composition", asyn assert.match(githubActions, /new GitHubActionsWorkStateService\(/); }); +test("GitHub Actions runner protocol is attached before the relay socket is accepted", async () => { + const [application, relay] = await Promise.all([ + readFile(new URL("../src/worker/github-actions-application.ts", import.meta.url), "utf8"), + readFile(new URL("../src/worker/session-control-do.ts", import.meta.url), "utf8"), + ]); + + assert.match(application, /stub\.fetch\(gitHubActionsRelayRunnerUrl\(request\)/); + const attach = relay.indexOf("attachGitHubActionsRunnerProtocol(server, runnerProtocol)"); + const accept = relay.indexOf( + 'this.ctx.acceptWebSocket(server, ["github-actions-runner"])', + attach, + ); + assert.notEqual(attach, -1); + assert.ok(accept > attach); +}); + test("worker entrypoint retains only routing and platform composition", async () => { const entrypoint = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); diff --git a/tests/github-actions-event-auth.test.ts b/tests/github-actions-event-auth.test.ts index 70109485..15dafb44 100644 --- a/tests/github-actions-event-auth.test.ts +++ b/tests/github-actions-event-auth.test.ts @@ -5,6 +5,7 @@ import { sha256 } from "../src/worker/crypto.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import { GitHubActionsApplication, + gitHubActionsRelayRunnerUrl, structuredEventRequestMaxBytes, } from "../src/worker/github-actions-application.ts"; import { terminalAgentEventGraceMs } from "../src/worker/session-agent-auth.ts"; @@ -143,6 +144,23 @@ test("GitHub Actions application rejects an event token issued to another sessio assert.equal(subject.mutationCount(), 0); }); +test("GitHub Actions application propagates only the exact runner protocol opt-in", () => { + const base = + "https://fleet.example/api/agent/interactive-sessions/IS-target/runner-pty?agentToken=secret"; + assert.equal( + gitHubActionsRelayRunnerUrl(new Request(base)), + "https://crabfleet.internal/api/session-control/github-actions/runner", + ); + assert.equal( + gitHubActionsRelayRunnerUrl(new Request(`${base}&runnerProtocol=cfr1-framed-io-v1`)), + "https://crabfleet.internal/api/session-control/github-actions/runner?runnerProtocol=cfr1-framed-io-v1", + ); + assert.equal( + gitHubActionsRelayRunnerUrl(new Request(`${base}&runnerProtocol=cfr1-framed-io-v2`)), + "https://crabfleet.internal/api/session-control/github-actions/runner", + ); +}); + test("agent event endpoint rejects a wrong-session token before persistence", async () => { const subject = await authEnvironment(); const application = new GitHubActionsApplication(subject.env, { audit: async () => undefined }); diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 3ac5b605..859cf5c0 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -3,14 +3,11 @@ import { test } from "node:test"; import { acceptGitHubActionsRunnerInput, - gitHubActionsRunnerProtocolAccepted, - negotiateGitHubActionsRunnerProtocol, sendGitHubActionsRunnerOutput, } from "../src/github-actions-runner.ts"; import { encodeGitHubActionsRelayInput, parseGitHubActionsRelayOutput, - parseGitHubActionsRunnerCapabilities, parseGitHubActionsRelayInputAcknowledgement, type GitHubActionsRelaySocket, } from "../src/github-actions-runtime.ts"; @@ -51,7 +48,7 @@ test("runner acknowledges input only after the PTY write completes", async () => }); }); -test("runner rejects failed framed writes and accepts legacy input during negotiation", async () => { +test("runner rejects failed writes and ignores unframed terminal data", async () => { const socket = relaySocket(); assert.equal( @@ -69,42 +66,17 @@ test("runner rejects failed framed writes and accepts legacy input during negoti accepted: false, error: "GitHub Actions runner did not accept terminal input", }); - let legacyInput = ""; - assert.equal( - await acceptGitHubActionsRunnerInput(socket, "raw input", async (payload) => { - legacyInput = new TextDecoder().decode(payload); - }), - true, - ); - assert.equal(legacyInput, "raw input"); + assert.equal(await acceptGitHubActionsRunnerInput(socket, "raw input", async () => {}), false); assert.equal(socket.sent.length, 1); }); -test("runner helpers stay raw until negotiation is accepted, then envelope PTY output", async () => { +test("runner output helper envelopes PTY bytes for connection-negotiated runners", () => { const socket = relaySocket(); - negotiateGitHubActionsRunnerProtocol(socket); - assert.deepEqual(parseGitHubActionsRunnerCapabilities(socket.sent[0]!), ["cfr1-framed-io-v1"]); - - sendGitHubActionsRunnerOutput(socket, "early", false); - assert.equal(socket.sent[1], "early"); - - const accepted = JSON.stringify({ - type: "crabfleet_runner_capabilities", - accepted: ["cfr1-framed-io-v1"], - }); - assert.equal(gitHubActionsRunnerProtocolAccepted(accepted), true); - assert.equal( - await acceptGitHubActionsRunnerInput(socket, accepted, async () => { - assert.fail("capability acceptance must not reach the PTY"); - }), - false, - ); - const collision = encodeGitHubActionsRelayInput("looks-like-input", "terminal bytes"); sendGitHubActionsRunnerOutput(socket, collision); assert.deepEqual( - new Uint8Array(parseGitHubActionsRelayOutput(socket.sent[2]!)!), + new Uint8Array(parseGitHubActionsRelayOutput(socket.sent[0]!)!), new Uint8Array(collision), ); }); diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 5107b2e8..19e32164 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -1,15 +1,18 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + attachGitHubActionsRunnerProtocol, buildGitHubActionsRunnerPtyUrl, encodeGitHubActionsRelayInput, encodeGitHubActionsRelayInputAcknowledgement, encodeGitHubActionsRelayOutput, - encodeGitHubActionsRunnerCapabilities, gitHubActionsSessionStatus, githubActionsCapabilities, + githubActionsFramedRunnerCapability, githubActionsRelayRole, + githubActionsRunnerProtocolQuery, githubActionsRuntimeLabel, + gitHubActionsRunnerUsesFramedProtocol, isGitHubActionsViewerControlMessage, isTerminalGitHubActionsWorkState, notifyGitHubActionsViewers, @@ -17,6 +20,7 @@ import { parseGitHubActionsRelayInput, parseGitHubActionsRelayInputAcknowledgement, parseGitHubActionsRelayOutput, + parseGitHubActionsRunnerProtocol, parseGitHubActionsWorkState, relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, @@ -68,6 +72,13 @@ test("runner URL works without custom WebSocket headers", () => { buildGitHubActionsRunnerPtyUrl("https://crabfleet.openclaw.ai", "IS-123", "token with spaces"), "wss://crabfleet.openclaw.ai/api/agent/interactive-sessions/IS-123/runner-pty?agentToken=token+with+spaces", ); + assert.equal(parseGitHubActionsRunnerProtocol(null), null); + assert.equal(parseGitHubActionsRunnerProtocol("cfr1-framed-io-v2"), null); + assert.equal( + parseGitHubActionsRunnerProtocol(githubActionsFramedRunnerCapability), + githubActionsFramedRunnerCapability, + ); + assert.equal(githubActionsRunnerProtocolQuery, "runnerProtocol"); }); test("work states preserve running phases and map terminal outcomes", () => { @@ -111,6 +122,24 @@ test("relay replaces the current runner and frames legacy raw runner output", () new TextDecoder().decode(parseGitHubActionsRelayOutput(viewerTwo.sent[0]!)!), "output", ); + + const oldCapabilityMessage = + '{"type":"crabfleet_runner_capabilities","capabilities":["cfr1-framed-io-v1"]}'; + assert.equal( + relayGitHubActionsWebSocketMessage( + "runner", + runner, + oldCapabilityMessage, + [runner], + [viewerOne], + ), + 1, + ); + assert.equal( + new TextDecoder().decode(parseGitHubActionsRelayOutput(viewerOne.sent[1]!)!), + oldCapabilityMessage, + ); + assert.equal(gitHubActionsRunnerUsesFramedProtocol(runner), false); }); test("legacy runners receive raw input and the relay acknowledges delivery", () => { @@ -126,24 +155,15 @@ test("legacy runners receive raw input and the relay acknowledges delivery", () }); }); -test("negotiated runners receive framed input without an early acknowledgement", () => { +test("connection-time opt-in frames the first input without a pending handshake", () => { const closedRunner = relaySocket(3); const openRunner = relaySocket(); const laterRunner = relaySocket(); const viewer = relaySocket(); const input = encodeGitHubActionsRelayInput("input-one", "steer"); - assert.equal( - relayGitHubActionsWebSocketMessage( - "runner", - openRunner, - encodeGitHubActionsRunnerCapabilities(), - [openRunner], - [], - ), - 0, - ); - openRunner.sent.length = 0; + attachGitHubActionsRunnerProtocol(openRunner, githubActionsFramedRunnerCapability); + assert.equal(gitHubActionsRunnerUsesFramedProtocol(openRunner), true); assert.equal( relayGitHubActionsWebSocketMessage( "viewer", @@ -196,14 +216,7 @@ test("runner acknowledgements retain correlation and fan out to viewers", () => inputId: "input-two", accepted: true, }); - relayGitHubActionsWebSocketMessage( - "runner", - runner, - encodeGitHubActionsRunnerCapabilities(), - [runner], - [], - ); - runner.sent.length = 0; + attachGitHubActionsRunnerProtocol(runner, githubActionsFramedRunnerCapability); assert.equal( relayGitHubActionsWebSocketMessage( @@ -230,17 +243,7 @@ test("negotiated runners frame output so control-shaped terminal bytes stay outp accepted: true, }); - assert.equal( - relayGitHubActionsWebSocketMessage( - "runner", - runner, - encodeGitHubActionsRunnerCapabilities(), - [runner], - [viewer], - ), - 0, - ); - assert.equal(typeof runner.sent[0], "string"); + attachGitHubActionsRunnerProtocol(runner, githubActionsFramedRunnerCapability); const output = encodeGitHubActionsRelayOutput(controlShapedOutput); assert.equal(relayGitHubActionsWebSocketMessage("runner", runner, output, [runner], [viewer]), 1); assert.deepEqual( From 6e60b01065b0b4bce3f795acb36c19b8c2376efe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:57:53 +0200 Subject: [PATCH 086/242] docs(actions): describe legacy runner compatibility --- docs/runs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/runs.md b/docs/runs.md index 31ee84da..a94f0d82 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -120,9 +120,9 @@ Terminal contract: GitHub Actions PTY contract: - OpenClaw registers or resumes work through `POST /api/openclaw/action-sessions`. -- The returned `runnerPtyUrl` is a `wss:` URL with a rotated session-scoped query credential. Node's global `WebSocket` can open it without custom headers, but the runner must implement the `CFR1` protocol. -- The Actions process sends unframed raw terminal output. Viewer input arrives in correlated binary `CFR1` frames, and the runner returns the matching acknowledgement only after its PTY accepts the write. -- Legacy runners that expect raw viewer input are incompatible; unframed viewer input is rejected. +- The returned `runnerPtyUrl` is a `wss:` URL with a rotated session-scoped query credential. Node's global `WebSocket` can open it without custom headers. +- Legacy runners open the returned URL unchanged and retain raw input/output with relay-level delivery reporting. +- Framed runners add the exact `runnerProtocol=cfr1-framed-io-v1` query before opening the socket. Viewer input and runner output then use collision-free binary `CFR1` frames, and the runner returns the matching acknowledgement only after its PTY accepts the write. - `SessionControlDO` allows one current runner and multiple viewers. A new runner replaces the previous runner; viewers remain connected and receive runner lifecycle events. - Authorized browser viewers attach through the existing `/api/terminal/ws` hub. Service and agent credentials are never included in viewer responses. - The runner updates `state`, `phase`, `summary`, Codex thread/turn IDs, and heartbeat through the agent work-state endpoint. `completed`, `blocked`, `failed`, and `canceled` are terminal. From c5a9da8e9d2b5e538dde86b71420b9b2a38e6811 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:04:45 +0200 Subject: [PATCH 087/242] fix(runtime): validate profile route templates --- src/worker/deployment.ts | 24 ++++++++++++++++++++++++ tests/deployment.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/worker/deployment.ts b/src/worker/deployment.ts index 4ca7710b..32494879 100644 --- a/src/worker/deployment.ts +++ b/src/worker/deployment.ts @@ -9,6 +9,7 @@ import { runtimeProfileByID, type RuntimeProfileDescriptor, } from "../runtime-profiles.ts"; +import { runtimeAdapterControlPlaneForProfile } from "../runtime-adapter.ts"; import { trustedProxyPublicOrigin, type TrustedProxyEnv } from "../trusted-proxy-auth.ts"; import { configuredHttpOrigin } from "../url-security.ts"; import { badRequest } from "./http.ts"; @@ -43,6 +44,8 @@ export type DeploymentEnv = TrustedProxyEnv & { CRABFLEET_INTERACTIVE_RUNTIMES?: string; CRABFLEET_DEFAULT_PROFILE?: string; CRABFLEET_RUNTIME_PROFILES_JSON?: string; + CRABBOX_RUNTIME_ADAPTER_URL?: string; + CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE?: string; }; export function deploymentConfig(env: DeploymentEnv): DeploymentConfig { @@ -52,6 +55,7 @@ export function deploymentConfig(env: DeploymentEnv): DeploymentConfig { if (runtimeProfiles.length > 0 && !runtimeProfileByID(runtimeProfiles, defaultProfile)) { throw new TypeError("CRABFLEET_DEFAULT_PROFILE must name a configured runtime profile"); } + validateRuntimeProfileRoutes(env, runtimeProfiles, defaultProfile); return { label: clean(env.CRABFLEET_LABEL, 80) || "Crabfleet", canonicalUrl: configuredHttpOrigin(env.CRABFLEET_CANONICAL_URL, appCanonicalOrigin), @@ -65,6 +69,26 @@ export function deploymentConfig(env: DeploymentEnv): DeploymentConfig { }; } +function validateRuntimeProfileRoutes( + env: DeploymentEnv, + runtimeProfiles: RuntimeProfileDescriptor[], + defaultProfile: string, +): void { + const direct = env.CRABBOX_RUNTIME_ADAPTER_URL; + const template = env.CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE; + if (!template || direct) return; + const profileIDs = + runtimeProfiles.length > 0 ? runtimeProfiles.map((profile) => profile.id) : [defaultProfile]; + const unroutable = profileIDs.find( + (profile) => !runtimeAdapterControlPlaneForProfile(undefined, template, profile), + ); + if (unroutable) { + throw new TypeError( + `runtime profile ${unroutable} cannot be routed by CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE`, + ); + } +} + export function selectedRuntimeProfile( deployment: DeploymentConfig, value: unknown, diff --git a/tests/deployment.test.ts b/tests/deployment.test.ts index 050a79b3..eca4f639 100644 --- a/tests/deployment.test.ts +++ b/tests/deployment.test.ts @@ -77,6 +77,30 @@ test("configured runtime profiles are allowlisted behaviorally", () => { ); }); +test("profile-routed deployments reject profiles the adapter template cannot address", () => { + assert.throws( + () => + deploymentConfig({ + CRABFLEET_DEFAULT_PROFILE: "Desktop.PROFILE_2026", + CRABFLEET_RUNTIME_PROFILES_JSON: JSON.stringify([ + { id: "Desktop.PROFILE_2026", label: "Desktop" }, + ]), + CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE: "https://controller.example.test/adapters/{profile}", + }), + /runtime profile Desktop\.PROFILE_2026 cannot be routed/, + ); + assert.equal( + deploymentConfig({ + CRABFLEET_DEFAULT_PROFILE: "Desktop.PROFILE_2026", + CRABFLEET_RUNTIME_PROFILES_JSON: JSON.stringify([ + { id: "Desktop.PROFILE_2026", label: "Desktop" }, + ]), + CRABBOX_RUNTIME_ADAPTER_URL: "https://controller.example.test/adapter", + }).defaultProfile, + "Desktop.PROFILE_2026", + ); +}); + test("public and client deployment views exclude server-only routing data", () => { const env: DeploymentEnv = { CRABFLEET_CANONICAL_URL: "https://backend.example", From 6dfc57b6a3aba17027175dd645ce82f496a70288 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:04:45 +0200 Subject: [PATCH 088/242] fix(image): preserve verified version overrides --- CHANGELOG.md | 4 ++-- Dockerfile | 27 ++++++++++++++++++++++----- tests/dockerfile.test.ts | 19 +++++++++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 tests/dockerfile.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b303b5bf..3ba1ffe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,11 @@ - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, rollback-restored Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. -- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. +- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers during deployment configuration, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. - Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. -- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly. +- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, retaining version-only image overrides by checking non-default releases against their published checksum manifests. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. - Add persisted send-only and receive-only clipboard directions to the native viewer's focus toolbar; automatic sync respects the direction while the explicit Send and Get actions keep working. diff --git a/Dockerfile b/Dockerfile index 1928ac62..dcb0d1b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ FROM docker.io/cloudflare/sandbox:0.10.1 USER root ARG CRABBOX_VERSION=0.17.1 -ARG CRABBOX_SHA256_AMD64=3c41839257e4622e28bcec8b0f0153f19d78d436fd548894a7c7d7726d922611 -ARG CRABBOX_SHA256_ARM64=4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2 +ARG CRABBOX_SHA256_AMD64= +ARG CRABBOX_SHA256_ARM64= RUN set -eux; \ apt-get update; \ @@ -64,12 +64,29 @@ RUN set -eux; \ RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "$arch" in \ - amd64) checksum="$CRABBOX_SHA256_AMD64" ;; \ - arm64) checksum="$CRABBOX_SHA256_ARM64" ;; \ + amd64) \ + checksum="$CRABBOX_SHA256_AMD64"; \ + pinned_checksum="3c41839257e4622e28bcec8b0f0153f19d78d436fd548894a7c7d7726d922611" \ + ;; \ + arm64) \ + checksum="$CRABBOX_SHA256_ARM64"; \ + pinned_checksum="4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2" \ + ;; \ *) echo "unsupported arch: $arch" >&2; exit 1 ;; \ esac; \ + archive="crabbox_${CRABBOX_VERSION}_linux_${arch}.tar.gz"; \ + if [ -z "$checksum" ]; then \ + if [ "$CRABBOX_VERSION" = "0.17.1" ]; then \ + checksum="$pinned_checksum"; \ + else \ + checksum="$(curl -fsSL \ + "https://github.com/openclaw/crabbox/releases/download/v${CRABBOX_VERSION}/checksums.txt" \ + | awk -v archive="$archive" '$2 == archive { print $1 }')"; \ + fi; \ + fi; \ + printf '%s\n' "$checksum" | grep -Eq '^[0-9a-f]{64}$'; \ curl -fsSL \ - "https://github.com/openclaw/crabbox/releases/download/v${CRABBOX_VERSION}/crabbox_${CRABBOX_VERSION}_linux_${arch}.tar.gz" \ + "https://github.com/openclaw/crabbox/releases/download/v${CRABBOX_VERSION}/${archive}" \ -o /tmp/crabbox.tar.gz; \ echo "$checksum /tmp/crabbox.tar.gz" | sha256sum -c -; \ tar -xzf /tmp/crabbox.tar.gz -C /usr/local/bin crabbox; \ diff --git a/tests/dockerfile.test.ts b/tests/dockerfile.test.ts new file mode 100644 index 00000000..c7c8fb85 --- /dev/null +++ b/tests/dockerfile.test.ts @@ -0,0 +1,19 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { test } from "node:test"; + +test("Crabbox image pins the default release and verifies version overrides", async () => { + const dockerfile = await readFile(new URL("../Dockerfile", import.meta.url), "utf8"); + + assert.match( + dockerfile, + /pinned_checksum="3c41839257e4622e28bcec8b0f0153f19d78d436fd548894a7c7d7726d922611"/, + ); + assert.match( + dockerfile, + /pinned_checksum="4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2"/, + ); + assert.match(dockerfile, /releases\/download\/v\$\{CRABBOX_VERSION\}\/checksums\.txt/); + assert.match(dockerfile, /grep -Eq '\^\[0-9a-f\]\{64\}\$'/); + assert.match(dockerfile, /sha256sum -c -/); +}); From a0113f0df03a377ea98b2b737b7c94be23ddf33f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:13:15 +0200 Subject: [PATCH 089/242] fix(runtime): reject ambiguous adapter urls --- src/worker/deployment.ts | 9 ++++++++- tests/deployment.test.ts | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/worker/deployment.ts b/src/worker/deployment.ts index 32494879..8b6978ac 100644 --- a/src/worker/deployment.ts +++ b/src/worker/deployment.ts @@ -76,7 +76,14 @@ function validateRuntimeProfileRoutes( ): void { const direct = env.CRABBOX_RUNTIME_ADAPTER_URL; const template = env.CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE; - if (!template || direct) return; + const hasDirect = typeof direct === "string" && direct.length > 0; + const hasTemplate = typeof template === "string" && template.length > 0; + if (hasDirect && hasTemplate) { + throw new TypeError( + "CRABBOX_RUNTIME_ADAPTER_URL and CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE are mutually exclusive", + ); + } + if (!hasTemplate) return; const profileIDs = runtimeProfiles.length > 0 ? runtimeProfiles.map((profile) => profile.id) : [defaultProfile]; const unroutable = profileIDs.find( diff --git a/tests/deployment.test.ts b/tests/deployment.test.ts index eca4f639..d0da1c25 100644 --- a/tests/deployment.test.ts +++ b/tests/deployment.test.ts @@ -78,6 +78,14 @@ test("configured runtime profiles are allowlisted behaviorally", () => { }); test("profile-routed deployments reject profiles the adapter template cannot address", () => { + assert.throws( + () => + deploymentConfig({ + CRABBOX_RUNTIME_ADAPTER_URL: "https://controller.example.test/adapter", + CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE: "https://controller.example.test/adapters/{profile}", + }), + /are mutually exclusive/, + ); assert.throws( () => deploymentConfig({ From 6c8a0babeca8fbb3696456100d8130e1e3aef2c2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:13:15 +0200 Subject: [PATCH 090/242] fix(image): require pinned override checksums --- CHANGELOG.md | 2 +- Dockerfile | 7 ++++--- tests/dockerfile.test.ts | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ba1ffe2..ac85b54b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. - Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. -- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, retaining version-only image overrides by checking non-default releases against their published checksum manifests. +- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, requiring an explicit architecture checksum when overriding the pinned default version. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. - Add persisted send-only and receive-only clipboard directions to the native viewer's focus toolbar; automatic sync respects the direction while the explicit Send and Get actions keep working. diff --git a/Dockerfile b/Dockerfile index dcb0d1b0..23e24c85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,10 +66,12 @@ RUN set -eux; \ case "$arch" in \ amd64) \ checksum="$CRABBOX_SHA256_AMD64"; \ + checksum_arg="CRABBOX_SHA256_AMD64"; \ pinned_checksum="3c41839257e4622e28bcec8b0f0153f19d78d436fd548894a7c7d7726d922611" \ ;; \ arm64) \ checksum="$CRABBOX_SHA256_ARM64"; \ + checksum_arg="CRABBOX_SHA256_ARM64"; \ pinned_checksum="4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2" \ ;; \ *) echo "unsupported arch: $arch" >&2; exit 1 ;; \ @@ -79,9 +81,8 @@ RUN set -eux; \ if [ "$CRABBOX_VERSION" = "0.17.1" ]; then \ checksum="$pinned_checksum"; \ else \ - checksum="$(curl -fsSL \ - "https://github.com/openclaw/crabbox/releases/download/v${CRABBOX_VERSION}/checksums.txt" \ - | awk -v archive="$archive" '$2 == archive { print $1 }')"; \ + echo "an explicit $checksum_arg is required for non-default versions" >&2; \ + exit 1; \ fi; \ fi; \ printf '%s\n' "$checksum" | grep -Eq '^[0-9a-f]{64}$'; \ diff --git a/tests/dockerfile.test.ts b/tests/dockerfile.test.ts index c7c8fb85..b458651f 100644 --- a/tests/dockerfile.test.ts +++ b/tests/dockerfile.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import { test } from "node:test"; -test("Crabbox image pins the default release and verifies version overrides", async () => { +test("Crabbox image pins the default release and requires pinned version overrides", async () => { const dockerfile = await readFile(new URL("../Dockerfile", import.meta.url), "utf8"); assert.match( @@ -13,7 +13,8 @@ test("Crabbox image pins the default release and verifies version overrides", as dockerfile, /pinned_checksum="4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2"/, ); - assert.match(dockerfile, /releases\/download\/v\$\{CRABBOX_VERSION\}\/checksums\.txt/); + assert.doesNotMatch(dockerfile, /checksums\.txt/); + assert.match(dockerfile, /an explicit \$checksum_arg is required/); assert.match(dockerfile, /grep -Eq '\^\[0-9a-f\]\{64\}\$'/); assert.match(dockerfile, /sha256sum -c -/); }); From 601ec041e39cbb0cea55b07866a037a13a2c8940 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:13:15 +0200 Subject: [PATCH 091/242] docs(actions): clarify raw runner fallback --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f295633..bc3f39b0 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Content-Type: application/json {"workKey":"openclaw/crabfleet:pr:42","workKind":"pr_repair","repo":"openclaw/crabfleet","branch":"fix/pr-42","owner":"operator@example.test","sourceUrl":"https://github.com/openclaw/crabfleet/pull/42","runUrl":"https://github.com/openclaw/crabfleet/actions/runs/123","purpose":"repair PR 42","summary":"starting repair"} ``` -The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and can be opened with Node's global `WebSocket` without custom headers, but it is not a raw duplex byte stream. The abbreviated runner handler is: +The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and can be opened unchanged as the legacy raw duplex byte stream. New runners opt into framed input, output, and acknowledgements by adding the exact query parameter shown below before opening the socket: ```js const framedRunnerPtyUrl = new URL(runnerPtyUrl); From 84c73169099ad7bcc953a10a1be108eef228686f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:15:04 +0200 Subject: [PATCH 092/242] fix(macos): await share cleanup on termination --- .../CrabfleetMac/CrabboxVNCBridge.swift | 9 +++- .../CrabfleetMac/CrabfleetMacApp.swift | 25 ++++++++++- .../PrivateMacShareController.swift | 6 +++ .../CrabfleetMac/SubprocessEnvironment.swift | 22 ++++++++++ .../CrabfleetMacTests/FleetModelsTests.swift | 21 ++++++++++ .../PrivateMacShareTests.swift | 41 +++++++++++++++++++ 6 files changed, 122 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift index d47a25f2..a29bf205 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabboxVNCBridge.swift @@ -36,6 +36,12 @@ private struct CrabboxVNCHandoff: Decodable { } final class CrabboxVNCBridge: @unchecked Sendable { + private static let configEnvironmentKeys = [ + "CRABBOX_CONFIG", + "XDG_CONFIG_HOME", + "XDG_STATE_HOME", + ] + private static let networkEnvironmentKeys = [ "ALL_PROXY", "HTTP_PROXY", @@ -263,7 +269,8 @@ final class CrabboxVNCBridge: @unchecked Sendable { SubprocessEnvironment.minimal( from: source, includeSSHAgent: true, - additionalInheritedKeys: networkEnvironmentKeys + additionalInheritedKeys: networkEnvironmentKeys, + additionalInheritedPathKeys: configEnvironmentKeys ) } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift index 0e5d6a89..5afca1bf 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift @@ -48,15 +48,26 @@ enum VNCConnectionLaunchMode { @MainActor final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { let shareController: PrivateMacShareController + private let replyToTerminationRequest: @MainActor (Bool) -> Void private var autoShareTask: Task? + private var terminationTask: Task? override init() { shareController = PrivateMacShareController() + replyToTerminationRequest = { shouldTerminate in + NSApp.reply(toApplicationShouldTerminate: shouldTerminate) + } super.init() } - init(shareController: PrivateMacShareController) { + init( + shareController: PrivateMacShareController, + replyToTerminationRequest: @escaping @MainActor (Bool) -> Void = { + NSApp.reply(toApplicationShouldTerminate: $0) + } + ) { self.shareController = shareController + self.replyToTerminationRequest = replyToTerminationRequest super.init() } @@ -70,6 +81,18 @@ final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { } } + func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply { + autoShareTask?.cancel() + guard terminationTask == nil else { return .terminateLater } + terminationTask = Task { [weak self] in + guard let self else { return } + await shareController.stopAndWaitForCleanup() + terminationTask = nil + replyToTerminationRequest(true) + } + return .terminateLater + } + func applicationWillTerminate(_ notification: Notification) { autoShareTask?.cancel() } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 9f95c195..5b0d6b4f 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -384,6 +384,12 @@ final class PrivateMacShareController: ObservableObject { phase = .idle } + func stopAndWaitForCleanup() async { + await stop() + let cleanupTask = registrationTask + await cleanupTask?.value + } + func openPrivacySettings(_ pane: PrivacyPane) { let value: String switch pane { diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift index 645810ca..609d1604 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/SubprocessEnvironment.swift @@ -1,3 +1,4 @@ +import Darwin import Foundation enum SubprocessEnvironment { @@ -15,6 +16,7 @@ enum SubprocessEnvironment { from source: [String: String], includeSSHAgent: Bool = false, additionalInheritedKeys: [String] = [], + additionalInheritedPathKeys: [String] = [], overrides: [String: String] = [:] ) -> [String: String] { var environment = Dictionary( @@ -25,6 +27,12 @@ enum SubprocessEnvironment { for key in additionalInheritedKeys { environment[key] = source[key] } + for key in additionalInheritedPathKeys { + environment.removeValue(forKey: key) + if let value = source[key], isSafeAbsolutePath(value) { + environment[key] = value + } + } environment["PATH"] = safePath if includeSSHAgent, let socket = source["SSH_AUTH_SOCK"], !socket.isEmpty { environment["SSH_AUTH_SOCK"] = socket @@ -34,4 +42,18 @@ enum SubprocessEnvironment { } return environment } + + private static func isSafeAbsolutePath(_ value: String) -> Bool { + guard + value.hasPrefix("/"), + value.utf8.count < Int(PATH_MAX), + value.unicodeScalars.allSatisfy({ + !CharacterSet.controlCharacters.contains($0) + }) + else { + return false + } + return !value.split(separator: "/", omittingEmptySubsequences: false) + .contains { $0 == "." || $0 == ".." } + } } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift index c35a7539..bd75521d 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/FleetModelsTests.swift @@ -27,6 +27,9 @@ struct FleetModelsTests { "NO_PROXY": "localhost,.example.test", "SSL_CERT_FILE": "/etc/ssl/custom-ca.pem", "SSL_CERT_DIR": "/etc/ssl/custom-certs", + "CRABBOX_CONFIG": "/Users/tester/.config/crabbox/config.yaml", + "XDG_CONFIG_HOME": "/Users/tester/.config", + "XDG_STATE_HOME": "/Users/tester/.local/state", "CRABFLEET_SESSION_COOKIE": "secret", "NODE_TLS_REJECT_UNAUTHORIZED": "0", ] @@ -39,10 +42,28 @@ struct FleetModelsTests { #expect(environment["NO_PROXY"] == "localhost,.example.test") #expect(environment["SSL_CERT_FILE"] == "/etc/ssl/custom-ca.pem") #expect(environment["SSL_CERT_DIR"] == "/etc/ssl/custom-certs") + #expect(environment["CRABBOX_CONFIG"] == "/Users/tester/.config/crabbox/config.yaml") + #expect(environment["XDG_CONFIG_HOME"] == "/Users/tester/.config") + #expect(environment["XDG_STATE_HOME"] == "/Users/tester/.local/state") #expect(environment["CRABFLEET_SESSION_COOKIE"] == nil) #expect(environment["NODE_TLS_REJECT_UNAUTHORIZED"] == nil) } + @Test + func crabboxRejectsUnsafeConfigEnvironmentPaths() { + let environment = CrabboxVNCBridge.commandEnvironment( + from: [ + "CRABBOX_CONFIG": "relative/config.yaml", + "XDG_CONFIG_HOME": "/Users/tester/../other-config", + "XDG_STATE_HOME": "/" + String(repeating: "a", count: Int(PATH_MAX)), + ] + ) + + #expect(environment["CRABBOX_CONFIG"] == nil) + #expect(environment["XDG_CONFIG_HOME"] == nil) + #expect(environment["XDG_STATE_HOME"] == nil) + } + @Test func sizesRemoteDesktopToEvenViewportPixelsWithinPerformanceCap() { #expect( diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 1b4d9ff1..8bd301d4 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -360,6 +360,47 @@ struct PrivateMacShareTests { #expect(delegate.shareController === controller) } + @Test @MainActor + func applicationTerminationWaitsForPrivateShareCleanup() async throws { + let registration = SuspendedDesktopCleanupRegistration() + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + let identity = desktopIdentity(name: "termination-cleanup", address: "100.64.12.44") + try await lifecycle.publish(identity: identity, port: 5_901) + + let runner = SuspendedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: registration, + registrationLifecycle: lifecycle, + defaults: defaults + ) + let startTask = Task { await controller.start() } + #expect(await waitUntilAsync { await runner.hasStarted }) + + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) } + ) + + #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) + #expect(await waitUntilAsync { await registration.hasStartedUnregistration }) + #expect(replies.isEmpty) + + await runner.resume( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + await startTask.value + await registration.finishUnregistration() + + #expect(await waitUntilAsync { replies == [true] }) + #expect(controller.phase == .idle) + #expect(controller.registryPhase == .notPublished) + } + @Test func privateShareCanStartViewOnlyWithoutAccessibility() { #expect( From bd0f7b7b6b3f29db1284555e864f020a4c16979a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:15:11 +0200 Subject: [PATCH 093/242] fix(credentials): claim rollback recovery ownership --- ...-credential-policy-registration-service.ts | 38 ++++++- .../sandbox-credential-policy-repository.ts | 38 +++++++ .../sandbox-credential-policy-scanner.ts | 36 +++---- ...ndbox-credential-policy-repository.test.ts | 100 ++++++++++++++++++ .../sandbox-credential-policy-scanner.test.ts | 28 ++++- 5 files changed, 210 insertions(+), 30 deletions(-) diff --git a/src/worker/sandbox-credential-policy-registration-service.ts b/src/worker/sandbox-credential-policy-registration-service.ts index 0cc215dd..a2b19c1c 100644 --- a/src/worker/sandbox-credential-policy-registration-service.ts +++ b/src/worker/sandbox-credential-policy-registration-service.ts @@ -28,10 +28,35 @@ import type { SandboxRuntimeSession } from "./sandbox-runtime.ts"; import { sandboxControlStub } from "./session-control-do.ts"; import type { SandboxCredentialPolicy, + SandboxCredentialPolicyRegistration, StoredSandboxCredentialPolicy, } from "./session-control-policy.ts"; import type { InteractiveSession } from "./session-model.ts"; +type RestoreSandboxCredentialPolicyRollback = typeof restoreSandboxCredentialPolicyRollback; + +export async function restoreSandboxCredentialPolicyRollbackIfOwned( + env: RuntimeEnv, + stub: Pick, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + rollbackJson: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + restoreRollback: RestoreSandboxCredentialPolicyRollback = restoreSandboxCredentialPolicyRollback, +): Promise { + const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + ); + if (!registrationExpiresAt) return false; + await restoreRollback(stub, registration, registrationExpiresAt, rollbackJson, sessionId); + return true; +} + export async function registerSandboxCredentialPolicy( env: RuntimeEnv, session: SandboxRuntimeSession, @@ -53,7 +78,6 @@ export async function registerSandboxCredentialPolicy( sandboxId, ownershipFence, ); - let latestRegistrationExpiresAt = 0; let rollbackJson: string | null = null; let registrationWriteStarted = false; try { @@ -121,7 +145,6 @@ export async function registerSandboxCredentialPolicy( if (!registrationExpiresAt) { throw new Error("sandbox credential policy registration claim was revoked"); } - latestRegistrationExpiresAt = registrationExpiresAt; registrationWriteStarted = true; const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { method: "POST", @@ -152,13 +175,18 @@ export async function registerSandboxCredentialPolicy( const message = clean(error instanceof Error ? error.message : String(error), 500); if (registrationWriteStarted && rollbackJson) { try { - await restoreSandboxCredentialPolicyRollback( + const restored = await restoreSandboxCredentialPolicyRollbackIfOwned( + env, stub, + session.id, + sandboxId, registration, - latestRegistrationExpiresAt, rollbackJson, - session.id, + ownershipFence, ); + if (!restored) { + throw new Error("sandbox credential policy registration claim was revoked"); + } } catch (rollbackError) { const rollbackMessage = clean( rollbackError instanceof Error ? rollbackError.message : String(rollbackError), diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 7a61e071..57e372d4 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -586,6 +586,44 @@ export async function renewSandboxCredentialPolicyRegistration( return Number(renewed.numUpdatedRows ?? 0n) === 1 ? registrationExpiresAt : null; } +export async function claimSandboxCredentialPolicyRegistrationRecovery( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + expiredRegistration: SandboxCredentialPolicyRegistration, + expiredRegistrationExpiresAt: number, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise<{ + registration: SandboxCredentialPolicyRegistration; + registrationExpiresAt: number; +} | null> { + const now = Date.now(); + const registration = { + ...expiredRegistration, + claim: `registration:${crypto.randomUUID()}`, + }; + const registrationExpiresAt = now + credentialPolicyRegistrationClaimMs; + const claimed = await database(env) + .updateTable("interactive_session_credential_policy_registrations") + .set({ + registration_claim: registration.claim, + registration_claim_expires_at: registrationExpiresAt, + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("state", "=", "registering") + .where("registration_generation", "=", expiredRegistration.generation) + .where("registration_claim", "=", expiredRegistration.claim) + .where("registration_claim_expires_at", "=", expiredRegistrationExpiresAt) + .where("registration_claim_expires_at", "<=", now) + .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .executeTakeFirst(); + return Number(claimed.numUpdatedRows ?? 0n) === 1 + ? { registration, registrationExpiresAt } + : null; +} + export async function recordSandboxCredentialPolicyRollback( env: RuntimeEnv, sessionId: string, diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index be2ebb2b..277dc3f5 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -10,9 +10,9 @@ import type { RuntimeEnv } from "./env.ts"; import type { InteractiveSessionStatus } from "./models.ts"; import { abandonSandboxCredentialPolicyRegistration, + claimSandboxCredentialPolicyRegistrationRecovery, finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, - renewSandboxCredentialPolicyRegistration, sandboxCredentialPolicyCleanupAuthorizedCondition, sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, @@ -350,13 +350,22 @@ async function scanStagedCredentialPolicyRegistrations( ); continue; } + const recovery = await claimSandboxCredentialPolicyRegistrationRecovery( + env, + row.session_id, + row.sandbox_id, + registration, + row.registration_claim_expires_at, + ownershipFence, + ); + if (!recovery) continue; if ( (await policyExists(env, row.sandbox_id, row.registration_generation)) && (await finishSandboxCredentialPolicyRegistration( env, row.session_id, row.sandbox_id, - registration, + recovery.registration, ownershipFence, )) ) { @@ -364,26 +373,9 @@ async function scanStagedCredentialPolicyRegistrations( } if (row.rollback_policies_json !== null) { if (!restoreRollback) throw new Error("sandbox credential policy rollback is unavailable"); - const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( - env, - row.session_id, - row.sandbox_id, - registration, - ownershipFence, - ); - if (!registrationExpiresAt) { - await abandonSandboxCredentialPolicyRegistration( - env, - row.session_id, - row.sandbox_id, - registration, - "sandbox credential policy owner changed before rollback", - ); - continue; - } await restoreRollback({ - registration, - registrationExpiresAt, + registration: recovery.registration, + registrationExpiresAt: recovery.registrationExpiresAt, rollbackJson: row.rollback_policies_json, sessionId: row.session_id, }); @@ -392,7 +384,7 @@ async function scanStagedCredentialPolicyRegistrations( env, row.session_id, row.sandbox_id, - registration, + recovery.registration, "sandbox credential policy registration did not complete", ); } catch (error) { diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 5b68790e..394265db 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -6,10 +6,12 @@ import { activeSandboxCredentialPolicyGeneration, abandonSandboxCredentialPolicyRegistration, beginSandboxCredentialPolicyRegistration, + claimSandboxCredentialPolicyRegistrationRecovery, currentSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, + renewSandboxCredentialPolicyRegistration, sandboxCredentialPolicyRegistrationQueries, sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, @@ -470,6 +472,104 @@ test("partial credential-policy rotation failure preserves the prior active gene ); }); +test("stale foreground rollback cannot renew after recovery takes its claim", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const expiredAt = 1; + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(expiredAt); + const recovery = await claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + staged, + expiredAt, + ownershipFence, + ); + assert.ok(recovery); + + const renewed = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ); + + assert.equal(renewed, null); +}); + +test("expired registration recovery grants one fresh exclusive claim", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const expiredAt = 1; + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(expiredAt); + + const claims = await Promise.all([ + claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + staged, + expiredAt, + ownershipFence, + ), + claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + staged, + expiredAt, + ownershipFence, + ), + ]); + const winner = claims.find((claim) => claim !== null); + + assert.equal(claims.filter((claim) => claim !== null).length, 1); + assert.ok(winner); + assert.notEqual(winner.registration.claim, staged.claim); + assert.ok(winner.registrationExpiresAt > expiredAt); + assert.deepEqual( + { + ...sqlite + .prepare(` + SELECT registration_generation, registration_claim, registration_claim_expires_at + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(), + }, + { + registration_generation: staged.generation, + registration_claim: winner.registration.claim, + registration_claim_expires_at: winner.registrationExpiresAt, + }, + ); +}); + test("completed credential-policy rotation atomically promotes every active lookup", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); diff --git a/tests/sandbox-credential-policy-scanner.test.ts b/tests/sandbox-credential-policy-scanner.test.ts index efa5911b..1b79d4a5 100644 --- a/tests/sandbox-credential-policy-scanner.test.ts +++ b/tests/sandbox-credential-policy-scanner.test.ts @@ -123,7 +123,7 @@ test("credential-policy scan rejects incomplete, expired, and mismatched ownersh ); }); -test("staged rollback reclaims current ownership before restoring policy", async () => { +test("staged recovery takes a fresh exclusive claim before promotion or rollback", async () => { const source = await readFile( new URL("../src/worker/sandbox-credential-policy-scanner.ts", import.meta.url), "utf8", @@ -136,10 +136,32 @@ test("staged rollback reclaims current ownership before restoring policy", async stagedRecovery.indexOf("if (!ownershipFence)") < stagedRecovery.indexOf("restoreRollback({"), ); assert.ok( - stagedRecovery.indexOf("renewSandboxCredentialPolicyRegistration(") < + stagedRecovery.indexOf("claimSandboxCredentialPolicyRegistrationRecovery(") < + stagedRecovery.indexOf("policyExists("), + ); + assert.ok( + stagedRecovery.indexOf("claimSandboxCredentialPolicyRegistrationRecovery(") < stagedRecovery.indexOf("restoreRollback({"), ); - assert.match(stagedRecovery, /if \(!registrationExpiresAt\)/); + assert.doesNotMatch(stagedRecovery, /renewSandboxCredentialPolicyRegistration/); +}); + +test("foreground rollback rechecks its exact claim before restoring policy", async () => { + const source = await readFile( + new URL("../src/worker/sandbox-credential-policy-registration-service.ts", import.meta.url), + "utf8", + ); + const start = source.indexOf( + "export async function restoreSandboxCredentialPolicyRollbackIfOwned", + ); + const end = source.indexOf("export async function registerSandboxCredentialPolicy", start); + const rollbackRecovery = source.slice(start, end); + + assert.ok( + rollbackRecovery.indexOf("renewSandboxCredentialPolicyRegistration(") < + rollbackRecovery.indexOf("restoreRollback("), + ); + assert.match(rollbackRecovery, /if \(!registrationExpiresAt\) return false/); }); test("credential-policy scan preserves live standalone and managed policies", () => { From efe22a4238074f5cf9fbdb1dc6ac2c23b73d81c6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:15:16 +0200 Subject: [PATCH 094/242] fix(desktop): return atomic registration row --- src/worker/desktop-host-repository.ts | 9 +--- tests/desktop-host-repository.test.ts | 75 ++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 1c5d03a4..83c7bd5b 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -50,7 +50,7 @@ export class DesktopHostRepository implements DesktopHostStore { } async upsert(host: DesktopHostWrite): Promise { - await database(this.env) + const row = await database(this.env) .insertInto("desktop_hosts") .values({ owner_subject: host.ownerSubject, @@ -73,12 +73,7 @@ export class DesktopHostRepository implements DesktopHostStore { updated_at: host.updatedAt, }), ) - .execute(); - const row = await database(this.env) - .selectFrom("desktop_hosts") - .selectAll() - .where("owner_subject", "=", host.ownerSubject) - .where("id", "=", host.id) + .returningAll() .executeTakeFirstOrThrow(); return { ownerSubject: row.owner_subject, diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index d48b56b5..853fad91 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -67,16 +67,77 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec }); assert.equal(upserted.id, "studio"); assert.match(executions[1]?.sql ?? "", /^insert into "desktop_hosts"/i); - assert.match(executions[2]?.sql ?? "", /where "owner_subject" = \? and "id" = \?/i); - assert.deepEqual(executions[2]?.parameters, ["github:1", "studio"]); + assert.match(executions[1]?.sql ?? "", /\breturning \*/i); await repository.remove("github:1", "studio", "ownership-token"); + assert.match(executions[2]?.sql ?? "", /^delete from "desktop_hosts"/i); + assert.match(executions[2]?.sql ?? "", /"ownership_token" = \?/i); + assert.deepEqual(executions[2]?.parameters, ["github:1", "studio", "ownership-token"]); + + await repository.remove("github:1", "legacy-studio", null); assert.match(executions[3]?.sql ?? "", /^delete from "desktop_hosts"/i); assert.match(executions[3]?.sql ?? "", /"ownership_token" = \?/i); - assert.deepEqual(executions[3]?.parameters, ["github:1", "studio", "ownership-token"]); + assert.deepEqual(executions[3]?.parameters, ["github:1", "legacy-studio", ""]); +}); - await repository.remove("github:1", "legacy-studio", null); - assert.match(executions[4]?.sql ?? "", /^delete from "desktop_hosts"/i); - assert.match(executions[4]?.sql ?? "", /"ownership_token" = \?/i); - assert.deepEqual(executions[4]?.parameters, ["github:1", "legacy-studio", ""]); +test("desktop host upsert returns the row written by the same atomic statement", async () => { + const executions: string[] = []; + const written = { + owner_subject: "github:1", + id: "studio", + owner: "alice", + name: "Host A", + address: "100.64.1.2", + port: 5901, + ownership_token: "token-a", + created_at: 1, + updated_at: 2, + }; + const competing = { + ...written, + owner: "bob", + name: "Host B", + address: "100.64.1.3", + ownership_token: "token-b", + updated_at: 3, + }; + const env = { + DB: { + prepare(sql: string) { + executions.push(sql); + return { + bind() { + return { + async all() { + return { + results: [/^insert into "desktop_hosts"/i.test(sql) ? written : competing], + meta: { changes: 1 }, + }; + }, + async run() { + return { meta: { changes: 1 } }; + }, + }; + }, + }; + }, + } as unknown as D1Database, + } as RuntimeEnv; + + const row = await new DesktopHostRepository(env).upsert({ + ownerSubject: written.owner_subject, + id: written.id, + owner: written.owner, + name: written.name, + address: written.address, + port: written.port, + ownershipToken: written.ownership_token, + createdAt: written.created_at, + updatedAt: written.updated_at, + }); + + assert.equal(executions.length, 1); + assert.match(executions[0] ?? "", /^insert into "desktop_hosts".*\breturning \*/is); + assert.equal(row.name, "Host A"); + assert.equal(row.ownershipToken, "token-a"); }); From 0f7ac2dba2d976087a20f692a37ca5834b570677 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:17:43 +0200 Subject: [PATCH 095/242] fix(terminal): route confirmations through one reader --- internal/terminalws/client.go | 313 ++++++++++++++++++++++------- internal/terminalws/client_test.go | 229 ++++++++++++++++++++- src/worker/terminal-hub.ts | 11 +- tests/terminal-hub.test.ts | 75 ++++++- 4 files changed, 548 insertions(+), 80 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 489ba1ef..4bb06b45 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -79,9 +79,16 @@ type Client struct { sessionID string supportsInputAcknowledgement bool cancel context.CancelFunc + readCancel context.CancelFunc canInput atomic.Bool lastSize atomic.Uint64 writeMu sync.Mutex + confirmMu sync.Mutex + stateMu sync.Mutex + inputWaiter chan error + attachment *terminalAttachment + readerDone chan struct{} + readerErr error } type frame struct { @@ -90,6 +97,11 @@ type frame struct { payload []byte } +type terminalAttachment struct { + frames chan frame + done chan struct{} +} + type eventPayload struct { Type string `json:"type"` Error string `json:"error"` @@ -222,6 +234,7 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option if setupTimer != nil { setupTimer.Stop() } + client.startReader() return client, nil } if event.Type == "closed" { @@ -232,6 +245,9 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option } func (c *Client) Close() error { + if c.readCancel != nil { + c.readCancel() + } err := c.conn.Close(websocket.StatusNormalClosure, "") if c.cancel != nil { c.cancel() @@ -260,43 +276,26 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { if !c.supportsInputAcknowledgement { return c.SendInput(ctx, payload) } + c.confirmMu.Lock() + defer c.confirmMu.Unlock() + + waiter := make(chan error, 1) + if err := c.registerInputWaiter(waiter); err != nil { + return err + } if err := c.SendInput(ctx, payload); err != nil { + c.clearInputWaiter(waiter) return err } - for { - current, err := c.read(ctx) - if err != nil { - return err - } - if current.sessionID != "" && current.sessionID != c.sessionID { - continue - } - switch current.messageType { - case messageOutput: - if err := c.write(ctx, frame{ - messageType: messageAck, - sessionID: c.sessionID, - payload: ackPayload(uint32(len(current.payload))), - }); err != nil { - return err - } - case messageError, messageControlRevoked: - c.canInput.Store(false) - return frameError(current, "terminal input rejected") - case messageControlGranted: - c.canInput.Store(true) - case messageEvent: - var event eventPayload - if err := json.Unmarshal(current.payload, &event); err != nil { - return fmt.Errorf("decode terminal event: %w", err) - } - switch event.Type { - case "input-accepted": - return nil - case "closed": - return errors.New("terminal closed before accepting input") - } - } + select { + case err := <-waiter: + return err + case <-c.readerDone: + c.clearInputWaiter(waiter) + return c.readerError() + case <-ctx.Done(): + c.clearInputWaiter(waiter) + return ctx.Err() } } @@ -319,6 +318,12 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c ctx, cancel := context.WithCancel(ctx) defer cancel() + attachment, err := c.registerAttachment() + if err != nil { + return err + } + defer c.clearAttachment(attachment) + var wg sync.WaitGroup canceler, cancelableRead := terminal.(readCanceler) cancelRead := func() { @@ -374,56 +379,53 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c go func() { defer wg.Done() for { - current, err := c.read(ctx) - if err != nil { - errCh <- err + select { + case <-ctx.Done(): return - } - if current.sessionID != "" && current.sessionID != c.sessionID { - continue - } - switch current.messageType { - case messageOutput: - if _, err := terminal.Write(current.payload); err != nil { - errCh <- err - return - } - if err := c.write(ctx, frame{ - messageType: messageAck, - sessionID: c.sessionID, - payload: ackPayload(uint32(len(current.payload))), - }); err != nil { - errCh <- err - return - } - case messageError: - errCh <- frameError(current, "terminal connection failed") + case <-c.readerDone: + errCh <- c.readerError() return - case messageControlRevoked: - c.canInput.Store(false) - case messageControlGranted: - c.canInput.Store(true) - if size := c.rememberedSize(); size.Cols > 0 && size.Rows > 0 { + case current := <-attachment.frames: + switch current.messageType { + case messageOutput: + if _, err := terminal.Write(current.payload); err != nil { + errCh <- err + return + } if err := c.write(ctx, frame{ - messageType: messageResize, + messageType: messageAck, sessionID: c.sessionID, - payload: resizePayload(size), + payload: ackPayload(uint32(len(current.payload))), }); err != nil { errCh <- err return } - } - case messageEvent: - var event eventPayload - if json.Unmarshal(current.payload, &event) == nil && event.Type == "closed" { - errCh <- nil + case messageError: + errCh <- frameError(current, "terminal connection failed") return + case messageControlGranted: + if size := c.rememberedSize(); size.Cols > 0 && size.Rows > 0 { + if err := c.write(ctx, frame{ + messageType: messageResize, + sessionID: c.sessionID, + payload: resizePayload(size), + }); err != nil { + errCh <- err + return + } + } + case messageEvent: + var event eventPayload + if json.Unmarshal(current.payload, &event) == nil && event.Type == "closed" { + errCh <- nil + return + } } } } }() - err := <-errCh + err = <-errCh cancelRead() if cancelableRead { wg.Wait() @@ -446,6 +448,177 @@ func (c *Client) write(ctx context.Context, current frame) error { return c.conn.Write(ctx, websocket.MessageBinary, encodeFrame(current)) } +func (c *Client) startReader() { + ctx, cancel := context.WithCancel(context.Background()) + c.readCancel = cancel + c.readerDone = make(chan struct{}) + go c.readLoop(ctx) +} + +func (c *Client) readLoop(ctx context.Context) { + for { + current, err := c.read(ctx) + if err != nil { + c.finishReader(err) + return + } + if err := c.handleFrame(ctx, current); err != nil { + c.finishReader(err) + return + } + } +} + +func (c *Client) handleFrame(ctx context.Context, current frame) error { + if current.sessionID != "" && current.sessionID != c.sessionID { + return nil + } + switch current.messageType { + case messageOutput: + if c.deliverAttachment(ctx, current) { + return nil + } + return c.write(ctx, frame{ + messageType: messageAck, + sessionID: c.sessionID, + payload: ackPayload(uint32(len(current.payload))), + }) + case messageError: + err := frameError(current, "terminal connection failed") + c.canInput.Store(false) + c.completeInput(err) + return err + case messageControlRevoked: + c.canInput.Store(false) + c.completeInput(frameError(current, "terminal input rejected")) + c.deliverAttachment(ctx, current) + case messageControlGranted: + c.canInput.Store(true) + c.deliverAttachment(ctx, current) + case messageEvent: + var event eventPayload + if err := json.Unmarshal(current.payload, &event); err != nil { + return fmt.Errorf("decode terminal event: %w", err) + } + switch event.Type { + case "subscribed": + c.canInput.Store(event.CanInput) + case "input-accepted": + c.completeInput(nil) + case "input-rejected": + c.completeInput(frameError(current, "terminal input rejected")) + case "closed": + c.completeInput(errors.New("terminal closed before accepting input")) + c.deliverAttachment(ctx, current) + default: + c.deliverAttachment(ctx, current) + } + default: + c.deliverAttachment(ctx, current) + } + return nil +} + +func (c *Client) registerInputWaiter(waiter chan error) error { + c.stateMu.Lock() + defer c.stateMu.Unlock() + select { + case <-c.readerDone: + return readerUnavailableError(c.readerErr) + default: + } + c.inputWaiter = waiter + return nil +} + +func (c *Client) clearInputWaiter(waiter chan error) { + c.stateMu.Lock() + defer c.stateMu.Unlock() + if c.inputWaiter == waiter { + c.inputWaiter = nil + } +} + +func (c *Client) completeInput(err error) { + c.stateMu.Lock() + waiter := c.inputWaiter + c.inputWaiter = nil + c.stateMu.Unlock() + if waiter != nil { + waiter <- err + } +} + +func (c *Client) registerAttachment() (*terminalAttachment, error) { + attachment := &terminalAttachment{ + frames: make(chan frame, 16), + done: make(chan struct{}), + } + c.stateMu.Lock() + defer c.stateMu.Unlock() + select { + case <-c.readerDone: + return nil, readerUnavailableError(c.readerErr) + default: + } + if c.attachment != nil { + return nil, errors.New("terminal client is already attached") + } + c.attachment = attachment + return attachment, nil +} + +func (c *Client) clearAttachment(attachment *terminalAttachment) { + c.stateMu.Lock() + if c.attachment == attachment { + c.attachment = nil + close(attachment.done) + } + c.stateMu.Unlock() +} + +func (c *Client) deliverAttachment(ctx context.Context, current frame) bool { + c.stateMu.Lock() + attachment := c.attachment + c.stateMu.Unlock() + if attachment == nil { + return false + } + select { + case attachment.frames <- current: + return true + case <-attachment.done: + return false + case <-ctx.Done(): + return false + } +} + +func (c *Client) finishReader(err error) { + c.stateMu.Lock() + c.readerErr = normalizeCloseError(err) + waiter := c.inputWaiter + c.inputWaiter = nil + close(c.readerDone) + c.stateMu.Unlock() + if waiter != nil { + waiter <- readerUnavailableError(c.readerErr) + } +} + +func (c *Client) readerError() error { + c.stateMu.Lock() + defer c.stateMu.Unlock() + return c.readerErr +} + +func readerUnavailableError(err error) error { + if err != nil { + return err + } + return errors.New("terminal connection closed") +} + func (c *Client) read(ctx context.Context) (frame, error) { messageType, payload, err := c.conn.Read(ctx) if err != nil { diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 7974b90b..5741f6ec 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -334,6 +334,221 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { } } +func TestSendInputConfirmedRejectionDoesNotRevokeControl(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-request-scoped", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + + for index := range 2 { + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + current, err := decodeFrame(payload) + if err != nil { + t.Error(err) + return + } + if current.messageType != messageInput { + t.Errorf("message type = %d", current.messageType) + return + } + event := eventPayload{Type: "input-accepted"} + if index == 0 { + event = eventPayload{ + Type: "input-rejected", + Error: "runner rejected terminal input", + } + } + encoded, _ := json.Marshal(event) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-request-scoped", + payload: encoded, + })); err != nil { + t.Error(err) + return + } + } + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-request-scoped", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + err = client.SendInputConfirmed(context.Background(), []byte("rejected\n")) + if err == nil || !strings.Contains(err.Error(), "runner rejected terminal input") { + t.Fatalf("error = %v", err) + } + if !client.canInput.Load() { + t.Fatal("request-scoped rejection revoked terminal control") + } + if err := client.SendInputConfirmed(context.Background(), []byte("accepted\n")); err != nil { + t.Fatal(err) + } +} + +func TestSendInputConfirmedSharesOneReaderWithAttach(t *testing.T) { + firstInput := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-concurrent", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + for index := range 2 { + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + current, err := decodeFrame(payload) + if err != nil { + t.Error(err) + return + } + if current.messageType != messageInput { + t.Errorf("message type = %d", current.messageType) + return + } + if index == 0 { + close(firstInput) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-concurrent", + payload: []byte("attached\n"), + })); err != nil { + t.Error(err) + return + } + _, acknowledgement, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + ack, err := decodeFrame(acknowledgement) + if err != nil || ack.messageType != messageAck { + t.Errorf("output acknowledgement = %#v, %v", ack, err) + return + } + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-concurrent", + payload: accepted, + })); err != nil { + t.Error(err) + return + } + } + closed, _ := json.Marshal(eventPayload{Type: "closed"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-concurrent", + payload: closed, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-concurrent", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + terminal := newBlockingTerminal() + attachDone := make(chan error, 1) + go func() { + attachDone <- client.Attach(context.Background(), terminal, nil) + }() + <-terminal.started + + sendDone := make(chan error, 2) + go func() { + sendDone <- client.SendInputConfirmed(context.Background(), []byte("first\n")) + }() + <-firstInput + go func() { + sendDone <- client.SendInputConfirmed(context.Background(), []byte("second\n")) + }() + for range 2 { + if err := <-sendDone; err != nil { + t.Fatal(err) + } + } + if err := <-attachDone; err != nil { + t.Fatal(err) + } + if terminal.String() != "attached\n" { + t.Fatalf("output = %q", terminal.String()) + } +} + func TestSendInputConfirmedReturnsImmediatelyForEmptyInput(t *testing.T) { client := &Client{supportsInputAcknowledgement: true} if err := client.SendInputConfirmed(context.Background(), nil); err != nil { @@ -826,16 +1041,24 @@ func (rw *readWriter) CancelRead() error { } type blockingTerminal struct { - closed chan struct{} - once sync.Once + closed chan struct{} + once sync.Once + started chan struct{} + startOnce sync.Once bytes.Buffer } func newBlockingTerminal() *blockingTerminal { - return &blockingTerminal{closed: make(chan struct{})} + return &blockingTerminal{ + closed: make(chan struct{}), + started: make(chan struct{}), + } } func (terminal *blockingTerminal) Read(_ []byte) (int, error) { + terminal.startOnce.Do(func() { + close(terminal.started) + }) <-terminal.closed return 0, io.ErrClosedPipe } diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 048ee982..6ddf902e 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -363,8 +363,12 @@ export class TerminalHub { }); return; } - if (subscriptions.has(id)) { - sendTerminalJson(client, TerminalMessageType.Event, id, { type: "subscribed" }); + const existingSubscription = subscriptions.get(id); + if (existingSubscription) { + sendTerminalJson(client, TerminalMessageType.Event, id, { + type: "subscribed", + canInput: existingSubscription.canInputGranted, + }); return; } @@ -689,7 +693,8 @@ function reportTerminalInputCompletion( if (socket.readyState !== WebSocket.OPEN) return; const rejection = results.find((result) => !result.accepted); if (rejection) { - sendTerminalJson(socket, TerminalMessageType.Error, sessionId, { + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-rejected", error: rejection.error ?? "terminal input was not accepted", }); return; diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 5965842a..b47f254f 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -389,6 +389,51 @@ test("terminal hub routes multiplex frames and explicit output acknowledgements" server.emit("close"); }); +test("duplicate subscriptions preserve the current input capability", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let upstreamOpens = 0; + const hub = new TerminalHub( + dependencies(client, server, upstream, { + inputGrant: () => async () => false, + async openUpstream() { + upstreamOpens += 1; + return { + socket: upstream, + outputAcknowledgements: true, + async markConnected() {}, + }; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + const subscribe = encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: session.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }); + + server.emit("message", { data: subscribe }); + await flushQueues(); + await flushQueues(); + server.emit("message", { data: subscribe }); + await flushQueues(); + await flushQueues(); + + assert.equal(upstreamOpens, 1); + assert.deepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "subscribed", + canInput: false, + }); + server.emit("close"); +}); + test("terminal hub publishes live controller downgrades and promotions", async () => { const client = socket(); const server = socket(); @@ -563,7 +608,7 @@ test("GitHub Actions input waits for the correlated runner acknowledgement", asy server.emit("close"); }); -test("GitHub Actions relay rejection becomes a terminal input error", async () => { +test("GitHub Actions relay rejection is request-scoped", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -613,10 +658,29 @@ test("GitHub Actions relay rejection becomes a terminal input error", async () = false, ); const rejected = messages.at(-1)!; - assert.equal(rejected.type, TerminalMessageType.Error); + assert.equal(rejected.type, TerminalMessageType.Event); assert.deepEqual(decodeJsonPayload(rejected.payload), { + type: "input-rejected", error: "GitHub Actions runner did not accept terminal input", }); + assert.equal(upstream.closed.length, 0); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("retry"), + }), + }); + await flushQueues(); + const retry = relayInput(upstream.sent.at(-1)!); + emitRelayAcknowledgement(upstream, retry.inputId, true); + await flushQueues(); + await flushQueues(); + + assert.deepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "input-accepted", + }); server.emit("close"); }); @@ -883,8 +947,9 @@ test("GitHub Actions send failure removes only its own acknowledgement waiter", await flushQueues(); const rejected = frame(server.sent.at(-1)!); - assert.equal(rejected.type, TerminalMessageType.Error); + assert.equal(rejected.type, TerminalMessageType.Event); assert.deepEqual(decodeJsonPayload(rejected.payload), { + type: "input-rejected", error: "terminal upstream send failed", }); server.emit("close"); @@ -943,7 +1008,9 @@ test("GitHub Actions close rejects every pending input acknowledgement", async ( assert.equal( messages.some( (message) => - message.type === TerminalMessageType.Error && + message.type === TerminalMessageType.Event && + (decodeJsonPayload(message.payload) as { type?: string; error?: string }).type === + "input-rejected" && (decodeJsonPayload(message.payload) as { error?: string }).error === "terminal upstream closed before accepting input", ), From 658efdf9a95c652801b867a33e8cd2cd25a38214 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:17:43 +0200 Subject: [PATCH 096/242] docs(changelog): record final concurrency fixes --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac85b54b..005e964b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,12 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, rollback-restored Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, and send attributed commands atomically to prevent interleaving. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers during deployment configuration, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, and custom-CA networking. -- Fence Share This Mac registry cleanup with per-registration ownership tokens so delayed shutdown from an older app process cannot remove a newer desktop host, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure and application-termination races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Fence Share This Mac registry cleanup with per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping publishers cannot displace cleanup authority, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, requiring an explicit architecture checksum when overriding the pinned default version. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. From 0f879b7e8084578bbf59d4a30675242573bc2cba Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:25:48 +0200 Subject: [PATCH 097/242] fix(image): keep default digest immutable --- Dockerfile | 13 +++++++------ tests/dockerfile.test.ts | 4 ++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23e24c85..78a4f335 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,22 +65,23 @@ RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "$arch" in \ amd64) \ - checksum="$CRABBOX_SHA256_AMD64"; \ + override_checksum="$CRABBOX_SHA256_AMD64"; \ checksum_arg="CRABBOX_SHA256_AMD64"; \ pinned_checksum="3c41839257e4622e28bcec8b0f0153f19d78d436fd548894a7c7d7726d922611" \ ;; \ arm64) \ - checksum="$CRABBOX_SHA256_ARM64"; \ + override_checksum="$CRABBOX_SHA256_ARM64"; \ checksum_arg="CRABBOX_SHA256_ARM64"; \ pinned_checksum="4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2" \ ;; \ *) echo "unsupported arch: $arch" >&2; exit 1 ;; \ esac; \ archive="crabbox_${CRABBOX_VERSION}_linux_${arch}.tar.gz"; \ - if [ -z "$checksum" ]; then \ - if [ "$CRABBOX_VERSION" = "0.17.1" ]; then \ - checksum="$pinned_checksum"; \ - else \ + if [ "$CRABBOX_VERSION" = "0.17.1" ]; then \ + checksum="$pinned_checksum"; \ + else \ + checksum="$override_checksum"; \ + if [ -z "$checksum" ]; then \ echo "an explicit $checksum_arg is required for non-default versions" >&2; \ exit 1; \ fi; \ diff --git a/tests/dockerfile.test.ts b/tests/dockerfile.test.ts index b458651f..6be638f8 100644 --- a/tests/dockerfile.test.ts +++ b/tests/dockerfile.test.ts @@ -14,6 +14,10 @@ test("Crabbox image pins the default release and requires pinned version overrid /pinned_checksum="4bf87a0d2365441ee2f8cb34183cfd9ebeb065111697eb2d8dc867b3a627fdd2"/, ); assert.doesNotMatch(dockerfile, /checksums\.txt/); + assert.match( + dockerfile, + /if \[ "\$CRABBOX_VERSION" = "0\.17\.1" \]; then \\\n\s+checksum="\$pinned_checksum"; \\\n\s+else \\\n\s+checksum="\$override_checksum";/, + ); assert.match(dockerfile, /an explicit \$checksum_arg is required/); assert.match(dockerfile, /grep -Eq '\^\[0-9a-f\]\{64\}\$'/); assert.match(dockerfile, /sha256sum -c -/); From 4a0adefd2dd243365747310dfb8d44fb5aa94afb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:25:48 +0200 Subject: [PATCH 098/242] docs(actions): reject split utf8 input frames --- README.md | 3 ++- docs/github-actions-sessions.md | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bc3f39b0..3ebc0ec2 100644 --- a/README.md +++ b/README.md @@ -101,13 +101,14 @@ const framedRunnerPtyUrl = new URL(runnerPtyUrl); framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); const terminal = new WebSocket(framedRunnerPtyUrl); terminal.binaryType = "arraybuffer"; +const inputDecoder = new TextDecoder("utf-8", { fatal: true }); pty.onData((output) => terminal.send(encodeCfr1Output(output))); terminal.onmessage = ({ data }) => { const input = decodeCfr1Input(data); if (!input) return; try { - pty.write(new TextDecoder().decode(input.payload)); + pty.write(inputDecoder.decode(input.payload)); terminal.send(encodeCfr1Ack(input.inputId, true)); } catch { terminal.send(encodeCfr1Ack(input.inputId, false)); diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 8514d4a8..31e64a8e 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -286,7 +286,8 @@ const runnerPtyUrl = process.env.CRABFLEET_RUNNER_PTY_URL; if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); const magic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); // CFR1 -const decoder = new TextDecoder(); +const inputIdDecoder = new TextDecoder(); +const inputDecoder = new TextDecoder("utf-8", { fatal: true }); const encoder = new TextEncoder(); const framedRunnerPtyUrl = new URL(runnerPtyUrl); framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); @@ -315,8 +316,8 @@ function acceptInput(data) { const input = decodeInput(data); if (!input) return; try { - // A successful node-pty write is this adapter's PTY acceptance point. - pty.write(decoder.decode(input.payload)); + // Reject frames that split or contain invalid UTF-8 instead of corrupting PTY input. + pty.write(inputDecoder.decode(input.payload)); terminal.send(encodeAck(input.inputId, true)); } catch { terminal.send(encodeAck(input.inputId, false)); @@ -334,7 +335,7 @@ function decodeInput(data) { if (!inputIdBytes || inputIdBytes > 80 || 6 + inputIdBytes > frame.byteLength) { return null; } - const inputId = decoder.decode(frame.subarray(6, 6 + inputIdBytes)); + const inputId = inputIdDecoder.decode(frame.subarray(6, 6 + inputIdBytes)); if (!/^[A-Za-z0-9_-]+$/.test(inputId)) return null; return { inputId, @@ -375,7 +376,9 @@ terminal.addEventListener("error", () => { Set `CRABFLEET_RUNNER_PTY_URL` to the `runnerPtyUrl` returned by registration. For a PTY API with an asynchronous write callback or promise, await that acceptance signal before sending `encodeAck(..., true)`. Do not acknowledge when -the WebSocket merely queues the input frame. +the WebSocket merely queues the input frame. This Node adapter also requires each +input frame to contain complete, valid UTF-8; invalid or split sequences receive +a negative acknowledgement and must be resent on valid boundaries. The protocol query is consumed during connection setup and is not forwarded as terminal data. There is no capability message or mode transition after the From 437a4f460caac27a63bcf7ad85a50ab51563648c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:26:29 +0200 Subject: [PATCH 099/242] test(actions): enforce byte-safe runner docs --- docs/github-actions-sessions.md | 5 ++++- tests/github-actions-docs.test.ts | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/github-actions-docs.test.ts diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 31e64a8e..805c9fc6 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -382,7 +382,10 @@ a negative acknowledgement and must be resent on valid boundaries. The protocol query is consumed during connection setup and is not forwarded as terminal data. There is no capability message or mode transition after the -socket opens. Each `CFR1` frame occupies one binary WebSocket message: +socket opens. Each `CFR1` frame occupies one binary WebSocket message. Payloads +are opaque bytes; adapters targeting string-only PTY APIs must either preserve +decoder state across acknowledgements or reject frames that end inside a text +encoding sequence, as the Node example does: | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------ | diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts new file mode 100644 index 00000000..119aa72b --- /dev/null +++ b/tests/github-actions-docs.test.ts @@ -0,0 +1,19 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { test } from "node:test"; + +test("the documented Node runner rejects split UTF-8 input frames", async () => { + const [readme, guide] = await Promise.all([ + readFile(new URL("../README.md", import.meta.url), "utf8"), + readFile(new URL("../docs/github-actions-sessions.md", import.meta.url), "utf8"), + ]); + + for (const documentation of [readme, guide]) { + assert.match(documentation, /new TextDecoder\("utf-8", \{ fatal: true \}\)/); + assert.match(documentation, /inputDecoder\.decode\(input\.payload\)/); + } + + const decoder = new TextDecoder("utf-8", { fatal: true }); + assert.throws(() => decoder.decode(Uint8Array.from([0xf0, 0x9f])), TypeError); + assert.throws(() => decoder.decode(Uint8Array.from([0xa6, 0x80])), TypeError); +}); From 461b200a7857faeac22bc7b443f239c03ce9db7d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:26:36 +0200 Subject: [PATCH 100/242] fix(macos): retry pending input releases --- .../Sources/CrabfleetMac/MacRemoteInput.swift | 117 +++++++++++++----- .../PrivateMacShareTests.swift | 97 +++++++++++++++ 2 files changed, 180 insertions(+), 34 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index fddc85f7..aeea22f1 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -15,20 +15,41 @@ extension RemoteInputForwarding { } final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable { + private static let releaseRetryDelay: DispatchTimeInterval = .milliseconds(250) + private let descriptor: CapturedDisplayDescriptor private let eventQueue = DispatchQueue( label: "org.openclaw.crabfleet.remote-input", qos: .userInteractive ) + private let accessibilityGranted: @Sendable () -> Bool + private let pendingReleaseRetryDelay: DispatchTimeInterval + private let keyEventPoster: (@Sendable (Bool, UInt32) -> Void)? + private let mouseEventPoster: + (@Sendable (CGEventType, CGPoint, CGMouseButton) -> Void)? private let frameSizeLock = NSLock() private var frameWidth: Int private var frameHeight: Int private var previousButtonMask: UInt8 = 0 private var previousPointerLocation: CGPoint private var pressedKeysyms: Set = [] + private var hasPendingRelease = false + private var pendingReleaseRetryScheduled = false - init(descriptor: CapturedDisplayDescriptor) { + init( + descriptor: CapturedDisplayDescriptor, + accessibilityGranted: @escaping @Sendable () -> Bool = { + MacRemoteInputController.isAccessibilityGranted + }, + pendingReleaseRetryDelay: DispatchTimeInterval = MacRemoteInputController.releaseRetryDelay, + keyEventPoster: (@Sendable (Bool, UInt32) -> Void)? = nil, + mouseEventPoster: (@Sendable (CGEventType, CGPoint, CGMouseButton) -> Void)? = nil + ) { self.descriptor = descriptor + self.accessibilityGranted = accessibilityGranted + self.pendingReleaseRetryDelay = pendingReleaseRetryDelay + self.keyEventPoster = keyEventPoster + self.mouseEventPoster = mouseEventPoster frameWidth = descriptor.frameWidth frameHeight = descriptor.frameHeight previousPointerLocation = descriptor.displayBounds.origin @@ -58,7 +79,8 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable func keyEvent(down: Bool, keysym: UInt32) { eventQueue.async { [self] in - guard Self.isAccessibilityGranted else { return } + guard accessibilityGranted() else { return } + flushPendingRelease() postKeyEvent(down: down, keysym: keysym) if down { pressedKeysyms.insert(keysym) @@ -70,7 +92,8 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable func pointerEvent(buttonMask: UInt8, x: UInt16, y: UInt16) { eventQueue.async { [self] in - guard Self.isAccessibilityGranted else { return } + guard accessibilityGranted() else { return } + flushPendingRelease() let location = mappedLocation(x: x, y: y) previousPointerLocation = location let changedButtons = previousButtonMask ^ buttonMask @@ -79,12 +102,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable for button in Self.mouseButtons where changedButtons & button.mask != 0 { let isDown = buttonMask & button.mask != 0 let type = isDown ? button.downType : button.upType - CGEvent( - mouseEventSource: eventSource(), - mouseType: type, - mouseCursorPosition: location, - mouseButton: button.button - )?.post(tap: .cghidEventTap) + postMouseEvent(type: type, location: location, button: button.button) postedButtonChange = true } @@ -99,12 +117,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable } else { moveType = .mouseMoved } - CGEvent( - mouseEventSource: eventSource(), - mouseType: moveType, - mouseCursorPosition: location, - mouseButton: .left - )?.post(tap: .cghidEventTap) + postMouseEvent(type: moveType, location: location, button: .left) } let newWheelBits = buttonMask & ~previousButtonMask @@ -118,29 +131,16 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable func releaseAllInput() { eventQueue.async { [self] in - let canPostEvents = Self.isAccessibilityGranted - if canPostEvents { - for keysym in pressedKeysyms { - postKeyEvent(down: false, keysym: keysym) - } - } - pressedKeysyms.removeAll() - - if canPostEvents { - for button in Self.mouseButtons where previousButtonMask & button.mask != 0 { - CGEvent( - mouseEventSource: eventSource(), - mouseType: button.upType, - mouseCursorPosition: previousPointerLocation, - mouseButton: button.button - )?.post(tap: .cghidEventTap) - } - } - previousButtonMask = 0 + hasPendingRelease = true + flushPendingRelease() } } private func postKeyEvent(down: Bool, keysym: UInt32) { + if let keyEventPoster { + keyEventPoster(down, keysym) + return + } let event: CGEvent? if let keyCode = Self.keyCode(for: keysym) { event = CGEvent(keyboardEventSource: eventSource(), virtualKey: keyCode, keyDown: down) @@ -158,6 +158,55 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable event?.post(tap: .cghidEventTap) } + private func postMouseEvent( + type: CGEventType, + location: CGPoint, + button: CGMouseButton + ) { + if let mouseEventPoster { + mouseEventPoster(type, location, button) + return + } + CGEvent( + mouseEventSource: eventSource(), + mouseType: type, + mouseCursorPosition: location, + mouseButton: button + )?.post(tap: .cghidEventTap) + } + + private func flushPendingRelease() { + guard hasPendingRelease else { return } + guard accessibilityGranted() else { + schedulePendingReleaseRetry() + return + } + + for keysym in pressedKeysyms { + postKeyEvent(down: false, keysym: keysym) + } + for button in Self.mouseButtons where previousButtonMask & button.mask != 0 { + postMouseEvent( + type: button.upType, + location: previousPointerLocation, + button: button.button + ) + } + pressedKeysyms.removeAll() + previousButtonMask = 0 + hasPendingRelease = false + } + + private func schedulePendingReleaseRetry() { + guard !pendingReleaseRetryScheduled else { return } + pendingReleaseRetryScheduled = true + eventQueue.asyncAfter(deadline: .now() + pendingReleaseRetryDelay) { [weak self] in + guard let self else { return } + self.pendingReleaseRetryScheduled = false + self.flushPendingRelease() + } + } + private func eventSource() -> CGEventSource? { CGEventSource(stateID: .hidSystemState) } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 8bd301d4..c9757e27 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -823,6 +823,52 @@ struct PrivateMacShareTests { #expect(MacRemoteInputController.keyCode(for: 0x1F980) == nil) } + @Test + func retriesHeldInputReleaseAfterAccessibilityReturns() async { + let trust = AccessibilityTrust(granted: true) + let events = RemoteInputEventRecorder() + let controller = MacRemoteInputController( + descriptor: CapturedDisplayDescriptor( + displayID: 1, + displayBounds: CGRect(x: 0, y: 0, width: 100, height: 100), + frameWidth: 100, + frameHeight: 100, + sourcePixelWidth: 100, + sourcePixelHeight: 100 + ), + accessibilityGranted: { trust.isGranted() }, + pendingReleaseRetryDelay: .milliseconds(10), + keyEventPoster: { down, keysym in + events.append(.key(down: down, keysym: keysym)) + }, + mouseEventPoster: { type, _, button in + events.append(.mouse(type: type, button: button)) + } + ) + + controller.keyEvent(down: true, keysym: 0x61) + controller.pointerEvent(buttonMask: 0x01, x: 50, y: 50) + #expect(await waitUntilAsync { + events.contains(.key(down: true, keysym: 0x61)) + && events.contains(.mouse(type: .leftMouseDown, button: .left)) + }) + + let checksBeforeRevocation = trust.checkCount + trust.setGranted(false) + controller.releaseAllInput() + #expect(await waitUntilAsync { + trust.checkCount > checksBeforeRevocation + }) + #expect(!events.contains(.key(down: false, keysym: 0x61))) + #expect(!events.contains(.mouse(type: .leftMouseUp, button: .left))) + + trust.setGranted(true) + #expect(await waitUntilAsync { + events.contains(.key(down: false, keysym: 0x61)) + && events.contains(.mouse(type: .leftMouseUp, button: .left)) + }) + } + @Test func decodesX11UnicodeKeysymsForMacInput() { #expect(MacRemoteInputController.unicodeScalar(for: 0x0100_03BB) == "λ") @@ -1302,6 +1348,57 @@ private final class RFBEventRecorder: @unchecked Sendable { } } +private enum RecordedRemoteInputEvent: Equatable { + case key(down: Bool, keysym: UInt32) + case mouse(type: CGEventType, button: CGMouseButton) +} + +private final class RemoteInputEventRecorder: @unchecked Sendable { + private let lock = NSLock() + private var events: [RecordedRemoteInputEvent] = [] + + func append(_ event: RecordedRemoteInputEvent) { + lock.lock() + events.append(event) + lock.unlock() + } + + func contains(_ event: RecordedRemoteInputEvent) -> Bool { + lock.lock() + defer { lock.unlock() } + return events.contains(event) + } +} + +private final class AccessibilityTrust: @unchecked Sendable { + private let lock = NSLock() + private var granted: Bool + private var checks = 0 + + init(granted: Bool) { + self.granted = granted + } + + var checkCount: Int { + lock.lock() + defer { lock.unlock() } + return checks + } + + func isGranted() -> Bool { + lock.lock() + defer { lock.unlock() } + checks += 1 + return granted + } + + func setGranted(_ granted: Bool) { + lock.lock() + self.granted = granted + lock.unlock() + } +} + private final class DesktopRegistrationTransport: HTTPDataTransport { private let handler: (URLRequest) throws -> (Data, HTTPURLResponse) From 2c98e22b0864de24a585721d3e637ecfc26d17e4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:26:56 +0200 Subject: [PATCH 101/242] fix(terminal): defer output ack until attach --- internal/terminalws/client.go | 51 ++++++++++----- internal/terminalws/client_test.go | 100 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 17 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 4bb06b45..af7174b0 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -87,6 +87,7 @@ type Client struct { stateMu sync.Mutex inputWaiter chan error attachment *terminalAttachment + pendingAttachmentFrames []frame readerDone chan struct{} readerErr error } @@ -475,14 +476,7 @@ func (c *Client) handleFrame(ctx context.Context, current frame) error { } switch current.messageType { case messageOutput: - if c.deliverAttachment(ctx, current) { - return nil - } - return c.write(ctx, frame{ - messageType: messageAck, - sessionID: c.sessionID, - payload: ackPayload(uint32(len(current.payload))), - }) + c.deliverOrQueueOutput(ctx, current) case messageError: err := frameError(current, "terminal connection failed") c.canInput.Store(false) @@ -550,20 +544,26 @@ func (c *Client) completeInput(err error) { } func (c *Client) registerAttachment() (*terminalAttachment, error) { - attachment := &terminalAttachment{ - frames: make(chan frame, 16), - done: make(chan struct{}), - } c.stateMu.Lock() defer c.stateMu.Unlock() - select { - case <-c.readerDone: - return nil, readerUnavailableError(c.readerErr) - default: - } if c.attachment != nil { return nil, errors.New("terminal client is already attached") } + if len(c.pendingAttachmentFrames) == 0 { + select { + case <-c.readerDone: + return nil, readerUnavailableError(c.readerErr) + default: + } + } + attachment := &terminalAttachment{ + frames: make(chan frame, len(c.pendingAttachmentFrames)+16), + done: make(chan struct{}), + } + for _, current := range c.pendingAttachmentFrames { + attachment.frames <- current + } + c.pendingAttachmentFrames = nil c.attachment = attachment return attachment, nil } @@ -577,6 +577,23 @@ func (c *Client) clearAttachment(attachment *terminalAttachment) { c.stateMu.Unlock() } +func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) { + c.stateMu.Lock() + attachment := c.attachment + if attachment == nil { + c.pendingAttachmentFrames = append(c.pendingAttachmentFrames, current) + c.stateMu.Unlock() + return + } + c.stateMu.Unlock() + select { + case attachment.frames <- current: + case <-attachment.done: + c.deliverOrQueueOutput(ctx, current) + case <-ctx.Done(): + } +} + func (c *Client) deliverAttachment(ctx context.Context, current frame) bool { c.stateMu.Lock() attachment := c.attachment diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 5741f6ec..2e55a1f9 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -275,6 +275,89 @@ func TestClientSubscribesSendsInputAndAcknowledgesOutput(t *testing.T) { } } +func TestClientDefersOutputAcknowledgementUntilAttach(t *testing.T) { + acknowledged := make(chan uint32, 1) + outputSent := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: false}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-before-attach", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-before-attach", + payload: []byte("early output\n"), + })); err != nil { + t.Error(err) + return + } + close(outputSent) + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + ack, err := decodeFrame(payload) + if err != nil || ack.messageType != messageAck { + t.Errorf("output acknowledgement = %#v, %v", ack, err) + return + } + acknowledged <- binary.LittleEndian.Uint32(ack.payload) + closed, _ := json.Marshal(eventPayload{Type: "closed"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-before-attach", + payload: closed, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-before-attach", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + <-outputSent + waitForPendingAttachmentFrames(t, client, 1) + select { + case bytes := <-acknowledged: + t.Fatalf("acknowledged %d bytes before attach", bytes) + default: + } + + terminal := newBlockingTerminal() + if err := client.Attach(context.Background(), terminal, nil); err != nil { + t.Fatal(err) + } + if terminal.String() != "early output\n" { + t.Fatalf("output = %q", terminal.String()) + } + if bytes := <-acknowledged; bytes != uint32(len("early output\n")) { + t.Fatalf("acknowledged = %d", bytes) + } +} + func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) @@ -1023,6 +1106,23 @@ func TestClientContinuesReadOnlyAndResumesControl(t *testing.T) { } } +func waitForPendingAttachmentFrames(t *testing.T, client *Client, count int) { + t.Helper() + deadline := time.Now().Add(time.Second) + for { + client.stateMu.Lock() + pending := len(client.pendingAttachmentFrames) + client.stateMu.Unlock() + if pending == count { + return + } + if time.Now().After(deadline) { + t.Fatalf("pending attachment frames = %d", pending) + } + time.Sleep(time.Millisecond) + } +} + type readWriter struct { reader io.Reader closer io.Closer From 515af603dab18199680300dd35f765602e46ee29 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:28:03 +0200 Subject: [PATCH 102/242] fix(macos): bound successful command drain --- .../CrabfleetMac/TailnetIdentity.swift | 52 ++++++++++++++----- .../PrivateMacShareTests.swift | 39 ++++++++++++++ 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift index def7f07a..9097e791 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift @@ -88,7 +88,7 @@ struct SystemTailscaleCommandRunner: TailscaleCommandRunning { } private final class TailscaleCommandExecution: @unchecked Sendable { - private static let stoppedProcessDrainTimeout: DispatchTimeInterval = .milliseconds(250) + private static let processDrainTimeout: DispatchTimeInterval = .milliseconds(250) private enum StopReason { case cancelled @@ -104,6 +104,7 @@ private final class TailscaleCommandExecution: @unchecked Sendable { private let timeout: TimeInterval private let maximumOutputBytes: Int private var stopReason: StopReason? + private var captureShouldStop = false private var standardOutput = Data() private var standardError = Data() @@ -182,15 +183,31 @@ private final class TailscaleCommandExecution: @unchecked Sendable { readGroup.enter() DispatchQueue.global(qos: .userInitiated).async { [self] in defer { readGroup.leave() } + let fileDescriptor = pipe.fileHandleForReading.fileDescriptor var data = Data() - while true { - let chunk = pipe.fileHandleForReading.readData(ofLength: 64 * 1_024) - if chunk.isEmpty { break } - guard chunk.count <= maximumOutputBytes - data.count else { + var buffer = [UInt8](repeating: 0, count: 64 * 1_024) + while !shouldStopCapture() { + var descriptor = pollfd(fd: fileDescriptor, events: Int16(POLLIN | POLLHUP), revents: 0) + let pollResult = Darwin.poll(&descriptor, 1, 50) + if pollResult == 0 { continue } + if pollResult < 0 { + if errno == EINTR { continue } + break + } + + let bytesRead = buffer.withUnsafeMutableBytes { + Darwin.read(fileDescriptor, $0.baseAddress, $0.count) + } + if bytesRead == 0 { break } + if bytesRead < 0 { + if errno == EINTR || errno == EAGAIN { continue } + break + } + guard bytesRead <= maximumOutputBytes - data.count else { stop(.outputTooLarge) break } - data.append(chunk) + data.append(buffer, count: bytesRead) } setCaptured(data, isStandardOutput: isStandardOutput) } @@ -221,6 +238,18 @@ private final class TailscaleCommandExecution: @unchecked Sendable { return stopReason } + private func shouldStopCapture() -> Bool { + lock.lock() + defer { lock.unlock() } + return captureShouldStop + } + + private func stopCapture() { + lock.lock() + captureShouldStop = true + lock.unlock() + } + private func stop(_ reason: StopReason) { lock.lock() if stopReason == nil { stopReason = reason } @@ -239,16 +268,11 @@ private final class TailscaleCommandExecution: @unchecked Sendable { } private func finishCapture() { - guard currentStopReason() != nil else { - readGroup.wait() - return - } - guard readGroup.wait(timeout: .now() + Self.stoppedProcessDrainTimeout) == .timedOut else { + guard readGroup.wait(timeout: .now() + Self.processDrainTimeout) == .timedOut else { return } - try? outputPipe.fileHandleForReading.close() - try? errorPipe.fileHandleForReading.close() - _ = readGroup.wait(timeout: .now() + Self.stoppedProcessDrainTimeout) + stopCapture() + _ = readGroup.wait(timeout: .now() + Self.processDrainTimeout) } } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index c9757e27..871d3bde 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -138,6 +138,45 @@ struct PrivateMacShareTests { #expect(elapsed < .seconds(2)) } + @Test + func successfulTailscaleCommandDoesNotWaitForDescendantPipeEOF() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("CrabfleetMacTests.\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let executable = directory.appendingPathComponent("tailscale") + let descendantPIDFile = directory.appendingPathComponent("descendant-pid") + try Data( + """ + #!/bin/sh + ( + trap '' HUP TERM + exec sleep 30 + ) & + printf '%s' "$!" > '\(descendantPIDFile.path)' + printf 'status complete' + exit 0 + """.utf8 + ).write(to: executable) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: executable.path) + + let runner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 5) + let clock = ContinuousClock() + let startedAt = clock.now + let result = try await runner.run(arguments: ["status"]) + let elapsed = startedAt.duration(to: clock.now) + + let descendantPID = try #require( + Int32(String(contentsOf: descendantPIDFile, encoding: .utf8)) + ) + defer { + _ = Darwin.kill(descendantPID, SIGKILL) + } + #expect(result.standardOutput == "status complete") + #expect(Darwin.kill(descendantPID, 0) == 0) + #expect(elapsed < .seconds(2)) + } + @Test @MainActor func stopInvalidatesAnInFlightPrivateShareStart() async throws { let runner = SuspendedTailscaleRunner() From d49d0e3492ea9e32af383fbf566496291cd9be17 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:28:47 +0200 Subject: [PATCH 103/242] fix(actions): negotiate viewer relay protocol --- CHANGELOG.md | 2 +- src/github-actions-runtime.ts | 144 ++++++++++++++++----- src/worker/interactive-terminal-service.ts | 12 +- src/worker/session-control-do.ts | 15 ++- tests/application-architecture.test.ts | 18 ++- tests/github-actions-runtime.test.ts | 98 ++++++++++++-- 6 files changed, 231 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 005e964b..02548933 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. - Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. -- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames for GitHub Actions runners, retain legacy raw runner compatibility, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, retain legacy raw peer compatibility during mixed deployments, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers during deployment configuration, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure and application-termination races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping publishers cannot displace cleanup authority, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index b8597ec9..7a8151fb 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -31,7 +31,10 @@ export type GitHubActionsRelayInput = { export const githubActionsFramedRunnerCapability = "cfr1-framed-io-v1"; export const githubActionsRunnerProtocolQuery = "runnerProtocol"; -export type GitHubActionsRunnerProtocol = typeof githubActionsFramedRunnerCapability; +export const githubActionsViewerProtocolQuery = "viewerProtocol"; +export type GitHubActionsRelayProtocol = typeof githubActionsFramedRunnerCapability; +export type GitHubActionsRunnerProtocol = GitHubActionsRelayProtocol; +export type GitHubActionsViewerProtocol = GitHubActionsRelayProtocol; export const githubActionsCapabilities = { terminal: true, @@ -82,8 +85,8 @@ const relayEvents = new Map( const encoder = new TextEncoder(); const decoder = new TextDecoder(); -type GitHubActionsRunnerAttachment = { - protocol?: GitHubActionsRunnerProtocol; +type GitHubActionsRelayAttachment = { + protocol?: GitHubActionsRelayProtocol; }; export function githubActionsRuntimeLabel(runtime: unknown): string { @@ -104,6 +107,12 @@ export function buildGitHubActionsRunnerPtyUrl( return url.toString(); } +export function buildGitHubActionsViewerRelayUrl(): string { + const url = new URL("https://crabfleet.internal/api/session-control/github-actions/viewer"); + url.searchParams.set(githubActionsViewerProtocolQuery, githubActionsFramedRunnerCapability); + return url.toString(); +} + export function parseGitHubActionsWorkState(value: unknown): GitHubActionsWorkState | null { const state = String(value ?? "").trim() as GitHubActionsWorkState; return workStates.has(state) ? state : null; @@ -179,51 +188,68 @@ export function relayGitHubActionsWebSocketMessage( ): number { if (sender === "viewer") { if (isGitHubActionsViewerControlMessage(message)) return 0; - const input = parseGitHubActionsRelayInput(message); - if (!input) return 0; + const framedViewer = gitHubActionsViewerUsesFramedProtocol(senderSocket); + const input = framedViewer ? parseGitHubActionsRelayInput(message) : null; + if (framedViewer && !input) return 0; const runner = runners.find((socket) => socket.readyState === webSocketOpen); if (!runner) { - sendGitHubActionsRelayInputAcknowledgement(senderSocket, { - inputId: input.inputId, - accepted: false, - }); + sendGitHubActionsViewerInputAcknowledgement(senderSocket, input?.inputId ?? null, false); return 0; } const framed = gitHubActionsRunnerUsesFramedProtocol(runner); try { - runner.send(framed ? message : input.payload); - if (!framed) { - sendGitHubActionsRelayInputAcknowledgement(senderSocket, { - inputId: input.inputId, - accepted: true, - }); + if (framed) { + runner.send( + framedViewer + ? message + : encodeGitHubActionsRelayInput(createGitHubActionsRelayInputId(), message), + ); + } else { + runner.send(framedViewer ? input!.payload : message); + } + if (!framedViewer || !framed) { + sendGitHubActionsViewerInputAcknowledgement(senderSocket, input?.inputId ?? null, true); } return 1; } catch { - sendGitHubActionsRelayInputAcknowledgement(senderSocket, { - inputId: input.inputId, - accepted: false, - }); + sendGitHubActionsViewerInputAcknowledgement(senderSocket, input?.inputId ?? null, false); return 0; } } if (gitHubActionsRunnerUsesFramedProtocol(senderSocket)) { - if ( - !parseGitHubActionsRelayInputAcknowledgement(message) && - !parseGitHubActionsRelayOutput(message) - ) { - return 0; + const acknowledgement = parseGitHubActionsRelayInputAcknowledgement(message); + const output = parseGitHubActionsRelayOutput(message); + if (!acknowledgement && !output) return 0; + let forwarded = 0; + for (const viewer of viewers) { + if (viewer.readyState !== webSocketOpen) continue; + if (acknowledgement && !gitHubActionsViewerUsesFramedProtocol(viewer)) continue; + try { + viewer.send(gitHubActionsViewerUsesFramedProtocol(viewer) ? message : output!); + forwarded += 1; + } catch { + // A failed viewer does not prevent delivery to the remaining viewers. + } } - return forwardGitHubActionsRelayMessage(sender, message, runners, viewers); + return forwarded; } - return forwardGitHubActionsRelayMessage( - sender, - encodeGitHubActionsRelayOutput(message), - runners, - viewers, - ); + let forwarded = 0; + for (const viewer of viewers) { + if (viewer.readyState !== webSocketOpen) continue; + try { + viewer.send( + gitHubActionsViewerUsesFramedProtocol(viewer) + ? encodeGitHubActionsRelayOutput(message) + : message, + ); + forwarded += 1; + } catch { + // A failed viewer does not prevent delivery to the remaining viewers. + } + } + return forwarded; } export function sendGitHubActionsRelayInputAcknowledgement( @@ -302,12 +328,27 @@ export function parseGitHubActionsRunnerProtocol( return value === githubActionsFramedRunnerCapability ? githubActionsFramedRunnerCapability : null; } +export function parseGitHubActionsViewerProtocol( + value: string | null, +): GitHubActionsViewerProtocol | null { + return value === githubActionsFramedRunnerCapability ? githubActionsFramedRunnerCapability : null; +} + export function attachGitHubActionsRunnerProtocol( socket: GitHubActionsRelaySocket, protocol: GitHubActionsRunnerProtocol | null, ): void { socket.serializeAttachment?.( - protocol ? ({ protocol } satisfies GitHubActionsRunnerAttachment) : {}, + protocol ? ({ protocol } satisfies GitHubActionsRelayAttachment) : {}, + ); +} + +export function attachGitHubActionsViewerProtocol( + socket: GitHubActionsRelaySocket, + protocol: GitHubActionsViewerProtocol | null, +): void { + socket.serializeAttachment?.( + protocol ? ({ protocol } satisfies GitHubActionsRelayAttachment) : {}, ); } @@ -354,7 +395,7 @@ export function notifyGitHubActionsViewers( viewers: readonly GitHubActionsRelaySocket[], type: "runner_connected" | "runner_disconnected" | "runner_waiting", ): number { - const payload = encodeGitHubActionsRelayFrame( + const framedPayload = encodeGitHubActionsRelayFrame( relayEventFrameType, "", new Uint8Array([relayEventCodes[type]]), @@ -362,7 +403,9 @@ export function notifyGitHubActionsViewers( let notified = 0; for (const socket of viewers) { if (socket.readyState !== webSocketOpen) continue; - socket.send(payload); + socket.send( + gitHubActionsViewerUsesFramedProtocol(socket) ? framedPayload : JSON.stringify({ type }), + ); notified += 1; } return notified; @@ -432,9 +475,40 @@ function requireGitHubActionsRelayInputId(inputId: string): void { } export function gitHubActionsRunnerUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { + return gitHubActionsRelayUsesFramedProtocol(socket); +} + +export function gitHubActionsViewerUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { + return gitHubActionsRelayUsesFramedProtocol(socket); +} + +function gitHubActionsRelayUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { const attachment = socket.deserializeAttachment?.(); if (!attachment || typeof attachment !== "object") return false; return ( - (attachment as GitHubActionsRunnerAttachment).protocol === githubActionsFramedRunnerCapability + (attachment as GitHubActionsRelayAttachment).protocol === githubActionsFramedRunnerCapability ); } + +function sendGitHubActionsViewerInputAcknowledgement( + viewer: GitHubActionsRelaySocket, + inputId: string | null, + accepted: boolean, +): boolean { + if (inputId) { + return sendGitHubActionsRelayInputAcknowledgement(viewer, { inputId, accepted }); + } + if (viewer.readyState !== webSocketOpen) return false; + try { + viewer.send( + JSON.stringify({ + type: "github_actions_input_ack", + accepted, + ...(accepted ? {} : { error: relayInputRejectedError }), + }), + ); + return true; + } catch { + return false; + } +} diff --git a/src/worker/interactive-terminal-service.ts b/src/worker/interactive-terminal-service.ts index 57b231a2..dc731ca0 100644 --- a/src/worker/interactive-terminal-service.ts +++ b/src/worker/interactive-terminal-service.ts @@ -6,7 +6,10 @@ import { terminalSubmittedLine, type TerminalInputState, } from "../terminal-multiplayer.ts"; -import { githubActionsRuntime } from "../github-actions-runtime.ts"; +import { + buildGitHubActionsViewerRelayUrl, + githubActionsRuntime, +} from "../github-actions-runtime.ts"; import { terminalFailureStatusForAdapter } from "../runtime-adapter.ts"; import { cachedBooleanGrant } from "../terminal-authorization.ts"; import { actor, requireRole } from "./auth.ts"; @@ -126,10 +129,9 @@ export class InteractiveTerminalService { if (session.runtime === githubActionsRuntime) { const stub = githubActionsRelayStub(this.env, session.id); if (!stub) throw serviceUnavailable("SESSION_CONTROL Durable Object is not configured"); - const upstreamResponse = await stub.fetch( - "https://crabfleet.internal/api/session-control/github-actions/viewer", - { headers: { upgrade: "websocket" } }, - ); + const upstreamResponse = await stub.fetch(buildGitHubActionsViewerRelayUrl(), { + headers: { upgrade: "websocket" }, + }); const upstream = upstreamResponse.webSocket; if (!upstream || upstreamResponse.status !== 101) { throw serviceUnavailable(`GitHub Actions relay HTTP ${upstreamResponse.status}`); diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index e7c004b3..4993b603 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -8,13 +8,16 @@ import { import type { FleetSandboxPolicySummary } from "../fleet-state.ts"; import { attachGitHubActionsRunnerProtocol, + attachGitHubActionsViewerProtocol, githubActionsRelayRole, githubActionsRunnerProtocolQuery, + githubActionsViewerProtocolQuery, notifyGitHubActionsViewers, parseGitHubActionsRunnerProtocol, + parseGitHubActionsViewerProtocol, relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, - type GitHubActionsRunnerProtocol, + type GitHubActionsRelayProtocol, } from "../github-actions-runtime.ts"; import type { RuntimeEnv } from "./env.ts"; import { json } from "./http.ts"; @@ -65,7 +68,10 @@ export class SessionControlDO extends DurableObject { request.method === "GET" && url.pathname === "/api/session-control/github-actions/viewer" ) { - return this.openGitHubActionsRelay("viewer"); + return this.openGitHubActionsRelay( + "viewer", + parseGitHubActionsViewerProtocol(url.searchParams.get(githubActionsViewerProtocolQuery)), + ); } if ( @@ -232,20 +238,21 @@ export class SessionControlDO extends DurableObject { private openGitHubActionsRelay( role: "runner" | "viewer", - runnerProtocol: GitHubActionsRunnerProtocol | null = null, + protocol: GitHubActionsRelayProtocol | null = null, ): Response { const pair = new WebSocketPair(); const client = pair[0]; const server = pair[1]; if (role === "runner") { replaceGitHubActionsRunner(this.ctx.getWebSockets("github-actions-runner")); - attachGitHubActionsRunnerProtocol(server, runnerProtocol); + attachGitHubActionsRunnerProtocol(server, protocol); this.ctx.acceptWebSocket(server, ["github-actions-runner"]); notifyGitHubActionsViewers( this.ctx.getWebSockets("github-actions-viewer"), "runner_connected", ); } else { + attachGitHubActionsViewerProtocol(server, protocol); this.ctx.acceptWebSocket(server, ["github-actions-viewer"]); if (this.ctx.getWebSockets("github-actions-runner").length === 0) { notifyGitHubActionsViewers([server], "runner_waiting"); diff --git a/tests/application-architecture.test.ts b/tests/application-architecture.test.ts index 4b546b29..734f92ca 100644 --- a/tests/application-architecture.test.ts +++ b/tests/application-architecture.test.ts @@ -66,7 +66,7 @@ test("GitHub Actions runner protocol is attached before the relay socket is acce ]); assert.match(application, /stub\.fetch\(gitHubActionsRelayRunnerUrl\(request\)/); - const attach = relay.indexOf("attachGitHubActionsRunnerProtocol(server, runnerProtocol)"); + const attach = relay.indexOf("attachGitHubActionsRunnerProtocol(server, protocol)"); const accept = relay.indexOf( 'this.ctx.acceptWebSocket(server, ["github-actions-runner"])', attach, @@ -75,6 +75,22 @@ test("GitHub Actions runner protocol is attached before the relay socket is acce assert.ok(accept > attach); }); +test("GitHub Actions viewer protocol is requested and attached before relay acceptance", async () => { + const [terminal, relay] = await Promise.all([ + readFile(new URL("../src/worker/interactive-terminal-service.ts", import.meta.url), "utf8"), + readFile(new URL("../src/worker/session-control-do.ts", import.meta.url), "utf8"), + ]); + + assert.match(terminal, /stub\.fetch\(\s*buildGitHubActionsViewerRelayUrl\(\)/); + const attach = relay.indexOf("attachGitHubActionsViewerProtocol(server, protocol)"); + const accept = relay.indexOf( + 'this.ctx.acceptWebSocket(server, ["github-actions-viewer"])', + attach, + ); + assert.notEqual(attach, -1); + assert.ok(accept > attach); +}); + test("worker entrypoint retains only routing and platform composition", async () => { const entrypoint = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 19e32164..ae003d31 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -2,7 +2,9 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { attachGitHubActionsRunnerProtocol, + attachGitHubActionsViewerProtocol, buildGitHubActionsRunnerPtyUrl, + buildGitHubActionsViewerRelayUrl, encodeGitHubActionsRelayInput, encodeGitHubActionsRelayInputAcknowledgement, encodeGitHubActionsRelayOutput, @@ -12,7 +14,9 @@ import { githubActionsRelayRole, githubActionsRunnerProtocolQuery, githubActionsRuntimeLabel, + githubActionsViewerProtocolQuery, gitHubActionsRunnerUsesFramedProtocol, + gitHubActionsViewerUsesFramedProtocol, isGitHubActionsViewerControlMessage, isTerminalGitHubActionsWorkState, notifyGitHubActionsViewers, @@ -21,6 +25,7 @@ import { parseGitHubActionsRelayInputAcknowledgement, parseGitHubActionsRelayOutput, parseGitHubActionsRunnerProtocol, + parseGitHubActionsViewerProtocol, parseGitHubActionsWorkState, relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, @@ -54,6 +59,12 @@ function relaySocket(readyState = 1): GitHubActionsRelaySocket & { }; } +function framedViewer() { + const viewer = relaySocket(); + attachGitHubActionsViewerProtocol(viewer, githubActionsFramedRunnerCapability); + return viewer; +} + test("github_actions exposes steerable terminal capabilities and label", () => { assert.equal(githubActionsRuntimeLabel("github_actions"), "GitHub Actions"); assert.equal(githubActionsRuntimeLabel("container"), ""); @@ -79,6 +90,17 @@ test("runner URL works without custom WebSocket headers", () => { githubActionsFramedRunnerCapability, ); assert.equal(githubActionsRunnerProtocolQuery, "runnerProtocol"); + assert.equal( + buildGitHubActionsViewerRelayUrl(), + "https://crabfleet.internal/api/session-control/github-actions/viewer?viewerProtocol=cfr1-framed-io-v1", + ); + assert.equal(parseGitHubActionsViewerProtocol(null), null); + assert.equal(parseGitHubActionsViewerProtocol("cfr1-framed-io-v2"), null); + assert.equal( + parseGitHubActionsViewerProtocol(githubActionsFramedRunnerCapability), + githubActionsFramedRunnerCapability, + ); + assert.equal(githubActionsViewerProtocolQuery, "viewerProtocol"); }); test("work states preserve running phases and map terminal outcomes", () => { @@ -95,8 +117,8 @@ test("work states preserve running phases and map terminal outcomes", () => { test("relay replaces the current runner and frames legacy raw runner output", () => { const oldRunner = relaySocket(); const runner = relaySocket(); - const viewerOne = relaySocket(); - const viewerTwo = relaySocket(); + const viewerOne = framedViewer(); + const viewerTwo = framedViewer(); assert.equal(replaceGitHubActionsRunner([oldRunner]), 1); assert.deepEqual(oldRunner.closed, [[1012, "runner replaced"]]); @@ -144,7 +166,7 @@ test("relay replaces the current runner and frames legacy raw runner output", () test("legacy runners receive raw input and the relay acknowledges delivery", () => { const runner = relaySocket(); - const viewer = relaySocket(); + const viewer = framedViewer(); const input = encodeGitHubActionsRelayInput("input-legacy", "steer"); assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, input, [runner], []), 1); @@ -159,7 +181,7 @@ test("connection-time opt-in frames the first input without a pending handshake" const closedRunner = relaySocket(3); const openRunner = relaySocket(); const laterRunner = relaySocket(); - const viewer = relaySocket(); + const viewer = framedViewer(); const input = encodeGitHubActionsRelayInput("input-one", "steer"); attachGitHubActionsRunnerProtocol(openRunner, githubActionsFramedRunnerCapability); @@ -189,7 +211,7 @@ test("relay rejects framed input only when no runner accepts the frame", () => { runner.send = () => { throw new Error("runner disconnected"); }; - const viewer = relaySocket(); + const viewer = framedViewer(); const input = encodeGitHubActionsRelayInput("input-failed", "steer"); assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, input, [runner], []), 0); @@ -199,7 +221,7 @@ test("relay rejects framed input only when no runner accepts the frame", () => { error: "GitHub Actions runner did not accept terminal input", }); - const waitingViewer = relaySocket(); + const waitingViewer = framedViewer(); assert.equal(relayGitHubActionsWebSocketMessage("viewer", waitingViewer, input, [], []), 0); assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(waitingViewer.sent[0]!), { inputId: "input-failed", @@ -210,8 +232,8 @@ test("relay rejects framed input only when no runner accepts the frame", () => { test("runner acknowledgements retain correlation and fan out to viewers", () => { const runner = relaySocket(); - const viewerOne = relaySocket(); - const viewerTwo = relaySocket(); + const viewerOne = framedViewer(); + const viewerTwo = framedViewer(); const acknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ inputId: "input-two", accepted: true, @@ -237,7 +259,7 @@ test("runner acknowledgements retain correlation and fan out to viewers", () => test("negotiated runners frame output so control-shaped terminal bytes stay output", () => { const runner = relaySocket(); - const viewer = relaySocket(); + const viewer = framedViewer(); const controlShapedOutput = encodeGitHubActionsRelayInputAcknowledgement({ inputId: "collision", accepted: true, @@ -253,7 +275,7 @@ test("negotiated runners frame output so control-shaped terminal bytes stay outp }); test("typed acknowledgements reject malformed ids and preserve collision-shaped terminal text", () => { - const viewer = relaySocket(); + const viewer = framedViewer(); const runner = relaySocket(); const collision = '{"type":"github_actions_input_ack","inputId":"input-three","accepted":true}'; @@ -280,7 +302,7 @@ test("typed acknowledgements reject malformed ids and preserve collision-shaped test("relay consumes viewer resize controls and rejects unframed input", () => { const runner = relaySocket(); - const viewer = relaySocket(); + const viewer = framedViewer(); const resize = JSON.stringify({ type: "resize", cols: 120, rows: 40 }); const typedJson = new TextEncoder().encode(resize).buffer; @@ -294,7 +316,7 @@ test("relay consumes viewer resize controls and rejects unframed input", () => { }); test("relay tags and runner lifecycle notifications stay explicit", () => { - const viewer = relaySocket(); + const viewer = framedViewer(); assert.equal(githubActionsRelayRole(["github-actions-runner"]), "runner"); assert.equal(githubActionsRelayRole(["github-actions-viewer"]), "viewer"); assert.equal(githubActionsRelayRole([]), null); @@ -303,3 +325,55 @@ test("relay tags and runner lifecycle notifications stay explicit", () => { type: "runner_waiting", }); }); + +test("unnegotiated viewers retain raw relay compatibility", () => { + const runner = relaySocket(); + const viewer = relaySocket(); + + assert.equal(gitHubActionsViewerUsesFramedProtocol(viewer), false); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, "steer", [runner], []), 1); + assert.deepEqual(runner.sent, ["steer"]); + assert.deepEqual(JSON.parse(viewer.sent[0] as string), { + type: "github_actions_input_ack", + accepted: true, + }); + + assert.equal( + relayGitHubActionsWebSocketMessage("runner", runner, "output", [runner], [viewer]), + 1, + ); + assert.equal(viewer.sent[1], "output"); + assert.equal(notifyGitHubActionsViewers([viewer], "runner_disconnected"), 1); + assert.deepEqual(JSON.parse(viewer.sent[2] as string), { + type: "runner_disconnected", + }); +}); + +test("raw viewers bridge through framed runners without receiving CFR1 controls", () => { + const runner = relaySocket(); + const viewer = relaySocket(); + attachGitHubActionsRunnerProtocol(runner, githubActionsFramedRunnerCapability); + + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, "steer", [runner], []), 1); + const input = parseGitHubActionsRelayInput(runner.sent[0]!); + assert.ok(input); + assert.equal(new TextDecoder().decode(input.payload), "steer"); + assert.deepEqual(JSON.parse(viewer.sent[0] as string), { + type: "github_actions_input_ack", + accepted: true, + }); + + const acknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: input.inputId, + accepted: true, + }); + assert.equal( + relayGitHubActionsWebSocketMessage("runner", runner, acknowledgement, [runner], [viewer]), + 0, + ); + assert.equal(viewer.sent.length, 1); + + const output = encodeGitHubActionsRelayOutput("output"); + assert.equal(relayGitHubActionsWebSocketMessage("runner", runner, output, [runner], [viewer]), 1); + assert.equal(new TextDecoder().decode(viewer.sent[1] as ArrayBuffer), "output"); +}); From 35931c05d2b451b295d62b6604480812a833508e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:30:30 +0200 Subject: [PATCH 104/242] fix(actions): confirm viewer relay negotiation --- src/github-actions-runtime.ts | 7 +++ src/worker/interactive-terminal-service.ts | 2 + src/worker/session-control-do.ts | 7 ++- src/worker/terminal-hub.ts | 4 +- tests/application-architecture.test.ts | 2 + tests/github-actions-runtime.test.ts | 14 ++++++ tests/terminal-hub.test.ts | 52 ++++++++++++++++++++++ 7 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 7a8151fb..6189221b 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -32,6 +32,7 @@ export type GitHubActionsRelayInput = { export const githubActionsFramedRunnerCapability = "cfr1-framed-io-v1"; export const githubActionsRunnerProtocolQuery = "runnerProtocol"; export const githubActionsViewerProtocolQuery = "viewerProtocol"; +export const githubActionsViewerProtocolHeader = "x-crabfleet-viewer-protocol"; export type GitHubActionsRelayProtocol = typeof githubActionsFramedRunnerCapability; export type GitHubActionsRunnerProtocol = GitHubActionsRelayProtocol; export type GitHubActionsViewerProtocol = GitHubActionsRelayProtocol; @@ -113,6 +114,12 @@ export function buildGitHubActionsViewerRelayUrl(): string { return url.toString(); } +export function gitHubActionsViewerResponseUsesFramedProtocol(response: Response): boolean { + return ( + response.headers.get(githubActionsViewerProtocolHeader) === githubActionsFramedRunnerCapability + ); +} + export function parseGitHubActionsWorkState(value: unknown): GitHubActionsWorkState | null { const state = String(value ?? "").trim() as GitHubActionsWorkState; return workStates.has(state) ? state : null; diff --git a/src/worker/interactive-terminal-service.ts b/src/worker/interactive-terminal-service.ts index dc731ca0..742f4c10 100644 --- a/src/worker/interactive-terminal-service.ts +++ b/src/worker/interactive-terminal-service.ts @@ -8,6 +8,7 @@ import { } from "../terminal-multiplayer.ts"; import { buildGitHubActionsViewerRelayUrl, + gitHubActionsViewerResponseUsesFramedProtocol, githubActionsRuntime, } from "../github-actions-runtime.ts"; import { terminalFailureStatusForAdapter } from "../runtime-adapter.ts"; @@ -139,6 +140,7 @@ export class InteractiveTerminalService { upstream.accept(); return { socket: upstream, + inputAcknowledgements: gitHubActionsViewerResponseUsesFramedProtocol(upstreamResponse), outputAcknowledgements: false, markConnected: () => markInteractiveTerminalConnected( diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index 4993b603..a7bfa898 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -11,6 +11,7 @@ import { attachGitHubActionsViewerProtocol, githubActionsRelayRole, githubActionsRunnerProtocolQuery, + githubActionsViewerProtocolHeader, githubActionsViewerProtocolQuery, notifyGitHubActionsViewers, parseGitHubActionsRunnerProtocol, @@ -258,7 +259,11 @@ export class SessionControlDO extends DurableObject { notifyGitHubActionsViewers([server], "runner_waiting"); } } - return new Response(null, { status: 101, webSocket: client }); + const responseInit: ResponseInit = { status: 101, webSocket: client }; + if (role === "viewer" && protocol) { + responseInit.headers = { [githubActionsViewerProtocolHeader]: protocol }; + } + return new Response(null, responseInit); } } diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 6ddf902e..22353a14 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -41,6 +41,7 @@ type PendingTerminalInputAcknowledgement = { export type TerminalUpstream = { socket: WebSocket; markConnected: () => Promise; + inputAcknowledgements?: boolean; outputAcknowledgements: boolean; }; @@ -465,7 +466,8 @@ export class TerminalHub { viewCheck, cols, rows, - inputAcknowledgements: session.runtime === githubActionsRuntime, + inputAcknowledgements: + upstreamConnection.inputAcknowledgements ?? session.runtime === githubActionsRuntime, pendingInputAcknowledgements: new Map(), outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, diff --git a/tests/application-architecture.test.ts b/tests/application-architecture.test.ts index 734f92ca..65ba3268 100644 --- a/tests/application-architecture.test.ts +++ b/tests/application-architecture.test.ts @@ -82,6 +82,7 @@ test("GitHub Actions viewer protocol is requested and attached before relay acce ]); assert.match(terminal, /stub\.fetch\(\s*buildGitHubActionsViewerRelayUrl\(\)/); + assert.match(terminal, /gitHubActionsViewerResponseUsesFramedProtocol\(upstreamResponse\)/); const attach = relay.indexOf("attachGitHubActionsViewerProtocol(server, protocol)"); const accept = relay.indexOf( 'this.ctx.acceptWebSocket(server, ["github-actions-viewer"])', @@ -89,6 +90,7 @@ test("GitHub Actions viewer protocol is requested and attached before relay acce ); assert.notEqual(attach, -1); assert.ok(accept > attach); + assert.match(relay, /\[githubActionsViewerProtocolHeader\]: protocol/); }); test("worker entrypoint retains only routing and platform composition", async () => { diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index ae003d31..1acb6dd9 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -14,7 +14,9 @@ import { githubActionsRelayRole, githubActionsRunnerProtocolQuery, githubActionsRuntimeLabel, + githubActionsViewerProtocolHeader, githubActionsViewerProtocolQuery, + gitHubActionsViewerResponseUsesFramedProtocol, gitHubActionsRunnerUsesFramedProtocol, gitHubActionsViewerUsesFramedProtocol, isGitHubActionsViewerControlMessage, @@ -101,6 +103,18 @@ test("runner URL works without custom WebSocket headers", () => { githubActionsFramedRunnerCapability, ); assert.equal(githubActionsViewerProtocolQuery, "viewerProtocol"); + assert.equal(githubActionsViewerProtocolHeader, "x-crabfleet-viewer-protocol"); + assert.equal( + gitHubActionsViewerResponseUsesFramedProtocol( + new Response(null, { + headers: { + [githubActionsViewerProtocolHeader]: githubActionsFramedRunnerCapability, + }, + }), + ), + true, + ); + assert.equal(gitHubActionsViewerResponseUsesFramedProtocol(new Response()), false); }); test("work states preserve running phases and map terminal outcomes", () => { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index b47f254f..e1345443 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -608,6 +608,58 @@ test("GitHub Actions input waits for the correlated runner acknowledgement", asy server.emit("close"); }); +test("GitHub Actions falls back to raw relay input when viewer negotiation is absent", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async openUpstream() { + return { + socket: upstream, + inputAcknowledgements: false, + outputAcknowledgements: false, + async markConnected() {}, + }; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("legacy"), + }), + }); + await flushQueues(); + await flushQueues(); + + assert.equal(new TextDecoder().decode(upstream.sent.at(-1) as Uint8Array), "legacy"); + assert.deepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "input-accepted", + }); + server.emit("close"); +}); + test("GitHub Actions relay rejection is request-scoped", async () => { const client = socket(); const server = socket(); From b317dc7b97863ddf51361d526c42087992fc1dde Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:30:40 +0200 Subject: [PATCH 105/242] docs(changelog): record final audit fixes --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02548933..83c631a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,13 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. -- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, retain legacy raw peer compatibility during mixed deployments, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Make terminal input delivery durable across multiplex subscribers, retain and acknowledge initial output only after an attachment owns it, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers during deployment configuration, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, stale desktop registrations including listener-failure and application-termination races, dropped auto-starts, stuck remote input, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping publishers cannot displace cleanup authority, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. -- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, requiring an explicit architecture checksum when overriding the pinned default version. +- Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. - Add persisted send-only and receive-only clipboard directions to the native viewer's focus toolbar; automatic sync respects the direction while the explicit Send and Get actions keep working. From b52c3a6677083c6cdca7eee559941a5669d08a80 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:33:09 +0200 Subject: [PATCH 106/242] test(macos): stabilize descendant drain fixture --- .../Tests/CrabfleetMacTests/PrivateMacShareTests.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 871d3bde..215e59a4 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -120,7 +120,8 @@ struct PrivateMacShareTests { ).write(to: executable) try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: executable.path) - let runner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 0.5) + // Give the helper time to spawn under a concurrently loaded Swift test runner. + let runner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 2) let clock = ContinuousClock() let startedAt = clock.now await #expect(throws: PrivateMacShareError.commandTimedOut) { @@ -135,7 +136,7 @@ struct PrivateMacShareTests { _ = Darwin.kill(descendantPID, SIGKILL) } #expect(Darwin.kill(descendantPID, 0) == 0) - #expect(elapsed < .seconds(2)) + #expect(elapsed < .seconds(4)) } @Test From 270ca5b6a49922abbdff5726d9f2c1def0f66be1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:38:24 +0200 Subject: [PATCH 107/242] fix(terminal): serialize confirmed input delivery --- internal/terminalws/client.go | 67 +++++++++++++++--------- internal/terminalws/client_test.go | 82 +++++++++++++++++++++++------- 2 files changed, 106 insertions(+), 43 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index af7174b0..3da77955 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -87,7 +87,7 @@ type Client struct { stateMu sync.Mutex inputWaiter chan error attachment *terminalAttachment - pendingAttachmentFrames []frame + attachmentReady chan struct{} readerDone chan struct{} readerErr error } @@ -296,6 +296,7 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { return c.readerError() case <-ctx.Done(): c.clearInputWaiter(waiter) + _ = c.Close() return ctx.Err() } } @@ -342,7 +343,7 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c for { count, err := terminal.Read(buffer) if count > 0 && c.canInput.Load() { - if writeErr := c.SendInput(ctx, buffer[:count]); writeErr != nil { + if writeErr := c.SendInputConfirmed(ctx, buffer[:count]); writeErr != nil { errCh <- writeErr return } @@ -452,7 +453,10 @@ func (c *Client) write(ctx context.Context, current frame) error { func (c *Client) startReader() { ctx, cancel := context.WithCancel(context.Background()) c.readCancel = cancel + c.stateMu.Lock() c.readerDone = make(chan struct{}) + c.attachmentReady = make(chan struct{}) + c.stateMu.Unlock() go c.readLoop(ctx) } @@ -549,22 +553,20 @@ func (c *Client) registerAttachment() (*terminalAttachment, error) { if c.attachment != nil { return nil, errors.New("terminal client is already attached") } - if len(c.pendingAttachmentFrames) == 0 { - select { - case <-c.readerDone: - return nil, readerUnavailableError(c.readerErr) - default: - } + select { + case <-c.readerDone: + return nil, readerUnavailableError(c.readerErr) + default: } attachment := &terminalAttachment{ - frames: make(chan frame, len(c.pendingAttachmentFrames)+16), + frames: make(chan frame, 16), done: make(chan struct{}), } - for _, current := range c.pendingAttachmentFrames { - attachment.frames <- current - } - c.pendingAttachmentFrames = nil c.attachment = attachment + if c.attachmentReady != nil { + close(c.attachmentReady) + c.attachmentReady = nil + } return attachment, nil } @@ -573,24 +575,39 @@ func (c *Client) clearAttachment(attachment *terminalAttachment) { if c.attachment == attachment { c.attachment = nil close(attachment.done) + select { + case <-c.readerDone: + default: + c.attachmentReady = make(chan struct{}) + } } c.stateMu.Unlock() } func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) { - c.stateMu.Lock() - attachment := c.attachment - if attachment == nil { - c.pendingAttachmentFrames = append(c.pendingAttachmentFrames, current) + for { + c.stateMu.Lock() + attachment := c.attachment + ready := c.attachmentReady c.stateMu.Unlock() - return - } - c.stateMu.Unlock() - select { - case attachment.frames <- current: - case <-attachment.done: - c.deliverOrQueueOutput(ctx, current) - case <-ctx.Done(): + if attachment == nil { + if ready == nil { + return + } + select { + case <-ready: + continue + case <-ctx.Done(): + return + } + } + select { + case attachment.frames <- current: + return + case <-attachment.done: + case <-ctx.Done(): + return + } } } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 2e55a1f9..2d3f0a97 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -339,7 +340,6 @@ func TestClientDefersOutputAcknowledgementUntilAttach(t *testing.T) { } defer client.Close() <-outputSent - waitForPendingAttachmentFrames(t, client, 1) select { case bytes := <-acknowledged: t.Fatalf("acknowledged %d bytes before attach", bytes) @@ -639,6 +639,69 @@ func TestSendInputConfirmedReturnsImmediatelyForEmptyInput(t *testing.T) { } } +func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { + inputReceived := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-confirm-timeout", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + close(inputReceived) + _, _, _ = conn.Read(r.Context()) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-confirm-timeout", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err = client.SendInputConfirmed(ctx, []byte("first\n")) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v", err) + } + <-inputReceived + if err := client.SendInputConfirmed(context.Background(), []byte("second\n")); err == nil { + t.Fatal("timed-out client accepted another input") + } +} + func TestSendInputConfirmedFallsBackWithoutServerCapability(t *testing.T) { receivedInput := make(chan []byte, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1106,23 +1169,6 @@ func TestClientContinuesReadOnlyAndResumesControl(t *testing.T) { } } -func waitForPendingAttachmentFrames(t *testing.T, client *Client, count int) { - t.Helper() - deadline := time.Now().Add(time.Second) - for { - client.stateMu.Lock() - pending := len(client.pendingAttachmentFrames) - client.stateMu.Unlock() - if pending == count { - return - } - if time.Now().After(deadline) { - t.Fatalf("pending attachment frames = %d", pending) - } - time.Sleep(time.Millisecond) - } -} - type readWriter struct { reader io.Reader closer io.Closer From bd51d0e9d078d9871090406e10658a0f67729683 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:39:27 +0200 Subject: [PATCH 108/242] fix(macos): await concurrent share teardown --- .../PrivateMacShareController.swift | 26 ++++++++++ .../PrivateMacShareTests.swift | 52 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 5b0d6b4f..32ddb5db 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -12,6 +12,25 @@ enum PrivateMacSharePermissionPolicy { } } +@MainActor +final class PrivateMacShareStopCoordinator { + private var operation: Task? + + func perform(_ body: @escaping @MainActor () async -> Void) async { + if let operation { + await operation.value + return + } + + let operation = Task { @MainActor in + await body() + } + self.operation = operation + await operation.value + self.operation = nil + } +} + @MainActor final class DesktopHostRegistrationLifecycle { private struct PublishedRegistration: Equatable { @@ -167,6 +186,7 @@ final class PrivateMacShareController: ObservableObject { private var registryOperationGeneration: UInt64 = 0 private var publishingServerGeneration: UInt64? private var refreshWaiters: [CheckedContinuation] = [] + private let stopCoordinator = PrivateMacShareStopCoordinator() init( runner: (any TailscaleCommandRunning)? = nil, @@ -363,6 +383,12 @@ final class PrivateMacShareController: ObservableObject { } func stop() async { + await stopCoordinator.perform { [weak self] in + await self?.performStop() + } + } + + private func performStop() async { guard phase.isRunning || phase == .failed else { return } let generation = beginLifecycleTransition() phase = .stopping diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 215e59a4..5f185387 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -338,6 +338,41 @@ struct PrivateMacShareTests { #expect(await waitUntilAsync { controller.registryPhase == .notPublished }) } + @Test @MainActor + func concurrentStopsAwaitTheSameInFlightOperation() async throws { + let coordinator = PrivateMacShareStopCoordinator() + let operation = SuspendedAsyncOperation() + let firstState = AsyncInvocationState() + let secondState = AsyncInvocationState() + + let first = Task { + await coordinator.perform { + await operation.run() + } + await firstState.markFinished() + } + #expect(await waitUntilAsync { await operation.invocationCount == 1 }) + + let second = Task { + await coordinator.perform { + await operation.run() + } + await secondState.markFinished() + } + try await Task.sleep(for: .milliseconds(20)) + + #expect(await operation.invocationCount == 1) + #expect(!(await firstState.finished)) + #expect(!(await secondState.finished)) + + await operation.finish() + await first.value + await second.value + + #expect(await firstState.finished) + #expect(await secondState.finished) + } + @Test @MainActor func failedDesktopPublicationIsNotUnregistered() async throws { let identity = desktopIdentity(name: "failed-publish", address: "100.64.12.40") @@ -1246,6 +1281,23 @@ private actor AsyncInvocationState { } } +private actor SuspendedAsyncOperation { + private var continuation: CheckedContinuation? + private(set) var invocationCount = 0 + + func run() async { + invocationCount += 1 + await withCheckedContinuation { continuation in + self.continuation = continuation + } + } + + func finish() { + continuation?.resume() + continuation = nil + } +} + private actor SuspendedDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case registerStarted From a372e514f1bc1020755d2344c5c4895294f1dc92 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:40:02 +0200 Subject: [PATCH 109/242] fix(runtime): preserve adapter migration fallback --- src/worker/deployment.ts | 7 +------ tests/deployment.test.ts | 15 +++++++-------- tests/session-create-request.test.ts | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/worker/deployment.ts b/src/worker/deployment.ts index 8b6978ac..f94ab9f2 100644 --- a/src/worker/deployment.ts +++ b/src/worker/deployment.ts @@ -78,12 +78,7 @@ function validateRuntimeProfileRoutes( const template = env.CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE; const hasDirect = typeof direct === "string" && direct.length > 0; const hasTemplate = typeof template === "string" && template.length > 0; - if (hasDirect && hasTemplate) { - throw new TypeError( - "CRABBOX_RUNTIME_ADAPTER_URL and CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE are mutually exclusive", - ); - } - if (!hasTemplate) return; + if (hasDirect || !hasTemplate) return; const profileIDs = runtimeProfiles.length > 0 ? runtimeProfiles.map((profile) => profile.id) : [defaultProfile]; const unroutable = profileIDs.find( diff --git a/tests/deployment.test.ts b/tests/deployment.test.ts index d0da1c25..bf59024b 100644 --- a/tests/deployment.test.ts +++ b/tests/deployment.test.ts @@ -77,14 +77,13 @@ test("configured runtime profiles are allowlisted behaviorally", () => { ); }); -test("profile-routed deployments reject profiles the adapter template cannot address", () => { - assert.throws( - () => - deploymentConfig({ - CRABBOX_RUNTIME_ADAPTER_URL: "https://controller.example.test/adapter", - CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE: "https://controller.example.test/adapters/{profile}", - }), - /are mutually exclusive/, +test("deployment reads tolerate adapter migration ambiguity and validate template routes", () => { + assert.equal( + deploymentConfig({ + CRABBOX_RUNTIME_ADAPTER_URL: "https://controller.example.test/adapter", + CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE: "https://controller.example.test/adapters/{profile}", + }).defaultProfile, + "default", ); assert.throws( () => diff --git a/tests/session-create-request.test.ts b/tests/session-create-request.test.ts index 8aa378c7..60041da0 100644 --- a/tests/session-create-request.test.ts +++ b/tests/session-create-request.test.ts @@ -76,6 +76,21 @@ test("session create requests enforce configured profiles and capability overlay }); test("session create requests fail before allocation when adapter routing is incomplete", () => { + assert.throws( + () => + resolveInteractiveSessionCreateRequest( + runtimeEnv({ + CRABBOX_RUNTIME_ADAPTER_URL: "https://adapter.example.test", + CRABBOX_RUNTIME_ADAPTER_URL_TEMPLATE: + "https://controller.example.test/adapters/{profile}", + CRABBOX_RUNTIME_ADAPTER_TOKEN: "adapter-token", + CRABBOX_RUNTIME_ADAPTER_NAMESPACE: "fleet", + }), + { repo: "openclaw/crabfleet" }, + { owner: "maintainer", createdBy: "maintainer" }, + ), + /runtime adapter URL or profile route template must be valid and unambiguous/, + ); assert.throws( () => resolveInteractiveSessionCreateRequest( From adf45ce3cd296f918d5aca41a3131b22c59e5df0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:40:06 +0200 Subject: [PATCH 110/242] docs(actions): clarify viewer relay negotiation --- docs/api.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 194bcb6e..f6c17203 100644 --- a/docs/api.md +++ b/docs/api.md @@ -608,8 +608,12 @@ Opening the returned URL unchanged selects legacy raw input and output. Adding the exact `runnerProtocol=cfr1-framed-io-v1` query selects framed input, output, acknowledgements, and relay control traffic. The application propagates only that exact value to `SessionControlDO`, which stores the mode on the server -socket before accepting it. The relay wraps legacy output before forwarding it -to viewers, so arbitrary raw PTY bytes cannot be consumed as control traffic. +socket before accepting it. Viewer framing is negotiated independently: framed +viewers receive `CFR1` output and control frames, while unnegotiated viewers +retain raw output and legacy JSON notices during rolling upgrades. The relay +therefore wraps legacy runner output only for framed viewers, and unwraps framed +runner output for raw viewers. Arbitrary raw PTY bytes cannot be consumed as +control traffic by framed viewers. Each `CFR1` frame occupies one binary WebSocket message and starts with: From 1bc068979894329acf3eeb76757ac7c29c20b91d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:41:11 +0200 Subject: [PATCH 111/242] fix(actions): fence writes to authenticated revision --- src/worker/github-actions-application.ts | 20 +++++-- src/worker/github-actions-repository.ts | 28 +++++++--- .../github-actions-runner-connection.ts | 28 ++++++---- .../github-actions-session-registration.ts | 3 +- .../github-actions-session-work-state.ts | 2 + tests/github-actions-repository.test.ts | 56 +++++++++++++------ .../github-actions-runner-connection.test.ts | 17 +++++- ...ithub-actions-session-registration.test.ts | 11 ++-- .../github-actions-session-work-state.test.ts | 27 ++++++++- 9 files changed, 145 insertions(+), 47 deletions(-) diff --git a/src/worker/github-actions-application.ts b/src/worker/github-actions-application.ts index 09d65feb..f81f7557 100644 --- a/src/worker/github-actions-application.ts +++ b/src/worker/github-actions-application.ts @@ -88,7 +88,11 @@ export class GitHubActionsApplication { nextSessionId: () => nextInteractiveSessionId(this.env), insertSession: (values) => repository.insertSession(values), readById: (id) => repository.readById(id), - updateSession: (id, values, expected) => repository.updateSession(id, values, expected), + updateSession: (id, values, expected) => + repository.updateSession(id, values, { + kind: "registration", + registration: expected, + }), isConstraintError, disconnectRunner: (id) => this.disconnectRunner(id), appendEvent: (id, message, now) => this.appendMessageEvent(id, user, message, now), @@ -108,8 +112,12 @@ export class GitHubActionsApplication { const store: GitHubActionsWorkStateStore = { now: () => Date.now(), readRow: (sessionId) => repository.readById(sessionId), - persist: (sessionId, values, expectedTerminalStatus) => - repository.updateSession(sessionId, values, undefined, expectedTerminalStatus), + persist: (sessionId, values, expectedRevision, expectedTerminalStatus) => + repository.updateSession(sessionId, values, { + kind: "authenticated", + revision: expectedRevision, + ...(expectedTerminalStatus ? { terminalStatus: expectedTerminalStatus } : {}), + }), appendEvent: (sessionId, message, now) => this.appendMessageEvent(sessionId, user, message, now), disconnectRunner: (sessionId) => this.disconnectRunner(sessionId), @@ -156,7 +164,11 @@ export class GitHubActionsApplication { const repository = this.repository(); const store: GitHubActionsRunnerConnectionStore = { now: () => Date.now(), - persist: (sessionId, values) => repository.updateSession(sessionId, values), + persist: (sessionId, values, expectedRevision) => + repository.updateSession(sessionId, values, { + kind: "authenticated", + revision: expectedRevision, + }), appendEvent: (sessionId, message, now) => this.appendMessageEvent(sessionId, user, message, now), }; diff --git a/src/worker/github-actions-repository.ts b/src/worker/github-actions-repository.ts index 65036d64..c349726b 100644 --- a/src/worker/github-actions-repository.ts +++ b/src/worker/github-actions-repository.ts @@ -17,6 +17,17 @@ type GitHubActionsSessionUpdate = | GitHubActionsWorkStateUpdate | GitHubActionsRunnerConnectionUpdate; +export type GitHubActionsSessionUpdateExpectation = + | { + kind: "registration"; + registration: GitHubActionsSessionRegistrationExpectation; + } + | { + kind: "authenticated"; + revision: number; + terminalStatus?: InteractiveSessionStatus; + }; + const terminalWorkStates = ["completed", "failed", "canceled", "blocked"]; const terminalSessionStatuses = ["stopped", "expired", "failed"] as const; @@ -54,8 +65,7 @@ export class GitHubActionsRepository { async updateSession( id: string, values: GitHubActionsSessionUpdate, - expectedRegistration?: GitHubActionsSessionRegistrationExpectation, - expectedTerminalStatus?: InteractiveSessionStatus, + expectation: GitHubActionsSessionUpdateExpectation, ): Promise { let update = database(this.env) .updateTable("interactive_sessions") @@ -64,9 +74,10 @@ export class GitHubActionsRepository { .where("runtime", "=", "github_actions"); if (isRegistrationUpdate(values)) { - if (!expectedRegistration) { + if (expectation.kind !== "registration") { throw new Error("GitHub Actions registration update requires expected state"); } + const expectedRegistration = expectation.registration; update = update .where("updated_at", "=", expectedRegistration.updated_at) .where("status", "=", expectedRegistration.status) @@ -78,12 +89,12 @@ export class GitHubActionsRepository { ? update.where("agent_token_hash", "is", null) : update.where("agent_token_hash", "=", expectedRegistration.agent_token_hash); } else if (isWorkStateUpdate(values) && terminalWorkStates.includes(values.work_state)) { - if (!expectedTerminalStatus) { + if (expectation.kind !== "authenticated" || !expectation.terminalStatus) { throw new Error("terminal GitHub Actions update requires expected session status"); } update = update - .where("updated_at", "<=", values.updated_at) - .where("status", "=", expectedTerminalStatus) + .where("updated_at", "=", expectation.revision) + .where("status", "=", expectation.terminalStatus) .where("status", "not in", terminalSessionStatuses) .where((expressions) => expressions.or([ @@ -92,8 +103,11 @@ export class GitHubActionsRepository { ]), ); } else { + if (expectation.kind !== "authenticated") { + throw new Error("GitHub Actions update requires authenticated revision"); + } update = update - .where("updated_at", "<=", values.updated_at) + .where("updated_at", "=", expectation.revision) .where("work_state", "not in", terminalWorkStates) .where("status", "not in", terminalSessionStatuses); } diff --git a/src/worker/github-actions-runner-connection.ts b/src/worker/github-actions-runner-connection.ts index ca9cdf53..3722c0ae 100644 --- a/src/worker/github-actions-runner-connection.ts +++ b/src/worker/github-actions-runner-connection.ts @@ -17,7 +17,11 @@ export type GitHubActionsRunnerConnectionUpdate = { export type GitHubActionsRunnerConnectionStore = { now(): number; - persist(id: string, values: GitHubActionsRunnerConnectionUpdate): Promise; + persist( + id: string, + values: GitHubActionsRunnerConnectionUpdate, + expectedRevision: number, + ): Promise; appendEvent(id: string, message: string, now: number): Promise; }; @@ -41,15 +45,19 @@ export class GitHubActionsRunnerConnectionService { : session.workPhase; const status = session.status === "attached" || session.status === "detached" ? session.status : "ready"; - await this.store.persist(session.id, { - status, - work_state: state, - work_phase: phase, - last_heartbeat_at: now, - last_seen_at: now, - updated_at: now, - last_event: githubActionsRunnerConnectedEvent, - }); + await this.store.persist( + session.id, + { + status, + work_state: state, + work_phase: phase, + last_heartbeat_at: now, + last_seen_at: now, + updated_at: now, + last_event: githubActionsRunnerConnectedEvent, + }, + session.updatedAt, + ); await this.store.appendEvent(session.id, githubActionsRunnerConnectedEvent, now); } } diff --git a/src/worker/github-actions-session-registration.ts b/src/worker/github-actions-session-registration.ts index 07d4c32c..b3e0bf28 100644 --- a/src/worker/github-actions-session-registration.ts +++ b/src/worker/github-actions-session-registration.ts @@ -157,6 +157,7 @@ export class GitHubActionsSessionRegistrationService { const resumed = existing.work_state !== "registered" || existing.status !== "ready"; const message = resumed ? "GitHub Actions work resumed" : "GitHub Actions work registered"; + const registrationRevision = Math.max(existing.updated_at + 1, now); await this.store.updateSession( existing.id, { @@ -174,7 +175,7 @@ export class GitHubActionsSessionRegistrationService { terminal_failure_reason: null, terminal_finalize_pending: 0, credential_cleanup_terminal_status: null, - updated_at: now, + updated_at: registrationRevision, last_seen_at: now, last_event: message, agent_token_hash: agentTokenHash, diff --git a/src/worker/github-actions-session-work-state.ts b/src/worker/github-actions-session-work-state.ts index 1ffe7fc4..7c1caf7a 100644 --- a/src/worker/github-actions-session-work-state.ts +++ b/src/worker/github-actions-session-work-state.ts @@ -41,6 +41,7 @@ export type GitHubActionsWorkStateStore = { persist( id: string, values: GitHubActionsWorkStateUpdate, + expectedRevision: number, expectedTerminalStatus?: InteractiveSessionStatus, ): Promise; appendEvent(id: string, message: string, now: number): Promise; @@ -111,6 +112,7 @@ export class GitHubActionsWorkStateService { updated_at: now, stopped_at: terminal ? now : null, }, + session.updatedAt, terminal ? row.status : undefined, ); if (changed) { diff --git a/tests/github-actions-repository.test.ts b/tests/github-actions-repository.test.ts index 19bd15e5..5873aed0 100644 --- a/tests/github-actions-repository.test.ts +++ b/tests/github-actions-repository.test.ts @@ -68,9 +68,19 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = now: 100, }), ); - await repository.updateSession("IS-101", registrationUpdate, registrationExpectation); - await repository.updateSession("IS-101", workStateUpdate, undefined, "attached"); - await repository.updateSession("IS-101", runnerConnectionUpdate); + await repository.updateSession("IS-101", registrationUpdate, { + kind: "registration", + registration: registrationExpectation, + }); + await repository.updateSession("IS-101", workStateUpdate, { + kind: "authenticated", + revision: 190, + terminalStatus: "attached", + }); + await repository.updateSession("IS-101", runnerConnectionUpdate, { + kind: "authenticated", + revision: 290, + }); assert.equal(executions.length, 6); assert.match(executions[0].sql, /select .* from "interactive_sessions"/i); @@ -93,12 +103,12 @@ test("GitHub Actions repository owns registration and lifecycle SQL", async () = assert.match(executions[3].sql, /"status" = \?/i); assert.match(executions[3].sql, /"work_state" = \?/i); assert.match(executions[3].sql, /"work_phase" = \?/i); - assert.match(executions[4].sql, /"updated_at" <= \?/i); + assert.match(executions[4].sql, /"updated_at" = \?/i); assert.match(executions[4].sql, /"status" = \?/i); assert.match(executions[4].sql, /"status" not in/i); assert.ok(executions[4].parameters.includes("attached")); assert.ok(executions[4].parameters.includes("expired")); - assert.match(executions[5].sql, /"updated_at" <= \?/i); + assert.match(executions[5].sql, /"updated_at" = \?/i); assert.match(executions[3].sql, /"owner_subject" = \?/i); assert.doesNotMatch(executions[3].sql, /"work_state" not in/i); assert.match(executions[4].sql, /"work_state" not in/i); @@ -111,13 +121,19 @@ test("GitHub Actions repository rejects stale or invalid state transitions", asy const executions: Execution[] = []; const repository = new GitHubActionsRepository(runtimeEnv(executions, 0)); - await assert.rejects(repository.updateSession("IS-101", runnerConnectionUpdate), (error) => { - assert.equal( - typeof error === "object" && error && "status" in error ? error.status : undefined, - 409, - ); - return true; - }); + await assert.rejects( + repository.updateSession("IS-101", runnerConnectionUpdate, { + kind: "authenticated", + revision: 290, + }), + (error) => { + assert.equal( + typeof error === "object" && error && "status" in error ? error.status : undefined, + 409, + ); + return true; + }, + ); assert.equal(executions.length, 1); }); @@ -126,12 +142,16 @@ test("terminal work-state updates require an observed non-terminal status", asyn const repository = new GitHubActionsRepository(runtimeEnv(executions)); await assert.rejects( - repository.updateSession("IS-101", { - ...workStateUpdate, - status: "stopped", - work_state: "completed", - stopped_at: 200, - }), + repository.updateSession( + "IS-101", + { + ...workStateUpdate, + status: "stopped", + work_state: "completed", + stopped_at: 200, + }, + { kind: "authenticated", revision: 190 }, + ), { message: "terminal GitHub Actions update requires expected session status", }, diff --git a/tests/github-actions-runner-connection.test.ts b/tests/github-actions-runner-connection.test.ts index b207e9ec..a669c0ab 100644 --- a/tests/github-actions-runner-connection.test.ts +++ b/tests/github-actions-runner-connection.test.ts @@ -27,21 +27,25 @@ function session(values: Parameters[0] = {}) { function connectionStore(): { store: GitHubActionsRunnerConnectionStore; updates: GitHubActionsRunnerConnectionUpdate[]; + expectedRevisions: number[]; events: string[]; operations: string[]; } { const updates: GitHubActionsRunnerConnectionUpdate[] = []; + const expectedRevisions: number[] = []; const events: string[] = []; const operations: string[] = []; return { updates, + expectedRevisions, events, operations, store: { now: () => 700, - persist: async (_id, values) => { + persist: async (_id, values, expectedRevision) => { operations.push("persist"); updates.push(values); + expectedRevisions.push(expectedRevision); }, appendEvent: async (_id, message) => { operations.push("event"); @@ -52,7 +56,7 @@ function connectionStore(): { } test("waiting runners become active with durable connection evidence", async () => { - const { store, updates, events, operations } = connectionStore(); + const { store, updates, expectedRevisions, events, operations } = connectionStore(); await new GitHubActionsRunnerConnectionService(store).connect( session({ status: "provisioning" }), ); @@ -69,6 +73,7 @@ test("waiting runners become active with durable connection evidence", async () }, ]); assert.deepEqual(events, [githubActionsRunnerConnectedEvent]); + assert.deepEqual(expectedRevisions, [session({ status: "provisioning" }).updatedAt]); assert.deepEqual(operations, ["persist", "event"]); }); @@ -87,6 +92,14 @@ test("reconnecting runners preserve active status, state, and phase", async () = assert.equal(updates[0]?.work_phase, "codex_turn"); }); +test("runner connections retain the exact revision authenticated before a token rotation", async () => { + const { store, expectedRevisions } = connectionStore(); + + await new GitHubActionsRunnerConnectionService(store).connect(session({ updated_at: 400 })); + + assert.deepEqual(expectedRevisions, [400]); +}); + test("runner connections reject non-work sessions", async () => { const { store } = connectionStore(); const service = new GitHubActionsRunnerConnectionService(store); diff --git a/tests/github-actions-session-registration.test.ts b/tests/github-actions-session-registration.test.ts index 809d96d8..481f4d19 100644 --- a/tests/github-actions-session-registration.test.ts +++ b/tests/github-actions-session-registration.test.ts @@ -364,7 +364,10 @@ test("registration adopts a concurrently inserted work key", async () => { assert.equal(result.session.id, "IS-concurrent"); assert.equal(state.workKeyReads, 2); assert.equal(state.updates[0]?.id, "IS-concurrent"); - assert.equal(state.updates[0]?.values.updated_at, 100); + assert.equal( + state.updates[0]?.values.updated_at, + Math.max(state.concurrentRow.updated_at + 1, 100), + ); assert.equal(state.updates[0]?.expected.agent_token_hash, state.concurrentRow.agent_token_hash); }); @@ -428,10 +431,10 @@ test("concurrent registration adoption rotates exactly one usable token", async state.rows.get(existing.id)?.agent_token_hash, `${fulfilled[0]?.value.agentToken}-hash`, ); - assert.equal(state.rows.get(existing.id)?.updated_at, 100); + assert.equal(state.rows.get(existing.id)?.updated_at, 101); }); -test("registration repairs a future timestamp without blocking immediate writers", async () => { +test("registration revisions advance monotonically when the stored clock is ahead", async () => { const existing = sessionRow({ id: "IS-future-revision", runtime: "github_actions", @@ -449,7 +452,7 @@ test("registration repairs a future timestamp without blocking immediate writers owner: "operator@example.test", }); - assert.equal(state.rows.get(existing.id)?.updated_at, 100); + assert.equal(state.rows.get(existing.id)?.updated_at, 501); }); test("registration rejects invalid input and work keys owned by another runtime", async () => { diff --git a/tests/github-actions-session-work-state.test.ts b/tests/github-actions-session-work-state.test.ts index 7be6baa4..19b5cbab 100644 --- a/tests/github-actions-session-work-state.test.ts +++ b/tests/github-actions-session-work-state.test.ts @@ -13,6 +13,7 @@ import { sessionRow } from "./helpers/session-row.ts"; type WorkStateStoreState = { row: InteractiveSessionRow | null; update: GitHubActionsWorkStateUpdate | null; + expectedRevision: number | undefined; expectedTerminalStatus: InteractiveSessionRow["status"] | undefined; events: string[]; operations: string[]; @@ -49,6 +50,7 @@ function workStateStore(values: Partial = {}): { ...values, }), update: null, + expectedRevision: undefined, expectedTerminalStatus: undefined, events: [], operations: [], @@ -57,9 +59,10 @@ function workStateStore(values: Partial = {}): { const store: GitHubActionsWorkStateStore = { now: () => 500, readRow: async () => state.row, - persist: async (_id, update, expectedTerminalStatus) => { + persist: async (_id, update, expectedRevision, expectedTerminalStatus) => { state.operations.push("persist"); state.update = update; + state.expectedRevision = expectedRevision; state.expectedTerminalStatus = expectedTerminalStatus; if (state.row) state.row = { ...state.row, ...update }; }, @@ -108,6 +111,7 @@ test("active work-state updates project fields and clear stale completion", asyn stopped_at: null, }); assert.deepEqual(state.events, ["running: codex_turn"]); + assert.equal(state.expectedRevision, workSession().updatedAt); assert.deepEqual(state.operations, ["persist", "event", "read"]); }); @@ -143,6 +147,7 @@ test("terminal work-state updates stop the session and disconnect the runner", a assert.equal(state.update?.completion_reason, "existing reason"); assert.equal(state.update?.stopped_at, 500); assert.equal(state.expectedTerminalStatus, "ready"); + assert.equal(state.expectedRevision, workSession().updatedAt); assert.deepEqual(state.events, ["failed: tests"]); assert.deepEqual(state.operations, ["persist", "event", "disconnect", "read"]); }); @@ -160,6 +165,26 @@ test("terminal work-state updates carry the status observed before persistence", assert.equal(state.expectedTerminalStatus, "attached"); }); +test("work-state updates retain the exact revision authenticated before a token rotation", async () => { + const authenticated = workSession({ updated_at: 400 }); + const { store, state } = workStateStore({ updated_at: 401, work_state: "registered" }); + store.persist = async (_id, _update, expectedRevision) => { + state.expectedRevision = expectedRevision; + if (state.row?.updated_at !== expectedRevision) { + throw new Error("GitHub Actions session changed; retry"); + } + }; + + await assert.rejects( + new GitHubActionsWorkStateService(store).update(authenticated, { + state: "running", + }), + { message: "GitHub Actions session changed; retry" }, + ); + + assert.equal(state.expectedRevision, 400); +}); + test("terminal runner disconnect races remain best effort", async () => { const { store, state } = workStateStore(); state.disconnectError = new Error("runner already disconnected"); From 66ab0091dfdefea9b8f9f480e7163161b6847c28 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:41:32 +0200 Subject: [PATCH 112/242] fix(desktop): negotiate registration ownership --- CHANGELOG.md | 2 +- docs/api.md | 17 ++-- .../CrabfleetDesktopRegistration.swift | 51 ++++++++--- .../PrivateMacShareController.swift | 2 +- .../PrivateMacShareTests.swift | 80 +++++++++++++++-- src/worker/desktop-host-service.ts | 14 ++- src/worker/routes/control-plane.ts | 7 ++ src/worker/worker-application.ts | 3 +- tests/control-plane-routes.test.ts | 56 +++++++++--- tests/desktop-host-service.test.ts | 90 ++++++++++++++----- 10 files changed, 260 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83c631a0..d9afb229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers during deployment configuration, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. -- Fence Share This Mac registry cleanup with per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping publishers cannot displace cleanup authority, while retaining owner-authenticated tokenless cleanup for migrated legacy registrations only. +- Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. diff --git a/docs/api.md b/docs/api.md index f6c17203..b60eae8c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1117,14 +1117,21 @@ The host is visible only to the same stable user, regardless of shared or private session-tenancy mode. Re-registering the same ID updates its name, address, port, and timestamp while preserving its creation time. +Clients opt into fenced registration by sending +`X-Crabfleet-Ownership-Mode: token-v1`. The response then includes an +`ownershipToken` required for deletion. Omitting the header preserves the +legacy `{ "host": ... }` response and stores a tokenless registration so older +clients can still clean up during rolling upgrades. Current clients tolerate a +legacy server response without `ownershipToken` and use tokenless cleanup for +that registration. + ### DELETE /api/desktop-hosts/:id Removes one registered desktop owned by the signed-in viewer. The route cannot -remove another user's record with the same ID. Registrations created by current -clients require the exact `X-Crabfleet-Ownership-Token` returned by `PUT`. -Older clients may omit the header only to remove a migrated legacy registration -whose stored ownership token is empty; omission never removes a tokenized -registration. +remove another user's record with the same ID. Fenced registrations require the +exact `X-Crabfleet-Ownership-Token` returned by `PUT`. Legacy clients may omit +the header only to remove a registration whose stored ownership token is empty; +omission never removes a tokenized registration. ## Static Routes diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index d087fafa..15c589c3 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -1,8 +1,8 @@ import Foundation protocol DesktopHostRegistering: Sendable { - func register(identity: TailnetIdentity, port: UInt16) async throws -> String - func unregister(identity: TailnetIdentity, ownershipToken: String) async throws + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws } actor DesktopHostRegistrationCoordinator { @@ -13,7 +13,7 @@ actor DesktopHostRegistrationCoordinator { self.registration = registration } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { let registration = self.registration let operation = enqueue { try await registration.register(identity: identity, port: port) @@ -21,7 +21,7 @@ actor DesktopHostRegistrationCoordinator { return try await operation.value } - func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { let registration = self.registration let operation = enqueue { try await registration.unregister(identity: identity, ownershipToken: ownershipToken) @@ -46,7 +46,26 @@ actor DesktopHostRegistrationCoordinator { struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable { private struct RegistrationResponse: Decodable { - let ownershipToken: String + private enum CodingKeys: String, CodingKey { + case host + case ownershipToken + } + + struct Host: Decodable { + let id: String + } + + let host: Host + let ownershipToken: String? + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + host = try container.decode(Host.self, forKey: .host) + ownershipToken = + container.contains(.ownershipToken) + ? try container.decode(String.self, forKey: .ownershipToken) + : nil + } } private struct RegistrationBody: Encodable { @@ -58,6 +77,8 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable private let baseURL: URL private let sessionCookie: String private let transport: any HTTPDataTransport + static let ownershipModeHeader = "X-Crabfleet-Ownership-Mode" + static let tokenOwnershipMode = "token-v1" init?( environment: [String: String] = ProcessInfo.processInfo.environment, @@ -84,20 +105,25 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable self.transport = transport } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { let request = try registrationRequest(identity: identity, port: port) let (data, http) = try await transport.data(for: request) try validate(response: http, for: request, acceptingNotFound: false) guard let response = try? JSONDecoder().decode(RegistrationResponse.self, from: data), - Self.isValidOwnershipToken(response.ownershipToken) + response.host.id == Self.hostID(identity: identity) else { throw DesktopHostRegistrationError.invalidResponse } + if let ownershipToken = response.ownershipToken, + !Self.isValidOwnershipToken(ownershipToken) + { + throw DesktopHostRegistrationError.invalidResponse + } return response.ownershipToken } - func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { let request = try removalRequest(identity: identity, ownershipToken: ownershipToken) let (_, http) = try await transport.data(for: request) try validate(response: http, for: request, acceptingNotFound: true) @@ -131,6 +157,7 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable request.setValue("application/json", forHTTPHeaderField: "Content-Type") request.setValue("application/json", forHTTPHeaderField: "Accept") request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") + request.setValue(Self.tokenOwnershipMode, forHTTPHeaderField: Self.ownershipModeHeader) request.httpBody = try JSONEncoder().encode( RegistrationBody( name: identity.hostName.isEmpty ? identity.dnsName : identity.hostName, @@ -140,8 +167,8 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable return request } - func removalRequest(identity: TailnetIdentity, ownershipToken: String) throws -> URLRequest { - guard Self.isValidOwnershipToken(ownershipToken) else { + func removalRequest(identity: TailnetIdentity, ownershipToken: String?) throws -> URLRequest { + if let ownershipToken, !Self.isValidOwnershipToken(ownershipToken) { throw DesktopHostRegistrationError.invalidResponse } let url = @@ -154,7 +181,9 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable request.timeoutInterval = 15 request.setValue("application/json", forHTTPHeaderField: "Accept") request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") - request.setValue(ownershipToken, forHTTPHeaderField: "X-Crabfleet-Ownership-Token") + if let ownershipToken { + request.setValue(ownershipToken, forHTTPHeaderField: "X-Crabfleet-Ownership-Token") + } return request } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 32ddb5db..08b493cc 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -35,7 +35,7 @@ final class PrivateMacShareStopCoordinator { final class DesktopHostRegistrationLifecycle { private struct PublishedRegistration: Equatable { let identity: TailnetIdentity - let ownershipToken: String + let ownershipToken: String? } private let coordinator: DesktopHostRegistrationCoordinator diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 5f185387..17845b20 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -617,6 +617,10 @@ struct PrivateMacShareTests { #expect(request.url?.absoluteString == "https://fleet.example/api/desktop-hosts/workstation-1") #expect(request.httpMethod == "PUT") #expect(request.value(forHTTPHeaderField: "Cookie") == "crabbox_session=secret") + #expect( + request.value(forHTTPHeaderField: CrabfleetDesktopRegistration.ownershipModeHeader) + == CrabfleetDesktopRegistration.tokenOwnershipMode + ) let body = try #require(request.httpBody) let json = try #require(JSONSerialization.jsonObject(with: body) as? [String: Any]) #expect(json["name"] as? String == "Workstation") @@ -642,7 +646,7 @@ struct PrivateMacShareTests { let transport = DesktopRegistrationTransport { request in let responseURL = try #require(request.url) return ( - Data(#"{"ownershipToken":"server-ownership-token"}"#.utf8), + Data(#"{"host":{"id":"workstation"},"ownershipToken":"server-ownership-token"}"#.utf8), try #require( HTTPURLResponse( url: responseURL, @@ -668,6 +672,66 @@ struct PrivateMacShareTests { ) } + @Test + func desktopRegistrationFallsBackToLegacyCleanupForOldServers() async throws { + let transport = DesktopRegistrationTransport { request in + let responseURL = try #require(request.url) + return ( + Data(#"{"host":{"id":"workstation"}}"#.utf8), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + #expect(try await registration.register(identity: identity, port: 5_901) == nil) + let removal = try registration.removalRequest(identity: identity, ownershipToken: nil) + #expect(removal.value(forHTTPHeaderField: "X-Crabfleet-Ownership-Token") == nil) + } + + @Test + func desktopRegistrationRejectsMalformedAdvertisedOwnershipTokens() async throws { + let transport = DesktopRegistrationTransport { request in + let responseURL = try #require(request.url) + return ( + Data(#"{"host":{"id":"workstation"},"ownershipToken":null}"#.utf8), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + await #expect(throws: DesktopHostRegistrationError.invalidResponse) { + try await registration.register(identity: identity, port: 5_901) + } + } + @Test func desktopRegistrationRejectsRedirectedResponses() async throws { let redirectedURL = try #require(URL(string: "https://login.example.test/desktop-host")) @@ -1312,7 +1376,7 @@ private actor SuspendedDesktopRegistration: DesktopHostRegistering { registrationContinuation != nil } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { events.append(.registerStarted) await withCheckedContinuation { continuation in registrationContinuation = continuation @@ -1321,7 +1385,7 @@ private actor SuspendedDesktopRegistration: DesktopHostRegistering { return "registration-token" } - func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { #expect(ownershipToken == "registration-token") events.append(.unregisterStarted) } @@ -1339,7 +1403,7 @@ private enum DesktopRegistrationTestError: Error { private actor RecordingDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String) - case unregister(String, String) + case unregister(String, String?) } private var registerFailures: [String: Int] @@ -1354,7 +1418,7 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { self.unregisterFailures = unregisterFailures } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { events.append(.register(identity.dnsName)) if consumeFailure(for: identity.dnsName, from: ®isterFailures) { throw DesktopRegistrationTestError.failed @@ -1362,7 +1426,7 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { return "token:\(identity.dnsName)" } - func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { events.append(.unregister(identity.dnsName, ownershipToken)) if consumeFailure(for: identity.dnsName, from: &unregisterFailures) { throw DesktopRegistrationTestError.failed @@ -1386,11 +1450,11 @@ private actor SuspendedDesktopCleanupRegistration: DesktopHostRegistering { unregistrationContinuation != nil } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String { + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { "slow-cleanup-token" } - func unregister(identity: TailnetIdentity, ownershipToken: String) async throws { + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { #expect(ownershipToken == "slow-cleanup-token") await withCheckedContinuation { continuation in unregistrationContinuation = continuation diff --git a/src/worker/desktop-host-service.ts b/src/worker/desktop-host-service.ts index b688aae4..c6a8e4fb 100644 --- a/src/worker/desktop-host-service.ts +++ b/src/worker/desktop-host-service.ts @@ -21,10 +21,13 @@ export type DesktopHost = { export type DesktopHostRegistration = { host: DesktopHost; - ownershipToken: string; + ownershipToken?: string; }; export const desktopHostOwnershipHeader = "x-crabfleet-ownership-token"; +export const desktopHostOwnershipModeHeader = "x-crabfleet-ownership-mode"; +export const desktopHostTokenOwnershipMode = "token-v1"; +export type DesktopHostOwnershipMode = "legacy" | typeof desktopHostTokenOwnershipMode; export class DesktopHostService { private readonly store: DesktopHostStore; @@ -50,13 +53,15 @@ export class DesktopHostService { user: User, rawID: string, input: DesktopHostInput, + ownershipMode: DesktopHostOwnershipMode = "legacy", ): Promise { const id = desktopHostID(rawID); const name = boundedText(input.name, "name", 100); const address = tailscaleIPv4(input.address); const port = desktopHostPort(input.port); const now = this.now(); - const ownershipToken = this.createOwnershipToken(); + const ownershipToken = + ownershipMode === desktopHostTokenOwnershipMode ? this.createOwnershipToken() : ""; const host: DesktopHostRow = { ownerSubject: tenantSubject(user), id, @@ -68,10 +73,11 @@ export class DesktopHostService { createdAt: now, updatedAt: now, }; - return { + const registration: DesktopHostRegistration = { host: presentDesktopHost(await this.store.upsert(host)), - ownershipToken, }; + if (ownershipToken) registration.ownershipToken = ownershipToken; + return registration; } async remove(user: User, rawID: string, rawOwnershipToken: unknown): Promise { diff --git a/src/worker/routes/control-plane.ts b/src/worker/routes/control-plane.ts index 5f409b65..5c97aa3c 100644 --- a/src/worker/routes/control-plane.ts +++ b/src/worker/routes/control-plane.ts @@ -7,7 +7,10 @@ import type { } from "../admin-service.ts"; import { desktopHostOwnershipHeader, + desktopHostOwnershipModeHeader, + desktopHostTokenOwnershipMode, type DesktopHostInput, + type DesktopHostOwnershipMode, type DesktopHostRegistration, } from "../desktop-host-service.ts"; import { json, notFound, readJson } from "../http.ts"; @@ -20,6 +23,7 @@ export type ControlPlaneRouteDependencies = { user: User, id: string, input: DesktopHostInput, + ownershipMode: DesktopHostOwnershipMode, ): Promise; removeDesktopHost(user: User, id: string, ownershipToken: string | null): Promise; searchGitHubRefs(number: unknown): Promise; @@ -56,6 +60,9 @@ export async function handleControlPlaneRoute( user, decoded(desktopHostMatch[1]), await readJson(request), + request.headers.get(desktopHostOwnershipModeHeader) === desktopHostTokenOwnershipMode + ? desktopHostTokenOwnershipMode + : "legacy", ); return json(registration); } diff --git a/src/worker/worker-application.ts b/src/worker/worker-application.ts index 8fc1b451..4ed453b2 100644 --- a/src/worker/worker-application.ts +++ b/src/worker/worker-application.ts @@ -162,7 +162,8 @@ export class WorkerApplication { return { readState: (request, user) => this.readState(request, user, context), readFleet: (user) => this.readFleetState(user, undefined, context), - registerDesktopHost: (user, id, input) => this.desktopHosts().register(user, id, input), + registerDesktopHost: (user, id, input, ownershipMode) => + this.desktopHosts().register(user, id, input, ownershipMode), removeDesktopHost: (user, id, ownershipToken) => this.desktopHosts().remove(user, id, ownershipToken), searchGitHubRefs: (number) => this.githubReferenceService().search(number), diff --git a/tests/control-plane-routes.test.ts b/tests/control-plane-routes.test.ts index 10f99c1b..2aa2390e 100644 --- a/tests/control-plane-routes.test.ts +++ b/tests/control-plane-routes.test.ts @@ -6,6 +6,10 @@ import { handleControlPlaneRoute, type ControlPlaneRouteDependencies, } from "../src/worker/routes/control-plane.ts"; +import { + desktopHostOwnershipModeHeader, + desktopHostTokenOwnershipMode, +} from "../src/worker/desktop-host-service.ts"; const viewer: User = { subject: "github:1", @@ -41,9 +45,9 @@ function dependencies(calls: string[]): ControlPlaneRouteDependencies { calls.push(`fleet:${user.login}`); return { handler: "fleet" }; }, - async registerDesktopHost(user, id, input) { - calls.push(`desktop-host:register:${user.login}:${id}:${input.name}`); - return { + async registerDesktopHost(user, id, input, ownershipMode) { + calls.push(`desktop-host:register:${user.login}:${id}:${input.name}:${ownershipMode}`); + const registration = { host: { id, owner: user.login ?? user.subject, @@ -53,8 +57,10 @@ function dependencies(calls: string[]): ControlPlaneRouteDependencies { createdAt: 1, updatedAt: 1, }, - ownershipToken: "ownership-token", }; + return ownershipMode === desktopHostTokenOwnershipMode + ? { ...registration, ownershipToken: "ownership-token" } + : registration; }, async removeDesktopHost(user, id, ownershipToken) { calls.push(`desktop-host:remove:${user.login}:${id}:${ownershipToken ?? "legacy"}`); @@ -154,10 +160,17 @@ test("control-plane read and card routes enforce their role boundaries", async ( test("desktop host routes register and remove only the authenticated user's host", async () => { const calls: string[] = []; const registered = await dispatch( - request("PUT", "/api/desktop-hosts/mac%2Dstudio", { - name: "Mac Studio", - address: "100.64.1.2", - port: 5901, + new Request("https://fleet.example/api/desktop-hosts/mac%2Dstudio", { + method: "PUT", + headers: { + "content-type": "application/json", + [desktopHostOwnershipModeHeader]: desktopHostTokenOwnershipMode, + }, + body: JSON.stringify({ + name: "Mac Studio", + address: "100.64.1.2", + port: 5901, + }), }), viewer, calls, @@ -186,18 +199,41 @@ test("desktop host routes register and remove only the authenticated user's host ); assert.equal(removed?.status, 200); assert.deepEqual(calls, [ - "desktop-host:register:viewer:mac-studio:Mac Studio", + "desktop-host:register:viewer:mac-studio:Mac Studio:token-v1", "desktop-host:remove:viewer:mac-studio:ownership-token", ]); const legacyCalls: string[] = []; + const legacyRegistered = await dispatch( + request("PUT", "/api/desktop-hosts/legacy%2Dstudio", { + name: "Legacy Studio", + address: "100.64.1.3", + port: 5901, + }), + viewer, + legacyCalls, + ); + assert.deepEqual(await legacyRegistered?.json(), { + host: { + id: "legacy-studio", + owner: "viewer", + name: "Legacy Studio", + address: "100.64.1.3", + port: 5901, + createdAt: 1, + updatedAt: 1, + }, + }); const legacyRemoved = await dispatch( request("DELETE", "/api/desktop-hosts/legacy%2Dstudio"), viewer, legacyCalls, ); assert.equal(legacyRemoved?.status, 200); - assert.deepEqual(legacyCalls, ["desktop-host:remove:viewer:legacy-studio:legacy"]); + assert.deepEqual(legacyCalls, [ + "desktop-host:register:viewer:legacy-studio:Legacy Studio:legacy", + "desktop-host:remove:viewer:legacy-studio:legacy", + ]); }); test("card actions derive viewer or maintainer authorization from the action", async () => { diff --git a/tests/desktop-host-service.test.ts b/tests/desktop-host-service.test.ts index f338dccb..18ca38d1 100644 --- a/tests/desktop-host-service.test.ts +++ b/tests/desktop-host-service.test.ts @@ -2,7 +2,10 @@ import assert from "node:assert/strict"; import test from "node:test"; import type { DesktopHostRow, DesktopHostStore } from "../src/worker/desktop-host-repository.ts"; -import { DesktopHostService } from "../src/worker/desktop-host-service.ts"; +import { + DesktopHostService, + desktopHostTokenOwnershipMode, +} from "../src/worker/desktop-host-service.ts"; import type { User } from "../src/worker/models.ts"; const alice: User = { @@ -53,11 +56,16 @@ test("desktop hosts are canonicalized and isolated to their stable owner", async () => now, () => tokens.shift() ?? "unexpected-token", ); - const registration = await service.register(alice, " Studio.ONE ", { - name: " Peter's Mac Studio ", - address: "100.68.201.40", - port: 5901, - }); + const registration = await service.register( + alice, + " Studio.ONE ", + { + name: " Peter's Mac Studio ", + address: "100.68.201.40", + port: 5901, + }, + desktopHostTokenOwnershipMode, + ); const host = registration.host; assert.deepEqual(host, { @@ -74,11 +82,16 @@ test("desktop hosts are canonicalized and isolated to their stable owner", async assert.deepEqual(await service.list(bob), []); now = 84; - const updatedRegistration = await service.register(alice, host.id, { - name: "Renamed Studio", - address: host.address, - port: host.port, - }); + const updatedRegistration = await service.register( + alice, + host.id, + { + name: "Renamed Studio", + address: host.address, + port: host.port, + }, + desktopHostTokenOwnershipMode, + ); const updated = updatedRegistration.host; assert.equal(updated.createdAt, 42); assert.equal(updated.updatedAt, 84); @@ -101,11 +114,21 @@ test("stale desktop host cleanup cannot remove a newer registration", async () = ); const input = { name: "Studio", address: "100.64.1.2", port: 5901 }; - const oldRegistration = await service.register(alice, "studio", input); - const newRegistration = await service.register(alice, "studio", { - ...input, - name: "New Studio Process", - }); + const oldRegistration = await service.register( + alice, + "studio", + input, + desktopHostTokenOwnershipMode, + ); + const newRegistration = await service.register( + alice, + "studio", + { + ...input, + name: "New Studio Process", + }, + desktopHostTokenOwnershipMode, + ); await service.remove(alice, "studio", oldRegistration.ownershipToken); assert.deepEqual(await service.list(alice), [newRegistration.host]); @@ -133,11 +156,16 @@ test("tokenless cleanup removes only migrated legacy desktop hosts", async () => updatedAt: 1, }; store.rows.set(`${alice.subject}:${legacy.id}`, legacy); - const registration = await service.register(alice, "new-studio", { - name: "New Studio", - address: "100.64.1.3", - port: 5901, - }); + const registration = await service.register( + alice, + "new-studio", + { + name: "New Studio", + address: "100.64.1.3", + port: 5901, + }, + desktopHostTokenOwnershipMode, + ); await service.remove(alice, legacy.id, null); await service.remove(alice, registration.host.id, null); @@ -145,6 +173,26 @@ test("tokenless cleanup removes only migrated legacy desktop hosts", async () => assert.deepEqual(await service.list(alice), [registration.host]); }); +test("legacy clients register tokenless rows they can remove after a server upgrade", async () => { + const store = new MemoryDesktopHostStore(); + const service = new DesktopHostService( + store, + () => 42, + () => "must-not-be-created", + ); + + const registration = await service.register(alice, "rolling-upgrade", { + name: "Rolling Upgrade", + address: "100.64.1.4", + port: 5901, + }); + + assert.equal(registration.ownershipToken, undefined); + assert.equal(store.rows.get(`${alice.subject}:rolling-upgrade`)?.ownershipToken, ""); + await service.remove(alice, registration.host.id, null); + assert.deepEqual(await service.list(alice), []); +}); + test("desktop hosts accept only bounded metadata and Tailscale IPv4 endpoints", async () => { const service = new DesktopHostService(new MemoryDesktopHostStore()); const valid = { name: "Studio", address: "100.127.255.254", port: 65_535 }; From 9660fe798a13dadfdbcc1d2f362506d4ab206d53 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:41:46 +0200 Subject: [PATCH 113/242] fix(credentials): fence legacy registration claims --- .../sandbox-credential-policy-repository.ts | 19 +++ ...ndbox-credential-policy-repository.test.ts | 136 +++++++++++++++--- 2 files changed, 138 insertions(+), 17 deletions(-) diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 57e372d4..4ae96723 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -432,6 +432,22 @@ export function sandboxCredentialPolicyOwnerCondition( )`; } +function noLivePolicyTableRegistrationCondition( + sessionId: string, + sandboxId: string, + now: number, +): RawBuilder { + return sql`NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'registering' + AND registration_claim IS NOT NULL + AND registration_claim_expires_at > ${now} + )`; +} + export function sandboxCredentialPolicyRegistrationQueries( sessionId: string, sandboxId: string, @@ -472,6 +488,7 @@ export function sandboxCredentialPolicyRegistrationQueries( ${now}, ${now} WHERE ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + AND ${noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)} AND NOT EXISTS ( SELECT 1 FROM interactive_session_credential_policies @@ -495,6 +512,7 @@ export function sandboxCredentialPolicyRegistrationQueries( OR interactive_session_credential_policy_registrations.registration_claim_expires_at <= ${now} ) AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + AND ${noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)} AND NOT EXISTS ( SELECT 1 FROM interactive_session_credential_policies @@ -761,6 +779,7 @@ export function sandboxCredentialPolicyPromotionQueries( AND registration_generation = ${registration.generation} AND registration_claim = ${registration.claim} ) + AND ${noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)} AND NOT EXISTS ( SELECT 1 FROM interactive_session_credential_policies diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 394265db..4f9e1da3 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import test from "node:test"; @@ -119,23 +120,6 @@ function credentialPolicyDatabase(): DatabaseSync { updated_at INTEGER NOT NULL, PRIMARY KEY (session_id, sandbox_id, lookup_id) ); - CREATE TABLE interactive_session_credential_policy_registrations ( - session_id TEXT NOT NULL, - sandbox_id TEXT NOT NULL, - state TEXT NOT NULL, - registration_generation TEXT NOT NULL, - registration_claim TEXT, - registration_claim_expires_at INTEGER, - attempt_count INTEGER NOT NULL DEFAULT 0, - last_attempt_at INTEGER, - last_error TEXT, - cleanup_claim TEXT, - cleanup_claim_expires_at INTEGER, - rollback_policies_json TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (session_id, sandbox_id) - ); INSERT INTO interactive_sessions ( id, adapter, @@ -165,6 +149,18 @@ function credentialPolicyDatabase(): DatabaseSync { ('IS-42', 'sandbox-1', 'sandbox-1', 'active', 'generation:existing', NULL, NULL, 1, 1), ('IS-42', 'sandbox-1', 'do-1', 'active', 'generation:existing', NULL, NULL, 1, 1); `); + db.exec( + readFileSync( + new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), + "utf8", + ), + ); + db.exec( + readFileSync( + new URL("../migrations/0035_credential_policy_registration_rollback.sql", import.meta.url), + "utf8", + ), + ); return db; } @@ -311,6 +307,8 @@ test("credential-policy registration SQL proves every supported ownership fence" assert.match(current.sql, /agent_token_hash is not null/i); assert.match(current.sql, /lease_id =/i); assert.match(current.sql, /sandbox_refresh_claim is null/i); + assert.match(current.sql, /state = 'registering'/i); + assert.match(current.sql, /registration_claim_expires_at >/i); assert.ok(current.parameters.includes("sandbox:sandbox-1:terminal-1:autostart-v4")); assert.doesNotMatch(current.sql, /1 = 1/); @@ -398,6 +396,110 @@ test("credential-policy rotation always claims a fresh generation", async () => assert.equal(rotated.generation, generation); }); +test("post-migration legacy registration claims block new staged generations", async () => { + const sqlite = credentialPolicyDatabase(); + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-worker', + registration_claim = 'legacy-registration', + registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + + await assert.rejects( + beginSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite), + "IS-42", + "sandbox-1", + ownershipFence, + ), + { message: "sandbox credential policy registration is unavailable" }, + ); + + assert.equal( + sqlite + .prepare("SELECT count(*) AS count FROM interactive_session_credential_policy_registrations") + .get()?.count, + 0, + ); + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => ({ + generation: row.registration_generation, + state: row.state, + claim: row.registration_claim, + })), + [ + { + generation: "generation:legacy-worker", + state: "registering", + claim: "legacy-registration", + }, + { + generation: "generation:legacy-worker", + state: "registering", + claim: "legacy-registration", + }, + ], + ); +}); + +test("legacy claims created after staging block promotion without interleaving generations", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-race', + registration_claim = 'legacy-race-claim', + registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + + assert.equal( + await finishSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + false, + ); + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => ({ + generation: row.registration_generation, + state: row.state, + })), + [ + { generation: "generation:legacy-race", state: "registering" }, + { generation: "generation:legacy-race", state: "registering" }, + ], + ); + assert.equal( + sqlite + .prepare(` + SELECT registration_generation + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get()?.registration_generation, + staged.generation, + ); +}); + test("partial credential-policy rotation failure preserves the prior active generation", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); From 7a441e4daaa073d68c625f39e9b811bb1a869b3e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:42:53 +0200 Subject: [PATCH 114/242] docs(changelog): record rollout safety fixes --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9afb229..64543ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, retain and acknowledge initial output only after an attachment owns it, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output, serialize every acknowledgement-aware input source, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. -- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable profile-routed identifiers during deployment configuration, malformed encoded session routes, and lossy or invalid-Unicode JSON event values; also reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes and lossy or invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. +- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races, concurrent teardown calls that could outpace application termination, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. From 2d64d0c4832dab34dc32d77cdaf8c2ad9075ba08 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:51:54 +0200 Subject: [PATCH 115/242] fix(actions): keep authenticated revisions monotonic --- src/worker/github-actions-runner-connection.ts | 3 ++- src/worker/github-actions-session-work-state.ts | 3 ++- tests/github-actions-runner-connection.test.ts | 9 +++++++++ tests/github-actions-session-work-state.test.ts | 12 ++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/worker/github-actions-runner-connection.ts b/src/worker/github-actions-runner-connection.ts index 3722c0ae..5eb37e40 100644 --- a/src/worker/github-actions-runner-connection.ts +++ b/src/worker/github-actions-runner-connection.ts @@ -37,6 +37,7 @@ export class GitHubActionsRunnerConnectionService { throw badRequest("session is not a GitHub Actions work session"); } const now = this.store.now(); + const revision = Math.max(session.updatedAt + 1, now); const state = session.workState === "registered" || !session.workState ? "running" : session.workState; const phase = @@ -53,7 +54,7 @@ export class GitHubActionsRunnerConnectionService { work_phase: phase, last_heartbeat_at: now, last_seen_at: now, - updated_at: now, + updated_at: revision, last_event: githubActionsRunnerConnectedEvent, }, session.updatedAt, diff --git a/src/worker/github-actions-session-work-state.ts b/src/worker/github-actions-session-work-state.ts index 7c1caf7a..a02e89e1 100644 --- a/src/worker/github-actions-session-work-state.ts +++ b/src/worker/github-actions-session-work-state.ts @@ -95,6 +95,7 @@ export class GitHubActionsWorkStateService { row.codex_turn_id !== codexTurnId || row.completion_reason !== completionReason; const now = this.store.now(); + const revision = Math.max(session.updatedAt + 1, now); await this.store.persist( session.id, @@ -109,7 +110,7 @@ export class GitHubActionsWorkStateService { completion_reason: completionReason, last_event: lastEvent, last_seen_at: now, - updated_at: now, + updated_at: revision, stopped_at: terminal ? now : null, }, session.updatedAt, diff --git a/tests/github-actions-runner-connection.test.ts b/tests/github-actions-runner-connection.test.ts index a669c0ab..0a916335 100644 --- a/tests/github-actions-runner-connection.test.ts +++ b/tests/github-actions-runner-connection.test.ts @@ -100,6 +100,15 @@ test("runner connections retain the exact revision authenticated before a token assert.deepEqual(expectedRevisions, [400]); }); +test("runner connections advance revisions when the authenticated clock is ahead", async () => { + const { store, updates, expectedRevisions } = connectionStore(); + + await new GitHubActionsRunnerConnectionService(store).connect(session({ updated_at: 800 })); + + assert.deepEqual(expectedRevisions, [800]); + assert.equal(updates[0]?.updated_at, 801); +}); + test("runner connections reject non-work sessions", async () => { const { store } = connectionStore(); const service = new GitHubActionsRunnerConnectionService(store); diff --git a/tests/github-actions-session-work-state.test.ts b/tests/github-actions-session-work-state.test.ts index 19b5cbab..4310d639 100644 --- a/tests/github-actions-session-work-state.test.ts +++ b/tests/github-actions-session-work-state.test.ts @@ -185,6 +185,18 @@ test("work-state updates retain the exact revision authenticated before a token assert.equal(state.expectedRevision, 400); }); +test("work-state updates advance revisions when the authenticated clock is ahead", async () => { + const authenticated = workSession({ updated_at: 800 }); + const { store, state } = workStateStore({ updated_at: 800 }); + + await new GitHubActionsWorkStateService(store).update(authenticated, { + state: "running", + }); + + assert.equal(state.expectedRevision, 800); + assert.equal(state.update?.updated_at, 801); +}); + test("terminal runner disconnect races remain best effort", async () => { const { store, state } = workStateStore(); state.disconnectError = new Error("runner already disconnected"); From a30a750486a629927a3306742519ec7685ff74c7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:52:33 +0200 Subject: [PATCH 116/242] fix(terminal): unblock confirmed message delivery --- internal/terminalws/client.go | 26 +++-- internal/terminalws/client_test.go | 162 +++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 7 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 3da77955..ee9bfa1b 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -427,7 +427,11 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c } }() - err = <-errCh + select { + case err = <-errCh: + case <-ctx.Done(): + err = ctx.Err() + } cancelRead() if cancelableRead { wg.Wait() @@ -480,7 +484,7 @@ func (c *Client) handleFrame(ctx context.Context, current frame) error { } switch current.messageType { case messageOutput: - c.deliverOrQueueOutput(ctx, current) + return c.deliverOrQueueOutput(ctx, current) case messageError: err := frameError(current, "terminal connection failed") c.canInput.Store(false) @@ -584,29 +588,37 @@ func (c *Client) clearAttachment(attachment *terminalAttachment) { c.stateMu.Unlock() } -func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) { +func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) error { for { c.stateMu.Lock() attachment := c.attachment ready := c.attachmentReady + discardOutput := c.inputWaiter != nil c.stateMu.Unlock() if attachment == nil { + if discardOutput { + return c.write(ctx, frame{ + messageType: messageAck, + sessionID: c.sessionID, + payload: ackPayload(uint32(len(current.payload))), + }) + } if ready == nil { - return + return nil } select { case <-ready: continue case <-ctx.Done(): - return + return ctx.Err() } } select { case attachment.frames <- current: - return + return nil case <-attachment.done: case <-ctx.Done(): - return + return ctx.Err() } } } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 2d3f0a97..b9752436 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -417,6 +417,90 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { } } +func TestSendInputConfirmedAcknowledgesOutputWithoutAttachment(t *testing.T) { + acknowledged := make(chan uint32, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-message", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-message", + payload: []byte("prompt\n"), + })); err != nil { + t.Error(err) + return + } + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + ack, err := decodeFrame(payload) + if err != nil || ack.messageType != messageAck { + t.Errorf("output acknowledgement = %#v, %v", ack, err) + return + } + acknowledged <- binary.LittleEndian.Uint32(ack.payload) + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-message", + payload: accepted, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-message", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := client.SendInputConfirmed(ctx, []byte("echo ready\n")); err != nil { + t.Fatal(err) + } + if bytes := <-acknowledged; bytes != uint32(len("prompt\n")) { + t.Fatalf("acknowledged = %d", bytes) + } +} + func TestSendInputConfirmedRejectionDoesNotRevokeControl(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) @@ -914,6 +998,62 @@ func TestAttachClosesCloseableTerminalAfterRemoteClosure(t *testing.T) { } } +func TestAttachReturnsWhenContextCancelsAnUncancelableRead(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-cancel", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + <-r.Context().Done() + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-cancel", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + terminal := newUncancelableTerminal() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- client.Attach(ctx, terminal, nil) + }() + <-terminal.started + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Attach did not return after context cancellation") + } + close(terminal.release) +} + func TestClientSubscribesReadOnlyAndSuppressesInput(t *testing.T) { acknowledged := make(chan uint32, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1194,6 +1334,28 @@ type blockingTerminal struct { bytes.Buffer } +type uncancelableTerminal struct { + started chan struct{} + startOnce sync.Once + release chan struct{} + bytes.Buffer +} + +func newUncancelableTerminal() *uncancelableTerminal { + return &uncancelableTerminal{ + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +func (terminal *uncancelableTerminal) Read(_ []byte) (int, error) { + terminal.startOnce.Do(func() { + close(terminal.started) + }) + <-terminal.release + return 0, io.EOF +} + func newBlockingTerminal() *blockingTerminal { return &blockingTerminal{ closed: make(chan struct{}), From 27058ad534e11d43b7fd53452ad6267670d0edd8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:53:14 +0200 Subject: [PATCH 117/242] fix(terminal): serialize client input completions --- src/worker/terminal-hub.ts | 138 ++++++++++++++++++++----------------- tests/terminal-hub.test.ts | 65 +++++++++++++++++ 2 files changed, 138 insertions(+), 65 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 22353a14..0c01652e 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -56,6 +56,7 @@ export type TerminalHubSubscription = { cols: number; rows: number; inputAcknowledgements: boolean; + inputQueue: Promise; pendingInputAcknowledgements: Map; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; @@ -234,62 +235,69 @@ export class TerminalHub { return; } if (frame.type === TerminalMessageType.Input || frame.type === TerminalMessageType.Key) { - const canInput = await subscription.canInput(); - updateTerminalInputCapability(server, subscription, canInput); - if (!canInput) { - return; - } - if (subscription.upstream.readyState !== WebSocket.OPEN) { - sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: "terminal upstream is not open", - }); - return; - } - const inputs = await this.dependencies.inputPayloads(subscription, user, frame.payload); - const acknowledgements: PendingTerminalInputAcknowledgement[] = []; - for (const [index, input] of inputs.entries()) { - if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); - if ( - subscriptions.get(frame.sessionId) !== subscription || - subscription.upstream.readyState !== WebSocket.OPEN - ) { - sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: "terminal upstream is not open", - }); - return; - } - const inputId = subscription.inputAcknowledgements - ? createGitHubActionsRelayInputId() - : null; - const acknowledgement = inputId - ? beginTerminalInputAcknowledgement(subscription, inputId) - : null; - if (acknowledgement) acknowledgements.push(acknowledgement); - try { - subscription.upstream.send( - inputId ? encodeGitHubActionsRelayInput(inputId, input) : input, - ); - } catch { - if (acknowledgement) { - completeTerminalInputAcknowledgement(subscription, acknowledgement.inputId, { - inputId: acknowledgement.inputId, - accepted: false, - error: "terminal upstream send failed", - }); - break; - } else { + subscription.inputQueue = subscription.inputQueue + .catch(() => undefined) + .then(async () => { + const canInput = await subscription.canInput(); + updateTerminalInputCapability(server, subscription, canInput); + if (!canInput) { + return; + } + if (subscription.upstream.readyState !== WebSocket.OPEN) { sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: "terminal upstream send failed", + error: "terminal upstream is not open", }); return; } - } - } - reportTerminalInputCompletion( - server, - frame.sessionId, - acknowledgements.map((acknowledgement) => acknowledgement.promise), - ); + const inputs = await this.dependencies.inputPayloads( + subscription, + user, + frame.payload, + ); + const acknowledgements: PendingTerminalInputAcknowledgement[] = []; + for (const [index, input] of inputs.entries()) { + if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); + if ( + subscriptions.get(frame.sessionId) !== subscription || + subscription.upstream.readyState !== WebSocket.OPEN + ) { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: "terminal upstream is not open", + }); + return; + } + const inputId = subscription.inputAcknowledgements + ? createGitHubActionsRelayInputId() + : null; + const acknowledgement = inputId + ? beginTerminalInputAcknowledgement(subscription, inputId) + : null; + if (acknowledgement) acknowledgements.push(acknowledgement); + try { + subscription.upstream.send( + inputId ? encodeGitHubActionsRelayInput(inputId, input) : input, + ); + } catch { + if (acknowledgement) { + completeTerminalInputAcknowledgement(subscription, acknowledgement.inputId, { + inputId: acknowledgement.inputId, + accepted: false, + error: "terminal upstream send failed", + }); + break; + } + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: "terminal upstream send failed", + }); + return; + } + } + await reportTerminalInputCompletion( + server, + frame.sessionId, + acknowledgements.map((acknowledgement) => acknowledgement.promise), + ); + }); return; } if (frame.type === TerminalMessageType.Resize) { @@ -468,6 +476,7 @@ export class TerminalHub { rows, inputAcknowledgements: upstreamConnection.inputAcknowledgements ?? session.runtime === githubActionsRuntime, + inputQueue: Promise.resolve(), pendingInputAcknowledgements: new Map(), outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, @@ -680,30 +689,29 @@ function completeAllTerminalInputAcknowledgements( return pending.length; } -function reportTerminalInputCompletion( +async function reportTerminalInputCompletion( socket: WebSocket, sessionId: string, acknowledgements: Promise[], -): void { +): Promise { if (acknowledgements.length === 0) { sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { type: "input-accepted", }); return; } - void Promise.all(acknowledgements).then((results) => { - if (socket.readyState !== WebSocket.OPEN) return; - const rejection = results.find((result) => !result.accepted); - if (rejection) { - sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { - type: "input-rejected", - error: rejection.error ?? "terminal input was not accepted", - }); - return; - } + const results = await Promise.all(acknowledgements); + if (socket.readyState !== WebSocket.OPEN) return; + const rejection = results.find((result) => !result.accepted); + if (rejection) { sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { - type: "input-accepted", + type: "input-rejected", + error: rejection.error ?? "terminal input was not accepted", }); + return; + } + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-accepted", }); } diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index e1345443..2422d756 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -804,6 +804,71 @@ test("GitHub Actions input acknowledgements correlate overlapping payloads out o server.emit("close"); }); +test("GitHub Actions serializes completion events for overlapping client inputs", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + for (const text of ["first", "second"]) { + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode(text), + }), + }); + } + await flushQueues(); + await flushQueues(); + + assert.equal(upstream.sent.length, 1); + const first = relayInput(upstream.sent[0]!); + assert.equal(first.text, "first"); + emitRelayAcknowledgement(upstream, first.inputId, false); + await flushQueues(); + await flushQueues(); + + assert.equal(upstream.sent.length, 2); + const second = relayInput(upstream.sent[1]!); + assert.equal(second.text, "second"); + emitRelayAcknowledgement(upstream, second.inputId, true); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.deepEqual( + completions.map((message) => message.type), + ["input-rejected", "input-accepted"], + ); + server.emit("close"); +}); + test("GitHub Actions framed output preserves control-shaped terminal bytes", async () => { const client = socket(); const server = socket(); From 867647e68590a309500b4a5c14a6de731828d4a2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:53:35 +0200 Subject: [PATCH 118/242] fix(http): reject rounded integer literals --- src/worker/http.ts | 75 ++++++++++++++++++++++++++++++++++++++++++++-- tests/http.test.ts | 19 +++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/worker/http.ts b/src/worker/http.ts index 26463928..c35bc98a 100644 --- a/src/worker/http.ts +++ b/src/worker/http.ts @@ -54,12 +54,14 @@ export function wantsMarkdown(request: Request): boolean { } export async function readJson(request: Request): Promise { + const source = await request.text(); let parsed: unknown; try { - parsed = JSON.parse(await request.text()) as unknown; + parsed = JSON.parse(source) as unknown; } catch { throw badRequest("invalid json"); } + assertRoundTrippableJsonIntegerLexemes(source); assertRoundTrippableJsonIntegers(parsed); return parsed as T; } @@ -100,12 +102,14 @@ export async function readBoundedJson(request: Request, maximumBytes: number) bytes.set(chunk, offset); offset += chunk.byteLength; } + const source = new TextDecoder().decode(bytes); let parsed: unknown; try { - parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; + parsed = JSON.parse(source) as unknown; } catch { throw badRequest("invalid json"); } + assertRoundTrippableJsonIntegerLexemes(source); assertRoundTrippableJsonIntegers(parsed); return parsed as T; } @@ -198,3 +202,70 @@ function assertRoundTrippableJsonIntegers(value: unknown): void { for (const item of Object.values(current)) pending.push(item); } } + +function assertRoundTrippableJsonIntegerLexemes(source: string): void { + const numberPattern = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y; + let inString = false; + let escaped = false; + for (let index = 0; index < source.length; index += 1) { + const character = source[index]!; + if (inString) { + if (escaped) { + escaped = false; + } else if (character === "\\") { + escaped = true; + } else if (character === '"') { + inString = false; + } + continue; + } + if (character === '"') { + inString = true; + continue; + } + if (character !== "-" && (character < "0" || character > "9")) continue; + numberPattern.lastIndex = index; + const match = numberPattern.exec(source); + if (!match) continue; + const token = match[0]; + const value = Number(token); + if ( + Number.isInteger(value) && + (!Number.isSafeInteger(value) || + Object.is(value, -0) || + exactJsonInteger(token) !== String(value)) + ) { + throw badRequest("json integers must be safe and round-trippable"); + } + index = numberPattern.lastIndex - 1; + } +} + +function exactJsonInteger(token: string): string | null { + const negative = token.startsWith("-"); + const unsigned = negative ? token.slice(1) : token; + const exponentIndex = unsigned.search(/[eE]/u); + const mantissa = exponentIndex === -1 ? unsigned : unsigned.slice(0, exponentIndex); + const exponentText = exponentIndex === -1 ? "" : unsigned.slice(exponentIndex + 1); + const decimalIndex = mantissa.indexOf("."); + const integerDigits = decimalIndex === -1 ? mantissa.length : decimalIndex; + const digits = + decimalIndex === -1 + ? mantissa + : mantissa.slice(0, decimalIndex) + mantissa.slice(decimalIndex + 1); + if (/^0+$/u.test(digits)) return negative ? "-0" : "0"; + + const exponent = exponentText ? Number(exponentText) : 0; + if (!Number.isSafeInteger(exponent)) return null; + const decimalPosition = integerDigits + exponent; + if (decimalPosition <= 0) return null; + if (decimalPosition < digits.length && !/^0+$/u.test(digits.slice(decimalPosition))) { + return null; + } + const exactDigits = + decimalPosition >= digits.length + ? digits + "0".repeat(decimalPosition - digits.length) + : digits.slice(0, decimalPosition); + const canonicalDigits = exactDigits.replace(/^0+/u, "") || "0"; + return negative ? `-${canonicalDigits}` : canonicalDigits; +} diff --git a/tests/http.test.ts b/tests/http.test.ts index 4b480d50..bdf39c84 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -114,7 +114,13 @@ test("bounded JSON parsing rejects declared and streamed bodies before unbounded }); test("JSON parsing rejects integers that cannot round-trip exactly", async () => { - for (const body of ['{"value":9007199254740993}', '{"value":-0}', '{"nested":[1e400]}']) { + for (const body of [ + '{"value":9007199254740993}', + '{"value":9007199254740991.1}', + '{"value":1.0000000000000001}', + '{"value":-0}', + '{"nested":[1e400]}', + ]) { for (const parse of [ () => readJson(new Request("https://fleet.example", { method: "POST", body })), () => @@ -135,6 +141,17 @@ test("JSON parsing rejects integers that cannot round-trip exactly", async () => } }); +test("JSON parsing accepts exact integer-equivalent numeric forms", async () => { + for (const body of ['{"value":1.0}', '{"value":1e0}', '{"value":100e-2}']) { + assert.deepEqual( + await readJson<{ value: number }>( + new Request("https://fleet.example", { method: "POST", body }), + ), + { value: 1 }, + ); + } +}); + test("JSON parsing handles deeply nested bounded payloads without exhausting the call stack", async () => { const depth = 20_000; const body = `${"[".repeat(depth)}0${"]".repeat(depth)}`; From b19dde976bbf7a1c164baaf5dd8d97c74b77bfc1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:53:54 +0200 Subject: [PATCH 119/242] docs(actions): close runner socket on PTY exit --- README.md | 3 +++ docs/github-actions-sessions.md | 4 ++++ tests/github-actions-docs.test.ts | 2 ++ 3 files changed, 9 insertions(+) diff --git a/README.md b/README.md index 3ebc0ec2..c2fad4bf 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,9 @@ terminal.binaryType = "arraybuffer"; const inputDecoder = new TextDecoder("utf-8", { fatal: true }); pty.onData((output) => terminal.send(encodeCfr1Output(output))); +pty.onExit(() => { + if (terminal.readyState < WebSocket.CLOSING) terminal.close(1000, "pty exited"); +}); terminal.onmessage = ({ data }) => { const input = decodeCfr1Input(data); if (!input) return; diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 805c9fc6..6d0b8bb5 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -308,6 +308,10 @@ pty.onData((output) => { terminal.send(encodeOutput(output)); }); +pty.onExit(() => { + if (terminal.readyState < WebSocket.CLOSING) terminal.close(1000, "pty exited"); +}); + terminal.addEventListener("message", (event) => { acceptInput(event.data); }); diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 119aa72b..64640591 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -11,6 +11,8 @@ test("the documented Node runner rejects split UTF-8 input frames", async () => for (const documentation of [readme, guide]) { assert.match(documentation, /new TextDecoder\("utf-8", \{ fatal: true \}\)/); assert.match(documentation, /inputDecoder\.decode\(input\.payload\)/); + assert.match(documentation, /pty\.onExit\(\(\) => \{/); + assert.match(documentation, /terminal\.close\(1000, "pty exited"\)/); } const decoder = new TextDecoder("utf-8", { fatal: true }); From 9131cedea5d72e96b129f851c5edd42366340c6a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:54:34 +0200 Subject: [PATCH 120/242] fix(macos): retry cleanup and unwind canceled starts --- .../PrivateMacShareController.swift | 28 ++++++-- .../PrivateMacShareTests.swift | 64 +++++++++++++++++++ 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 08b493cc..14b2d4ff 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -309,13 +309,13 @@ final class PrivateMacShareController: ObservableObject { streamStats = nil registryPhase = desktopRegistration == nil ? .notConfigured : .registering await waitForRefreshCompletion() - guard isCurrent(generation), phase == .starting, !Task.isCancelled else { return } + guard canContinueStarting(generation) else { return } do { let loadedIdentity = try await fetchIdentity() - guard isCurrent(generation), phase == .starting else { return } + guard canContinueStarting(generation) else { return } identity = loadedIdentity } catch { - guard isCurrent(generation), phase == .starting else { return } + guard canContinueStarting(generation) else { return } identity = nil phase = .failed notice = error.localizedDescription @@ -345,7 +345,7 @@ final class PrivateMacShareController: ObservableObject { let capture = MacScreenCapture() do { let descriptor = try await capture.start(displayID: selectedDisplayID) - guard isCurrent(generation), phase == .starting else { + guard canContinueStarting(generation) else { await capture.stop() return } @@ -371,7 +371,7 @@ final class PrivateMacShareController: ObservableObject { self.clipboardBridge = bridge activeIdentity = identity } catch { - guard isCurrent(generation) else { + guard canContinueStarting(generation) else { await capture.stop() return } @@ -414,6 +414,14 @@ final class PrivateMacShareController: ObservableObject { await stop() let cleanupTask = registrationTask await cleanupTask?.value + guard let desktopRegistrationLifecycle else { return } + do { + try await desktopRegistrationLifecycle.removePublishedIdentities() + registryPhase = .notPublished + } catch { + registryPhase = .failed(error.localizedDescription) + notice = error.localizedDescription + } } func openPrivacySettings(_ pane: PrivacyPane) { @@ -595,6 +603,16 @@ final class PrivateMacShareController: ObservableObject { lifecycleGeneration == generation } + private func canContinueStarting(_ generation: UInt64) -> Bool { + guard isCurrent(generation), phase == .starting else { return false } + guard !Task.isCancelled else { + phase = .idle + registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished + return false + } + return true + } + @discardableResult private func beginRegistryOperation() -> UInt64 { registryOperationGeneration &+= 1 diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 17845b20..ab406fe4 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -272,6 +272,35 @@ struct PrivateMacShareTests { #expect(controller.phase == .idle) } + @Test @MainActor + func cancellationRestoresIdleWhileStartWaitsForRefresh() async throws { + let runner = SequencedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: nil, + defaults: defaults + ) + + let refreshTask = Task { await controller.refresh() } + #expect(await waitUntilAsync { await runner.callCount == 1 }) + + let startTask = Task { await controller.start() } + #expect(await waitUntilAsync { controller.phase == .starting }) + startTask.cancel() + + await runner.resumeNext( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + await refreshTask.value + await startTask.value + + #expect(await runner.callCount == 1) + #expect(controller.phase == .idle) + } + @Test func desktopRemovalWaitsForACommittedRegistrationAfterCancellation() async throws { let registration = SuspendedDesktopRegistration() @@ -420,6 +449,41 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func terminationRetriesRetainedDesktopCleanup() async throws { + let identity = desktopIdentity(name: "retry-cleanup", address: "100.64.12.45") + let registration = RecordingDesktopRegistration( + unregisterFailures: [identity.dnsName: 1] + ) + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + try await lifecycle.publish(identity: identity, port: 5_901) + await #expect(throws: DesktopRegistrationTestError.failed) { + try await lifecycle.removePublishedIdentities() + } + + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: StaticTailscaleRunner(output: statusJSON()), + desktopRegistration: registration, + registrationLifecycle: lifecycle, + defaults: defaults + ) + + await controller.stopAndWaitForCleanup() + + #expect(controller.registryPhase == .notPublished) + #expect( + await registration.events + == [ + .register(identity.dnsName), + .unregister(identity.dnsName, "token:\(identity.dnsName)"), + .unregister(identity.dnsName, "token:\(identity.dnsName)"), + ] + ) + } + @Test @MainActor func applicationDelegateOwnsTheShareControllerUsedByTheApp() throws { let defaults = try #require( From 5b74e414c0d1e3eba1c5e43dc888f92465611747 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:54:53 +0200 Subject: [PATCH 121/242] docs(changelog): record final audit repairs --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64543ba8..e82976a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output, serialize every acknowledgement-aware input source, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. -- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. -- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes and lossy or invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races, concurrent teardown calls that could outpace application termination, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while discarding and acknowledging output for one-shot confirmed messages, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, concurrent teardown calls that could outpace application termination, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. From 170547004ed20efa33813818546e40399f945f1a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:07:18 +0200 Subject: [PATCH 122/242] fix(terminal): wake confirmed input delivery --- internal/terminalws/client.go | 6 +- internal/terminalws/client_test.go | 148 +++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index ee9bfa1b..8cfda7fd 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -293,7 +293,7 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { return err case <-c.readerDone: c.clearInputWaiter(waiter) - return c.readerError() + return readerUnavailableError(c.readerError()) case <-ctx.Done(): c.clearInputWaiter(waiter) _ = c.Close() @@ -530,6 +530,10 @@ func (c *Client) registerInputWaiter(waiter chan error) error { default: } c.inputWaiter = waiter + if c.attachment == nil && c.attachmentReady != nil { + close(c.attachmentReady) + c.attachmentReady = make(chan struct{}) + } return nil } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index b9752436..0f7f23aa 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -501,6 +501,154 @@ func TestSendInputConfirmedAcknowledgesOutputWithoutAttachment(t *testing.T) { } } +func TestSendInputConfirmedWakesOutputWaitingForAttachment(t *testing.T) { + outputSent := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-output-first", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-output-first", + payload: []byte("prompt\n"), + })); err != nil { + t.Error(err) + return + } + close(outputSent) + + seenInput := false + seenAcknowledgement := false + for !seenInput || !seenAcknowledgement { + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + current, err := decodeFrame(payload) + if err != nil { + t.Error(err) + return + } + switch current.messageType { + case messageInput: + seenInput = true + case messageAck: + seenAcknowledgement = true + default: + t.Errorf("message type = %d", current.messageType) + return + } + } + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-output-first", + payload: accepted, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-output-first", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + <-outputSent + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := client.SendInputConfirmed(ctx, []byte("echo ready\n")); err != nil { + t.Fatal(err) + } +} + +func TestSendInputConfirmedFailsWhenConnectionClosesBeforeAcknowledgement(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-close-before-ack", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + _ = conn.Close(websocket.StatusNormalClosure, "") + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-close-before-ack", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err = client.SendInputConfirmed(ctx, []byte("echo ready\n")) + if err == nil { + t.Fatal("normal close before acknowledgement reported success") + } +} + func TestSendInputConfirmedRejectionDoesNotRevokeControl(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) From d522fbd561f51a148bae207ef5aa9afe7251a543 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:07:18 +0200 Subject: [PATCH 123/242] fix(macos): clear completed stop coordination --- .../PrivateMacShareController.swift | 21 +++++++++------- .../PrivateMacShareTests.swift | 24 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 14b2d4ff..ef12e2d8 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -14,20 +14,25 @@ enum PrivateMacSharePermissionPolicy { @MainActor final class PrivateMacShareStopCoordinator { - private var operation: Task? + private var isPerforming = false + private var waiters: [CheckedContinuation] = [] func perform(_ body: @escaping @MainActor () async -> Void) async { - if let operation { - await operation.value + if isPerforming { + await withCheckedContinuation { continuation in + self.waiters.append(continuation) + } return } - let operation = Task { @MainActor in - await body() + isPerforming = true + await body() + isPerforming = false + let waiters = waiters + self.waiters.removeAll() + for waiter in waiters { + waiter.resume() } - self.operation = operation - await operation.value - self.operation = nil } } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index ab406fe4..bee3cca9 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -402,6 +402,30 @@ struct PrivateMacShareTests { #expect(await secondState.finished) } + @Test @MainActor + func completedStopDoesNotCoalesceWithTheNextOperation() async { + let coordinator = PrivateMacShareStopCoordinator() + let operation = SuspendedAsyncOperation() + + let first = Task { + await coordinator.perform { + await operation.run() + } + } + #expect(await waitUntilAsync { await operation.invocationCount == 1 }) + await operation.finish() + await first.value + + let second = Task { + await coordinator.perform { + await operation.run() + } + } + #expect(await waitUntilAsync { await operation.invocationCount == 2 }) + await operation.finish() + await second.value + } + @Test @MainActor func failedDesktopPublicationIsNotUnregistered() async throws { let identity = desktopIdentity(name: "failed-publish", address: "100.64.12.40") From e947561d987d60993f17624eb9ae01dd1c66269d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:07:18 +0200 Subject: [PATCH 124/242] fix(vnc): close parser and credential lifetime gaps --- .../RoyalVNCKit/Compression/ZlibStream.swift | 6 ++- .../SDK/Connection/VNCConnection+API.swift | 4 ++ .../SDK/Connection/VNCConnection.swift | 5 +- .../RoyalVNCKitTests/AuditFindingsTests.swift | 50 +++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift index 8fb21617..cffbba17 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Compression/ZlibStream.swift @@ -91,12 +91,14 @@ extension ZlibStream { let actualOut = bufferSize - UInt(stream.availOut) if actualOut > 0 { - guard decompressedData.count <= maximumOutputSize - Int(actualOut) else { + let actualOutCount = Int(actualOut) + guard decompressedData.count <= maximumOutputSize, + actualOutCount <= maximumOutputSize - decompressedData.count else { throw VNCError.protocol(.zlibDecompress( underlyingError: ZlibStreamError.decompressedDataOverflow )) } - decompressedData.append(buffer, count: Int(actualOut)) + decompressedData.append(buffer, count: actualOutCount) } if isDone { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index d602886b..fd57ceae 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -370,6 +370,10 @@ extension VNCConnection { framebufferRequestLock.unlock() return } + guard pixelFormatTransitionFenceWasSent else { + framebufferRequestLock.unlock() + throw VNCError.protocol(.invalidData) + } let requiredFlags = pixelFormatTransitionRequiredFenceFlags guard fence.flags.intersection(requiredFlags) == requiredFlags else { framebufferRequestLock.unlock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 257dec31..96e62428 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -526,6 +526,7 @@ final class PendingCredentialRequest: @unchecked Sendable { lock.lock() if isResolved { let credential = resolvedCredential + resolvedCredential = nil lock.unlock() continuation.resume(returning: credential) } else { @@ -545,9 +546,11 @@ final class PendingCredentialRequest: @unchecked Sendable { return } isResolved = true - resolvedCredential = credential let continuation = self.continuation self.continuation = nil + if continuation == nil { + resolvedCredential = credential + } let onResolution = self.onResolution self.onResolution = nil lock.unlock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 4391eb10..63adabc2 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -62,6 +62,18 @@ struct AuditFindingsTests { #expect(actual == expected) } + @Test + func rejectsInflatedChunkLargerThanOutputLimitWithoutTrapping() throws { + let compressed = try ZlibOneShot.deflate(Data(repeating: 0xA5, count: 385)) + + #expect(throws: (any Error).self) { + _ = try ZlibStream().decompressedData( + compressedData: compressed, + maximumOutputSize: 384 + ) + } + } + @Test func consumesSyncFlushBytesAfterFixedSizeOutputIsFull() throws { let firstCompressed = Data([ @@ -381,6 +393,29 @@ struct AuditFindingsTests { #expect(connection.connectionState.status == .connected) } + @Test + func rejectsPixelFormatFenceResponseBeforeRequestIsSent() async throws { + let connection = try await makeFenceCapableConnection() + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + let payload = try #require(connection.pixelFormatTransitionFencePayload) + + #expect(throws: (any Error).self) { + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: payload + ) + ) + } + #expect(connection.state.pixelFormat?.depth == 24) + + try await queued.message.send(connection: AuditWritingConnection()) + connection.cancelFramebufferUpdateScheduling() + } + @Test func waitsForSlowFramebufferBoundaryBeforeArmingTransitionDeadline() async throws { let connection = try await makeFenceCapableConnection() @@ -637,6 +672,21 @@ struct AuditFindingsTests { delegate.completion?(VNCPasswordCredential(password: "late")) } + @Test + func releasesResolvedCredentialAfterValueIsConsumed() async { + let request = PendingCredentialRequest(onResolution: {}) + var credential: VNCPasswordCredential? = VNCPasswordCredential(password: "secret") + weak var weakCredential = credential + + request.resolve(with: credential) + credential = nil + var received = await request.value() as? VNCPasswordCredential + + #expect(received === weakCredential) + received = nil + #expect(weakCredential == nil) + } + @Test func preservesAlreadyRGBAFormattedCursorChannels() { let cursor = VNCCursor( From 139dd1448cd4a7827a8177e8b33f21d70408e8f2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:07:28 +0200 Subject: [PATCH 125/242] docs(changelog): record final race repairs --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e82976a7..ad095530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,12 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while discarding and acknowledging output for one-shot confirmed messages, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, reject connection closure before an input acknowledgement, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, concurrent teardown calls that could outpace application termination, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, bounded zlib streams, RFB Fence-synchronized color-depth transitions with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation, and cursor channel preservation. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions that reject premature responses with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From 6c53bcc04285df07896887d1c5cbc4ff01377c66 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:18:02 +0200 Subject: [PATCH 126/242] fix(desktop): preserve legacy ownership fences --- src/worker/desktop-host-repository.ts | 2 +- tests/desktop-host-repository.test.ts | 47 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 83c7bd5b..33408281 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -69,7 +69,7 @@ export class DesktopHostRepository implements DesktopHostStore { name: host.name, address: host.address, port: host.port, - ownership_token: host.ownershipToken, + ...(host.ownershipToken ? { ownership_token: host.ownershipToken } : {}), updated_at: host.updatedAt, }), ) diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index 853fad91..d9c6d1d0 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -141,3 +141,50 @@ test("desktop host upsert returns the row written by the same atomic statement", assert.equal(row.name, "Host A"); assert.equal(row.ownershipToken, "token-a"); }); + +test("legacy desktop host upserts preserve token ownership", async () => { + let statement = ""; + const stored = { + owner_subject: "github:1", + id: "studio", + owner: "alice", + name: "Legacy Studio", + address: "100.64.1.2", + port: 5901, + ownership_token: "current-token", + created_at: 1, + updated_at: 2, + }; + const env = { + DB: { + prepare(sql: string) { + statement = sql; + return { + bind() { + return { + async all() { + return { results: [stored], meta: { changes: 1 } }; + }, + }; + }, + }; + }, + } as unknown as D1Database, + } as RuntimeEnv; + + const row = await new DesktopHostRepository(env).upsert({ + ownerSubject: stored.owner_subject, + id: stored.id, + owner: stored.owner, + name: stored.name, + address: stored.address, + port: stored.port, + ownershipToken: "", + createdAt: stored.created_at, + updatedAt: stored.updated_at, + }); + + const updateClause = statement.split(/do update set/i)[1] ?? ""; + assert.doesNotMatch(updateClause, /ownership_token/i); + assert.equal(row.ownershipToken, "current-token"); +}); From b17c5c84e59540c274c7620fa777420a203d6095 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:18:03 +0200 Subject: [PATCH 127/242] fix(terminal): close remaining delivery races --- internal/terminalws/client.go | 28 ++++- internal/terminalws/client_test.go | 157 ++++++++++++++++++++++++++++- src/worker/terminal-hub.ts | 4 + tests/terminal-hub.test.ts | 6 +- 4 files changed, 189 insertions(+), 6 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 8cfda7fd..c8afe408 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -88,6 +88,7 @@ type Client struct { inputWaiter chan error attachment *terminalAttachment attachmentReady chan struct{} + terminalErr error readerDone chan struct{} readerErr error } @@ -292,6 +293,11 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { case err := <-waiter: return err case <-c.readerDone: + select { + case err := <-waiter: + return err + default: + } c.clearInputWaiter(waiter) return readerUnavailableError(c.readerError()) case <-ctx.Done(): @@ -510,6 +516,8 @@ func (c *Client) handleFrame(ctx context.Context, current frame) error { case "input-rejected": c.completeInput(frameError(current, "terminal input rejected")) case "closed": + c.canInput.Store(false) + c.markTerminalClosed(errors.New("terminal closed")) c.completeInput(errors.New("terminal closed before accepting input")) c.deliverAttachment(ctx, current) default: @@ -529,6 +537,9 @@ func (c *Client) registerInputWaiter(waiter chan error) error { return readerUnavailableError(c.readerErr) default: } + if c.terminalErr != nil { + return c.terminalErr + } c.inputWaiter = waiter if c.attachment == nil && c.attachmentReady != nil { close(c.attachmentReady) @@ -561,13 +572,16 @@ func (c *Client) registerAttachment() (*terminalAttachment, error) { if c.attachment != nil { return nil, errors.New("terminal client is already attached") } + if c.terminalErr != nil { + return nil, c.terminalErr + } select { case <-c.readerDone: return nil, readerUnavailableError(c.readerErr) default: } attachment := &terminalAttachment{ - frames: make(chan frame, 16), + frames: make(chan frame), done: make(chan struct{}), } c.attachment = attachment @@ -592,6 +606,14 @@ func (c *Client) clearAttachment(attachment *terminalAttachment) { c.stateMu.Unlock() } +func (c *Client) markTerminalClosed(err error) { + c.stateMu.Lock() + if c.terminalErr == nil { + c.terminalErr = err + } + c.stateMu.Unlock() +} + func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) error { for { c.stateMu.Lock() @@ -649,11 +671,11 @@ func (c *Client) finishReader(err error) { c.readerErr = normalizeCloseError(err) waiter := c.inputWaiter c.inputWaiter = nil - close(c.readerDone) - c.stateMu.Unlock() if waiter != nil { waiter <- readerUnavailableError(c.readerErr) } + close(c.readerDone) + c.stateMu.Unlock() } func (c *Client) readerError() error { diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 0f7f23aa..7c98222e 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -649,6 +649,159 @@ func TestSendInputConfirmedFailsWhenConnectionClosesBeforeAcknowledgement(t *tes } } +func TestSendInputConfirmedPrefersAcceptanceBeforeReaderClose(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-accepted-before-close", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-accepted-before-close", + payload: accepted, + })); err != nil { + t.Error(err) + return + } + _ = conn.Close(websocket.StatusNormalClosure, "") + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-accepted-before-close", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + if err := client.SendInputConfirmed(context.Background(), []byte("echo ready\n")); err != nil { + t.Fatal(err) + } +} + +func TestAttachRejectsSessionClosedBeforeAttachment(t *testing.T) { + closedSent := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-closed-before-attach", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + closed, _ := json.Marshal(eventPayload{Type: "closed"}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-closed-before-attach", + payload: closed, + })); err != nil { + t.Error(err) + return + } + close(closedSent) + <-r.Context().Done() + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-closed-before-attach", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + <-closedSent + deadline := time.Now().Add(time.Second) + for { + client.stateMu.Lock() + terminalErr := client.terminalErr + client.stateMu.Unlock() + if terminalErr != nil { + break + } + if time.Now().After(deadline) { + t.Fatal("terminal close was not retained") + } + time.Sleep(time.Millisecond) + } + + err = client.Attach(context.Background(), newBlockingTerminal(), nil) + if err == nil || !strings.Contains(err.Error(), "terminal closed") { + t.Fatalf("error = %v", err) + } +} + +func TestRetiredAttachmentCannotAcceptBufferedFrames(t *testing.T) { + attachment := &terminalAttachment{ + frames: make(chan frame), + done: make(chan struct{}), + } + close(attachment.done) + client := &Client{attachment: attachment} + + for range 100 { + if client.deliverAttachment(context.Background(), frame{messageType: messageOutput}) { + t.Fatal("retired attachment accepted output") + } + } +} + func TestSendInputConfirmedRejectionDoesNotRevokeControl(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) @@ -1452,8 +1605,8 @@ func TestClientContinuesReadOnlyAndResumesControl(t *testing.T) { if size := <-resumedSize; size != (Size{Cols: 132, Rows: 43}) { t.Fatalf("resumed size = %#v", size) } - if !client.canInput.Load() { - t.Fatal("client did not resume input control") + if client.canInput.Load() { + t.Fatal("closed terminal retained input control") } } diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 0c01652e..344b7b34 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -241,6 +241,10 @@ export class TerminalHub { const canInput = await subscription.canInput(); updateTerminalInputCapability(server, subscription, canInput); if (!canInput) { + sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { + type: "input-rejected", + error: "terminal control is not granted", + }); return; } if (subscription.upstream.readyState !== WebSocket.OPEN) { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 2422d756..3ca809fc 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -469,7 +469,11 @@ test("terminal hub publishes live controller downgrades and promotions", async ( }), }); await flushQueues(); - assert.equal(frame(server.sent.at(-1)!).type, TerminalMessageType.ControlRevoked); + assert.equal(frame(server.sent.at(-2)!).type, TerminalMessageType.ControlRevoked); + assert.deepEqual(decodeJsonPayload(frame(server.sent.at(-1)!).payload), { + type: "input-rejected", + error: "terminal control is not granted", + }); assert.deepEqual(upstream.sent, []); canInput = true; From b200f1f3c33edd35ef7354894d90a7ff421548f6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:18:05 +0200 Subject: [PATCH 128/242] fix(macos): retain teardown input releases --- .../Sources/CrabfleetMac/MacRemoteInput.swift | 3 +- .../PrivateMacShareTests.swift | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index aeea22f1..ec58dbb5 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -200,8 +200,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable private func schedulePendingReleaseRetry() { guard !pendingReleaseRetryScheduled else { return } pendingReleaseRetryScheduled = true - eventQueue.asyncAfter(deadline: .now() + pendingReleaseRetryDelay) { [weak self] in - guard let self else { return } + eventQueue.asyncAfter(deadline: .now() + pendingReleaseRetryDelay) { [self] in self.pendingReleaseRetryScheduled = false self.flushPendingRelease() } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index bee3cca9..dde37ea3 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -1096,6 +1096,47 @@ struct PrivateMacShareTests { }) } + @Test + func pendingInputReleaseRetainsControllerThroughTeardown() async { + let trust = AccessibilityTrust(granted: true) + let events = RemoteInputEventRecorder() + var controller: MacRemoteInputController? = MacRemoteInputController( + descriptor: CapturedDisplayDescriptor( + displayID: 1, + displayBounds: CGRect(x: 0, y: 0, width: 100, height: 100), + frameWidth: 100, + frameHeight: 100, + sourcePixelWidth: 100, + sourcePixelHeight: 100 + ), + accessibilityGranted: { trust.isGranted() }, + pendingReleaseRetryDelay: .milliseconds(10), + keyEventPoster: { down, keysym in + events.append(.key(down: down, keysym: keysym)) + } + ) + weak var retainedController = controller + + controller?.keyEvent(down: true, keysym: 0x61) + #expect(await waitUntilAsync { + events.contains(.key(down: true, keysym: 0x61)) + }) + let checksBeforeRevocation = trust.checkCount + trust.setGranted(false) + controller?.releaseAllInput() + #expect(await waitUntilAsync { + trust.checkCount > checksBeforeRevocation + }) + controller = nil + + #expect(retainedController != nil) + trust.setGranted(true) + #expect(await waitUntilAsync { + events.contains(.key(down: false, keysym: 0x61)) + }) + #expect(await waitUntilAsync { retainedController == nil }) + } + @Test func decodesX11UnicodeKeysymsForMacInput() { #expect(MacRemoteInputController.unicodeScalar(for: 0x0100_03BB) == "λ") From 7d7dd97bf992a7942878933e2878eed29ebef7df Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:18:06 +0200 Subject: [PATCH 129/242] fix(vnc): arm transition fences before writes --- .../SDK/Connection/VNCConnection+API.swift | 43 ++++++++++++++- .../RoyalVNCKitTests/AuditFindingsTests.swift | 55 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index fd57ceae..ec76fe38 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -9,7 +9,9 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { let pixelFormatMessage: VNCProtocol.SetPixelFormat let willSend: () -> Void let didSend: () -> Void + let willSendFence: () -> Void let didSendFence: () -> Void + let didFailFence: () -> Void var messageType: UInt8 { fenceMessage?.messageType ?? pixelFormatMessage.messageType } var data: Data { @@ -19,8 +21,17 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { func send(connection: NetworkConnectionWriting) async throws { if fenceMessage == nil { willSend() + } else { + willSendFence() + } + do { + try await connection.write(data: data) + } catch { + if fenceMessage != nil { + didFailFence() + } + throw error } - try await connection.write(data: data) if fenceMessage == nil { didSend() } else { @@ -199,27 +210,53 @@ extension VNCConnection { didSend: { [weak self] in self?.completePixelFormatTransition() }, + willSendFence: { [weak self] in + guard let payload = transition.fencePayload else { return } + self?.beginPixelFormatTransitionFenceWrite(payload: payload) + }, didSendFence: { [weak self] in guard let payload = transition.fencePayload else { return } self?.schedulePixelFormatTransitionDeadline(payload: payload) + }, + didFailFence: { [weak self] in + guard let payload = transition.fencePayload else { return } + self?.cancelPixelFormatTransitionFenceWrite(payload: payload) } ) enqueueClientToServerMessage(message) } - private func schedulePixelFormatTransitionDeadline(payload: Data) { + private func beginPixelFormatTransitionFenceWrite(payload: Data) { framebufferRequestLock.lock() defer { framebufferRequestLock.unlock() } guard isPixelFormatTransitionInFlight, pixelFormatTransitionFencePayload == payload else { return } - pixelFormatTransitionFenceWasSent = true + } + + private func schedulePixelFormatTransitionDeadline(payload: Data) { + framebufferRequestLock.lock() + defer { framebufferRequestLock.unlock() } + guard isPixelFormatTransitionInFlight, + pixelFormatTransitionFencePayload == payload, + pixelFormatTransitionFenceWasSent else { + return + } + schedulePixelFormatTransitionDeadlineIfReadyLocked(payload: payload) } + private func cancelPixelFormatTransitionFenceWrite(payload: Data) { + framebufferRequestLock.lock() + defer { framebufferRequestLock.unlock() } + guard pixelFormatTransitionFencePayload == payload else { return } + pixelFormatTransitionFenceWasSent = false + cancelPixelFormatTransitionDeadlineLocked() + } + private func schedulePixelFormatTransitionDeadlineIfReadyLocked(payload: Data) { guard pixelFormatTransitionFenceWasSent, !framebufferUpdateRequestOutstanding, diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 63adabc2..353d9832 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -416,6 +416,53 @@ struct AuditFindingsTests { connection.cancelFramebufferUpdateScheduling() } + @Test + func acceptsPixelFormatFenceResponseWhileWriteCompletes() async throws { + let connection = try await makeFenceCapableConnection() + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + let payload = try #require(connection.pixelFormatTransitionFencePayload) + connection.completeFramebufferUpdateRequest() + var responseError: Error? + let writer = AuditWritingConnection { + do { + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: payload + ) + ) + } catch { + responseError = error + } + } + + try await queued.message.send(connection: writer) + + #expect(responseError == nil) + #expect(connection.state.pixelFormat?.depth == 8) + #expect(!connection.pixelFormatTransitionFenceWasSent) + } + + @Test + func rollsBackPixelFormatFenceWhenWriteFails() async throws { + let connection = try await makeFenceCapableConnection() + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + + await #expect(throws: AuditWriteError.self) { + try await queued.message.send(connection: AuditFailingWritingConnection()) + } + + #expect(connection.isPixelFormatTransitionInFlight) + #expect(!connection.pixelFormatTransitionFenceWasSent) + #expect(connection.pixelFormatTransitionDeadlineTask == nil) + connection.cancelFramebufferUpdateScheduling() + } + @Test func waitsForSlowFramebufferBoundaryBeforeArmingTransitionDeadline() async throws { let connection = try await makeFenceCapableConnection() @@ -809,6 +856,14 @@ private final class AuditWritingConnection: NetworkConnectionWriting { } } +private struct AuditWriteError: Error {} + +private final class AuditFailingWritingConnection: NetworkConnectionWriting { + func write(data: Data) async throws { + throw AuditWriteError() + } +} + @MainActor private final class PendingCredentialDelegate: VNCConnectionDelegate { var completion: ((VNCCredential?) -> Void)? From 64deb4889dfa7b4269895d698ba46d5131c53206 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:18:13 +0200 Subject: [PATCH 130/242] docs(actions): make framed runner byte-safe --- README.md | 18 +----------------- docs/github-actions-sessions.md | 4 ++-- tests/github-actions-docs.test.ts | 18 +++++++++--------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c2fad4bf..bc7fbe44 100644 --- a/README.md +++ b/README.md @@ -101,22 +101,6 @@ const framedRunnerPtyUrl = new URL(runnerPtyUrl); framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); const terminal = new WebSocket(framedRunnerPtyUrl); terminal.binaryType = "arraybuffer"; -const inputDecoder = new TextDecoder("utf-8", { fatal: true }); - -pty.onData((output) => terminal.send(encodeCfr1Output(output))); -pty.onExit(() => { - if (terminal.readyState < WebSocket.CLOSING) terminal.close(1000, "pty exited"); -}); -terminal.onmessage = ({ data }) => { - const input = decodeCfr1Input(data); - if (!input) return; - try { - pty.write(inputDecoder.decode(input.payload)); - terminal.send(encodeCfr1Ack(input.inputId, true)); - } catch { - terminal.send(encodeCfr1Ack(input.inputId, false)); - } -}; ``` Existing runners retain raw input and output by opening the returned URL @@ -125,7 +109,7 @@ acknowledgement frames by adding the exact `runnerProtocol=cfr1-framed-io-v1` query before opening the socket. The relay selects that mode before accepting the connection, so there is no pending handshake. Framed runners acknowledge only after their PTY accepts the input; -the complete encoder, decoder, and Node runner example are in +the complete byte-safe encoder, decoder, and Node PTY runner are in [`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 6d0b8bb5..8d6e3aa4 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -320,8 +320,8 @@ function acceptInput(data) { const input = decodeInput(data); if (!input) return; try { - // Reject frames that split or contain invalid UTF-8 instead of corrupting PTY input. - pty.write(inputDecoder.decode(input.payload)); + const text = inputDecoder.decode(input.payload, { stream: true }); + if (text) pty.write(text); terminal.send(encodeAck(input.inputId, true)); } catch { terminal.send(encodeAck(input.inputId, false)); diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 64640591..5d8888cf 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -2,20 +2,20 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import { test } from "node:test"; -test("the documented Node runner rejects split UTF-8 input frames", async () => { +test("the documented Node runner preserves split UTF-8 input frames", async () => { const [readme, guide] = await Promise.all([ readFile(new URL("../README.md", import.meta.url), "utf8"), readFile(new URL("../docs/github-actions-sessions.md", import.meta.url), "utf8"), ]); - for (const documentation of [readme, guide]) { - assert.match(documentation, /new TextDecoder\("utf-8", \{ fatal: true \}\)/); - assert.match(documentation, /inputDecoder\.decode\(input\.payload\)/); - assert.match(documentation, /pty\.onExit\(\(\) => \{/); - assert.match(documentation, /terminal\.close\(1000, "pty exited"\)/); - } + assert.match(readme, /complete byte-safe encoder, decoder, and Node PTY runner/); + assert.doesNotMatch(readme, /encodeCfr1Output|decodeCfr1Input|encodeCfr1Ack/); + assert.match(guide, /new TextDecoder\("utf-8", \{ fatal: true \}\)/); + assert.match(guide, /inputDecoder\.decode\(input\.payload, \{ stream: true \}\)/); + assert.match(guide, /pty\.onExit\(\(\) => \{/); + assert.match(guide, /terminal\.close\(1000, "pty exited"\)/); const decoder = new TextDecoder("utf-8", { fatal: true }); - assert.throws(() => decoder.decode(Uint8Array.from([0xf0, 0x9f])), TypeError); - assert.throws(() => decoder.decode(Uint8Array.from([0xa6, 0x80])), TypeError); + assert.equal(decoder.decode(Uint8Array.from([0xf0, 0x9f]), { stream: true }), ""); + assert.equal(decoder.decode(Uint8Array.from([0xa6, 0x80]), { stream: true }), "🦀"); }); From 394aeaa391a802ecc39200c7f00be7a755e75ac8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:18:14 +0200 Subject: [PATCH 131/242] docs(changelog): record remaining audit repairs --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad095530..ecd38cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,10 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, reject connection closure before an input acknowledgement, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. -- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, reject split UTF-8 in the string-only Node adapter, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, preserve split UTF-8 across the string-only Node adapter, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases delayed by revoked Accessibility trust, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers erasing current ownership fences, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions that reject premature responses with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. From 3052af1b4a572fd00c7be809feca129c1cc5f2aa Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:21:11 +0200 Subject: [PATCH 132/242] fix(ssh): snapshot connection limits before handlers --- cmd/crabbox-ssh-gateway/main.go | 58 +++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/cmd/crabbox-ssh-gateway/main.go b/cmd/crabbox-ssh-gateway/main.go index a4970a83..17e07501 100644 --- a/cmd/crabbox-ssh-gateway/main.go +++ b/cmd/crabbox-ssh-gateway/main.go @@ -98,6 +98,22 @@ type sessionPTY struct { resizes chan fleetapi.TerminalSize } +type sshConnectionSettings struct { + handshakeTimeout time.Duration + connectionIdle time.Duration + sessionIdle time.Duration + sessionChannels int +} + +func currentSSHConnectionSettings() sshConnectionSettings { + return sshConnectionSettings{ + handshakeTimeout: sshHandshakeTimeout, + connectionIdle: sshConnectionIdle, + sessionIdle: sshSessionIdleTimer, + sessionChannels: sshSessionChannels, + } +} + func main() { var addr string var apiURL string @@ -176,6 +192,7 @@ func main() { } func acceptConn(raw net.Conn, config *ssh.ServerConfig, client *apiClient) { + settings := currentSSHConnectionSettings() if !sshConnectionSlots.acquire() { log.Printf("connection limit reached for %s", raw.RemoteAddr()) raw.Close() @@ -187,17 +204,25 @@ func acceptConn(raw net.Conn, config *ssh.ServerConfig, client *apiClient) { raw.Close() return } - go handleConnWithRelease(raw, config, client, sshHandshakeSlots.release, sshConnectionSlots.release) + go handleConnWithRelease( + raw, + config, + client, + settings, + sshHandshakeSlots.release, + sshConnectionSlots.release, + ) } func handleConn(raw net.Conn, config *ssh.ServerConfig, client *apiClient) { - handleConnWithRelease(raw, config, client, nil, nil) + handleConnWithRelease(raw, config, client, currentSSHConnectionSettings(), nil, nil) } func handleConnWithRelease( raw net.Conn, config *ssh.ServerConfig, client *apiClient, + settings sshConnectionSettings, releaseHandshake func(), releaseConnection func(), ) { @@ -212,7 +237,7 @@ func handleConnWithRelease( } }() defer raw.Close() - if err := raw.SetDeadline(time.Now().Add(sshHandshakeTimeout)); err != nil { + if err := raw.SetDeadline(time.Now().Add(settings.handshakeTimeout)); err != nil { log.Printf("handshake deadline %s: %v", raw.RemoteAddr(), err) } conn, chans, reqs, err := ssh.NewServerConn(raw, config) @@ -234,12 +259,12 @@ func handleConnWithRelease( if permissions == nil { permissions = &ssh.Permissions{Extensions: map[string]string{}} } - sessionSlots := newConnectionLimiter(sshSessionChannels) + sessionSlots := newConnectionLimiter(settings.sessionChannels) sessionDone := make(chan struct{}) connectionClosed := make(chan struct{}) defer close(connectionClosed) activeSessions := 0 - connectionIdleTimer, connectionIdle := newConnectionIdleTimer() + connectionIdleTimer, connectionIdle := newIdleTimer(settings.connectionIdle) defer stopTimer(connectionIdleTimer) for { select { @@ -251,7 +276,7 @@ func handleConnWithRelease( } if activeSessions == 0 { stopTimer(connectionIdleTimer) - connectionIdleTimer, connectionIdle = newConnectionIdleTimer() + connectionIdleTimer, connectionIdle = newIdleTimer(settings.connectionIdle) } case ch, ok := <-chans: if !ok { @@ -277,12 +302,12 @@ func handleConnWithRelease( activeSessions-- } if activeSessions == 0 { - connectionIdleTimer, connectionIdle = newConnectionIdleTimer() + connectionIdleTimer, connectionIdle = newIdleTimer(settings.connectionIdle) } log.Printf("channel accept: %v", err) continue } - go handleSession(channel, requests, permissions, client, func() { + go handleSession(channel, requests, permissions, client, settings.sessionIdle, func() { sessionSlots.release() select { case sessionDone <- struct{}{}: @@ -298,6 +323,7 @@ func handleSession( requests <-chan *ssh.Request, perms *ssh.Permissions, client *apiClient, + idleTimeout time.Duration, release func(), ) { if release != nil { @@ -306,7 +332,7 @@ func handleSession( defer channel.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - idleTimer, idle := newSessionIdleTimer() + idleTimer, idle := newIdleTimer(idleTimeout) defer stopTimer(idleTimer) pty := sessionPTY{ cols: 120, @@ -398,19 +424,11 @@ func handleSession( } } -func newSessionIdleTimer() (*time.Timer, <-chan time.Time) { - if sshSessionIdleTimer <= 0 { - return nil, nil - } - timer := time.NewTimer(sshSessionIdleTimer) - return timer, timer.C -} - -func newConnectionIdleTimer() (*time.Timer, <-chan time.Time) { - if sshConnectionIdle <= 0 { +func newIdleTimer(timeout time.Duration) (*time.Timer, <-chan time.Time) { + if timeout <= 0 { return nil, nil } - timer := time.NewTimer(sshConnectionIdle) + timer := time.NewTimer(timeout) return timer, timer.C } From 255825a375eec4928868cba1dfe2efa80681779d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:21:12 +0200 Subject: [PATCH 133/242] docs(changelog): record ssh limit snapshotting --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd38cb1..81e481b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments, snapshot SSH connection limits before launching handlers, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, preserve split UTF-8 across the string-only Node adapter, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers erasing current ownership fences, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. From c2600f10be5ca847196d1332e060c0d128453f04 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:34:46 +0200 Subject: [PATCH 134/242] fix(migrations): fence mixed-version ownership --- migrations/0033_desktop_host_ownership.sql | 28 ++++ ...credential_policy_registration_staging.sql | 30 +--- src/worker/desktop-host-repository.ts | 80 +++++++--- tests/desktop-host-migration.test.ts | 97 ++++++++++++ tests/desktop-host-repository.test.ts | 143 +++++++++++++++++- ...ndbox-credential-policy-repository.test.ts | 90 +++++++++-- 6 files changed, 405 insertions(+), 63 deletions(-) diff --git a/migrations/0033_desktop_host_ownership.sql b/migrations/0033_desktop_host_ownership.sql index f9aa13c2..92d18468 100644 --- a/migrations/0033_desktop_host_ownership.sql +++ b/migrations/0033_desktop_host_ownership.sql @@ -1,2 +1,30 @@ ALTER TABLE desktop_hosts ADD COLUMN ownership_token TEXT NOT NULL DEFAULT ''; + +CREATE TRIGGER IF NOT EXISTS protect_token_owned_desktop_host_update +BEFORE UPDATE ON desktop_hosts +WHEN OLD.ownership_token <> '' + AND NEW.ownership_token = OLD.ownership_token + AND ( + NEW.owner_subject IS NOT OLD.owner_subject + OR NEW.id IS NOT OLD.id + OR NEW.owner IS NOT OLD.owner + OR NEW.name IS NOT OLD.name + OR NEW.address IS NOT OLD.address + OR NEW.port IS NOT OLD.port + OR NEW.created_at IS NOT OLD.created_at + OR NEW.updated_at IS NOT OLD.updated_at + ) +BEGIN + SELECT RAISE(IGNORE); +END; + +-- Token-aware workers replace the exact token with this transient marker and +-- delete it in the same atomic batch. Legacy workers can never create it. +CREATE TRIGGER IF NOT EXISTS protect_token_owned_desktop_host_delete +BEFORE DELETE ON desktop_hosts +WHEN OLD.ownership_token <> '' + AND OLD.ownership_token NOT GLOB 'delete-authorized:*' +BEGIN + SELECT RAISE(IGNORE); +END; diff --git a/migrations/0034_credential_policy_registration_staging.sql b/migrations/0034_credential_policy_registration_staging.sql index e0587917..1455ea6b 100644 --- a/migrations/0034_credential_policy_registration_staging.sql +++ b/migrations/0034_credential_policy_registration_staging.sql @@ -41,30 +41,6 @@ CREATE INDEX IF NOT EXISTS idx_credential_policy_registration_expiry updated_at ); -INSERT OR IGNORE INTO interactive_session_credential_policy_registrations ( - session_id, - sandbox_id, - state, - registration_generation, - registration_claim, - registration_claim_expires_at, - created_at, - updated_at -) -SELECT - session_id, - sandbox_id, - 'registering', - MIN(registration_generation), - MIN(registration_claim), - MIN(registration_claim_expires_at), - MIN(created_at), - MAX(updated_at) -FROM interactive_session_credential_policies -WHERE state = 'registering' - AND registration_claim IS NOT NULL - AND registration_claim_expires_at IS NOT NULL -GROUP BY session_id, sandbox_id -HAVING count(DISTINCT registration_generation) = 1 - AND count(DISTINCT registration_claim) = 1 - AND count(DISTINCT registration_claim_expires_at) = 1; +-- Legacy workers renew registration claims only in the policy table. Leaving +-- those claims unstaged prevents the new scanner from recovering a stale +-- snapshot while the legacy registration is still live. diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 33408281..5e1f840a 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -1,4 +1,6 @@ -import { database } from "./database.ts"; +import { sql } from "kysely"; + +import { database, executeBatch } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; export type DesktopHostRow = { @@ -63,16 +65,40 @@ export class DesktopHostRepository implements DesktopHostStore { created_at: host.createdAt, updated_at: host.updatedAt, }) - .onConflict((conflict) => - conflict.columns(["owner_subject", "id"]).doUpdateSet({ - owner: host.owner, - name: host.name, - address: host.address, - port: host.port, - ...(host.ownershipToken ? { ownership_token: host.ownershipToken } : {}), - updated_at: host.updatedAt, - }), - ) + .onConflict((conflict) => { + const update = conflict.columns(["owner_subject", "id"]); + return host.ownershipToken + ? update.doUpdateSet({ + owner: host.owner, + name: host.name, + address: host.address, + port: host.port, + ownership_token: host.ownershipToken, + updated_at: host.updatedAt, + }) + : update.doUpdateSet({ + owner: sql`CASE + WHEN desktop_hosts.ownership_token = '' THEN excluded.owner + ELSE desktop_hosts.owner + END`, + name: sql`CASE + WHEN desktop_hosts.ownership_token = '' THEN excluded.name + ELSE desktop_hosts.name + END`, + address: sql`CASE + WHEN desktop_hosts.ownership_token = '' THEN excluded.address + ELSE desktop_hosts.address + END`, + port: sql`CASE + WHEN desktop_hosts.ownership_token = '' THEN excluded.port + ELSE desktop_hosts.port + END`, + updated_at: sql`CASE + WHEN desktop_hosts.ownership_token = '' THEN excluded.updated_at + ELSE desktop_hosts.updated_at + END`, + }); + }) .returningAll() .executeTakeFirstOrThrow(); return { @@ -89,11 +115,31 @@ export class DesktopHostRepository implements DesktopHostStore { } async remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise { - await database(this.env) - .deleteFrom("desktop_hosts") - .where("owner_subject", "=", ownerSubject) - .where("id", "=", id) - .where("ownership_token", "=", ownershipToken ?? "") - .execute(); + const db = database(this.env); + if (!ownershipToken) { + await db + .deleteFrom("desktop_hosts") + .where("owner_subject", "=", ownerSubject) + .where("id", "=", id) + .where("ownership_token", "=", "") + .execute(); + return; + } + const deleteMarker = `delete-authorized:${crypto.randomUUID()}`; + // The migration trigger permits this marker only; the atomic batch keeps it + // invisible to legacy workers between authorization and deletion. + await executeBatch(this.env, [ + db + .updateTable("desktop_hosts") + .set({ ownership_token: deleteMarker }) + .where("owner_subject", "=", ownerSubject) + .where("id", "=", id) + .where("ownership_token", "=", ownershipToken), + db + .deleteFrom("desktop_hosts") + .where("owner_subject", "=", ownerSubject) + .where("id", "=", id) + .where("ownership_token", "=", deleteMarker), + ]); } } diff --git a/tests/desktop-host-migration.test.ts b/tests/desktop-host-migration.test.ts index c9d82cd8..0efc821d 100644 --- a/tests/desktop-host-migration.test.ts +++ b/tests/desktop-host-migration.test.ts @@ -47,3 +47,100 @@ test("desktop host migration creates an owner-scoped registry with bounded ports "idx_desktop_hosts_owner_updated", ); }); + +test("desktop host ownership migration blocks old-worker mutations of token-owned rows", () => { + const database = new DatabaseSync(":memory:"); + database.exec( + readFileSync(new URL("../migrations/0030_desktop_hosts.sql", import.meta.url), "utf8"), + ); + database.exec( + readFileSync(new URL("../migrations/0033_desktop_host_ownership.sql", import.meta.url), "utf8"), + ); + database.exec(` + INSERT INTO desktop_hosts ( + owner_subject, id, owner, name, address, port, ownership_token, created_at, updated_at + ) VALUES + ('github:1', 'owned', 'alice', 'Owned Studio', '100.64.1.2', 5901, 'token-1', 1, 2), + ('github:1', 'legacy', 'alice', 'Legacy Studio', '100.64.1.3', 5901, '', 1, 2); + `); + + const oldWorkerUpsert = database.prepare(` + INSERT INTO desktop_hosts ( + owner_subject, id, owner, name, address, port, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(owner_subject, id) DO UPDATE SET + owner = excluded.owner, + name = excluded.name, + address = excluded.address, + port = excluded.port, + updated_at = excluded.updated_at + `); + oldWorkerUpsert.run( + "github:1", + "owned", + "old-worker", + "Overwritten", + "100.64.1.99", + 5902, + 10, + 20, + ); + oldWorkerUpsert.run( + "github:1", + "legacy", + "old-worker", + "Updated Legacy", + "100.64.1.4", + 5902, + 10, + 20, + ); + + assert.deepEqual( + { + ...database + .prepare(` + SELECT owner, name, address, port, ownership_token, created_at, updated_at + FROM desktop_hosts + WHERE id = 'owned' + `) + .get(), + }, + { + owner: "alice", + name: "Owned Studio", + address: "100.64.1.2", + port: 5901, + ownership_token: "token-1", + created_at: 1, + updated_at: 2, + }, + ); + assert.equal( + database.prepare("SELECT name FROM desktop_hosts WHERE id = 'legacy'").get()?.name, + "Updated Legacy", + ); + + database.exec("DELETE FROM desktop_hosts WHERE owner_subject = 'github:1' AND id = 'owned'"); + database.exec("DELETE FROM desktop_hosts WHERE owner_subject = 'github:1' AND id = 'legacy'"); + assert.equal( + database.prepare("SELECT count(*) AS count FROM desktop_hosts WHERE id = 'owned'").get()?.count, + 1, + ); + assert.equal( + database.prepare("SELECT count(*) AS count FROM desktop_hosts WHERE id = 'legacy'").get() + ?.count, + 0, + ); + + database.exec(` + UPDATE desktop_hosts + SET ownership_token = 'delete-authorized:test' + WHERE owner_subject = 'github:1' AND id = 'owned' AND ownership_token = 'token-1'; + DELETE FROM desktop_hosts + WHERE owner_subject = 'github:1' + AND id = 'owned' + AND ownership_token = 'delete-authorized:test'; + `); + assert.equal(database.prepare("SELECT count(*) AS count FROM desktop_hosts").get()?.count, 0); +}); diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index d9c6d1d0..c14bea90 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -1,9 +1,72 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; import test from "node:test"; import { DesktopHostRepository } from "../src/worker/desktop-host-repository.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; +type BoundStatement = { + execute(): { + results: Record[]; + success: true; + meta: { changes: number; last_row_id?: number }; + }; +}; + +function sqliteRuntimeEnv(sqlite: DatabaseSync): RuntimeEnv { + function execute(sql: string, parameters: unknown[]) { + const statement = sqlite.prepare(sql); + if (/^\s*(?:select|pragma|with)\b|\breturning\b/i.test(sql)) { + const results = statement.all(...parameters).map((row) => ({ ...row })); + const changes = Number(sqlite.prepare("SELECT changes() AS changes").get()?.changes ?? 0); + return { results, success: true as const, meta: { changes } }; + } + const result = statement.run(...parameters); + return { + results: [], + success: true as const, + meta: { + changes: Number(result.changes), + last_row_id: Number(result.lastInsertRowid), + }, + }; + } + return { + DB: { + prepare(sql: string) { + return { + bind(...parameters: unknown[]) { + const bound = { + execute: () => execute(sql, parameters), + async all() { + return bound.execute(); + }, + async run() { + return bound.execute(); + }, + }; + return bound; + }, + }; + }, + async batch(statements: D1PreparedStatement[]) { + sqlite.exec("BEGIN IMMEDIATE"); + try { + const results = statements.map((statement) => + (statement as unknown as BoundStatement).execute(), + ); + sqlite.exec("COMMIT"); + return results; + } catch (error) { + sqlite.exec("ROLLBACK"); + throw error; + } + }, + } as unknown as D1Database, + } as RuntimeEnv; +} + test("desktop host repository scopes reads, upserts, and deletes by owner subject", async () => { const executions: Array<{ sql: string; parameters: unknown[] }> = []; const stored = { @@ -34,6 +97,9 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec }, }; }, + async batch() { + return []; + }, } as unknown as D1Database, } as RuntimeEnv; const repository = new DesktopHostRepository(env); @@ -70,14 +136,18 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec assert.match(executions[1]?.sql ?? "", /\breturning \*/i); await repository.remove("github:1", "studio", "ownership-token"); - assert.match(executions[2]?.sql ?? "", /^delete from "desktop_hosts"/i); + assert.match(executions[2]?.sql ?? "", /^update "desktop_hosts"/i); assert.match(executions[2]?.sql ?? "", /"ownership_token" = \?/i); - assert.deepEqual(executions[2]?.parameters, ["github:1", "studio", "ownership-token"]); + assert.deepEqual(executions[2]?.parameters.slice(1), ["github:1", "studio", "ownership-token"]); + const deleteMarker = executions[2]?.parameters[0]; + assert.match(String(deleteMarker), /^delete-authorized:/); + assert.match(executions[3]?.sql ?? "", /^delete from "desktop_hosts"/i); + assert.deepEqual(executions[3]?.parameters, ["github:1", "studio", deleteMarker]); await repository.remove("github:1", "legacy-studio", null); - assert.match(executions[3]?.sql ?? "", /^delete from "desktop_hosts"/i); - assert.match(executions[3]?.sql ?? "", /"ownership_token" = \?/i); - assert.deepEqual(executions[3]?.parameters, ["github:1", "legacy-studio", ""]); + assert.match(executions[4]?.sql ?? "", /^delete from "desktop_hosts"/i); + assert.match(executions[4]?.sql ?? "", /"ownership_token" = \?/i); + assert.deepEqual(executions[4]?.parameters, ["github:1", "legacy-studio", ""]); }); test("desktop host upsert returns the row written by the same atomic statement", async () => { @@ -185,6 +255,67 @@ test("legacy desktop host upserts preserve token ownership", async () => { }); const updateClause = statement.split(/do update set/i)[1] ?? ""; - assert.doesNotMatch(updateClause, /ownership_token/i); + for (const column of ["owner", "name", "address", "port", "updated_at"]) { + assert.match( + updateClause, + new RegExp( + `"${column}" = CASE\\s+WHEN desktop_hosts\\.ownership_token = '' THEN excluded\\.${column}\\s+ELSE desktop_hosts\\.${column}\\s+END`, + "i", + ), + ); + } assert.equal(row.ownershipToken, "current-token"); }); + +test("legacy desktop host writes and cleanup cannot mutate token-owned rows", async () => { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec( + readFileSync(new URL("../migrations/0030_desktop_hosts.sql", import.meta.url), "utf8"), + ); + sqlite.exec( + readFileSync(new URL("../migrations/0033_desktop_host_ownership.sql", import.meta.url), "utf8"), + ); + sqlite.exec(` + INSERT INTO desktop_hosts ( + owner_subject, id, owner, name, address, port, ownership_token, created_at, updated_at + ) VALUES ( + 'github:1', 'studio', 'alice', 'Token Studio', '100.64.1.2', 5901, + 'current-token', 1, 2 + ) + `); + const repository = new DesktopHostRepository(sqliteRuntimeEnv(sqlite)); + + const preserved = await repository.upsert({ + ownerSubject: "github:1", + id: "studio", + owner: "legacy-worker", + name: "Overwritten Studio", + address: "100.64.1.99", + port: 5902, + ownershipToken: "", + createdAt: 10, + updatedAt: 20, + }); + assert.deepEqual(preserved, { + ownerSubject: "github:1", + id: "studio", + owner: "alice", + name: "Token Studio", + address: "100.64.1.2", + port: 5901, + ownershipToken: "current-token", + createdAt: 1, + updatedAt: 2, + }); + + await repository.remove("github:1", "studio", null); + await repository.remove("github:1", "studio", "stale-token"); + assert.equal( + sqlite.prepare("SELECT ownership_token FROM desktop_hosts WHERE id = 'studio'").get() + ?.ownership_token, + "current-token", + ); + + await repository.remove("github:1", "studio", "current-token"); + assert.equal(sqlite.prepare("SELECT count(*) AS count FROM desktop_hosts").get()?.count, 0); +}); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 4f9e1da3..f125e920 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -89,7 +89,7 @@ function runtimeEnv( } as RuntimeEnv; } -function credentialPolicyDatabase(): DatabaseSync { +function credentialPolicyDatabase(options: { applyMigrations?: boolean } = {}): DatabaseSync { const db = new DatabaseSync(":memory:"); db.exec(` CREATE TABLE interactive_sessions ( @@ -149,18 +149,20 @@ function credentialPolicyDatabase(): DatabaseSync { ('IS-42', 'sandbox-1', 'sandbox-1', 'active', 'generation:existing', NULL, NULL, 1, 1), ('IS-42', 'sandbox-1', 'do-1', 'active', 'generation:existing', NULL, NULL, 1, 1); `); - db.exec( - readFileSync( - new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), - "utf8", - ), - ); - db.exec( - readFileSync( - new URL("../migrations/0035_credential_policy_registration_rollback.sql", import.meta.url), - "utf8", - ), - ); + if (options.applyMigrations !== false) { + db.exec( + readFileSync( + new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), + "utf8", + ), + ); + db.exec( + readFileSync( + new URL("../migrations/0035_credential_policy_registration_rollback.sql", import.meta.url), + "utf8", + ), + ); + } return db; } @@ -396,6 +398,68 @@ test("credential-policy rotation always claims a fresh generation", async () => assert.equal(rotated.generation, generation); }); +test("migration leaves live legacy registrations unstaged while old workers renew them", () => { + const sqlite = credentialPolicyDatabase({ applyMigrations: false }); + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-worker', + registration_claim = 'legacy-registration', + registration_claim_expires_at = 1000, + updated_at = 500 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + sqlite.exec( + readFileSync( + new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), + "utf8", + ), + ); + + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET registration_claim_expires_at = 3000, updated_at = 2000 + WHERE session_id = 'IS-42' + AND sandbox_id = 'sandbox-1' + AND registration_claim = 'legacy-registration' + `) + .run(); + + assert.equal( + sqlite + .prepare(` + SELECT count(*) AS count + FROM interactive_session_credential_policy_registrations + WHERE state = 'registering' AND registration_claim_expires_at <= 2000 + `) + .get()?.count, + 0, + ); + assert.deepEqual( + sqlite + .prepare(` + SELECT DISTINCT state, registration_generation, registration_claim, + registration_claim_expires_at + FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .all() + .map((row) => ({ ...row })), + [ + { + state: "registering", + registration_generation: "generation:legacy-worker", + registration_claim: "legacy-registration", + registration_claim_expires_at: 3000, + }, + ], + ); +}); + test("post-migration legacy registration claims block new staged generations", async () => { const sqlite = credentialPolicyDatabase(); sqlite From 3f9185ce97c3c0c3f8b805cdda6df7bdc6113f50 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:34:48 +0200 Subject: [PATCH 135/242] fix(terminal): bound confirmation serialization --- internal/terminalws/client.go | 54 ++++++++- internal/terminalws/client_test.go | 173 +++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 5 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index c8afe408..84f18c34 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -23,6 +23,8 @@ const ( maxFrameBytes = 16 * 1024 * 1024 maxErrorBytes = 512 + defaultInputConfirmationTimeout = 5 * time.Second + messageHello = 1 messageWelcome = 2 messageSubscribe = 10 @@ -83,7 +85,9 @@ type Client struct { canInput atomic.Bool lastSize atomic.Uint64 writeMu sync.Mutex - confirmMu sync.Mutex + confirmOnce sync.Once + confirmGate chan struct{} + confirmationTimeout time.Duration stateMu sync.Mutex inputWaiter chan error attachment *terminalAttachment @@ -183,7 +187,12 @@ func Dial(ctx context.Context, endpoint string, sessionID string, options Option return nil, err } conn.SetReadLimit(maxFrameBytes) - client := &Client{conn: conn, sessionID: sessionID, cancel: setupCancel} + client := &Client{ + conn: conn, + sessionID: sessionID, + cancel: setupCancel, + confirmationTimeout: defaultInputConfirmationTimeout, + } client.rememberSize(Size{Cols: options.Cols, Rows: options.Rows}) closeWithError := func(err error) (*Client, error) { setupFinished.Store(true) @@ -278,8 +287,10 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { if !c.supportsInputAcknowledgement { return c.SendInput(ctx, payload) } - c.confirmMu.Lock() - defer c.confirmMu.Unlock() + if err := c.acquireConfirmation(ctx); err != nil { + return err + } + defer c.releaseConfirmation() waiter := make(chan error, 1) if err := c.registerInputWaiter(waiter); err != nil { @@ -307,6 +318,33 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { } } +func (c *Client) acquireConfirmation(ctx context.Context) error { + c.confirmOnce.Do(func() { + c.confirmGate = make(chan struct{}, 1) + }) + select { + case c.confirmGate <- struct{}{}: + if err := ctx.Err(); err != nil { + <-c.confirmGate + return err + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (c *Client) releaseConfirmation() { + <-c.confirmGate +} + +func (c *Client) inputConfirmationTimeout() time.Duration { + if c.confirmationTimeout > 0 { + return c.confirmationTimeout + } + return defaultInputConfirmationTimeout +} + func (c *Client) Resize(ctx context.Context, size Size) error { if size.Cols == 0 || size.Rows == 0 { return nil @@ -349,7 +387,13 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c for { count, err := terminal.Read(buffer) if count > 0 && c.canInput.Load() { - if writeErr := c.SendInputConfirmed(ctx, buffer[:count]); writeErr != nil { + confirmationCtx, confirmationCancel := context.WithTimeout( + ctx, + c.inputConfirmationTimeout(), + ) + writeErr := c.SendInputConfirmed(confirmationCtx, buffer[:count]) + confirmationCancel() + if writeErr != nil { errCh <- writeErr return } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 7c98222e..25602ea4 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -1017,6 +1017,112 @@ func TestSendInputConfirmedSharesOneReaderWithAttach(t *testing.T) { } } +func TestSendInputConfirmedCanCancelWhileWaitingForPreviousConfirmation(t *testing.T) { + firstInput := make(chan struct{}) + releaseFirst := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-cancel-wait", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + for index := range 2 { + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + current, err := decodeFrame(payload) + if err != nil { + t.Error(err) + return + } + if current.messageType != messageInput { + t.Errorf("message type = %d", current.messageType) + return + } + if index == 0 { + close(firstInput) + <-releaseFirst + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-cancel-wait", + payload: accepted, + })); err != nil { + t.Error(err) + return + } + } + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-cancel-wait", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + firstCtx, firstCancel := context.WithTimeout(context.Background(), time.Second) + defer firstCancel() + firstDone := make(chan error, 1) + go func() { + firstDone <- client.SendInputConfirmed(firstCtx, []byte("first\n")) + }() + <-firstInput + + waitCtx, waitCancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer waitCancel() + started := time.Now() + err = client.SendInputConfirmed(waitCtx, []byte("canceled\n")) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v", err) + } + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + t.Fatalf("waiting cancellation took %s", elapsed) + } + + close(releaseFirst) + if err := <-firstDone; err != nil { + t.Fatal(err) + } + finalCtx, finalCancel := context.WithTimeout(context.Background(), time.Second) + defer finalCancel() + if err := client.SendInputConfirmed(finalCtx, []byte("final\n")); err != nil { + t.Fatal(err) + } +} + func TestSendInputConfirmedReturnsImmediatelyForEmptyInput(t *testing.T) { client := &Client{supportsInputAcknowledgement: true} if err := client.SendInputConfirmed(context.Background(), nil); err != nil { @@ -1087,6 +1193,73 @@ func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { } } +func TestAttachBoundsInputConfirmationAndRetiresConnection(t *testing.T) { + inputReceived := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-attach-confirm-timeout", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + close(inputReceived) + _, _, _ = conn.Read(r.Context()) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-attach-confirm-timeout", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + client.confirmationTimeout = 20 * time.Millisecond + + terminal := &readWriter{reader: strings.NewReader("blocked\n")} + started := time.Now() + err = client.Attach(context.Background(), terminal, nil) + if err == nil { + t.Fatal("attachment succeeded without input confirmation") + } + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + t.Fatalf("attachment confirmation timeout took %s", elapsed) + } + <-inputReceived + if err := client.SendInputConfirmed(context.Background(), []byte("second\n")); err == nil { + t.Fatal("timed-out attachment left the connection reusable") + } +} + func TestSendInputConfirmedFallsBackWithoutServerCapability(t *testing.T) { receivedInput := make(chan []byte, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 8bd9aa871498ffd9cdc44ec99846fb5aa5d2992d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:34:50 +0200 Subject: [PATCH 136/242] docs(actions): acknowledge only delivered input --- docs/github-actions-sessions.md | 47 ++++++++++++++++++++++++------- tests/github-actions-docs.test.ts | 22 +++++++++++---- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 8d6e3aa4..5faefc94 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -287,8 +287,9 @@ if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); const magic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); // CFR1 const inputIdDecoder = new TextDecoder(); -const inputDecoder = new TextDecoder("utf-8", { fatal: true }); const encoder = new TextEncoder(); +let pendingInputs = []; +let pendingInputBytes = 0; const framedRunnerPtyUrl = new URL(runnerPtyUrl); framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); const terminal = new WebSocket(framedRunnerPtyUrl); @@ -319,13 +320,38 @@ terminal.addEventListener("message", (event) => { function acceptInput(data) { const input = decodeInput(data); if (!input) return; + pendingInputs.push(input); + pendingInputBytes += input.payload.byteLength; + + const payload = new Uint8Array(pendingInputBytes); + let offset = 0; + for (const pending of pendingInputs) { + payload.set(pending.payload, offset); + offset += pending.payload.byteLength; + } + try { - const text = inputDecoder.decode(input.payload, { stream: true }); - if (text) pty.write(text); - terminal.send(encodeAck(input.inputId, true)); + const text = decodeCompleteUtf8(payload); + if (text === null) return; + pty.write(text); + settlePendingInputs(true); } catch { - terminal.send(encodeAck(input.inputId, false)); + settlePendingInputs(false); + } +} + +function decodeCompleteUtf8(payload) { + const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const text = decoder.decode(payload, { stream: true }); + return encoder.encode(text).byteLength === payload.byteLength ? text : null; +} + +function settlePendingInputs(accepted) { + for (const input of pendingInputs) { + terminal.send(encodeAck(input.inputId, accepted)); } + pendingInputs = []; + pendingInputBytes = 0; } function decodeInput(data) { @@ -380,16 +406,17 @@ terminal.addEventListener("error", () => { Set `CRABFLEET_RUNNER_PTY_URL` to the `runnerPtyUrl` returned by registration. For a PTY API with an asynchronous write callback or promise, await that acceptance signal before sending `encodeAck(..., true)`. Do not acknowledge when -the WebSocket merely queues the input frame. This Node adapter also requires each -input frame to contain complete, valid UTF-8; invalid or split sequences receive -a negative acknowledgement and must be resent on valid boundaries. +the WebSocket merely queues the input frame. This Node adapter buffers a valid +incomplete UTF-8 suffix together with every affected input ID. It writes and +positively acknowledges those frames only after a later frame completes the +sequence. Invalid UTF-8 rejects the buffered group without delivering any of it. The protocol query is consumed during connection setup and is not forwarded as terminal data. There is no capability message or mode transition after the socket opens. Each `CFR1` frame occupies one binary WebSocket message. Payloads are opaque bytes; adapters targeting string-only PTY APIs must either preserve -decoder state across acknowledgements or reject frames that end inside a text -encoding sequence, as the Node example does: +bytes and correlation IDs until a complete character sequence can be delivered, +as the Node example does: | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------ | diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 5d8888cf..cdbad598 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import { test } from "node:test"; -test("the documented Node runner preserves split UTF-8 input frames", async () => { +test("the documented Node runner acknowledges only delivered UTF-8 input", async () => { const [readme, guide] = await Promise.all([ readFile(new URL("../README.md", import.meta.url), "utf8"), readFile(new URL("../docs/github-actions-sessions.md", import.meta.url), "utf8"), @@ -10,12 +10,22 @@ test("the documented Node runner preserves split UTF-8 input frames", async () = assert.match(readme, /complete byte-safe encoder, decoder, and Node PTY runner/); assert.doesNotMatch(readme, /encodeCfr1Output|decodeCfr1Input|encodeCfr1Ack/); - assert.match(guide, /new TextDecoder\("utf-8", \{ fatal: true \}\)/); - assert.match(guide, /inputDecoder\.decode\(input\.payload, \{ stream: true \}\)/); + assert.match(guide, /let pendingInputs = \[\]/); + assert.match(guide, /pendingInputs\.push\(input\)/); + assert.match(guide, /const text = decodeCompleteUtf8\(payload\)/); + assert.match(guide, /if \(text === null\) return/); + assert.match(guide, /pty\.write\(text\);\s+settlePendingInputs\(true\)/); + assert.match(guide, /new TextDecoder\("utf-8", \{ fatal: true, ignoreBOM: true \}\)/); + assert.doesNotMatch(guide, /inputDecoder\.decode/); assert.match(guide, /pty\.onExit\(\(\) => \{/); assert.match(guide, /terminal\.close\(1000, "pty exited"\)/); - const decoder = new TextDecoder("utf-8", { fatal: true }); - assert.equal(decoder.decode(Uint8Array.from([0xf0, 0x9f]), { stream: true }), ""); - assert.equal(decoder.decode(Uint8Array.from([0xa6, 0x80]), { stream: true }), "🦀"); + const decodeCompleteUtf8 = (payload: Uint8Array) => { + const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const text = decoder.decode(payload, { stream: true }); + return new TextEncoder().encode(text).byteLength === payload.byteLength ? text : null; + }; + assert.equal(decodeCompleteUtf8(Uint8Array.from([0xe2, 0x82])), null); + assert.equal(decodeCompleteUtf8(Uint8Array.from([0xe2, 0x82, 0xac])), "\u20ac"); + assert.throws(() => decodeCompleteUtf8(Uint8Array.from([0xe2, 0x28, 0xa1]))); }); From db2bd15077ae99cf362a240246ed226d72598718 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:34:52 +0200 Subject: [PATCH 137/242] fix(vnc): publish fence capability atomically --- .../SDK/Connection/VNCConnection+API.swift | 19 ++++-- .../Connection/VNCConnection+Receive.swift | 5 +- .../RoyalVNCKitTests/AuditFindingsTests.swift | 67 +++++++++++++++++++ 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index ec76fe38..535e064b 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -293,16 +293,27 @@ extension VNCConnection { handleBreakingError(VNCError.protocol(.pixelFormatTransitionTimedOut)) } - func probePixelFormatFenceSupport() { + func publishPixelFormatFenceSupport() -> Bool { framebufferRequestLock.lock() + defer { framebufferRequestLock.unlock() } + guard !state.areFencesSupported else { return false } + cancelPixelFormatFenceNegotiationTimeoutLocked() - guard pixelFormatFenceCapabilityProbePayload == nil, + if pixelFormatFenceCapabilityProbePayload == nil, + state.pixelFormat != nil { + pixelFormatFenceCapabilityProbePayload = Data("royalvnc-pixel-format".utf8) + } + state.areFencesSupported = true + return true + } + + func enqueuePixelFormatFenceSupportProbe() { + framebufferRequestLock.lock() + guard let payload = pixelFormatFenceCapabilityProbePayload, let pixelFormat = state.pixelFormat else { framebufferRequestLock.unlock() return } - let payload = Data("royalvnc-pixel-format".utf8) - pixelFormatFenceCapabilityProbePayload = payload framebufferRequestLock.unlock() enqueueClientToServerMessage( diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift index 5bab385b..9de4541c 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+Receive.swift @@ -207,8 +207,7 @@ private extension VNCConnection { extension VNCConnection { func handleServerFence(_ fence: VNCProtocol.ServerFence) throws { - let first = !state.areFencesSupported - state.areFencesSupported = true + let first = publishPixelFormatFenceSupport() if fence.flags.contains(.request) { let responseFlags = fence.flags.intersection(.blockBefore) @@ -221,7 +220,7 @@ extension VNCConnection { if first { logger.logDebug("Fence supported (server sent ServerFence)") - probePixelFormatFenceSupport() + enqueuePixelFormatFenceSupportProbe() } } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 353d9832..1a8c506f 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -182,6 +182,60 @@ struct AuditFindingsTests { #expect(connection.clientToServerMessageQueue.dequeue() != nil) } + @Test + func publishesFenceNegotiationBeforeExposingSupport() async throws { + let logger = AuditCallbackLogger() + let connection = VNCConnection( + settings: makeSettings(), + logger: logger, + framebufferAllocator: VNCFramebufferMallocAllocator(), + context: nil + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.state.areContinuousUpdatesEnabled = true + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + logger.onDebug = { message in + guard message == "Fence supported (server sent ServerFence)" else { return } + connection.updateColorDepth(.depth8Bit) + } + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + + #expect(connection.state.areFencesSupported) + #expect(connection.pixelFormatFenceCapabilityProbePayload != nil) + #expect(connection.pendingPixelFormatTransition?.depth == 8) + #expect(connection.state.pixelFormat?.depth == 24) + + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityWriter = AuditWritingConnection() + try await capabilityProbe.message.send(connection: capabilityWriter) + let capabilityLength = Int(capabilityWriter.data[8]) + let capabilityPayload = Data(capabilityWriter.data[9..<(9 + capabilityLength)]) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: capabilityPayload + ) + ) + + #expect(connection.pendingPixelFormatTransition == nil) + #expect(connection.clientToServerMessageQueue.dequeue() != nil) + logger.onDebug = nil + connection.cancelFramebufferUpdateScheduling() + } + @Test func startsFenceCapabilityTimeoutAfterDelayedProbeSend() async throws { let connection = VNCConnection( @@ -856,6 +910,19 @@ private final class AuditWritingConnection: NetworkConnectionWriting { } } +private final class AuditCallbackLogger: VNCLogger { + var isDebugLoggingEnabled = false + var onDebug: ((String) -> Void)? + + func logDebug(_ message: @autoclosure () -> String) { + onDebug?(message()) + } + + func logInfo(_ message: String) {} + func logWarning(_ message: String) {} + func logError(_ message: String) {} +} + private struct AuditWriteError: Error {} private final class AuditFailingWritingConnection: NetworkConnectionWriting { From d1808ef2aa3820c99e214418d97b9c3fa2214530 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:34:54 +0200 Subject: [PATCH 138/242] docs(changelog): record rollout boundary fixes --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81e481b8..7a00d944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,13 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments, snapshot SSH connection limits before launching handlers, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. -- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, preserve split UTF-8 across the string-only Node adapter, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, buffer split UTF-8 until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers erasing current ownership fences, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. -- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions that reject premature responses with fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. +- Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions with atomic capability publication, premature-response rejection, and fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. - Exchange full UTF-8 clipboard text between the native Mac viewer, Share This Mac hosts, and any Extended Clipboard-capable VNC server by completing the RoyalVNCKit fork's extension stub, keeping Latin-1 cut text as the fallback and dropping malformed extension bodies without tearing down the connection. From b30814e879ba030738298b86e567da684b2a12ca Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:48:18 +0200 Subject: [PATCH 139/242] fix(macos): skip empty input release retries --- .../Sources/CrabfleetMac/MacRemoteInput.swift | 1 + .../PrivateMacShareTests.swift | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index ec58dbb5..5a078627 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -131,6 +131,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable func releaseAllInput() { eventQueue.async { [self] in + guard !pressedKeysyms.isEmpty || previousButtonMask != 0 else { return } hasPendingRelease = true flushPendingRelease() } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index dde37ea3..b6debdf4 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -1137,6 +1137,31 @@ struct PrivateMacShareTests { #expect(await waitUntilAsync { retainedController == nil }) } + @Test + func emptyInputReleaseDoesNotRetainController() async { + let trust = AccessibilityTrust(granted: false) + var controller: MacRemoteInputController? = MacRemoteInputController( + descriptor: CapturedDisplayDescriptor( + displayID: 1, + displayBounds: CGRect(x: 0, y: 0, width: 100, height: 100), + frameWidth: 100, + frameHeight: 100, + sourcePixelWidth: 100, + sourcePixelHeight: 100 + ), + accessibilityGranted: { trust.isGranted() }, + pendingReleaseRetryDelay: .milliseconds(10) + ) + weak var retainedController = controller + let checksBeforeRelease = trust.checkCount + + controller?.releaseAllInput() + controller = nil + + #expect(await waitUntilAsync { retainedController == nil }) + #expect(trust.checkCount == checksBeforeRelease) + } + @Test func decodesX11UnicodeKeysymsForMacInput() { #expect(MacRemoteInputController.unicodeScalar(for: 0x0100_03BB) == "λ") From cc7ff39de7cfb8f3064385a8a8b05808c52e8fd2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:48:36 +0200 Subject: [PATCH 140/242] fix(terminal): await attachment frame consumer --- internal/terminalws/client.go | 3 ++ internal/terminalws/client_test.go | 81 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 84f18c34..7a035874 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -380,6 +380,7 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c } errCh := make(chan error, 3) + frameConsumerDone := make(chan struct{}) wg.Add(1) go func() { defer wg.Done() @@ -430,6 +431,7 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c wg.Add(1) go func() { defer wg.Done() + defer close(frameConsumerDone) for { select { case <-ctx.Done(): @@ -483,6 +485,7 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c err = ctx.Err() } cancelRead() + <-frameConsumerDone if cancelableRead { wg.Wait() } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 25602ea4..95f901f8 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -1528,6 +1528,51 @@ func TestAttachReturnsWhenContextCancelsAnUncancelableRead(t *testing.T) { close(terminal.release) } +func TestAttachWaitsForFrameConsumerWithUncancelableRead(t *testing.T) { + client := &Client{ + readerDone: make(chan struct{}), + attachmentReady: make(chan struct{}), + } + terminal := newUncancelableReadBlockingWriteTerminal() + ctx, cancel := context.WithCancel(context.Background()) + attachDone := make(chan error, 1) + go func() { + attachDone <- client.Attach(ctx, terminal, nil) + }() + + <-terminal.readStarted + client.stateMu.Lock() + attachment := client.attachment + client.stateMu.Unlock() + if attachment == nil { + t.Fatal("attachment was not registered") + } + delivered := make(chan bool, 1) + go func() { + delivered <- client.deliverAttachment(context.Background(), frame{ + messageType: messageOutput, + payload: []byte("old output\n"), + }) + }() + <-terminal.writeStarted + if !<-delivered { + t.Fatal("output was not delivered to the attachment") + } + + cancel() + select { + case err := <-attachDone: + t.Fatalf("Attach retired before its frame consumer exited: %v", err) + case <-time.After(20 * time.Millisecond): + } + + close(terminal.releaseWrite) + if err := <-attachDone; !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + close(terminal.releaseRead) +} + func TestClientSubscribesReadOnlyAndSuppressesInput(t *testing.T) { acknowledged := make(chan uint32, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1815,6 +1860,42 @@ type uncancelableTerminal struct { bytes.Buffer } +var errBlockedTerminalWrite = errors.New("blocked terminal write") + +type uncancelableReadBlockingWriteTerminal struct { + readStarted chan struct{} + readOnce sync.Once + releaseRead chan struct{} + writeStarted chan struct{} + writeOnce sync.Once + releaseWrite chan struct{} +} + +func newUncancelableReadBlockingWriteTerminal() *uncancelableReadBlockingWriteTerminal { + return &uncancelableReadBlockingWriteTerminal{ + readStarted: make(chan struct{}), + releaseRead: make(chan struct{}), + writeStarted: make(chan struct{}), + releaseWrite: make(chan struct{}), + } +} + +func (terminal *uncancelableReadBlockingWriteTerminal) Read(_ []byte) (int, error) { + terminal.readOnce.Do(func() { + close(terminal.readStarted) + }) + <-terminal.releaseRead + return 0, io.EOF +} + +func (terminal *uncancelableReadBlockingWriteTerminal) Write(payload []byte) (int, error) { + terminal.writeOnce.Do(func() { + close(terminal.writeStarted) + }) + <-terminal.releaseWrite + return 0, errBlockedTerminalWrite +} + func newUncancelableTerminal() *uncancelableTerminal { return &uncancelableTerminal{ started: make(chan struct{}), From 66251dd66b1db2ac8076a9240d5ed041cbff9c45 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:48:38 +0200 Subject: [PATCH 141/242] docs(actions): bound framed runner buffering --- docs/architecture.md | 4 +-- docs/github-actions-sessions.md | 48 ++++++++++++++++++++++--------- tests/github-actions-docs.test.ts | 27 +++++++++++++++++ 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 0b1d3f68..371e97e5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ D1 is canonical for product metadata: ### Durable Objects - `Sandbox` runs first-party Cloudflare Sandbox workspaces. -- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; exact connection-query opt-in selects correlated binary `CFR1` input, output, and acknowledgement frames before socket acceptance. Viewer-bound output is always framed so terminal bytes cannot collide with control traffic. +- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; exact connection-query opt-in selects correlated binary `CFR1` input, output, and acknowledgement frames before socket acceptance. Viewer framing is negotiated independently: opted-in viewers receive `CFR1` terminal, lifecycle, and acknowledgement frames, while legacy viewers retain raw terminal output and JSON control-message fallbacks. There is no `BoardDO` or `RunDO`. General Board/Fleet state is D1 plus REST polling. @@ -137,7 +137,7 @@ Interactive sessions are the live execution plane. Supported paths: - **Built-in Sandbox:** Worker provisions a Cloudflare Sandbox, prepares the repo, starts a Codex-capable shell, and proxies PTY traffic. - **Versioned runtime adapter:** Worker durably registers a tenant-namespaced workspace ID, creates and reconciles the provider workspace, proxies PTY access, mints transient desktop links, and confirms provider release before terminal state. -- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, and either retains legacy raw terminal traffic or opts into correlated `CFR1` browser input/output through the connection URL. Framed runners acknowledge each PTY write before Crabfleet reports input acceptance. +- **GitHub Actions:** OpenClaw automation registers a logical work key; an Actions runner connects outbound to `SessionControlDO`, reports work state, and either retains legacy raw terminal traffic or opts into correlated `CFR1` browser input/output through the connection URL. Framed runners acknowledge each PTY write before Crabfleet reports input acceptance. Viewers separately negotiate framed output and control traffic, with raw terminal output and JSON notices preserved for legacy viewers. Sessions can carry a stable tenant owner, parent/root lineage, purpose, summary, named grants, public share state, delegated control, multiplayer mode, archive metadata, and runtime-specific capability state. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 5faefc94..94575769 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -288,8 +288,12 @@ if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); const magic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); // CFR1 const inputIdDecoder = new TextDecoder(); const encoder = new TextEncoder(); +const maxPendingInputBytes = 16 * 1024; +const maxPendingInputFrames = 32; +const maxPendingInputAgeMs = 1_000; let pendingInputs = []; let pendingInputBytes = 0; +let pendingInputTimer; const framedRunnerPtyUrl = new URL(runnerPtyUrl); framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); const terminal = new WebSocket(framedRunnerPtyUrl); @@ -305,8 +309,8 @@ const pty = spawn(process.env.SHELL || "/bin/bash", [], { env: process.env, }); -pty.onData((output) => { - terminal.send(encodeOutput(output)); +pty.onData((outputText) => { + terminal.send(encodeUtf8Output(outputText)); }); pty.onExit(() => { @@ -322,6 +326,13 @@ function acceptInput(data) { if (!input) return; pendingInputs.push(input); pendingInputBytes += input.payload.byteLength; + if (pendingInputs.length === 1) { + pendingInputTimer = setTimeout(() => settlePendingInputs(false), maxPendingInputAgeMs); + } + if (pendingInputBytes > maxPendingInputBytes || pendingInputs.length > maxPendingInputFrames) { + settlePendingInputs(false); + return; + } const payload = new Uint8Array(pendingInputBytes); let offset = 0; @@ -347,6 +358,8 @@ function decodeCompleteUtf8(payload) { } function settlePendingInputs(accepted) { + if (pendingInputTimer) clearTimeout(pendingInputTimer); + pendingInputTimer = undefined; for (const input of pendingInputs) { terminal.send(encodeAck(input.inputId, accepted)); } @@ -384,8 +397,8 @@ function encodeAck(inputId, accepted) { return frame; } -function encodeOutput(output) { - const payload = encoder.encode(output); +function encodeUtf8Output(outputText) { + const payload = encoder.encode(outputText); const frame = new Uint8Array(6 + payload.byteLength); frame.set(magic); frame[4] = 0x04; @@ -410,13 +423,17 @@ the WebSocket merely queues the input frame. This Node adapter buffers a valid incomplete UTF-8 suffix together with every affected input ID. It writes and positively acknowledges those frames only after a later frame completes the sequence. Invalid UTF-8 rejects the buffered group without delivering any of it. +The adapter also rejects the whole pending group when it exceeds 16 KiB, 32 +frames, or one second, bounding memory, copy work, and acknowledgement latency. The protocol query is consumed during connection setup and is not forwarded as terminal data. There is no capability message or mode transition after the -socket opens. Each `CFR1` frame occupies one binary WebSocket message. Payloads -are opaque bytes; adapters targeting string-only PTY APIs must either preserve -bytes and correlation IDs until a complete character sequence can be delivered, -as the Node example does: +socket opens. Each `CFR1` frame occupies one binary WebSocket message. At the +wire level, input and output payloads are opaque terminal bytes. The example is +deliberately a UTF-8 text adapter because `@lydell/node-pty` exposes input and +output as JavaScript strings: it rejects input that is not complete valid UTF-8 +and encodes each output string as UTF-8. Deployments that require lossless +arbitrary PTY bytes must use a byte-oriented PTY adapter instead of this example. | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------ | @@ -443,12 +460,17 @@ Properties: send raw output. - Framed runners add the exact protocol query before connecting, receive framed input immediately, and wrap every output payload in a `0x04` frame. +- Framed viewers add `viewerProtocol=cfr1-framed-io-v1` before connecting. They + receive `CFR1` output, lifecycle, and acknowledgement frames regardless of the + runner's mode. +- Legacy viewers omit that query. They receive raw terminal output plus JSON + lifecycle and input-acknowledgement messages for compatibility. - Negotiated input produces `input-accepted` only after the correlated runner acknowledgement. Legacy input reports acceptance after relay delivery. -- Runner lifecycle events are typed binary frames even while no runner is - connected. -- The relay frames legacy output internally, so raw bytes beginning with - `CFR1` cannot collide with acknowledgements or lifecycle events. +- Framed viewer lifecycle events remain typed binary frames while no runner is + connected. Legacy viewers receive the JSON fallback. +- When runner and viewer modes differ, the relay wraps or unwraps terminal + output at the viewer boundary. The relay does not interpret Codex JSON-RPC. The runner-side integration decides how accepted terminal input maps to model steering. @@ -493,7 +515,7 @@ ClawSweeper integration, the runner: 1. Accepts the framed bytes into its input handler and acknowledges that input ID. 2. Collects printable input until Enter. -3. Echoes `[steer] ` to the terminal as raw output. +3. Echoes `[steer] ` to the terminal as UTF-8 terminal output. 4. Calls Codex `turn/steer` with the active thread and expected turn ID. 5. Reports rejection or no-active-turn conditions in the terminal. diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index cdbad598..6587e893 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -15,8 +15,21 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.match(guide, /const text = decodeCompleteUtf8\(payload\)/); assert.match(guide, /if \(text === null\) return/); assert.match(guide, /pty\.write\(text\);\s+settlePendingInputs\(true\)/); + assert.match(guide, /const maxPendingInputBytes = 16 \* 1024/); + assert.match(guide, /const maxPendingInputFrames = 32/); + assert.match(guide, /const maxPendingInputAgeMs = 1_000/); + assert.match(guide, /setTimeout\(\(\) => settlePendingInputs\(false\), maxPendingInputAgeMs\)/); + assert.match(guide, /pendingInputBytes > maxPendingInputBytes/); + assert.match(guide, /pendingInputs\.length > maxPendingInputFrames/); + assert.match(guide, /clearTimeout\(pendingInputTimer\)/); assert.match(guide, /new TextDecoder\("utf-8", \{ fatal: true, ignoreBOM: true \}\)/); assert.doesNotMatch(guide, /inputDecoder\.decode/); + assert.match(guide, /pty\.onData\(\(outputText\) => \{/); + assert.match(guide, /encodeUtf8Output\(outputText\)/); + assert.match(guide, /deliberately a UTF-8 text adapter/); + assert.match(guide, /lossless\s+arbitrary PTY bytes must use a byte-oriented PTY adapter/); + assert.match(guide, /Framed viewers add\s+`viewerProtocol=cfr1-framed-io-v1`/); + assert.match(guide, /Legacy viewers omit that query/); assert.match(guide, /pty\.onExit\(\(\) => \{/); assert.match(guide, /terminal\.close\(1000, "pty exited"\)/); @@ -29,3 +42,17 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.equal(decodeCompleteUtf8(Uint8Array.from([0xe2, 0x82, 0xac])), "\u20ac"); assert.throws(() => decodeCompleteUtf8(Uint8Array.from([0xe2, 0x28, 0xa1]))); }); + +test("the architecture documents negotiated and legacy viewer output", async () => { + const architecture = await readFile(new URL("../docs/architecture.md", import.meta.url), "utf8"); + + assert.match(architecture, /Viewer framing is negotiated independently/); + assert.match( + architecture, + /opted-in viewers receive `CFR1` terminal, lifecycle, and acknowledgement/, + ); + assert.match( + architecture, + /legacy viewers retain raw terminal output and JSON control-message fallbacks/, + ); +}); From 5b71ba132d1e731cc30e22cd6d22fa9709f98acb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:49:24 +0200 Subject: [PATCH 142/242] fix(credentials): fence staged rotations from legacy writers --- ...credential_policy_registration_staging.sql | 44 +++++++ .../sandbox-credential-policy-repository.ts | 11 ++ ...ndbox-credential-policy-repository.test.ts | 108 +++++++++++++++--- 3 files changed, 145 insertions(+), 18 deletions(-) diff --git a/migrations/0034_credential_policy_registration_staging.sql b/migrations/0034_credential_policy_registration_staging.sql index 1455ea6b..2e4f329b 100644 --- a/migrations/0034_credential_policy_registration_staging.sql +++ b/migrations/0034_credential_policy_registration_staging.sql @@ -41,6 +41,50 @@ CREATE INDEX IF NOT EXISTS idx_credential_policy_registration_expiry updated_at ); +-- Once a new worker stages a rotation, legacy workers must not claim, promote, +-- or remove the policy rows underneath its rollback snapshot. Cleanup may +-- still transition an older generation into cleanup_pending, but its rows stay +-- fenced until the staged registration is removed. +CREATE TRIGGER IF NOT EXISTS fence_staged_credential_policy_insert +BEFORE INSERT ON interactive_session_credential_policies +WHEN NEW.state != 'cleanup_pending' + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = NEW.session_id + AND staged.sandbox_id = NEW.sandbox_id + AND staged.registration_generation != NEW.registration_generation + ) +BEGIN + SELECT RAISE(IGNORE); +END; + +CREATE TRIGGER IF NOT EXISTS fence_staged_credential_policy_update +BEFORE UPDATE ON interactive_session_credential_policies +WHEN NOT (OLD.state != 'cleanup_pending' AND NEW.state = 'cleanup_pending') + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = NEW.session_id + AND staged.sandbox_id = NEW.sandbox_id + AND staged.registration_generation != NEW.registration_generation + ) +BEGIN + SELECT RAISE(IGNORE); +END; + +CREATE TRIGGER IF NOT EXISTS fence_staged_credential_policy_delete +BEFORE DELETE ON interactive_session_credential_policies +WHEN EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = OLD.session_id + AND staged.sandbox_id = OLD.sandbox_id +) +BEGIN + SELECT RAISE(IGNORE); +END; + -- Legacy workers renew registration claims only in the policy table. Leaving -- those claims unstaged prevents the new scanner from recovering a stale -- snapshot while the legacy registration is still live. diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 4ae96723..1452627c 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -188,6 +188,16 @@ export function sandboxCredentialPolicyRefQueries( now: number, authorizationCondition: RawBuilder, ): CompilableQuery[] { + const stagedWriteAllowed = + state === "cleanup_pending" + ? sql`1 = 1` + : sql`NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = ${sessionId} + AND staged.sandbox_id = ${sandboxId} + AND staged.registration_generation != ${generation} + )`; return sandboxLookupIds(env, sandboxId).map( (lookupId) => sql` INSERT INTO interactive_session_credential_policies ( @@ -221,6 +231,7 @@ export function sandboxCredentialPolicyRefQueries( ${now}, ${now} WHERE ${authorizationCondition} + AND ${stagedWriteAllowed} ON CONFLICT(session_id, sandbox_id, lookup_id) DO UPDATE SET state = CASE WHEN interactive_session_credential_policies.state = 'cleanup_pending' diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index f125e920..8211682c 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -511,7 +511,7 @@ test("post-migration legacy registration claims block new staged generations", a ); }); -test("legacy claims created after staging block promotion without interleaving generations", async () => { +test("staged rotations fence old-worker writes until the staged row is removed", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); const staged = await beginSandboxCredentialPolicyRegistration( @@ -520,48 +520,120 @@ test("legacy claims created after staging block promotion without interleaving g "sandbox-1", ownershipFence, ); - sqlite + const legacyClaim = sqlite .prepare(` UPDATE interactive_session_credential_policies SET state = 'registering', registration_generation = 'generation:legacy-race', registration_claim = 'legacy-race-claim', - registration_claim_expires_at = ? + registration_claim_expires_at = ?, + updated_at = 2000 WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' `) .run(Number.MAX_SAFE_INTEGER); + assert.equal(legacyClaim.changes, 0); - assert.equal( - await finishSandboxCredentialPolicyRegistration( - env, - "IS-42", - "sandbox-1", - staged, - ownershipFence, - ), - false, - ); assert.deepEqual( activeCredentialPolicyRows(sqlite).map((row) => ({ generation: row.registration_generation, state: row.state, })), [ - { generation: "generation:legacy-race", state: "registering" }, - { generation: "generation:legacy-race", state: "registering" }, + { generation: "generation:existing", state: "active" }, + { generation: "generation:existing", state: "active" }, ], ); + + await abandonSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + "simulated registration failure after rollback", + ); + + const legacyCompletion = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'active', + registration_claim = NULL, + registration_claim_expires_at = NULL, + updated_at = 3000 + WHERE session_id = 'IS-42' + AND sandbox_id = 'sandbox-1' + AND registration_generation = 'generation:legacy-race' + AND registration_claim = 'legacy-race-claim' + `) + .run(); + assert.equal(legacyCompletion.changes, 0); + + const legacyDelete = sqlite + .prepare(` + DELETE FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + assert.equal(legacyDelete.changes, 0); + + const legacyInsert = sqlite + .prepare(` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at + ) VALUES ( + 'IS-42', + 'sandbox-1', + 'legacy-extra', + 'registering', + 'generation:legacy-race', + 'legacy-race-claim', + ?, + 2000, + 2000 + ) + `) + .run(Number.MAX_SAFE_INTEGER); + assert.equal(legacyInsert.changes, 0); + assert.equal( sqlite .prepare(` - SELECT registration_generation + SELECT state FROM interactive_session_credential_policy_registrations WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' `) - .get()?.registration_generation, - staged.generation, + .get()?.state, + "cleanup_pending", ); + + sqlite + .prepare(` + DELETE FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + const postFenceClaim = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-after-fence', + registration_claim = 'legacy-after-fence-claim', + registration_claim_expires_at = ?, + updated_at = 4000 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + assert.equal(postFenceClaim.changes, 2); }); test("partial credential-policy rotation failure preserves the prior active generation", async () => { From 0d0c32c17016c53d3d7cb22f21060cd6df14e373 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:49:54 +0200 Subject: [PATCH 143/242] docs(changelog): record final audit repairs --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a00d944..f86f07d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. -- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, buffer split UTF-8 until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown without scheduling retries when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions with atomic capability publication, premature-response rejection, and fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. From 999797cc2daf9fa1517f659646f903679a56e08f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:02:34 +0200 Subject: [PATCH 144/242] fix(credentials): repair incomplete lookup sets --- .../0036_credential_policy_lookup_repair.sql | 50 ++++ ...-credential-policy-registration-service.ts | 122 ++++++++- .../sandbox-credential-policy-repository.ts | 171 ++++++++++++ ...ndbox-credential-policy-repository.test.ts | 246 ++++++++++++++++++ 4 files changed, 584 insertions(+), 5 deletions(-) create mode 100644 migrations/0036_credential_policy_lookup_repair.sql diff --git a/migrations/0036_credential_policy_lookup_repair.sql b/migrations/0036_credential_policy_lookup_repair.sql new file mode 100644 index 00000000..fdc4437e --- /dev/null +++ b/migrations/0036_credential_policy_lookup_repair.sql @@ -0,0 +1,50 @@ +ALTER TABLE interactive_session_credential_policy_registrations + ADD COLUMN repair_generation TEXT; + +DROP TRIGGER IF EXISTS fence_staged_credential_policy_insert; +DROP TRIGGER IF EXISTS fence_staged_credential_policy_update; + +CREATE TRIGGER fence_staged_credential_policy_insert +BEFORE INSERT ON interactive_session_credential_policies +WHEN NEW.state != 'cleanup_pending' + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = NEW.session_id + AND staged.sandbox_id = NEW.sandbox_id + AND staged.registration_generation != NEW.registration_generation + AND NOT ( + staged.state = 'registering' + AND staged.repair_generation = NEW.registration_generation + AND staged.registration_claim = NEW.registration_claim + AND staged.registration_claim_expires_at = NEW.registration_claim_expires_at + AND NEW.state = 'registering' + ) + ) +BEGIN + SELECT RAISE(IGNORE); +END; + +CREATE TRIGGER fence_staged_credential_policy_update +BEFORE UPDATE ON interactive_session_credential_policies +WHEN NOT (OLD.state != 'cleanup_pending' AND NEW.state = 'cleanup_pending') + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = NEW.session_id + AND staged.sandbox_id = NEW.sandbox_id + AND staged.registration_generation != NEW.registration_generation + AND NOT ( + staged.state = 'registering' + AND staged.repair_generation = NEW.registration_generation + AND staged.registration_claim = OLD.registration_claim + AND staged.registration_claim_expires_at = OLD.registration_claim_expires_at + AND OLD.state = 'registering' + AND NEW.state = 'active' + AND NEW.registration_claim IS NULL + AND NEW.registration_claim_expires_at IS NULL + ) + ) +BEGIN + SELECT RAISE(IGNORE); +END; diff --git a/src/worker/sandbox-credential-policy-registration-service.ts b/src/worker/sandbox-credential-policy-registration-service.ts index a2b19c1c..4e4e3ca4 100644 --- a/src/worker/sandbox-credential-policy-registration-service.ts +++ b/src/worker/sandbox-credential-policy-registration-service.ts @@ -8,9 +8,12 @@ import { deferSandboxCredentialPolicyRollback, existingSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, + incompleteSandboxCredentialPolicyGeneration, recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, + repairSandboxCredentialPolicyReferences, renewSandboxCredentialPolicyRegistration, + stageSandboxCredentialPolicyReferenceRepair, standaloneSandboxPolicyExpiresAt, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; @@ -35,6 +38,110 @@ import type { InteractiveSession } from "./session-model.ts"; type RestoreSandboxCredentialPolicyRollback = typeof restoreSandboxCredentialPolicyRollback; +async function repairIncompleteSandboxCredentialPolicyLookupSet( + env: RuntimeEnv, + stub: Pick, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const repairGeneration = await incompleteSandboxCredentialPolicyGeneration( + env, + sessionId, + sandboxId, + ); + if (!repairGeneration) return null; + const records = await Promise.all( + registration.lookupIds.map(async (lookupId) => { + const response = await stub.fetch( + `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, + ); + if (response.status === 404) return null; + if (!response.ok) throw new Error("sandbox credential policy repair snapshot failed"); + const generation = response.headers.get("x-crabfleet-policy-generation"); + const policy = (await response.json()) as SandboxCredentialPolicy; + if ( + generation !== repairGeneration || + policy.sessionId !== sessionId || + policy.sandboxId !== lookupId + ) { + throw new Error("sandbox credential policy repair snapshot is inconsistent"); + } + return { generation, policy }; + }), + ); + const surviving = records.filter((record) => record !== null); + if (surviving.length === 0) return null; + const source = surviving[0]!.policy; + if ( + surviving.some( + (record) => + JSON.stringify({ ...record.policy, sandboxId: "" }) !== + JSON.stringify({ ...source, sandboxId: "" }), + ) + ) { + throw new Error("sandbox credential policy repair snapshot is inconsistent"); + } + let registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + ); + if ( + !registrationExpiresAt || + !(await stageSandboxCredentialPolicyReferenceRepair( + env, + sessionId, + sandboxId, + registration, + repairGeneration, + ownershipFence, + )) + ) { + throw new Error("sandbox credential policy registration claim was revoked"); + } + const repairExpiresAt = registrationExpiresAt - 1; + for (const [index, lookupId] of registration.lookupIds.entries()) { + if (records[index]) continue; + const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: repairGeneration, + registrationClaim: registration.claim, + registrationExpiresAt: repairExpiresAt, + policy: { ...source, sandboxId: lookupId }, + } satisfies StoredSandboxCredentialPolicy), + headers: { "content-type": "application/json" }, + }); + if (!response.ok) throw new Error("sandbox credential policy lookup repair failed"); + } + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + ); + if ( + !registrationExpiresAt || + !(await repairSandboxCredentialPolicyReferences( + env, + sessionId, + sandboxId, + registration, + repairGeneration, + ownershipFence, + registrationExpiresAt, + )) + ) { + throw new Error("sandbox credential policy lookup references were not repaired"); + } + return repairGeneration; +} + export async function restoreSandboxCredentialPolicyRollbackIfOwned( env: RuntimeEnv, stub: Pick, @@ -81,11 +188,16 @@ export async function registerSandboxCredentialPolicy( let rollbackJson: string | null = null; let registrationWriteStarted = false; try { - const activeGeneration = await activeSandboxCredentialPolicyGeneration( - env, - session.id, - sandboxId, - ); + const activeGeneration = + (await activeSandboxCredentialPolicyGeneration(env, session.id, sandboxId)) ?? + (await repairIncompleteSandboxCredentialPolicyLookupSet( + env, + stub, + session.id, + sandboxId, + registration, + ownershipFence, + )); const rollback = await captureSandboxCredentialPolicyRollback( stub, registration.lookupIds, diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 1452627c..3e889b94 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -107,6 +107,36 @@ export async function activeSandboxCredentialPolicyGeneration( return generation; } +export async function incompleteSandboxCredentialPolicyGeneration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, +): Promise { + const rows = await database(env) + .selectFrom("interactive_session_credential_policies") + .select(["lookup_id", "state", "registration_generation", "registration_claim"]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .execute(); + const expected = new Set(sandboxLookupIds(env, sandboxId)); + const generation = rows[0]?.registration_generation; + if ( + rows.length === 0 || + rows.length >= expected.size || + !isCurrentCredentialPolicyGeneration(generation) || + rows.some( + (row) => + !expected.has(row.lookup_id) || + row.state !== "active" || + row.registration_generation !== generation || + row.registration_claim !== null, + ) + ) { + return null; + } + return generation; +} + export async function sandboxCredentialPolicyHasDurableOwner( env: RuntimeEnv, lookupId: string, @@ -512,6 +542,7 @@ export function sandboxCredentialPolicyRegistrationQueries( registration_generation = excluded.registration_generation, registration_claim = excluded.registration_claim, registration_claim_expires_at = excluded.registration_claim_expires_at, + repair_generation = NULL, rollback_policies_json = NULL, last_error = NULL, cleanup_claim = NULL, @@ -679,6 +710,146 @@ export async function recordSandboxCredentialPolicyRollback( return Number(recorded.numUpdatedRows ?? 0n) === 1; } +export async function stageSandboxCredentialPolicyReferenceRepair( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + repairGeneration: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const now = Date.now(); + const staged = await sql<{ repair_generation: string }>` + UPDATE interactive_session_credential_policy_registrations + SET repair_generation = ${repairGeneration}, updated_at = ${now} + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'registering' + AND registration_generation = ${registration.generation} + AND registration_claim = ${registration.claim} + AND registration_claim_expires_at > ${now} + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + RETURNING repair_generation + `.execute(database(env)); + return staged.rows.length === 1 && staged.rows[0]?.repair_generation === repairGeneration; +} + +export async function repairSandboxCredentialPolicyReferences( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + repairGeneration: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + registrationExpiresAt: number, +): Promise { + const now = Date.now(); + const repairAuthorized = sql` + EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = ${sessionId} + AND staged.sandbox_id = ${sandboxId} + AND staged.state = 'registering' + AND staged.registration_generation = ${registration.generation} + AND staged.registration_claim = ${registration.claim} + AND staged.registration_claim_expires_at = ${registrationExpiresAt} + AND staged.registration_claim_expires_at > ${now} + AND staged.repair_generation = ${repairGeneration} + ) + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS surviving + WHERE surviving.session_id = ${sessionId} + AND surviving.sandbox_id = ${sandboxId} + AND surviving.state = 'active' + AND surviving.registration_generation = ${repairGeneration} + AND surviving.registration_claim IS NULL + ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS conflicting + WHERE conflicting.session_id = ${sessionId} + AND conflicting.sandbox_id = ${sandboxId} + AND ( + conflicting.lookup_id NOT IN (${sql.join(registration.lookupIds)}) + OR conflicting.registration_generation != ${repairGeneration} + OR NOT ( + ( + conflicting.state = 'active' + AND conflicting.registration_claim IS NULL + ) + OR ( + conflicting.state = 'registering' + AND conflicting.registration_claim = ${registration.claim} + AND conflicting.registration_claim_expires_at = ${registrationExpiresAt} + ) + ) + ) + ) + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + `; + const inserts = registration.lookupIds.map( + (lookupId) => sql` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + attempt_count, + last_attempt_at, + last_error, + cleanup_claim, + cleanup_claim_expires_at, + created_at, + updated_at + ) + SELECT + ${sessionId}, + ${sandboxId}, + ${lookupId}, + 'registering', + ${repairGeneration}, + ${registration.claim}, + ${registrationExpiresAt}, + 0, + NULL, + NULL, + NULL, + NULL, + ${now}, + ${now} + WHERE ${repairAuthorized} + ON CONFLICT(session_id, sandbox_id, lookup_id) DO NOTHING + `, + ); + const promotions = registration.lookupIds.map( + (lookupId) => sql` + UPDATE interactive_session_credential_policies + SET + state = 'active', + registration_claim = NULL, + registration_claim_expires_at = NULL, + updated_at = ${now} + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND lookup_id = ${lookupId} + AND state = 'registering' + AND registration_generation = ${repairGeneration} + AND registration_claim = ${registration.claim} + AND registration_claim_expires_at = ${registrationExpiresAt} + AND ${repairAuthorized} + `, + ); + await executeBatch(env, [...inserts, ...promotions]); + return ( + (await activeSandboxCredentialPolicyGeneration(env, sessionId, sandboxId)) === repairGeneration + ); +} + export async function deferSandboxCredentialPolicyRollback( env: RuntimeEnv, sessionId: string, diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 8211682c..651b7366 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -10,16 +10,22 @@ import { claimSandboxCredentialPolicyRegistrationRecovery, currentSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, + incompleteSandboxCredentialPolicyGeneration, recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, + repairSandboxCredentialPolicyReferences, renewSandboxCredentialPolicyRegistration, sandboxCredentialPolicyRegistrationQueries, sandboxLookupIds, + stageSandboxCredentialPolicyReferenceRepair, type SandboxCredentialPolicyOwnershipFence, } from "../src/worker/sandbox-credential-policy-repository.ts"; +import { captureSandboxCredentialPolicyRollback } from "../src/worker/sandbox-credential-policy-rollback.ts"; +import { credentialPolicyRegistrationAccepted } from "../src/credential-policy-fence.ts"; import { database } from "../src/worker/database.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import type { SandboxCredentialPolicyRegistration } from "../src/worker/session-control-policy.ts"; +import type { StoredSandboxCredentialPolicy } from "../src/worker/session-control-policy.ts"; type PreparedStatement = { sql: string; @@ -162,6 +168,12 @@ function credentialPolicyDatabase(options: { applyMigrations?: boolean } = {}): "utf8", ), ); + db.exec( + readFileSync( + new URL("../migrations/0036_credential_policy_lookup_repair.sql", import.meta.url), + "utf8", + ), + ); } return db; } @@ -912,6 +924,240 @@ test("active credential-policy generation requires every exact lookup row", asyn assert.equal(await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), null); }); +test("credential refresh repairs an incomplete legacy lookup set before rotation", async () => { + const sqlite = credentialPolicyDatabase(); + sqlite + .prepare(` + DELETE FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' AND lookup_id = 'do-1' + `) + .run(); + const now = Date.now(); + const policies = new Map([ + [ + "sandbox-1", + { + generation: "generation:existing", + registrationClaim: "registration:legacy", + registrationExpiresAt: now + 30_000, + policy: { + allowedHosts: [], + githubCredentialSource: "none", + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: "sandbox-1", + sessionId: "IS-42", + }, + }, + ], + ]); + const stub = { + async fetch(input: RequestInfo | URL, init?: RequestInit): Promise { + const url = new URL(String(input)); + const egress = url.pathname.match(/^\/api\/session-control\/egress\/([^/]+)$/); + if (egress && (!init?.method || init.method === "GET")) { + const current = policies.get(decodeURIComponent(egress[1] ?? "")); + return current + ? Response.json(current.policy, { + headers: { "x-crabfleet-policy-generation": current.generation }, + }) + : Response.json({ error: "not found" }, { status: 404 }); + } + if (url.pathname === "/api/session-control/register" && init?.method === "POST") { + const incoming = JSON.parse(String(init.body)) as StoredSandboxCredentialPolicy; + const current = policies.get(incoming.policy.sandboxId); + if (!credentialPolicyRegistrationAccepted(current, undefined, incoming, Date.now())) { + return Response.json({ error: "conflict" }, { status: 409 }); + } + policies.set(incoming.policy.sandboxId, incoming); + return Response.json({ ok: true }); + } + return Response.json({ error: "not found" }, { status: 404 }); + }, + }; + const env = sqliteRuntimeEnv(sqlite); + + assert.equal(await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), null); + assert.equal( + await incompleteSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), + "generation:existing", + ); + await assert.rejects( + captureSandboxCredentialPolicyRollback(stub, sandboxLookupIds(env, "sandbox-1"), null, "IS-42"), + /no durable rollback owner/, + ); + + const registration = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const repairGeneration = await incompleteSandboxCredentialPolicyGeneration( + env, + "IS-42", + "sandbox-1", + ); + assert.equal(repairGeneration, "generation:existing"); + let registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.equal( + await stageSandboxCredentialPolicyReferenceRepair( + env, + "IS-42", + "sandbox-1", + registration, + repairGeneration, + ownershipFence, + ), + true, + ); + const legacyUpdate = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_claim = 'registration:legacy-race', + registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + assert.equal(legacyUpdate.changes, 0); + const legacyInsert = sqlite + .prepare(` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at + ) VALUES ( + 'IS-42', + 'sandbox-1', + 'do-1', + 'registering', + 'generation:existing', + 'registration:legacy-race', + ?, + 1, + 1 + ) + `) + .run(Number.MAX_SAFE_INTEGER); + assert.equal(legacyInsert.changes, 0); + assert.equal( + ( + await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: repairGeneration, + registrationClaim: registration.claim, + registrationExpiresAt: registrationExpiresAt - 1, + policy: { ...policies.get("sandbox-1")!.policy, sandboxId: "do-1" }, + } satisfies StoredSandboxCredentialPolicy), + }) + ).ok, + true, + ); + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.equal( + await repairSandboxCredentialPolicyReferences( + env, + "IS-42", + "sandbox-1", + registration, + repairGeneration, + ownershipFence, + registrationExpiresAt, + ), + true, + ); + const rollback = await captureSandboxCredentialPolicyRollback( + stub, + registration.lookupIds, + repairGeneration, + "IS-42", + ); + assert.equal(rollback.length, 2); + assert.equal( + await recordSandboxCredentialPolicyRollback( + env, + "IS-42", + "sandbox-1", + registration, + rollback, + ownershipFence, + ), + true, + ); + for (const lookupId of registration.lookupIds) { + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.equal( + ( + await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: registration.generation, + registrationClaim: registration.claim, + registrationExpiresAt, + policy: { ...policies.get(lookupId)!.policy, sandboxId: lookupId }, + } satisfies StoredSandboxCredentialPolicy), + }) + ).ok, + true, + ); + } + assert.equal( + await finishSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ), + true, + ); + + const generation = await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"); + assert.match(generation ?? "", /^generation:/); + assert.notEqual(generation, "generation:existing"); + assert.deepEqual( + [...policies.entries()].map(([lookupId, policy]) => ({ + lookupId, + generation: policy.generation, + sandboxId: policy.policy.sandboxId, + })), + [ + { lookupId: "sandbox-1", generation, sandboxId: "sandbox-1" }, + { lookupId: "do-1", generation, sandboxId: "do-1" }, + ], + ); +}); + test("recording active policy refs promotes then upserts every lookup under one fence", async () => { const rows = [ { From a238dd84dd0023e73de822026355ff99e06f3e91 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:03:04 +0200 Subject: [PATCH 145/242] fix(terminal): bound serialized input backlog --- src/worker/terminal-hub.ts | 153 ++++++++++++++++++++++++------------- tests/terminal-hub.test.ts | 145 +++++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+), 54 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 344b7b34..c0ae95a5 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -28,7 +28,10 @@ import type { User } from "./models.ts"; import type { InteractiveSession } from "./session-model.ts"; const encoder = new TextEncoder(); -const terminalFrameLimits = { maxFrameBytes: 16 * 1024 * 1024 }; +const terminalMaxFrameBytes = 16 * 1024 * 1024; +const terminalFrameLimits = { maxFrameBytes: terminalMaxFrameBytes }; +const terminalInputQueueMaxBytes = terminalMaxFrameBytes; +const terminalInputQueueMaxFrames = 32; const terminalInputAcknowledgementTimeoutMs = 5_000; type PendingTerminalInputAcknowledgement = { @@ -57,6 +60,9 @@ export type TerminalHubSubscription = { rows: number; inputAcknowledgements: boolean; inputQueue: Promise; + inputQueueBytes: number; + inputQueueFrames: number; + inputQueueRejectionScheduled: boolean; pendingInputAcknowledgements: Map; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; @@ -235,72 +241,90 @@ export class TerminalHub { return; } if (frame.type === TerminalMessageType.Input || frame.type === TerminalMessageType.Key) { + if ( + subscription.inputQueueFrames >= terminalInputQueueMaxFrames || + subscription.inputQueueBytes + frame.payload.byteLength > terminalInputQueueMaxBytes + ) { + scheduleTerminalInputBacklogRejection(server, subscription, frame.sessionId); + return; + } + subscription.inputQueueBytes += frame.payload.byteLength; + subscription.inputQueueFrames += 1; subscription.inputQueue = subscription.inputQueue .catch(() => undefined) .then(async () => { - const canInput = await subscription.canInput(); - updateTerminalInputCapability(server, subscription, canInput); - if (!canInput) { - sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { - type: "input-rejected", - error: "terminal control is not granted", - }); - return; - } - if (subscription.upstream.readyState !== WebSocket.OPEN) { - sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: "terminal upstream is not open", - }); - return; - } - const inputs = await this.dependencies.inputPayloads( - subscription, - user, - frame.payload, - ); - const acknowledgements: PendingTerminalInputAcknowledgement[] = []; - for (const [index, input] of inputs.entries()) { - if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); - if ( - subscriptions.get(frame.sessionId) !== subscription || - subscription.upstream.readyState !== WebSocket.OPEN - ) { + try { + const canInput = await subscription.canInput(); + updateTerminalInputCapability(server, subscription, canInput); + if (!canInput) { + sendTerminalJson(server, TerminalMessageType.Event, frame.sessionId, { + type: "input-rejected", + error: "terminal control is not granted", + }); + return; + } + if (subscription.upstream.readyState !== WebSocket.OPEN) { sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { error: "terminal upstream is not open", }); return; } - const inputId = subscription.inputAcknowledgements - ? createGitHubActionsRelayInputId() - : null; - const acknowledgement = inputId - ? beginTerminalInputAcknowledgement(subscription, inputId) - : null; - if (acknowledgement) acknowledgements.push(acknowledgement); - try { - subscription.upstream.send( - inputId ? encodeGitHubActionsRelayInput(inputId, input) : input, - ); - } catch { - if (acknowledgement) { - completeTerminalInputAcknowledgement(subscription, acknowledgement.inputId, { - inputId: acknowledgement.inputId, - accepted: false, + const inputs = await this.dependencies.inputPayloads( + subscription, + user, + frame.payload, + ); + const acknowledgements: PendingTerminalInputAcknowledgement[] = []; + for (const [index, input] of inputs.entries()) { + if (index > 0) await sleep(index === inputs.length - 1 ? 80 : 2); + if ( + subscriptions.get(frame.sessionId) !== subscription || + subscription.upstream.readyState !== WebSocket.OPEN + ) { + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { + error: "terminal upstream is not open", + }); + return; + } + const inputId = subscription.inputAcknowledgements + ? createGitHubActionsRelayInputId() + : null; + const acknowledgement = inputId + ? beginTerminalInputAcknowledgement(subscription, inputId) + : null; + if (acknowledgement) acknowledgements.push(acknowledgement); + try { + subscription.upstream.send( + inputId ? encodeGitHubActionsRelayInput(inputId, input) : input, + ); + } catch { + if (acknowledgement) { + completeTerminalInputAcknowledgement( + subscription, + acknowledgement.inputId, + { + inputId: acknowledgement.inputId, + accepted: false, + error: "terminal upstream send failed", + }, + ); + break; + } + sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { error: "terminal upstream send failed", }); - break; + return; } - sendTerminalJson(server, TerminalMessageType.Error, frame.sessionId, { - error: "terminal upstream send failed", - }); - return; } + await reportTerminalInputCompletion( + server, + frame.sessionId, + acknowledgements.map((acknowledgement) => acknowledgement.promise), + ); + } finally { + subscription.inputQueueBytes -= frame.payload.byteLength; + subscription.inputQueueFrames -= 1; } - await reportTerminalInputCompletion( - server, - frame.sessionId, - acknowledgements.map((acknowledgement) => acknowledgement.promise), - ); }); return; } @@ -481,6 +505,9 @@ export class TerminalHub { inputAcknowledgements: upstreamConnection.inputAcknowledgements ?? session.runtime === githubActionsRuntime, inputQueue: Promise.resolve(), + inputQueueBytes: 0, + inputQueueFrames: 0, + inputQueueRejectionScheduled: false, pendingInputAcknowledgements: new Map(), outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, @@ -667,6 +694,24 @@ function beginTerminalInputAcknowledgement( return pending; } +function scheduleTerminalInputBacklogRejection( + socket: WebSocket, + subscription: TerminalHubSubscription, + sessionId: string, +): void { + if (subscription.inputQueueRejectionScheduled) return; + subscription.inputQueueRejectionScheduled = true; + subscription.inputQueue = subscription.inputQueue + .catch(() => undefined) + .then(() => { + subscription.inputQueueRejectionScheduled = false; + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-rejected", + error: "terminal input backlog exceeded", + }); + }); +} + function completeTerminalInputAcknowledgement( subscription: TerminalHubSubscription, inputId: string, diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 3ca809fc..ee64d626 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -873,6 +873,151 @@ test("GitHub Actions serializes completion events for overlapping client inputs" server.emit("close"); }); +test("terminal input queue rejects excess frames after earlier completions", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let releaseFirstInput: (() => void) | undefined; + const firstInput = new Promise((resolve) => { + releaseFirstInput = resolve; + }); + let inputCalls = 0; + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async inputPayloads(_subscription, _user, payload) { + inputCalls += 1; + if (inputCalls === 1) await firstInput; + return [payload]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: session.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + for (let index = 0; index < 128; index += 1) { + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: session.id, + payload: new Uint8Array([index]), + }), + }); + } + await flushQueues(); + await flushQueues(); + + assert.equal(inputCalls, 1); + assert.equal(upstream.sent.length, 0); + assert.equal( + server.sent.some( + (payload) => + (decodeJsonPayload(frame(payload).payload) as { error?: string }).error === + "terminal input backlog exceeded", + ), + false, + ); + + releaseFirstInput?.(); + await flushQueues(); + await flushQueues(); + await flushQueues(); + + assert.equal(inputCalls, 32); + assert.equal(upstream.sent.length, 32); + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.equal(completions.length, 33); + assert.deepEqual(completions.slice(0, 32), Array(32).fill({ type: "input-accepted" })); + assert.deepEqual(completions[32], { + type: "input-rejected", + error: "terminal input backlog exceeded", + }); + server.emit("close"); +}); + +test("terminal input queue enforces the protocol-sized byte budget", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let releaseFirstInput: (() => void) | undefined; + const firstInput = new Promise((resolve) => { + releaseFirstInput = resolve; + }); + let inputCalls = 0; + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async inputPayloads(_subscription, _user, payload) { + inputCalls += 1; + if (inputCalls === 1) await firstInput; + return [payload]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: session.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + for (const payload of [new Uint8Array(9 * 1024 * 1024), new Uint8Array(8 * 1024 * 1024)]) { + server.emit("message", { + data: encodeTerminalFrame( + { + type: TerminalMessageType.Input, + sessionId: session.id, + payload, + }, + { maxFrameBytes: 16 * 1024 * 1024 }, + ), + }); + } + await flushQueues(); + await flushQueues(); + assert.equal(inputCalls, 1); + + releaseFirstInput?.(); + await flushQueues(); + await flushQueues(); + + assert.equal(inputCalls, 1); + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.deepEqual(completions, [ + { type: "input-accepted" }, + { type: "input-rejected", error: "terminal input backlog exceeded" }, + ]); + server.emit("close"); +}); + test("GitHub Actions framed output preserves control-shaped terminal bytes", async () => { const client = socket(); const server = socket(); From 0eac78f59f3043f43ec265f5741bb67a3bbda00d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:03:34 +0200 Subject: [PATCH 146/242] fix(actions): retire acknowledgements on runner failover --- src/worker/terminal-hub.ts | 11 +++ tests/terminal-hub.test.ts | 165 +++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index c0ae95a5..05afd6e0 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -563,6 +563,17 @@ export class TerminalHub { } const relayEvent = parseGitHubActionsRelayEvent(data); if (relayEvent) { + if (relayEvent.type === "runner_disconnected") { + completeAllTerminalInputAcknowledgements(activeSubscription, { + accepted: false, + error: "GitHub Actions runner disconnected before accepting input", + }); + } else if (relayEvent.type === "runner_connected") { + completeAllTerminalInputAcknowledgements(activeSubscription, { + accepted: false, + error: "GitHub Actions runner was replaced before accepting input", + }); + } sendTerminalJson(client, TerminalMessageType.Event, id, relayEvent); return; } diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index ee64d626..e026d66e 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -11,8 +11,11 @@ import { encodeTerminalFrame, } from "@openclaw/libterminal/protocol"; import { + attachGitHubActionsViewerProtocol, encodeGitHubActionsRelayInputAcknowledgement, encodeGitHubActionsRelayOutput, + githubActionsFramedRunnerCapability, + notifyGitHubActionsViewers, parseGitHubActionsRelayInput, } from "../src/github-actions-runtime.ts"; import type { User } from "../src/worker/models.ts"; @@ -27,6 +30,7 @@ class TestSocket { readonly sent: Array = []; readonly closed: Array<{ code?: number; reason?: string }> = []; accepted = false; + private attachment: unknown; private readonly listeners = new Map(); accept(): void { @@ -48,6 +52,14 @@ class TestSocket { this.readyState = WebSocket.CLOSED; } + serializeAttachment(attachment: unknown): void { + this.attachment = attachment; + } + + deserializeAttachment(): unknown { + return this.attachment; + } + emit(type: string, values: Record = {}): void { for (const listener of this.listeners.get(type) ?? []) { listener(Object.assign(new Event(type), values)); @@ -82,6 +94,16 @@ function emitRelayAcknowledgement(upstream: TestSocket, inputId: string, accepte }); } +function emitRelayEvent( + upstream: TestSocket, + type: "runner_connected" | "runner_disconnected" | "runner_waiting", +): void { + const source = socket(); + attachGitHubActionsViewerProtocol(source, githubActionsFramedRunnerCapability); + notifyGitHubActionsViewers([source], type); + upstream.emit("message", { data: source.sent[0] }); +} + async function flushQueues(): Promise { await new Promise((resolve) => setImmediate(resolve)); } @@ -1285,6 +1307,149 @@ test("GitHub Actions close rejects every pending input acknowledgement", async ( server.emit("close"); }); +test("GitHub Actions runner disconnect rejects pending input without closing the viewer relay", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("pending"), + }), + }); + await flushQueues(); + + emitRelayEvent(upstream, "runner_disconnected"); + await flushQueues(); + await flushQueues(); + await flushQueues(); + + assert.deepEqual(upstream.closed, []); + const disconnectEvents = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload)); + assert.equal( + disconnectEvents.some( + (event) => + (event as { type?: string }).type === "input-rejected" && + (event as { error?: string }).error === + "GitHub Actions runner disconnected before accepting input", + ), + true, + JSON.stringify(disconnectEvents), + ); + server.emit("close"); +}); + +test("GitHub Actions runner replacement rejects old input and accepts new input", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("old runner"), + }), + }); + await flushQueues(); + const oldInput = relayInput(upstream.sent.at(-1)!); + + emitRelayEvent(upstream, "runner_connected"); + await flushQueues(); + await flushQueues(); + await flushQueues(); + assert.deepEqual(upstream.closed, []); + const replacementEvents = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload)); + assert.equal( + replacementEvents.some( + (event) => + (event as { type?: string }).type === "input-rejected" && + (event as { error?: string }).error === + "GitHub Actions runner was replaced before accepting input", + ), + true, + JSON.stringify(replacementEvents), + ); + + emitRelayAcknowledgement(upstream, oldInput.inputId, true); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("new runner"), + }), + }); + await flushQueues(); + await flushQueues(); + const newInput = relayInput(upstream.sent.at(-1)!); + assert.equal(newInput.text, "new runner"); + emitRelayAcknowledgement(upstream, newInput.inputId, true); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.deepEqual( + completions.map((message) => message.type), + ["input-rejected", "input-accepted"], + ); + assert.deepEqual(upstream.closed, []); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From cc38033e3f694c52e08bec25de672847c8bded37 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:04:05 +0200 Subject: [PATCH 147/242] docs(changelog): record migration and relay fixes --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f86f07d5..a9d556c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, ownership-fenced repair of incomplete legacy lookup sets before rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes, reject pending acknowledgements immediately when GitHub Actions runners disconnect or are replaced, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown without scheduling retries when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. From e33d43d6cf2a8ab569c8ee7a0c2b07f26aa5e5cd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:19 +0200 Subject: [PATCH 148/242] fix(credentials): persist staged lookup identity --- ...dential_policy_registration_lookup_ids.sql | 30 +++++++++++++++++++ src/worker/database.ts | 2 ++ src/worker/session-control-policy.ts | 25 ++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 migrations/0037_credential_policy_registration_lookup_ids.sql diff --git a/migrations/0037_credential_policy_registration_lookup_ids.sql b/migrations/0037_credential_policy_registration_lookup_ids.sql new file mode 100644 index 00000000..fe3b8614 --- /dev/null +++ b/migrations/0037_credential_policy_registration_lookup_ids.sql @@ -0,0 +1,30 @@ +ALTER TABLE interactive_session_credential_policy_registrations + ADD COLUMN lookup_ids_json TEXT; + +UPDATE interactive_session_credential_policy_registrations AS registration +SET lookup_ids_json = ( + SELECT json_group_array(lookup_id) + FROM ( + SELECT DISTINCT lookup_id + FROM ( + SELECT policy.lookup_id + FROM interactive_session_credential_policies AS policy + WHERE policy.session_id = registration.session_id + AND policy.sandbox_id = registration.sandbox_id + UNION ALL + SELECT json_extract(rollback.value, '$.policy.sandboxId') + FROM json_each( + CASE + WHEN json_valid(registration.rollback_policies_json) + THEN registration.rollback_policies_json + ELSE '[]' + END + ) AS rollback + WHERE json_type(rollback.value, '$.policy.sandboxId') = 'text' + UNION ALL + SELECT registration.sandbox_id + ) + WHERE typeof(lookup_id) = 'text' AND length(lookup_id) > 0 + ORDER BY lookup_id + ) +); diff --git a/src/worker/database.ts b/src/worker/database.ts index 98a432f3..ed9cd46e 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -314,6 +314,8 @@ export type InteractiveSessionCredentialPolicyRegistrationTable = { cleanup_claim: string | null; cleanup_claim_expires_at: number | null; rollback_policies_json: Generated; + lookup_ids_json: Generated; + repair_generation: Generated; created_at: number; updated_at: number; }; diff --git a/src/worker/session-control-policy.ts b/src/worker/session-control-policy.ts index f927ed78..6fb2b255 100644 --- a/src/worker/session-control-policy.ts +++ b/src/worker/session-control-policy.ts @@ -29,6 +29,31 @@ export type SandboxCredentialPolicyRegistration = { lookupIds: string[]; }; +export function sandboxCredentialPolicyRegistrationLookupIds( + value: string | null | undefined, + sandboxId: string, +): string[] { + if (value) { + try { + const parsed = JSON.parse(value) as unknown; + if ( + Array.isArray(parsed) && + parsed.length > 0 && + parsed.every( + (lookupId) => + typeof lookupId === "string" && lookupId.length > 0 && lookupId.length <= 200, + ) + ) { + const lookupIds = [...new Set(parsed)]; + if (lookupIds.includes(sandboxId)) return lookupIds; + } + } catch { + // Upgraded rows are backfilled by migration; malformed rows retain the stable sandbox key. + } + } + return [sandboxId]; +} + export function storedSandboxCredentialPolicy( value: unknown, ): StoredSandboxCredentialPolicy | undefined { From 52f1828fd46da2a2ba15e0842a64beae2b16af07 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:22 +0200 Subject: [PATCH 149/242] fix(macos): reconcile uncertain desktop publication --- .../CrabfleetDesktopRegistration.swift | 33 +++++++-- .../PrivateMacShareController.swift | 42 +++++++++++- .../PrivateMacShareTests.swift | 68 ++++++++++++++++++- 3 files changed, 135 insertions(+), 8 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index 15c589c3..b2f19e48 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -5,6 +5,12 @@ protocol DesktopHostRegistering: Sendable { func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws } +struct DesktopHostRegistrationResultUncertainError: LocalizedError, Equatable, Sendable { + let message: String + + var errorDescription: String? { message } +} + actor DesktopHostRegistrationCoordinator { private let registration: any DesktopHostRegistering private var pendingOperation: Task? @@ -107,18 +113,37 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { let request = try registrationRequest(identity: identity, port: port) - let (data, http) = try await transport.data(for: request) - try validate(response: http, for: request, acceptingNotFound: false) + let data: Data + let http: HTTPURLResponse + do { + (data, http) = try await transport.data(for: request) + } catch { + throw DesktopHostRegistrationResultUncertainError(message: error.localizedDescription) + } + do { + try validate(response: http, for: request, acceptingNotFound: false) + } catch let error as DesktopHostRegistrationError { + if case .httpStatus(let status) = error, status >= 500 { + throw DesktopHostRegistrationResultUncertainError( + message: error.localizedDescription + ) + } + throw error + } guard let response = try? JSONDecoder().decode(RegistrationResponse.self, from: data), response.host.id == Self.hostID(identity: identity) else { - throw DesktopHostRegistrationError.invalidResponse + throw DesktopHostRegistrationResultUncertainError( + message: DesktopHostRegistrationError.invalidResponse.localizedDescription + ) } if let ownershipToken = response.ownershipToken, !Self.isValidOwnershipToken(ownershipToken) { - throw DesktopHostRegistrationError.invalidResponse + throw DesktopHostRegistrationResultUncertainError( + message: DesktopHostRegistrationError.invalidResponse.localizedDescription + ) } return response.ownershipToken } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index ef12e2d8..791d08a9 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -38,6 +38,11 @@ final class PrivateMacShareStopCoordinator { @MainActor final class DesktopHostRegistrationLifecycle { + private struct RegistrationTarget: Equatable { + let identity: TailnetIdentity + let port: UInt16 + } + private struct PublishedRegistration: Equatable { let identity: TailnetIdentity let ownershipToken: String? @@ -45,6 +50,7 @@ final class DesktopHostRegistrationLifecycle { private let coordinator: DesktopHostRegistrationCoordinator private var publishedRegistration: PublishedRegistration? + private var uncertainRegistrations: [RegistrationTarget] = [] private var pendingRemovals: [PublishedRegistration] = [] init(registration: any DesktopHostRegistering) { @@ -52,7 +58,19 @@ final class DesktopHostRegistrationLifecycle { } func publish(identity: TailnetIdentity, port: UInt16) async throws { - let ownershipToken = try await coordinator.register(identity: identity, port: port) + let target = RegistrationTarget(identity: identity, port: port) + let ownershipToken: String? + do { + ownershipToken = try await coordinator.register(identity: identity, port: port) + } catch { + if error is DesktopHostRegistrationResultUncertainError, + !uncertainRegistrations.contains(target) + { + uncertainRegistrations.append(target) + } + throw error + } + uncertainRegistrations.removeAll { $0 == target } if let publishedRegistration, publishedRegistration.identity != identity, !pendingRemovals.contains(publishedRegistration) { @@ -66,6 +84,27 @@ final class DesktopHostRegistrationLifecycle { } func removePublishedIdentities() async throws { + var firstError: Error? + let uncertainRegistrations = uncertainRegistrations + for target in uncertainRegistrations { + do { + let ownershipToken = try await coordinator.register( + identity: target.identity, + port: target.port + ) + let recovered = PublishedRegistration( + identity: target.identity, + ownershipToken: ownershipToken + ) + if !pendingRemovals.contains(recovered) { + pendingRemovals.append(recovered) + } + self.uncertainRegistrations.removeAll { $0 == target } + } catch { + firstError = firstError ?? error + } + } + if let publishedRegistration { if !pendingRemovals.contains(publishedRegistration) { pendingRemovals.append(publishedRegistration) @@ -73,7 +112,6 @@ final class DesktopHostRegistrationLifecycle { self.publishedRegistration = nil } - var firstError: Error? let removals = pendingRemovals for removal in removals { do { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index b6debdf4..7c9f9565 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -440,6 +440,27 @@ struct PrivateMacShareTests { #expect(await registration.events == [.register(identity.dnsName)]) } + @Test @MainActor + func ambiguousDesktopPublicationIsReacquiredBeforeCleanup() async throws { + let identity = desktopIdentity(name: "ambiguous-publish", address: "100.64.12.46") + let registration = AmbiguousDesktopRegistration() + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + try await lifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(identity.dnsName), + .register(identity.dnsName), + .unregister(identity.dnsName, "reacquired-token"), + ] + ) + } + @Test @MainActor func failedDesktopRemovalSurvivesLaterIdentityChanges() async throws { let first = desktopIdentity(name: "first-host", address: "100.64.12.41") @@ -791,7 +812,7 @@ struct PrivateMacShareTests { } @Test - func desktopRegistrationRejectsMalformedAdvertisedOwnershipTokens() async throws { + func desktopRegistrationTreatsMalformedCommittedResponsesAsUncertain() async throws { let transport = DesktopRegistrationTransport { request in let responseURL = try #require(request.url) return ( @@ -815,7 +836,27 @@ struct PrivateMacShareTests { )) let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) - await #expect(throws: DesktopHostRegistrationError.invalidResponse) { + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await registration.register(identity: identity, port: 5_901) + } + } + + @Test + func desktopRegistrationTreatsTransportFailuresAsUncertain() async throws { + let transport = DesktopRegistrationTransport { _ in + throw URLError(.timedOut) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { try await registration.register(identity: identity, port: 5_901) } } @@ -1597,6 +1638,29 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { } } +private actor AmbiguousDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register(String) + case unregister(String, String?) + } + + private(set) var events: [Event] = [] + private var registerCount = 0 + + func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { + events.append(.register(identity.dnsName)) + registerCount += 1 + if registerCount == 1 { + throw DesktopHostRegistrationResultUncertainError(message: "response lost") + } + return "reacquired-token" + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + events.append(.unregister(identity.dnsName, ownershipToken)) + } +} + private actor SuspendedDesktopCleanupRegistration: DesktopHostRegistering { private var unregistrationContinuation: CheckedContinuation? From ebd81b7187705f4289eee7d0fd0fb6009c85effe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:24 +0200 Subject: [PATCH 150/242] fix(credentials): recover staged rotations idempotently --- ...ndbox-credential-policy-cleanup-service.ts | 9 +- .../sandbox-credential-policy-repository.ts | 61 +++++-- .../sandbox-credential-policy-scanner.ts | 18 +- .../sandbox-credential-policy-cleanup.test.ts | 15 ++ ...ndbox-credential-policy-repository.test.ts | 157 +++++++++++++++++- .../sandbox-credential-policy-scanner.test.ts | 2 + 6 files changed, 242 insertions(+), 20 deletions(-) diff --git a/src/worker/sandbox-credential-policy-cleanup-service.ts b/src/worker/sandbox-credential-policy-cleanup-service.ts index 96dc18dd..ac9d9212 100644 --- a/src/worker/sandbox-credential-policy-cleanup-service.ts +++ b/src/worker/sandbox-credential-policy-cleanup-service.ts @@ -23,6 +23,7 @@ import { scanCredentialPolicyCleanupPage } from "./sandbox-credential-policy-sca import { isCurrentSandboxLease, sandboxLeaseInfo } from "./sandbox-lease.ts"; import { isSandboxSessionAlreadyGone } from "./sandbox-session-errors.ts"; import { sandboxControlStub } from "./session-control-do.ts"; +import { sandboxCredentialPolicyRegistrationLookupIds } from "./session-control-policy.ts"; import { finalizeTerminalInteractiveSession } from "./session-terminal-finalization.ts"; const credentialPolicyCleanupLimit = 8; @@ -32,11 +33,12 @@ export async function sandboxCredentialPolicyExists( env: RuntimeEnv, sandboxId: string, generation: string, + lookupIds: readonly string[] = sandboxLookupIds(env, sandboxId), ): Promise { const stub = sandboxControlStub(env); if (!stub) return false; const responses = await Promise.all( - sandboxLookupIds(env, sandboxId).map((lookupId) => + lookupIds.map((lookupId) => stub.fetch( `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, ), @@ -129,7 +131,10 @@ async function reconcileStagedCredentialPolicyRegistration( if ((claimed.numUpdatedRows ?? 0n) === 0n) return; try { await Promise.all( - sandboxLookupIds(env, registration.sandbox_id).map((lookupId) => + sandboxCredentialPolicyRegistrationLookupIds( + registration.lookup_ids_json, + registration.sandbox_id, + ).map((lookupId) => unregisterSandboxCredentialPolicyLookup( env, lookupId, diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 3e889b94..1025c580 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -506,6 +506,7 @@ export function sandboxCredentialPolicyRegistrationQueries( registration_generation, registration_claim, registration_claim_expires_at, + lookup_ids_json, attempt_count, last_attempt_at, last_error, @@ -521,6 +522,7 @@ export function sandboxCredentialPolicyRegistrationQueries( ${registration.generation}, ${registration.claim}, ${registrationExpiresAt}, + ${JSON.stringify(registration.lookupIds)}, 0, NULL, NULL, @@ -542,6 +544,7 @@ export function sandboxCredentialPolicyRegistrationQueries( registration_generation = excluded.registration_generation, registration_claim = excluded.registration_claim, registration_claim_expires_at = excluded.registration_claim_expires_at, + lookup_ids_json = excluded.lookup_ids_json, repair_generation = NULL, rollback_policies_json = NULL, last_error = NULL, @@ -599,6 +602,7 @@ export async function beginSandboxCredentialPolicyRegistration( "registration_generation", "registration_claim", "registration_claim_expires_at", + "lookup_ids_json", ]) .where("session_id", "=", sessionId) .where("sandbox_id", "=", sandboxId) @@ -607,7 +611,8 @@ export async function beginSandboxCredentialPolicyRegistration( claimed?.state !== "registering" || claimed.registration_generation !== registration.generation || claimed.registration_claim !== registration.claim || - claimed.registration_claim_expires_at !== registrationExpiresAt + claimed.registration_claim_expires_at !== registrationExpiresAt || + claimed.lookup_ids_json !== JSON.stringify(registration.lookupIds) ) { await abandonSandboxCredentialPolicyRegistration( env, @@ -882,18 +887,50 @@ export async function finishSandboxCredentialPolicyRegistration( ownershipFence: SandboxCredentialPolicyOwnershipFence, ): Promise { const now = Date.now(); - const db = database(env); - await executeBatch( - env, - sandboxCredentialPolicyPromotionQueries( + let batchError: unknown; + try { + await executeBatch( env, - sessionId, - sandboxId, - registration, - ownershipFence, - now, - ), - ); + sandboxCredentialPolicyPromotionQueries( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + now, + ), + ); + } catch (error) { + batchError = error; + } + try { + if (await sandboxCredentialPolicyPromotionCompleted(env, sessionId, sandboxId, registration)) { + return true; + } + } catch (readError) { + try { + if ( + await sandboxCredentialPolicyPromotionCompleted(env, sessionId, sandboxId, registration) + ) { + return true; + } + } catch { + // Preserve the first observable failure when verification remains unavailable. + } + if (batchError) throw batchError; + throw readError; + } + if (batchError) throw batchError; + return false; +} + +async function sandboxCredentialPolicyPromotionCompleted( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, +): Promise { + const db = database(env); const active = await db .selectFrom("interactive_session_credential_policies") .select(["lookup_id", "state", "registration_generation", "registration_claim"]) diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index 277dc3f5..6bd17feb 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -14,11 +14,13 @@ import { finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyCleanupAuthorizedCondition, - sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; import { sandboxLeaseInfo, sandboxLeasePrefix } from "./sandbox-lease.ts"; -import type { SandboxCredentialPolicyRegistration } from "./session-control-policy.ts"; +import { + sandboxCredentialPolicyRegistrationLookupIds, + type SandboxCredentialPolicyRegistration, +} from "./session-control-policy.ts"; const credentialPolicyScanLimit = 32; export const credentialPolicyProvisioningStaleMs = 15 * 60_000; @@ -70,6 +72,7 @@ type StagedCredentialPolicyScanRow = CredentialPolicyOwnershipRow & { registration_generation: string; registration_claim: string; registration_claim_expires_at: number; + lookup_ids_json: string | null; rollback_policies_json: string | null; }; @@ -77,6 +80,7 @@ export type SandboxCredentialPolicyExists = ( env: RuntimeEnv, sandboxId: string, generation: string, + lookupIds?: readonly string[], ) => Promise; export type RestoreSandboxCredentialPolicyRollback = (input: { @@ -310,6 +314,7 @@ async function scanStagedCredentialPolicyRegistrations( registration.registration_generation, registration.registration_claim, registration.registration_claim_expires_at, + registration.lookup_ids_json, registration.rollback_policies_json, session.id AS matched_session_id, session.adapter AS session_adapter, @@ -336,7 +341,7 @@ async function scanStagedCredentialPolicyRegistrations( const registration: SandboxCredentialPolicyRegistration = { generation: row.registration_generation, claim: row.registration_claim, - lookupIds: sandboxLookupIds(env, row.sandbox_id), + lookupIds: sandboxCredentialPolicyRegistrationLookupIds(row.lookup_ids_json, row.sandbox_id), }; try { const ownershipFence = credentialPolicyScanOwnershipFence(row, now); @@ -360,7 +365,12 @@ async function scanStagedCredentialPolicyRegistrations( ); if (!recovery) continue; if ( - (await policyExists(env, row.sandbox_id, row.registration_generation)) && + (await policyExists( + env, + row.sandbox_id, + row.registration_generation, + registration.lookupIds, + )) && (await finishSandboxCredentialPolicyRegistration( env, row.session_id, diff --git a/tests/sandbox-credential-policy-cleanup.test.ts b/tests/sandbox-credential-policy-cleanup.test.ts index 5089da29..368f62b0 100644 --- a/tests/sandbox-credential-policy-cleanup.test.ts +++ b/tests/sandbox-credential-policy-cleanup.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; import test from "node:test"; import { @@ -212,3 +213,17 @@ test("terminal cleanup atomically stages the session and credential-policy refs" assert.ok(parameters.includes("sandbox-1")); assert.ok(parameters.includes(leaseId)); }); + +test("staged cleanup uses the persisted lookup set", async () => { + const source = await readFile( + new URL("../src/worker/sandbox-credential-policy-cleanup-service.ts", import.meta.url), + "utf8", + ); + const start = source.indexOf("async function reconcileStagedCredentialPolicyRegistration"); + const end = source.indexOf("async function normalizeCredentialPolicyCleanupGroups", start); + const stagedCleanup = source.slice(start, end); + + assert.match(stagedCleanup, /sandboxCredentialPolicyRegistrationLookupIds/); + assert.match(stagedCleanup, /registration\.lookup_ids_json/); + assert.doesNotMatch(stagedCleanup, /sandboxLookupIds\(env, registration\.sandbox_id\)/); +}); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 651b7366..0575178e 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -25,6 +25,7 @@ import { credentialPolicyRegistrationAccepted } from "../src/credential-policy-f import { database } from "../src/worker/database.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import type { SandboxCredentialPolicyRegistration } from "../src/worker/session-control-policy.ts"; +import { sandboxCredentialPolicyRegistrationLookupIds } from "../src/worker/session-control-policy.ts"; import type { StoredSandboxCredentialPolicy } from "../src/worker/session-control-policy.ts"; type PreparedStatement = { @@ -174,15 +175,33 @@ function credentialPolicyDatabase(options: { applyMigrations?: boolean } = {}): "utf8", ), ); + db.exec( + readFileSync( + new URL( + "../migrations/0037_credential_policy_registration_lookup_ids.sql", + import.meta.url, + ), + "utf8", + ), + ); } return db; } function sqliteRuntimeEnv( sqlite: DatabaseSync, - options: { interruptAfterStatement?: number } = {}, + options: { + interruptAfterStatement?: number; + throwAfterCommit?: boolean; + failNextReadAfterBatch?: boolean; + } = {}, ): RuntimeEnv { + let failNextRead = false; function execute(sql: string, parameters: unknown[]) { + if (failNextRead && /^\s*select\b/i.test(sql)) { + failNextRead = false; + throw new Error("simulated committed read failure"); + } const statement = sqlite.prepare(sql) as unknown as SqliteStatement; if (/^\s*(?:select|pragma|with)\b|\breturning\b/i.test(sql)) { const results = statement.all(...parameters).map((row) => ({ ...row })); @@ -230,9 +249,13 @@ function sqliteRuntimeEnv( } } sqlite.exec("COMMIT"); + failNextRead = options.failNextReadAfterBatch ?? false; + if (options.throwAfterCommit) { + throw new Error("simulated ambiguous committed batch"); + } return results; } catch (error) { - sqlite.exec("ROLLBACK"); + if (sqlite.isTransaction) sqlite.exec("ROLLBACK"); throw error; } }, @@ -293,6 +316,21 @@ test("credential-policy lookup identity includes the Sandbox durable object id e ]); }); +test("staged lookup identity decoder requires the stable sandbox lookup", () => { + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds('["sandbox-1","do-old"]', "sandbox-1"), + ["sandbox-1", "do-old"], + ); + assert.deepEqual(sandboxCredentialPolicyRegistrationLookupIds('["do-old"]', "sandbox-1"), [ + "sandbox-1", + ]); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds('["sandbox-1","sandbox-1"]', "sandbox-1"), + ["sandbox-1"], + ); + assert.deepEqual(sandboxCredentialPolicyRegistrationLookupIds(null, "sandbox-1"), ["sandbox-1"]); +}); + test("credential-policy generations reuse exactly one current identity", () => { assert.equal(currentSandboxCredentialPolicyGeneration([]), null); assert.equal( @@ -370,6 +408,7 @@ test("credential-policy rotation always claims a fresh generation", async () => registration_generation: generation, registration_claim: claim, registration_claim_expires_at: registrationExpiresAt, + lookup_ids_json: '["sandbox-1"]', }, ], }; @@ -472,6 +511,78 @@ test("migration leaves live legacy registrations unstaged while old workers rene ); }); +test("lookup identity migration backfills the exact staged legacy lookup set", () => { + const sqlite = credentialPolicyDatabase({ applyMigrations: false }); + sqlite.exec( + readFileSync( + new URL("../migrations/0034_credential_policy_registration_staging.sql", import.meta.url), + "utf8", + ), + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0035_credential_policy_registration_rollback.sql", import.meta.url), + "utf8", + ), + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0036_credential_policy_lookup_repair.sql", import.meta.url), + "utf8", + ), + ); + sqlite + .prepare(` + INSERT INTO interactive_session_credential_policy_registrations ( + session_id, + sandbox_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + rollback_policies_json, + created_at, + updated_at + ) VALUES (?, ?, 'registering', ?, ?, ?, ?, 1, 1) + `) + .run( + "IS-42", + "sandbox-1", + "generation:staged", + "registration:staged", + Number.MAX_SAFE_INTEGER, + JSON.stringify([ + { + generation: "generation:existing", + policy: { + allowedHosts: [], + githubCredentialSource: "none", + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: "do-old", + sessionId: "IS-42", + }, + }, + ]), + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0037_credential_policy_registration_lookup_ids.sql", import.meta.url), + "utf8", + ), + ); + + const row = sqlite + .prepare(` + SELECT lookup_ids_json, repair_generation + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(); + assert.deepEqual(JSON.parse(String(row?.lookup_ids_json)), ["do-1", "do-old", "sandbox-1"]); + assert.equal(row?.repair_generation, null); +}); + test("post-migration legacy registration claims block new staged generations", async () => { const sqlite = credentialPolicyDatabase(); sqlite @@ -858,6 +969,48 @@ test("completed credential-policy rotation atomically promotes every active look ); }); +test("completed credential-policy rotation tolerates an ambiguous committed batch", async () => { + const sqlite = credentialPolicyDatabase(); + const staged = await beginSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite), + "IS-42", + "sandbox-1", + ownershipFence, + ); + + assert.equal( + await finishSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite, { throwAfterCommit: true }), + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + true, + ); +}); + +test("completed credential-policy rotation retries an ambiguous verification read", async () => { + const sqlite = credentialPolicyDatabase(); + const staged = await beginSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite), + "IS-42", + "sandbox-1", + ownershipFence, + ); + + assert.equal( + await finishSandboxCredentialPolicyRegistration( + sqliteRuntimeEnv(sqlite, { failNextReadAfterBatch: true }), + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + true, + ); +}); + test("interrupted credential-policy promotion rolls back every active lookup", async () => { const sqlite = credentialPolicyDatabase(); const staged = await beginSandboxCredentialPolicyRegistration( diff --git a/tests/sandbox-credential-policy-scanner.test.ts b/tests/sandbox-credential-policy-scanner.test.ts index 1b79d4a5..22cd4565 100644 --- a/tests/sandbox-credential-policy-scanner.test.ts +++ b/tests/sandbox-credential-policy-scanner.test.ts @@ -143,6 +143,8 @@ test("staged recovery takes a fresh exclusive claim before promotion or rollback stagedRecovery.indexOf("claimSandboxCredentialPolicyRegistrationRecovery(") < stagedRecovery.indexOf("restoreRollback({"), ); + assert.match(stagedRecovery, /sandboxCredentialPolicyRegistrationLookupIds/); + assert.match(stagedRecovery, /registration\.lookupIds/); assert.doesNotMatch(stagedRecovery, /renewSandboxCredentialPolicyRegistration/); }); From 286e05910b197922393ab09512f748538ce1df58 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:30 +0200 Subject: [PATCH 151/242] fix(macos): bound teardown input release retries --- .../Sources/CrabfleetMac/MacRemoteInput.swift | 10 ++++- .../PrivateMacShareTests.swift | 38 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index 5a078627..b216a27a 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -16,6 +16,7 @@ extension RemoteInputForwarding { final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable { private static let releaseRetryDelay: DispatchTimeInterval = .milliseconds(250) + private static let releaseRetryLimit = 120 private let descriptor: CapturedDisplayDescriptor private let eventQueue = DispatchQueue( @@ -24,6 +25,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable ) private let accessibilityGranted: @Sendable () -> Bool private let pendingReleaseRetryDelay: DispatchTimeInterval + private let pendingReleaseRetryLimit: Int private let keyEventPoster: (@Sendable (Bool, UInt32) -> Void)? private let mouseEventPoster: (@Sendable (CGEventType, CGPoint, CGMouseButton) -> Void)? @@ -35,6 +37,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable private var pressedKeysyms: Set = [] private var hasPendingRelease = false private var pendingReleaseRetryScheduled = false + private var pendingReleaseRetriesRemaining = 0 init( descriptor: CapturedDisplayDescriptor, @@ -42,12 +45,14 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable MacRemoteInputController.isAccessibilityGranted }, pendingReleaseRetryDelay: DispatchTimeInterval = MacRemoteInputController.releaseRetryDelay, + pendingReleaseRetryLimit: Int = MacRemoteInputController.releaseRetryLimit, keyEventPoster: (@Sendable (Bool, UInt32) -> Void)? = nil, mouseEventPoster: (@Sendable (CGEventType, CGPoint, CGMouseButton) -> Void)? = nil ) { self.descriptor = descriptor self.accessibilityGranted = accessibilityGranted self.pendingReleaseRetryDelay = pendingReleaseRetryDelay + self.pendingReleaseRetryLimit = max(pendingReleaseRetryLimit, 0) self.keyEventPoster = keyEventPoster self.mouseEventPoster = mouseEventPoster frameWidth = descriptor.frameWidth @@ -133,6 +138,7 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable eventQueue.async { [self] in guard !pressedKeysyms.isEmpty || previousButtonMask != 0 else { return } hasPendingRelease = true + pendingReleaseRetriesRemaining = pendingReleaseRetryLimit flushPendingRelease() } } @@ -196,10 +202,12 @@ final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable pressedKeysyms.removeAll() previousButtonMask = 0 hasPendingRelease = false + pendingReleaseRetriesRemaining = 0 } private func schedulePendingReleaseRetry() { - guard !pendingReleaseRetryScheduled else { return } + guard !pendingReleaseRetryScheduled, pendingReleaseRetriesRemaining > 0 else { return } + pendingReleaseRetriesRemaining -= 1 pendingReleaseRetryScheduled = true eventQueue.asyncAfter(deadline: .now() + pendingReleaseRetryDelay) { [self] in self.pendingReleaseRetryScheduled = false diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 7c9f9565..fe117efa 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -1178,6 +1178,44 @@ struct PrivateMacShareTests { #expect(await waitUntilAsync { retainedController == nil }) } + @Test + func pendingInputReleaseStopsRetryingAfterTeardownBudgetExpires() async { + let trust = AccessibilityTrust(granted: true) + let events = RemoteInputEventRecorder() + var controller: MacRemoteInputController? = MacRemoteInputController( + descriptor: CapturedDisplayDescriptor( + displayID: 1, + displayBounds: CGRect(x: 0, y: 0, width: 100, height: 100), + frameWidth: 100, + frameHeight: 100, + sourcePixelWidth: 100, + sourcePixelHeight: 100 + ), + accessibilityGranted: { trust.isGranted() }, + pendingReleaseRetryDelay: .milliseconds(10), + pendingReleaseRetryLimit: 2, + keyEventPoster: { down, keysym in + events.append(.key(down: down, keysym: keysym)) + } + ) + weak var retainedController = controller + + controller?.keyEvent(down: true, keysym: 0x61) + #expect(await waitUntilAsync { + events.contains(.key(down: true, keysym: 0x61)) + }) + trust.setGranted(false) + let checksBeforeRelease = trust.checkCount + controller?.releaseAllInput() + controller = nil + + #expect(await waitUntilAsync { + trust.checkCount >= checksBeforeRelease + 3 + }) + #expect(await waitUntilAsync { retainedController == nil }) + #expect(!events.contains(.key(down: false, keysym: 0x61))) + } + @Test func emptyInputReleaseDoesNotRetainController() async { let trust = AccessibilityTrust(granted: false) From f9b06f743dc05a076e7ccc88113b4a70bc8c2a4a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:16:02 +0200 Subject: [PATCH 152/242] fix(terminal): fence runner input generations --- src/worker/terminal-hub.ts | 86 ++++++++++++++++++++++++++----- tests/terminal-hub.test.ts | 100 +++++++++++++++++++++++++++++++++++-- 2 files changed, 169 insertions(+), 17 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 05afd6e0..2392bedd 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -36,6 +36,7 @@ const terminalInputAcknowledgementTimeoutMs = 5_000; type PendingTerminalInputAcknowledgement = { inputId: string; + runnerGeneration: number; promise: Promise; resolve(result: GitHubActionsRelayInputAcknowledgement): void; timeout: ReturnType; @@ -62,8 +63,10 @@ export type TerminalHubSubscription = { inputQueue: Promise; inputQueueBytes: number; inputQueueFrames: number; + inputQueueRejections: number; inputQueueRejectionScheduled: boolean; pendingInputAcknowledgements: Map; + runnerGeneration: number; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; }; @@ -290,7 +293,11 @@ export class TerminalHub { ? createGitHubActionsRelayInputId() : null; const acknowledgement = inputId - ? beginTerminalInputAcknowledgement(subscription, inputId) + ? beginTerminalInputAcknowledgement( + subscription, + inputId, + subscription.runnerGeneration, + ) : null; if (acknowledgement) acknowledgements.push(acknowledgement); try { @@ -507,8 +514,10 @@ export class TerminalHub { inputQueue: Promise.resolve(), inputQueueBytes: 0, inputQueueFrames: 0, + inputQueueRejections: 0, inputQueueRejectionScheduled: false, pendingInputAcknowledgements: new Map(), + runnerGeneration: 0, outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; @@ -546,6 +555,13 @@ export class TerminalHub { }); upstream.addEventListener("message", (event) => { const raw = event.data; + const receivedRelayEvent = activeSubscription.inputAcknowledgements + ? parseSynchronousGitHubActionsRelayEvent(raw) + : null; + let runnerGenerationAtReceipt = + receivedRelayEvent?.type === "runner_connected" + ? ++activeSubscription.runnerGeneration + : activeSubscription.runnerGeneration; outputQueue = outputQueue .catch(() => undefined) .then(async () => { @@ -561,7 +577,7 @@ export class TerminalHub { ); return; } - const relayEvent = parseGitHubActionsRelayEvent(data); + const relayEvent = receivedRelayEvent ?? parseGitHubActionsRelayEvent(data); if (relayEvent) { if (relayEvent.type === "runner_disconnected") { completeAllTerminalInputAcknowledgements(activeSubscription, { @@ -569,10 +585,17 @@ export class TerminalHub { error: "GitHub Actions runner disconnected before accepting input", }); } else if (relayEvent.type === "runner_connected") { - completeAllTerminalInputAcknowledgements(activeSubscription, { - accepted: false, - error: "GitHub Actions runner was replaced before accepting input", - }); + if (!receivedRelayEvent) { + runnerGenerationAtReceipt = ++activeSubscription.runnerGeneration; + } + completeTerminalInputAcknowledgementsBeforeGeneration( + activeSubscription, + runnerGenerationAtReceipt, + { + accepted: false, + error: "GitHub Actions runner was replaced before accepting input", + }, + ); } sendTerminalJson(client, TerminalMessageType.Event, id, relayEvent); return; @@ -678,6 +701,7 @@ export class TerminalHub { function beginTerminalInputAcknowledgement( subscription: TerminalHubSubscription, inputId: string, + runnerGeneration: number, ): PendingTerminalInputAcknowledgement { let resolve!: (result: GitHubActionsRelayInputAcknowledgement) => void; const promise = new Promise((complete) => { @@ -685,6 +709,7 @@ function beginTerminalInputAcknowledgement( }); const pending: PendingTerminalInputAcknowledgement = { inputId, + runnerGeneration, promise, resolve, timeout: setTimeout(() => { @@ -710,16 +735,21 @@ function scheduleTerminalInputBacklogRejection( subscription: TerminalHubSubscription, sessionId: string, ): void { + subscription.inputQueueRejections += 1; if (subscription.inputQueueRejectionScheduled) return; subscription.inputQueueRejectionScheduled = true; subscription.inputQueue = subscription.inputQueue .catch(() => undefined) .then(() => { + const rejections = subscription.inputQueueRejections; + subscription.inputQueueRejections = 0; subscription.inputQueueRejectionScheduled = false; - sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { - type: "input-rejected", - error: "terminal input backlog exceeded", - }); + for (let index = 0; index < rejections; index += 1) { + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-rejected", + error: "terminal input backlog exceeded", + }); + } }); } @@ -740,15 +770,47 @@ function completeAllTerminalInputAcknowledgements( subscription: TerminalHubSubscription, result: Omit, ): number { - const pending = [...subscription.pendingInputAcknowledgements.values()]; - subscription.pendingInputAcknowledgements.clear(); + return completeTerminalInputAcknowledgements(subscription, () => true, result); +} + +function completeTerminalInputAcknowledgementsBeforeGeneration( + subscription: TerminalHubSubscription, + runnerGeneration: number, + result: Omit, +): number { + return completeTerminalInputAcknowledgements( + subscription, + (pending) => pending.runnerGeneration < runnerGeneration, + result, + ); +} + +function completeTerminalInputAcknowledgements( + subscription: TerminalHubSubscription, + matches: (pending: PendingTerminalInputAcknowledgement) => boolean, + result: Omit, +): number { + const pending = [...subscription.pendingInputAcknowledgements.values()].filter(matches); for (const acknowledgement of pending) { + subscription.pendingInputAcknowledgements.delete(acknowledgement.inputId); clearTimeout(acknowledgement.timeout); acknowledgement.resolve({ inputId: acknowledgement.inputId, ...result }); } return pending.length; } +function parseSynchronousGitHubActionsRelayEvent( + data: unknown, +): ReturnType { + if (typeof data === "string" || data instanceof ArrayBuffer) { + return parseGitHubActionsRelayEvent(data); + } + if (!ArrayBuffer.isView(data)) return null; + const copied = new Uint8Array(data.byteLength); + copied.set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)); + return parseGitHubActionsRelayEvent(copied.buffer); +} + async function reportTerminalInputCompletion( socket: WebSocket, sessionId: string, diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index e026d66e..cbca7acf 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -964,12 +964,15 @@ test("terminal input queue rejects excess frames after earlier completions", asy .filter((message) => message.type === TerminalMessageType.Event) .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); - assert.equal(completions.length, 33); + assert.equal(completions.length, 128); assert.deepEqual(completions.slice(0, 32), Array(32).fill({ type: "input-accepted" })); - assert.deepEqual(completions[32], { - type: "input-rejected", - error: "terminal input backlog exceeded", - }); + assert.deepEqual( + completions.slice(32), + Array(96).fill({ + type: "input-rejected", + error: "terminal input backlog exceeded", + }), + ); server.emit("close"); }); @@ -1450,6 +1453,93 @@ test("GitHub Actions runner replacement rejects old input and accepts new input" server.emit("close"); }); +test("queued runner replacement rejects only acknowledgements sent to the old generation", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let releaseBlockedOutput: ((output: ArrayBuffer) => void) | undefined; + const blockedOutput = new Promise((resolve) => { + releaseBlockedOutput = resolve; + }); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async inputPayloads() { + return [new TextEncoder().encode("old runner"), new TextEncoder().encode("new runner")]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + upstream.emit("message", { data: { arrayBuffer: () => blockedOutput } }); + const send = upstream.send.bind(upstream); + upstream.send = (data) => { + send(data); + if (upstream.sent.length === 1) emitRelayEvent(upstream, "runner_connected"); + }; + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("split input"), + }), + }); + await waitForInputPayloads(); + + assert.equal(upstream.sent.length, 2); + const oldInput = relayInput(upstream.sent[0]!); + const replacementInput = relayInput(upstream.sent[1]!); + releaseBlockedOutput?.(encodeGitHubActionsRelayOutput("blocked output")); + await flushQueues(); + await flushQueues(); + await flushQueues(); + + emitRelayAcknowledgement(upstream, oldInput.inputId, true); + await flushQueues(); + assert.equal( + server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string }) + .some((message) => message.type === "input-accepted" || message.type === "input-rejected"), + false, + ); + + emitRelayAcknowledgement(upstream, replacementInput.inputId, true); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.deepEqual(completions, [ + { + type: "input-rejected", + error: "GitHub Actions runner was replaced before accepting input", + }, + ]); + assert.deepEqual(upstream.closed, []); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From 59e251c8c6778f7e1bd53a486677a0f69844887c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:17:24 +0200 Subject: [PATCH 153/242] fix(runtime): persist superseded workspace cleanup --- ...0037_runtime_adapter_workspace_cleanup.sql | 30 ++ src/worker/database.ts | 20 ++ .../runtime-adapter-release-repository.ts | 164 +++++++++++ .../runtime-adapter-release-service.ts | 87 +++++- src/worker/runtime-adapter-workspaces.ts | 17 +- src/worker/runtime-application.ts | 22 ++ tests/runtime-adapter-release-service.test.ts | 273 ++++++++++++++++++ tests/runtime-adapter-workspaces.test.ts | 47 +++ 8 files changed, 645 insertions(+), 15 deletions(-) create mode 100644 migrations/0037_runtime_adapter_workspace_cleanup.sql create mode 100644 src/worker/provisioning/runtime-adapter-release-repository.ts diff --git a/migrations/0037_runtime_adapter_workspace_cleanup.sql b/migrations/0037_runtime_adapter_workspace_cleanup.sql new file mode 100644 index 00000000..866cb03a --- /dev/null +++ b/migrations/0037_runtime_adapter_workspace_cleanup.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS runtime_adapter_workspace_cleanups ( + session_id TEXT NOT NULL, + adapter_workspace_id TEXT NOT NULL, + profile TEXT, + control_plane TEXT, + create_pending INTEGER NOT NULL CHECK (create_pending IN (0, 1)), + message TEXT NOT NULL, + reconcile_error TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + next_attempt_at INTEGER NOT NULL, + cleanup_claim TEXT, + cleanup_claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, adapter_workspace_id), + CHECK ( + (cleanup_claim IS NULL AND cleanup_claim_expires_at IS NULL) + OR (cleanup_claim IS NOT NULL AND cleanup_claim_expires_at IS NOT NULL) + ) +); + +CREATE INDEX IF NOT EXISTS idx_runtime_adapter_workspace_cleanup_due + ON runtime_adapter_workspace_cleanups( + next_attempt_at, + cleanup_claim_expires_at, + updated_at, + session_id, + adapter_workspace_id + ); diff --git a/src/worker/database.ts b/src/worker/database.ts index ed9cd46e..e219fdeb 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -221,6 +221,25 @@ export type InteractiveSessionTable = { export type InteractiveSessionRow = Selectable; +export type RuntimeAdapterWorkspaceCleanupTable = { + session_id: string; + adapter_workspace_id: string; + profile: string | null; + control_plane: string | null; + create_pending: number; + message: string; + reconcile_error: string | null; + attempt_count: Generated; + last_attempt_at: number | null; + next_attempt_at: number; + cleanup_claim: string | null; + cleanup_claim_expires_at: number | null; + created_at: number; + updated_at: number; +}; + +export type RuntimeAdapterWorkspaceCleanupRow = Selectable; + export type InteractiveSessionGrantTable = { session_id: string; subject: string; @@ -395,6 +414,7 @@ export type Database = { cards: CardTable; run_attempts: RunAttemptTable; interactive_sessions: InteractiveSessionTable; + runtime_adapter_workspace_cleanups: RuntimeAdapterWorkspaceCleanupTable; interactive_session_grants: InteractiveSessionGrantTable; openclaw_request_replays: OpenClawRequestReplayTable; interactive_session_events: InteractiveSessionEventTable; diff --git a/src/worker/provisioning/runtime-adapter-release-repository.ts b/src/worker/provisioning/runtime-adapter-release-repository.ts new file mode 100644 index 00000000..03082562 --- /dev/null +++ b/src/worker/provisioning/runtime-adapter-release-repository.ts @@ -0,0 +1,164 @@ +import { sql } from "kysely"; + +import { database, type RuntimeAdapterWorkspaceCleanupRow } from "../database.ts"; +import type { RuntimeEnv } from "../env.ts"; +import type { + RuntimeAdapterWorkspaceCleanup, + RuntimeAdapterWorkspaceRegistration, +} from "./runtime-adapter-release-service.ts"; + +const cleanupClaimTtlMs = 60_000; +const cleanupRetryDelayMs = 15_000; + +export async function stageRuntimeAdapterWorkspaceCleanup( + env: RuntimeEnv, + input: { + sessionId: string; + adapterWorkspaceId: string; + registration: RuntimeAdapterWorkspaceRegistration | null; + createPending: boolean; + now: number; + }, +): Promise { + await sql` + INSERT INTO runtime_adapter_workspace_cleanups ( + session_id, + adapter_workspace_id, + profile, + control_plane, + create_pending, + message, + reconcile_error, + next_attempt_at, + created_at, + updated_at + ) VALUES ( + ${input.sessionId}, + ${input.adapterWorkspaceId}, + ${input.registration?.profile ?? null}, + ${input.registration?.controlPlane ?? null}, + ${input.createPending ? 1 : 0}, + 'superseded runtime adapter cleanup pending', + NULL, + ${input.now}, + ${input.now}, + ${input.now} + ) + ON CONFLICT(session_id, adapter_workspace_id) DO NOTHING + `.execute(database(env)); +} + +export async function claimRuntimeAdapterWorkspaceCleanup( + env: RuntimeEnv, + sessionId: string, + adapterWorkspaceId: string, + now: number, +): Promise { + const claim = `runtime-cleanup:${crypto.randomUUID()}`; + const row = await database(env) + .updateTable("runtime_adapter_workspace_cleanups") + .set({ + cleanup_claim: claim, + cleanup_claim_expires_at: now + cleanupClaimTtlMs, + attempt_count: sql`attempt_count + 1`, + last_attempt_at: now, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where("session_id", "=", sessionId) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .where("next_attempt_at", "<=", now) + .where((expression) => + expression.or([ + expression("cleanup_claim", "is", null), + expression("cleanup_claim_expires_at", "<=", now), + ]), + ) + .returningAll() + .executeTakeFirst(); + return row ? cleanupClaim(row) : null; +} + +export async function claimRuntimeAdapterWorkspaceCleanupBatch( + env: RuntimeEnv, + now: number, + limit: number, +): Promise { + const candidates = await database(env) + .selectFrom("runtime_adapter_workspace_cleanups") + .select(["session_id", "adapter_workspace_id"]) + .where("next_attempt_at", "<=", now) + .where((expression) => + expression.or([ + expression("cleanup_claim", "is", null), + expression("cleanup_claim_expires_at", "<=", now), + ]), + ) + .orderBy("next_attempt_at", "asc") + .orderBy("updated_at", "asc") + .orderBy("session_id", "asc") + .orderBy("adapter_workspace_id", "asc") + .limit(limit) + .execute(); + const claims: RuntimeAdapterWorkspaceCleanup[] = []; + for (const candidate of candidates) { + const claimed = await claimRuntimeAdapterWorkspaceCleanup( + env, + candidate.session_id, + candidate.adapter_workspace_id, + now, + ); + if (claimed) claims.push(claimed); + } + return claims; +} + +export async function persistRuntimeAdapterWorkspaceCleanupEvidence( + env: RuntimeEnv, + cleanup: RuntimeAdapterWorkspaceCleanup, + message: string, + now: number, + reconcileError: string | null, +): Promise { + await database(env) + .updateTable("runtime_adapter_workspace_cleanups") + .set({ + message, + reconcile_error: reconcileError, + next_attempt_at: now + cleanupRetryDelayMs, + cleanup_claim: null, + cleanup_claim_expires_at: null, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where("session_id", "=", cleanup.sessionId) + .where("adapter_workspace_id", "=", cleanup.adapterWorkspaceId) + .where("cleanup_claim", "=", cleanup.claim) + .execute(); +} + +export async function completeRuntimeAdapterWorkspaceCleanup( + env: RuntimeEnv, + cleanup: RuntimeAdapterWorkspaceCleanup, +): Promise { + await database(env) + .deleteFrom("runtime_adapter_workspace_cleanups") + .where("session_id", "=", cleanup.sessionId) + .where("adapter_workspace_id", "=", cleanup.adapterWorkspaceId) + .where("cleanup_claim", "=", cleanup.claim) + .execute(); +} + +function cleanupClaim(row: RuntimeAdapterWorkspaceCleanupRow): RuntimeAdapterWorkspaceCleanup { + return { + sessionId: row.session_id, + adapterWorkspaceId: row.adapter_workspace_id, + registration: + row.profile && row.control_plane + ? { + profile: row.profile, + controlPlane: row.control_plane, + } + : null, + createPending: row.create_pending === 1, + claim: row.cleanup_claim ?? "", + }; +} diff --git a/src/worker/provisioning/runtime-adapter-release-service.ts b/src/worker/provisioning/runtime-adapter-release-service.ts index b769720d..f9305597 100644 --- a/src/worker/provisioning/runtime-adapter-release-service.ts +++ b/src/worker/provisioning/runtime-adapter-release-service.ts @@ -5,7 +5,35 @@ export type RuntimeAdapterWorkspaceRegistration = { controlPlane: string; }; +export type RuntimeAdapterWorkspaceCleanup = { + sessionId: string; + adapterWorkspaceId: string; + registration: RuntimeAdapterWorkspaceRegistration | null; + createPending: boolean; + claim: string; +}; + export type RuntimeAdapterReleaseServiceDependencies = { + stageCleanup(input: { + sessionId: string; + adapterWorkspaceId: string; + registration: RuntimeAdapterWorkspaceRegistration | null; + createPending: boolean; + now: number; + }): Promise; + claimCleanup( + sessionId: string, + adapterWorkspaceId: string, + now: number, + ): Promise; + claimPendingCleanups(now: number): Promise; + persistCleanupEvidence( + cleanup: RuntimeAdapterWorkspaceCleanup, + message: string, + now: number, + reconcileError: string | null, + ): Promise; + completeCleanup(cleanup: RuntimeAdapterWorkspaceCleanup): Promise; clearCreatePending(sessionId: string, adapterWorkspaceId: string): Promise; stopWorkspace( sessionId: string, @@ -43,11 +71,32 @@ export class RuntimeAdapterReleaseService { createPending: boolean; now: number; }): Promise { - const { sessionId, adapterWorkspaceId, registration, createPending, now } = input; - if (!createPending) { - await this.dependencies.clearCreatePending(sessionId, adapterWorkspaceId); + await this.dependencies.stageCleanup(input); + const cleanup = await this.dependencies.claimCleanup( + input.sessionId, + input.adapterWorkspaceId, + input.now, + ); + if (!cleanup) return; + await this.releaseCleanup(cleanup, input.now); + } + + async retryPending(now: number): Promise { + const cleanups = await this.dependencies.claimPendingCleanups(now); + for (const cleanup of cleanups) { + await this.releaseCleanup(cleanup, now); } + } + + private async releaseCleanup( + cleanup: RuntimeAdapterWorkspaceCleanup, + now: number, + ): Promise { + const { sessionId, adapterWorkspaceId, registration, createPending } = cleanup; try { + if (!createPending) { + await this.dependencies.clearCreatePending(sessionId, adapterWorkspaceId); + } const release = await this.dependencies.stopWorkspace( sessionId, adapterWorkspaceId, @@ -56,24 +105,36 @@ export class RuntimeAdapterReleaseService { ); if (release.status === "stopped") { await this.dependencies.confirmRelease(sessionId, adapterWorkspaceId, now, release.message); + await this.dependencies.completeCleanup(cleanup); return; } - await this.dependencies.persistStopEvidence( - sessionId, - adapterWorkspaceId, - release.message, - now, - null, - ); + await this.persistEvidence(cleanup, release.message, now, null); } catch (error) { const message = this.dependencies.providerError(error, adapterWorkspaceId); - await this.dependencies.persistStopEvidence( - sessionId, - adapterWorkspaceId, + await this.persistEvidence( + cleanup, `superseded runtime adapter stop pending: ${message}`, now, message, ); } } + + private async persistEvidence( + cleanup: RuntimeAdapterWorkspaceCleanup, + message: string, + now: number, + reconcileError: string | null, + ): Promise { + await this.dependencies.persistCleanupEvidence(cleanup, message, now, reconcileError); + await this.dependencies + .persistStopEvidence( + cleanup.sessionId, + cleanup.adapterWorkspaceId, + message, + now, + reconcileError, + ) + .catch(() => undefined); + } } diff --git a/src/worker/runtime-adapter-workspaces.ts b/src/worker/runtime-adapter-workspaces.ts index 6ef12771..eb674a6e 100644 --- a/src/worker/runtime-adapter-workspaces.ts +++ b/src/worker/runtime-adapter-workspaces.ts @@ -190,6 +190,7 @@ export class RuntimeAdapterWorkspaceLifecycle { retainedRegistration?: RuntimeAdapterWorkspaceRegistration | null, retainedCreatePending?: boolean, ): Promise { + const supersededCleanup = retainedRegistration !== undefined; const registration = retainedRegistration ? { adapter_control_plane: retainedRegistration.controlPlane, @@ -208,13 +209,18 @@ export class RuntimeAdapterWorkspaceLifecycle { registration?.profile ?? "", registration?.adapter_control_plane, ); - if (registration?.adapter_create_pending !== 0) { + if (registration?.adapter_create_pending !== 0 && !supersededCleanup) { return { status: "stopping", message: "runtime adapter stop waiting for create resolution", }; } - return this.stopWorkspace(registration?.profile ?? "", controlPlane, adapterWorkspaceId); + return this.stopWorkspace( + registration?.profile ?? "", + controlPlane, + adapterWorkspaceId, + supersededCleanup && registration?.adapter_create_pending !== 0, + ); } private async reconcileStopping( @@ -523,6 +529,7 @@ export class RuntimeAdapterWorkspaceLifecycle { profile: string, registeredControlPlane: string, adapterWorkspaceId: string, + retryMissing = false, ): Promise { const controlPlane = requireRegisteredRuntimeAdapterControlPlane( this.env, @@ -546,6 +553,12 @@ export class RuntimeAdapterWorkspaceLifecycle { const message = parsed?.message ?? redactedAdapterResponseMessage(body, fallbackMessage, [adapterWorkspaceId]); + if (response.status === 404 && retryMissing) { + return { + status: "stopping", + message: "runtime adapter workspace not yet visible; cleanup retry pending", + }; + } if (response.status === 404 || response.status === 204) { return { status: "stopped", message }; } diff --git a/src/worker/runtime-application.ts b/src/worker/runtime-application.ts index 5870ce10..50b6f27b 100644 --- a/src/worker/runtime-application.ts +++ b/src/worker/runtime-application.ts @@ -3,6 +3,13 @@ import { mapWithConcurrency } from "./concurrency.ts"; import type { InteractiveSessionRow } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; import { readAbandonedInteractiveSessionReservations } from "./openclaw-repository.ts"; +import { + claimRuntimeAdapterWorkspaceCleanup, + claimRuntimeAdapterWorkspaceCleanupBatch, + completeRuntimeAdapterWorkspaceCleanup, + persistRuntimeAdapterWorkspaceCleanupEvidence, + stageRuntimeAdapterWorkspaceCleanup, +} from "./provisioning/runtime-adapter-release-repository.ts"; import { safeProviderError } from "./provisioning/result.ts"; import { RuntimeAdapterReleaseService } from "./provisioning/runtime-adapter-release-service.ts"; import { @@ -186,6 +193,20 @@ export class RuntimeApplication { services.release, () => new RuntimeAdapterReleaseService({ + stageCleanup: (input) => stageRuntimeAdapterWorkspaceCleanup(this.env, input), + claimCleanup: (sessionId, adapterWorkspaceId, now) => + claimRuntimeAdapterWorkspaceCleanup(this.env, sessionId, adapterWorkspaceId, now), + claimPendingCleanups: (now) => + claimRuntimeAdapterWorkspaceCleanupBatch(this.env, now, runtimeAdapterReconcileLimit), + persistCleanupEvidence: (cleanup, message, now, reconcileError) => + persistRuntimeAdapterWorkspaceCleanupEvidence( + this.env, + cleanup, + message, + now, + reconcileError, + ), + completeCleanup: (cleanup) => completeRuntimeAdapterWorkspaceCleanup(this.env, cleanup), clearCreatePending: (sessionId, adapterWorkspaceId) => clearRuntimeAdapterCreatePending(this.env, sessionId, adapterWorkspaceId), stopWorkspace: (sessionId, adapterWorkspaceId, registration, createPending) => @@ -375,6 +396,7 @@ export class RuntimeApplication { } private async cleanupAbandonedPreparations(now: number): Promise { + await this.release().retryPending(now); const rows = await readAbandonedInteractiveSessionReservations( this.env, now - interactiveSessionPreparationStaleMs, diff --git a/tests/runtime-adapter-release-service.test.ts b/tests/runtime-adapter-release-service.test.ts index bc4fb143..985293a0 100644 --- a/tests/runtime-adapter-release-service.test.ts +++ b/tests/runtime-adapter-release-service.test.ts @@ -1,7 +1,16 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; import test from "node:test"; import type { RuntimeEnv } from "../src/worker/env.ts"; +import { + claimRuntimeAdapterWorkspaceCleanup, + claimRuntimeAdapterWorkspaceCleanupBatch, + completeRuntimeAdapterWorkspaceCleanup, + persistRuntimeAdapterWorkspaceCleanupEvidence, + stageRuntimeAdapterWorkspaceCleanup, +} from "../src/worker/provisioning/runtime-adapter-release-repository.ts"; import { clearRuntimeAdapterCreatePending, confirmRuntimeAdapterRelease, @@ -10,6 +19,7 @@ import { import { RuntimeAdapterReleaseService, type RuntimeAdapterReleaseServiceDependencies, + type RuntimeAdapterWorkspaceCleanup, type RuntimeAdapterWorkspaceRegistration, } from "../src/worker/provisioning/runtime-adapter-release-service.ts"; @@ -28,7 +38,25 @@ type PreparedStatement = { function releaseDependencies( overrides: Partial = {}, ): RuntimeAdapterReleaseServiceDependencies { + let stagedCleanup: RuntimeAdapterWorkspaceCleanup | null = null; return { + async stageCleanup(input) { + stagedCleanup = { + sessionId: input.sessionId, + adapterWorkspaceId: input.adapterWorkspaceId, + registration: input.registration, + createPending: input.createPending, + claim: "claim-1", + }; + }, + async claimCleanup() { + return stagedCleanup; + }, + async claimPendingCleanups() { + return []; + }, + async persistCleanupEvidence() {}, + async completeCleanup() {}, async clearCreatePending() {}, async stopWorkspace() { return { status: "stopped", message: "runtime workspace released" }; @@ -78,6 +106,18 @@ test("superseded release clears the create marker before stopping and confirming const calls: string[] = []; const service = new RuntimeAdapterReleaseService( releaseDependencies({ + async stageCleanup(input) { + calls.push(`stage:${input.sessionId}:${input.adapterWorkspaceId}`); + }, + async claimCleanup(sessionId, adapterWorkspaceId) { + return { + sessionId, + adapterWorkspaceId, + registration, + createPending: false, + claim: "claim-1", + }; + }, async clearCreatePending(sessionId, adapterWorkspaceId) { calls.push(`clear:${sessionId}:${adapterWorkspaceId}`); }, @@ -91,6 +131,9 @@ test("superseded release clears the create marker before stopping and confirming calls.push(`confirm:${sessionId}:${adapterWorkspaceId}:${now}:${message}`); return "stopped"; }, + async completeCleanup(cleanup) { + calls.push(`complete:${cleanup.sessionId}:${cleanup.adapterWorkspaceId}`); + }, }), ); @@ -103,9 +146,11 @@ test("superseded release clears the create marker before stopping and confirming }); assert.deepEqual(calls, [ + "stage:IS-101:fleet-a-is-101", "clear:IS-101:fleet-a-is-101", "stop:IS-101:fleet-a-is-101:default:https://adapter.example.test/:false", "confirm:IS-101:fleet-a-is-101:200:runtime workspace released", + "complete:IS-101:fleet-a-is-101", ]); }); @@ -170,6 +215,173 @@ test("superseded release records redacted provider failures for retry", async () ]); }); +test("superseded cleanup survives ownership loss and retries only the old workspace", async () => { + const replacementWorkspaceId = "fleet-a-is-101-replacement"; + const cleanupRows = new Map(); + const stopped: string[] = []; + const sessionEvidence: string[] = []; + let stopAttempts = 0; + const service = new RuntimeAdapterReleaseService( + releaseDependencies({ + async stageCleanup(input) { + cleanupRows.set(input.adapterWorkspaceId, { + sessionId: input.sessionId, + adapterWorkspaceId: input.adapterWorkspaceId, + registration: input.registration, + createPending: input.createPending, + claim: "claim-1", + }); + }, + async claimCleanup(_sessionId, adapterWorkspaceId) { + return cleanupRows.get(adapterWorkspaceId) ?? null; + }, + async claimPendingCleanups() { + return [...cleanupRows.values()].map((cleanup) => ({ + ...cleanup, + claim: "claim-2", + })); + }, + async stopWorkspace(_sessionId, adapterWorkspaceId) { + stopped.push(adapterWorkspaceId); + stopAttempts += 1; + return stopAttempts === 1 + ? { status: "stopping", message: "provider stop pending" } + : { status: "stopped", message: "provider workspace released" }; + }, + async persistCleanupEvidence(cleanup, message) { + cleanupRows.set(cleanup.adapterWorkspaceId, { + ...cleanup, + claim: "", + }); + assert.equal(message, "provider stop pending"); + }, + async persistStopEvidence(_sessionId, adapterWorkspaceId) { + if (adapterWorkspaceId === replacementWorkspaceId) { + sessionEvidence.push(adapterWorkspaceId); + } + }, + async confirmRelease(_sessionId, adapterWorkspaceId) { + assert.notEqual(adapterWorkspaceId, replacementWorkspaceId); + return null; + }, + async completeCleanup(cleanup) { + cleanupRows.delete(cleanup.adapterWorkspaceId); + }, + }), + ); + + await service.stopSuperseded({ + sessionId: "IS-101", + adapterWorkspaceId: "fleet-a-is-101-old", + registration, + createPending: true, + now: 200, + }); + assert.equal(cleanupRows.size, 1); + + await service.retryPending(300); + + assert.deepEqual(stopped, ["fleet-a-is-101-old", "fleet-a-is-101-old"]); + assert.deepEqual(sessionEvidence, []); + assert.equal(cleanupRows.size, 0); +}); + +test("superseded provider failures remain independently retryable", async () => { + const cleanupRows: RuntimeAdapterWorkspaceCleanup[] = []; + let fail = true; + const service = new RuntimeAdapterReleaseService( + releaseDependencies({ + async stageCleanup(input) { + cleanupRows.push({ + sessionId: input.sessionId, + adapterWorkspaceId: input.adapterWorkspaceId, + registration: input.registration, + createPending: input.createPending, + claim: "claim-1", + }); + }, + async claimCleanup() { + return cleanupRows[0] ?? null; + }, + async claimPendingCleanups() { + return cleanupRows; + }, + async stopWorkspace() { + if (fail) { + fail = false; + throw new Error("provider unavailable"); + } + return { status: "stopped", message: "provider workspace released" }; + }, + async persistCleanupEvidence(cleanup, message, _now, reconcileError) { + assert.equal(cleanup.adapterWorkspaceId, "fleet-a-is-101-old"); + assert.equal(message, "superseded runtime adapter stop pending: provider unavailable"); + assert.equal(reconcileError, "provider unavailable"); + }, + async completeCleanup() { + cleanupRows.length = 0; + }, + }), + ); + + await service.stopSuperseded({ + sessionId: "IS-101", + adapterWorkspaceId: "fleet-a-is-101-old", + registration, + createPending: true, + now: 200, + }); + assert.equal(cleanupRows.length, 1); + + await service.retryPending(300); + assert.equal(cleanupRows.length, 0); +}); + +test("runtime adapter cleanup storage is independent and claim fenced", async () => { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec( + readFileSync( + new URL("../migrations/0037_runtime_adapter_workspace_cleanup.sql", import.meta.url), + "utf8", + ), + ); + const env = sqliteRuntimeEnv(sqlite); + await stageRuntimeAdapterWorkspaceCleanup(env, { + sessionId: "IS-101", + adapterWorkspaceId: "fleet-a-is-101-old", + registration, + createPending: true, + now: 200, + }); + + const claimed = await claimRuntimeAdapterWorkspaceCleanup( + env, + "IS-101", + "fleet-a-is-101-old", + 200, + ); + assert.ok(claimed); + assert.equal(claimed.createPending, true); + assert.deepEqual(claimed.registration, registration); + assert.equal((await claimRuntimeAdapterWorkspaceCleanupBatch(env, 200, 3)).length, 0); + + await persistRuntimeAdapterWorkspaceCleanupEvidence( + env, + claimed, + "provider stop pending", + 200, + null, + ); + assert.equal((await claimRuntimeAdapterWorkspaceCleanupBatch(env, 15_199, 3)).length, 0); + const retry = await claimRuntimeAdapterWorkspaceCleanupBatch(env, 15_200, 3); + assert.equal(retry.length, 1); + await completeRuntimeAdapterWorkspaceCleanup(env, retry[0]); + assert.equal( + sqlite.prepare("SELECT COUNT(*) AS count FROM runtime_adapter_workspace_cleanups").get()?.count, + 0, + ); +}); + test("confirmed release waits for create resolution behind an exact lifecycle fence", async () => { let statements: PreparedStatement[] = []; const effects: string[] = []; @@ -325,3 +537,64 @@ function releaseEffects(calls: string[]): RuntimeAdapterReleaseEffects { }, }; } + +type BoundStatement = { + execute(): { + results: Record[]; + success: true; + meta: { changes: number; last_row_id?: number }; + }; +}; + +function sqliteRuntimeEnv(sqlite: DatabaseSync): RuntimeEnv { + function execute(sql: string, parameters: unknown[]) { + const statement = sqlite.prepare(sql); + if (/^\s*(?:select|pragma|with)\b|\breturning\b/i.test(sql)) { + const results = statement.all(...parameters).map((row) => ({ ...row })); + const changes = Number(sqlite.prepare("SELECT changes() AS changes").get()?.changes ?? 0); + return { results, success: true as const, meta: { changes } }; + } + const result = statement.run(...parameters); + return { + results: [], + success: true as const, + meta: { + changes: Number(result.changes), + last_row_id: Number(result.lastInsertRowid), + }, + }; + } + return { + DB: { + prepare(sql: string) { + return { + bind(...parameters: unknown[]) { + const bound = { + execute: () => execute(sql, parameters), + async all() { + return bound.execute(); + }, + async run() { + return bound.execute(); + }, + }; + return bound; + }, + }; + }, + async batch(statements: D1PreparedStatement[]) { + sqlite.exec("BEGIN IMMEDIATE"); + try { + const results = statements.map((statement) => + (statement as unknown as BoundStatement).execute(), + ); + sqlite.exec("COMMIT"); + return results; + } catch (error) { + sqlite.exec("ROLLBACK"); + throw error; + } + }, + } as unknown as D1Database, + } as RuntimeEnv; +} diff --git a/tests/runtime-adapter-workspaces.test.ts b/tests/runtime-adapter-workspaces.test.ts index c72bcd8b..5333af7f 100644 --- a/tests/runtime-adapter-workspaces.test.ts +++ b/tests/runtime-adapter-workspaces.test.ts @@ -401,6 +401,53 @@ test("superseded stop uses retained registration after the session row moves on" }); }); +test("superseded pending creates retry DELETE until the old workspace becomes visible", async () => { + const requests: Array<{ url: string; method: string | undefined }> = []; + let responseStatus = 404; + const service = new RuntimeAdapterWorkspaceLifecycle( + runtimeEnv(), + dependencies({ + async fetch(input, init) { + requests.push({ url: input, method: init.method }); + return responseStatus === 204 + ? new Response(null, { status: 204 }) + : Response.json({ message: "workspace not found" }, { status: responseStatus }); + }, + }), + ); + const registration = { + profile: "default", + controlPlane: "https://adapter.example.test/", + }; + + assert.deepEqual( + await service.stopForSession("IS-42", "workspace-superseded", registration, true), + { + status: "stopping", + message: "runtime adapter workspace not yet visible; cleanup retry pending", + }, + ); + + responseStatus = 204; + assert.deepEqual( + await service.stopForSession("IS-42", "workspace-superseded", registration, true), + { + status: "stopped", + message: "runtime adapter workspace released", + }, + ); + assert.deepEqual(requests, [ + { + url: "https://adapter.example.test/v1/workspaces/workspace-superseded", + method: "DELETE", + }, + { + url: "https://adapter.example.test/v1/workspaces/workspace-superseded", + method: "DELETE", + }, + ]); +}); + test("session-bound stop redacts provider credentials from failures", async () => { const env = runtimeEnv(() => [ { From 6d0216ca0d5e42712d5e1401b2d8ba09b988179c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:17:54 +0200 Subject: [PATCH 154/242] docs(changelog): record final lifecycle hardening --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9d556c0..c354b1c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, ownership-fenced repair of incomplete legacy lookup sets before rotation, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes, reject pending acknowledgements immediately when GitHub Actions runners disconnect or are replaced, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, generation-fence pending acknowledgements when GitHub Actions runners disconnect or are replaced, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. -- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown without scheduling retries when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; durably claim and retry superseded workspace cleanup without touching the replacement workspace; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure, ambiguous committed publication, and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown with bounded retries and no retry when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions with atomic capability publication, premature-response rejection, and fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. From aeb10c834c004196456e1ba566d37a43f9b91e69 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:32:14 +0200 Subject: [PATCH 155/242] fix(desktop): recover publication ownership safely --- ...0038_desktop_host_publication_identity.sql | 19 +++++++ src/worker/database.ts | 1 + src/worker/desktop-host-repository.ts | 26 +++++++++ src/worker/desktop-host-service.ts | 40 ++++++++++++++ src/worker/routes/control-plane.ts | 15 ++++++ src/worker/worker-application.ts | 6 ++- tests/control-plane-routes.test.ts | 26 +++++++-- tests/desktop-host-migration.test.ts | 44 +++++++++++++++ tests/desktop-host-repository.test.ts | 54 ++++++++++++++++++- tests/desktop-host-service.test.ts | 38 +++++++++++++ 10 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 migrations/0038_desktop_host_publication_identity.sql diff --git a/migrations/0038_desktop_host_publication_identity.sql b/migrations/0038_desktop_host_publication_identity.sql new file mode 100644 index 00000000..bc93a4cb --- /dev/null +++ b/migrations/0038_desktop_host_publication_identity.sql @@ -0,0 +1,19 @@ +ALTER TABLE desktop_hosts + ADD COLUMN publication_id TEXT NOT NULL DEFAULT ''; + +-- Older token-aware workers replace ownership_token without knowing about +-- publication_id. Clear the stale identity so a previous publisher cannot +-- recover authority over the replacement row. +CREATE TRIGGER IF NOT EXISTS clear_stale_desktop_host_publication_identity +AFTER UPDATE ON desktop_hosts +WHEN OLD.publication_id <> '' + AND NEW.ownership_token <> OLD.ownership_token + AND NEW.publication_id = OLD.publication_id +BEGIN + UPDATE desktop_hosts + SET publication_id = '' + WHERE owner_subject = NEW.owner_subject + AND id = NEW.id + AND ownership_token = NEW.ownership_token + AND publication_id = NEW.publication_id; +END; diff --git a/src/worker/database.ts b/src/worker/database.ts index e219fdeb..0608d6f8 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -70,6 +70,7 @@ export type DesktopHostTable = { address: string; port: number; ownership_token: string; + publication_id: string; created_at: number; updated_at: number; }; diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 5e1f840a..b24f30f2 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -11,6 +11,7 @@ export type DesktopHostRow = { address: string; port: number; ownershipToken: string; + publicationID: string; createdAt: number; updatedAt: number; }; @@ -20,6 +21,11 @@ export type DesktopHostWrite = DesktopHostRow; export interface DesktopHostStore { list(ownerSubject: string): Promise; upsert(host: DesktopHostWrite): Promise; + ownershipTokenForPublication( + ownerSubject: string, + id: string, + publicationID: string, + ): Promise; remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise; } @@ -46,6 +52,7 @@ export class DesktopHostRepository implements DesktopHostStore { address: row.address, port: row.port, ownershipToken: row.ownership_token, + publicationID: row.publication_id, createdAt: row.created_at, updatedAt: row.updated_at, })); @@ -62,6 +69,7 @@ export class DesktopHostRepository implements DesktopHostStore { address: host.address, port: host.port, ownership_token: host.ownershipToken, + publication_id: host.publicationID, created_at: host.createdAt, updated_at: host.updatedAt, }) @@ -74,6 +82,7 @@ export class DesktopHostRepository implements DesktopHostStore { address: host.address, port: host.port, ownership_token: host.ownershipToken, + publication_id: host.publicationID, updated_at: host.updatedAt, }) : update.doUpdateSet({ @@ -109,11 +118,28 @@ export class DesktopHostRepository implements DesktopHostStore { address: row.address, port: row.port, ownershipToken: row.ownership_token, + publicationID: row.publication_id, createdAt: row.created_at, updatedAt: row.updated_at, }; } + async ownershipTokenForPublication( + ownerSubject: string, + id: string, + publicationID: string, + ): Promise { + const row = await database(this.env) + .selectFrom("desktop_hosts") + .select("ownership_token") + .where("owner_subject", "=", ownerSubject) + .where("id", "=", id) + .where("publication_id", "=", publicationID) + .where("ownership_token", "<>", "") + .executeTakeFirst(); + return row?.ownership_token ?? null; + } + async remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise { const db = database(this.env); if (!ownershipToken) { diff --git a/src/worker/desktop-host-service.ts b/src/worker/desktop-host-service.ts index c6a8e4fb..3a8ed0bb 100644 --- a/src/worker/desktop-host-service.ts +++ b/src/worker/desktop-host-service.ts @@ -26,6 +26,7 @@ export type DesktopHostRegistration = { export const desktopHostOwnershipHeader = "x-crabfleet-ownership-token"; export const desktopHostOwnershipModeHeader = "x-crabfleet-ownership-mode"; +export const desktopHostPublicationHeader = "x-crabfleet-publication-id"; export const desktopHostTokenOwnershipMode = "token-v1"; export type DesktopHostOwnershipMode = "legacy" | typeof desktopHostTokenOwnershipMode; @@ -54,6 +55,7 @@ export class DesktopHostService { rawID: string, input: DesktopHostInput, ownershipMode: DesktopHostOwnershipMode = "legacy", + rawPublicationID: unknown = null, ): Promise { const id = desktopHostID(rawID); const name = boundedText(input.name, "name", 100); @@ -62,6 +64,10 @@ export class DesktopHostService { const now = this.now(); const ownershipToken = ownershipMode === desktopHostTokenOwnershipMode ? this.createOwnershipToken() : ""; + const publicationID = + ownershipMode === desktopHostTokenOwnershipMode + ? optionalDesktopHostPublicationID(rawPublicationID) + : ""; const host: DesktopHostRow = { ownerSubject: tenantSubject(user), id, @@ -70,6 +76,7 @@ export class DesktopHostService { address, port, ownershipToken, + publicationID, createdAt: now, updatedAt: now, }; @@ -80,6 +87,19 @@ export class DesktopHostService { return registration; } + async recover( + user: User, + rawID: string, + rawPublicationID: unknown, + ): Promise<{ ownershipToken: string | null }> { + const ownershipToken = await this.store.ownershipTokenForPublication( + tenantSubject(user), + desktopHostID(rawID), + desktopHostPublicationID(rawPublicationID), + ); + return { ownershipToken }; + } + async remove(user: User, rawID: string, rawOwnershipToken: unknown): Promise { await this.store.remove( tenantSubject(user), @@ -173,3 +193,23 @@ function desktopHostOwnershipToken(value: unknown): string | null { } return value; } + +function desktopHostPublicationID(value: unknown): string { + if ( + typeof value !== "string" || + value.length === 0 || + new TextEncoder().encode(value).byteLength > 200 || + [...value].some((character) => { + const codePoint = character.codePointAt(0) ?? 0; + return codePoint <= 0x20 || codePoint === 0x7f; + }) + ) { + throw badRequest("desktop host publication id is required"); + } + return value; +} + +function optionalDesktopHostPublicationID(value: unknown): string { + if (value === null || value === undefined) return ""; + return desktopHostPublicationID(value); +} diff --git a/src/worker/routes/control-plane.ts b/src/worker/routes/control-plane.ts index 5c97aa3c..7995ea3f 100644 --- a/src/worker/routes/control-plane.ts +++ b/src/worker/routes/control-plane.ts @@ -8,6 +8,7 @@ import type { import { desktopHostOwnershipHeader, desktopHostOwnershipModeHeader, + desktopHostPublicationHeader, desktopHostTokenOwnershipMode, type DesktopHostInput, type DesktopHostOwnershipMode, @@ -24,7 +25,13 @@ export type ControlPlaneRouteDependencies = { id: string, input: DesktopHostInput, ownershipMode: DesktopHostOwnershipMode, + publicationID: string | null, ): Promise; + recoverDesktopHost( + user: User, + id: string, + publicationID: unknown, + ): Promise<{ ownershipToken: string | null }>; removeDesktopHost(user: User, id: string, ownershipToken: string | null): Promise; searchGitHubRefs(number: unknown): Promise; createCard(request: Request, user: User): Promise; @@ -63,9 +70,17 @@ export async function handleControlPlaneRoute( request.headers.get(desktopHostOwnershipModeHeader) === desktopHostTokenOwnershipMode ? desktopHostTokenOwnershipMode : "legacy", + request.headers.get(desktopHostPublicationHeader), ); return json(registration); } + if (request.method === "POST" && desktopHostMatch && url.searchParams.get("recover") === "1") { + requireRole(user, "viewer"); + const body = await readJson<{ publicationID?: unknown }>(request); + return json( + await dependencies.recoverDesktopHost(user, decoded(desktopHostMatch[1]), body.publicationID), + ); + } if (request.method === "DELETE" && desktopHostMatch) { requireRole(user, "viewer"); const ownershipToken = request.headers.get(desktopHostOwnershipHeader); diff --git a/src/worker/worker-application.ts b/src/worker/worker-application.ts index 4ed453b2..87a60fe8 100644 --- a/src/worker/worker-application.ts +++ b/src/worker/worker-application.ts @@ -162,8 +162,10 @@ export class WorkerApplication { return { readState: (request, user) => this.readState(request, user, context), readFleet: (user) => this.readFleetState(user, undefined, context), - registerDesktopHost: (user, id, input, ownershipMode) => - this.desktopHosts().register(user, id, input, ownershipMode), + registerDesktopHost: (user, id, input, ownershipMode, publicationID) => + this.desktopHosts().register(user, id, input, ownershipMode, publicationID), + recoverDesktopHost: (user, id, publicationID) => + this.desktopHosts().recover(user, id, publicationID), removeDesktopHost: (user, id, ownershipToken) => this.desktopHosts().remove(user, id, ownershipToken), searchGitHubRefs: (number) => this.githubReferenceService().search(number), diff --git a/tests/control-plane-routes.test.ts b/tests/control-plane-routes.test.ts index 2aa2390e..7a5c00f1 100644 --- a/tests/control-plane-routes.test.ts +++ b/tests/control-plane-routes.test.ts @@ -45,8 +45,10 @@ function dependencies(calls: string[]): ControlPlaneRouteDependencies { calls.push(`fleet:${user.login}`); return { handler: "fleet" }; }, - async registerDesktopHost(user, id, input, ownershipMode) { - calls.push(`desktop-host:register:${user.login}:${id}:${input.name}:${ownershipMode}`); + async registerDesktopHost(user, id, input, ownershipMode, publicationID) { + calls.push( + `desktop-host:register:${user.login}:${id}:${input.name}:${ownershipMode}:${publicationID ?? "none"}`, + ); const registration = { host: { id, @@ -62,6 +64,12 @@ function dependencies(calls: string[]): ControlPlaneRouteDependencies { ? { ...registration, ownershipToken: "ownership-token" } : registration; }, + async recoverDesktopHost(user, id, publicationID) { + calls.push(`desktop-host:recover:${user.login}:${id}:${String(publicationID)}`); + return { + ownershipToken: publicationID === "publication-id" ? "ownership-token" : null, + }; + }, async removeDesktopHost(user, id, ownershipToken) { calls.push(`desktop-host:remove:${user.login}:${id}:${ownershipToken ?? "legacy"}`); }, @@ -165,6 +173,7 @@ test("desktop host routes register and remove only the authenticated user's host headers: { "content-type": "application/json", [desktopHostOwnershipModeHeader]: desktopHostTokenOwnershipMode, + "x-crabfleet-publication-id": "publication-id", }, body: JSON.stringify({ name: "Mac Studio", @@ -198,9 +207,18 @@ test("desktop host routes register and remove only the authenticated user's host calls, ); assert.equal(removed?.status, 200); + const recovered = await dispatch( + request("POST", "/api/desktop-hosts/mac%2Dstudio?recover=1", { + publicationID: "publication-id", + }), + viewer, + calls, + ); + assert.deepEqual(await recovered?.json(), { ownershipToken: "ownership-token" }); assert.deepEqual(calls, [ - "desktop-host:register:viewer:mac-studio:Mac Studio:token-v1", + "desktop-host:register:viewer:mac-studio:Mac Studio:token-v1:publication-id", "desktop-host:remove:viewer:mac-studio:ownership-token", + "desktop-host:recover:viewer:mac-studio:publication-id", ]); const legacyCalls: string[] = []; @@ -231,7 +249,7 @@ test("desktop host routes register and remove only the authenticated user's host ); assert.equal(legacyRemoved?.status, 200); assert.deepEqual(legacyCalls, [ - "desktop-host:register:viewer:legacy-studio:Legacy Studio:legacy", + "desktop-host:register:viewer:legacy-studio:Legacy Studio:legacy:none", "desktop-host:remove:viewer:legacy-studio:legacy", ]); }); diff --git a/tests/desktop-host-migration.test.ts b/tests/desktop-host-migration.test.ts index 0efc821d..e4a6863a 100644 --- a/tests/desktop-host-migration.test.ts +++ b/tests/desktop-host-migration.test.ts @@ -16,6 +16,12 @@ test("desktop host migration creates an owner-scoped registry with bounded ports database.exec(migration); database.exec(migration); database.exec(ownershipMigration); + database.exec( + readFileSync( + new URL("../migrations/0038_desktop_host_publication_identity.sql", import.meta.url), + "utf8", + ), + ); const insert = database.prepare(` INSERT INTO desktop_hosts @@ -36,6 +42,12 @@ test("desktop host migration creates an owner-scoped registry with bounded ports .get()?.ownership_token, "", ); + assert.equal( + database + .prepare("SELECT publication_id FROM desktop_hosts WHERE owner_subject = 'github:1'") + .get()?.publication_id, + "", + ); assert.throws( () => insert.run("github:3", "bad", "bad", "Bad", "100.64.1.4", 0, 1, 1), /constraint/i, @@ -48,6 +60,38 @@ test("desktop host migration creates an owner-scoped registry with bounded ports ); }); +test("desktop host publication migration clears identities rotated by old workers", () => { + const database = new DatabaseSync(":memory:"); + for (const migration of [ + "0030_desktop_hosts.sql", + "0033_desktop_host_ownership.sql", + "0038_desktop_host_publication_identity.sql", + ]) { + database.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); + } + database.exec(` + INSERT INTO desktop_hosts ( + owner_subject, id, owner, name, address, port, ownership_token, publication_id, + created_at, updated_at + ) VALUES ( + 'github:1', 'studio', 'alice', 'Studio', '100.64.1.2', 5901, + 'token-a', 'publication-a', 1, 2 + ); + UPDATE desktop_hosts + SET ownership_token = 'token-b' + WHERE owner_subject = 'github:1' AND id = 'studio'; + `); + + assert.deepEqual( + { + ...database + .prepare("SELECT ownership_token, publication_id FROM desktop_hosts WHERE id = 'studio'") + .get(), + }, + { ownership_token: "token-b", publication_id: "" }, + ); +}); + test("desktop host ownership migration blocks old-worker mutations of token-owned rows", () => { const database = new DatabaseSync(":memory:"); database.exec( diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index c14bea90..1048c4fb 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -77,6 +77,7 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec address: "100.64.1.2", port: 5901, ownership_token: "ownership-token", + publication_id: "publication-id", created_at: 1, updated_at: 2, }; @@ -113,6 +114,7 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec address: "100.64.1.2", port: 5901, ownershipToken: "ownership-token", + publicationID: "publication-id", createdAt: 1, updatedAt: 2, }, @@ -128,6 +130,7 @@ test("desktop host repository scopes reads, upserts, and deletes by owner subjec address: "100.64.1.2", port: 5901, ownershipToken: "ownership-token", + publicationID: "publication-id", createdAt: 1, updatedAt: 2, }); @@ -160,6 +163,7 @@ test("desktop host upsert returns the row written by the same atomic statement", address: "100.64.1.2", port: 5901, ownership_token: "token-a", + publication_id: "publication-a", created_at: 1, updated_at: 2, }; @@ -202,6 +206,7 @@ test("desktop host upsert returns the row written by the same atomic statement", address: written.address, port: written.port, ownershipToken: written.ownership_token, + publicationID: written.publication_id, createdAt: written.created_at, updatedAt: written.updated_at, }); @@ -222,6 +227,7 @@ test("legacy desktop host upserts preserve token ownership", async () => { address: "100.64.1.2", port: 5901, ownership_token: "current-token", + publication_id: "current-publication", created_at: 1, updated_at: 2, }; @@ -250,6 +256,7 @@ test("legacy desktop host upserts preserve token ownership", async () => { address: stored.address, port: stored.port, ownershipToken: "", + publicationID: "", createdAt: stored.created_at, updatedAt: stored.updated_at, }); @@ -275,12 +282,19 @@ test("legacy desktop host writes and cleanup cannot mutate token-owned rows", as sqlite.exec( readFileSync(new URL("../migrations/0033_desktop_host_ownership.sql", import.meta.url), "utf8"), ); + sqlite.exec( + readFileSync( + new URL("../migrations/0038_desktop_host_publication_identity.sql", import.meta.url), + "utf8", + ), + ); sqlite.exec(` INSERT INTO desktop_hosts ( - owner_subject, id, owner, name, address, port, ownership_token, created_at, updated_at + owner_subject, id, owner, name, address, port, ownership_token, publication_id, + created_at, updated_at ) VALUES ( 'github:1', 'studio', 'alice', 'Token Studio', '100.64.1.2', 5901, - 'current-token', 1, 2 + 'current-token', 'current-publication', 1, 2 ) `); const repository = new DesktopHostRepository(sqliteRuntimeEnv(sqlite)); @@ -293,6 +307,7 @@ test("legacy desktop host writes and cleanup cannot mutate token-owned rows", as address: "100.64.1.99", port: 5902, ownershipToken: "", + publicationID: "", createdAt: 10, updatedAt: 20, }); @@ -304,6 +319,7 @@ test("legacy desktop host writes and cleanup cannot mutate token-owned rows", as address: "100.64.1.2", port: 5901, ownershipToken: "current-token", + publicationID: "current-publication", createdAt: 1, updatedAt: 2, }); @@ -319,3 +335,37 @@ test("legacy desktop host writes and cleanup cannot mutate token-owned rows", as await repository.remove("github:1", "studio", "current-token"); assert.equal(sqlite.prepare("SELECT count(*) AS count FROM desktop_hosts").get()?.count, 0); }); + +test("desktop host publication recovery matches only the current publication", async () => { + const sqlite = new DatabaseSync(":memory:"); + for (const migration of [ + "0030_desktop_hosts.sql", + "0033_desktop_host_ownership.sql", + "0038_desktop_host_publication_identity.sql", + ]) { + sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); + } + const repository = new DesktopHostRepository(sqliteRuntimeEnv(sqlite)); + + await repository.upsert({ + ownerSubject: "github:1", + id: "studio", + owner: "alice", + name: "Studio", + address: "100.64.1.2", + port: 5901, + ownershipToken: "token-b", + publicationID: "publication-b", + createdAt: 1, + updatedAt: 2, + }); + + assert.equal( + await repository.ownershipTokenForPublication("github:1", "studio", "publication-a"), + null, + ); + assert.equal( + await repository.ownershipTokenForPublication("github:1", "studio", "publication-b"), + "token-b", + ); +}); diff --git a/tests/desktop-host-service.test.ts b/tests/desktop-host-service.test.ts index 18ca38d1..8c840d90 100644 --- a/tests/desktop-host-service.test.ts +++ b/tests/desktop-host-service.test.ts @@ -39,6 +39,15 @@ class MemoryDesktopHostStore implements DesktopHostStore { return stored; } + async ownershipTokenForPublication( + ownerSubject: string, + id: string, + publicationID: string, + ): Promise { + const row = this.rows.get(`${ownerSubject}:${id}`); + return row?.publicationID === publicationID ? row.ownershipToken : null; + } + async remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise { const key = `${ownerSubject}:${id}`; if (this.rows.get(key)?.ownershipToken === (ownershipToken ?? "")) { @@ -152,6 +161,7 @@ test("tokenless cleanup removes only migrated legacy desktop hosts", async () => address: "100.64.1.2", port: 5901, ownershipToken: "", + publicationID: "", createdAt: 1, updatedAt: 1, }; @@ -173,6 +183,34 @@ test("tokenless cleanup removes only migrated legacy desktop hosts", async () => assert.deepEqual(await service.list(alice), [registration.host]); }); +test("ambiguous desktop recovery cannot acquire a newer publication", async () => { + const store = new MemoryDesktopHostStore(); + const tokens = ["old-process-token", "new-process-token"]; + const service = new DesktopHostService( + store, + () => 42, + () => tokens.shift() ?? "unexpected-token", + ); + const input = { name: "Studio", address: "100.64.1.2", port: 5901 }; + + await service.register(alice, "studio", input, desktopHostTokenOwnershipMode, "publication-a"); + const newer = await service.register( + alice, + "studio", + { ...input, name: "Newer Studio" }, + desktopHostTokenOwnershipMode, + "publication-b", + ); + + assert.deepEqual(await service.recover(alice, "studio", "publication-a"), { + ownershipToken: null, + }); + assert.deepEqual(await service.list(alice), [newer.host]); + assert.deepEqual(await service.recover(alice, "studio", "publication-b"), { + ownershipToken: newer.ownershipToken, + }); +}); + test("legacy clients register tokenless rows they can remove after a server upgrade", async () => { const store = new MemoryDesktopHostStore(); const service = new DesktopHostService( From b11b9b5d5b6e04ed75494f540bf5840e459f56fe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:32:22 +0200 Subject: [PATCH 156/242] fix(macos): reconcile uncertain desktop publications --- .../CrabfleetDesktopRegistration.swift | 128 ++++++++- .../PrivateMacShareController.swift | 56 +++- .../PrivateMacShareTests.swift | 251 ++++++++++++++++-- 3 files changed, 397 insertions(+), 38 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index b2f19e48..de41c2f7 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -1,7 +1,12 @@ import Foundation protocol DesktopHostRegistering: Sendable { - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws } @@ -11,6 +16,12 @@ struct DesktopHostRegistrationResultUncertainError: LocalizedError, Equatable, S var errorDescription: String? { message } } +struct DesktopHostRegistrationSupersededError: LocalizedError, Equatable, Sendable { + var errorDescription: String? { + "The previous desktop publication is no longer current." + } +} + actor DesktopHostRegistrationCoordinator { private let registration: any DesktopHostRegistering private var pendingOperation: Task? @@ -19,10 +30,26 @@ actor DesktopHostRegistrationCoordinator { self.registration = registration } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + let registration = self.registration + let operation = enqueue { + try await registration.register( + identity: identity, + port: port, + publicationID: publicationID + ) + } + return try await operation.value + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { let registration = self.registration let operation = enqueue { - try await registration.register(identity: identity, port: port) + try await registration.recover(identity: identity, publicationID: publicationID) } return try await operation.value } @@ -80,10 +107,19 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable let port: UInt16 } + private struct RecoveryBody: Encodable { + let publicationID: String + } + + private struct RecoveryResponse: Decodable { + let ownershipToken: String? + } + private let baseURL: URL private let sessionCookie: String private let transport: any HTTPDataTransport static let ownershipModeHeader = "X-Crabfleet-Ownership-Mode" + static let publicationIDHeader = "X-Crabfleet-Publication-ID" static let tokenOwnershipMode = "token-v1" init?( @@ -111,8 +147,16 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable self.transport = transport } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { - let request = try registrationRequest(identity: identity, port: port) + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + let request = try registrationRequest( + identity: identity, + port: port, + publicationID: publicationID + ) let data: Data let http: HTTPURLResponse do { @@ -148,6 +192,44 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable return response.ownershipToken } + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + let request = try recoveryRequest(identity: identity, publicationID: publicationID) + let data: Data + let http: HTTPURLResponse + do { + (data, http) = try await transport.data(for: request) + } catch { + throw DesktopHostRegistrationResultUncertainError(message: error.localizedDescription) + } + if http.statusCode == 404 { + try validate(response: http, for: request, acceptingNotFound: true) + return nil + } + do { + try validate(response: http, for: request, acceptingNotFound: false) + } catch let error as DesktopHostRegistrationError { + if case .httpStatus(let status) = error, status >= 500 { + throw DesktopHostRegistrationResultUncertainError( + message: error.localizedDescription + ) + } + throw error + } + guard let response = try? JSONDecoder().decode(RecoveryResponse.self, from: data) else { + throw DesktopHostRegistrationResultUncertainError( + message: DesktopHostRegistrationError.invalidResponse.localizedDescription + ) + } + if let ownershipToken = response.ownershipToken, + !Self.isValidOwnershipToken(ownershipToken) + { + throw DesktopHostRegistrationResultUncertainError( + message: DesktopHostRegistrationError.invalidResponse.localizedDescription + ) + } + return response.ownershipToken + } + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { let request = try removalRequest(identity: identity, ownershipToken: ownershipToken) let (_, http) = try await transport.data(for: request) @@ -169,7 +251,14 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable } } - func registrationRequest(identity: TailnetIdentity, port: UInt16) throws -> URLRequest { + func registrationRequest( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) throws -> URLRequest { + guard Self.isValidOwnershipToken(publicationID) else { + throw DesktopHostRegistrationError.invalidResponse + } let hostID = Self.hostID(identity: identity) let url = baseURL @@ -183,6 +272,7 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable request.setValue("application/json", forHTTPHeaderField: "Accept") request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") request.setValue(Self.tokenOwnershipMode, forHTTPHeaderField: Self.ownershipModeHeader) + request.setValue(publicationID, forHTTPHeaderField: Self.publicationIDHeader) request.httpBody = try JSONEncoder().encode( RegistrationBody( name: identity.hostName.isEmpty ? identity.dnsName : identity.hostName, @@ -192,6 +282,32 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable return request } + func recoveryRequest(identity: TailnetIdentity, publicationID: String) throws -> URLRequest { + guard Self.isValidOwnershipToken(publicationID) else { + throw DesktopHostRegistrationError.invalidResponse + } + var components = URLComponents( + url: + baseURL + .appending(path: "api") + .appending(path: "desktop-hosts") + .appending(path: Self.hostID(identity: identity)), + resolvingAgainstBaseURL: false + ) + components?.queryItems = [URLQueryItem(name: "recover", value: "1")] + guard let url = components?.url else { + throw DesktopHostRegistrationError.invalidResponse + } + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.timeoutInterval = 15 + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") + request.httpBody = try JSONEncoder().encode(RecoveryBody(publicationID: publicationID)) + return request + } + func removalRequest(identity: TailnetIdentity, ownershipToken: String?) throws -> URLRequest { if let ownershipToken, !Self.isValidOwnershipToken(ownershipToken) { throw DesktopHostRegistrationError.invalidResponse diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 791d08a9..81aa721e 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -41,6 +41,7 @@ final class DesktopHostRegistrationLifecycle { private struct RegistrationTarget: Equatable { let identity: TailnetIdentity let port: UInt16 + let publicationID: String } private struct PublishedRegistration: Equatable { @@ -49,19 +50,47 @@ final class DesktopHostRegistrationLifecycle { } private let coordinator: DesktopHostRegistrationCoordinator + private let createPublicationID: () -> String private var publishedRegistration: PublishedRegistration? private var uncertainRegistrations: [RegistrationTarget] = [] private var pendingRemovals: [PublishedRegistration] = [] - init(registration: any DesktopHostRegistering) { + init( + registration: any DesktopHostRegistering, + createPublicationID: @escaping () -> String = { UUID().uuidString } + ) { coordinator = DesktopHostRegistrationCoordinator(registration: registration) + self.createPublicationID = createPublicationID } func publish(identity: TailnetIdentity, port: UInt16) async throws { - let target = RegistrationTarget(identity: identity, port: port) + let target = + uncertainRegistrations.first { + $0.identity == identity && $0.port == port + } + ?? RegistrationTarget( + identity: identity, + port: port, + publicationID: createPublicationID() + ) let ownershipToken: String? do { - ownershipToken = try await coordinator.register(identity: identity, port: port) + if uncertainRegistrations.contains(target) { + ownershipToken = try await coordinator.recover( + identity: identity, + publicationID: target.publicationID + ) + guard ownershipToken != nil else { + uncertainRegistrations.removeAll { $0 == target } + throw DesktopHostRegistrationSupersededError() + } + } else { + ownershipToken = try await coordinator.register( + identity: identity, + port: port, + publicationID: target.publicationID + ) + } } catch { if error is DesktopHostRegistrationResultUncertainError, !uncertainRegistrations.contains(target) @@ -88,16 +117,18 @@ final class DesktopHostRegistrationLifecycle { let uncertainRegistrations = uncertainRegistrations for target in uncertainRegistrations { do { - let ownershipToken = try await coordinator.register( - identity: target.identity, - port: target.port - ) - let recovered = PublishedRegistration( + let ownershipToken = try await coordinator.recover( identity: target.identity, - ownershipToken: ownershipToken + publicationID: target.publicationID ) - if !pendingRemovals.contains(recovered) { - pendingRemovals.append(recovered) + if let ownershipToken { + let recovered = PublishedRegistration( + identity: target.identity, + ownershipToken: ownershipToken + ) + if !pendingRemovals.contains(recovered) { + pendingRemovals.append(recovered) + } } self.uncertainRegistrations.removeAll { $0 == target } } catch { @@ -239,7 +270,8 @@ final class PrivateMacShareController: ObservableObject { ) { self.desktopRegistration = desktopRegistration desktopRegistrationLifecycle = - registrationLifecycle ?? desktopRegistration.map(DesktopHostRegistrationLifecycle.init) + registrationLifecycle + ?? desktopRegistration.map { DesktopHostRegistrationLifecycle(registration: $0) } self.defaults = defaults registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished let savedDisplayID = defaults.object(forKey: Self.selectedDisplayDefaultsKey) as? Int diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index fe117efa..eb9904f3 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -308,7 +308,11 @@ struct PrivateMacShareTests { let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) let publish = Task { - _ = try await coordinator.register(identity: identity, port: 5_901) + _ = try await coordinator.register( + identity: identity, + port: 5_901, + publicationID: "publication-id" + ) } #expect(await waitUntilAsync { await registration.hasStartedRegistration }) publish.cancel() @@ -441,7 +445,7 @@ struct PrivateMacShareTests { } @Test @MainActor - func ambiguousDesktopPublicationIsReacquiredBeforeCleanup() async throws { + func ambiguousDesktopPublicationIsRecoveredBeforeCleanup() async throws { let identity = desktopIdentity(name: "ambiguous-publish", address: "100.64.12.46") let registration = AmbiguousDesktopRegistration() let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) @@ -455,12 +459,45 @@ struct PrivateMacShareTests { await registration.events == [ .register(identity.dnsName), - .register(identity.dnsName), - .unregister(identity.dnsName, "reacquired-token"), + .recover(identity.dnsName), + .unregister(identity.dnsName, "recovered-token"), ] ) } + @Test @MainActor + func ambiguousDesktopCleanupPreservesANewerPublisher() async throws { + let identity = desktopIdentity(name: "shared-host", address: "100.64.12.47") + let registration = TwoProcessDesktopRegistration(lostPublicationID: "publication-a") + let firstLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "publication-a" } + ) + let secondLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "publication-b" } + ) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await firstLifecycle.publish(identity: identity, port: 5_901) + } + try await secondLifecycle.publish(identity: identity, port: 5_901) + try await firstLifecycle.removePublishedIdentities() + + #expect(await registration.activePublicationID == "publication-b") + #expect( + await registration.events + == [ + .register("publication-a"), + .register("publication-b"), + .recover("publication-a"), + ] + ) + + try await secondLifecycle.removePublishedIdentities() + #expect(await registration.activePublicationID == nil) + } + @Test @MainActor func failedDesktopRemovalSurvivesLaterIdentityChanges() async throws { let first = desktopIdentity(name: "first-host", address: "100.64.12.41") @@ -722,7 +759,11 @@ struct PrivateMacShareTests { userID: 42 ) - let request = try registration.registrationRequest(identity: identity, port: 5901) + let request = try registration.registrationRequest( + identity: identity, + port: 5901, + publicationID: "publication-id" + ) #expect(request.url?.absoluteString == "https://fleet.example/api/desktop-hosts/workstation-1") #expect(request.httpMethod == "PUT") #expect(request.value(forHTTPHeaderField: "Cookie") == "crabbox_session=secret") @@ -730,6 +771,10 @@ struct PrivateMacShareTests { request.value(forHTTPHeaderField: CrabfleetDesktopRegistration.ownershipModeHeader) == CrabfleetDesktopRegistration.tokenOwnershipMode ) + #expect( + request.value(forHTTPHeaderField: CrabfleetDesktopRegistration.publicationIDHeader) + == "publication-id" + ) let body = try #require(request.httpBody) let json = try #require(JSONSerialization.jsonObject(with: body) as? [String: Any]) #expect(json["name"] as? String == "Workstation") @@ -776,7 +821,11 @@ struct PrivateMacShareTests { let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) #expect( - try await registration.register(identity: identity, port: 5_901) + try await registration.register( + identity: identity, + port: 5_901, + publicationID: "publication-id" + ) == "server-ownership-token" ) } @@ -806,11 +855,88 @@ struct PrivateMacShareTests { )) let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) - #expect(try await registration.register(identity: identity, port: 5_901) == nil) + #expect( + try await registration.register( + identity: identity, + port: 5_901, + publicationID: "publication-id" + ) == nil + ) let removal = try registration.removalRequest(identity: identity, ownershipToken: nil) #expect(removal.value(forHTTPHeaderField: "X-Crabfleet-Ownership-Token") == nil) } + @Test + func desktopRegistrationRecoversOnlyTheMatchingPublication() async throws { + let transport = DesktopRegistrationTransport { request in + let responseURL = try #require(request.url) + #expect(request.httpMethod == "POST") + #expect(responseURL.query == "recover=1") + let body = try #require(request.httpBody) + let json = try #require(JSONSerialization.jsonObject(with: body) as? [String: Any]) + #expect(json["publicationID"] as? String == "publication-id") + return ( + Data(#"{"ownershipToken":"server-ownership-token"}"#.utf8), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + #expect( + try await registration.recover( + identity: identity, + publicationID: "publication-id" + ) == "server-ownership-token" + ) + } + + @Test + func desktopRegistrationTreatsMissingRecoveryRouteAsNoOwnership() async throws { + let transport = DesktopRegistrationTransport { request in + let responseURL = try #require(request.url) + return ( + Data(), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 404, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + #expect( + try await registration.recover( + identity: identity, + publicationID: "publication-id" + ) == nil + ) + } + @Test func desktopRegistrationTreatsMalformedCommittedResponsesAsUncertain() async throws { let transport = DesktopRegistrationTransport { request in @@ -837,7 +963,11 @@ struct PrivateMacShareTests { let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { - try await registration.register(identity: identity, port: 5_901) + try await registration.register( + identity: identity, + port: 5_901, + publicationID: "publication-id" + ) } } @@ -857,7 +987,11 @@ struct PrivateMacShareTests { let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { - try await registration.register(identity: identity, port: 5_901) + try await registration.register( + identity: identity, + port: 5_901, + publicationID: "publication-id" + ) } } @@ -887,7 +1021,11 @@ struct PrivateMacShareTests { let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) await #expect(throws: DesktopHostRegistrationError.redirectRejected) { - try await registration.register(identity: identity, port: 5_901) + try await registration.register( + identity: identity, + port: 5_901, + publicationID: "publication-id" + ) } } @@ -1609,7 +1747,11 @@ private actor SuspendedDesktopRegistration: DesktopHostRegistering { registrationContinuation != nil } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { events.append(.registerStarted) await withCheckedContinuation { continuation in registrationContinuation = continuation @@ -1618,6 +1760,10 @@ private actor SuspendedDesktopRegistration: DesktopHostRegistering { return "registration-token" } + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + nil + } + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { #expect(ownershipToken == "registration-token") events.append(.unregisterStarted) @@ -1651,7 +1797,11 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { self.unregisterFailures = unregisterFailures } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { events.append(.register(identity.dnsName)) if consumeFailure(for: identity.dnsName, from: ®isterFailures) { throw DesktopRegistrationTestError.failed @@ -1659,6 +1809,10 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { return "token:\(identity.dnsName)" } + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + nil + } + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { events.append(.unregister(identity.dnsName, ownershipToken)) if consumeFailure(for: identity.dnsName, from: &unregisterFailures) { @@ -1679,23 +1833,72 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { private actor AmbiguousDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String) + case recover(String) case unregister(String, String?) } private(set) var events: [Event] = [] - private var registerCount = 0 - - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { events.append(.register(identity.dnsName)) - registerCount += 1 - if registerCount == 1 { + throw DesktopHostRegistrationResultUncertainError(message: "response lost") + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + events.append(.recover(identity.dnsName)) + return "recovered-token" + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + events.append(.unregister(identity.dnsName, ownershipToken)) + } +} + +private actor TwoProcessDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register(String) + case recover(String) + case unregister(String) + } + + private let lostPublicationID: String + private var activeOwnershipToken: String? + private(set) var activePublicationID: String? + private(set) var events: [Event] = [] + + init(lostPublicationID: String) { + self.lostPublicationID = lostPublicationID + } + + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + events.append(.register(publicationID)) + let ownershipToken = "token:\(publicationID)" + activePublicationID = publicationID + activeOwnershipToken = ownershipToken + if publicationID == lostPublicationID { throw DesktopHostRegistrationResultUncertainError(message: "response lost") } - return "reacquired-token" + return ownershipToken + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + events.append(.recover(publicationID)) + guard activePublicationID == publicationID else { return nil } + return activeOwnershipToken } func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { - events.append(.unregister(identity.dnsName, ownershipToken)) + guard ownershipToken == activeOwnershipToken else { return } + events.append(.unregister(ownershipToken ?? "")) + activePublicationID = nil + activeOwnershipToken = nil } } @@ -1706,10 +1909,18 @@ private actor SuspendedDesktopCleanupRegistration: DesktopHostRegistering { unregistrationContinuation != nil } - func register(identity: TailnetIdentity, port: UInt16) async throws -> String? { + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { "slow-cleanup-token" } + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + nil + } + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { #expect(ownershipToken == "slow-cleanup-token") await withCheckedContinuation { continuation in From 816dc731c81d26e70eb01f0755cf11a6ffe8669e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:36:10 +0200 Subject: [PATCH 157/242] fix(actions): fence relay runner generations --- src/github-actions-runner.ts | 2 + src/github-actions-runtime.ts | 241 ++++++++++++++++++--- src/worker/interactive-terminal-service.ts | 2 + src/worker/session-control-do.ts | 20 +- src/worker/terminal-hub.ts | 103 +++++++-- tests/application-architecture.test.ts | 2 +- tests/github-actions-event-auth.test.ts | 4 + tests/github-actions-runner.test.ts | 18 ++ tests/github-actions-runtime.test.ts | 148 ++++++++++++- tests/terminal-hub.test.ts | 124 ++++++++++- 10 files changed, 603 insertions(+), 61 deletions(-) diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 788927b1..8b54ff7d 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -24,11 +24,13 @@ export async function acceptGitHubActionsRunnerInput( sendGitHubActionsRelayInputAcknowledgement(socket, { inputId: input.inputId, accepted: true, + ...(input.generation ? { generation: input.generation } : {}), }); } catch { sendGitHubActionsRelayInputAcknowledgement(socket, { inputId: input.inputId, accepted: false, + ...(input.generation ? { generation: input.generation } : {}), }); } return true; diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 6189221b..5b5b47ab 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -21,19 +21,24 @@ export type GitHubActionsRelaySocket = { export type GitHubActionsRelayInputAcknowledgement = { accepted: boolean; error?: string; + generation?: string; inputId: string; }; export type GitHubActionsRelayInput = { + generation?: string; inputId: string; payload: ArrayBuffer; }; export const githubActionsFramedRunnerCapability = "cfr1-framed-io-v1"; +export const githubActionsGenerationFencedCapability = "cfr1-framed-io-v2"; export const githubActionsRunnerProtocolQuery = "runnerProtocol"; export const githubActionsViewerProtocolQuery = "viewerProtocol"; export const githubActionsViewerProtocolHeader = "x-crabfleet-viewer-protocol"; -export type GitHubActionsRelayProtocol = typeof githubActionsFramedRunnerCapability; +export type GitHubActionsRelayProtocol = + | typeof githubActionsFramedRunnerCapability + | typeof githubActionsGenerationFencedCapability; export type GitHubActionsRunnerProtocol = GitHubActionsRelayProtocol; export type GitHubActionsViewerProtocol = GitHubActionsRelayProtocol; @@ -70,8 +75,14 @@ const relayInputFrameType = 1; const relayInputAcknowledgementFrameType = 2; const relayEventFrameType = 3; const relayOutputFrameType = 4; +const relayGenerationInputFrameType = 5; +const relayGenerationInputAcknowledgementFrameType = 6; +const relayGenerationEventFrameType = 7; const relayInputIdMaximumBytes = 80; const relayInputIdPattern = /^[A-Za-z0-9_-]+$/; +const relayGenerationMaximumBytes = 80; +const relayGenerationPattern = /^[A-Za-z0-9_-]+$/; +export const githubActionsLegacyRelayGeneration = "legacy"; const relayEventCodes = { runner_connected: 1, runner_disconnected: 2, @@ -87,6 +98,7 @@ const encoder = new TextEncoder(); const decoder = new TextDecoder(); type GitHubActionsRelayAttachment = { + generation?: string; protocol?: GitHubActionsRelayProtocol; }; @@ -110,13 +122,18 @@ export function buildGitHubActionsRunnerPtyUrl( export function buildGitHubActionsViewerRelayUrl(): string { const url = new URL("https://crabfleet.internal/api/session-control/github-actions/viewer"); - url.searchParams.set(githubActionsViewerProtocolQuery, githubActionsFramedRunnerCapability); + url.searchParams.set(githubActionsViewerProtocolQuery, githubActionsGenerationFencedCapability); return url.toString(); } export function gitHubActionsViewerResponseUsesFramedProtocol(response: Response): boolean { + return isGitHubActionsRelayProtocol(response.headers.get(githubActionsViewerProtocolHeader)); +} + +export function gitHubActionsViewerResponseUsesGenerations(response: Response): boolean { return ( - response.headers.get(githubActionsViewerProtocolHeader) === githubActionsFramedRunnerCapability + response.headers.get(githubActionsViewerProtocolHeader) === + githubActionsGenerationFencedCapability ); } @@ -200,40 +217,91 @@ export function relayGitHubActionsWebSocketMessage( if (framedViewer && !input) return 0; const runner = runners.find((socket) => socket.readyState === webSocketOpen); if (!runner) { - sendGitHubActionsViewerInputAcknowledgement(senderSocket, input?.inputId ?? null, false); + sendGitHubActionsViewerInputAcknowledgement( + senderSocket, + input?.inputId ?? null, + false, + input?.generation, + ); + return 0; + } + const generation = gitHubActionsRelayGeneration(runner) ?? githubActionsLegacyRelayGeneration; + if ( + gitHubActionsRelayUsesGenerations(senderSocket) && + (!input?.generation || input.generation !== generation) + ) { + sendGitHubActionsViewerInputAcknowledgement( + senderSocket, + input?.inputId ?? null, + false, + input?.generation, + ); return 0; } const framed = gitHubActionsRunnerUsesFramedProtocol(runner); try { if (framed) { runner.send( - framedViewer - ? message - : encodeGitHubActionsRelayInput(createGitHubActionsRelayInputId(), message), + encodeGitHubActionsRelayInput( + input?.inputId ?? createGitHubActionsRelayInputId(), + input?.payload ?? message, + gitHubActionsRelayUsesGenerations(runner) ? generation : undefined, + ), ); } else { runner.send(framedViewer ? input!.payload : message); } if (!framedViewer || !framed) { - sendGitHubActionsViewerInputAcknowledgement(senderSocket, input?.inputId ?? null, true); + sendGitHubActionsViewerInputAcknowledgement( + senderSocket, + input?.inputId ?? null, + true, + input?.generation, + ); } return 1; } catch { - sendGitHubActionsViewerInputAcknowledgement(senderSocket, input?.inputId ?? null, false); + sendGitHubActionsViewerInputAcknowledgement( + senderSocket, + input?.inputId ?? null, + false, + input?.generation, + ); return 0; } } + if (runners.find((socket) => socket.readyState === webSocketOpen) !== senderSocket) { + return 0; + } + if (gitHubActionsRunnerUsesFramedProtocol(senderSocket)) { const acknowledgement = parseGitHubActionsRelayInputAcknowledgement(message); const output = parseGitHubActionsRelayOutput(message); if (!acknowledgement && !output) return 0; + const generation = gitHubActionsRelayGeneration(senderSocket); + if ( + acknowledgement && + gitHubActionsRelayUsesGenerations(senderSocket) && + acknowledgement.generation !== generation + ) { + return 0; + } let forwarded = 0; for (const viewer of viewers) { if (viewer.readyState !== webSocketOpen) continue; if (acknowledgement && !gitHubActionsViewerUsesFramedProtocol(viewer)) continue; try { - viewer.send(gitHubActionsViewerUsesFramedProtocol(viewer) ? message : output!); + viewer.send( + acknowledgement + ? encodeGitHubActionsRelayInputAcknowledgement({ + ...acknowledgement, + ...(gitHubActionsRelayUsesGenerations(viewer) && generation ? { generation } : {}), + }) + : gitHubActionsViewerUsesFramedProtocol(viewer) + ? message + : output!, + ); forwarded += 1; } catch { // A failed viewer does not prevent delivery to the remaining viewers. @@ -275,7 +343,15 @@ export function sendGitHubActionsRelayInputAcknowledgement( export function parseGitHubActionsRelayInputAcknowledgement( message: string | ArrayBuffer, ): GitHubActionsRelayInputAcknowledgement | null { - const frame = decodeGitHubActionsRelayFrame(message, relayInputAcknowledgementFrameType); + const legacyFrame = decodeGitHubActionsRelayFrame(message, relayInputAcknowledgementFrameType); + const generatedFrame = decodeGitHubActionsRelayFrame( + message, + relayGenerationInputAcknowledgementFrameType, + ); + const decoded = generatedFrame + ? decodeGitHubActionsRelayGeneration(generatedFrame.payload) + : null; + const frame = legacyFrame ?? (decoded ? { ...generatedFrame!, payload: decoded.payload } : null); if (!frame?.inputId || frame.payload.byteLength < 1) return null; const acceptedByte = frame.payload[0]; if (acceptedByte !== 0 && acceptedByte !== 1) return null; @@ -284,6 +360,7 @@ export function parseGitHubActionsRelayInputAcknowledgement( return { inputId: frame.inputId, accepted: true, + ...(decoded ? { generation: decoded.generation } : {}), }; } const error = decoder.decode(frame.payload.subarray(1)).trim(); @@ -291,6 +368,7 @@ export function parseGitHubActionsRelayInputAcknowledgement( inputId: frame.inputId, accepted: false, error: error || relayInputRejectedError, + ...(decoded ? { generation: decoded.generation } : {}), }; } @@ -301,9 +379,16 @@ export function createGitHubActionsRelayInputId(): string { export function encodeGitHubActionsRelayInput( inputId: string, payload: string | ArrayBuffer | ArrayBufferView, + generation?: string, ): ArrayBuffer { requireGitHubActionsRelayInputId(inputId); - return encodeGitHubActionsRelayFrame(relayInputFrameType, inputId, messageBytes(payload)); + return encodeGitHubActionsRelayFrame( + generation ? relayGenerationInputFrameType : relayInputFrameType, + inputId, + generation + ? encodeGitHubActionsRelayGeneration(generation, messageBytes(payload)) + : messageBytes(payload), + ); } export function encodeGitHubActionsRelayOutput( @@ -321,32 +406,44 @@ export function parseGitHubActionsRelayOutput(message: string | ArrayBuffer): Ar export function parseGitHubActionsRelayInput( message: string | ArrayBuffer, ): GitHubActionsRelayInput | null { - const frame = decodeGitHubActionsRelayFrame(message, relayInputFrameType); + const legacyFrame = decodeGitHubActionsRelayFrame(message, relayInputFrameType); + const generatedFrame = decodeGitHubActionsRelayFrame(message, relayGenerationInputFrameType); + const decoded = generatedFrame + ? decodeGitHubActionsRelayGeneration(generatedFrame.payload) + : null; + const frame = legacyFrame ?? (decoded ? { ...generatedFrame!, payload: decoded.payload } : null); if (!frame?.inputId) return null; return { inputId: frame.inputId, payload: Uint8Array.from(frame.payload).buffer, + ...(decoded ? { generation: decoded.generation } : {}), }; } export function parseGitHubActionsRunnerProtocol( value: string | null, ): GitHubActionsRunnerProtocol | null { - return value === githubActionsFramedRunnerCapability ? githubActionsFramedRunnerCapability : null; + return isGitHubActionsRelayProtocol(value) ? value : null; } export function parseGitHubActionsViewerProtocol( value: string | null, ): GitHubActionsViewerProtocol | null { - return value === githubActionsFramedRunnerCapability ? githubActionsFramedRunnerCapability : null; + return isGitHubActionsRelayProtocol(value) ? value : null; } export function attachGitHubActionsRunnerProtocol( socket: GitHubActionsRelaySocket, protocol: GitHubActionsRunnerProtocol | null, + generation?: string, ): void { socket.serializeAttachment?.( - protocol ? ({ protocol } satisfies GitHubActionsRelayAttachment) : {}, + protocol || generation + ? ({ + ...(protocol ? { protocol } : {}), + ...(generation ? { generation } : {}), + } satisfies GitHubActionsRelayAttachment) + : {}, ); } @@ -371,19 +468,28 @@ export function encodeGitHubActionsRelayInputAcknowledgement( payload[0] = acknowledgement.accepted ? 1 : 0; payload.set(error, 1); return encodeGitHubActionsRelayFrame( - relayInputAcknowledgementFrameType, + acknowledgement.generation + ? relayGenerationInputAcknowledgementFrameType + : relayInputAcknowledgementFrameType, acknowledgement.inputId, - payload, + acknowledgement.generation + ? encodeGitHubActionsRelayGeneration(acknowledgement.generation, payload) + : payload, ); } export function parseGitHubActionsRelayEvent( message: string | ArrayBuffer, -): { type: keyof typeof relayEventCodes } | null { - const frame = decodeGitHubActionsRelayFrame(message, relayEventFrameType); +): { generation?: string; type: keyof typeof relayEventCodes } | null { + const legacyFrame = decodeGitHubActionsRelayFrame(message, relayEventFrameType); + const generatedFrame = decodeGitHubActionsRelayFrame(message, relayGenerationEventFrameType); + const decoded = generatedFrame + ? decodeGitHubActionsRelayGeneration(generatedFrame.payload) + : null; + const frame = legacyFrame ?? (decoded ? { ...generatedFrame!, payload: decoded.payload } : null); if (!frame || frame.inputId || frame.payload.byteLength !== 1) return null; const type = relayEvents.get(frame.payload[0] ?? 0); - return type ? { type } : null; + return type ? { type, ...(decoded ? { generation: decoded.generation } : {}) } : null; } export function isGitHubActionsViewerControlMessage(message: string | ArrayBuffer): boolean { @@ -401,15 +507,22 @@ export function isGitHubActionsViewerControlMessage(message: string | ArrayBuffe export function notifyGitHubActionsViewers( viewers: readonly GitHubActionsRelaySocket[], type: "runner_connected" | "runner_disconnected" | "runner_waiting", + generation?: string, ): number { - const framedPayload = encodeGitHubActionsRelayFrame( - relayEventFrameType, - "", - new Uint8Array([relayEventCodes[type]]), - ); let notified = 0; for (const socket of viewers) { if (socket.readyState !== webSocketOpen) continue; + const usesGenerations = gitHubActionsRelayUsesGenerations(socket); + const framedPayload = encodeGitHubActionsRelayFrame( + usesGenerations ? relayGenerationEventFrameType : relayEventFrameType, + "", + usesGenerations + ? encodeGitHubActionsRelayGeneration( + generation ?? "none", + new Uint8Array([relayEventCodes[type]]), + ) + : new Uint8Array([relayEventCodes[type]]), + ); socket.send( gitHubActionsViewerUsesFramedProtocol(socket) ? framedPayload : JSON.stringify({ type }), ); @@ -489,21 +602,44 @@ export function gitHubActionsViewerUsesFramedProtocol(socket: GitHubActionsRelay return gitHubActionsRelayUsesFramedProtocol(socket); } -function gitHubActionsRelayUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { +export function gitHubActionsRelayGeneration(socket: GitHubActionsRelaySocket): string | undefined { + const attachment = socket.deserializeAttachment?.(); + if (!attachment || typeof attachment !== "object") return undefined; + const generation = (attachment as GitHubActionsRelayAttachment).generation; + return isGitHubActionsRelayGeneration(generation) ? generation : undefined; +} + +export function gitHubActionsRelayUsesGenerations(socket: GitHubActionsRelaySocket): boolean { const attachment = socket.deserializeAttachment?.(); if (!attachment || typeof attachment !== "object") return false; return ( - (attachment as GitHubActionsRelayAttachment).protocol === githubActionsFramedRunnerCapability + (attachment as GitHubActionsRelayAttachment).protocol === + githubActionsGenerationFencedCapability ); } +export function createGitHubActionsRelayGeneration(): string { + return crypto.randomUUID().replaceAll("-", ""); +} + +function gitHubActionsRelayUsesFramedProtocol(socket: GitHubActionsRelaySocket): boolean { + const attachment = socket.deserializeAttachment?.(); + if (!attachment || typeof attachment !== "object") return false; + return isGitHubActionsRelayProtocol((attachment as GitHubActionsRelayAttachment).protocol); +} + function sendGitHubActionsViewerInputAcknowledgement( viewer: GitHubActionsRelaySocket, inputId: string | null, accepted: boolean, + generation?: string, ): boolean { if (inputId) { - return sendGitHubActionsRelayInputAcknowledgement(viewer, { inputId, accepted }); + return sendGitHubActionsRelayInputAcknowledgement(viewer, { + inputId, + accepted, + ...(gitHubActionsRelayUsesGenerations(viewer) ? { generation: generation ?? "none" } : {}), + }); } if (viewer.readyState !== webSocketOpen) return false; try { @@ -519,3 +655,50 @@ function sendGitHubActionsViewerInputAcknowledgement( return false; } } + +function isGitHubActionsRelayProtocol(value: unknown): value is GitHubActionsRelayProtocol { + return ( + value === githubActionsFramedRunnerCapability || + value === githubActionsGenerationFencedCapability + ); +} + +function isGitHubActionsRelayGeneration(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + encoder.encode(value).byteLength <= relayGenerationMaximumBytes && + relayGenerationPattern.test(value) + ); +} + +function encodeGitHubActionsRelayGeneration(generation: string, payload: Uint8Array): Uint8Array { + if (!isGitHubActionsRelayGeneration(generation)) { + throw new Error("invalid GitHub Actions relay generation"); + } + const generationBytes = encoder.encode(generation); + const generatedPayload = new Uint8Array(1 + generationBytes.byteLength + payload.byteLength); + generatedPayload[0] = generationBytes.byteLength; + generatedPayload.set(generationBytes, 1); + generatedPayload.set(payload, 1 + generationBytes.byteLength); + return generatedPayload; +} + +function decodeGitHubActionsRelayGeneration( + payload: Uint8Array, +): { generation: string; payload: Uint8Array } | null { + const generationBytes = payload[0] ?? 0; + if ( + generationBytes === 0 || + generationBytes > relayGenerationMaximumBytes || + 1 + generationBytes > payload.byteLength + ) { + return null; + } + const generation = decoder.decode(payload.subarray(1, 1 + generationBytes)); + if (!isGitHubActionsRelayGeneration(generation)) return null; + return { + generation, + payload: payload.subarray(1 + generationBytes), + }; +} diff --git a/src/worker/interactive-terminal-service.ts b/src/worker/interactive-terminal-service.ts index 742f4c10..4f133115 100644 --- a/src/worker/interactive-terminal-service.ts +++ b/src/worker/interactive-terminal-service.ts @@ -9,6 +9,7 @@ import { import { buildGitHubActionsViewerRelayUrl, gitHubActionsViewerResponseUsesFramedProtocol, + gitHubActionsViewerResponseUsesGenerations, githubActionsRuntime, } from "../github-actions-runtime.ts"; import { terminalFailureStatusForAdapter } from "../runtime-adapter.ts"; @@ -141,6 +142,7 @@ export class InteractiveTerminalService { return { socket: upstream, inputAcknowledgements: gitHubActionsViewerResponseUsesFramedProtocol(upstreamResponse), + inputGenerations: gitHubActionsViewerResponseUsesGenerations(upstreamResponse), outputAcknowledgements: false, markConnected: () => markInteractiveTerminalConnected( diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index a7bfa898..8c102a39 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -9,6 +9,10 @@ import type { FleetSandboxPolicySummary } from "../fleet-state.ts"; import { attachGitHubActionsRunnerProtocol, attachGitHubActionsViewerProtocol, + createGitHubActionsRelayGeneration, + gitHubActionsRelayGeneration, + gitHubActionsRelayUsesGenerations, + githubActionsLegacyRelayGeneration, githubActionsRelayRole, githubActionsRunnerProtocolQuery, githubActionsViewerProtocolHeader, @@ -229,6 +233,7 @@ export class SessionControlDO extends DurableObject { notifyGitHubActionsViewers( this.ctx.getWebSockets("github-actions-viewer"), "runner_disconnected", + gitHubActionsRelayGeneration(socket) ?? githubActionsLegacyRelayGeneration, ); } } @@ -245,18 +250,29 @@ export class SessionControlDO extends DurableObject { const client = pair[0]; const server = pair[1]; if (role === "runner") { + const generation = createGitHubActionsRelayGeneration(); replaceGitHubActionsRunner(this.ctx.getWebSockets("github-actions-runner")); - attachGitHubActionsRunnerProtocol(server, protocol); + attachGitHubActionsRunnerProtocol(server, protocol, generation); this.ctx.acceptWebSocket(server, ["github-actions-runner"]); notifyGitHubActionsViewers( this.ctx.getWebSockets("github-actions-viewer"), "runner_connected", + generation, ); } else { attachGitHubActionsViewerProtocol(server, protocol); this.ctx.acceptWebSocket(server, ["github-actions-viewer"]); - if (this.ctx.getWebSockets("github-actions-runner").length === 0) { + const runner = this.ctx + .getWebSockets("github-actions-runner") + .find((socket) => socket.readyState === WebSocket.OPEN); + if (!runner) { notifyGitHubActionsViewers([server], "runner_waiting"); + } else if (gitHubActionsRelayUsesGenerations(server)) { + notifyGitHubActionsViewers( + [server], + "runner_connected", + gitHubActionsRelayGeneration(runner) ?? githubActionsLegacyRelayGeneration, + ); } } const responseInit: ResponseInit = { status: 101, webSocket: client }; diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 2392bedd..4405fbea 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -36,7 +36,7 @@ const terminalInputAcknowledgementTimeoutMs = 5_000; type PendingTerminalInputAcknowledgement = { inputId: string; - runnerGeneration: number; + runnerGeneration: number | string; promise: Promise; resolve(result: GitHubActionsRelayInputAcknowledgement): void; timeout: ReturnType; @@ -46,6 +46,7 @@ export type TerminalUpstream = { socket: WebSocket; markConnected: () => Promise; inputAcknowledgements?: boolean; + inputGenerations?: boolean; outputAcknowledgements: boolean; }; @@ -65,8 +66,9 @@ export type TerminalHubSubscription = { inputQueueFrames: number; inputQueueRejections: number; inputQueueRejectionScheduled: boolean; + inputGenerations: boolean; pendingInputAcknowledgements: Map; - runnerGeneration: number; + runnerGeneration: number | string; outputAcknowledgements: boolean; outputAcknowledgementBytes: number; }; @@ -302,7 +304,15 @@ export class TerminalHub { if (acknowledgement) acknowledgements.push(acknowledgement); try { subscription.upstream.send( - inputId ? encodeGitHubActionsRelayInput(inputId, input) : input, + inputId + ? encodeGitHubActionsRelayInput( + inputId, + input, + subscription.inputGenerations + ? String(subscription.runnerGeneration) + : undefined, + ) + : input, ); } catch { if (acknowledgement) { @@ -516,8 +526,9 @@ export class TerminalHub { inputQueueFrames: 0, inputQueueRejections: 0, inputQueueRejectionScheduled: false, + inputGenerations: upstreamConnection.inputGenerations ?? false, pendingInputAcknowledgements: new Map(), - runnerGeneration: 0, + runnerGeneration: upstreamConnection.inputGenerations ? "none" : 0, outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; @@ -558,10 +569,22 @@ export class TerminalHub { const receivedRelayEvent = activeSubscription.inputAcknowledgements ? parseSynchronousGitHubActionsRelayEvent(raw) : null; - let runnerGenerationAtReceipt = - receivedRelayEvent?.type === "runner_connected" - ? ++activeSubscription.runnerGeneration - : activeSubscription.runnerGeneration; + let runnerGenerationAtReceipt = activeSubscription.runnerGeneration; + if (receivedRelayEvent?.generation) { + runnerGenerationAtReceipt = receivedRelayEvent.generation; + if (receivedRelayEvent.type === "runner_disconnected") { + if (activeSubscription.runnerGeneration === receivedRelayEvent.generation) { + activeSubscription.runnerGeneration = "none"; + } + } else { + activeSubscription.runnerGeneration = receivedRelayEvent.generation; + } + } else if ( + receivedRelayEvent?.type === "runner_connected" && + !activeSubscription.inputGenerations + ) { + runnerGenerationAtReceipt = ++(activeSubscription.runnerGeneration as number); + } outputQueue = outputQueue .catch(() => undefined) .then(async () => { @@ -580,22 +603,55 @@ export class TerminalHub { const relayEvent = receivedRelayEvent ?? parseGitHubActionsRelayEvent(data); if (relayEvent) { if (relayEvent.type === "runner_disconnected") { + if (relayEvent.generation && activeSubscription.inputGenerations) { + completeTerminalInputAcknowledgements( + activeSubscription, + (pending) => pending.runnerGeneration === relayEvent.generation, + { + accepted: false, + error: "GitHub Actions runner disconnected before accepting input", + }, + ); + } else { + completeAllTerminalInputAcknowledgements(activeSubscription, { + accepted: false, + error: "GitHub Actions runner disconnected before accepting input", + }); + } + } else if (relayEvent.type === "runner_connected") { + if (relayEvent.generation && activeSubscription.inputGenerations) { + runnerGenerationAtReceipt = relayEvent.generation; + activeSubscription.runnerGeneration = relayEvent.generation; + completeTerminalInputAcknowledgements( + activeSubscription, + (pending) => pending.runnerGeneration !== relayEvent.generation, + { + accepted: false, + error: "GitHub Actions runner was replaced before accepting input", + }, + ); + } else { + if (!receivedRelayEvent) { + runnerGenerationAtReceipt = ++(activeSubscription.runnerGeneration as number); + } + completeTerminalInputAcknowledgementsBeforeGeneration( + activeSubscription, + runnerGenerationAtReceipt as number, + { + accepted: false, + error: "GitHub Actions runner was replaced before accepting input", + }, + ); + } + } else if ( + relayEvent.type === "runner_waiting" && + activeSubscription.inputGenerations + ) { + activeSubscription.runnerGeneration = relayEvent.generation ?? "none"; completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, error: "GitHub Actions runner disconnected before accepting input", }); - } else if (relayEvent.type === "runner_connected") { - if (!receivedRelayEvent) { - runnerGenerationAtReceipt = ++activeSubscription.runnerGeneration; - } - completeTerminalInputAcknowledgementsBeforeGeneration( - activeSubscription, - runnerGenerationAtReceipt, - { - accepted: false, - error: "GitHub Actions runner was replaced before accepting input", - }, - ); } sendTerminalJson(client, TerminalMessageType.Event, id, relayEvent); return; @@ -701,7 +757,7 @@ export class TerminalHub { function beginTerminalInputAcknowledgement( subscription: TerminalHubSubscription, inputId: string, - runnerGeneration: number, + runnerGeneration: number | string, ): PendingTerminalInputAcknowledgement { let resolve!: (result: GitHubActionsRelayInputAcknowledgement) => void; const promise = new Promise((complete) => { @@ -760,6 +816,9 @@ function completeTerminalInputAcknowledgement( ): boolean { const pending = subscription.pendingInputAcknowledgements.get(inputId); if (!pending) return false; + if (subscription.inputGenerations && result.generation !== String(pending.runnerGeneration)) { + return false; + } subscription.pendingInputAcknowledgements.delete(inputId); clearTimeout(pending.timeout); pending.resolve(result); @@ -780,7 +839,7 @@ function completeTerminalInputAcknowledgementsBeforeGeneration( ): number { return completeTerminalInputAcknowledgements( subscription, - (pending) => pending.runnerGeneration < runnerGeneration, + (pending) => (pending.runnerGeneration as number) < runnerGeneration, result, ); } diff --git a/tests/application-architecture.test.ts b/tests/application-architecture.test.ts index 65ba3268..085229ac 100644 --- a/tests/application-architecture.test.ts +++ b/tests/application-architecture.test.ts @@ -66,7 +66,7 @@ test("GitHub Actions runner protocol is attached before the relay socket is acce ]); assert.match(application, /stub\.fetch\(gitHubActionsRelayRunnerUrl\(request\)/); - const attach = relay.indexOf("attachGitHubActionsRunnerProtocol(server, protocol)"); + const attach = relay.indexOf("attachGitHubActionsRunnerProtocol(server, protocol, generation)"); const accept = relay.indexOf( 'this.ctx.acceptWebSocket(server, ["github-actions-runner"])', attach, diff --git a/tests/github-actions-event-auth.test.ts b/tests/github-actions-event-auth.test.ts index 15dafb44..a764a129 100644 --- a/tests/github-actions-event-auth.test.ts +++ b/tests/github-actions-event-auth.test.ts @@ -157,6 +157,10 @@ test("GitHub Actions application propagates only the exact runner protocol opt-i ); assert.equal( gitHubActionsRelayRunnerUrl(new Request(`${base}&runnerProtocol=cfr1-framed-io-v2`)), + "https://crabfleet.internal/api/session-control/github-actions/runner?runnerProtocol=cfr1-framed-io-v2", + ); + assert.equal( + gitHubActionsRelayRunnerUrl(new Request(`${base}&runnerProtocol=cfr1-framed-io-v3`)), "https://crabfleet.internal/api/session-control/github-actions/runner", ); }); diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 859cf5c0..07573586 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -48,6 +48,24 @@ test("runner acknowledges input only after the PTY write completes", async () => }); }); +test("runner copies the relay generation into its acknowledgement", async () => { + const socket = relaySocket(); + + assert.equal( + await acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-generated", "steer", "generation-one"), + async () => {}, + ), + true, + ); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(socket.sent[0]!), { + inputId: "input-generated", + accepted: true, + generation: "generation-one", + }); +}); + test("runner rejects failed writes and ignores unframed terminal data", async () => { const socket = relaySocket(); diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 1acb6dd9..eb8a188a 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -5,18 +5,23 @@ import { attachGitHubActionsViewerProtocol, buildGitHubActionsRunnerPtyUrl, buildGitHubActionsViewerRelayUrl, + createGitHubActionsRelayGeneration, encodeGitHubActionsRelayInput, encodeGitHubActionsRelayInputAcknowledgement, encodeGitHubActionsRelayOutput, + gitHubActionsRelayGeneration, + gitHubActionsRelayUsesGenerations, gitHubActionsSessionStatus, githubActionsCapabilities, githubActionsFramedRunnerCapability, + githubActionsGenerationFencedCapability, githubActionsRelayRole, githubActionsRunnerProtocolQuery, githubActionsRuntimeLabel, githubActionsViewerProtocolHeader, githubActionsViewerProtocolQuery, gitHubActionsViewerResponseUsesFramedProtocol, + gitHubActionsViewerResponseUsesGenerations, gitHubActionsRunnerUsesFramedProtocol, gitHubActionsViewerUsesFramedProtocol, isGitHubActionsViewerControlMessage, @@ -67,6 +72,12 @@ function framedViewer() { return viewer; } +function generatedViewer() { + const viewer = relaySocket(); + attachGitHubActionsViewerProtocol(viewer, githubActionsGenerationFencedCapability); + return viewer; +} + test("github_actions exposes steerable terminal capabilities and label", () => { assert.equal(githubActionsRuntimeLabel("github_actions"), "GitHub Actions"); assert.equal(githubActionsRuntimeLabel("container"), ""); @@ -86,7 +97,10 @@ test("runner URL works without custom WebSocket headers", () => { "wss://crabfleet.openclaw.ai/api/agent/interactive-sessions/IS-123/runner-pty?agentToken=token+with+spaces", ); assert.equal(parseGitHubActionsRunnerProtocol(null), null); - assert.equal(parseGitHubActionsRunnerProtocol("cfr1-framed-io-v2"), null); + assert.equal( + parseGitHubActionsRunnerProtocol(githubActionsGenerationFencedCapability), + githubActionsGenerationFencedCapability, + ); assert.equal( parseGitHubActionsRunnerProtocol(githubActionsFramedRunnerCapability), githubActionsFramedRunnerCapability, @@ -94,14 +108,17 @@ test("runner URL works without custom WebSocket headers", () => { assert.equal(githubActionsRunnerProtocolQuery, "runnerProtocol"); assert.equal( buildGitHubActionsViewerRelayUrl(), - "https://crabfleet.internal/api/session-control/github-actions/viewer?viewerProtocol=cfr1-framed-io-v1", + "https://crabfleet.internal/api/session-control/github-actions/viewer?viewerProtocol=cfr1-framed-io-v2", ); assert.equal(parseGitHubActionsViewerProtocol(null), null); - assert.equal(parseGitHubActionsViewerProtocol("cfr1-framed-io-v2"), null); assert.equal( parseGitHubActionsViewerProtocol(githubActionsFramedRunnerCapability), githubActionsFramedRunnerCapability, ); + assert.equal( + parseGitHubActionsViewerProtocol(githubActionsGenerationFencedCapability), + githubActionsGenerationFencedCapability, + ); assert.equal(githubActionsViewerProtocolQuery, "viewerProtocol"); assert.equal(githubActionsViewerProtocolHeader, "x-crabfleet-viewer-protocol"); assert.equal( @@ -115,6 +132,13 @@ test("runner URL works without custom WebSocket headers", () => { true, ); assert.equal(gitHubActionsViewerResponseUsesFramedProtocol(new Response()), false); + const generatedResponse = new Response(null, { + headers: { + [githubActionsViewerProtocolHeader]: githubActionsGenerationFencedCapability, + }, + }); + assert.equal(gitHubActionsViewerResponseUsesFramedProtocol(generatedResponse), true); + assert.equal(gitHubActionsViewerResponseUsesGenerations(generatedResponse), true); }); test("work states preserve running phases and map terminal outcomes", () => { @@ -271,6 +295,110 @@ test("runner acknowledgements retain correlation and fan out to viewers", () => assert.deepEqual(viewerTwo.sent, [acknowledgement]); }); +test("relay-owned generations fence stale input and bridge framed protocol versions", () => { + const runner = relaySocket(); + const viewer = generatedViewer(); + const generation = createGitHubActionsRelayGeneration(); + attachGitHubActionsRunnerProtocol(runner, githubActionsFramedRunnerCapability, generation); + + assert.equal(gitHubActionsRelayGeneration(runner), generation); + assert.equal(gitHubActionsRelayUsesGenerations(runner), false); + assert.equal(gitHubActionsRelayUsesGenerations(viewer), true); + + const staleInput = encodeGitHubActionsRelayInput("input-stale", "old", "old-generation"); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, staleInput, [runner], []), 0); + assert.deepEqual(runner.sent, []); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + inputId: "input-stale", + accepted: false, + error: "GitHub Actions runner did not accept terminal input", + generation: "old-generation", + }); + + const currentInput = encodeGitHubActionsRelayInput("input-current", "new", generation); + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, currentInput, [runner], []), 1); + assert.deepEqual(parseGitHubActionsRelayInput(runner.sent[0]!), { + inputId: "input-current", + payload: new TextEncoder().encode("new").buffer, + }); + + const acknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "input-current", + accepted: true, + }); + assert.equal( + relayGitHubActionsWebSocketMessage("runner", runner, acknowledgement, [runner], [viewer]), + 1, + ); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[1]!), { + inputId: "input-current", + accepted: true, + generation, + }); +}); + +test("replacement relay drops acknowledgements from the superseded runner", () => { + const oldRunner = relaySocket(); + const replacement = relaySocket(); + const viewer = generatedViewer(); + attachGitHubActionsRunnerProtocol( + oldRunner, + githubActionsGenerationFencedCapability, + "old-generation", + ); + attachGitHubActionsRunnerProtocol( + replacement, + githubActionsGenerationFencedCapability, + "new-generation", + ); + const acknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "input-old", + accepted: true, + generation: "old-generation", + }); + + assert.equal( + relayGitHubActionsWebSocketMessage( + "runner", + oldRunner, + acknowledgement, + [replacement], + [viewer], + ), + 0, + ); + assert.deepEqual(viewer.sent, []); +}); + +test("generation-fenced runners echo only their relay-owned generation", () => { + const runner = relaySocket(); + const viewer = generatedViewer(); + attachGitHubActionsRunnerProtocol( + runner, + githubActionsGenerationFencedCapability, + "current-generation", + ); + const input = encodeGitHubActionsRelayInput("input-current", "steer", "current-generation"); + + assert.equal(relayGitHubActionsWebSocketMessage("viewer", viewer, input, [runner], []), 1); + assert.deepEqual(parseGitHubActionsRelayInput(runner.sent[0]!), { + inputId: "input-current", + generation: "current-generation", + payload: new TextEncoder().encode("steer").buffer, + }); + + const staleAcknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "input-current", + accepted: true, + generation: "stale-generation", + }); + assert.equal( + relayGitHubActionsWebSocketMessage("runner", runner, staleAcknowledgement, [runner], [viewer]), + 0, + ); + assert.deepEqual(viewer.sent, []); +}); + test("negotiated runners frame output so control-shaped terminal bytes stay output", () => { const runner = relaySocket(); const viewer = framedViewer(); @@ -340,6 +468,20 @@ test("relay tags and runner lifecycle notifications stay explicit", () => { }); }); +test("generation-fenced lifecycle events identify the relay runner", () => { + const viewer = generatedViewer(); + assert.equal(notifyGitHubActionsViewers([viewer], "runner_connected", "generation-one"), 1); + assert.deepEqual(parseGitHubActionsRelayEvent(viewer.sent[0]!), { + type: "runner_connected", + generation: "generation-one", + }); + assert.equal(notifyGitHubActionsViewers([viewer], "runner_waiting"), 1); + assert.deepEqual(parseGitHubActionsRelayEvent(viewer.sent[1]!), { + type: "runner_waiting", + generation: "none", + }); +}); + test("unnegotiated viewers retain raw relay compatibility", () => { const runner = relaySocket(); const viewer = relaySocket(); diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index cbca7acf..123dfca3 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -15,6 +15,7 @@ import { encodeGitHubActionsRelayInputAcknowledgement, encodeGitHubActionsRelayOutput, githubActionsFramedRunnerCapability, + githubActionsGenerationFencedCapability, notifyGitHubActionsViewers, parseGitHubActionsRelayInput, } from "../src/github-actions-runtime.ts"; @@ -84,23 +85,37 @@ function relayInput(value: string | ArrayBuffer | ArrayBufferView | Blob) { assert.ok(input); return { inputId: input.inputId, + generation: input.generation, text: new TextDecoder().decode(input.payload), }; } -function emitRelayAcknowledgement(upstream: TestSocket, inputId: string, accepted: boolean): void { +function emitRelayAcknowledgement( + upstream: TestSocket, + inputId: string, + accepted: boolean, + generation?: string, +): void { upstream.emit("message", { - data: encodeGitHubActionsRelayInputAcknowledgement({ inputId, accepted }), + data: encodeGitHubActionsRelayInputAcknowledgement({ + inputId, + accepted, + ...(generation ? { generation } : {}), + }), }); } function emitRelayEvent( upstream: TestSocket, type: "runner_connected" | "runner_disconnected" | "runner_waiting", + generation?: string, ): void { const source = socket(); - attachGitHubActionsViewerProtocol(source, githubActionsFramedRunnerCapability); - notifyGitHubActionsViewers([source], type); + attachGitHubActionsViewerProtocol( + source, + generation ? githubActionsGenerationFencedCapability : githubActionsFramedRunnerCapability, + ); + notifyGitHubActionsViewers([source], type, generation); upstream.emit("message", { data: source.sent[0] }); } @@ -1540,6 +1555,107 @@ test("queued runner replacement rejects only acknowledgements sent to the old ge server.emit("close"); }); +test("relay generations bind interleaved replacement input before lifecycle processing", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let releaseBlockedOutput: ((output: ArrayBuffer) => void) | undefined; + const blockedOutput = new Promise((resolve) => { + releaseBlockedOutput = resolve; + }); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async openUpstream() { + return { + socket: upstream, + inputAcknowledgements: true, + inputGenerations: true, + outputAcknowledgements: false, + async markConnected() {}, + }; + }, + async inputPayloads() { + return [new TextEncoder().encode("old runner"), new TextEncoder().encode("new runner")]; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + emitRelayEvent(upstream, "runner_connected", "generation-old"); + await flushQueues(); + await flushQueues(); + + upstream.emit("message", { data: { arrayBuffer: () => blockedOutput } }); + const send = upstream.send.bind(upstream); + upstream.send = (data) => { + send(data); + if (upstream.sent.length === 1) { + emitRelayEvent(upstream, "runner_connected", "generation-new"); + } + }; + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("split input"), + }), + }); + await waitForInputPayloads(); + + assert.equal(upstream.sent.length, 2); + const oldInput = relayInput(upstream.sent[0]!); + const replacementInput = relayInput(upstream.sent[1]!); + assert.equal(oldInput.generation, "generation-old"); + assert.equal(replacementInput.generation, "generation-new"); + + releaseBlockedOutput?.(encodeGitHubActionsRelayOutput("blocked output")); + await flushQueues(); + await flushQueues(); + await flushQueues(); + assert.equal( + server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string }) + .some((message) => message.type === "input-accepted" || message.type === "input-rejected"), + false, + ); + + emitRelayAcknowledgement(upstream, replacementInput.inputId, true, "generation-new"); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.deepEqual(completions, [ + { + type: "input-rejected", + error: "GitHub Actions runner was replaced before accepting input", + }, + ]); + assert.deepEqual(upstream.closed, []); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From a7630a2a9326c0777896ab576c055f3275ca5064 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:36:12 +0200 Subject: [PATCH 158/242] docs(actions): define generation-fenced relay protocol --- docs/api.md | 48 ++++++++++-------- docs/architecture.md | 2 +- docs/github-actions-sessions.md | 84 ++++++++++++++++++++----------- docs/runs.md | 2 +- docs/spec.md | 2 +- tests/github-actions-docs.test.ts | 4 +- 6 files changed, 88 insertions(+), 54 deletions(-) diff --git a/docs/api.md b/docs/api.md index b60eae8c..f6ccf7b2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -598,22 +598,22 @@ Response: } ``` -Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output by opening it unchanged; new runners add the exact `runnerProtocol=cfr1-framed-io-v1` query to opt into the framed contract below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. +Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output by opening it unchanged; new runners add the exact `runnerProtocol=cfr1-framed-io-v2` query to opt into the generation-fenced contract below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. ### GET /api/agent/interactive-sessions/:id/runner-pty WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses the scoped `agentToken` query parameter embedded in `runnerPtyUrl`. One runner is current; a reconnect replaces the previous runner while browser viewers remain attached. Opening the returned URL unchanged selects legacy raw input and output. Adding -the exact `runnerProtocol=cfr1-framed-io-v1` query selects framed input, output, -acknowledgements, and relay control traffic. The application propagates only -that exact value to `SessionControlDO`, which stores the mode on the server -socket before accepting it. Viewer framing is negotiated independently: framed -viewers receive `CFR1` output and control frames, while unnegotiated viewers -retain raw output and legacy JSON notices during rolling upgrades. The relay -therefore wraps legacy runner output only for framed viewers, and unwraps framed -runner output for raw viewers. Arbitrary raw PTY bytes cannot be consumed as -control traffic by framed viewers. +the exact `runnerProtocol=cfr1-framed-io-v2` query selects generation-fenced +input, output, acknowledgements, and relay control traffic. `SessionControlDO` +stores the mode and a relay-owned runner generation on the server socket before +accepting it. Viewer framing is negotiated independently: v2 viewers receive +`CFR1` output and generation-bearing control frames, while unnegotiated viewers +retain raw output and legacy JSON notices. The relay translates the earlier +`cfr1-framed-io-v1` format and raw sockets at each boundary during rolling +upgrades. Arbitrary raw PTY bytes cannot be consumed as control traffic by +framed viewers. Each `CFR1` frame occupies one binary WebSocket message and starts with: @@ -626,22 +626,26 @@ Each `CFR1` frame occupies one binary WebSocket message and starts with: Input IDs are nonempty ASCII `[A-Za-z0-9_-]` values of at most 80 bytes. -| Type | Direction | Payload | -| ---------------------- | ---------------- | ----------------------------------------------------------------------------- | -| `0x01` input | relay to runner | raw terminal input bytes | -| `0x02` acknowledgement | runner to relay | one byte: `1` accepted or `0` rejected, followed by optional UTF-8 error text | -| `0x03` lifecycle event | relay to viewers | empty input ID and one event-code byte | -| `0x04` output | runner to relay | empty input ID followed by raw terminal output bytes | +| Type | Direction | Payload | +| ---------------------- | ---------------- | --------------------------------------------------------------------------------------------- | +| `0x05` input | relay to runner | generation-length byte, generation, then raw terminal input bytes | +| `0x06` acknowledgement | runner to relay | generation envelope, then `1` accepted or `0` rejected, followed by optional UTF-8 error text | +| `0x07` lifecycle event | relay to viewers | empty input ID, generation envelope, and one event-code byte | +| `0x04` output | runner to relay | empty input ID followed by raw terminal output bytes | Lifecycle event codes are `0x01` runner connected, `0x02` runner disconnected, and `0x03` runner waiting. -The runner must copy the input frame's ID into its acknowledgement. It must send -an accepted acknowledgement only after its PTY write API has accepted the -payload. Queueing the frame in `WebSocket.send()` is not acceptance. Crabfleet -generates a rejected acknowledgement only when no current runner is available -to receive the input frame or the relay send fails. Stale or mismatched -acknowledgement IDs do not complete another pending input. +The runner must copy the input frame's ID and generation into its +acknowledgement. It must send an accepted acknowledgement only after its PTY +write API has accepted the payload. Queueing the frame in `WebSocket.send()` is +not acceptance. Crabfleet rejects stale-generation input before forwarding it, +and rejects input when no current runner is available or the relay send fails. +Stale or mismatched acknowledgement IDs or generations do not complete another +pending input. + +The v1 `0x01`, `0x02`, and `0x03` frames omit generations. They remain accepted +for mixed-version deployments and are translated by the relay. For legacy connections, the relay unwraps viewer input to raw bytes and reports acceptance once the runner socket accepts the send. Framed connections provide diff --git a/docs/architecture.md b/docs/architecture.md index 371e97e5..85d959a2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ D1 is canonical for product metadata: ### Durable Objects - `Sandbox` runs first-party Cloudflare Sandbox workspaces. -- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; exact connection-query opt-in selects correlated binary `CFR1` input, output, and acknowledgement frames before socket acceptance. Viewer framing is negotiated independently: opted-in viewers receive `CFR1` terminal, lifecycle, and acknowledgement frames, while legacy viewers retain raw terminal output and JSON control-message fallbacks. +- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; exact v2 connection-query opt-in selects relay-generation-fenced binary `CFR1` input, lifecycle, and acknowledgement frames before socket acceptance. Viewer framing is negotiated independently: opted-in viewers receive `CFR1` terminal, lifecycle, and acknowledgement frames, legacy viewers retain raw terminal output and JSON control-message fallbacks, and the relay translates v1 framed peers during rolling upgrades. There is no `BoardDO` or `RunDO`. General Board/Fleet state is D1 plus REST polling. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 94575769..05ba0994 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -267,11 +267,12 @@ The Action connects outbound to the returned `runnerPtyUrl`. Node's global The returned URL opens a legacy raw-input/raw-output socket. A runner opts into collision-free framed I/O by adding the exact -`runnerProtocol=cfr1-framed-io-v1` query before opening the socket. The relay -records that mode before accepting the connection. Viewer input then arrives in -a binary `CFR1` frame carrying a correlation ID, and runner output uses a -distinct `CFR1` output frame. The runner returns a correlated acknowledgement -only after its PTY accepts the input write. +`runnerProtocol=cfr1-framed-io-v2` query before opening the socket. The relay +records that mode and a relay-owned runner generation before accepting the +connection. Viewer input then arrives in a binary `CFR1` frame carrying a +correlation ID and that generation, and runner output uses a distinct `CFR1` +output frame. The runner copies the generation into its correlated +acknowledgement only after its PTY accepts the input write. Complete Node runner integration: @@ -295,7 +296,7 @@ let pendingInputs = []; let pendingInputBytes = 0; let pendingInputTimer; const framedRunnerPtyUrl = new URL(runnerPtyUrl); -framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); +framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v2"); const terminal = new WebSocket(framedRunnerPtyUrl); terminal.binaryType = "arraybuffer"; @@ -361,7 +362,7 @@ function settlePendingInputs(accepted) { if (pendingInputTimer) clearTimeout(pendingInputTimer); pendingInputTimer = undefined; for (const input of pendingInputs) { - terminal.send(encodeAck(input.inputId, accepted)); + terminal.send(encodeAck(input.inputId, input.generation, accepted)); } pendingInputs = []; pendingInputBytes = 0; @@ -373,27 +374,45 @@ function decodeInput(data) { if (frame.byteLength < 7 || !magic.every((value, index) => frame[index] === value)) { return null; } - if (frame[4] !== 0x01) return null; + if (frame[4] !== 0x05) return null; const inputIdBytes = frame[5]; if (!inputIdBytes || inputIdBytes > 80 || 6 + inputIdBytes > frame.byteLength) { return null; } const inputId = inputIdDecoder.decode(frame.subarray(6, 6 + inputIdBytes)); if (!/^[A-Za-z0-9_-]+$/.test(inputId)) return null; + const generationOffset = 6 + inputIdBytes; + const generationBytes = frame[generationOffset]; + if ( + !generationBytes || + generationBytes > 80 || + generationOffset + 1 + generationBytes > frame.byteLength + ) { + return null; + } + const generation = inputIdDecoder.decode( + frame.subarray(generationOffset + 1, generationOffset + 1 + generationBytes), + ); + if (!/^[A-Za-z0-9_-]+$/.test(generation)) return null; return { inputId, - payload: frame.slice(6 + inputIdBytes), + generation, + payload: frame.slice(generationOffset + 1 + generationBytes), }; } -function encodeAck(inputId, accepted) { +function encodeAck(inputId, generation, accepted) { const inputIdBytes = encoder.encode(inputId); - const frame = new Uint8Array(7 + inputIdBytes.byteLength); + const generationBytes = encoder.encode(generation); + const frame = new Uint8Array(8 + inputIdBytes.byteLength + generationBytes.byteLength); frame.set(magic); - frame[4] = 0x02; + frame[4] = 0x06; frame[5] = inputIdBytes.byteLength; frame.set(inputIdBytes, 6); - frame[6 + inputIdBytes.byteLength] = accepted ? 1 : 0; + const generationOffset = 6 + inputIdBytes.byteLength; + frame[generationOffset] = generationBytes.byteLength; + frame.set(generationBytes, generationOffset + 1); + frame[generationOffset + 1 + generationBytes.byteLength] = accepted ? 1 : 0; return frame; } @@ -435,18 +454,25 @@ output as JavaScript strings: it rejects input that is not complete valid UTF-8 and encodes each output string as UTF-8. Deployments that require lossless arbitrary PTY bytes must use a byte-oriented PTY adapter instead of this example. -| Offset | Size | Value | -| ------ | -------- | ------------------------------------------------------------------------------ | -| 0 | 4 | ASCII `CFR1` | -| 4 | 1 | `0x01` input, `0x02` acknowledgement, `0x03` lifecycle event, or `0x04` output | -| 5 | 1 | input ID byte length | -| 6 | variable | input ID followed by the type-specific payload | - -Input payloads are raw terminal bytes. An acknowledgement payload starts with -`1` for accepted or `0` for rejected and may include UTF-8 error text after the -status byte. Lifecycle events use an empty input ID and event code `0x01` for -runner connected, `0x02` for runner disconnected, or `0x03` for runner waiting. -Output uses an empty input ID followed by raw terminal bytes. +| Offset | Size | Value | +| ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| 0 | 4 | ASCII `CFR1` | +| 4 | 1 | v2 `0x05` input, `0x06` acknowledgement, `0x07` lifecycle event, or shared `0x04` output | +| 5 | 1 | input ID byte length | +| 6 | variable | input ID, then one generation-length byte, the relay generation, and the type-specific payload; output omits the generation envelope | + +Input payloads are raw terminal bytes after the generation envelope. An +acknowledgement payload starts with the generation envelope, then `1` for +accepted or `0` for rejected, followed by optional UTF-8 error text. Lifecycle +events use an empty input ID, the generation envelope, and event code `0x01` +for runner connected, `0x02` for runner disconnected, or `0x03` for runner +waiting. Output uses an empty input ID followed by raw terminal bytes. + +The earlier `cfr1-framed-io-v1` mode remains accepted during rolling upgrades. +Its `0x01`, `0x02`, and `0x03` frames omit generations. The relay translates +between v1 and v2 at each socket boundary. A v2 viewer must send the generation +from its latest lifecycle event; stale-generation input is rejected before it +can reach the replacement runner. The full wire contract is also specified in [API](/api/#get-api-agent-interactive-sessions-id-runner-pty). @@ -460,9 +486,11 @@ Properties: send raw output. - Framed runners add the exact protocol query before connecting, receive framed input immediately, and wrap every output payload in a `0x04` frame. -- Framed viewers add `viewerProtocol=cfr1-framed-io-v1` before connecting. They - receive `CFR1` output, lifecycle, and acknowledgement frames regardless of the - runner's mode. +- Generation-fenced viewers add `viewerProtocol=cfr1-framed-io-v2` before + connecting. They receive `CFR1` output plus relay-generated lifecycle and + acknowledgement frames regardless of the runner's mode. +- Existing v1 runners and viewers remain interoperable through relay-side + frame translation. - Legacy viewers omit that query. They receive raw terminal output plus JSON lifecycle and input-acknowledgement messages for compatibility. - Negotiated input produces `input-accepted` only after the correlated runner diff --git a/docs/runs.md b/docs/runs.md index a94f0d82..62faf0b7 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -122,7 +122,7 @@ GitHub Actions PTY contract: - OpenClaw registers or resumes work through `POST /api/openclaw/action-sessions`. - The returned `runnerPtyUrl` is a `wss:` URL with a rotated session-scoped query credential. Node's global `WebSocket` can open it without custom headers. - Legacy runners open the returned URL unchanged and retain raw input/output with relay-level delivery reporting. -- Framed runners add the exact `runnerProtocol=cfr1-framed-io-v1` query before opening the socket. Viewer input and runner output then use collision-free binary `CFR1` frames, and the runner returns the matching acknowledgement only after its PTY accepts the write. +- Generation-fenced runners add the exact `runnerProtocol=cfr1-framed-io-v2` query before opening the socket. Viewer input and acknowledgements carry the relay-owned runner generation, stale input is rejected before forwarding, and the runner returns the matching generation and correlation ID only after its PTY accepts the write. The relay continues translating v1 framed and raw sockets during rolling upgrades. - `SessionControlDO` allows one current runner and multiple viewers. A new runner replaces the previous runner; viewers remain connected and receive runner lifecycle events. - Authorized browser viewers attach through the existing `/api/terminal/ws` hub. Service and agent credentials are never included in viewer responses. - The runner updates `state`, `phase`, `summary`, Codex thread/turn IDs, and heartbeat through the agent work-state endpoint. `completed`, `blocked`, `failed`, and `canceled` are terminal. diff --git a/docs/spec.md b/docs/spec.md index 22873c9e..52170db7 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -178,7 +178,7 @@ Crabfleet owns: - session identity and metadata; - rotating scoped agent token; -- outbound runner relay through `SessionControlDO`, preserving legacy raw runner traffic while exact connection-query opt-in selects correlated binary `CFR1` input, output, acknowledgement, and lifecycle frames; +- outbound runner relay through `SessionControlDO`, preserving raw and v1 framed traffic while exact v2 connection-query opt-in selects relay-generation-fenced binary `CFR1` input, acknowledgement, and lifecycle frames plus framed output; - browser terminal steering; - work-state heartbeats; - event and transcript finalization. diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 6587e893..8bbc3a62 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -28,7 +28,9 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.match(guide, /encodeUtf8Output\(outputText\)/); assert.match(guide, /deliberately a UTF-8 text adapter/); assert.match(guide, /lossless\s+arbitrary PTY bytes must use a byte-oriented PTY adapter/); - assert.match(guide, /Framed viewers add\s+`viewerProtocol=cfr1-framed-io-v1`/); + assert.match(guide, /Generation-fenced viewers add `viewerProtocol=cfr1-framed-io-v2`/); + assert.match(guide, /stale-generation input is rejected before it\s+can reach/); + assert.match(guide, /encodeAck\(input\.inputId, input\.generation, accepted\)/); assert.match(guide, /Legacy viewers omit that query/); assert.match(guide, /pty\.onExit\(\(\) => \{/); assert.match(guide, /terminal\.close\(1000, "pty exited"\)/); From 10d912c928238923d7b9c4fb64b083b327b4208a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:36:51 +0200 Subject: [PATCH 159/242] docs(changelog): record ownership race fixes --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c354b1c6..886dbef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,10 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, generation-fence pending acknowledgements when GitHub Actions runners disconnect or are replaced, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. +- Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; durably claim and retry superseded workspace cleanup without touching the replacement workspace; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure, ambiguous committed publication, and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown with bounded retries and no retry when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure, ambiguous committed publication reconciled by stable publication identity, and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown with bounded retries and no retry when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions with atomic capability publication, premature-response rejection, and fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. From b907745a8c4f9e24eab02076d1a2e91007a0bf35 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:49:07 +0200 Subject: [PATCH 160/242] fix(api): reject malformed desktop recovery ids --- src/worker/routes/control-plane.ts | 8 ++++++-- tests/control-plane-routes.test.ts | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/worker/routes/control-plane.ts b/src/worker/routes/control-plane.ts index 7995ea3f..e8cc15d8 100644 --- a/src/worker/routes/control-plane.ts +++ b/src/worker/routes/control-plane.ts @@ -14,7 +14,7 @@ import { type DesktopHostOwnershipMode, type DesktopHostRegistration, } from "../desktop-host-service.ts"; -import { json, notFound, readJson } from "../http.ts"; +import { badRequest, json, notFound, readJson } from "../http.ts"; import type { User } from "../models.ts"; export type ControlPlaneRouteDependencies = { @@ -154,5 +154,9 @@ export async function handleControlPlaneRoute( } function decoded(value: string | undefined): string { - return decodeURIComponent(value ?? ""); + try { + return decodeURIComponent(value ?? ""); + } catch { + throw badRequest("invalid path identifier"); + } } diff --git a/tests/control-plane-routes.test.ts b/tests/control-plane-routes.test.ts index 7a5c00f1..7ad1c3cd 100644 --- a/tests/control-plane-routes.test.ts +++ b/tests/control-plane-routes.test.ts @@ -254,6 +254,24 @@ test("desktop host routes register and remove only the authenticated user's host ]); }); +test("desktop host recovery rejects malformed encoded ids with a client error", async () => { + const calls: string[] = []; + await assert.rejects( + dispatch( + request("POST", "/api/desktop-hosts/%?recover=1", { + publicationID: "publication-id", + }), + viewer, + calls, + ), + (error) => { + assert.equal(status(error), 400); + return true; + }, + ); + assert.deepEqual(calls, []); +}); + test("card actions derive viewer or maintainer authorization from the action", async () => { for (const action of ["attach", "watch"]) { const calls: string[] = []; From 8a4588a37bfaa32127d34c25ea4eddaed79e4323 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:49:07 +0200 Subject: [PATCH 161/242] docs(actions): align generation and recovery contracts --- README.md | 10 ++++----- docs/api.md | 34 +++++++++++++++++++++++++------ tests/github-actions-docs.test.ts | 3 +++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index bc7fbe44..a71755fd 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New reg ```js const framedRunnerPtyUrl = new URL(runnerPtyUrl); -framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v1"); +framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v2"); const terminal = new WebSocket(framedRunnerPtyUrl); terminal.binaryType = "arraybuffer"; ``` @@ -106,10 +106,10 @@ terminal.binaryType = "arraybuffer"; Existing runners retain raw input and output by opening the returned URL unchanged. A new runner opts into correlated binary `CFR1` input, output, and acknowledgement frames by adding the exact -`runnerProtocol=cfr1-framed-io-v1` query before opening the socket. The relay -selects that mode before accepting the connection, so there is no pending -handshake. Framed runners acknowledge only after their PTY accepts the input; -the complete byte-safe encoder, decoder, and Node PTY runner are in +`runnerProtocol=cfr1-framed-io-v2` query before opening the socket. The relay +selects that mode before accepting the connection and fences runner input with +the relay-owned connection generation. Framed runners acknowledge only after +their PTY accepts the input. The complete byte-safe encoder, decoder, and Node PTY runner are in [`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. diff --git a/docs/api.md b/docs/api.md index f6ccf7b2..b11e2922 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1122,12 +1122,34 @@ private session-tenancy mode. Re-registering the same ID updates its name, address, port, and timestamp while preserving its creation time. Clients opt into fenced registration by sending -`X-Crabfleet-Ownership-Mode: token-v1`. The response then includes an -`ownershipToken` required for deletion. Omitting the header preserves the -legacy `{ "host": ... }` response and stores a tokenless registration so older -clients can still clean up during rolling upgrades. Current clients tolerate a -legacy server response without `ownershipToken` and use tokenless cleanup for -that registration. +`X-Crabfleet-Ownership-Mode: token-v1` and a stable +`X-Crabfleet-Publication-ID`. The publication ID identifies one client's +attempt across retries and restarts; it must satisfy the same 1-80 character +identifier rules as the host ID. The response includes an `ownershipToken` +required for deletion. Omitting the ownership-mode header preserves the legacy +`{ "host": ... }` response and stores a tokenless registration so older clients +can still clean up during rolling upgrades. Current clients tolerate a legacy +server response without `ownershipToken` and retain enough state for guarded +legacy cleanup. + +### POST /api/desktop-hosts/:id?recover=1 + +Recovers the ownership token after a fenced `PUT` may have committed but its +response was lost. The signed-in viewer and host ID must match the original +registration, and the JSON body carries the same stable publication ID: + +```json +{ + "publicationID": "01JZDESKTOPPUBLICATION" +} +``` + +The response is `{ "ownershipToken": "..." }` when that publication still owns +the host, or `{ "ownershipToken": null }` when it does not. Recovery never +reassigns ownership and cannot replace a newer publisher. A route-level `404` +means the server predates publication recovery, not that the original `PUT` +definitely failed; rolling-upgrade clients must preserve the uncertain +registration or use guarded legacy cleanup rather than silently discarding it. ### DELETE /api/desktop-hosts/:id diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 8bbc3a62..9f045d43 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -9,6 +9,9 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async ]); assert.match(readme, /complete byte-safe encoder, decoder, and Node PTY runner/); + assert.match(readme, /runnerProtocol", "cfr1-framed-io-v2"/); + assert.match(readme, /`runnerProtocol=cfr1-framed-io-v2` query/); + assert.doesNotMatch(readme, /New runners opt into[\s\S]*cfr1-framed-io-v1/); assert.doesNotMatch(readme, /encodeCfr1Output|decodeCfr1Input|encodeCfr1Ack/); assert.match(guide, /let pendingInputs = \[\]/); assert.match(guide, /pendingInputs\.push\(input\)/); From 5d63e4823ddeb88bacf41cb10726090f6d629a7a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:49:32 +0200 Subject: [PATCH 162/242] fix(actions): serialize runner input delivery --- src/github-actions-runner.ts | 39 ++++++++++++------ src/worker/terminal-hub.ts | 3 ++ tests/github-actions-runner.test.ts | 42 +++++++++++++++++++ tests/terminal-hub.test.ts | 62 +++++++++++++++++++++++++++++ 4 files changed, 134 insertions(+), 12 deletions(-) diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 8b54ff7d..09b5825c 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -5,6 +5,8 @@ import { type GitHubActionsRelaySocket, } from "./github-actions-runtime.ts"; +const runnerInputQueues = new WeakMap>(); + export function sendGitHubActionsRunnerOutput( socket: GitHubActionsRelaySocket, output: string | ArrayBuffer | ArrayBufferView, @@ -19,19 +21,32 @@ export async function acceptGitHubActionsRunnerInput( ): Promise { const input = parseGitHubActionsRelayInput(message); if (!input) return false; - try { - await writeToPty(input.payload); - sendGitHubActionsRelayInputAcknowledgement(socket, { - inputId: input.inputId, - accepted: true, - ...(input.generation ? { generation: input.generation } : {}), - }); - } catch { - sendGitHubActionsRelayInputAcknowledgement(socket, { - inputId: input.inputId, - accepted: false, - ...(input.generation ? { generation: input.generation } : {}), + + const queued = (runnerInputQueues.get(socket) ?? Promise.resolve()) + .catch(() => undefined) + .then(async () => { + try { + await writeToPty(input.payload); + sendGitHubActionsRelayInputAcknowledgement(socket, { + inputId: input.inputId, + accepted: true, + ...(input.generation ? { generation: input.generation } : {}), + }); + } catch { + sendGitHubActionsRelayInputAcknowledgement(socket, { + inputId: input.inputId, + accepted: false, + ...(input.generation ? { generation: input.generation } : {}), + }); + } }); + runnerInputQueues.set(socket, queued); + try { + await queued; + } finally { + if (runnerInputQueues.get(socket) === queued) { + runnerInputQueues.delete(socket); + } } return true; } diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 4405fbea..7dde562b 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -323,6 +323,9 @@ export class TerminalHub { inputId: acknowledgement.inputId, accepted: false, error: "terminal upstream send failed", + ...(subscription.inputGenerations + ? { generation: String(acknowledgement.runnerGeneration) } + : {}), }, ); break; diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 07573586..7ed2e524 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -48,6 +48,48 @@ test("runner acknowledges input only after the PTY write completes", async () => }); }); +test("runner serializes concurrent input writes and acknowledgements per socket", async () => { + const socket = relaySocket(); + const writes: string[] = []; + let completeFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + completeFirstWrite = resolve; + }); + + const first = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-first", "first"), + async (payload) => { + writes.push(`${new TextDecoder().decode(payload)}:start`); + await firstWrite; + writes.push("first:end"); + }, + ); + const second = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-second", "second"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.deepEqual(writes, ["first:start"]); + assert.deepEqual(socket.sent, []); + + completeFirstWrite(); + assert.deepEqual(await Promise.all([first, second]), [true, true]); + assert.deepEqual(writes, ["first:start", "first:end", "second"]); + assert.deepEqual( + socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), + [ + { inputId: "input-first", accepted: true }, + { inputId: "input-second", accepted: true }, + ], + ); +}); + test("runner copies the relay generation into its acknowledgement", async () => { const socket = relaySocket(); diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 123dfca3..09c3656c 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -1261,6 +1261,68 @@ test("GitHub Actions send failure removes only its own acknowledgement waiter", server.emit("close"); }); +test("generation-fenced send failure completes its matching acknowledgement", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async openUpstream() { + return { + socket: upstream, + inputAcknowledgements: true, + inputGenerations: true, + outputAcknowledgements: false, + async markConnected() {}, + }; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + emitRelayEvent(upstream, "runner_connected", "generation-one"); + await flushQueues(); + await flushQueues(); + + upstream.send = () => { + throw new Error("runner disconnected"); + }; + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("input"), + }), + }); + await waitForInputPayloads(); + await flushQueues(); + + const rejected = frame(server.sent.at(-1)!); + assert.equal(rejected.type, TerminalMessageType.Event); + assert.deepEqual(decodeJsonPayload(rejected.payload), { + type: "input-rejected", + error: "terminal upstream send failed", + }); + assert.deepEqual(upstream.closed, []); + server.emit("close"); +}); + test("GitHub Actions close rejects every pending input acknowledgement", async () => { const client = socket(); const server = socket(); From 45270f43557c8e7f4f8f061d01bf4720fc15a952 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:49:54 +0200 Subject: [PATCH 163/242] fix(terminal): bound blocked attachment shutdown --- internal/terminalws/client.go | 26 +++++++++++++++++++--- internal/terminalws/client_test.go | 35 ++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 7a035874..f1384456 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -23,7 +23,8 @@ const ( maxFrameBytes = 16 * 1024 * 1024 maxErrorBytes = 512 - defaultInputConfirmationTimeout = 5 * time.Second + defaultInputConfirmationTimeout = 5 * time.Second + defaultAttachmentShutdownTimeout = 250 * time.Millisecond messageHello = 1 messageWelcome = 2 @@ -88,6 +89,7 @@ type Client struct { confirmOnce sync.Once confirmGate chan struct{} confirmationTimeout time.Duration + attachmentShutdownTimeout time.Duration stateMu sync.Mutex inputWaiter chan error attachment *terminalAttachment @@ -485,13 +487,31 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c err = ctx.Err() } cancelRead() - <-frameConsumerDone - if cancelableRead { + // Let completed writes preserve their acknowledgement ordering, but retire the + // attachment if its owner must close the terminal to unblock a write. + shutdownTimer := time.NewTimer(c.frameConsumerShutdownTimeout()) + frameConsumerStopped := false + select { + case <-frameConsumerDone: + frameConsumerStopped = true + if !shutdownTimer.Stop() { + <-shutdownTimer.C + } + case <-shutdownTimer.C: + } + if cancelableRead && frameConsumerStopped { wg.Wait() } return normalizeCloseError(err) } +func (c *Client) frameConsumerShutdownTimeout() time.Duration { + if c.attachmentShutdownTimeout > 0 { + return c.attachmentShutdownTimeout + } + return defaultAttachmentShutdownTimeout +} + func (c *Client) rememberSize(size Size) { c.lastSize.Store(uint64(size.Cols)<<32 | uint64(size.Rows)) } diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 95f901f8..664f0494 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -1528,10 +1528,11 @@ func TestAttachReturnsWhenContextCancelsAnUncancelableRead(t *testing.T) { close(terminal.release) } -func TestAttachWaitsForFrameConsumerWithUncancelableRead(t *testing.T) { +func TestAttachBoundsBlockedFrameConsumerShutdown(t *testing.T) { client := &Client{ - readerDone: make(chan struct{}), - attachmentReady: make(chan struct{}), + readerDone: make(chan struct{}), + attachmentReady: make(chan struct{}), + attachmentShutdownTimeout: 10 * time.Millisecond, } terminal := newUncancelableReadBlockingWriteTerminal() ctx, cancel := context.WithCancel(context.Background()) @@ -1562,13 +1563,30 @@ func TestAttachWaitsForFrameConsumerWithUncancelableRead(t *testing.T) { cancel() select { case err := <-attachDone: - t.Fatalf("Attach retired before its frame consumer exited: %v", err) - case <-time.After(20 * time.Millisecond): + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Attach did not retire the blocked frame consumer") + } + + replacement := newBlockingTerminal() + replacementCtx, replacementCancel := context.WithCancel(context.Background()) + replacementDone := make(chan error, 1) + go func() { + replacementDone <- client.Attach(replacementCtx, replacement, nil) + }() + <-replacement.started + replacementCancel() + if err := <-replacementDone; !errors.Is(err, context.Canceled) { + t.Fatalf("replacement error = %v", err) } close(terminal.releaseWrite) - if err := <-attachDone; !errors.Is(err, context.Canceled) { - t.Fatalf("error = %v", err) + select { + case <-terminal.writeDone: + case <-time.After(time.Second): + t.Fatal("blocked frame consumer did not exit after terminal shutdown") } close(terminal.releaseRead) } @@ -1869,6 +1887,7 @@ type uncancelableReadBlockingWriteTerminal struct { writeStarted chan struct{} writeOnce sync.Once releaseWrite chan struct{} + writeDone chan struct{} } func newUncancelableReadBlockingWriteTerminal() *uncancelableReadBlockingWriteTerminal { @@ -1877,6 +1896,7 @@ func newUncancelableReadBlockingWriteTerminal() *uncancelableReadBlockingWriteTe releaseRead: make(chan struct{}), writeStarted: make(chan struct{}), releaseWrite: make(chan struct{}), + writeDone: make(chan struct{}), } } @@ -1889,6 +1909,7 @@ func (terminal *uncancelableReadBlockingWriteTerminal) Read(_ []byte) (int, erro } func (terminal *uncancelableReadBlockingWriteTerminal) Write(payload []byte) (int, error) { + defer close(terminal.writeDone) terminal.writeOnce.Do(func() { close(terminal.writeStarted) }) From 25791a18fd039ae07ea2ccca73c9cfffc6b41e49 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:50:27 +0200 Subject: [PATCH 164/242] fix(macos): preserve legacy uncertain publications --- .../CrabfleetDesktopRegistration.swift | 4 +- .../PrivateMacShareTests.swift | 96 ++++++++++++++++++- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index de41c2f7..6a9b1445 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -203,7 +203,9 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable } if http.statusCode == 404 { try validate(response: http, for: request, acceptingNotFound: true) - return nil + throw DesktopHostRegistrationResultUncertainError( + message: "Desktop publication recovery is unavailable on this server." + ) } do { try validate(response: http, for: request, acceptingNotFound: false) diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index eb9904f3..d8e6953b 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -905,7 +905,7 @@ struct PrivateMacShareTests { } @Test - func desktopRegistrationTreatsMissingRecoveryRouteAsNoOwnership() async throws { + func desktopRegistrationTreatsMissingRecoveryRouteAsUncertain() async throws { let transport = DesktopRegistrationTransport { request in let responseURL = try #require(request.url) return ( @@ -929,12 +929,47 @@ struct PrivateMacShareTests { )) let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) - #expect( + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { try await registration.recover( identity: identity, publicationID: "publication-id" - ) == nil + ) + } + } + + @Test @MainActor + func legacyRecoveryRoutePreservesUncertainPublicationWithoutDeletingNewerPublisher() + async throws + { + let transport = LegacyDesktopServerTransport() + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "legacy-publication" } ) + let identity = try TailnetIdentityPolicy.identity(from: statusDocument()) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + await transport.publishNewerEndpoint() + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.removePublishedIdentities() + } + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.removePublishedIdentities() + } + + #expect(await transport.activeEndpoint == "newer-publisher") + #expect(await transport.events == [.register, .recover, .recover]) } @Test @@ -2036,6 +2071,61 @@ private final class DesktopRegistrationTransport: HTTPDataTransport { func close() {} } +private actor LegacyDesktopServerTransport: HTTPDataTransport { + enum Event: Equatable { + case register + case recover + case unregister + } + + private(set) var activeEndpoint: String? + private(set) var events: [Event] = [] + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let responseURL = try #require(request.url) + switch request.httpMethod { + case "PUT": + events.append(.register) + activeEndpoint = "legacy-publisher" + throw URLError(.networkConnectionLost) + case "POST": + events.append(.recover) + return ( + Data(), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 404, + httpVersion: nil, + headerFields: nil + )) + ) + case "DELETE": + events.append(.unregister) + activeEndpoint = nil + return ( + Data(), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + default: + Issue.record("unexpected legacy desktop request method") + throw URLError(.badURL) + } + } + + func publishNewerEndpoint() { + activeEndpoint = "newer-publisher" + } + + nonisolated func close() {} +} + private func waitUntilAsync( timeout: Duration = .seconds(2), condition: @escaping () async -> Bool From 351acb9e1d1fce5c9bbfe9faa8b868282fc69915 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:50:58 +0200 Subject: [PATCH 165/242] fix(actions): carry initial viewer generation --- src/github-actions-runtime.ts | 6 ++ src/worker/interactive-terminal-service.ts | 2 + src/worker/session-control-do.ts | 20 +++++-- src/worker/terminal-hub.ts | 5 +- tests/application-architecture.test.ts | 5 ++ tests/github-actions-runtime.test.ts | 13 ++++ tests/terminal-hub.test.ts | 69 ++++++++++++++++++++++ 7 files changed, 113 insertions(+), 7 deletions(-) diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 5b5b47ab..d93f7725 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -36,6 +36,7 @@ export const githubActionsGenerationFencedCapability = "cfr1-framed-io-v2"; export const githubActionsRunnerProtocolQuery = "runnerProtocol"; export const githubActionsViewerProtocolQuery = "viewerProtocol"; export const githubActionsViewerProtocolHeader = "x-crabfleet-viewer-protocol"; +export const githubActionsViewerGenerationHeader = "x-crabfleet-runner-generation"; export type GitHubActionsRelayProtocol = | typeof githubActionsFramedRunnerCapability | typeof githubActionsGenerationFencedCapability; @@ -137,6 +138,11 @@ export function gitHubActionsViewerResponseUsesGenerations(response: Response): ); } +export function gitHubActionsViewerResponseGeneration(response: Response): string | null { + const generation = response.headers.get(githubActionsViewerGenerationHeader); + return isGitHubActionsRelayGeneration(generation) ? generation : null; +} + export function parseGitHubActionsWorkState(value: unknown): GitHubActionsWorkState | null { const state = String(value ?? "").trim() as GitHubActionsWorkState; return workStates.has(state) ? state : null; diff --git a/src/worker/interactive-terminal-service.ts b/src/worker/interactive-terminal-service.ts index 4f133115..bea36e7e 100644 --- a/src/worker/interactive-terminal-service.ts +++ b/src/worker/interactive-terminal-service.ts @@ -8,6 +8,7 @@ import { } from "../terminal-multiplayer.ts"; import { buildGitHubActionsViewerRelayUrl, + gitHubActionsViewerResponseGeneration, gitHubActionsViewerResponseUsesFramedProtocol, gitHubActionsViewerResponseUsesGenerations, githubActionsRuntime, @@ -143,6 +144,7 @@ export class InteractiveTerminalService { socket: upstream, inputAcknowledgements: gitHubActionsViewerResponseUsesFramedProtocol(upstreamResponse), inputGenerations: gitHubActionsViewerResponseUsesGenerations(upstreamResponse), + initialRunnerGeneration: gitHubActionsViewerResponseGeneration(upstreamResponse), outputAcknowledgements: false, markConnected: () => markInteractiveTerminalConnected( diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index 8c102a39..2cfc2a14 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -15,6 +15,7 @@ import { githubActionsLegacyRelayGeneration, githubActionsRelayRole, githubActionsRunnerProtocolQuery, + githubActionsViewerGenerationHeader, githubActionsViewerProtocolHeader, githubActionsViewerProtocolQuery, notifyGitHubActionsViewers, @@ -249,6 +250,7 @@ export class SessionControlDO extends DurableObject { const pair = new WebSocketPair(); const client = pair[0]; const server = pair[1]; + let initialRunnerGeneration: string | undefined; if (role === "runner") { const generation = createGitHubActionsRelayGeneration(); replaceGitHubActionsRunner(this.ctx.getWebSockets("github-actions-runner")); @@ -266,18 +268,24 @@ export class SessionControlDO extends DurableObject { .getWebSockets("github-actions-runner") .find((socket) => socket.readyState === WebSocket.OPEN); if (!runner) { + if (gitHubActionsRelayUsesGenerations(server)) { + initialRunnerGeneration = "none"; + } notifyGitHubActionsViewers([server], "runner_waiting"); } else if (gitHubActionsRelayUsesGenerations(server)) { - notifyGitHubActionsViewers( - [server], - "runner_connected", - gitHubActionsRelayGeneration(runner) ?? githubActionsLegacyRelayGeneration, - ); + initialRunnerGeneration = + gitHubActionsRelayGeneration(runner) ?? githubActionsLegacyRelayGeneration; + notifyGitHubActionsViewers([server], "runner_connected", initialRunnerGeneration); } } const responseInit: ResponseInit = { status: 101, webSocket: client }; if (role === "viewer" && protocol) { - responseInit.headers = { [githubActionsViewerProtocolHeader]: protocol }; + responseInit.headers = { + [githubActionsViewerProtocolHeader]: protocol, + ...(initialRunnerGeneration + ? { [githubActionsViewerGenerationHeader]: initialRunnerGeneration } + : {}), + }; } return new Response(null, responseInit); } diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 7dde562b..f79ae393 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -47,6 +47,7 @@ export type TerminalUpstream = { markConnected: () => Promise; inputAcknowledgements?: boolean; inputGenerations?: boolean; + initialRunnerGeneration?: string | null; outputAcknowledgements: boolean; }; @@ -531,7 +532,9 @@ export class TerminalHub { inputQueueRejectionScheduled: false, inputGenerations: upstreamConnection.inputGenerations ?? false, pendingInputAcknowledgements: new Map(), - runnerGeneration: upstreamConnection.inputGenerations ? "none" : 0, + runnerGeneration: upstreamConnection.inputGenerations + ? (upstreamConnection.initialRunnerGeneration ?? "none") + : 0, outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; diff --git a/tests/application-architecture.test.ts b/tests/application-architecture.test.ts index 085229ac..251c862b 100644 --- a/tests/application-architecture.test.ts +++ b/tests/application-architecture.test.ts @@ -91,6 +91,11 @@ test("GitHub Actions viewer protocol is requested and attached before relay acce assert.notEqual(attach, -1); assert.ok(accept > attach); assert.match(relay, /\[githubActionsViewerProtocolHeader\]: protocol/); + assert.match(relay, /\[githubActionsViewerGenerationHeader\]: initialRunnerGeneration/); + assert.match( + terminal, + /initialRunnerGeneration: gitHubActionsViewerResponseGeneration\(upstreamResponse\)/, + ); }); test("worker entrypoint retains only routing and platform composition", async () => { diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index eb8a188a..d922c90f 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -18,8 +18,10 @@ import { githubActionsRelayRole, githubActionsRunnerProtocolQuery, githubActionsRuntimeLabel, + githubActionsViewerGenerationHeader, githubActionsViewerProtocolHeader, githubActionsViewerProtocolQuery, + gitHubActionsViewerResponseGeneration, gitHubActionsViewerResponseUsesFramedProtocol, gitHubActionsViewerResponseUsesGenerations, gitHubActionsRunnerUsesFramedProtocol, @@ -121,6 +123,7 @@ test("runner URL works without custom WebSocket headers", () => { ); assert.equal(githubActionsViewerProtocolQuery, "viewerProtocol"); assert.equal(githubActionsViewerProtocolHeader, "x-crabfleet-viewer-protocol"); + assert.equal(githubActionsViewerGenerationHeader, "x-crabfleet-runner-generation"); assert.equal( gitHubActionsViewerResponseUsesFramedProtocol( new Response(null, { @@ -134,11 +137,21 @@ test("runner URL works without custom WebSocket headers", () => { assert.equal(gitHubActionsViewerResponseUsesFramedProtocol(new Response()), false); const generatedResponse = new Response(null, { headers: { + [githubActionsViewerGenerationHeader]: "generation-one", [githubActionsViewerProtocolHeader]: githubActionsGenerationFencedCapability, }, }); assert.equal(gitHubActionsViewerResponseUsesFramedProtocol(generatedResponse), true); assert.equal(gitHubActionsViewerResponseUsesGenerations(generatedResponse), true); + assert.equal(gitHubActionsViewerResponseGeneration(generatedResponse), "generation-one"); + assert.equal( + gitHubActionsViewerResponseGeneration( + new Response(null, { + headers: { [githubActionsViewerGenerationHeader]: "invalid generation" }, + }), + ), + null, + ); }); test("work states preserve running phases and map terminal outcomes", () => { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 09c3656c..c9552429 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -1718,6 +1718,75 @@ test("relay generations bind interleaved replacement input before lifecycle proc server.emit("close"); }); +test("viewer carries its initial runner generation through authorization setup", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + let releaseView!: (allowed: boolean) => void; + const viewAllowed = new Promise((resolve) => { + releaseView = resolve; + }); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + viewGrant: () => () => viewAllowed, + async openUpstream() { + return { + socket: upstream, + inputAcknowledgements: true, + inputGenerations: true, + initialRunnerGeneration: "generation-initial", + outputAcknowledgements: false, + async markConnected() {}, + }; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + emitRelayEvent(upstream, "runner_connected", "generation-initial"); + releaseView(true); + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("first input"), + }), + }); + await waitForInputPayloads(); + + const input = relayInput(upstream.sent.at(-1)!); + assert.equal(input.generation, "generation-initial"); + emitRelayAcknowledgement(upstream, input.inputId, true, "generation-initial"); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string }) + .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + assert.deepEqual( + completions.map((message) => message.type), + ["input-accepted"], + ); + server.emit("close"); +}); + test("terminal hub immediately acknowledges upstream output when the client opts out", async () => { const client = socket(); const server = socket(); From 88664d652dbe27a24e6b004d5a3bd1c5d29535e6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:51:36 +0200 Subject: [PATCH 166/242] docs(changelog): record final race fixes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 886dbef8..6dc6b336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, serializing per-runner PTY writes and acknowledgements, matching generation-fenced local send failures, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; durably claim and retry superseded workspace cleanup without touching the replacement workspace; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. From c1f477b49b54cb5b95f6a8944fd54e02a6aa32d2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:03:22 +0200 Subject: [PATCH 167/242] fix(desktop): preserve idempotent publication recovery --- ...0038_desktop_host_publication_identity.sql | 16 +++-- src/worker/database.ts | 1 + src/worker/desktop-host-repository.ts | 2 + tests/desktop-host-migration.test.ts | 12 ++-- tests/desktop-host-repository.test.ts | 64 +++++++++++++++++++ 5 files changed, 86 insertions(+), 9 deletions(-) diff --git a/migrations/0038_desktop_host_publication_identity.sql b/migrations/0038_desktop_host_publication_identity.sql index bc93a4cb..52f1b1fd 100644 --- a/migrations/0038_desktop_host_publication_identity.sql +++ b/migrations/0038_desktop_host_publication_identity.sql @@ -1,19 +1,25 @@ ALTER TABLE desktop_hosts ADD COLUMN publication_id TEXT NOT NULL DEFAULT ''; --- Older token-aware workers replace ownership_token without knowing about --- publication_id. Clear the stale identity so a previous publisher cannot --- recover authority over the replacement row. +ALTER TABLE desktop_hosts + ADD COLUMN publication_write_token TEXT NOT NULL DEFAULT ''; + +-- New workers rotate publication_write_token with ownership_token. Older +-- token-aware workers leave it unchanged, which identifies a token-only write +-- that must invalidate recovery authority for the previous publisher. CREATE TRIGGER IF NOT EXISTS clear_stale_desktop_host_publication_identity AFTER UPDATE ON desktop_hosts WHEN OLD.publication_id <> '' AND NEW.ownership_token <> OLD.ownership_token AND NEW.publication_id = OLD.publication_id + AND NEW.publication_write_token = OLD.publication_write_token BEGIN UPDATE desktop_hosts - SET publication_id = '' + SET publication_id = '', + publication_write_token = '' WHERE owner_subject = NEW.owner_subject AND id = NEW.id AND ownership_token = NEW.ownership_token - AND publication_id = NEW.publication_id; + AND publication_id = NEW.publication_id + AND publication_write_token = NEW.publication_write_token; END; diff --git a/src/worker/database.ts b/src/worker/database.ts index 0608d6f8..1fc0c6bd 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -71,6 +71,7 @@ export type DesktopHostTable = { port: number; ownership_token: string; publication_id: string; + publication_write_token: Generated; created_at: number; updated_at: number; }; diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index b24f30f2..6e6b5499 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -70,6 +70,7 @@ export class DesktopHostRepository implements DesktopHostStore { port: host.port, ownership_token: host.ownershipToken, publication_id: host.publicationID, + publication_write_token: host.ownershipToken, created_at: host.createdAt, updated_at: host.updatedAt, }) @@ -83,6 +84,7 @@ export class DesktopHostRepository implements DesktopHostStore { port: host.port, ownership_token: host.ownershipToken, publication_id: host.publicationID, + publication_write_token: host.ownershipToken, updated_at: host.updatedAt, }) : update.doUpdateSet({ diff --git a/tests/desktop-host-migration.test.ts b/tests/desktop-host-migration.test.ts index e4a6863a..2476ef9a 100644 --- a/tests/desktop-host-migration.test.ts +++ b/tests/desktop-host-migration.test.ts @@ -72,10 +72,10 @@ test("desktop host publication migration clears identities rotated by old worker database.exec(` INSERT INTO desktop_hosts ( owner_subject, id, owner, name, address, port, ownership_token, publication_id, - created_at, updated_at + publication_write_token, created_at, updated_at ) VALUES ( 'github:1', 'studio', 'alice', 'Studio', '100.64.1.2', 5901, - 'token-a', 'publication-a', 1, 2 + 'token-a', 'publication-a', 'token-a', 1, 2 ); UPDATE desktop_hosts SET ownership_token = 'token-b' @@ -85,10 +85,14 @@ test("desktop host publication migration clears identities rotated by old worker assert.deepEqual( { ...database - .prepare("SELECT ownership_token, publication_id FROM desktop_hosts WHERE id = 'studio'") + .prepare(` + SELECT ownership_token, publication_id, publication_write_token + FROM desktop_hosts + WHERE id = 'studio' + `) .get(), }, - { ownership_token: "token-b", publication_id: "" }, + { ownership_token: "token-b", publication_id: "", publication_write_token: "" }, ); }); diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index 1048c4fb..dbe296a2 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -369,3 +369,67 @@ test("desktop host publication recovery matches only the current publication", a "token-b", ); }); + +test("same-publication retries remain recoverable after the publication migration", async () => { + const sqlite = new DatabaseSync(":memory:"); + for (const migration of [ + "0030_desktop_hosts.sql", + "0033_desktop_host_ownership.sql", + "0038_desktop_host_publication_identity.sql", + ]) { + sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); + } + const repository = new DesktopHostRepository(sqliteRuntimeEnv(sqlite)); + const host = { + ownerSubject: "github:1", + id: "studio", + owner: "alice", + name: "Studio", + address: "100.64.1.2", + port: 5901, + publicationID: "publication-a", + createdAt: 1, + }; + + await repository.upsert({ + ...host, + ownershipToken: "token-a", + updatedAt: 2, + }); + await repository.upsert({ + ...host, + ownershipToken: "token-b", + updatedAt: 3, + }); + + assert.deepEqual( + { + ...sqlite + .prepare(` + SELECT ownership_token, publication_id, publication_write_token + FROM desktop_hosts + WHERE owner_subject = 'github:1' AND id = 'studio' + `) + .get(), + }, + { + ownership_token: "token-b", + publication_id: "publication-a", + publication_write_token: "token-b", + }, + ); + assert.equal( + await repository.ownershipTokenForPublication("github:1", "studio", "publication-a"), + "token-b", + ); + + sqlite.exec(` + UPDATE desktop_hosts + SET ownership_token = 'token-c' + WHERE owner_subject = 'github:1' AND id = 'studio' + `); + assert.equal( + await repository.ownershipTokenForPublication("github:1", "studio", "publication-a"), + null, + ); +}); From f5f3bda44a990dcad82a46e629d37dcc6b9ca6d7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:03:59 +0200 Subject: [PATCH 168/242] fix(actions): bound runner input delivery --- src/github-actions-runner.ts | 113 ++++++++++++++++++----- src/github-actions-runtime.ts | 4 +- tests/github-actions-runner.test.ts | 131 +++++++++++++++++++++++++++ tests/github-actions-runtime.test.ts | 32 +++++++ 4 files changed, 257 insertions(+), 23 deletions(-) diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 09b5825c..3a0e78ef 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -5,7 +5,19 @@ import { type GitHubActionsRelaySocket, } from "./github-actions-runtime.ts"; -const runnerInputQueues = new WeakMap>(); +const runnerInputQueueMaxBytes = 16 * 1024 * 1024; +const runnerInputQueueMaxFrames = 32; +const runnerInputQueueMaxAgeMs = 5_000; +const runnerInputBacklogError = "GitHub Actions runner input backlog exceeded"; +const runnerInputExpiredError = "GitHub Actions runner input expired"; + +type RunnerInputQueue = { + bytes: number; + frames: number; + tail: Promise; +}; + +const runnerInputQueues = new WeakMap(); export function sendGitHubActionsRunnerOutput( socket: GitHubActionsRelaySocket, @@ -14,39 +26,96 @@ export function sendGitHubActionsRunnerOutput( socket.send(encodeGitHubActionsRelayOutput(output)); } -export async function acceptGitHubActionsRunnerInput( +export function acceptGitHubActionsRunnerInput( socket: GitHubActionsRelaySocket, message: string | ArrayBuffer, writeToPty: (payload: ArrayBuffer) => void | Promise, + now: () => number = Date.now, ): Promise { const input = parseGitHubActionsRelayInput(message); - if (!input) return false; + if (!input) return Promise.resolve(false); - const queued = (runnerInputQueues.get(socket) ?? Promise.resolve()) + const queue = runnerInputQueues.get(socket) ?? { + bytes: 0, + frames: 0, + tail: Promise.resolve(), + }; + runnerInputQueues.set(socket, queue); + + if ( + queue.frames >= runnerInputQueueMaxFrames || + queue.bytes + input.payload.byteLength > runnerInputQueueMaxBytes + ) { + const { generation, inputId } = input; + const rejected = queue.tail + .catch(() => undefined) + .then(() => { + sendRunnerInputAcknowledgement(socket, inputId, generation, false, runnerInputBacklogError); + }); + queue.tail = rejected; + return rejected + .finally(() => { + deleteIdleRunnerInputQueue(socket, queue, rejected); + }) + .then(() => true); + } + + const queuedAt = now(); + queue.frames += 1; + queue.bytes += input.payload.byteLength; + const queued = queue.tail .catch(() => undefined) .then(async () => { + if (now() - queuedAt >= runnerInputQueueMaxAgeMs) { + sendRunnerInputAcknowledgement( + socket, + input.inputId, + input.generation, + false, + runnerInputExpiredError, + ); + return; + } try { await writeToPty(input.payload); - sendGitHubActionsRelayInputAcknowledgement(socket, { - inputId: input.inputId, - accepted: true, - ...(input.generation ? { generation: input.generation } : {}), - }); + sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, true); } catch { - sendGitHubActionsRelayInputAcknowledgement(socket, { - inputId: input.inputId, - accepted: false, - ...(input.generation ? { generation: input.generation } : {}), - }); + sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, false); } + }) + .finally(() => { + queue.frames -= 1; + queue.bytes -= input.payload.byteLength; }); - runnerInputQueues.set(socket, queued); - try { - await queued; - } finally { - if (runnerInputQueues.get(socket) === queued) { - runnerInputQueues.delete(socket); - } + queue.tail = queued; + return queued + .finally(() => { + deleteIdleRunnerInputQueue(socket, queue, queued); + }) + .then(() => true); +} + +function sendRunnerInputAcknowledgement( + socket: GitHubActionsRelaySocket, + inputId: string, + generation: string | undefined, + accepted: boolean, + error?: string, +): void { + sendGitHubActionsRelayInputAcknowledgement(socket, { + inputId, + accepted, + ...(error ? { error } : {}), + ...(generation ? { generation } : {}), + }); +} + +function deleteIdleRunnerInputQueue( + socket: GitHubActionsRelaySocket, + queue: RunnerInputQueue, + tail: Promise, +): void { + if (runnerInputQueues.get(socket) === queue && queue.tail === tail && queue.frames === 0) { + runnerInputQueues.delete(socket); } - return true; } diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index d93f7725..6bce6496 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -301,7 +301,9 @@ export function relayGitHubActionsWebSocketMessage( viewer.send( acknowledgement ? encodeGitHubActionsRelayInputAcknowledgement({ - ...acknowledgement, + inputId: acknowledgement.inputId, + accepted: acknowledgement.accepted, + ...(acknowledgement.error ? { error: acknowledgement.error } : {}), ...(gitHubActionsRelayUsesGenerations(viewer) && generation ? { generation } : {}), }) : gitHubActionsViewerUsesFramedProtocol(viewer) diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 7ed2e524..4f943aae 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -90,6 +90,137 @@ test("runner serializes concurrent input writes and acknowledgements per socket" ); }); +test("runner bounds queued frames and rejects overflow in acknowledgement order", async () => { + const socket = relaySocket(); + let completeFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + completeFirstWrite = resolve; + }); + const writes: number[] = []; + const pending = Array.from({ length: 33 }, (_, index) => + acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput(`input-${index}`, new Uint8Array([index])), + async (payload) => { + writes.push(new Uint8Array(payload)[0]!); + if (index === 0) await firstWrite; + }, + ), + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.deepEqual(writes, [0]); + completeFirstWrite(); + assert.deepEqual(await Promise.all(pending), Array(33).fill(true)); + assert.deepEqual( + writes, + Array.from({ length: 32 }, (_, index) => index), + ); + assert.deepEqual( + socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), + [ + ...Array.from({ length: 32 }, (_, index) => ({ + inputId: `input-${index}`, + accepted: true, + })), + { + inputId: "input-32", + accepted: false, + error: "GitHub Actions runner input backlog exceeded", + }, + ], + ); +}); + +test("runner bounds queued bytes while a PTY write is stalled", async () => { + const socket = relaySocket(); + let completeFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + completeFirstWrite = resolve; + }); + let writes = 0; + const first = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-first", new Uint8Array(9 * 1024 * 1024)), + async () => { + writes += 1; + await firstWrite; + }, + ); + const overflow = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-overflow", new Uint8Array(8 * 1024 * 1024)), + async () => { + writes += 1; + }, + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.equal(writes, 1); + completeFirstWrite(); + assert.deepEqual(await Promise.all([first, overflow]), [true, true]); + assert.equal(writes, 1); + assert.deepEqual( + socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), + [ + { inputId: "input-first", accepted: true }, + { + inputId: "input-overflow", + accepted: false, + error: "GitHub Actions runner input backlog exceeded", + }, + ], + ); +}); + +test("runner rejects queued input that outlives the viewer acknowledgement timeout", async () => { + const socket = relaySocket(); + let completeFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + completeFirstWrite = resolve; + }); + let now = 0; + const writes: string[] = []; + const first = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-first", "first"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + await firstWrite; + }, + () => now, + ); + const expired = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-expired", "expired"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + () => now, + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.deepEqual(writes, ["first"]); + now = 5_000; + completeFirstWrite(); + assert.deepEqual(await Promise.all([first, expired]), [true, true]); + assert.deepEqual(writes, ["first"]); + assert.deepEqual( + socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), + [ + { inputId: "input-first", accepted: true }, + { + inputId: "input-expired", + accepted: false, + error: "GitHub Actions runner input expired", + }, + ], + ); +}); + test("runner copies the relay generation into its acknowledgement", async () => { const socket = relaySocket(); diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index d922c90f..196f3e7a 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -350,6 +350,38 @@ test("relay-owned generations fence stale input and bridge framed protocol versi }); }); +test("generation acknowledgements bridge to exact v1 frames for v1 viewers", () => { + const runner = relaySocket(); + const viewer = framedViewer(); + attachGitHubActionsRunnerProtocol( + runner, + githubActionsGenerationFencedCapability, + "current-generation", + ); + const acknowledgement = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "input-current", + accepted: false, + error: "rejected", + generation: "current-generation", + }); + + assert.equal( + relayGitHubActionsWebSocketMessage("runner", runner, acknowledgement, [runner], [viewer]), + 1, + ); + const expected = encodeGitHubActionsRelayInputAcknowledgement({ + inputId: "input-current", + accepted: false, + error: "rejected", + }); + assert.deepEqual(new Uint8Array(viewer.sent[0] as ArrayBuffer), new Uint8Array(expected)); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(viewer.sent[0]!), { + inputId: "input-current", + accepted: false, + error: "rejected", + }); +}); + test("replacement relay drops acknowledgements from the superseded runner", () => { const oldRunner = relaySocket(); const replacement = relaySocket(); From c44c36772a0a5984a9ee92695b2ce9c685ef5e2e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:04:01 +0200 Subject: [PATCH 169/242] fix(terminal): serialize raw input acknowledgements --- internal/terminalws/client.go | 38 +++++++- internal/terminalws/client_test.go | 140 +++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index f1384456..19125100 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -272,6 +272,31 @@ func (c *Client) SendInput(ctx context.Context, payload []byte) error { if len(payload) == 0 { return nil } + if !c.canInput.Load() { + return errors.New("terminal control has not been granted") + } + if !c.supportsInputAcknowledgement { + return c.writeInput(ctx, payload) + } + if err := c.acquireConfirmation(ctx); err != nil { + return err + } + + waiter := make(chan error, 1) + if err := c.registerInputWaiter(waiter); err != nil { + c.releaseConfirmation() + return err + } + if err := c.writeInput(ctx, payload); err != nil { + c.clearInputWaiter(waiter) + c.releaseConfirmation() + return err + } + go c.drainInputConfirmation(waiter) + return nil +} + +func (c *Client) writeInput(ctx context.Context, payload []byte) error { if !c.canInput.Load() { return errors.New("terminal control has not been granted") } @@ -298,10 +323,21 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { if err := c.registerInputWaiter(waiter); err != nil { return err } - if err := c.SendInput(ctx, payload); err != nil { + if err := c.writeInput(ctx, payload); err != nil { c.clearInputWaiter(waiter) return err } + return c.waitForInputConfirmation(ctx, waiter) +} + +func (c *Client) drainInputConfirmation(waiter chan error) { + defer c.releaseConfirmation() + ctx, cancel := context.WithTimeout(context.Background(), c.inputConfirmationTimeout()) + defer cancel() + _ = c.waitForInputConfirmation(ctx, waiter) +} + +func (c *Client) waitForInputConfirmation(ctx context.Context, waiter chan error) error { select { case err := <-waiter: return err diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 664f0494..20cc3cb2 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -1017,6 +1017,146 @@ func TestSendInputConfirmedSharesOneReaderWithAttach(t *testing.T) { } } +func TestSendInputDrainsAcknowledgementBeforeConfirmedSend(t *testing.T) { + rawReceived := make(chan struct{}) + confirmedReceived := make(chan struct{}) + releaseRawAcknowledgement := make(chan struct{}) + releaseConfirmedAcknowledgement := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-raw-confirmed", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + + inputs := make(chan frame, 2) + go func() { + for range 2 { + _, payload, readErr := conn.Read(r.Context()) + if readErr != nil { + return + } + current, decodeErr := decodeFrame(payload) + if decodeErr != nil { + t.Error(decodeErr) + return + } + inputs <- current + } + }() + + raw := <-inputs + if raw.messageType != messageInput || string(raw.payload) != "raw\n" { + t.Errorf("raw input = %#v", raw) + return + } + close(rawReceived) + select { + case confirmed := <-inputs: + if confirmed.messageType != messageInput || string(confirmed.payload) != "confirmed\n" { + t.Errorf("confirmed input = %#v", confirmed) + return + } + close(confirmedReceived) + case <-releaseRawAcknowledgement: + } + + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-raw-confirmed", + payload: accepted, + })); err != nil { + t.Error(err) + return + } + select { + case <-confirmedReceived: + default: + confirmed := <-inputs + if confirmed.messageType != messageInput || string(confirmed.payload) != "confirmed\n" { + t.Errorf("confirmed input = %#v", confirmed) + return + } + close(confirmedReceived) + } + <-releaseConfirmedAcknowledgement + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-raw-confirmed", + payload: accepted, + })); err != nil { + t.Error(err) + } + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-raw-confirmed", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + if err := client.SendInput(context.Background(), []byte("raw\n")); err != nil { + t.Fatal(err) + } + <-rawReceived + + confirmedStarted := make(chan struct{}) + confirmedDone := make(chan error, 1) + go func() { + close(confirmedStarted) + confirmedDone <- client.SendInputConfirmed(context.Background(), []byte("confirmed\n")) + }() + <-confirmedStarted + select { + case <-confirmedReceived: + t.Fatal("confirmed input was written before the raw acknowledgement") + case <-time.After(100 * time.Millisecond): + } + + close(releaseRawAcknowledgement) + <-confirmedReceived + select { + case err := <-confirmedDone: + t.Fatalf("confirmed send completed from the raw acknowledgement: %v", err) + case <-time.After(100 * time.Millisecond): + } + + close(releaseConfirmedAcknowledgement) + if err := <-confirmedDone; err != nil { + t.Fatal(err) + } +} + func TestSendInputConfirmedCanCancelWhileWaitingForPreviousConfirmation(t *testing.T) { firstInput := make(chan struct{}) releaseFirst := make(chan struct{}) From 31e6983cca364f16443ed2f700314074bdf866e8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:04:29 +0200 Subject: [PATCH 170/242] docs(changelog): record protocol follow-up fixes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc6b336..07f49a3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. -- Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, serializing per-runner PTY writes and acknowledgements, matching generation-fenced local send failures, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, and retaining uncertain Share This Mac publications when older servers lack the recovery route. +- Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; durably claim and retry superseded workspace cleanup without touching the replacement workspace; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. From f543d271af0b428b48ef194426c154b327410e47 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:14:59 +0200 Subject: [PATCH 171/242] docs(actions): restrict runner input to steering --- docs/github-actions-sessions.md | 60 +++++++++++++++---------------- tests/github-actions-docs.test.ts | 10 +++--- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 05ba0994..7b9d9f80 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -274,14 +274,15 @@ correlation ID and that generation, and runner output uses a distinct `CFR1` output frame. The runner copies the generation into its correlated acknowledgement only after its PTY accepts the input write. -Complete Node runner integration: - -```sh -npm install @lydell/node-pty -``` +Complete Node framing adapter around a restricted Codex steering handler: ```js -import { spawn } from "@lydell/node-pty"; +import { + closeSteering, + deliverSteeringInput, + subscribeSteeringExit, + subscribeSteeringOutput, +} from "./restricted-codex-steering.js"; const runnerPtyUrl = process.env.CRABFLEET_RUNNER_PTY_URL; if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); @@ -305,24 +306,19 @@ await new Promise((resolve, reject) => { terminal.addEventListener("error", reject, { once: true }); }); -const pty = spawn(process.env.SHELL || "/bin/bash", [], { - cwd: process.cwd(), - env: process.env, -}); - -pty.onData((outputText) => { +subscribeSteeringOutput((outputText) => { terminal.send(encodeUtf8Output(outputText)); }); -pty.onExit(() => { +subscribeSteeringExit(() => { if (terminal.readyState < WebSocket.CLOSING) terminal.close(1000, "pty exited"); }); terminal.addEventListener("message", (event) => { - acceptInput(event.data); + void acceptInput(event.data); }); -function acceptInput(data) { +async function acceptInput(data) { const input = decodeInput(data); if (!input) return; pendingInputs.push(input); @@ -345,7 +341,7 @@ function acceptInput(data) { try { const text = decodeCompleteUtf8(payload); if (text === null) return; - pty.write(text); + await deliverSteeringInput(text); settlePendingInputs(true); } catch { settlePendingInputs(false); @@ -427,32 +423,36 @@ function encodeUtf8Output(outputText) { } terminal.addEventListener("close", () => { - pty.kill(); + closeSteering(); }); terminal.addEventListener("error", () => { - pty.kill(); + closeSteering(); }); ``` Set `CRABFLEET_RUNNER_PTY_URL` to the `runnerPtyUrl` returned by registration. -For a PTY API with an asynchronous write callback or promise, await that -acceptance signal before sending `encodeAck(..., true)`. Do not acknowledge when -the WebSocket merely queues the input frame. This Node adapter buffers a valid -incomplete UTF-8 suffix together with every affected input ID. It writes and -positively acknowledges those frames only after a later frame completes the -sequence. Invalid UTF-8 rejects the buffered group without delivering any of it. -The adapter also rejects the whole pending group when it exceeds 16 KiB, 32 -frames, or one second, bounding memory, copy work, and acknowledgement latency. +Implement `restricted-codex-steering.js` as the integration's narrow +`turn/steer` and `turn/interrupt` adapter. `deliverSteeringInput` must consume +browser input as steering instructions; it must never forward that input to a +shell or subprocess, and the adapter must not expose the GitHub Actions +environment. Await the steering acceptance signal before sending +`encodeAck(..., true)`. Do not acknowledge when the WebSocket merely queues the +input frame. This Node adapter buffers a valid incomplete UTF-8 suffix together +with every affected input ID. It delivers and positively acknowledges those +frames only after a later frame completes the sequence. Invalid UTF-8 rejects +the buffered group without delivering any of it. The adapter also rejects the +whole pending group when it exceeds 16 KiB, 32 frames, or one second, bounding +memory, copy work, and acknowledgement latency. The protocol query is consumed during connection setup and is not forwarded as terminal data. There is no capability message or mode transition after the socket opens. Each `CFR1` frame occupies one binary WebSocket message. At the wire level, input and output payloads are opaque terminal bytes. The example is -deliberately a UTF-8 text adapter because `@lydell/node-pty` exposes input and -output as JavaScript strings: it rejects input that is not complete valid UTF-8 -and encodes each output string as UTF-8. Deployments that require lossless -arbitrary PTY bytes must use a byte-oriented PTY adapter instead of this example. +deliberately a UTF-8 text adapter for the integration's string-based steering +surface: it rejects input that is not complete valid UTF-8 and encodes each +output string as UTF-8. Deployments that require lossless arbitrary terminal +bytes must use a byte-oriented restricted steering adapter instead. | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------ | diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 9f045d43..d124f102 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -17,7 +17,7 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.match(guide, /pendingInputs\.push\(input\)/); assert.match(guide, /const text = decodeCompleteUtf8\(payload\)/); assert.match(guide, /if \(text === null\) return/); - assert.match(guide, /pty\.write\(text\);\s+settlePendingInputs\(true\)/); + assert.match(guide, /await deliverSteeringInput\(text\);\s+settlePendingInputs\(true\)/); assert.match(guide, /const maxPendingInputBytes = 16 \* 1024/); assert.match(guide, /const maxPendingInputFrames = 32/); assert.match(guide, /const maxPendingInputAgeMs = 1_000/); @@ -27,16 +27,18 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.match(guide, /clearTimeout\(pendingInputTimer\)/); assert.match(guide, /new TextDecoder\("utf-8", \{ fatal: true, ignoreBOM: true \}\)/); assert.doesNotMatch(guide, /inputDecoder\.decode/); - assert.match(guide, /pty\.onData\(\(outputText\) => \{/); + assert.match(guide, /subscribeSteeringOutput\(\(outputText\) => \{/); assert.match(guide, /encodeUtf8Output\(outputText\)/); assert.match(guide, /deliberately a UTF-8 text adapter/); - assert.match(guide, /lossless\s+arbitrary PTY bytes must use a byte-oriented PTY adapter/); + assert.match(guide, /byte-oriented restricted steering adapter/); assert.match(guide, /Generation-fenced viewers add `viewerProtocol=cfr1-framed-io-v2`/); assert.match(guide, /stale-generation input is rejected before it\s+can reach/); assert.match(guide, /encodeAck\(input\.inputId, input\.generation, accepted\)/); assert.match(guide, /Legacy viewers omit that query/); - assert.match(guide, /pty\.onExit\(\(\) => \{/); + assert.match(guide, /subscribeSteeringExit\(\(\) => \{/); assert.match(guide, /terminal\.close\(1000, "pty exited"\)/); + assert.match(guide, /must never forward that input to a\s+shell or subprocess/); + assert.doesNotMatch(guide, /spawn\(process\.env\.SHELL|env:\s*process\.env|pty\.write/); const decodeCompleteUtf8 = (payload: Uint8Array) => { const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); From 66a2e5dece419a7c91109fc9005809590e64d78e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:15:52 +0200 Subject: [PATCH 172/242] fix(terminal): fence retired attachment delivery --- internal/terminalws/client.go | 54 +++++++++++++++--- internal/terminalws/client_test.go | 92 +++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 10 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 19125100..134b7312 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -106,10 +106,15 @@ type frame struct { } type terminalAttachment struct { - frames chan frame + frames chan attachmentDelivery done chan struct{} } +type attachmentDelivery struct { + frame frame + accepted chan bool +} + type eventPayload struct { Type string `json:"type"` Error string `json:"error"` @@ -477,7 +482,11 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c case <-c.readerDone: errCh <- c.readerError() return - case current := <-attachment.frames: + case delivery := <-attachment.frames: + if !c.acceptAttachmentDelivery(attachment, delivery) { + return + } + current := delivery.frame switch current.messageType { case messageOutput: if _, err := terminal.Write(current.payload); err != nil { @@ -684,7 +693,7 @@ func (c *Client) registerAttachment() (*terminalAttachment, error) { default: } attachment := &terminalAttachment{ - frames: make(chan frame), + frames: make(chan attachmentDelivery), done: make(chan struct{}), } c.attachment = attachment @@ -742,11 +751,10 @@ func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) error return ctx.Err() } } - select { - case attachment.frames <- current: + if c.deliverToAttachment(ctx, attachment, current) { return nil - case <-attachment.done: - case <-ctx.Done(): + } + if err := ctx.Err(); err != nil { return ctx.Err() } } @@ -759,14 +767,42 @@ func (c *Client) deliverAttachment(ctx context.Context, current frame) bool { if attachment == nil { return false } + return c.deliverToAttachment(ctx, attachment, current) +} + +func (c *Client) deliverToAttachment( + ctx context.Context, + attachment *terminalAttachment, + current frame, +) bool { + delivery := attachmentDelivery{ + frame: current, + accepted: make(chan bool, 1), + } select { - case attachment.frames <- current: - return true + case attachment.frames <- delivery: case <-attachment.done: return false case <-ctx.Done(): return false } + select { + case accepted := <-delivery.accepted: + return accepted + case <-ctx.Done(): + return false + } +} + +func (c *Client) acceptAttachmentDelivery( + attachment *terminalAttachment, + delivery attachmentDelivery, +) bool { + c.stateMu.Lock() + accepted := c.attachment == attachment + c.stateMu.Unlock() + delivery.accepted <- accepted + return accepted } func (c *Client) finishReader(err error) { diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 20cc3cb2..be66c16c 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -789,7 +789,7 @@ func TestAttachRejectsSessionClosedBeforeAttachment(t *testing.T) { func TestRetiredAttachmentCannotAcceptBufferedFrames(t *testing.T) { attachment := &terminalAttachment{ - frames: make(chan frame), + frames: make(chan attachmentDelivery), done: make(chan struct{}), } close(attachment.done) @@ -802,6 +802,96 @@ func TestRetiredAttachmentCannotAcceptBufferedFrames(t *testing.T) { } } +func TestRetiredAttachmentRejectsReceivedDelivery(t *testing.T) { + oldAttachment := &terminalAttachment{ + frames: make(chan attachmentDelivery), + done: make(chan struct{}), + } + replacement := &terminalAttachment{ + frames: make(chan attachmentDelivery), + done: make(chan struct{}), + } + client := &Client{attachment: replacement} + close(oldAttachment.done) + + for range 1_000 { + delivery := attachmentDelivery{ + frame: frame{messageType: messageOutput, payload: []byte("replacement output\n")}, + accepted: make(chan bool, 1), + } + if client.acceptAttachmentDelivery(oldAttachment, delivery) { + t.Fatal("retired attachment accepted a received delivery") + } + if accepted := <-delivery.accepted; accepted { + t.Fatal("retired attachment acknowledged a received delivery") + } + } +} + +func TestStaleAttachmentDeliveryRetriesReplacement(t *testing.T) { + oldAttachment := &terminalAttachment{ + frames: make(chan attachmentDelivery), + done: make(chan struct{}), + } + replacement := &terminalAttachment{ + frames: make(chan attachmentDelivery), + done: make(chan struct{}), + } + client := &Client{ + attachment: oldAttachment, + attachmentReady: make(chan struct{}), + } + + staleCaptured := make(chan struct{}) + releaseStale := make(chan struct{}) + go func() { + delivery := <-oldAttachment.frames + close(staleCaptured) + <-releaseStale + client.acceptAttachmentDelivery(oldAttachment, delivery) + }() + + delivered := make(chan error, 1) + go func() { + delivered <- client.deliverOrQueueOutput(context.Background(), frame{ + messageType: messageOutput, + payload: []byte("replacement output\n"), + }) + }() + <-staleCaptured + + client.stateMu.Lock() + client.attachment = replacement + close(oldAttachment.done) + client.stateMu.Unlock() + + replacementReceived := make(chan frame, 1) + go func() { + delivery := <-replacement.frames + if client.acceptAttachmentDelivery(replacement, delivery) { + replacementReceived <- delivery.frame + } + }() + close(releaseStale) + + select { + case err := <-delivered: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("stale delivery did not retry the replacement attachment") + } + select { + case current := <-replacementReceived: + if got := string(current.payload); got != "replacement output\n" { + t.Fatalf("replacement payload = %q", got) + } + case <-time.After(time.Second): + t.Fatal("replacement attachment did not receive retried output") + } +} + func TestSendInputConfirmedRejectionDoesNotRevokeControl(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) From 1907d7fc23588bf2f3cff2d009eb82cf916ff799 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:16:04 +0200 Subject: [PATCH 173/242] fix(actions): retire stale runner input queues --- src/github-actions-runner.ts | 89 +++++++++++--- tests/github-actions-runner.test.ts | 182 ++++++++++++++++++++++++++-- 2 files changed, 242 insertions(+), 29 deletions(-) diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 3a0e78ef..7d6d1552 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -10,10 +10,13 @@ const runnerInputQueueMaxFrames = 32; const runnerInputQueueMaxAgeMs = 5_000; const runnerInputBacklogError = "GitHub Actions runner input backlog exceeded"; const runnerInputExpiredError = "GitHub Actions runner input expired"; +const runnerInputGenerationError = "GitHub Actions runner generation changed"; type RunnerInputQueue = { bytes: number; frames: number; + generation: string | undefined; + retired: boolean; tail: Promise; }; @@ -35,29 +38,46 @@ export function acceptGitHubActionsRunnerInput( const input = parseGitHubActionsRelayInput(message); if (!input) return Promise.resolve(false); - const queue = runnerInputQueues.get(socket) ?? { - bytes: 0, - frames: 0, - tail: Promise.resolve(), - }; - runnerInputQueues.set(socket, queue); + const existingQueue = runnerInputQueues.get(socket); + if (existingQueue?.retired || socket.readyState !== WebSocket.OPEN) { + return Promise.resolve(true); + } + if (existingQueue && existingQueue.generation !== input.generation) { + existingQueue.retired = true; + sendRunnerInputAcknowledgement( + socket, + input.inputId, + input.generation, + false, + runnerInputGenerationError, + ); + closeRunnerInputSocket(socket, runnerInputGenerationError); + return Promise.resolve(true); + } + + const queue = + existingQueue ?? + ({ + bytes: 0, + frames: 0, + generation: input.generation, + retired: false, + tail: Promise.resolve(), + } satisfies RunnerInputQueue); + if (!existingQueue) runnerInputQueues.set(socket, queue); if ( queue.frames >= runnerInputQueueMaxFrames || queue.bytes + input.payload.byteLength > runnerInputQueueMaxBytes ) { - const { generation, inputId } = input; - const rejected = queue.tail - .catch(() => undefined) - .then(() => { - sendRunnerInputAcknowledgement(socket, inputId, generation, false, runnerInputBacklogError); - }); - queue.tail = rejected; - return rejected - .finally(() => { - deleteIdleRunnerInputQueue(socket, queue, rejected); - }) - .then(() => true); + sendRunnerInputAcknowledgement( + socket, + input.inputId, + input.generation, + false, + runnerInputBacklogError, + ); + return Promise.resolve(true); } const queuedAt = now(); @@ -66,6 +86,7 @@ export function acceptGitHubActionsRunnerInput( const queued = queue.tail .catch(() => undefined) .then(async () => { + if (!isActiveRunnerInputQueue(socket, queue)) return; if (now() - queuedAt >= runnerInputQueueMaxAgeMs) { sendRunnerInputAcknowledgement( socket, @@ -78,9 +99,13 @@ export function acceptGitHubActionsRunnerInput( } try { await writeToPty(input.payload); - sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, true); + if (isActiveRunnerInputQueue(socket, queue)) { + sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, true); + } } catch { - sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, false); + if (isActiveRunnerInputQueue(socket, queue)) { + sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, false); + } } }) .finally(() => { @@ -110,6 +135,30 @@ function sendRunnerInputAcknowledgement( }); } +function closeRunnerInputSocket(socket: GitHubActionsRelaySocket, reason: string): void { + if (socket.readyState !== WebSocket.OPEN) return; + try { + socket.close(1012, reason); + } catch { + // Queue retirement still prevents later writes when the socket cannot be closed cleanly. + } +} + +function isActiveRunnerInputQueue( + socket: GitHubActionsRelaySocket, + queue: RunnerInputQueue, +): boolean { + if ( + runnerInputQueues.get(socket) !== queue || + queue.retired || + socket.readyState !== WebSocket.OPEN + ) { + queue.retired = true; + return false; + } + return true; +} + function deleteIdleRunnerInputQueue( socket: GitHubActionsRelaySocket, queue: RunnerInputQueue, diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 4f943aae..06b7d4a1 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -12,14 +12,21 @@ import { type GitHubActionsRelaySocket, } from "../src/github-actions-runtime.ts"; -function relaySocket(): GitHubActionsRelaySocket & { sent: Array } { +function relaySocket(): GitHubActionsRelaySocket & { + closes: Array<{ code: number | undefined; reason: string | undefined }>; + sent: Array; +} { return { readyState: WebSocket.OPEN, + closes: [], sent: [], send(message) { this.sent.push(message); }, - close() {}, + close(code, reason) { + this.closes.push({ code, reason }); + this.readyState = WebSocket.CLOSED; + }, }; } @@ -90,7 +97,7 @@ test("runner serializes concurrent input writes and acknowledgements per socket" ); }); -test("runner bounds queued frames and rejects overflow in acknowledgement order", async () => { +test("runner bounds queued frames and rejects overflow without extending a stalled tail", async () => { const socket = relaySocket(); let completeFirstWrite!: () => void; const firstWrite = new Promise((resolve) => { @@ -111,6 +118,11 @@ test("runner bounds queued frames and rejects overflow in acknowledgement order" await Promise.resolve(); await Promise.resolve(); assert.deepEqual(writes, [0]); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(socket.sent[0]!), { + inputId: "input-32", + accepted: false, + error: "GitHub Actions runner input backlog exceeded", + }); completeFirstWrite(); assert.deepEqual(await Promise.all(pending), Array(33).fill(true)); assert.deepEqual( @@ -120,19 +132,66 @@ test("runner bounds queued frames and rejects overflow in acknowledgement order" assert.deepEqual( socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), [ - ...Array.from({ length: 32 }, (_, index) => ({ - inputId: `input-${index}`, - accepted: true, - })), { inputId: "input-32", accepted: false, error: "GitHub Actions runner input backlog exceeded", }, + ...Array.from({ length: 32 }, (_, index) => ({ + inputId: `input-${index}`, + accepted: true, + })), ], ); }); +test("runner keeps overflow floods off the accepted input queue", async () => { + const socket = relaySocket(); + let completeFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + completeFirstWrite = resolve; + }); + const writes: number[] = []; + const accepted = Array.from({ length: 32 }, (_, index) => + acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput(`accepted-${index}`, new Uint8Array([index])), + async (payload) => { + writes.push(new Uint8Array(payload)[0]!); + if (index === 0) await firstWrite; + }, + ), + ); + + let settledOverflows = 0; + const overflows = Array.from({ length: 512 }, (_, index) => + acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput(`overflow-${index}`, new Uint8Array([index])), + async () => { + assert.fail("overflow input must not reach the PTY"); + }, + ).then((handled) => { + settledOverflows += 1; + return handled; + }), + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.equal(settledOverflows, 512); + assert.deepEqual(await Promise.all(overflows), Array(512).fill(true)); + assert.deepEqual(writes, [0]); + + completeFirstWrite(); + assert.deepEqual(await Promise.all(accepted), Array(32).fill(true)); + assert.deepEqual( + writes, + Array.from({ length: 32 }, (_, index) => index), + ); + assert.equal(socket.sent.length, 544); +}); + test("runner bounds queued bytes while a PTY write is stalled", async () => { const socket = relaySocket(); let completeFirstWrite!: () => void; @@ -159,22 +218,127 @@ test("runner bounds queued bytes while a PTY write is stalled", async () => { await Promise.resolve(); await Promise.resolve(); assert.equal(writes, 1); + assert.equal(await overflow, true); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(socket.sent[0]!), { + inputId: "input-overflow", + accepted: false, + error: "GitHub Actions runner input backlog exceeded", + }); completeFirstWrite(); - assert.deepEqual(await Promise.all([first, overflow]), [true, true]); + assert.equal(await first, true); assert.equal(writes, 1); assert.deepEqual( socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), [ - { inputId: "input-first", accepted: true }, { inputId: "input-overflow", accepted: false, error: "GitHub Actions runner input backlog exceeded", }, + { inputId: "input-first", accepted: true }, ], ); }); +test("runner does not execute queued input after its socket is replaced", async () => { + const replaced = relaySocket(); + const replacement = relaySocket(); + let completeBlockedWrite!: () => void; + const blockedWrite = new Promise((resolve) => { + completeBlockedWrite = resolve; + }); + const writes: string[] = []; + const first = acceptGitHubActionsRunnerInput( + replaced, + encodeGitHubActionsRelayInput("old-first", "old-first", "old-generation"), + async (payload) => { + writes.push(`${new TextDecoder().decode(payload)}:start`); + await blockedWrite; + writes.push("old-first:end"); + }, + ); + const queued = acceptGitHubActionsRunnerInput( + replaced, + encodeGitHubActionsRelayInput("old-queued", "old-queued", "old-generation"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.deepEqual(writes, ["old-first:start"]); + replaced.close(1012, "runner replaced"); + const current = acceptGitHubActionsRunnerInput( + replacement, + encodeGitHubActionsRelayInput("new-input", "new-input", "new-generation"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + ); + assert.equal(await current, true); + + completeBlockedWrite(); + assert.deepEqual(await Promise.all([first, queued]), [true, true]); + assert.deepEqual(writes, ["old-first:start", "new-input", "old-first:end"]); + assert.deepEqual(replaced.sent, []); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(replacement.sent[0]!), { + inputId: "new-input", + accepted: true, + generation: "new-generation", + }); +}); + +test("runner retires queued input when its relay generation changes", async () => { + const socket = relaySocket(); + let completeBlockedWrite!: () => void; + const blockedWrite = new Promise((resolve) => { + completeBlockedWrite = resolve; + }); + const writes: string[] = []; + const first = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("first", "first", "generation-one"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + await blockedWrite; + }, + ); + const queued = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("queued", "queued", "generation-one"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + ); + + await Promise.resolve(); + await Promise.resolve(); + assert.equal( + await acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("replacement", "replacement", "generation-two"), + async () => { + assert.fail("replacement input must not reach the retired socket"); + }, + ), + true, + ); + assert.deepEqual(socket.closes, [ + { code: 1012, reason: "GitHub Actions runner generation changed" }, + ]); + + completeBlockedWrite(); + assert.deepEqual(await Promise.all([first, queued]), [true, true]); + assert.deepEqual(writes, ["first"]); + assert.deepEqual(parseGitHubActionsRelayInputAcknowledgement(socket.sent[0]!), { + inputId: "replacement", + accepted: false, + error: "GitHub Actions runner generation changed", + generation: "generation-two", + }); +}); + test("runner rejects queued input that outlives the viewer acknowledgement timeout", async () => { const socket = relaySocket(); let completeFirstWrite!: () => void; From 647ee25f60e00425897f9a499053c15f5f70fbff Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:16:39 +0200 Subject: [PATCH 174/242] fix(runtime): make cleanup deletion crash-idempotent --- CHANGELOG.md | 2 +- ...time_adapter_cleanup_deletion_observed.sql | 3 + src/worker/database.ts | 1 + .../runtime-adapter-release-repository.ts | 20 +++++ .../runtime-adapter-release-service.ts | 10 ++- src/worker/runtime-adapter-workspaces.ts | 4 +- src/worker/runtime-application.ts | 7 +- tests/runtime-adapter-release-service.test.ts | 77 +++++++++++++++++++ tests/runtime-adapter-workspaces.test.ts | 27 +++++++ 9 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 migrations/0039_runtime_adapter_cleanup_deletion_observed.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f49a3f..46bd2df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, and retained registration data for superseded runtime workspace cleanup. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. diff --git a/migrations/0039_runtime_adapter_cleanup_deletion_observed.sql b/migrations/0039_runtime_adapter_cleanup_deletion_observed.sql new file mode 100644 index 00000000..e93c3b52 --- /dev/null +++ b/migrations/0039_runtime_adapter_cleanup_deletion_observed.sql @@ -0,0 +1,3 @@ +ALTER TABLE runtime_adapter_workspace_cleanups + ADD COLUMN deletion_observed INTEGER NOT NULL DEFAULT 0 + CHECK (deletion_observed IN (0, 1)); diff --git a/src/worker/database.ts b/src/worker/database.ts index 1fc0c6bd..971807e5 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -229,6 +229,7 @@ export type RuntimeAdapterWorkspaceCleanupTable = { profile: string | null; control_plane: string | null; create_pending: number; + deletion_observed: Generated; message: string; reconcile_error: string | null; attempt_count: Generated; diff --git a/src/worker/provisioning/runtime-adapter-release-repository.ts b/src/worker/provisioning/runtime-adapter-release-repository.ts index 03082562..2441eb00 100644 --- a/src/worker/provisioning/runtime-adapter-release-repository.ts +++ b/src/worker/provisioning/runtime-adapter-release-repository.ts @@ -135,6 +135,25 @@ export async function persistRuntimeAdapterWorkspaceCleanupEvidence( .execute(); } +export async function markRuntimeAdapterWorkspaceCleanupDeletionObserved( + env: RuntimeEnv, + cleanup: RuntimeAdapterWorkspaceCleanup, + now: number, +): Promise { + const row = await database(env) + .updateTable("runtime_adapter_workspace_cleanups") + .set({ + deletion_observed: 1, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where("session_id", "=", cleanup.sessionId) + .where("adapter_workspace_id", "=", cleanup.adapterWorkspaceId) + .where("cleanup_claim", "=", cleanup.claim) + .returning("deletion_observed") + .executeTakeFirst(); + if (!row) throw new Error("runtime adapter cleanup ownership changed"); +} + export async function completeRuntimeAdapterWorkspaceCleanup( env: RuntimeEnv, cleanup: RuntimeAdapterWorkspaceCleanup, @@ -159,6 +178,7 @@ function cleanupClaim(row: RuntimeAdapterWorkspaceCleanupRow): RuntimeAdapterWor } : null, createPending: row.create_pending === 1, + deletionObserved: row.deletion_observed === 1, claim: row.cleanup_claim ?? "", }; } diff --git a/src/worker/provisioning/runtime-adapter-release-service.ts b/src/worker/provisioning/runtime-adapter-release-service.ts index f9305597..5bd2add3 100644 --- a/src/worker/provisioning/runtime-adapter-release-service.ts +++ b/src/worker/provisioning/runtime-adapter-release-service.ts @@ -10,6 +10,7 @@ export type RuntimeAdapterWorkspaceCleanup = { adapterWorkspaceId: string; registration: RuntimeAdapterWorkspaceRegistration | null; createPending: boolean; + deletionObserved: boolean; claim: string; }; @@ -33,13 +34,14 @@ export type RuntimeAdapterReleaseServiceDependencies = { now: number, reconcileError: string | null, ): Promise; + markCleanupDeletionObserved(cleanup: RuntimeAdapterWorkspaceCleanup, now: number): Promise; completeCleanup(cleanup: RuntimeAdapterWorkspaceCleanup): Promise; clearCreatePending(sessionId: string, adapterWorkspaceId: string): Promise; stopWorkspace( sessionId: string, adapterWorkspaceId: string, registration: RuntimeAdapterWorkspaceRegistration | null, - createPending: boolean, + retryMissing: boolean, ): Promise; confirmRelease( sessionId: string, @@ -92,7 +94,8 @@ export class RuntimeAdapterReleaseService { cleanup: RuntimeAdapterWorkspaceCleanup, now: number, ): Promise { - const { sessionId, adapterWorkspaceId, registration, createPending } = cleanup; + const { sessionId, adapterWorkspaceId, registration, createPending, deletionObserved } = + cleanup; try { if (!createPending) { await this.dependencies.clearCreatePending(sessionId, adapterWorkspaceId); @@ -101,9 +104,10 @@ export class RuntimeAdapterReleaseService { sessionId, adapterWorkspaceId, registration, - createPending, + createPending && !deletionObserved, ); if (release.status === "stopped") { + await this.dependencies.markCleanupDeletionObserved(cleanup, now); await this.dependencies.confirmRelease(sessionId, adapterWorkspaceId, now, release.message); await this.dependencies.completeCleanup(cleanup); return; diff --git a/src/worker/runtime-adapter-workspaces.ts b/src/worker/runtime-adapter-workspaces.ts index eb674a6e..54918133 100644 --- a/src/worker/runtime-adapter-workspaces.ts +++ b/src/worker/runtime-adapter-workspaces.ts @@ -188,13 +188,13 @@ export class RuntimeAdapterWorkspaceLifecycle { sessionId: string, adapterWorkspaceId: string, retainedRegistration?: RuntimeAdapterWorkspaceRegistration | null, - retainedCreatePending?: boolean, + retryMissing?: boolean, ): Promise { const supersededCleanup = retainedRegistration !== undefined; const registration = retainedRegistration ? { adapter_control_plane: retainedRegistration.controlPlane, - adapter_create_pending: retainedCreatePending ? 1 : 0, + adapter_create_pending: retryMissing ? 1 : 0, profile: retainedRegistration.profile, } : await database(this.env) diff --git a/src/worker/runtime-application.ts b/src/worker/runtime-application.ts index 50b6f27b..0bad2787 100644 --- a/src/worker/runtime-application.ts +++ b/src/worker/runtime-application.ts @@ -7,6 +7,7 @@ import { claimRuntimeAdapterWorkspaceCleanup, claimRuntimeAdapterWorkspaceCleanupBatch, completeRuntimeAdapterWorkspaceCleanup, + markRuntimeAdapterWorkspaceCleanupDeletionObserved, persistRuntimeAdapterWorkspaceCleanupEvidence, stageRuntimeAdapterWorkspaceCleanup, } from "./provisioning/runtime-adapter-release-repository.ts"; @@ -206,15 +207,17 @@ export class RuntimeApplication { now, reconcileError, ), + markCleanupDeletionObserved: (cleanup, now) => + markRuntimeAdapterWorkspaceCleanupDeletionObserved(this.env, cleanup, now), completeCleanup: (cleanup) => completeRuntimeAdapterWorkspaceCleanup(this.env, cleanup), clearCreatePending: (sessionId, adapterWorkspaceId) => clearRuntimeAdapterCreatePending(this.env, sessionId, adapterWorkspaceId), - stopWorkspace: (sessionId, adapterWorkspaceId, registration, createPending) => + stopWorkspace: (sessionId, adapterWorkspaceId, registration, retryMissing) => this.workspaceLifecycle().stopForSession( sessionId, adapterWorkspaceId, registration, - createPending, + retryMissing, ), confirmRelease: (sessionId, adapterWorkspaceId, now, message) => confirmRuntimeAdapterRelease(this.env, sessionId, adapterWorkspaceId, now, message), diff --git a/tests/runtime-adapter-release-service.test.ts b/tests/runtime-adapter-release-service.test.ts index 985293a0..08723383 100644 --- a/tests/runtime-adapter-release-service.test.ts +++ b/tests/runtime-adapter-release-service.test.ts @@ -8,6 +8,7 @@ import { claimRuntimeAdapterWorkspaceCleanup, claimRuntimeAdapterWorkspaceCleanupBatch, completeRuntimeAdapterWorkspaceCleanup, + markRuntimeAdapterWorkspaceCleanupDeletionObserved, persistRuntimeAdapterWorkspaceCleanupEvidence, stageRuntimeAdapterWorkspaceCleanup, } from "../src/worker/provisioning/runtime-adapter-release-repository.ts"; @@ -46,6 +47,7 @@ function releaseDependencies( adapterWorkspaceId: input.adapterWorkspaceId, registration: input.registration, createPending: input.createPending, + deletionObserved: false, claim: "claim-1", }; }, @@ -56,6 +58,7 @@ function releaseDependencies( return []; }, async persistCleanupEvidence() {}, + async markCleanupDeletionObserved() {}, async completeCleanup() {}, async clearCreatePending() {}, async stopWorkspace() { @@ -115,6 +118,7 @@ test("superseded release clears the create marker before stopping and confirming adapterWorkspaceId, registration, createPending: false, + deletionObserved: false, claim: "claim-1", }; }, @@ -229,6 +233,7 @@ test("superseded cleanup survives ownership loss and retries only the old worksp adapterWorkspaceId: input.adapterWorkspaceId, registration: input.registration, createPending: input.createPending, + deletionObserved: false, claim: "claim-1", }); }, @@ -297,6 +302,7 @@ test("superseded provider failures remain independently retryable", async () => adapterWorkspaceId: input.adapterWorkspaceId, registration: input.registration, createPending: input.createPending, + deletionObserved: false, claim: "claim-1", }); }, @@ -337,6 +343,68 @@ test("superseded provider failures remain independently retryable", async () => assert.equal(cleanupRows.length, 0); }); +test("observed create-pending deletion survives completion persistence failure", async () => { + let cleanup: RuntimeAdapterWorkspaceCleanup | null = null; + const retryMissing: boolean[] = []; + let completionAttempts = 0; + const service = new RuntimeAdapterReleaseService( + releaseDependencies({ + async stageCleanup(input) { + cleanup = { + sessionId: input.sessionId, + adapterWorkspaceId: input.adapterWorkspaceId, + registration: input.registration, + createPending: input.createPending, + deletionObserved: false, + claim: "claim-1", + }; + }, + async claimCleanup() { + return cleanup; + }, + async claimPendingCleanups() { + return cleanup ? [{ ...cleanup, claim: "claim-2" }] : []; + }, + async stopWorkspace(_sessionId, _adapterWorkspaceId, _registration, retry) { + retryMissing.push(retry); + return { + status: "stopped", + message: retry + ? "runtime adapter workspace released" + : "runtime adapter workspace already gone", + }; + }, + async markCleanupDeletionObserved(current) { + cleanup = { ...current, deletionObserved: true }; + }, + async completeCleanup() { + completionAttempts += 1; + if (completionAttempts === 1) throw new Error("completion persistence unavailable"); + cleanup = null; + }, + providerError(error) { + assert.ok(error instanceof Error); + return error.message; + }, + }), + ); + + await service.stopSuperseded({ + sessionId: "IS-101", + adapterWorkspaceId: "fleet-a-is-101-old", + registration, + createPending: true, + now: 200, + }); + assert.equal(cleanup?.deletionObserved, true); + + await service.retryPending(15_200); + + assert.deepEqual(retryMissing, [true, false]); + assert.equal(completionAttempts, 2); + assert.equal(cleanup, null); +}); + test("runtime adapter cleanup storage is independent and claim fenced", async () => { const sqlite = new DatabaseSync(":memory:"); sqlite.exec( @@ -345,6 +413,12 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () "utf8", ), ); + sqlite.exec( + readFileSync( + new URL("../migrations/0039_runtime_adapter_cleanup_deletion_observed.sql", import.meta.url), + "utf8", + ), + ); const env = sqliteRuntimeEnv(sqlite); await stageRuntimeAdapterWorkspaceCleanup(env, { sessionId: "IS-101", @@ -362,9 +436,11 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () ); assert.ok(claimed); assert.equal(claimed.createPending, true); + assert.equal(claimed.deletionObserved, false); assert.deepEqual(claimed.registration, registration); assert.equal((await claimRuntimeAdapterWorkspaceCleanupBatch(env, 200, 3)).length, 0); + await markRuntimeAdapterWorkspaceCleanupDeletionObserved(env, claimed, 201); await persistRuntimeAdapterWorkspaceCleanupEvidence( env, claimed, @@ -375,6 +451,7 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () assert.equal((await claimRuntimeAdapterWorkspaceCleanupBatch(env, 15_199, 3)).length, 0); const retry = await claimRuntimeAdapterWorkspaceCleanupBatch(env, 15_200, 3); assert.equal(retry.length, 1); + assert.equal(retry[0].deletionObserved, true); await completeRuntimeAdapterWorkspaceCleanup(env, retry[0]); assert.equal( sqlite.prepare("SELECT COUNT(*) AS count FROM runtime_adapter_workspace_cleanups").get()?.count, diff --git a/tests/runtime-adapter-workspaces.test.ts b/tests/runtime-adapter-workspaces.test.ts index 5333af7f..2104dcf8 100644 --- a/tests/runtime-adapter-workspaces.test.ts +++ b/tests/runtime-adapter-workspaces.test.ts @@ -448,6 +448,33 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi ]); }); +test("superseded cleanup accepts missing workspaces after deletion was observed", async () => { + const service = new RuntimeAdapterWorkspaceLifecycle( + runtimeEnv(), + dependencies({ + async fetch() { + return Response.json({ message: "workspace not found" }, { status: 404 }); + }, + }), + ); + + assert.deepEqual( + await service.stopForSession( + "IS-42", + "workspace-superseded", + { + profile: "default", + controlPlane: "https://adapter.example.test/", + }, + false, + ), + { + status: "stopped", + message: "workspace not found", + }, + ); +}); + test("session-bound stop redacts provider credentials from failures", async () => { const env = runtimeEnv(() => [ { From 95b73909bff6c302ead70fe06fd4a93eb808509b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:16:39 +0200 Subject: [PATCH 175/242] fix(terminal): capture relay replacement during authorization --- src/worker/terminal-hub.ts | 56 +++++++++++++++++++++++++++++++++----- tests/terminal-hub.test.ts | 26 +++++++++++++++--- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index f79ae393..5ffcecc8 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -33,6 +33,7 @@ const terminalFrameLimits = { maxFrameBytes: terminalMaxFrameBytes }; const terminalInputQueueMaxBytes = terminalMaxFrameBytes; const terminalInputQueueMaxFrames = 32; const terminalInputAcknowledgementTimeoutMs = 5_000; +const terminalPreAuthorizationRelayEventMax = 32; type PendingTerminalInputAcknowledgement = { inputId: string; @@ -500,7 +501,46 @@ export class TerminalHub { return; } const upstream = upstreamConnection.socket; - if (!(await canView())) { + const inputAcknowledgements = + upstreamConnection.inputAcknowledgements ?? session.runtime === githubActionsRuntime; + const inputGenerations = upstreamConnection.inputGenerations ?? false; + let runnerGeneration: number | string = inputGenerations + ? (upstreamConnection.initialRunnerGeneration ?? "none") + : 0; + const bufferedRelayEvents: Array< + NonNullable> + > = []; + let captureRelayEvents = true; + upstream.addEventListener("message", (event) => { + if (!captureRelayEvents || !inputAcknowledgements) return; + const relayEvent = parseSynchronousGitHubActionsRelayEvent(event.data); + if (!relayEvent) return; + if (bufferedRelayEvents.length === terminalPreAuthorizationRelayEventMax) { + bufferedRelayEvents.shift(); + } + bufferedRelayEvents.push(relayEvent); + if (relayEvent.generation) { + if (relayEvent.type === "runner_disconnected") { + if (runnerGeneration === relayEvent.generation) runnerGeneration = "none"; + } else { + runnerGeneration = relayEvent.generation; + } + } else if (relayEvent.type === "runner_connected" && !inputGenerations) { + runnerGeneration = (runnerGeneration as number) + 1; + } + }); + let canViewNow: boolean; + try { + canViewNow = await canView(); + } catch (error) { + captureRelayEvents = false; + if (upstream.readyState < WebSocket.CLOSING) { + upstream.close(1011, "view authorization failed"); + } + throw error; + } + if (!canViewNow) { + captureRelayEvents = false; if (upstream.readyState < WebSocket.CLOSING) upstream.close(1008, "share revoked"); sendTerminalJson(client, TerminalMessageType.Error, id, { error: "interactive session not found", @@ -508,6 +548,7 @@ export class TerminalHub { return; } if (!isHubOpen() || client.readyState !== WebSocket.OPEN) { + captureRelayEvents = false; if (upstream.readyState < WebSocket.CLOSING) upstream.close(1000, "client closed"); return; } @@ -523,18 +564,15 @@ export class TerminalHub { viewCheck, cols, rows, - inputAcknowledgements: - upstreamConnection.inputAcknowledgements ?? session.runtime === githubActionsRuntime, + inputAcknowledgements, inputQueue: Promise.resolve(), inputQueueBytes: 0, inputQueueFrames: 0, inputQueueRejections: 0, inputQueueRejectionScheduled: false, - inputGenerations: upstreamConnection.inputGenerations ?? false, + inputGenerations, pendingInputAcknowledgements: new Map(), - runnerGeneration: upstreamConnection.inputGenerations - ? (upstreamConnection.initialRunnerGeneration ?? "none") - : 0, + runnerGeneration, outputAcknowledgements: outputAcknowledgements && upstreamConnection.outputAcknowledgements, outputAcknowledgementBytes: 0, }; @@ -564,6 +602,7 @@ export class TerminalHub { }); }, 5000); activeSubscription.viewCheck = viewCheck; + captureRelayEvents = false; subscriptions.set(id, activeSubscription); let outputQueue = Promise.resolve(); sendTerminalJson(client, TerminalMessageType.Event, id, { @@ -699,6 +738,9 @@ export class TerminalHub { } }); }); + for (const relayEvent of bufferedRelayEvents) { + sendTerminalJson(client, TerminalMessageType.Event, id, relayEvent); + } upstream.addEventListener("close", (event) => { completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index c9552429..97bcf3bb 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -1718,7 +1718,7 @@ test("relay generations bind interleaved replacement input before lifecycle proc server.emit("close"); }); -test("viewer carries its initial runner generation through authorization setup", async () => { +test("viewer captures runner replacement during authorization setup", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -1755,11 +1755,29 @@ test("viewer carries its initial runner generation through authorization setup", }), }); await flushQueues(); - emitRelayEvent(upstream, "runner_connected", "generation-initial"); + emitRelayEvent(upstream, "runner_connected", "generation-replacement"); + upstream.emit("message", { + data: encodeGitHubActionsRelayOutput("output-before-authorization"), + }); releaseView(true); await flushQueues(); await flushQueues(); + const messages = server.sent.map((payload) => frame(payload)); + assert.deepEqual( + messages + .filter((message) => message.type === TerminalMessageType.Event) + .map( + (message) => decodeJsonPayload(message.payload) as { type?: string; generation?: string }, + ) + .filter((message) => message.type === "runner_connected"), + [{ type: "runner_connected", generation: "generation-replacement" }], + ); + assert.equal( + messages.some((message) => message.type === TerminalMessageType.Output), + false, + ); + server.emit("message", { data: encodeTerminalFrame({ type: TerminalMessageType.Input, @@ -1770,8 +1788,8 @@ test("viewer carries its initial runner generation through authorization setup", await waitForInputPayloads(); const input = relayInput(upstream.sent.at(-1)!); - assert.equal(input.generation, "generation-initial"); - emitRelayAcknowledgement(upstream, input.inputId, true, "generation-initial"); + assert.equal(input.generation, "generation-replacement"); + emitRelayAcknowledgement(upstream, input.inputId, true, "generation-replacement"); await flushQueues(); await flushQueues(); From a89ba66f88ad2d7797053a4ab07dbc5a76e12a70 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:16:53 +0200 Subject: [PATCH 176/242] fix(actions): negotiate runner protocol explicitly --- README.md | 21 ++++++++------ docs/api.md | 17 +++++++---- docs/architecture.md | 2 +- docs/runs.md | 2 +- docs/spec.md | 11 ++++---- src/github-actions-runtime.ts | 13 +++++++++ src/worker/github-actions-application.ts | 13 ++++++++- src/worker/session-control-do.ts | 18 ++++++++++-- tests/application-architecture.test.ts | 5 ++-- tests/github-actions-docs.test.ts | 5 ++-- tests/github-actions-event-auth.test.ts | 36 ++++++++++++++++++++++++ tests/github-actions-runtime.test.ts | 11 ++++++++ 12 files changed, 126 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a71755fd..aace4a72 100644 --- a/README.md +++ b/README.md @@ -94,22 +94,25 @@ Content-Type: application/json {"workKey":"openclaw/crabfleet:pr:42","workKind":"pr_repair","repo":"openclaw/crabfleet","branch":"fix/pr-42","owner":"operator@example.test","sourceUrl":"https://github.com/openclaw/crabfleet/pull/42","runUrl":"https://github.com/openclaw/crabfleet/actions/runs/123","purpose":"repair PR 42","summary":"starting repair"} ``` -The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and can be opened unchanged as the legacy raw duplex byte stream. New runners opt into framed input, output, and acknowledgements by adding the exact query parameter shown below before opening the socket: +The response contains `{session, agentToken, runnerPtyUrl, browserUrl}`. New registrations and resumes require `owner` to resolve to one active Crabfleet user; resumes must prove the same stable owner subject already recorded on the `workKey`. The stable subject owns browser visibility while the OpenClaw service retains lifecycle authority for its session. `runnerPtyUrl` includes the rotated session-scoped query credential and can be opened unchanged as the legacy raw duplex byte stream. New runners offer the framed protocol as a WebSocket subprotocol: ```js -const framedRunnerPtyUrl = new URL(runnerPtyUrl); -framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v2"); -const terminal = new WebSocket(framedRunnerPtyUrl); +const terminal = new WebSocket(runnerPtyUrl, "cfr1-framed-io-v2"); terminal.binaryType = "arraybuffer"; +terminal.addEventListener("open", () => { + const framed = terminal.protocol === "cfr1-framed-io-v2"; + // Use CFR1 only when framed is true; otherwise retain raw compatibility. +}); ``` Existing runners retain raw input and output by opening the returned URL unchanged. A new runner opts into correlated binary `CFR1` input, output, and -acknowledgement frames by adding the exact -`runnerProtocol=cfr1-framed-io-v2` query before opening the socket. The relay -selects that mode before accepting the connection and fences runner input with -the relay-owned connection generation. Framed runners acknowledge only after -their PTY accepts the input. The complete byte-safe encoder, decoder, and Node PTY runner are in +acknowledgement frames only when the relay selects the +`cfr1-framed-io-v2` WebSocket subprotocol in the upgrade response. Relays that +ignore the offer leave `WebSocket.protocol` empty, so new runners retain raw +compatibility. The relay fences negotiated runner input with its connection +generation. Framed runners acknowledge only after their PTY accepts the input. +The complete byte-safe encoder, decoder, and Node PTY runner are in [`docs/github-actions-sessions.md`](docs/github-actions-sessions.md#runner-pty). The runner reports heartbeat and durable progress with bearer `agentToken` to `POST /api/agent/interactive-sessions/:id/work-state`. Terminal states are `completed`, `blocked`, `failed`, and `canceled`; active work uses `registered` or `running` plus a specific `phase`. diff --git a/docs/api.md b/docs/api.md index b11e2922..998f43ac 100644 --- a/docs/api.md +++ b/docs/api.md @@ -598,23 +598,30 @@ Response: } ``` -Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output by opening it unchanged; new runners add the exact `runnerProtocol=cfr1-framed-io-v2` query to opt into the generation-fenced contract below. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. +Every new registration and every resume requires `owner`; it must resolve to exactly one active Crabfleet user by login, email, or stable subject. Existing work keys resume only when the supplied owner resolves to the same stable owner subject already stored on the work key. Ownerless resumes fail closed before token rotation, and a work key cannot transfer to a different stable owner. `runnerPtyUrl` can be opened with Node's global `WebSocket` without custom headers. Existing runners retain raw input/output by opening it unchanged; new runners offer `cfr1-framed-io-v2` as a WebSocket subprotocol and enter the generation-fenced contract below only when the relay selects it. The query credential is session-scoped, rotates on registration, is stored only as a hash, and is not exposed through viewer/session APIs. ### GET /api/agent/interactive-sessions/:id/runner-pty WebSocket endpoint for the outbound GitHub Actions runner. Authentication uses the scoped `agentToken` query parameter embedded in `runnerPtyUrl`. One runner is current; a reconnect replaces the previous runner while browser viewers remain attached. Opening the returned URL unchanged selects legacy raw input and output. Adding -the exact `runnerProtocol=cfr1-framed-io-v2` query selects generation-fenced -input, output, acknowledgements, and relay control traffic. `SessionControlDO` -stores the mode and a relay-owned runner generation on the server socket before -accepting it. Viewer framing is negotiated independently: v2 viewers receive +`cfr1-framed-io-v2` to the `WebSocket` constructor's protocol list offers +generation-fenced input, output, acknowledgements, and relay control traffic. +The runner switches formats only when `WebSocket.protocol` confirms that exact +selection. An older relay that ignores the offer therefore remains a raw +connection. `SessionControlDO` stores the selected mode and a relay-owned runner +generation on the server socket before accepting it. Viewer framing is +negotiated independently: v2 viewers receive `CFR1` output and generation-bearing control frames, while unnegotiated viewers retain raw output and legacy JSON notices. The relay translates the earlier `cfr1-framed-io-v1` format and raw sockets at each boundary during rolling upgrades. Arbitrary raw PTY bytes cannot be consumed as control traffic by framed viewers. +The `runnerProtocol` query remains accepted for compatibility with already +deployed query-aware runners. New runners must use subprotocol negotiation so +they do not switch formats against a relay that did not explicitly confirm v2. + Each `CFR1` frame occupies one binary WebSocket message and starts with: | Offset | Size | Value | diff --git a/docs/architecture.md b/docs/architecture.md index 85d959a2..55201059 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ D1 is canonical for product metadata: ### Durable Objects - `Sandbox` runs first-party Cloudflare Sandbox workspaces. -- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; exact v2 connection-query opt-in selects relay-generation-fenced binary `CFR1` input, lifecycle, and acknowledgement frames before socket acceptance. Viewer framing is negotiated independently: opted-in viewers receive `CFR1` terminal, lifecycle, and acknowledgement frames, legacy viewers retain raw terminal output and JSON control-message fallbacks, and the relay translates v1 framed peers during rolling upgrades. +- `SessionControlDO` stores generation-fenced Sandbox credential/checkpoint state and relays one current GitHub Actions runner to multiple viewers. Existing runners retain raw input/output at the runner boundary; new runners use relay-generation-fenced binary `CFR1` input, lifecycle, and acknowledgement frames only after the upgrade response selects the offered v2 WebSocket subprotocol. Ignored offers remain raw-compatible. Viewer framing is negotiated independently: opted-in viewers receive `CFR1` terminal, lifecycle, and acknowledgement frames, legacy viewers retain raw terminal output and JSON control-message fallbacks, and the relay translates v1 framed peers during rolling upgrades. There is no `BoardDO` or `RunDO`. General Board/Fleet state is D1 plus REST polling. diff --git a/docs/runs.md b/docs/runs.md index 62faf0b7..1a104345 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -122,7 +122,7 @@ GitHub Actions PTY contract: - OpenClaw registers or resumes work through `POST /api/openclaw/action-sessions`. - The returned `runnerPtyUrl` is a `wss:` URL with a rotated session-scoped query credential. Node's global `WebSocket` can open it without custom headers. - Legacy runners open the returned URL unchanged and retain raw input/output with relay-level delivery reporting. -- Generation-fenced runners add the exact `runnerProtocol=cfr1-framed-io-v2` query before opening the socket. Viewer input and acknowledgements carry the relay-owned runner generation, stale input is rejected before forwarding, and the runner returns the matching generation and correlation ID only after its PTY accepts the write. The relay continues translating v1 framed and raw sockets during rolling upgrades. +- Generation-fenced runners offer `cfr1-framed-io-v2` as a WebSocket subprotocol and use `CFR1` only when the upgrade response selects it. A relay that ignores the offer leaves `WebSocket.protocol` empty, so the runner remains raw-compatible during rolling upgrades. Viewer input and acknowledgements carry the relay-owned runner generation, stale input is rejected before forwarding, and the runner returns the matching generation and correlation ID only after its PTY accepts the write. The relay continues translating v1 framed and raw sockets. - `SessionControlDO` allows one current runner and multiple viewers. A new runner replaces the previous runner; viewers remain connected and receive runner lifecycle events. - Authorized browser viewers attach through the existing `/api/terminal/ws` hub. Service and agent credentials are never included in viewer responses. - The runner updates `state`, `phase`, `summary`, Codex thread/turn IDs, and heartbeat through the agent work-state endpoint. `completed`, `blocked`, `failed`, and `canceled` are terminal. diff --git a/docs/spec.md b/docs/spec.md index 52170db7..238b9f23 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -178,7 +178,7 @@ Crabfleet owns: - session identity and metadata; - rotating scoped agent token; -- outbound runner relay through `SessionControlDO`, preserving raw and v1 framed traffic while exact v2 connection-query opt-in selects relay-generation-fenced binary `CFR1` input, acknowledgement, and lifecycle frames plus framed output; +- outbound runner relay through `SessionControlDO`, preserving raw and v1 framed traffic while an explicitly selected v2 WebSocket subprotocol enables relay-generation-fenced binary `CFR1` input, acknowledgement, and lifecycle frames plus framed output; - browser terminal steering; - work-state heartbeats; - event and transcript finalization. @@ -186,10 +186,11 @@ Crabfleet owns: The Action remains the execution host and mutation authority. A framed runner acknowledges viewer input only after its PTY accepts the correlated write; relay queueing is not acceptance. Legacy runners keep raw input/output -and relay-level delivery reporting. The exact protocol query selects the mode -before the runner socket is accepted, so there is no in-band handshake or -mode-transition race. Ending the Crabfleet session does not cancel the workflow -run. +and relay-level delivery reporting. New runners offer the v2 WebSocket +subprotocol and switch formats only when the relay selects it in the upgrade +response. Relays that ignore the offer therefore remain raw-compatible, with no +in-band handshake or mode-transition race. Ending the Crabfleet session does +not cancel the workflow run. ## Session Lifecycle diff --git a/src/github-actions-runtime.ts b/src/github-actions-runtime.ts index 6bce6496..fdcaa8bf 100644 --- a/src/github-actions-runtime.ts +++ b/src/github-actions-runtime.ts @@ -33,6 +33,7 @@ export type GitHubActionsRelayInput = { export const githubActionsFramedRunnerCapability = "cfr1-framed-io-v1"; export const githubActionsGenerationFencedCapability = "cfr1-framed-io-v2"; +export const githubActionsRunnerProtocolHeader = "sec-websocket-protocol"; export const githubActionsRunnerProtocolQuery = "runnerProtocol"; export const githubActionsViewerProtocolQuery = "viewerProtocol"; export const githubActionsViewerProtocolHeader = "x-crabfleet-viewer-protocol"; @@ -434,6 +435,18 @@ export function parseGitHubActionsRunnerProtocol( return isGitHubActionsRelayProtocol(value) ? value : null; } +export function parseGitHubActionsRunnerProtocolOffer( + value: string | null, +): typeof githubActionsGenerationFencedCapability | null { + if (!value) return null; + return value + .split(",") + .map((protocol) => protocol.trim()) + .includes(githubActionsGenerationFencedCapability) + ? githubActionsGenerationFencedCapability + : null; +} + export function parseGitHubActionsViewerProtocol( value: string | null, ): GitHubActionsViewerProtocol | null { diff --git a/src/worker/github-actions-application.ts b/src/worker/github-actions-application.ts index f81f7557..ac8e27ce 100644 --- a/src/worker/github-actions-application.ts +++ b/src/worker/github-actions-application.ts @@ -1,7 +1,9 @@ import type { GitHubActionsSessionRegistrationInput } from "./github-actions-session-registration.ts"; import { + githubActionsRunnerProtocolHeader, githubActionsRunnerProtocolQuery, parseGitHubActionsRunnerProtocol, + parseGitHubActionsRunnerProtocolOffer, } from "../github-actions-runtime.ts"; import { AdminRepository } from "./admin-repository.ts"; import { @@ -174,7 +176,7 @@ export class GitHubActionsApplication { }; await new GitHubActionsRunnerConnectionService(store).connect(session); return stub.fetch(gitHubActionsRelayRunnerUrl(request), { - headers: { upgrade: "websocket" }, + headers: gitHubActionsRelayRunnerHeaders(request), }); } @@ -226,6 +228,15 @@ export function gitHubActionsRelayRunnerUrl(request: Request): string { return relayUrl.toString(); } +export function gitHubActionsRelayRunnerHeaders(request: Request): Headers { + const headers = new Headers({ upgrade: "websocket" }); + const offeredProtocol = parseGitHubActionsRunnerProtocolOffer( + request.headers.get(githubActionsRunnerProtocolHeader), + ); + if (offeredProtocol) headers.set(githubActionsRunnerProtocolHeader, offeredProtocol); + return headers; +} + function isConstraintError(error: unknown): boolean { return error instanceof Error && /constraint|unique/i.test(error.message); } diff --git a/src/worker/session-control-do.ts b/src/worker/session-control-do.ts index 2cfc2a14..03cf5d35 100644 --- a/src/worker/session-control-do.ts +++ b/src/worker/session-control-do.ts @@ -14,12 +14,14 @@ import { gitHubActionsRelayUsesGenerations, githubActionsLegacyRelayGeneration, githubActionsRelayRole, + githubActionsRunnerProtocolHeader, githubActionsRunnerProtocolQuery, githubActionsViewerGenerationHeader, githubActionsViewerProtocolHeader, githubActionsViewerProtocolQuery, notifyGitHubActionsViewers, parseGitHubActionsRunnerProtocol, + parseGitHubActionsRunnerProtocolOffer, parseGitHubActionsViewerProtocol, relayGitHubActionsWebSocketMessage, replaceGitHubActionsRunner, @@ -64,9 +66,16 @@ export class SessionControlDO extends DurableObject { request.method === "GET" && url.pathname === "/api/session-control/github-actions/runner" ) { + const offeredProtocol = parseGitHubActionsRunnerProtocolOffer( + request.headers.get(githubActionsRunnerProtocolHeader), + ); return this.openGitHubActionsRelay( "runner", - parseGitHubActionsRunnerProtocol(url.searchParams.get(githubActionsRunnerProtocolQuery)), + offeredProtocol ?? + parseGitHubActionsRunnerProtocol( + url.searchParams.get(githubActionsRunnerProtocolQuery), + ), + offeredProtocol, ); } @@ -246,6 +255,7 @@ export class SessionControlDO extends DurableObject { private openGitHubActionsRelay( role: "runner" | "viewer", protocol: GitHubActionsRelayProtocol | null = null, + confirmedRunnerProtocol: GitHubActionsRelayProtocol | null = null, ): Response { const pair = new WebSocketPair(); const client = pair[0]; @@ -279,7 +289,11 @@ export class SessionControlDO extends DurableObject { } } const responseInit: ResponseInit = { status: 101, webSocket: client }; - if (role === "viewer" && protocol) { + if (role === "runner" && confirmedRunnerProtocol) { + responseInit.headers = { + [githubActionsRunnerProtocolHeader]: confirmedRunnerProtocol, + }; + } else if (role === "viewer" && protocol) { responseInit.headers = { [githubActionsViewerProtocolHeader]: protocol, ...(initialRunnerGeneration diff --git a/tests/application-architecture.test.ts b/tests/application-architecture.test.ts index 251c862b..2181449c 100644 --- a/tests/application-architecture.test.ts +++ b/tests/application-architecture.test.ts @@ -59,13 +59,13 @@ test("worker entrypoint delegates OpenClaw and GitHub Actions composition", asyn assert.match(githubActions, /new GitHubActionsWorkStateService\(/); }); -test("GitHub Actions runner protocol is attached before the relay socket is accepted", async () => { +test("GitHub Actions runner protocol is confirmed before the relay socket is accepted", async () => { const [application, relay] = await Promise.all([ readFile(new URL("../src/worker/github-actions-application.ts", import.meta.url), "utf8"), readFile(new URL("../src/worker/session-control-do.ts", import.meta.url), "utf8"), ]); - assert.match(application, /stub\.fetch\(gitHubActionsRelayRunnerUrl\(request\)/); + assert.match(application, /headers: gitHubActionsRelayRunnerHeaders\(request\)/); const attach = relay.indexOf("attachGitHubActionsRunnerProtocol(server, protocol, generation)"); const accept = relay.indexOf( 'this.ctx.acceptWebSocket(server, ["github-actions-runner"])', @@ -73,6 +73,7 @@ test("GitHub Actions runner protocol is attached before the relay socket is acce ); assert.notEqual(attach, -1); assert.ok(accept > attach); + assert.match(relay, /\[githubActionsRunnerProtocolHeader\]: confirmedRunnerProtocol/); }); test("GitHub Actions viewer protocol is requested and attached before relay acceptance", async () => { diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index d124f102..15ee6d1e 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -9,8 +9,9 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async ]); assert.match(readme, /complete byte-safe encoder, decoder, and Node PTY runner/); - assert.match(readme, /runnerProtocol", "cfr1-framed-io-v2"/); - assert.match(readme, /`runnerProtocol=cfr1-framed-io-v2` query/); + assert.match(readme, /new WebSocket\(runnerPtyUrl, "cfr1-framed-io-v2"\)/); + assert.match(readme, /terminal\.protocol === "cfr1-framed-io-v2"/); + assert.match(readme, /ignore the offer leave `WebSocket\.protocol` empty/); assert.doesNotMatch(readme, /New runners opt into[\s\S]*cfr1-framed-io-v1/); assert.doesNotMatch(readme, /encodeCfr1Output|decodeCfr1Input|encodeCfr1Ack/); assert.match(guide, /let pendingInputs = \[\]/); diff --git a/tests/github-actions-event-auth.test.ts b/tests/github-actions-event-auth.test.ts index a764a129..1e63dcd6 100644 --- a/tests/github-actions-event-auth.test.ts +++ b/tests/github-actions-event-auth.test.ts @@ -5,9 +5,14 @@ import { sha256 } from "../src/worker/crypto.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import { GitHubActionsApplication, + gitHubActionsRelayRunnerHeaders, gitHubActionsRelayRunnerUrl, structuredEventRequestMaxBytes, } from "../src/worker/github-actions-application.ts"; +import { + githubActionsGenerationFencedCapability, + githubActionsRunnerProtocolHeader, +} from "../src/github-actions-runtime.ts"; import { terminalAgentEventGraceMs } from "../src/worker/session-agent-auth.ts"; import { handleServiceSessionRoute, @@ -163,6 +168,37 @@ test("GitHub Actions application propagates only the exact runner protocol opt-i gitHubActionsRelayRunnerUrl(new Request(`${base}&runnerProtocol=cfr1-framed-io-v3`)), "https://crabfleet.internal/api/session-control/github-actions/runner", ); + + const offered = gitHubActionsRelayRunnerHeaders( + new Request(base, { + headers: { + [githubActionsRunnerProtocolHeader]: `ignored, ${githubActionsGenerationFencedCapability}`, + }, + }), + ); + assert.equal( + gitHubActionsRelayRunnerUrl( + new Request(base, { + headers: { + [githubActionsRunnerProtocolHeader]: githubActionsGenerationFencedCapability, + }, + }), + ), + "https://crabfleet.internal/api/session-control/github-actions/runner", + ); + assert.equal(offered.get("upgrade"), "websocket"); + assert.equal( + offered.get(githubActionsRunnerProtocolHeader), + githubActionsGenerationFencedCapability, + ); + assert.equal( + gitHubActionsRelayRunnerHeaders( + new Request(base, { + headers: { [githubActionsRunnerProtocolHeader]: "cfr1-framed-io-v1" }, + }), + ).get(githubActionsRunnerProtocolHeader), + null, + ); }); test("agent event endpoint rejects a wrong-session token before persistence", async () => { diff --git a/tests/github-actions-runtime.test.ts b/tests/github-actions-runtime.test.ts index 196f3e7a..3fea9a1a 100644 --- a/tests/github-actions-runtime.test.ts +++ b/tests/github-actions-runtime.test.ts @@ -16,6 +16,7 @@ import { githubActionsFramedRunnerCapability, githubActionsGenerationFencedCapability, githubActionsRelayRole, + githubActionsRunnerProtocolHeader, githubActionsRunnerProtocolQuery, githubActionsRuntimeLabel, githubActionsViewerGenerationHeader, @@ -34,6 +35,7 @@ import { parseGitHubActionsRelayInputAcknowledgement, parseGitHubActionsRelayOutput, parseGitHubActionsRunnerProtocol, + parseGitHubActionsRunnerProtocolOffer, parseGitHubActionsViewerProtocol, parseGitHubActionsWorkState, relayGitHubActionsWebSocketMessage, @@ -107,6 +109,15 @@ test("runner URL works without custom WebSocket headers", () => { parseGitHubActionsRunnerProtocol(githubActionsFramedRunnerCapability), githubActionsFramedRunnerCapability, ); + assert.equal(githubActionsRunnerProtocolHeader, "sec-websocket-protocol"); + assert.equal(parseGitHubActionsRunnerProtocolOffer(null), null); + assert.equal(parseGitHubActionsRunnerProtocolOffer(githubActionsFramedRunnerCapability), null); + assert.equal( + parseGitHubActionsRunnerProtocolOffer( + `other-protocol, ${githubActionsGenerationFencedCapability}`, + ), + githubActionsGenerationFencedCapability, + ); assert.equal(githubActionsRunnerProtocolQuery, "runnerProtocol"); assert.equal( buildGitHubActionsViewerRelayUrl(), From 249751b4f443774238b6912b4247cce7244c6708 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:18:12 +0200 Subject: [PATCH 177/242] fix(macos): validate ARD Diffie-Hellman parameters --- .../RoyalVNCKit/Encryption/BigNum.swift | 5 ++ .../ARDDiffieHellmanKeyAgreement.swift | 25 ++++++- .../SecurityAndInputTests.swift | 69 +++++++++++++++++-- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift index a0dfc797..829e5f31 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Encryption/BigNum.swift @@ -24,6 +24,11 @@ extension BigNum { self.bigInt > 1 } + func isValidDiffieHellmanElement(modulus: BigNum) -> Bool { + guard modulus.bigInt > 2 else { return false } + return self.bigInt > 1 && self.bigInt < modulus.bigInt - 1 + } + var isZero: Bool { let isIt = self.bigInt == 0 diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift index 8ad964e3..fa6ddc0d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift @@ -19,7 +19,10 @@ extension VNCProtocol.ARDAuthentication { generator: Data, peerKey: Data, keyLength: Int) { - guard keyLength > 0 else { + guard Self.validParameters(prime: prime, + generator: generator, + peerKey: peerKey, + keyLength: keyLength) else { return nil } @@ -52,6 +55,26 @@ private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { let privateKey: Data } + static func validParameters(prime: Data, + generator: Data, + peerKey: Data, + keyLength: Int) -> Bool { + guard keyLength >= 128, + keyLength <= 512, + prime.count == keyLength, + peerKey.count == keyLength, + let bigPrime = BigNum(data: prime), + bigPrime.bitsCount >= 1_024, + let bigGenerator = BigNum(data: generator), + bigGenerator.isValidDiffieHellmanElement(modulus: bigPrime), + let bigPeerKey = BigNum(data: peerKey), + bigPeerKey.isValidDiffieHellmanElement(modulus: bigPrime) else { + return false + } + + return true + } + static func generateKeyPair(generator: Data, prime: Data, keyLength: Int) -> KeyPair? { diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift index 32e60f9c..f118b78a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -9,19 +9,76 @@ struct SecurityAndInputTests { VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAgreement.UltraVNCBigNum @Test - func rejectsZeroAndOneAppleRemoteDesktopModuli() { - for prime in [Data([0]), Data([1]), Data([0, 1])] { + func rejectsWeakAppleRemoteDesktopModuli() { + for prime in [ + Data(repeating: 0, count: 128), + Data(repeating: 1, count: 128), + Data([0]) + Data(repeating: 0xFF, count: 127), + ] { let agreement = ARDKeyAgreement( prime: prime, generator: Data([2]), - peerKey: Data([2]), - keyLength: prime.count + peerKey: paddedARDValue(2), + keyLength: 128 ) #expect(agreement.map { _ in true } == nil) } } + @Test + func rejectsAppleRemoteDesktopElementsOutsideTheSafeRange() { + let prime = Data(repeating: 0xFF, count: 128) + let primeMinusOne = Data(repeating: 0xFF, count: 127) + Data([0xFE]) + + for generator in [ + Data([0]), + Data([1]), + primeMinusOne, + prime, + ] { + #expect( + ARDKeyAgreement( + prime: prime, + generator: generator, + peerKey: paddedARDValue(2), + keyLength: 128 + ) == nil + ) + } + + for peerKey in [ + paddedARDValue(0), + paddedARDValue(1), + primeMinusOne, + prime, + ] { + #expect( + ARDKeyAgreement( + prime: prime, + generator: Data([2]), + peerKey: peerKey, + keyLength: 128 + ) == nil + ) + } + } + + @Test + func acceptsAppleRemoteDesktopKeyMaterialAtAndAboveTheMinimum() { + for keyLength in [128, 256] { + let agreement = ARDKeyAgreement( + prime: Data(repeating: 0xFF, count: keyLength), + generator: Data([2]), + peerKey: Data(repeating: 0, count: keyLength - 1) + Data([2]), + keyLength: keyLength + ) + + #expect(agreement?.publicKey.count == keyLength) + #expect(agreement?.secretKey.count == keyLength) + } + } + @Test func computesUltraVNCModularArithmeticKnownAnswers() { #expect( @@ -92,4 +149,8 @@ struct SecurityAndInputTests { #expect(_ObjC_VNCKeyCode.ansiKeypadDecimal == X11KeySymbols.XK_KP_Decimal) #endif } + + private func paddedARDValue(_ value: UInt8) -> Data { + Data(repeating: 0, count: 127) + Data([value]) + } } From e85c048aa048a17b760119a30137f14d7410323a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:18:45 +0200 Subject: [PATCH 178/242] fix(credentials): recover namespace rotation safely --- CHANGELOG.md | 2 +- ...credential_policy_registration_staging.sql | 54 ++- .../0036_credential_policy_lookup_repair.sql | 32 ++ ...dential_policy_registration_lookup_ids.sql | 44 +++ ...-credential-policy-registration-service.ts | 122 +++++- .../sandbox-credential-policy-repository.ts | 132 ++++++- ...ndbox-credential-policy-repository.test.ts | 359 ++++++++++++++++++ 7 files changed, 714 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46bd2df9..44d6dcb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation for the full rollback lifetime, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete legacy lookup sets before rotation, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. diff --git a/migrations/0034_credential_policy_registration_staging.sql b/migrations/0034_credential_policy_registration_staging.sql index 2e4f329b..53d62cc6 100644 --- a/migrations/0034_credential_policy_registration_staging.sql +++ b/migrations/0034_credential_policy_registration_staging.sql @@ -42,9 +42,9 @@ CREATE INDEX IF NOT EXISTS idx_credential_policy_registration_expiry ); -- Once a new worker stages a rotation, legacy workers must not claim, promote, --- or remove the policy rows underneath its rollback snapshot. Cleanup may --- still transition an older generation into cleanup_pending, but its rows stay --- fenced until the staged registration is removed. +-- or remove the policy rows underneath its rollback snapshot. Expired claims +-- and abandoned cleanup rows eventually release the compatibility fence so a +-- rollback to legacy worker code cannot wedge this policy group forever. CREATE TRIGGER IF NOT EXISTS fence_staged_credential_policy_insert BEFORE INSERT ON interactive_session_credential_policies WHEN NEW.state != 'cleanup_pending' @@ -54,6 +54,22 @@ WHEN NEW.state != 'cleanup_pending' WHERE staged.session_id = NEW.session_id AND staged.sandbox_id = NEW.sandbox_id AND staged.registration_generation != NEW.registration_generation + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) ) BEGIN SELECT RAISE(IGNORE); @@ -68,6 +84,22 @@ WHEN NOT (OLD.state != 'cleanup_pending' AND NEW.state = 'cleanup_pending') WHERE staged.session_id = NEW.session_id AND staged.sandbox_id = NEW.sandbox_id AND staged.registration_generation != NEW.registration_generation + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) ) BEGIN SELECT RAISE(IGNORE); @@ -80,6 +112,22 @@ WHEN EXISTS ( FROM interactive_session_credential_policy_registrations AS staged WHERE staged.session_id = OLD.session_id AND staged.sandbox_id = OLD.sandbox_id + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) ) BEGIN SELECT RAISE(IGNORE); diff --git a/migrations/0036_credential_policy_lookup_repair.sql b/migrations/0036_credential_policy_lookup_repair.sql index fdc4437e..50325c7c 100644 --- a/migrations/0036_credential_policy_lookup_repair.sql +++ b/migrations/0036_credential_policy_lookup_repair.sql @@ -13,6 +13,22 @@ WHEN NEW.state != 'cleanup_pending' WHERE staged.session_id = NEW.session_id AND staged.sandbox_id = NEW.sandbox_id AND staged.registration_generation != NEW.registration_generation + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) AND NOT ( staged.state = 'registering' AND staged.repair_generation = NEW.registration_generation @@ -34,6 +50,22 @@ WHEN NOT (OLD.state != 'cleanup_pending' AND NEW.state = 'cleanup_pending') WHERE staged.session_id = NEW.session_id AND staged.sandbox_id = NEW.sandbox_id AND staged.registration_generation != NEW.registration_generation + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) AND NOT ( staged.state = 'registering' AND staged.repair_generation = NEW.registration_generation diff --git a/migrations/0037_credential_policy_registration_lookup_ids.sql b/migrations/0037_credential_policy_registration_lookup_ids.sql index fe3b8614..1bad836f 100644 --- a/migrations/0037_credential_policy_registration_lookup_ids.sql +++ b/migrations/0037_credential_policy_registration_lookup_ids.sql @@ -28,3 +28,47 @@ SET lookup_ids_json = ( ORDER BY lookup_id ) ); + +DROP TRIGGER IF EXISTS fence_staged_credential_policy_delete; + +CREATE TRIGGER fence_staged_credential_policy_delete +BEFORE DELETE ON interactive_session_credential_policies +WHEN EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = OLD.session_id + AND staged.sandbox_id = OLD.sandbox_id + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) + AND NOT ( + staged.state = 'registering' + AND staged.repair_generation = OLD.registration_generation + AND staged.registration_claim = OLD.cleanup_claim + AND staged.registration_claim_expires_at = OLD.cleanup_claim_expires_at + AND OLD.state = 'cleanup_pending' + AND json_valid(staged.lookup_ids_json) + AND NOT EXISTS ( + SELECT 1 + FROM json_each(staged.lookup_ids_json) AS current_lookup + WHERE current_lookup.type = 'text' + AND current_lookup.value = OLD.lookup_id + ) + ) +) +BEGIN + SELECT RAISE(IGNORE); +END; diff --git a/src/worker/sandbox-credential-policy-registration-service.ts b/src/worker/sandbox-credential-policy-registration-service.ts index 4e4e3ca4..9f9d0734 100644 --- a/src/worker/sandbox-credential-policy-registration-service.ts +++ b/src/worker/sandbox-credential-policy-registration-service.ts @@ -5,6 +5,7 @@ import { activeSandboxCredentialPolicyGeneration, abandonSandboxCredentialPolicyRegistration, beginSandboxCredentialPolicyRegistration, + claimObsoleteSandboxCredentialPolicyReferences, deferSandboxCredentialPolicyRollback, existingSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, @@ -12,7 +13,9 @@ import { recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, repairSandboxCredentialPolicyReferences, + retireObsoleteSandboxCredentialPolicyReference, renewSandboxCredentialPolicyRegistration, + sandboxCredentialPolicyLookupIdsForGeneration, stageSandboxCredentialPolicyReferenceRepair, standaloneSandboxPolicyExpiresAt, type SandboxCredentialPolicyOwnershipFence, @@ -52,26 +55,35 @@ async function repairIncompleteSandboxCredentialPolicyLookupSet( sandboxId, ); if (!repairGeneration) return null; - const records = await Promise.all( - registration.lookupIds.map(async (lookupId) => { - const response = await stub.fetch( - `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, - ); - if (response.status === 404) return null; - if (!response.ok) throw new Error("sandbox credential policy repair snapshot failed"); - const generation = response.headers.get("x-crabfleet-policy-generation"); - const policy = (await response.json()) as SandboxCredentialPolicy; - if ( - generation !== repairGeneration || - policy.sessionId !== sessionId || - policy.sandboxId !== lookupId - ) { - throw new Error("sandbox credential policy repair snapshot is inconsistent"); - } - return { generation, policy }; - }), + const historicalLookupIds = await sandboxCredentialPolicyLookupIdsForGeneration( + env, + sessionId, + sandboxId, + repairGeneration, + ); + const repairLookupIds = [...new Set([...historicalLookupIds, ...registration.lookupIds])]; + const records = new Map( + await Promise.all( + repairLookupIds.map(async (lookupId) => { + const response = await stub.fetch( + `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, + ); + if (response.status === 404) return [lookupId, null] as const; + if (!response.ok) throw new Error("sandbox credential policy repair snapshot failed"); + const generation = response.headers.get("x-crabfleet-policy-generation"); + const policy = (await response.json()) as SandboxCredentialPolicy; + if ( + generation !== repairGeneration || + policy.sessionId !== sessionId || + policy.sandboxId !== lookupId + ) { + throw new Error("sandbox credential policy repair snapshot is inconsistent"); + } + return [lookupId, { generation, policy }] as const; + }), + ), ); - const surviving = records.filter((record) => record !== null); + const surviving = [...records.values()].filter((record) => record !== null); if (surviving.length === 0) return null; const source = surviving[0]!.policy; if ( @@ -104,8 +116,8 @@ async function repairIncompleteSandboxCredentialPolicyLookupSet( throw new Error("sandbox credential policy registration claim was revoked"); } const repairExpiresAt = registrationExpiresAt - 1; - for (const [index, lookupId] of registration.lookupIds.entries()) { - if (records[index]) continue; + for (const lookupId of registration.lookupIds) { + if (records.get(lookupId)) continue; const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { method: "POST", body: JSON.stringify({ @@ -139,6 +151,74 @@ async function repairIncompleteSandboxCredentialPolicyLookupSet( ) { throw new Error("sandbox credential policy lookup references were not repaired"); } + const currentLookupIds = new Set(registration.lookupIds); + const obsoleteLookupIds = historicalLookupIds.filter( + (lookupId) => !currentLookupIds.has(lookupId), + ); + if (obsoleteLookupIds.length > 0) { + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + ); + if (!registrationExpiresAt) { + throw new Error("sandbox credential policy registration claim was revoked"); + } + const claimed = await claimObsoleteSandboxCredentialPolicyReferences( + env, + sessionId, + sandboxId, + registration, + repairGeneration, + obsoleteLookupIds, + ownershipFence, + registrationExpiresAt, + ); + if ( + claimed.length !== obsoleteLookupIds.length || + obsoleteLookupIds.some((lookupId) => !claimed.includes(lookupId)) + ) { + throw new Error("obsolete sandbox credential policy references were not claimed"); + } + for (const lookupId of obsoleteLookupIds) { + const response = await stub.fetch( + `https://crabfleet.internal/api/session-control/sandbox/${encodeURIComponent(lookupId)}`, + { + method: "DELETE", + body: JSON.stringify({ + generation: repairGeneration, + sessionId, + tombstonedAt: Date.now(), + }), + headers: { "content-type": "application/json" }, + }, + ); + if (!response.ok) { + throw new Error("obsolete sandbox credential policy retirement failed"); + } + if ( + !(await retireObsoleteSandboxCredentialPolicyReference( + env, + sessionId, + sandboxId, + registration, + repairGeneration, + lookupId, + ownershipFence, + registrationExpiresAt, + )) + ) { + throw new Error("obsolete sandbox credential policy reference was not retired"); + } + } + } + if ( + (await activeSandboxCredentialPolicyGeneration(env, sessionId, sandboxId)) !== repairGeneration + ) { + throw new Error("sandbox credential policy namespace repair did not converge"); + } return repairGeneration; } diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 1025c580..e20989dc 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -122,21 +122,40 @@ export async function incompleteSandboxCredentialPolicyGeneration( const generation = rows[0]?.registration_generation; if ( rows.length === 0 || - rows.length >= expected.size || !isCurrentCredentialPolicyGeneration(generation) || + !rows.some((row) => row.lookup_id === sandboxId) || rows.some( (row) => - !expected.has(row.lookup_id) || row.state !== "active" || row.registration_generation !== generation || row.registration_claim !== null, - ) + ) || + (rows.length === expected.size && rows.every((row) => expected.has(row.lookup_id))) ) { return null; } return generation; } +export async function sandboxCredentialPolicyLookupIdsForGeneration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + generation: string, +): Promise { + const rows = await database(env) + .selectFrom("interactive_session_credential_policies") + .select("lookup_id") + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("state", "=", "active") + .where("registration_generation", "=", generation) + .where("registration_claim", "is", null) + .orderBy("lookup_id") + .execute(); + return rows.map((row) => row.lookup_id); +} + export async function sandboxCredentialPolicyHasDurableOwner( env: RuntimeEnv, lookupId: string, @@ -777,8 +796,7 @@ export async function repairSandboxCredentialPolicyReferences( WHERE conflicting.session_id = ${sessionId} AND conflicting.sandbox_id = ${sandboxId} AND ( - conflicting.lookup_id NOT IN (${sql.join(registration.lookupIds)}) - OR conflicting.registration_generation != ${repairGeneration} + conflicting.registration_generation != ${repairGeneration} OR NOT ( ( conflicting.state = 'active' @@ -850,11 +868,113 @@ export async function repairSandboxCredentialPolicyReferences( `, ); await executeBatch(env, [...inserts, ...promotions]); + const refs = await database(env) + .selectFrom("interactive_session_credential_policies") + .select(["lookup_id", "state", "registration_generation", "registration_claim"]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .execute(); return ( - (await activeSandboxCredentialPolicyGeneration(env, sessionId, sandboxId)) === repairGeneration + registration.lookupIds.every((lookupId) => + refs.some( + (ref) => + ref.lookup_id === lookupId && + ref.state === "active" && + ref.registration_generation === repairGeneration && + ref.registration_claim === null, + ), + ) && + refs.every( + (ref) => + ref.state === "active" && + ref.registration_generation === repairGeneration && + ref.registration_claim === null, + ) ); } +export async function claimObsoleteSandboxCredentialPolicyReferences( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + repairGeneration: string, + obsoleteLookupIds: readonly string[], + ownershipFence: SandboxCredentialPolicyOwnershipFence, + registrationExpiresAt: number, +): Promise { + if (obsoleteLookupIds.length === 0) return []; + const now = Date.now(); + const claimed = await sql<{ lookup_id: string }>` + UPDATE interactive_session_credential_policies + SET + state = 'cleanup_pending', + cleanup_claim = ${registration.claim}, + cleanup_claim_expires_at = ${registrationExpiresAt}, + last_error = 'obsolete sandbox credential policy namespace', + updated_at = ${now} + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND lookup_id IN (${sql.join(obsoleteLookupIds)}) + AND state = 'active' + AND registration_generation = ${repairGeneration} + AND registration_claim IS NULL + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = ${sessionId} + AND staged.sandbox_id = ${sandboxId} + AND staged.state = 'registering' + AND staged.registration_generation = ${registration.generation} + AND staged.registration_claim = ${registration.claim} + AND staged.registration_claim_expires_at = ${registrationExpiresAt} + AND staged.registration_claim_expires_at > ${now} + AND staged.repair_generation = ${repairGeneration} + ) + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + RETURNING lookup_id + `.execute(database(env)); + return claimed.rows.map((row) => row.lookup_id).sort(); +} + +export async function retireObsoleteSandboxCredentialPolicyReference( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + repairGeneration: string, + lookupId: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + registrationExpiresAt: number, +): Promise { + const now = Date.now(); + const retired = await sql<{ lookup_id: string }>` + DELETE FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND lookup_id = ${lookupId} + AND state = 'cleanup_pending' + AND registration_generation = ${repairGeneration} + AND cleanup_claim = ${registration.claim} + AND cleanup_claim_expires_at = ${registrationExpiresAt} + AND EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = ${sessionId} + AND staged.sandbox_id = ${sandboxId} + AND staged.state = 'registering' + AND staged.registration_generation = ${registration.generation} + AND staged.registration_claim = ${registration.claim} + AND staged.registration_claim_expires_at = ${registrationExpiresAt} + AND staged.registration_claim_expires_at > ${now} + AND staged.repair_generation = ${repairGeneration} + ) + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + RETURNING lookup_id + `.execute(database(env)); + return retired.rows[0]?.lookup_id === lookupId; +} + export async function deferSandboxCredentialPolicyRollback( env: RuntimeEnv, sessionId: string, diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 0575178e..2548a931 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -7,6 +7,7 @@ import { activeSandboxCredentialPolicyGeneration, abandonSandboxCredentialPolicyRegistration, beginSandboxCredentialPolicyRegistration, + claimObsoleteSandboxCredentialPolicyReferences, claimSandboxCredentialPolicyRegistrationRecovery, currentSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, @@ -14,7 +15,9 @@ import { recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, repairSandboxCredentialPolicyReferences, + retireObsoleteSandboxCredentialPolicyReference, renewSandboxCredentialPolicyRegistration, + sandboxCredentialPolicyLookupIdsForGeneration, sandboxCredentialPolicyRegistrationQueries, sandboxLookupIds, stageSandboxCredentialPolicyReferenceRepair, @@ -759,6 +762,82 @@ test("staged rotations fence old-worker writes until the staged row is removed", assert.equal(postFenceClaim.changes, 2); }); +test("expired staged rotations release the legacy worker compatibility fence", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + await beginSandboxCredentialPolicyRegistration(env, "IS-42", "sandbox-1", ownershipFence); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = 0, updated_at = 0 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + + const legacyClaim = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-rollback', + registration_claim = 'legacy-rollback-claim', + registration_claim_expires_at = ?, + updated_at = 1 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + + assert.equal(legacyClaim.changes, 2); + assert.equal( + sqlite + .prepare("SELECT count(*) AS count FROM interactive_session_credential_policy_registrations") + .get()?.count, + 1, + ); +}); + +test("stale staged cleanup releases legacy deletion but an active cleanup claim stays fenced", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + await beginSandboxCredentialPolicyRegistration(env, "IS-42", "sandbox-1", ownershipFence); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET + state = 'cleanup_pending', + registration_claim = NULL, + registration_claim_expires_at = NULL, + cleanup_claim = 'cleanup:new-worker', + cleanup_claim_expires_at = ?, + updated_at = 0 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + + const fencedDelete = sqlite + .prepare(` + DELETE FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + assert.equal(fencedDelete.changes, 0); + + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET cleanup_claim = NULL, cleanup_claim_expires_at = NULL, updated_at = 0 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + const legacyDelete = sqlite + .prepare(` + DELETE FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + assert.equal(legacyDelete.changes, 2); +}); + test("partial credential-policy rotation failure preserves the prior active generation", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); @@ -1311,6 +1390,286 @@ test("credential refresh repairs an incomplete legacy lookup set before rotation ); }); +test("credential refresh replaces an obsolete durable namespace without losing rollback coverage", async () => { + const sqlite = credentialPolicyDatabase(); + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET lookup_id = 'do-old' + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' AND lookup_id = 'do-1' + `) + .run(); + const now = Date.now(); + const policy = { + allowedHosts: [], + githubCredentialSource: "none" as const, + githubRepo: "openclaw/crabfleet", + owner: "operator", + sessionId: "IS-42", + }; + const policies = new Map( + ["sandbox-1", "do-old"].map((lookupId) => [ + lookupId, + { + generation: "generation:existing", + registrationClaim: "registration:legacy", + registrationExpiresAt: now + 1_000, + policy: { ...policy, sandboxId: lookupId }, + }, + ]), + ); + const tombstones = new Map(); + const stub = { + async fetch(input: RequestInfo | URL, init?: RequestInit): Promise { + const url = new URL(String(input)); + const egress = url.pathname.match(/^\/api\/session-control\/egress\/([^/]+)$/); + if (egress && (!init?.method || init.method === "GET")) { + const current = policies.get(decodeURIComponent(egress[1] ?? "")); + return current + ? Response.json(current.policy, { + headers: { "x-crabfleet-policy-generation": current.generation }, + }) + : Response.json({ error: "not found" }, { status: 404 }); + } + if (url.pathname === "/api/session-control/register" && init?.method === "POST") { + const incoming = JSON.parse(String(init.body)) as StoredSandboxCredentialPolicy; + const lookupId = incoming.policy.sandboxId; + if ( + tombstones.get(lookupId) === incoming.generation || + !credentialPolicyRegistrationAccepted( + policies.get(lookupId), + undefined, + incoming, + Date.now(), + ) + ) { + return Response.json({ error: "conflict" }, { status: 409 }); + } + policies.set(lookupId, incoming); + return Response.json({ ok: true }); + } + const removal = url.pathname.match(/^\/api\/session-control\/sandbox\/([^/]+)$/); + if (removal && init?.method === "DELETE") { + const lookupId = decodeURIComponent(removal[1] ?? ""); + const tombstone = JSON.parse(String(init.body)) as { + generation: string; + sessionId: string; + }; + tombstones.set(lookupId, tombstone.generation); + const current = policies.get(lookupId); + if ( + current?.generation === tombstone.generation && + current.policy.sessionId === tombstone.sessionId + ) { + policies.delete(lookupId); + } + return Response.json({ ok: true }); + } + return Response.json({ error: "not found" }, { status: 404 }); + }, + }; + const env = sqliteRuntimeEnv(sqlite); + + assert.equal(await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), null); + assert.equal( + await incompleteSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), + "generation:existing", + ); + const registration = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const repairGeneration = await incompleteSandboxCredentialPolicyGeneration( + env, + "IS-42", + "sandbox-1", + ); + assert.equal(repairGeneration, "generation:existing"); + assert.deepEqual( + await sandboxCredentialPolicyLookupIdsForGeneration( + env, + "IS-42", + "sandbox-1", + repairGeneration, + ), + ["do-old", "sandbox-1"], + ); + let registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.equal( + await stageSandboxCredentialPolicyReferenceRepair( + env, + "IS-42", + "sandbox-1", + registration, + repairGeneration, + ownershipFence, + ), + true, + ); + assert.equal( + ( + await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: repairGeneration, + registrationClaim: registration.claim, + registrationExpiresAt: registrationExpiresAt - 1, + policy: { ...policy, sandboxId: "do-1" }, + } satisfies StoredSandboxCredentialPolicy), + }) + ).ok, + true, + ); + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.equal( + await repairSandboxCredentialPolicyReferences( + env, + "IS-42", + "sandbox-1", + registration, + repairGeneration, + ownershipFence, + registrationExpiresAt, + ), + true, + ); + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.deepEqual( + await claimObsoleteSandboxCredentialPolicyReferences( + env, + "IS-42", + "sandbox-1", + registration, + repairGeneration, + ["do-old"], + ownershipFence, + registrationExpiresAt, + ), + ["do-old"], + ); + assert.equal( + ( + await stub.fetch("https://crabfleet.internal/api/session-control/sandbox/do-old", { + method: "DELETE", + body: JSON.stringify({ + generation: repairGeneration, + sessionId: "IS-42", + tombstonedAt: Date.now(), + }), + }) + ).ok, + true, + ); + assert.equal( + await retireObsoleteSandboxCredentialPolicyReference( + env, + "IS-42", + "sandbox-1", + registration, + repairGeneration, + "do-old", + ownershipFence, + registrationExpiresAt, + ), + true, + ); + assert.equal( + await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), + repairGeneration, + ); + const rollback = await captureSandboxCredentialPolicyRollback( + stub, + registration.lookupIds, + repairGeneration, + "IS-42", + ); + assert.equal( + await recordSandboxCredentialPolicyRollback( + env, + "IS-42", + "sandbox-1", + registration, + rollback, + ownershipFence, + ), + true, + ); + for (const lookupId of registration.lookupIds) { + registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); + assert.equal( + ( + await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: registration.generation, + registrationClaim: registration.claim, + registrationExpiresAt, + policy: { ...policy, sandboxId: lookupId }, + } satisfies StoredSandboxCredentialPolicy), + }) + ).ok, + true, + ); + } + assert.equal( + await finishSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ), + true, + ); + + const generation = await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"); + assert.match(generation ?? "", /^generation:/); + assert.notEqual(generation, "generation:existing"); + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => row.lookup_id), + ["do-1", "sandbox-1"], + ); + assert.deepEqual([...policies.keys()].sort(), ["do-1", "sandbox-1"]); + assert.equal(tombstones.get("do-old"), "generation:existing"); + assert.equal(policies.has("do-old"), false); + assert.equal( + sqlite + .prepare("SELECT count(*) AS count FROM interactive_session_credential_policy_registrations") + .get()?.count, + 0, + ); +}); + test("recording active policy refs promotes then upserts every lookup under one fence", async () => { const rows = [ { From 6476c819181d4dbb165c0e7ae9177cc32fa1f289 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:18:56 +0200 Subject: [PATCH 179/242] fix(macos): key publication cleanup by host ID --- .../PrivateMacShareController.swift | 12 +++- .../PrivateMacShareTests.swift | 59 +++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 81aa721e..f9b9a191 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -40,12 +40,14 @@ final class PrivateMacShareStopCoordinator { final class DesktopHostRegistrationLifecycle { private struct RegistrationTarget: Equatable { let identity: TailnetIdentity + let hostID: String let port: UInt16 let publicationID: String } private struct PublishedRegistration: Equatable { let identity: TailnetIdentity + let hostID: String let ownershipToken: String? } @@ -64,12 +66,14 @@ final class DesktopHostRegistrationLifecycle { } func publish(identity: TailnetIdentity, port: UInt16) async throws { + let hostID = CrabfleetDesktopRegistration.hostID(identity: identity) let target = uncertainRegistrations.first { - $0.identity == identity && $0.port == port + $0.hostID == hostID && $0.port == port } ?? RegistrationTarget( identity: identity, + hostID: hostID, port: port, publicationID: createPublicationID() ) @@ -100,14 +104,15 @@ final class DesktopHostRegistrationLifecycle { throw error } uncertainRegistrations.removeAll { $0 == target } - if let publishedRegistration, publishedRegistration.identity != identity, + if let publishedRegistration, publishedRegistration.hostID != hostID, !pendingRemovals.contains(publishedRegistration) { pendingRemovals.append(publishedRegistration) } - pendingRemovals.removeAll { $0.identity == identity } + pendingRemovals.removeAll { $0.hostID == hostID } publishedRegistration = PublishedRegistration( identity: identity, + hostID: hostID, ownershipToken: ownershipToken ) } @@ -124,6 +129,7 @@ final class DesktopHostRegistrationLifecycle { if let ownershipToken { let recovered = PublishedRegistration( identity: target.identity, + hostID: target.hostID, ownershipToken: ownershipToken ) if !pendingRemovals.contains(recovered) { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index d8e6953b..a4b9270f 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -498,6 +498,39 @@ struct PrivateMacShareTests { #expect(await registration.activePublicationID == nil) } + @Test @MainActor + func desktopPublicationReplacementUsesSanitizedHostIDAcrossIdentityChanges() async throws { + let first = desktopIdentity(name: "shared-host", address: "100.64.12.48") + let second = TailnetIdentity( + tailnetName: first.tailnetName, + loginName: first.loginName, + dnsName: first.dnsName, + hostName: "Shared Host Renamed", + ipv4Address: "100.64.12.49", + userID: first.userID + ) + #expect(first != second) + #expect( + CrabfleetDesktopRegistration.hostID(identity: first) + == CrabfleetDesktopRegistration.hostID(identity: second) + ) + let registration = MutableIdentityDesktopRegistration() + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + + try await lifecycle.publish(identity: first, port: 5_901) + try await lifecycle.publish(identity: second, port: 5_901) + try await lifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(first.ipv4Address), + .register(second.ipv4Address), + .unregister(second.ipv4Address, "token:\(second.ipv4Address)"), + ] + ) + } + @Test @MainActor func failedDesktopRemovalSurvivesLaterIdentityChanges() async throws { let first = desktopIdentity(name: "first-host", address: "100.64.12.41") @@ -1865,6 +1898,32 @@ private actor RecordingDesktopRegistration: DesktopHostRegistering { } } +private actor MutableIdentityDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register(String) + case unregister(String, String?) + } + + private(set) var events: [Event] = [] + + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + events.append(.register(identity.ipv4Address)) + return "token:\(identity.ipv4Address)" + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + nil + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + events.append(.unregister(identity.ipv4Address, ownershipToken)) + } +} + private actor AmbiguousDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String) From c9d055b371ec68b4da8395ade2580b626356d299 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:19:41 +0200 Subject: [PATCH 180/242] docs(changelog): record final audit fixes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44d6dcb3..5d9519d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, persisting successful runtime-adapter deletion, validating Apple Remote Desktop Diffie-Hellman groups, and keying desktop publication cleanup by the API host ID; the runner guide now demonstrates restricted steering instead of shell access. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. From 6773dae047dc57faa5cb84dbb28bf85b72ab29d1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:30:17 +0200 Subject: [PATCH 181/242] docs(actions): serialize negotiated steering example --- CHANGELOG.md | 2 +- docs/github-actions-sessions.md | 74 +++++++++++++++++++++---------- tests/github-actions-docs.test.ts | 31 +++++++++++-- 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d9519d2..1710b29e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, persisting successful runtime-adapter deletion, validating Apple Remote Desktop Diffie-Hellman groups, and keying desktop publication cleanup by the API host ID; the runner guide now demonstrates restricted steering instead of shell access. +- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, persisting successful runtime-adapter deletion, validating Apple Remote Desktop Diffie-Hellman groups, and keying desktop publication cleanup by the API host ID; the runner guide now demonstrates negotiated, serialized restricted steering instead of shell access. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 7b9d9f80..bae404ae 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -265,14 +265,14 @@ returns only the sanitized event. The Action connects outbound to the returned `runnerPtyUrl`. Node's global `WebSocket` can open the URL without custom headers. -The returned URL opens a legacy raw-input/raw-output socket. A runner opts into -collision-free framed I/O by adding the exact -`runnerProtocol=cfr1-framed-io-v2` query before opening the socket. The relay -records that mode and a relay-owned runner generation before accepting the -connection. Viewer input then arrives in a binary `CFR1` frame carrying a -correlation ID and that generation, and runner output uses a distinct `CFR1` -output frame. The runner copies the generation into its correlated -acknowledgement only after its PTY accepts the input write. +The returned URL opens a legacy raw-input/raw-output socket. A runner offers +`cfr1-framed-io-v2` as a WebSocket subprotocol and switches to collision-free +framed I/O only when the upgrade response selects it. The relay records that +mode and a relay-owned runner generation before accepting the connection. +Viewer input then arrives in a binary `CFR1` frame carrying a correlation ID +and that generation, and runner output uses a distinct `CFR1` output frame. The +runner copies the generation into its correlated acknowledgement only after its +restricted steering handler accepts the input. Complete Node framing adapter around a restricted Codex steering handler: @@ -296,15 +296,18 @@ const maxPendingInputAgeMs = 1_000; let pendingInputs = []; let pendingInputBytes = 0; let pendingInputTimer; -const framedRunnerPtyUrl = new URL(runnerPtyUrl); -framedRunnerPtyUrl.searchParams.set("runnerProtocol", "cfr1-framed-io-v2"); -const terminal = new WebSocket(framedRunnerPtyUrl); +let inputQueue = Promise.resolve(); +const terminal = new WebSocket(runnerPtyUrl, "cfr1-framed-io-v2"); terminal.binaryType = "arraybuffer"; await new Promise((resolve, reject) => { terminal.addEventListener("open", resolve, { once: true }); terminal.addEventListener("error", reject, { once: true }); }); +if (terminal.protocol !== "cfr1-framed-io-v2") { + terminal.close(1002, "framed protocol not negotiated"); + throw new Error("relay did not negotiate cfr1-framed-io-v2"); +} subscribeSteeringOutput((outputText) => { terminal.send(encodeUtf8Output(outputText)); @@ -315,7 +318,11 @@ subscribeSteeringExit(() => { }); terminal.addEventListener("message", (event) => { - void acceptInput(event.data); + inputQueue = inputQueue + .then(() => acceptInput(event.data)) + .catch(() => { + settleInputs(takePendingInputs(), false); + }); }); async function acceptInput(data) { @@ -323,11 +330,8 @@ async function acceptInput(data) { if (!input) return; pendingInputs.push(input); pendingInputBytes += input.payload.byteLength; - if (pendingInputs.length === 1) { - pendingInputTimer = setTimeout(() => settlePendingInputs(false), maxPendingInputAgeMs); - } if (pendingInputBytes > maxPendingInputBytes || pendingInputs.length > maxPendingInputFrames) { - settlePendingInputs(false); + settleInputs(takePendingInputs(), false); return; } @@ -338,13 +342,23 @@ async function acceptInput(data) { offset += pending.payload.byteLength; } + let text; + try { + text = decodeCompleteUtf8(payload); + } catch { + settleInputs(takePendingInputs(), false); + return; + } + if (text === null) { + armPendingInputTimer(); + return; + } + const inputs = takePendingInputs(); try { - const text = decodeCompleteUtf8(payload); - if (text === null) return; await deliverSteeringInput(text); - settlePendingInputs(true); + settleInputs(inputs, true); } catch { - settlePendingInputs(false); + settleInputs(inputs, false); } } @@ -354,14 +368,26 @@ function decodeCompleteUtf8(payload) { return encoder.encode(text).byteLength === payload.byteLength ? text : null; } -function settlePendingInputs(accepted) { +function armPendingInputTimer() { + if (pendingInputTimer) return; + pendingInputTimer = setTimeout(() => { + settleInputs(takePendingInputs(), false); + }, maxPendingInputAgeMs); +} + +function takePendingInputs() { if (pendingInputTimer) clearTimeout(pendingInputTimer); pendingInputTimer = undefined; - for (const input of pendingInputs) { - terminal.send(encodeAck(input.inputId, input.generation, accepted)); - } + const inputs = pendingInputs; pendingInputs = []; pendingInputBytes = 0; + return inputs; +} + +function settleInputs(inputs, accepted) { + for (const input of inputs) { + terminal.send(encodeAck(input.inputId, input.generation, accepted)); + } } function decodeInput(data) { diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 15ee6d1e..70f05528 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -14,18 +14,34 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.match(readme, /ignore the offer leave `WebSocket\.protocol` empty/); assert.doesNotMatch(readme, /New runners opt into[\s\S]*cfr1-framed-io-v1/); assert.doesNotMatch(readme, /encodeCfr1Output|decodeCfr1Input|encodeCfr1Ack/); + assert.match(guide, /new WebSocket\(runnerPtyUrl, "cfr1-framed-io-v2"\)/); + assert.match(guide, /terminal\.protocol !== "cfr1-framed-io-v2"/); + assert.doesNotMatch(guide, /searchParams\.set\("runnerProtocol"/); assert.match(guide, /let pendingInputs = \[\]/); + assert.match(guide, /let inputQueue = Promise\.resolve\(\)/); + assert.match( + guide, + /inputQueue = inputQueue\s+\.then\(\(\) => acceptInput\(event\.data\)\)\s+\.catch/, + ); assert.match(guide, /pendingInputs\.push\(input\)/); - assert.match(guide, /const text = decodeCompleteUtf8\(payload\)/); - assert.match(guide, /if \(text === null\) return/); - assert.match(guide, /await deliverSteeringInput\(text\);\s+settlePendingInputs\(true\)/); + assert.match(guide, /text = decodeCompleteUtf8\(payload\)/); + assert.match(guide, /if \(text === null\) \{\s+armPendingInputTimer\(\);\s+return;/); + assert.match( + guide, + /const inputs = takePendingInputs\(\);\s+try \{\s+await deliverSteeringInput\(text\);\s+settleInputs\(inputs, true\)/, + ); + assert.match(guide, /catch \{\s+settleInputs\(inputs, false\);/); assert.match(guide, /const maxPendingInputBytes = 16 \* 1024/); assert.match(guide, /const maxPendingInputFrames = 32/); assert.match(guide, /const maxPendingInputAgeMs = 1_000/); - assert.match(guide, /setTimeout\(\(\) => settlePendingInputs\(false\), maxPendingInputAgeMs\)/); + assert.match(guide, /settleInputs\(takePendingInputs\(\), false\)/); assert.match(guide, /pendingInputBytes > maxPendingInputBytes/); assert.match(guide, /pendingInputs\.length > maxPendingInputFrames/); assert.match(guide, /clearTimeout\(pendingInputTimer\)/); + assert.match( + guide, + /const inputs = pendingInputs;\s+pendingInputs = \[\];\s+pendingInputBytes = 0;\s+return inputs;/, + ); assert.match(guide, /new TextDecoder\("utf-8", \{ fatal: true, ignoreBOM: true \}\)/); assert.doesNotMatch(guide, /inputDecoder\.decode/); assert.match(guide, /subscribeSteeringOutput\(\(outputText\) => \{/); @@ -41,6 +57,13 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.match(guide, /must never forward that input to a\s+shell or subprocess/); assert.doesNotMatch(guide, /spawn\(process\.env\.SHELL|env:\s*process\.env|pty\.write/); + const timerArm = guide.indexOf("armPendingInputTimer();"); + const batchSnapshot = guide.indexOf("const inputs = takePendingInputs();"); + const delivery = guide.indexOf("await deliverSteeringInput(text);"); + assert.ok(timerArm > guide.indexOf("if (text === null)")); + assert.ok(batchSnapshot > timerArm); + assert.ok(delivery > batchSnapshot); + const decodeCompleteUtf8 = (payload: Uint8Array) => { const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); const text = decoder.decode(payload, { stream: true }); From 490d07dee354a5b810bbe254a8d9b7e881fdc06b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:41:55 +0200 Subject: [PATCH 182/242] fix(desktop): require token publication identity --- CHANGELOG.md | 2 +- src/worker/desktop-host-service.ts | 11 ++----- tests/desktop-host-service.test.ts | 48 ++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1710b29e..66d0db9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; durably claim and retry superseded workspace cleanup without touching the replacement workspace; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. - Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure, ambiguous committed publication reconciled by stable publication identity, and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown with bounded retries and no retry when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. -- Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. +- Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens, require a valid publication identity before selecting token ownership, and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions with atomic capability publication, premature-response rejection, and fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. - Add a VideoToolbox-backed Open H.264 RFB pipeline for Share This Mac with up to 60 fps capture, adaptive 1.5–30 Mbit/s rate control, automatic Tight/JPEG fallback, live stream stats, larger resize limits, and a persisted host-enforced view-only mode. diff --git a/src/worker/desktop-host-service.ts b/src/worker/desktop-host-service.ts index 3a8ed0bb..fbc7d322 100644 --- a/src/worker/desktop-host-service.ts +++ b/src/worker/desktop-host-service.ts @@ -62,12 +62,12 @@ export class DesktopHostService { const address = tailscaleIPv4(input.address); const port = desktopHostPort(input.port); const now = this.now(); - const ownershipToken = - ownershipMode === desktopHostTokenOwnershipMode ? this.createOwnershipToken() : ""; const publicationID = ownershipMode === desktopHostTokenOwnershipMode - ? optionalDesktopHostPublicationID(rawPublicationID) + ? desktopHostPublicationID(rawPublicationID) : ""; + const ownershipToken = + ownershipMode === desktopHostTokenOwnershipMode ? this.createOwnershipToken() : ""; const host: DesktopHostRow = { ownerSubject: tenantSubject(user), id, @@ -208,8 +208,3 @@ function desktopHostPublicationID(value: unknown): string { } return value; } - -function optionalDesktopHostPublicationID(value: unknown): string { - if (value === null || value === undefined) return ""; - return desktopHostPublicationID(value); -} diff --git a/tests/desktop-host-service.test.ts b/tests/desktop-host-service.test.ts index 8c840d90..97359a2a 100644 --- a/tests/desktop-host-service.test.ts +++ b/tests/desktop-host-service.test.ts @@ -74,6 +74,7 @@ test("desktop hosts are canonicalized and isolated to their stable owner", async port: 5901, }, desktopHostTokenOwnershipMode, + "publication-1", ); const host = registration.host; @@ -100,6 +101,7 @@ test("desktop hosts are canonicalized and isolated to their stable owner", async port: host.port, }, desktopHostTokenOwnershipMode, + "publication-2", ); const updated = updatedRegistration.host; assert.equal(updated.createdAt, 42); @@ -128,6 +130,7 @@ test("stale desktop host cleanup cannot remove a newer registration", async () = "studio", input, desktopHostTokenOwnershipMode, + "old-publication", ); const newRegistration = await service.register( alice, @@ -137,6 +140,7 @@ test("stale desktop host cleanup cannot remove a newer registration", async () = name: "New Studio Process", }, desktopHostTokenOwnershipMode, + "new-publication", ); await service.remove(alice, "studio", oldRegistration.ownershipToken); @@ -175,6 +179,7 @@ test("tokenless cleanup removes only migrated legacy desktop hosts", async () => port: 5901, }, desktopHostTokenOwnershipMode, + "new-studio-publication", ); await service.remove(alice, legacy.id, null); @@ -209,6 +214,49 @@ test("ambiguous desktop recovery cannot acquire a newer publication", async () = assert.deepEqual(await service.recover(alice, "studio", "publication-b"), { ownershipToken: newer.ownershipToken, }); + assert.deepEqual(await service.recover(bob, "studio", "publication-b"), { + ownershipToken: null, + }); + for (const publicationID of [null, undefined, "", "bad publication", "a".repeat(201)]) { + await assert.rejects( + service.recover(alice, "studio", publicationID), + /desktop host publication id/, + ); + } +}); + +test("token ownership requires a valid publication before minting or persistence", async () => { + const store = new MemoryDesktopHostStore(); + let tokenCreations = 0; + const service = new DesktopHostService( + store, + () => 42, + () => { + tokenCreations += 1; + return "ownership-token"; + }, + ); + const input = { name: "Studio", address: "100.64.1.2", port: 5901 }; + + for (const publicationID of [ + null, + undefined, + "", + "bad publication", + "bad\npublication", + "a".repeat(201), + ]) { + await assert.rejects( + service.register(alice, "studio", input, desktopHostTokenOwnershipMode, publicationID), + /desktop host publication id/, + ); + } + assert.equal(tokenCreations, 0); + assert.equal(store.rows.size, 0); + + const legacy = await service.register(alice, "studio", input, "legacy", "ignored publication"); + assert.equal(legacy.ownershipToken, undefined); + assert.equal(store.rows.get(`${alice.subject}:studio`)?.publicationID, ""); }); test("legacy clients register tokenless rows they can remove after a server upgrade", async () => { From eefc11b2100a0ba41b12f3350b6d513fbf5a98f6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:42:15 +0200 Subject: [PATCH 183/242] fix(terminal): report ambiguous steering timeouts --- src/worker/terminal-hub.ts | 37 +++++++++++++++++------- tests/terminal-hub.test.ts | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 5ffcecc8..f14127f8 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -38,11 +38,15 @@ const terminalPreAuthorizationRelayEventMax = 32; type PendingTerminalInputAcknowledgement = { inputId: string; runnerGeneration: number | string; - promise: Promise; - resolve(result: GitHubActionsRelayInputAcknowledgement): void; + promise: Promise; + resolve(result: TerminalInputAcknowledgementResult): void; timeout: ReturnType; }; +type TerminalInputAcknowledgementResult = GitHubActionsRelayInputAcknowledgement & { + deliveryUnknown?: boolean; +}; + export type TerminalUpstream = { socket: WebSocket; markConnected: () => Promise; @@ -121,6 +125,7 @@ export type TerminalHubDependencies = { error?: unknown, ): Promise; markDetached(user: User | null, sessionId: string, message: string): Promise; + inputAcknowledgementTimeoutMs?: number; }; type PendingTerminalSubscription = { @@ -301,6 +306,8 @@ export class TerminalHub { subscription, inputId, subscription.runnerGeneration, + this.dependencies.inputAcknowledgementTimeoutMs ?? + terminalInputAcknowledgementTimeoutMs, ) : null; if (acknowledgement) acknowledgements.push(acknowledgement); @@ -806,9 +813,10 @@ function beginTerminalInputAcknowledgement( subscription: TerminalHubSubscription, inputId: string, runnerGeneration: number | string, + timeoutMs: number, ): PendingTerminalInputAcknowledgement { - let resolve!: (result: GitHubActionsRelayInputAcknowledgement) => void; - const promise = new Promise((complete) => { + let resolve!: (result: TerminalInputAcknowledgementResult) => void; + const promise = new Promise((complete) => { resolve = complete; }); const pending: PendingTerminalInputAcknowledgement = { @@ -820,7 +828,8 @@ function beginTerminalInputAcknowledgement( if ( completeAllTerminalInputAcknowledgements(subscription, { accepted: false, - error: "terminal input delivery was not acknowledged", + deliveryUnknown: true, + error: "terminal input delivery outcome is unknown; the runner may still complete it", }) === 0 ) { return; @@ -828,7 +837,7 @@ function beginTerminalInputAcknowledgement( if (subscription.upstream.readyState === WebSocket.OPEN) { subscription.upstream.close(1011, "input acknowledgement timed out"); } - }, terminalInputAcknowledgementTimeoutMs), + }, timeoutMs), }; subscription.pendingInputAcknowledgements.set(inputId, pending); return pending; @@ -875,7 +884,7 @@ function completeTerminalInputAcknowledgement( function completeAllTerminalInputAcknowledgements( subscription: TerminalHubSubscription, - result: Omit, + result: Omit, ): number { return completeTerminalInputAcknowledgements(subscription, () => true, result); } @@ -883,7 +892,7 @@ function completeAllTerminalInputAcknowledgements( function completeTerminalInputAcknowledgementsBeforeGeneration( subscription: TerminalHubSubscription, runnerGeneration: number, - result: Omit, + result: Omit, ): number { return completeTerminalInputAcknowledgements( subscription, @@ -895,7 +904,7 @@ function completeTerminalInputAcknowledgementsBeforeGeneration( function completeTerminalInputAcknowledgements( subscription: TerminalHubSubscription, matches: (pending: PendingTerminalInputAcknowledgement) => boolean, - result: Omit, + result: Omit, ): number { const pending = [...subscription.pendingInputAcknowledgements.values()].filter(matches); for (const acknowledgement of pending) { @@ -921,7 +930,7 @@ function parseSynchronousGitHubActionsRelayEvent( async function reportTerminalInputCompletion( socket: WebSocket, sessionId: string, - acknowledgements: Promise[], + acknowledgements: Promise[], ): Promise { if (acknowledgements.length === 0) { sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { @@ -931,6 +940,14 @@ async function reportTerminalInputCompletion( } const results = await Promise.all(acknowledgements); if (socket.readyState !== WebSocket.OPEN) return; + const unknown = results.find((result) => result.deliveryUnknown); + if (unknown) { + sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { + type: "input-delivery-unknown", + error: unknown.error ?? "terminal input delivery outcome is unknown", + }); + return; + } const rejection = results.find((result) => !result.accepted); if (rejection) { sendTerminalJson(socket, TerminalMessageType.Event, sessionId, { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 97bcf3bb..b66937db 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -649,6 +649,65 @@ test("GitHub Actions input waits for the correlated runner acknowledgement", asy server.emit("close"); }); +test("GitHub Actions acknowledgement timeout reports an ambiguous delivery outcome", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + inputAcknowledgementTimeoutMs: 1, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("possibly-delivered"), + }), + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushQueues(); + + assert.deepEqual(upstream.closed, [{ code: 1011, reason: "input acknowledgement timed out" }]); + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload)) + .filter( + (message) => + typeof message === "object" && + message !== null && + "type" in message && + String(message.type).startsWith("input-"), + ); + assert.deepEqual(completions, [ + { + type: "input-delivery-unknown", + error: "terminal input delivery outcome is unknown; the runner may still complete it", + }, + ]); + server.emit("close"); +}); + test("GitHub Actions falls back to raw relay input when viewer negotiation is absent", async () => { const client = socket(); const server = socket(); From 339e0ddc2285f1e926564e145ed35227c806d1f2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:42:46 +0200 Subject: [PATCH 184/242] fix(macos): republish rotated desktop identities --- CHANGELOG.md | 2 +- .../PrivateMacShareController.swift | 2 +- .../PrivateMacShareTests.swift | 136 ++++++++++++++++++ 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d0db9f..4ff87ca8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, persisting successful runtime-adapter deletion, validating Apple Remote Desktop Diffie-Hellman groups, and keying desktop publication cleanup by the API host ID; the runner guide now demonstrates negotiated, serialized restricted steering instead of shell access. +- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, persisting successful runtime-adapter deletion, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index f9b9a191..153e3efa 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -69,7 +69,7 @@ final class DesktopHostRegistrationLifecycle { let hostID = CrabfleetDesktopRegistration.hostID(identity: identity) let target = uncertainRegistrations.first { - $0.hostID == hostID && $0.port == port + $0.hostID == hostID && $0.identity == identity && $0.port == port } ?? RegistrationTarget( identity: identity, diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index a4b9270f..2d5ac2b0 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -465,6 +465,106 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func ambiguousDesktopPublicationRetryRecoversOnlyTheExactIdentity() async throws { + let identity = desktopIdentity(name: "retry-publish", address: "100.64.12.50") + let registration = IdentityAwareAmbiguousDesktopRegistration( + uncertainPublicationIDs: ["publication-a"] + ) + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "publication-a" } + ) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + try await lifecycle.publish(identity: identity, port: 5_901) + try await lifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(identity.ipv4Address, 5_901, "publication-a"), + .recover(identity.ipv4Address, "publication-a"), + .unregister(identity.ipv4Address, "recovered:publication-a"), + ] + ) + } + + @Test @MainActor + func addressRotationRepublishesInsteadOfRecoveringAnUncertainDesktop() async throws { + let first = desktopIdentity(name: "rotating-host", address: "100.64.12.51") + let second = TailnetIdentity( + tailnetName: first.tailnetName, + loginName: first.loginName, + dnsName: first.dnsName, + hostName: first.hostName, + ipv4Address: "100.64.12.52", + userID: first.userID + ) + #expect(first != second) + #expect( + CrabfleetDesktopRegistration.hostID(identity: first) + == CrabfleetDesktopRegistration.hostID(identity: second) + ) + let registration = IdentityAwareAmbiguousDesktopRegistration( + uncertainPublicationIDs: ["publication-a"] + ) + var publicationIDs = ["publication-a", "publication-b"] + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { publicationIDs.removeFirst() } + ) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: first, port: 5_901) + } + try await lifecycle.publish(identity: second, port: 5_901) + try await lifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(first.ipv4Address, 5_901, "publication-a"), + .register(second.ipv4Address, 5_901, "publication-b"), + .recover(first.ipv4Address, "publication-a"), + .unregister(first.ipv4Address, "recovered:publication-a"), + .unregister(second.ipv4Address, "token:publication-b"), + ] + ) + } + + @Test @MainActor + func portRotationRepublishesInsteadOfRecoveringAnUncertainDesktop() async throws { + let identity = desktopIdentity(name: "rotating-port", address: "100.64.12.53") + let registration = IdentityAwareAmbiguousDesktopRegistration( + uncertainPublicationIDs: ["publication-a"] + ) + var publicationIDs = ["publication-a", "publication-b"] + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { publicationIDs.removeFirst() } + ) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + try await lifecycle.publish(identity: identity, port: 5_902) + try await lifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(identity.ipv4Address, 5_901, "publication-a"), + .register(identity.ipv4Address, 5_902, "publication-b"), + .recover(identity.ipv4Address, "publication-a"), + .unregister(identity.ipv4Address, "recovered:publication-a"), + .unregister(identity.ipv4Address, "token:publication-b"), + ] + ) + } + @Test @MainActor func ambiguousDesktopCleanupPreservesANewerPublisher() async throws { let identity = desktopIdentity(name: "shared-host", address: "100.64.12.47") @@ -1951,6 +2051,42 @@ private actor AmbiguousDesktopRegistration: DesktopHostRegistering { } } +private actor IdentityAwareAmbiguousDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register(String, UInt16, String) + case recover(String, String) + case unregister(String, String?) + } + + private let uncertainPublicationIDs: Set + private(set) var events: [Event] = [] + + init(uncertainPublicationIDs: Set) { + self.uncertainPublicationIDs = uncertainPublicationIDs + } + + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + events.append(.register(identity.ipv4Address, port, publicationID)) + if uncertainPublicationIDs.contains(publicationID) { + throw DesktopHostRegistrationResultUncertainError(message: "response lost") + } + return "token:\(publicationID)" + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + events.append(.recover(identity.ipv4Address, publicationID)) + return "recovered:\(publicationID)" + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + events.append(.unregister(identity.ipv4Address, ownershipToken)) + } +} + private actor TwoProcessDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String) From 1654b80a16cb5b3c289c2ebfd955d4a5639d615a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:43:37 +0200 Subject: [PATCH 185/242] fix(runtime): persist idempotent cleanup deletes --- ...ime_adapter_cleanup_delete_idempotency.sql | 2 + src/worker/database.ts | 1 + .../runtime-adapter-release-repository.ts | 11 +++ .../runtime-adapter-release-service.ts | 13 ++- src/worker/runtime-adapter-workspaces.ts | 8 +- src/worker/runtime-application.ts | 9 +- tests/runtime-adapter-release-service.test.ts | 92 ++++++++++++++++++- tests/runtime-adapter-workspaces.test.ts | 81 +++++++++++++++- 8 files changed, 205 insertions(+), 12 deletions(-) create mode 100644 migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql diff --git a/migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql b/migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql new file mode 100644 index 00000000..67a50d33 --- /dev/null +++ b/migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql @@ -0,0 +1,2 @@ +ALTER TABLE runtime_adapter_workspace_cleanups + ADD COLUMN delete_idempotency_key TEXT; diff --git a/src/worker/database.ts b/src/worker/database.ts index 971807e5..13ed8620 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -230,6 +230,7 @@ export type RuntimeAdapterWorkspaceCleanupTable = { control_plane: string | null; create_pending: number; deletion_observed: Generated; + delete_idempotency_key: string | null; message: string; reconcile_error: string | null; attempt_count: Generated; diff --git a/src/worker/provisioning/runtime-adapter-release-repository.ts b/src/worker/provisioning/runtime-adapter-release-repository.ts index 2441eb00..6ce8ca41 100644 --- a/src/worker/provisioning/runtime-adapter-release-repository.ts +++ b/src/worker/provisioning/runtime-adapter-release-repository.ts @@ -20,6 +20,7 @@ export async function stageRuntimeAdapterWorkspaceCleanup( now: number; }, ): Promise { + const deleteIdempotencyKey = `runtime-cleanup-delete:${crypto.randomUUID()}`; await sql` INSERT INTO runtime_adapter_workspace_cleanups ( session_id, @@ -27,6 +28,7 @@ export async function stageRuntimeAdapterWorkspaceCleanup( profile, control_plane, create_pending, + delete_idempotency_key, message, reconcile_error, next_attempt_at, @@ -38,6 +40,7 @@ export async function stageRuntimeAdapterWorkspaceCleanup( ${input.registration?.profile ?? null}, ${input.registration?.controlPlane ?? null}, ${input.createPending ? 1 : 0}, + ${deleteIdempotencyKey}, 'superseded runtime adapter cleanup pending', NULL, ${input.now}, @@ -55,11 +58,13 @@ export async function claimRuntimeAdapterWorkspaceCleanup( now: number, ): Promise { const claim = `runtime-cleanup:${crypto.randomUUID()}`; + const deleteIdempotencyKey = `runtime-cleanup-delete:${crypto.randomUUID()}`; const row = await database(env) .updateTable("runtime_adapter_workspace_cleanups") .set({ cleanup_claim: claim, cleanup_claim_expires_at: now + cleanupClaimTtlMs, + delete_idempotency_key: sql`COALESCE(delete_idempotency_key, ${deleteIdempotencyKey})`, attempt_count: sql`attempt_count + 1`, last_attempt_at: now, updated_at: sql`MAX(updated_at + 1, ${now})`, @@ -179,6 +184,12 @@ function cleanupClaim(row: RuntimeAdapterWorkspaceCleanupRow): RuntimeAdapterWor : null, createPending: row.create_pending === 1, deletionObserved: row.deletion_observed === 1, + deleteIdempotencyKey: requiredDeleteIdempotencyKey(row.delete_idempotency_key), claim: row.cleanup_claim ?? "", }; } + +function requiredDeleteIdempotencyKey(value: string | null): string { + if (!value) throw new Error("runtime adapter cleanup delete idempotency key is missing"); + return value; +} diff --git a/src/worker/provisioning/runtime-adapter-release-service.ts b/src/worker/provisioning/runtime-adapter-release-service.ts index 5bd2add3..ca12f893 100644 --- a/src/worker/provisioning/runtime-adapter-release-service.ts +++ b/src/worker/provisioning/runtime-adapter-release-service.ts @@ -11,6 +11,7 @@ export type RuntimeAdapterWorkspaceCleanup = { registration: RuntimeAdapterWorkspaceRegistration | null; createPending: boolean; deletionObserved: boolean; + deleteIdempotencyKey: string; claim: string; }; @@ -42,6 +43,7 @@ export type RuntimeAdapterReleaseServiceDependencies = { adapterWorkspaceId: string, registration: RuntimeAdapterWorkspaceRegistration | null, retryMissing: boolean, + deleteIdempotencyKey: string, ): Promise; confirmRelease( sessionId: string, @@ -94,8 +96,14 @@ export class RuntimeAdapterReleaseService { cleanup: RuntimeAdapterWorkspaceCleanup, now: number, ): Promise { - const { sessionId, adapterWorkspaceId, registration, createPending, deletionObserved } = - cleanup; + const { + sessionId, + adapterWorkspaceId, + registration, + createPending, + deletionObserved, + deleteIdempotencyKey, + } = cleanup; try { if (!createPending) { await this.dependencies.clearCreatePending(sessionId, adapterWorkspaceId); @@ -105,6 +113,7 @@ export class RuntimeAdapterReleaseService { adapterWorkspaceId, registration, createPending && !deletionObserved, + deleteIdempotencyKey, ); if (release.status === "stopped") { await this.dependencies.markCleanupDeletionObserved(cleanup, now); diff --git a/src/worker/runtime-adapter-workspaces.ts b/src/worker/runtime-adapter-workspaces.ts index 54918133..35e321d4 100644 --- a/src/worker/runtime-adapter-workspaces.ts +++ b/src/worker/runtime-adapter-workspaces.ts @@ -189,6 +189,7 @@ export class RuntimeAdapterWorkspaceLifecycle { adapterWorkspaceId: string, retainedRegistration?: RuntimeAdapterWorkspaceRegistration | null, retryMissing?: boolean, + deleteIdempotencyKey?: string, ): Promise { const supersededCleanup = retainedRegistration !== undefined; const registration = retainedRegistration @@ -220,6 +221,7 @@ export class RuntimeAdapterWorkspaceLifecycle { controlPlane, adapterWorkspaceId, supersededCleanup && registration?.adapter_create_pending !== 0, + deleteIdempotencyKey, ); } @@ -530,6 +532,7 @@ export class RuntimeAdapterWorkspaceLifecycle { registeredControlPlane: string, adapterWorkspaceId: string, retryMissing = false, + deleteIdempotencyKey?: string, ): Promise { const controlPlane = requireRegisteredRuntimeAdapterControlPlane( this.env, @@ -538,7 +541,10 @@ export class RuntimeAdapterWorkspaceLifecycle { ); const response = await this.dependencies.fetch( runtimeAdapterWorkspaceUrl(controlPlane, adapterWorkspaceId), - { method: "DELETE" }, + { + method: "DELETE", + ...(deleteIdempotencyKey ? { headers: { "idempotency-key": deleteIdempotencyKey } } : {}), + }, ); const body = response.status === 204 ? null : await this.dependencies.readResponseBody(response); diff --git a/src/worker/runtime-application.ts b/src/worker/runtime-application.ts index 0bad2787..07723566 100644 --- a/src/worker/runtime-application.ts +++ b/src/worker/runtime-application.ts @@ -212,12 +212,19 @@ export class RuntimeApplication { completeCleanup: (cleanup) => completeRuntimeAdapterWorkspaceCleanup(this.env, cleanup), clearCreatePending: (sessionId, adapterWorkspaceId) => clearRuntimeAdapterCreatePending(this.env, sessionId, adapterWorkspaceId), - stopWorkspace: (sessionId, adapterWorkspaceId, registration, retryMissing) => + stopWorkspace: ( + sessionId, + adapterWorkspaceId, + registration, + retryMissing, + deleteIdempotencyKey, + ) => this.workspaceLifecycle().stopForSession( sessionId, adapterWorkspaceId, registration, retryMissing, + deleteIdempotencyKey, ), confirmRelease: (sessionId, adapterWorkspaceId, now, message) => confirmRuntimeAdapterRelease(this.env, sessionId, adapterWorkspaceId, now, message), diff --git a/tests/runtime-adapter-release-service.test.ts b/tests/runtime-adapter-release-service.test.ts index 08723383..a67444fd 100644 --- a/tests/runtime-adapter-release-service.test.ts +++ b/tests/runtime-adapter-release-service.test.ts @@ -28,6 +28,7 @@ const registration: RuntimeAdapterWorkspaceRegistration = { profile: "default", controlPlane: "https://adapter.example.test/", }; +const deleteIdempotencyKey = "runtime-cleanup-delete:test-operation"; type PreparedStatement = { sql: string; @@ -48,6 +49,7 @@ function releaseDependencies( registration: input.registration, createPending: input.createPending, deletionObserved: false, + deleteIdempotencyKey, claim: "claim-1", }; }, @@ -119,15 +121,16 @@ test("superseded release clears the create marker before stopping and confirming registration, createPending: false, deletionObserved: false, + deleteIdempotencyKey, claim: "claim-1", }; }, async clearCreatePending(sessionId, adapterWorkspaceId) { calls.push(`clear:${sessionId}:${adapterWorkspaceId}`); }, - async stopWorkspace(sessionId, adapterWorkspaceId, retained, createPending) { + async stopWorkspace(sessionId, adapterWorkspaceId, retained, createPending, idempotencyKey) { calls.push( - `stop:${sessionId}:${adapterWorkspaceId}:${retained?.profile}:${retained?.controlPlane}:${createPending}`, + `stop:${sessionId}:${adapterWorkspaceId}:${retained?.profile}:${retained?.controlPlane}:${createPending}:${idempotencyKey}`, ); return { status: "stopped", message: "runtime workspace released" }; }, @@ -152,7 +155,7 @@ test("superseded release clears the create marker before stopping and confirming assert.deepEqual(calls, [ "stage:IS-101:fleet-a-is-101", "clear:IS-101:fleet-a-is-101", - "stop:IS-101:fleet-a-is-101:default:https://adapter.example.test/:false", + `stop:IS-101:fleet-a-is-101:default:https://adapter.example.test/:false:${deleteIdempotencyKey}`, "confirm:IS-101:fleet-a-is-101:200:runtime workspace released", "complete:IS-101:fleet-a-is-101", ]); @@ -234,6 +237,7 @@ test("superseded cleanup survives ownership loss and retries only the old worksp registration: input.registration, createPending: input.createPending, deletionObserved: false, + deleteIdempotencyKey, claim: "claim-1", }); }, @@ -303,6 +307,7 @@ test("superseded provider failures remain independently retryable", async () => registration: input.registration, createPending: input.createPending, deletionObserved: false, + deleteIdempotencyKey, claim: "claim-1", }); }, @@ -356,6 +361,7 @@ test("observed create-pending deletion survives completion persistence failure", registration: input.registration, createPending: input.createPending, deletionObserved: false, + deleteIdempotencyKey, claim: "claim-1", }; }, @@ -365,7 +371,8 @@ test("observed create-pending deletion survives completion persistence failure", async claimPendingCleanups() { return cleanup ? [{ ...cleanup, claim: "claim-2" }] : []; }, - async stopWorkspace(_sessionId, _adapterWorkspaceId, _registration, retry) { + async stopWorkspace(_sessionId, _adapterWorkspaceId, _registration, retry, idempotencyKey) { + assert.equal(idempotencyKey, deleteIdempotencyKey); retryMissing.push(retry); return { status: "stopped", @@ -419,6 +426,12 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () "utf8", ), ); + sqlite.exec( + readFileSync( + new URL("../migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql", import.meta.url), + "utf8", + ), + ); const env = sqliteRuntimeEnv(sqlite); await stageRuntimeAdapterWorkspaceCleanup(env, { sessionId: "IS-101", @@ -437,6 +450,7 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () assert.ok(claimed); assert.equal(claimed.createPending, true); assert.equal(claimed.deletionObserved, false); + assert.match(claimed.deleteIdempotencyKey, /^runtime-cleanup-delete:/u); assert.deepEqual(claimed.registration, registration); assert.equal((await claimRuntimeAdapterWorkspaceCleanupBatch(env, 200, 3)).length, 0); @@ -452,6 +466,7 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () const retry = await claimRuntimeAdapterWorkspaceCleanupBatch(env, 15_200, 3); assert.equal(retry.length, 1); assert.equal(retry[0].deletionObserved, true); + assert.equal(retry[0].deleteIdempotencyKey, claimed.deleteIdempotencyKey); await completeRuntimeAdapterWorkspaceCleanup(env, retry[0]); assert.equal( sqlite.prepare("SELECT COUNT(*) AS count FROM runtime_adapter_workspace_cleanups").get()?.count, @@ -459,6 +474,75 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () ); }); +test("legacy cleanup rows receive one stable delete idempotency key on claim", async () => { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec( + readFileSync( + new URL("../migrations/0037_runtime_adapter_workspace_cleanup.sql", import.meta.url), + "utf8", + ), + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0039_runtime_adapter_cleanup_deletion_observed.sql", import.meta.url), + "utf8", + ), + ); + sqlite + .prepare( + `INSERT INTO runtime_adapter_workspace_cleanups ( + session_id, + adapter_workspace_id, + profile, + control_plane, + create_pending, + message, + next_attempt_at, + created_at, + updated_at + ) VALUES (?, ?, ?, ?, 1, ?, 200, 200, 200)`, + ) + .run( + "IS-legacy", + "fleet-a-is-legacy", + registration.profile, + registration.controlPlane, + "superseded runtime adapter cleanup pending", + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql", import.meta.url), + "utf8", + ), + ); + const env = sqliteRuntimeEnv(sqlite); + + const claimed = await claimRuntimeAdapterWorkspaceCleanup( + env, + "IS-legacy", + "fleet-a-is-legacy", + 200, + ); + assert.ok(claimed); + assert.match(claimed.deleteIdempotencyKey, /^runtime-cleanup-delete:/u); + + await persistRuntimeAdapterWorkspaceCleanupEvidence( + env, + claimed, + "provider stop pending", + 200, + null, + ); + const retry = await claimRuntimeAdapterWorkspaceCleanup( + env, + "IS-legacy", + "fleet-a-is-legacy", + 15_200, + ); + assert.ok(retry); + assert.equal(retry.deleteIdempotencyKey, claimed.deleteIdempotencyKey); +}); + test("confirmed release waits for create resolution behind an exact lifecycle fence", async () => { let statements: PreparedStatement[] = []; const effects: string[] = []; diff --git a/tests/runtime-adapter-workspaces.test.ts b/tests/runtime-adapter-workspaces.test.ts index 2104dcf8..ee866431 100644 --- a/tests/runtime-adapter-workspaces.test.ts +++ b/tests/runtime-adapter-workspaces.test.ts @@ -402,13 +402,21 @@ test("superseded stop uses retained registration after the session row moves on" }); test("superseded pending creates retry DELETE until the old workspace becomes visible", async () => { - const requests: Array<{ url: string; method: string | undefined }> = []; + const requests: Array<{ + url: string; + method: string | undefined; + idempotencyKey: string | null; + }> = []; let responseStatus = 404; const service = new RuntimeAdapterWorkspaceLifecycle( runtimeEnv(), dependencies({ async fetch(input, init) { - requests.push({ url: input, method: init.method }); + requests.push({ + url: input, + method: init.method, + idempotencyKey: new Headers(init.headers).get("idempotency-key"), + }); return responseStatus === 204 ? new Response(null, { status: 204 }) : Response.json({ message: "workspace not found" }, { status: responseStatus }); @@ -419,9 +427,16 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi profile: "default", controlPlane: "https://adapter.example.test/", }; + const deleteIdempotencyKey = "runtime-cleanup-delete:test-operation"; assert.deepEqual( - await service.stopForSession("IS-42", "workspace-superseded", registration, true), + await service.stopForSession( + "IS-42", + "workspace-superseded", + registration, + true, + deleteIdempotencyKey, + ), { status: "stopping", message: "runtime adapter workspace not yet visible; cleanup retry pending", @@ -430,7 +445,13 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi responseStatus = 204; assert.deepEqual( - await service.stopForSession("IS-42", "workspace-superseded", registration, true), + await service.stopForSession( + "IS-42", + "workspace-superseded", + registration, + true, + deleteIdempotencyKey, + ), { status: "stopped", message: "runtime adapter workspace released", @@ -440,14 +461,66 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi { url: "https://adapter.example.test/v1/workspaces/workspace-superseded", method: "DELETE", + idempotencyKey: deleteIdempotencyKey, }, { url: "https://adapter.example.test/v1/workspaces/workspace-superseded", method: "DELETE", + idempotencyKey: deleteIdempotencyKey, }, ]); }); +test("superseded cleanup replays a committed DELETE after response loss", async () => { + const deleteIdempotencyKey = "runtime-cleanup-delete:committed-operation"; + const committedDeletes = new Set(); + let loseFirstResponse = true; + const service = new RuntimeAdapterWorkspaceLifecycle( + runtimeEnv(), + dependencies({ + async fetch(_input, init) { + const key = new Headers(init.headers).get("idempotency-key"); + assert.equal(key, deleteIdempotencyKey); + if (loseFirstResponse) { + loseFirstResponse = false; + committedDeletes.add(key); + throw new Error("response lost after delete commit"); + } + assert.equal(committedDeletes.has(key), true); + return new Response(null, { status: 204 }); + }, + }), + ); + const registration = { + profile: "default", + controlPlane: "https://adapter.example.test/", + }; + + await assert.rejects( + service.stopForSession( + "IS-42", + "workspace-superseded", + registration, + true, + deleteIdempotencyKey, + ), + /response lost after delete commit/u, + ); + assert.deepEqual( + await service.stopForSession( + "IS-42", + "workspace-superseded", + registration, + true, + deleteIdempotencyKey, + ), + { + status: "stopped", + message: "runtime adapter workspace released", + }, + ); +}); + test("superseded cleanup accepts missing workspaces after deletion was observed", async () => { const service = new RuntimeAdapterWorkspaceLifecycle( runtimeEnv(), From 160e393a55e4e591677b034efdc83c6f2c8bac6b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:44:37 +0200 Subject: [PATCH 186/242] docs(actions): harden runner input fallback --- docs/github-actions-sessions.md | 145 ++++++++++++++++++++++-------- tests/github-actions-docs.test.ts | 52 +++++++++-- 2 files changed, 150 insertions(+), 47 deletions(-) diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index bae404ae..bfd53b6c 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -290,9 +290,11 @@ if (!runnerPtyUrl) throw new Error("CRABFLEET_RUNNER_PTY_URL is required"); const magic = new Uint8Array([0x43, 0x46, 0x52, 0x31]); // CFR1 const inputIdDecoder = new TextDecoder(); const encoder = new TextEncoder(); -const maxPendingInputBytes = 16 * 1024; -const maxPendingInputFrames = 32; +const maxAdmittedInputBytes = 16 * 1024; +const maxAdmittedInputFrames = 32; const maxPendingInputAgeMs = 1_000; +let admittedInputBytes = 0; +let admittedInputFrames = 0; let pendingInputs = []; let pendingInputBytes = 0; let pendingInputTimer; @@ -304,13 +306,11 @@ await new Promise((resolve, reject) => { terminal.addEventListener("open", resolve, { once: true }); terminal.addEventListener("error", reject, { once: true }); }); -if (terminal.protocol !== "cfr1-framed-io-v2") { - terminal.close(1002, "framed protocol not negotiated"); - throw new Error("relay did not negotiate cfr1-framed-io-v2"); -} +const framed = terminal.protocol === "cfr1-framed-io-v2"; +// An empty protocol means an older relay kept this socket in legacy raw mode. subscribeSteeringOutput((outputText) => { - terminal.send(encodeUtf8Output(outputText)); + terminal.send(framed ? encodeUtf8Output(outputText) : outputText); }); subscribeSteeringExit(() => { @@ -318,22 +318,14 @@ subscribeSteeringExit(() => { }); terminal.addEventListener("message", (event) => { - inputQueue = inputQueue - .then(() => acceptInput(event.data)) - .catch(() => { - settleInputs(takePendingInputs(), false); - }); + const input = admitInput(event.data); + if (!input) return; + inputQueue = inputQueue.then(() => acceptInput(input)); }); -async function acceptInput(data) { - const input = decodeInput(data); - if (!input) return; +async function acceptInput(input) { pendingInputs.push(input); pendingInputBytes += input.payload.byteLength; - if (pendingInputBytes > maxPendingInputBytes || pendingInputs.length > maxPendingInputFrames) { - settleInputs(takePendingInputs(), false); - return; - } const payload = new Uint8Array(pendingInputBytes); let offset = 0; @@ -346,7 +338,7 @@ async function acceptInput(data) { try { text = decodeCompleteUtf8(payload); } catch { - settleInputs(takePendingInputs(), false); + rejectInputs(takePendingInputs(), 1007, "invalid UTF-8 input"); return; } if (text === null) { @@ -358,8 +350,36 @@ async function acceptInput(data) { await deliverSteeringInput(text); settleInputs(inputs, true); } catch { - settleInputs(inputs, false); + rejectInputs(inputs, 1011, "steering rejected input"); + } +} + +function admitInput(data) { + if (!framed && typeof data === "string" && data.length > maxAdmittedInputBytes) { + closeRawOverflow(); + return null; + } + const input = framed ? decodeInput(data) : decodeRawInput(data); + if (!input) return null; + const nextBytes = admittedInputBytes + input.payload.byteLength; + const nextFrames = admittedInputFrames + 1; + if (nextBytes > maxAdmittedInputBytes || nextFrames > maxAdmittedInputFrames) { + if (framed) { + sendAck(input, false); + } else { + closeRawOverflow(); + } + return null; } + admittedInputBytes = nextBytes; + admittedInputFrames = nextFrames; + return input; +} + +function decodeRawInput(data) { + if (typeof data === "string") return { payload: encoder.encode(data) }; + if (data instanceof ArrayBuffer) return { payload: new Uint8Array(data) }; + return null; } function decodeCompleteUtf8(payload) { @@ -371,7 +391,7 @@ function decodeCompleteUtf8(payload) { function armPendingInputTimer() { if (pendingInputTimer) return; pendingInputTimer = setTimeout(() => { - settleInputs(takePendingInputs(), false); + rejectInputs(takePendingInputs(), 1007, "incomplete UTF-8 input"); }, maxPendingInputAgeMs); } @@ -385,8 +405,39 @@ function takePendingInputs() { } function settleInputs(inputs, accepted) { + releaseInputs(inputs); + if (!framed) return; + for (const input of inputs) { + sendAck(input, accepted); + } +} + +function rejectInputs(inputs, rawCloseCode, rawCloseReason) { + settleInputs(inputs, false); + if (!framed && terminal.readyState < WebSocket.CLOSING) { + terminal.close(rawCloseCode, rawCloseReason); + } +} + +function releaseInputs(inputs) { for (const input of inputs) { + admittedInputBytes -= input.payload.byteLength; + } + admittedInputFrames -= inputs.length; +} + +function sendAck(input, accepted) { + if (terminal.readyState !== WebSocket.OPEN) return; + try { terminal.send(encodeAck(input.inputId, input.generation, accepted)); + } catch { + closeSteering(); + } +} + +function closeRawOverflow() { + if (terminal.readyState < WebSocket.CLOSING) { + terminal.close(1009, "input backlog exceeded"); } } @@ -419,7 +470,7 @@ function decodeInput(data) { return { inputId, generation, - payload: frame.slice(generationOffset + 1 + generationBytes), + payload: frame.subarray(generationOffset + 1 + generationBytes), }; } @@ -467,18 +518,29 @@ environment. Await the steering acceptance signal before sending input frame. This Node adapter buffers a valid incomplete UTF-8 suffix together with every affected input ID. It delivers and positively acknowledges those frames only after a later frame completes the sequence. Invalid UTF-8 rejects -the buffered group without delivering any of it. The adapter also rejects the -whole pending group when it exceeds 16 KiB, 32 frames, or one second, bounding -memory, copy work, and acknowledgement latency. - -The protocol query is consumed during connection setup and is not forwarded as -terminal data. There is no capability message or mode transition after the -socket opens. Each `CFR1` frame occupies one binary WebSocket message. At the -wire level, input and output payloads are opaque terminal bytes. The example is -deliberately a UTF-8 text adapter for the integration's string-based steering -surface: it rejects input that is not complete valid UTF-8 and encodes each -output string as UTF-8. Deployments that require lossless arbitrary terminal -bytes must use a byte-oriented restricted steering adapter instead. +the buffered group without delivering any of it. Every message is decoded and +admitted against the shared 16 KiB and 32-frame limits before it enters the +serialized delivery tail. Those counters retain ownership while input is +pending, queued, or blocked in `deliverSteeringInput`, so a stalled steering +call cannot retain an unbounded sequence of `MessageEvent` payloads. Framed +overflow receives a negative acknowledgement; raw overflow closes the socket +because legacy mode has no acknowledgement channel. An incomplete UTF-8 group +expires after one second. + +WebSocket subprotocol selection is fixed during the opening handshake. There is +no capability message or mode transition after the socket opens. Older relays +that ignore the offered subprotocol leave `WebSocket.protocol` empty; the +adapter then receives raw input and sends raw string output. The +`runnerProtocol` query remains compatibility-only for already-deployed runners. +New runners must not add it, close, or reconnect solely because +`WebSocket.protocol` is empty: a query cannot confirm that the relay selected +framed I/O, while the existing socket is the required raw fallback. Each `CFR1` +frame occupies one binary WebSocket message. At the wire level, input and output +payloads are opaque terminal bytes. The example is deliberately a UTF-8 text +adapter for the integration's string-based steering surface: it rejects input +that is not complete valid UTF-8 and encodes each output string as UTF-8. +Deployments that require lossless arbitrary terminal bytes must use a +byte-oriented restricted steering adapter instead. | Offset | Size | Value | | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------ | @@ -510,8 +572,9 @@ Properties: - Multiple browser viewers may remain connected. - Legacy runners open the returned URL unchanged, receive raw viewer input, and send raw output. -- Framed runners add the exact protocol query before connecting, receive framed - input immediately, and wrap every output payload in a `0x04` frame. +- Framed runners offer the exact WebSocket subprotocol before connecting, + confirm its selection through `WebSocket.protocol`, and wrap every output + payload in a `0x04` frame. - Generation-fenced viewers add `viewerProtocol=cfr1-framed-io-v2` before connecting. They receive `CFR1` output plus relay-generated lifecycle and acknowledgement frames regardless of the runner's mode. @@ -520,7 +583,13 @@ Properties: - Legacy viewers omit that query. They receive raw terminal output plus JSON lifecycle and input-acknowledgement messages for compatibility. - Negotiated input produces `input-accepted` only after the correlated runner - acknowledgement. Legacy input reports acceptance after relay delivery. + acknowledgement. A definitive negative acknowledgement produces + `input-rejected`. If the terminal hub's acknowledgement deadline expires + while the runner write may still be in flight, it produces + `input-delivery-unknown`, not + `input-rejected`, because that write may still complete. Legacy input reports + acceptance after relay delivery. The unknown-delivery JSON control event + carries `{"type":"input-delivery-unknown","error":"terminal input delivery outcome is unknown; the runner may still complete it"}`. - Framed viewer lifecycle events remain typed binary frames while no runner is connected. Legacy viewers receive the JSON fallback. - When runner and viewer modes differ, the relay wraps or unwraps terminal diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index 70f05528..f41ece97 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -15,14 +15,22 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.doesNotMatch(readme, /New runners opt into[\s\S]*cfr1-framed-io-v1/); assert.doesNotMatch(readme, /encodeCfr1Output|decodeCfr1Input|encodeCfr1Ack/); assert.match(guide, /new WebSocket\(runnerPtyUrl, "cfr1-framed-io-v2"\)/); - assert.match(guide, /terminal\.protocol !== "cfr1-framed-io-v2"/); + assert.match(guide, /const framed = terminal\.protocol === "cfr1-framed-io-v2"/); + assert.match(guide, /empty protocol means an older relay kept this socket in legacy raw mode/); + assert.match(guide, /terminal\.send\(framed \? encodeUtf8Output\(outputText\) : outputText\)/); + assert.doesNotMatch(guide, /close\(1002, "framed protocol not negotiated"\)/); + assert.doesNotMatch(guide, /throw new Error\("relay did not negotiate cfr1-framed-io-v2"\)/); assert.doesNotMatch(guide, /searchParams\.set\("runnerProtocol"/); + assert.match(guide, /`runnerProtocol` query remains compatibility-only/); + assert.match(guide, /New runners must not add it, close, or reconnect solely because/); + assert.equal(guide.match(/runnerProtocol/g)?.length, 1); assert.match(guide, /let pendingInputs = \[\]/); assert.match(guide, /let inputQueue = Promise\.resolve\(\)/); assert.match( guide, - /inputQueue = inputQueue\s+\.then\(\(\) => acceptInput\(event\.data\)\)\s+\.catch/, + /const input = admitInput\(event\.data\);\s+if \(!input\) return;\s+inputQueue = inputQueue\.then\(\(\) => acceptInput\(input\)\)/, ); + assert.doesNotMatch(guide, /\.then\(\(\) => acceptInput\(event\.data\)\)/); assert.match(guide, /pendingInputs\.push\(input\)/); assert.match(guide, /text = decodeCompleteUtf8\(payload\)/); assert.match(guide, /if \(text === null\) \{\s+armPendingInputTimer\(\);\s+return;/); @@ -30,13 +38,21 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async guide, /const inputs = takePendingInputs\(\);\s+try \{\s+await deliverSteeringInput\(text\);\s+settleInputs\(inputs, true\)/, ); - assert.match(guide, /catch \{\s+settleInputs\(inputs, false\);/); - assert.match(guide, /const maxPendingInputBytes = 16 \* 1024/); - assert.match(guide, /const maxPendingInputFrames = 32/); + assert.match(guide, /catch \{\s+rejectInputs\(inputs, 1011, "steering rejected input"\);/); + assert.match(guide, /const maxAdmittedInputBytes = 16 \* 1024/); + assert.match(guide, /const maxAdmittedInputFrames = 32/); assert.match(guide, /const maxPendingInputAgeMs = 1_000/); - assert.match(guide, /settleInputs\(takePendingInputs\(\), false\)/); - assert.match(guide, /pendingInputBytes > maxPendingInputBytes/); - assert.match(guide, /pendingInputs\.length > maxPendingInputFrames/); + assert.match(guide, /const nextBytes = admittedInputBytes \+ input\.payload\.byteLength/); + assert.match(guide, /const nextFrames = admittedInputFrames \+ 1/); + assert.match(guide, /nextBytes > maxAdmittedInputBytes/); + assert.match(guide, /nextFrames > maxAdmittedInputFrames/); + assert.match(guide, /if \(framed\) \{\s+sendAck\(input, false\);/); + assert.match(guide, /terminal\.close\(1009, "input backlog exceeded"\)/); + assert.match(guide, /function decodeRawInput\(data\)/); + assert.match(guide, /typeof data === "string"/); + assert.match(guide, /data instanceof ArrayBuffer/); + assert.match(guide, /admittedInputBytes -= input\.payload\.byteLength/); + assert.match(guide, /admittedInputFrames -= inputs\.length/); assert.match(guide, /clearTimeout\(pendingInputTimer\)/); assert.match( guide, @@ -46,17 +62,35 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.doesNotMatch(guide, /inputDecoder\.decode/); assert.match(guide, /subscribeSteeringOutput\(\(outputText\) => \{/); assert.match(guide, /encodeUtf8Output\(outputText\)/); - assert.match(guide, /deliberately a UTF-8 text adapter/); + assert.match(guide, /deliberately\s+a UTF-8 text\s+adapter/); assert.match(guide, /byte-oriented restricted steering adapter/); assert.match(guide, /Generation-fenced viewers add `viewerProtocol=cfr1-framed-io-v2`/); assert.match(guide, /stale-generation input is rejected before it\s+can reach/); assert.match(guide, /encodeAck\(input\.inputId, input\.generation, accepted\)/); assert.match(guide, /Legacy viewers omit that query/); + assert.match( + guide, + /acknowledgement deadline expires\s+while the runner write may still be in flight[\s\S]*`input-delivery-unknown`, not\s+`input-rejected`, because that write may still complete/, + ); + assert.match( + guide, + /\{"type":"input-delivery-unknown","error":"terminal input delivery outcome is unknown; the runner may still complete it"\}/, + ); assert.match(guide, /subscribeSteeringExit\(\(\) => \{/); assert.match(guide, /terminal\.close\(1000, "pty exited"\)/); assert.match(guide, /must never forward that input to a\s+shell or subprocess/); assert.doesNotMatch(guide, /spawn\(process\.env\.SHELL|env:\s*process\.env|pty\.write/); + const messageHandler = guide.indexOf('terminal.addEventListener("message"'); + const admission = guide.indexOf("const input = admitInput(event.data);", messageHandler); + const serialization = guide.indexOf( + "inputQueue = inputQueue.then(() => acceptInput(input));", + messageHandler, + ); + assert.ok(messageHandler >= 0); + assert.ok(admission > messageHandler); + assert.ok(serialization > admission); + const timerArm = guide.indexOf("armPendingInputTimer();"); const batchSnapshot = guide.indexOf("const inputs = takePendingInputs();"); const delivery = guide.indexOf("await deliverSteeringInput(text);"); From 790f7cf51df1b1391b3c06df846aad50bede9e98 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:48:12 +0200 Subject: [PATCH 187/242] fix(runtime): require replayable delete tombstones --- docs/api.md | 2 +- docs/architecture.md | 3 +- ...ime_adapter_cleanup_delete_idempotency.sql | 2 - src/worker/database.ts | 1 - .../runtime-adapter-release-repository.ts | 11 -- .../runtime-adapter-release-service.ts | 13 +- src/worker/runtime-adapter-workspaces.ts | 10 +- src/worker/runtime-application.ts | 9 +- tests/runtime-adapter-release-service.test.ts | 171 +++++++++--------- tests/runtime-adapter-workspaces.test.ts | 81 +++------ 10 files changed, 119 insertions(+), 184 deletions(-) delete mode 100644 migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql diff --git a/docs/api.md b/docs/api.md index 998f43ac..a6798f8a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -507,7 +507,7 @@ Crabfleet authenticates every adapter request with `Authorization: Bearer CRABBO - `POST /v1/workspaces`: idempotent create. Crabfleet persists the deterministic adapter identity, TTL, idle timeout, requested capabilities, and exact serialized create payload before the request, then sends the same namespaced DNS-safe lowercase `id` and `Idempotency-Key`, plus repo, branch, runtime, opaque profile, command, prompt, ownership/lineage, and lifecycle settings. A definitive non-2xx response to the initial request is read once, sanitized, and durably recorded as the failure reason before provider release begins. After an ambiguous result, a bounded reconciliation pass retries only that immutable payload and key before any inspect; later edits to session metadata do not alter it. Replay-time authentication, routing, validation, or other non-success responses cannot prove the original request failed and therefore keep create ambiguity pending. - An adapter that finds the requested ID already bound to a different immutable request returns `409` with `error.code = "workspace_id_conflict"`. Crabfleet marks only its local session failed and atomically drops that adapter identity when the exact pending create attempt still owns the lifecycle revision and reconciliation claim; a stale conflict response is ignored. It never adopts, inspects, or deletes the pre-existing workspace. Other `409` responses remain ambiguous and retryable. - `GET /v1/workspaces/:id`: inspect current status, capabilities, terminal URL, expiry, and provider resource identity. Status-only responses preserve previously stored capabilities and expiry; explicit `null` clears those fields. Active external sessions are reconciled in bounded batches; state responses wait only for a short foreground budget while remaining work continues in the Worker background. -- `DELETE /v1/workspaces/:id`: stop/release. Crabfleet enters `stopping` before calling the adapter and marks the session stopped only after `204`, `404`, or a valid exact-ID terminal response confirms release; malformed successful bodies remain `stopping`. Plain-text and malformed-JSON responses are read once and sanitized before their evidence is retained. An explicit stop whose ownership claim loses returns success only when the exact workspace is already stopping or terminal; otherwise it returns a lifecycle conflict. +- `DELETE /v1/workspaces/:id`: idempotent stop/release. Before provider release, the adapter must durably retain the exact workspace identity and stopping intent. Once a DELETE is accepted, every retry for that immutable ID must return `204` or a valid exact-ID `stopping`, `stopped`, or `expired` response, including after provider deletion or adapter restart; it must not collapse that lifecycle tombstone into `404`. This lets a caller recover when deletion commits but the response is lost without treating a pre-visibility `404` from an ambiguous create as release proof. Crabfleet enters `stopping` before calling the adapter and marks the session stopped only after `204`, a `404` when create ambiguity is absent or prior deletion evidence is durable, or a valid exact-ID terminal response confirms release; malformed successful bodies remain `stopping`. Plain-text and malformed-JSON responses are read once and sanitized before their evidence is retained. An explicit stop whose ownership claim loses returns success only when the exact workspace is already stopping or terminal; otherwise it returns a lifecycle conflict. - `POST /v1/workspaces/:id/connections/desktop`: mint a current transient desktop URL. The request has no body. `expiresAt` is optional; when present it must be in the future and no more than 15 minutes away. Accepted HTTPS URLs are treated as opaque signed connection material and redirected byte-for-byte without URL normalization. After minting, Crabfleet re-reads the exact current session status, control grant, capabilities, and registered adapter identity before redirecting; a concurrent stop, revocation, capability withdrawal, or lifecycle replacement discards the URL and denies access. - `POST /v1/workspaces/:id/connections/native-vnc`: mint a short-lived, single-use native VNC grant. The response must use the `crabbox/native-vnc-grant/v1` schema, an HTTPS broker URL (literal loopback HTTP is allowed for development), the exact opaque lease ID, a 32-byte-hex `native_vnc_` ticket, and an expiry no more than two minutes away. Crabfleet never exposes the provider lease ID in Fleet state and requests this grant only after revalidating current session control and the persisted adapter identity. diff --git a/docs/architecture.md b/docs/architecture.md index 55201059..52dc6317 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -157,7 +157,7 @@ An optional `CRABFLEET_RUNTIME_PROFILES_JSON` allowlist exposes generic Crabbox - `POST /v1/workspaces`: idempotent create using an immutable namespaced ID and request snapshot. - `GET /v1/workspaces/:id`: inspect status, capabilities, expiry, provider identity, and terminal connection. Native-only VNC uses a separate `nativeVnc` capability and does not imply the browser desktop endpoint. -- `DELETE /v1/workspaces/:id`: release the provider workspace. +- `DELETE /v1/workspaces/:id`: idempotently release the provider workspace while retaining an exact-ID stopping or terminal tombstone for retries after response loss. - `POST /v1/workspaces/:id/connections/desktop`: mint a short-lived desktop URL. Important invariants: @@ -167,6 +167,7 @@ Important invariants: - Redirects are rejected so bearer credentials cannot cross origins. - Request/response bodies are bounded before parsing. - Create ambiguity replays the exact original idempotent request before inspection. +- Delete retries replay the retained exact-ID stopping or terminal lifecycle instead of degrading to an ambiguous 404. - An explicit workspace-ID conflict never adopts or deletes the existing provider workspace. - Provider failure is not terminal until DELETE confirms release. - Status, capabilities, expiry, terminal state, and cleanup use compare-and-swap ownership fences. diff --git a/migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql b/migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql deleted file mode 100644 index 67a50d33..00000000 --- a/migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE runtime_adapter_workspace_cleanups - ADD COLUMN delete_idempotency_key TEXT; diff --git a/src/worker/database.ts b/src/worker/database.ts index 13ed8620..971807e5 100644 --- a/src/worker/database.ts +++ b/src/worker/database.ts @@ -230,7 +230,6 @@ export type RuntimeAdapterWorkspaceCleanupTable = { control_plane: string | null; create_pending: number; deletion_observed: Generated; - delete_idempotency_key: string | null; message: string; reconcile_error: string | null; attempt_count: Generated; diff --git a/src/worker/provisioning/runtime-adapter-release-repository.ts b/src/worker/provisioning/runtime-adapter-release-repository.ts index 6ce8ca41..2441eb00 100644 --- a/src/worker/provisioning/runtime-adapter-release-repository.ts +++ b/src/worker/provisioning/runtime-adapter-release-repository.ts @@ -20,7 +20,6 @@ export async function stageRuntimeAdapterWorkspaceCleanup( now: number; }, ): Promise { - const deleteIdempotencyKey = `runtime-cleanup-delete:${crypto.randomUUID()}`; await sql` INSERT INTO runtime_adapter_workspace_cleanups ( session_id, @@ -28,7 +27,6 @@ export async function stageRuntimeAdapterWorkspaceCleanup( profile, control_plane, create_pending, - delete_idempotency_key, message, reconcile_error, next_attempt_at, @@ -40,7 +38,6 @@ export async function stageRuntimeAdapterWorkspaceCleanup( ${input.registration?.profile ?? null}, ${input.registration?.controlPlane ?? null}, ${input.createPending ? 1 : 0}, - ${deleteIdempotencyKey}, 'superseded runtime adapter cleanup pending', NULL, ${input.now}, @@ -58,13 +55,11 @@ export async function claimRuntimeAdapterWorkspaceCleanup( now: number, ): Promise { const claim = `runtime-cleanup:${crypto.randomUUID()}`; - const deleteIdempotencyKey = `runtime-cleanup-delete:${crypto.randomUUID()}`; const row = await database(env) .updateTable("runtime_adapter_workspace_cleanups") .set({ cleanup_claim: claim, cleanup_claim_expires_at: now + cleanupClaimTtlMs, - delete_idempotency_key: sql`COALESCE(delete_idempotency_key, ${deleteIdempotencyKey})`, attempt_count: sql`attempt_count + 1`, last_attempt_at: now, updated_at: sql`MAX(updated_at + 1, ${now})`, @@ -184,12 +179,6 @@ function cleanupClaim(row: RuntimeAdapterWorkspaceCleanupRow): RuntimeAdapterWor : null, createPending: row.create_pending === 1, deletionObserved: row.deletion_observed === 1, - deleteIdempotencyKey: requiredDeleteIdempotencyKey(row.delete_idempotency_key), claim: row.cleanup_claim ?? "", }; } - -function requiredDeleteIdempotencyKey(value: string | null): string { - if (!value) throw new Error("runtime adapter cleanup delete idempotency key is missing"); - return value; -} diff --git a/src/worker/provisioning/runtime-adapter-release-service.ts b/src/worker/provisioning/runtime-adapter-release-service.ts index ca12f893..5bd2add3 100644 --- a/src/worker/provisioning/runtime-adapter-release-service.ts +++ b/src/worker/provisioning/runtime-adapter-release-service.ts @@ -11,7 +11,6 @@ export type RuntimeAdapterWorkspaceCleanup = { registration: RuntimeAdapterWorkspaceRegistration | null; createPending: boolean; deletionObserved: boolean; - deleteIdempotencyKey: string; claim: string; }; @@ -43,7 +42,6 @@ export type RuntimeAdapterReleaseServiceDependencies = { adapterWorkspaceId: string, registration: RuntimeAdapterWorkspaceRegistration | null, retryMissing: boolean, - deleteIdempotencyKey: string, ): Promise; confirmRelease( sessionId: string, @@ -96,14 +94,8 @@ export class RuntimeAdapterReleaseService { cleanup: RuntimeAdapterWorkspaceCleanup, now: number, ): Promise { - const { - sessionId, - adapterWorkspaceId, - registration, - createPending, - deletionObserved, - deleteIdempotencyKey, - } = cleanup; + const { sessionId, adapterWorkspaceId, registration, createPending, deletionObserved } = + cleanup; try { if (!createPending) { await this.dependencies.clearCreatePending(sessionId, adapterWorkspaceId); @@ -113,7 +105,6 @@ export class RuntimeAdapterReleaseService { adapterWorkspaceId, registration, createPending && !deletionObserved, - deleteIdempotencyKey, ); if (release.status === "stopped") { await this.dependencies.markCleanupDeletionObserved(cleanup, now); diff --git a/src/worker/runtime-adapter-workspaces.ts b/src/worker/runtime-adapter-workspaces.ts index 35e321d4..f25f0631 100644 --- a/src/worker/runtime-adapter-workspaces.ts +++ b/src/worker/runtime-adapter-workspaces.ts @@ -189,7 +189,6 @@ export class RuntimeAdapterWorkspaceLifecycle { adapterWorkspaceId: string, retainedRegistration?: RuntimeAdapterWorkspaceRegistration | null, retryMissing?: boolean, - deleteIdempotencyKey?: string, ): Promise { const supersededCleanup = retainedRegistration !== undefined; const registration = retainedRegistration @@ -221,7 +220,6 @@ export class RuntimeAdapterWorkspaceLifecycle { controlPlane, adapterWorkspaceId, supersededCleanup && registration?.adapter_create_pending !== 0, - deleteIdempotencyKey, ); } @@ -532,7 +530,6 @@ export class RuntimeAdapterWorkspaceLifecycle { registeredControlPlane: string, adapterWorkspaceId: string, retryMissing = false, - deleteIdempotencyKey?: string, ): Promise { const controlPlane = requireRegisteredRuntimeAdapterControlPlane( this.env, @@ -541,10 +538,7 @@ export class RuntimeAdapterWorkspaceLifecycle { ); const response = await this.dependencies.fetch( runtimeAdapterWorkspaceUrl(controlPlane, adapterWorkspaceId), - { - method: "DELETE", - ...(deleteIdempotencyKey ? { headers: { "idempotency-key": deleteIdempotencyKey } } : {}), - }, + { method: "DELETE" }, ); const body = response.status === 204 ? null : await this.dependencies.readResponseBody(response); @@ -560,6 +554,8 @@ export class RuntimeAdapterWorkspaceLifecycle { parsed?.message ?? redactedAdapterResponseMessage(body, fallbackMessage, [adapterWorkspaceId]); if (response.status === 404 && retryMissing) { + // An ambiguous create may still appear. Accepted DELETE retries must replay + // the adapter's retained stopping or terminal tombstone instead. return { status: "stopping", message: "runtime adapter workspace not yet visible; cleanup retry pending", diff --git a/src/worker/runtime-application.ts b/src/worker/runtime-application.ts index 07723566..0bad2787 100644 --- a/src/worker/runtime-application.ts +++ b/src/worker/runtime-application.ts @@ -212,19 +212,12 @@ export class RuntimeApplication { completeCleanup: (cleanup) => completeRuntimeAdapterWorkspaceCleanup(this.env, cleanup), clearCreatePending: (sessionId, adapterWorkspaceId) => clearRuntimeAdapterCreatePending(this.env, sessionId, adapterWorkspaceId), - stopWorkspace: ( - sessionId, - adapterWorkspaceId, - registration, - retryMissing, - deleteIdempotencyKey, - ) => + stopWorkspace: (sessionId, adapterWorkspaceId, registration, retryMissing) => this.workspaceLifecycle().stopForSession( sessionId, adapterWorkspaceId, registration, retryMissing, - deleteIdempotencyKey, ), confirmRelease: (sessionId, adapterWorkspaceId, now, message) => confirmRuntimeAdapterRelease(this.env, sessionId, adapterWorkspaceId, now, message), diff --git a/tests/runtime-adapter-release-service.test.ts b/tests/runtime-adapter-release-service.test.ts index a67444fd..d00cf163 100644 --- a/tests/runtime-adapter-release-service.test.ts +++ b/tests/runtime-adapter-release-service.test.ts @@ -28,7 +28,6 @@ const registration: RuntimeAdapterWorkspaceRegistration = { profile: "default", controlPlane: "https://adapter.example.test/", }; -const deleteIdempotencyKey = "runtime-cleanup-delete:test-operation"; type PreparedStatement = { sql: string; @@ -49,7 +48,6 @@ function releaseDependencies( registration: input.registration, createPending: input.createPending, deletionObserved: false, - deleteIdempotencyKey, claim: "claim-1", }; }, @@ -121,16 +119,15 @@ test("superseded release clears the create marker before stopping and confirming registration, createPending: false, deletionObserved: false, - deleteIdempotencyKey, claim: "claim-1", }; }, async clearCreatePending(sessionId, adapterWorkspaceId) { calls.push(`clear:${sessionId}:${adapterWorkspaceId}`); }, - async stopWorkspace(sessionId, adapterWorkspaceId, retained, createPending, idempotencyKey) { + async stopWorkspace(sessionId, adapterWorkspaceId, retained, createPending) { calls.push( - `stop:${sessionId}:${adapterWorkspaceId}:${retained?.profile}:${retained?.controlPlane}:${createPending}:${idempotencyKey}`, + `stop:${sessionId}:${adapterWorkspaceId}:${retained?.profile}:${retained?.controlPlane}:${createPending}`, ); return { status: "stopped", message: "runtime workspace released" }; }, @@ -155,7 +152,7 @@ test("superseded release clears the create marker before stopping and confirming assert.deepEqual(calls, [ "stage:IS-101:fleet-a-is-101", "clear:IS-101:fleet-a-is-101", - `stop:IS-101:fleet-a-is-101:default:https://adapter.example.test/:false:${deleteIdempotencyKey}`, + "stop:IS-101:fleet-a-is-101:default:https://adapter.example.test/:false", "confirm:IS-101:fleet-a-is-101:200:runtime workspace released", "complete:IS-101:fleet-a-is-101", ]); @@ -237,7 +234,6 @@ test("superseded cleanup survives ownership loss and retries only the old worksp registration: input.registration, createPending: input.createPending, deletionObserved: false, - deleteIdempotencyKey, claim: "claim-1", }); }, @@ -307,7 +303,6 @@ test("superseded provider failures remain independently retryable", async () => registration: input.registration, createPending: input.createPending, deletionObserved: false, - deleteIdempotencyKey, claim: "claim-1", }); }, @@ -348,6 +343,85 @@ test("superseded provider failures remain independently retryable", async () => assert.equal(cleanupRows.length, 0); }); +test("create-pending cleanup recovers from a crash before DELETE success is persisted", async () => { + let cleanup: RuntimeAdapterWorkspaceCleanup | null = null; + const retryMissing: boolean[] = []; + let stopAttempt = 0; + let deletionPersistenceAttempt = 0; + const service = new RuntimeAdapterReleaseService( + releaseDependencies({ + async stageCleanup(input) { + cleanup = { + sessionId: input.sessionId, + adapterWorkspaceId: input.adapterWorkspaceId, + registration: input.registration, + createPending: input.createPending, + deletionObserved: false, + claim: "claim-1", + }; + }, + async claimCleanup() { + return cleanup; + }, + async claimPendingCleanups() { + return cleanup ? [{ ...cleanup, claim: `claim-${stopAttempt + 1}` }] : []; + }, + async stopWorkspace(_sessionId, _adapterWorkspaceId, _registration, retry) { + retryMissing.push(retry); + stopAttempt += 1; + if (stopAttempt === 1) { + return { + status: "stopping", + message: "runtime adapter workspace not yet visible; cleanup retry pending", + }; + } + return { + status: "stopped", + message: + stopAttempt === 2 + ? "runtime adapter workspace released" + : "workspace stopped tombstone", + }; + }, + async persistCleanupEvidence(current) { + cleanup = { ...current, claim: "" }; + }, + async markCleanupDeletionObserved(current) { + deletionPersistenceAttempt += 1; + if (deletionPersistenceAttempt === 1) { + throw new Error("crash before DELETE success persistence"); + } + cleanup = { ...current, deletionObserved: true }; + }, + async completeCleanup() { + cleanup = null; + }, + providerError(error) { + assert.ok(error instanceof Error); + return error.message; + }, + }), + ); + + await service.stopSuperseded({ + sessionId: "IS-101", + adapterWorkspaceId: "fleet-a-is-101-old", + registration, + createPending: true, + now: 200, + }); + assert.ok(cleanup); + + await service.retryPending(15_200); + assert.ok(cleanup); + + await service.retryPending(30_200); + + assert.deepEqual(retryMissing, [true, true, true]); + assert.equal(deletionPersistenceAttempt, 2); + assert.equal(cleanup, null); +}); + test("observed create-pending deletion survives completion persistence failure", async () => { let cleanup: RuntimeAdapterWorkspaceCleanup | null = null; const retryMissing: boolean[] = []; @@ -361,7 +435,6 @@ test("observed create-pending deletion survives completion persistence failure", registration: input.registration, createPending: input.createPending, deletionObserved: false, - deleteIdempotencyKey, claim: "claim-1", }; }, @@ -371,8 +444,7 @@ test("observed create-pending deletion survives completion persistence failure", async claimPendingCleanups() { return cleanup ? [{ ...cleanup, claim: "claim-2" }] : []; }, - async stopWorkspace(_sessionId, _adapterWorkspaceId, _registration, retry, idempotencyKey) { - assert.equal(idempotencyKey, deleteIdempotencyKey); + async stopWorkspace(_sessionId, _adapterWorkspaceId, _registration, retry) { retryMissing.push(retry); return { status: "stopped", @@ -426,12 +498,6 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () "utf8", ), ); - sqlite.exec( - readFileSync( - new URL("../migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql", import.meta.url), - "utf8", - ), - ); const env = sqliteRuntimeEnv(sqlite); await stageRuntimeAdapterWorkspaceCleanup(env, { sessionId: "IS-101", @@ -450,7 +516,6 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () assert.ok(claimed); assert.equal(claimed.createPending, true); assert.equal(claimed.deletionObserved, false); - assert.match(claimed.deleteIdempotencyKey, /^runtime-cleanup-delete:/u); assert.deepEqual(claimed.registration, registration); assert.equal((await claimRuntimeAdapterWorkspaceCleanupBatch(env, 200, 3)).length, 0); @@ -466,7 +531,6 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () const retry = await claimRuntimeAdapterWorkspaceCleanupBatch(env, 15_200, 3); assert.equal(retry.length, 1); assert.equal(retry[0].deletionObserved, true); - assert.equal(retry[0].deleteIdempotencyKey, claimed.deleteIdempotencyKey); await completeRuntimeAdapterWorkspaceCleanup(env, retry[0]); assert.equal( sqlite.prepare("SELECT COUNT(*) AS count FROM runtime_adapter_workspace_cleanups").get()?.count, @@ -474,75 +538,6 @@ test("runtime adapter cleanup storage is independent and claim fenced", async () ); }); -test("legacy cleanup rows receive one stable delete idempotency key on claim", async () => { - const sqlite = new DatabaseSync(":memory:"); - sqlite.exec( - readFileSync( - new URL("../migrations/0037_runtime_adapter_workspace_cleanup.sql", import.meta.url), - "utf8", - ), - ); - sqlite.exec( - readFileSync( - new URL("../migrations/0039_runtime_adapter_cleanup_deletion_observed.sql", import.meta.url), - "utf8", - ), - ); - sqlite - .prepare( - `INSERT INTO runtime_adapter_workspace_cleanups ( - session_id, - adapter_workspace_id, - profile, - control_plane, - create_pending, - message, - next_attempt_at, - created_at, - updated_at - ) VALUES (?, ?, ?, ?, 1, ?, 200, 200, 200)`, - ) - .run( - "IS-legacy", - "fleet-a-is-legacy", - registration.profile, - registration.controlPlane, - "superseded runtime adapter cleanup pending", - ); - sqlite.exec( - readFileSync( - new URL("../migrations/0040_runtime_adapter_cleanup_delete_idempotency.sql", import.meta.url), - "utf8", - ), - ); - const env = sqliteRuntimeEnv(sqlite); - - const claimed = await claimRuntimeAdapterWorkspaceCleanup( - env, - "IS-legacy", - "fleet-a-is-legacy", - 200, - ); - assert.ok(claimed); - assert.match(claimed.deleteIdempotencyKey, /^runtime-cleanup-delete:/u); - - await persistRuntimeAdapterWorkspaceCleanupEvidence( - env, - claimed, - "provider stop pending", - 200, - null, - ); - const retry = await claimRuntimeAdapterWorkspaceCleanup( - env, - "IS-legacy", - "fleet-a-is-legacy", - 15_200, - ); - assert.ok(retry); - assert.equal(retry.deleteIdempotencyKey, claimed.deleteIdempotencyKey); -}); - test("confirmed release waits for create resolution behind an exact lifecycle fence", async () => { let statements: PreparedStatement[] = []; const effects: string[] = []; diff --git a/tests/runtime-adapter-workspaces.test.ts b/tests/runtime-adapter-workspaces.test.ts index ee866431..631109a9 100644 --- a/tests/runtime-adapter-workspaces.test.ts +++ b/tests/runtime-adapter-workspaces.test.ts @@ -402,21 +402,13 @@ test("superseded stop uses retained registration after the session row moves on" }); test("superseded pending creates retry DELETE until the old workspace becomes visible", async () => { - const requests: Array<{ - url: string; - method: string | undefined; - idempotencyKey: string | null; - }> = []; + const requests: Array<{ url: string; method: string | undefined }> = []; let responseStatus = 404; const service = new RuntimeAdapterWorkspaceLifecycle( runtimeEnv(), dependencies({ async fetch(input, init) { - requests.push({ - url: input, - method: init.method, - idempotencyKey: new Headers(init.headers).get("idempotency-key"), - }); + requests.push({ url: input, method: init.method }); return responseStatus === 204 ? new Response(null, { status: 204 }) : Response.json({ message: "workspace not found" }, { status: responseStatus }); @@ -427,16 +419,9 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi profile: "default", controlPlane: "https://adapter.example.test/", }; - const deleteIdempotencyKey = "runtime-cleanup-delete:test-operation"; assert.deepEqual( - await service.stopForSession( - "IS-42", - "workspace-superseded", - registration, - true, - deleteIdempotencyKey, - ), + await service.stopForSession("IS-42", "workspace-superseded", registration, true), { status: "stopping", message: "runtime adapter workspace not yet visible; cleanup retry pending", @@ -445,13 +430,7 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi responseStatus = 204; assert.deepEqual( - await service.stopForSession( - "IS-42", - "workspace-superseded", - registration, - true, - deleteIdempotencyKey, - ), + await service.stopForSession("IS-42", "workspace-superseded", registration, true), { status: "stopped", message: "runtime adapter workspace released", @@ -461,33 +440,32 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi { url: "https://adapter.example.test/v1/workspaces/workspace-superseded", method: "DELETE", - idempotencyKey: deleteIdempotencyKey, }, { url: "https://adapter.example.test/v1/workspaces/workspace-superseded", method: "DELETE", - idempotencyKey: deleteIdempotencyKey, }, ]); }); -test("superseded cleanup replays a committed DELETE after response loss", async () => { - const deleteIdempotencyKey = "runtime-cleanup-delete:committed-operation"; - const committedDeletes = new Set(); - let loseFirstResponse = true; +test("create-pending cleanup recovers a lost DELETE response from the terminal tombstone", async () => { + let attempt = 0; const service = new RuntimeAdapterWorkspaceLifecycle( runtimeEnv(), dependencies({ - async fetch(_input, init) { - const key = new Headers(init.headers).get("idempotency-key"); - assert.equal(key, deleteIdempotencyKey); - if (loseFirstResponse) { - loseFirstResponse = false; - committedDeletes.add(key); + async fetch() { + attempt += 1; + if (attempt === 1) { + return Response.json({ message: "workspace not found" }, { status: 404 }); + } + if (attempt === 2) { throw new Error("response lost after delete commit"); } - assert.equal(committedDeletes.has(key), true); - return new Response(null, { status: 204 }); + return Response.json({ + id: "workspace-superseded", + status: "stopped", + message: "workspace stopped", + }); }, }), ); @@ -496,27 +474,22 @@ test("superseded cleanup replays a committed DELETE after response loss", async controlPlane: "https://adapter.example.test/", }; + assert.deepEqual( + await service.stopForSession("IS-42", "workspace-superseded", registration, true), + { + status: "stopping", + message: "runtime adapter workspace not yet visible; cleanup retry pending", + }, + ); await assert.rejects( - service.stopForSession( - "IS-42", - "workspace-superseded", - registration, - true, - deleteIdempotencyKey, - ), + service.stopForSession("IS-42", "workspace-superseded", registration, true), /response lost after delete commit/u, ); assert.deepEqual( - await service.stopForSession( - "IS-42", - "workspace-superseded", - registration, - true, - deleteIdempotencyKey, - ), + await service.stopForSession("IS-42", "workspace-superseded", registration, true), { status: "stopped", - message: "runtime adapter workspace released", + message: "workspace stopped", }, ); }); From 9ec7b69ba39795790d65fbff528dd95e6f29ed67 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:48:44 +0200 Subject: [PATCH 188/242] docs(changelog): record final recovery contracts --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff87ca8..3fa02d31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, persisting successful runtime-adapter deletion, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. +- Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, requiring replayable runtime-adapter deletion tombstones, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. From 55acdb5e80754efc4567111ab00645ac773d6228 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:00:11 +0200 Subject: [PATCH 189/242] test(docs): verify embedded spec generation --- .gitignore | 1 + tests/generated-assets.test.ts | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/generated-assets.test.ts diff --git a/.gitignore b/.gitignore index 91afe0bc..b7e803ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules src/spec.generated.ts +# Rebuilt from canonical sources by pnpm check/build/deploy; do not commit. src/generated.ts dist/ .wrangler/ diff --git a/tests/generated-assets.test.ts b/tests/generated-assets.test.ts new file mode 100644 index 00000000..041baf12 --- /dev/null +++ b/tests/generated-assets.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { SPEC_MARKDOWN } from "../src/generated.ts"; + +test("generated embedded specification matches the canonical markdown", async () => { + const source = await readFile(new URL("../docs/spec.md", import.meta.url), "utf8"); + const markdown = source.replace(/^---\n[\s\S]*?\n---\n+/, ""); + + assert.equal(SPEC_MARKDOWN, markdown); + assert.match(SPEC_MARKDOWN, /relay-generation-fenced binary `CFR1` input/); +}); From 43e61bcae0f1799973d9977974fb6b51c50f0517 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:01:20 +0200 Subject: [PATCH 190/242] fix(credentials): fence staging claim renewal --- CHANGELOG.md | 2 +- ...ndbox-credential-policy-cleanup-service.ts | 1 + .../sandbox-credential-policy-repository.ts | 3 + .../sandbox-credential-policy-scanner.ts | 7 +- src/worker/session-control-policy.ts | 9 ++- .../sandbox-credential-policy-cleanup.test.ts | 4 +- ...ndbox-credential-policy-repository.test.ts | 75 +++++++++++++++++-- 7 files changed, 88 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fa02d31..96c9dcf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, requiring replayable runtime-adapter deletion tombstones, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. -- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. +- Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes with exact current-identity fallback for mixed-version rows, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. diff --git a/src/worker/sandbox-credential-policy-cleanup-service.ts b/src/worker/sandbox-credential-policy-cleanup-service.ts index ac9d9212..0772527b 100644 --- a/src/worker/sandbox-credential-policy-cleanup-service.ts +++ b/src/worker/sandbox-credential-policy-cleanup-service.ts @@ -134,6 +134,7 @@ async function reconcileStagedCredentialPolicyRegistration( sandboxCredentialPolicyRegistrationLookupIds( registration.lookup_ids_json, registration.sandbox_id, + sandboxLookupIds(env, registration.sandbox_id), ).map((lookupId) => unregisterSandboxCredentialPolicyLookup( env, diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index e20989dc..a198c701 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -665,7 +665,9 @@ export async function renewSandboxCredentialPolicyRegistration( .where("state", "=", "registering") .where("registration_generation", "=", registration.generation) .where("registration_claim", "=", registration.claim) + .where("registration_claim_expires_at", ">", now) .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .where(noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)) .executeTakeFirst(); return Number(renewed.numUpdatedRows ?? 0n) === 1 ? registrationExpiresAt : null; } @@ -702,6 +704,7 @@ export async function claimSandboxCredentialPolicyRegistrationRecovery( .where("registration_claim_expires_at", "=", expiredRegistrationExpiresAt) .where("registration_claim_expires_at", "<=", now) .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .where(noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)) .executeTakeFirst(); return Number(claimed.numUpdatedRows ?? 0n) === 1 ? { registration, registrationExpiresAt } diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index 6bd17feb..35c63c3a 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -14,6 +14,7 @@ import { finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyCleanupAuthorizedCondition, + sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; import { sandboxLeaseInfo, sandboxLeasePrefix } from "./sandbox-lease.ts"; @@ -341,7 +342,11 @@ async function scanStagedCredentialPolicyRegistrations( const registration: SandboxCredentialPolicyRegistration = { generation: row.registration_generation, claim: row.registration_claim, - lookupIds: sandboxCredentialPolicyRegistrationLookupIds(row.lookup_ids_json, row.sandbox_id), + lookupIds: sandboxCredentialPolicyRegistrationLookupIds( + row.lookup_ids_json, + row.sandbox_id, + sandboxLookupIds(env, row.sandbox_id), + ), }; try { const ownershipFence = credentialPolicyScanOwnershipFence(row, now); diff --git a/src/worker/session-control-policy.ts b/src/worker/session-control-policy.ts index 6fb2b255..26596aa4 100644 --- a/src/worker/session-control-policy.ts +++ b/src/worker/session-control-policy.ts @@ -32,8 +32,9 @@ export type SandboxCredentialPolicyRegistration = { export function sandboxCredentialPolicyRegistrationLookupIds( value: string | null | undefined, sandboxId: string, + expectedLookupIds: readonly string[], ): string[] { - if (value) { + if (value !== null && value !== undefined) { try { const parsed = JSON.parse(value) as unknown; if ( @@ -48,10 +49,12 @@ export function sandboxCredentialPolicyRegistrationLookupIds( if (lookupIds.includes(sandboxId)) return lookupIds; } } catch { - // Upgraded rows are backfilled by migration; malformed rows retain the stable sandbox key. + // Malformed persisted state cannot authorize additional lookup identities. } + return [sandboxId]; } - return [sandboxId]; + const fallbackLookupIds = [...new Set(expectedLookupIds)]; + return fallbackLookupIds.includes(sandboxId) ? fallbackLookupIds : [sandboxId]; } export function storedSandboxCredentialPolicy( diff --git a/tests/sandbox-credential-policy-cleanup.test.ts b/tests/sandbox-credential-policy-cleanup.test.ts index 368f62b0..ea7d2f4b 100644 --- a/tests/sandbox-credential-policy-cleanup.test.ts +++ b/tests/sandbox-credential-policy-cleanup.test.ts @@ -214,7 +214,7 @@ test("terminal cleanup atomically stages the session and credential-policy refs" assert.ok(parameters.includes(leaseId)); }); -test("staged cleanup uses the persisted lookup set", async () => { +test("staged cleanup uses the persisted lookup set with an exact current fallback", async () => { const source = await readFile( new URL("../src/worker/sandbox-credential-policy-cleanup-service.ts", import.meta.url), "utf8", @@ -225,5 +225,5 @@ test("staged cleanup uses the persisted lookup set", async () => { assert.match(stagedCleanup, /sandboxCredentialPolicyRegistrationLookupIds/); assert.match(stagedCleanup, /registration\.lookup_ids_json/); - assert.doesNotMatch(stagedCleanup, /sandboxLookupIds\(env, registration\.sandbox_id\)/); + assert.match(stagedCleanup, /sandboxLookupIds\(env, registration\.sandbox_id\)/); }); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 2548a931..e2aa9fc6 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -321,17 +321,38 @@ test("credential-policy lookup identity includes the Sandbox durable object id e test("staged lookup identity decoder requires the stable sandbox lookup", () => { assert.deepEqual( - sandboxCredentialPolicyRegistrationLookupIds('["sandbox-1","do-old"]', "sandbox-1"), + sandboxCredentialPolicyRegistrationLookupIds('["sandbox-1","do-old"]', "sandbox-1", [ + "sandbox-1", + "do-current", + ]), ["sandbox-1", "do-old"], ); - assert.deepEqual(sandboxCredentialPolicyRegistrationLookupIds('["do-old"]', "sandbox-1"), [ - "sandbox-1", - ]); assert.deepEqual( - sandboxCredentialPolicyRegistrationLookupIds('["sandbox-1","sandbox-1"]', "sandbox-1"), + sandboxCredentialPolicyRegistrationLookupIds('["do-old"]', "sandbox-1", [ + "sandbox-1", + "do-current", + ]), + ["sandbox-1"], + ); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds('["sandbox-1","sandbox-1"]', "sandbox-1", [ + "sandbox-1", + "do-current", + ]), + ["sandbox-1"], + ); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds(null, "sandbox-1", ["sandbox-1", "do-current"]), + ["sandbox-1", "do-current"], + ); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds(null, "sandbox-1", ["do-current"]), + ["sandbox-1"], + ); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds("{", "sandbox-1", ["sandbox-1", "do-current"]), ["sandbox-1"], ); - assert.deepEqual(sandboxCredentialPolicyRegistrationLookupIds(null, "sandbox-1"), ["sandbox-1"]); }); test("credential-policy generations reuse exactly one current identity", () => { @@ -586,6 +607,48 @@ test("lookup identity migration backfills the exact staged legacy lookup set", ( assert.equal(row?.repair_generation, null); }); +test("post-migration legacy staging recovers the current sandbox lookup set", () => { + const sqlite = credentialPolicyDatabase(); + sqlite + .prepare(` + INSERT INTO interactive_session_credential_policy_registrations ( + session_id, + sandbox_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at + ) VALUES (?, ?, 'registering', ?, ?, ?, 1, 1) + `) + .run( + "IS-42", + "sandbox-1", + "generation:legacy-staged", + "registration:legacy-staged", + Number.MAX_SAFE_INTEGER, + ); + + const row = sqlite + .prepare(` + SELECT lookup_ids_json + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(); + const env = sqliteRuntimeEnv(sqlite); + assert.equal(row?.lookup_ids_json, null); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds( + row?.lookup_ids_json as string | null, + "sandbox-1", + sandboxLookupIds(env, "sandbox-1"), + ), + ["sandbox-1", "do-1"], + ); +}); + test("post-migration legacy registration claims block new staged generations", async () => { const sqlite = credentialPolicyDatabase(); sqlite From 54e7dd7a187185807f9a39c0d78eedeb757ab9f9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:01:38 +0200 Subject: [PATCH 191/242] test(credentials): prove single claim ownership --- ...ndbox-credential-policy-repository.test.ts | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index e2aa9fc6..3bcae3b9 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -1013,6 +1013,194 @@ test("stale foreground rollback cannot renew after recovery takes its claim", as assert.equal(renewed, null); }); +test("expired staged registration claims cannot be revived by renewal", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const expiredAt = 1; + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(expiredAt); + + assert.equal( + await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + null, + ); + assert.equal( + sqlite + .prepare(` + SELECT registration_claim_expires_at + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get()?.registration_claim_expires_at, + expiredAt, + ); +}); + +test("staged renewal cannot extend across a live legacy registration claim", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = 1 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + const legacyClaim = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-renewal', + registration_claim = 'registration:legacy-renewal', + registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + assert.equal(legacyClaim.changes, 2); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + + assert.equal( + await renewSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + null, + ); + assert.deepEqual( + sqlite + .prepare(` + SELECT DISTINCT registration_generation, registration_claim, registration_claim_expires_at + FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .all() + .map((row) => ({ ...row })), + [ + { + registration_generation: "generation:legacy-renewal", + registration_claim: "registration:legacy-renewal", + registration_claim_expires_at: Number.MAX_SAFE_INTEGER, + }, + ], + ); +}); + +test("expired staged recovery cannot race a live legacy registration owner", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const expiredAt = 1; + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(expiredAt); + const legacyClaim = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-recovery', + registration_claim = 'registration:legacy-recovery', + registration_claim_expires_at = ? + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + assert.equal(legacyClaim.changes, 2); + + const recoveries = await Promise.all([ + claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + staged, + expiredAt, + ownershipFence, + ), + claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + staged, + expiredAt, + ownershipFence, + ), + ]); + + assert.deepEqual(recoveries, [null, null]); + assert.deepEqual( + { + ...sqlite + .prepare(` + SELECT registration_claim, registration_claim_expires_at + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(), + }, + { + registration_claim: staged.claim, + registration_claim_expires_at: expiredAt, + }, + ); + assert.deepEqual( + activeCredentialPolicyRows(sqlite).map((row) => ({ + generation: row.registration_generation, + claim: row.registration_claim, + })), + [ + { + generation: "generation:legacy-recovery", + claim: "registration:legacy-recovery", + }, + { + generation: "generation:legacy-recovery", + claim: "registration:legacy-recovery", + }, + ], + ); +}); + test("expired registration recovery grants one fresh exclusive claim", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); From e8d0075639f9f33164cf93ee9e8d5e79a97beb86 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:02:07 +0200 Subject: [PATCH 192/242] fix(vnc): renegotiate pixel format encodings --- .../SDK/Connection/VNCConnection+API.swift | 21 ++- .../SDK/Connection/VNCConnection.swift | 13 +- .../RoyalVNCKitTests/AuditFindingsTests.swift | 125 +++++++++++++++++- 3 files changed, 148 insertions(+), 11 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 535e064b..d87ebf5a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -7,6 +7,8 @@ import Foundation private struct PixelFormatTransitionMessage: VNCSendableMessage { let fenceMessage: VNCProtocol.ClientFence? let pixelFormatMessage: VNCProtocol.SetPixelFormat + let encodingsMessage: VNCProtocol.SetEncodings + let willSendPixelFormat: () throws -> Void let willSend: () -> Void let didSend: () -> Void let willSendFence: () -> Void @@ -15,10 +17,11 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { var messageType: UInt8 { fenceMessage?.messageType ?? pixelFormatMessage.messageType } var data: Data { - (fenceMessage?.data ?? Data()) + pixelFormatMessage.data + (fenceMessage?.data ?? Data()) + pixelFormatMessage.data + encodingsMessage.data } func send(connection: NetworkConnectionWriting) async throws { + try willSendPixelFormat() if fenceMessage == nil { willSend() } else { @@ -49,12 +52,14 @@ private struct PixelFormatTransition { private struct FenceCapabilityProbeMessage: VNCSendableMessage { let fenceMessage: VNCProtocol.ClientFence let pixelFormatMessage: VNCProtocol.SetPixelFormat + let willSendPixelFormat: () throws -> Void let didSend: () -> Void var messageType: UInt8 { fenceMessage.messageType } var data: Data { fenceMessage.data + pixelFormatMessage.data } func send(connection: NetworkConnectionWriting) async throws { + try willSendPixelFormat() try await connection.write(data: data) didSend() } @@ -198,12 +203,23 @@ extension VNCConnection { } private func enqueuePixelFormatTransition(_ transition: PixelFormatTransition) { + let encodingTypes: [VNCEncodingType] + do { + encodingTypes = try orderedEncodingTypes(pixelFormat: transition.pixelFormat) + } catch { + handleBreakingError(error) + return + } let fenceMessage = transition.fencePayload.map { VNCProtocol.ClientFence(flags: transition.fenceFlags, payload: $0) } let message = PixelFormatTransitionMessage( fenceMessage: fenceMessage, pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: transition.pixelFormat), + encodingsMessage: VNCProtocol.SetEncodings(encodingTypes: encodingTypes), + willSendPixelFormat: { [weak self] in + try self?.resetZRLECompressionState() + }, willSend: { [weak self] in self?.beginPixelFormatTransition(transition.pixelFormat) }, @@ -323,6 +339,9 @@ extension VNCConnection { payload: payload ), pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat), + willSendPixelFormat: { [weak self] in + try self?.resetZRLECompressionState() + }, didSend: { [weak self] in self?.didSendPixelFormatFenceCapabilityProbe(payload: payload) } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 96e62428..ba1c27c2 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -234,7 +234,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { return enabledEncodings }() - func orderedEncodingTypes() throws -> [VNCEncodingType] { + func orderedEncodingTypes(pixelFormat: VNCProtocol.PixelFormat? = nil) throws -> [VNCEncodingType] { // Frame Encodings (Required) var encs: [VNCEncodingType] = [ VNCFrameEncodingType.copyRect.rawValue @@ -242,22 +242,23 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { // Frame Encodings (Customizable) var customizedFrameEncodings = settings.frameEncodings.map({ $0.rawValue }) + let negotiatedPixelFormat = pixelFormat ?? state.pixelFormat // TODO: Remove once we support ZRLE for non-24-bit pixel formats - if let pixelFormat = state.pixelFormat, + if let pixelFormat = negotiatedPixelFormat, customizedFrameEncodings.contains(VNCFrameEncodingType.zrle.rawValue), !VNCProtocol.ZRLEEncoding.supportsPixelFormat(pixelFormat) { customizedFrameEncodings.removeAll(where: { $0 == VNCFrameEncodingType.zrle.rawValue }) } - if let pixelFormat = state.pixelFormat, + if let pixelFormat = negotiatedPixelFormat, customizedFrameEncodings.contains(VNCFrameEncodingType.tight.rawValue), !VNCProtocol.TightEncoding.supportsPixelFormat(pixelFormat) { customizedFrameEncodings.removeAll(where: { $0 == VNCFrameEncodingType.tight.rawValue }) } #if canImport(VideoToolbox) - if let pixelFormat = state.pixelFormat, + if let pixelFormat = negotiatedPixelFormat, customizedFrameEncodings.contains(VNCFrameEncodingType.openH264.rawValue), !VNCProtocol.OpenH264Encoding.supportsPixelFormat(pixelFormat) { customizedFrameEncodings.removeAll(where: { $0 == VNCFrameEncodingType.openH264.rawValue }) @@ -305,6 +306,10 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { return uniqueEncs } + func resetZRLECompressionState() throws { + try sharedZRLEZStream.reset() + } + // MARK: - Public Initializers public init(settings: Settings, logger: VNCLogger, diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 1a8c506f..c491c2e7 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -423,11 +423,12 @@ struct AuditFindingsTests { try await queued.message.send(connection: writer) #expect(connection.pixelFormatTransitionDeadlineTask == nil) - #expect(writer.data.count == 37) #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) #expect(writer.data[4..<8] == Data([0x80, 0, 0, 5])) #expect(writer.data[8] == 8) #expect(writer.data[17] == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType) + #expect(writer.data[37] == VNCProtocol.SetEncodings(encodingTypes: []).messageType) + #expect(setEncodingValues(in: writer.data, at: 37).contains(Int32(VNCFrameEncodingType.raw.rawValue.rawValue))) #expect(connection.state.pixelFormat?.depth == 24) let payload = Data(writer.data[9..<17]) @@ -447,6 +448,88 @@ struct AuditFindingsTests { #expect(connection.connectionState.status == .connected) } + @Test + func renegotiatesEncodingsForSynchronizedPixelFormatTransition() async throws { + let connection = try await makeFenceCapableConnection( + settings: makeSettings(frameEncodings: [.tight, .zrle, .openH264, .raw]) + ) + + let depth24Encodings = try connection.orderedEncodingTypes( + pixelFormat: VNCProtocol.PixelFormat(depth: 24) + ) + #expect(depth24Encodings.contains(VNCFrameEncodingType.tight.rawValue)) + #expect(depth24Encodings.contains(VNCFrameEncodingType.zrle.rawValue)) +#if canImport(VideoToolbox) + #expect(depth24Encodings.contains(VNCFrameEncodingType.openH264.rawValue)) +#endif + + connection.updateColorDepth(.depth8Bit) + let queued = try #require(connection.clientToServerMessageQueue.dequeue()) + let writer = AuditWritingConnection { + #expect(connection.state.pixelFormat?.depth == 24) + } + try await queued.message.send(connection: writer) + + #expect(writer.data[0] == VNCProtocol.ClientFence.messageType) + #expect(writer.data[17] == VNCProtocol.SetPixelFormat(pixelFormat: VNCProtocol.PixelFormat(depth: 8)).messageType) + #expect(writer.data[37] == VNCProtocol.SetEncodings(encodingTypes: []).messageType) + let values = setEncodingValues(in: writer.data, at: 37) + #expect(!values.contains(Int32(VNCFrameEncodingType.tight.rawValue.rawValue))) + #expect(!values.contains(Int32(VNCFrameEncodingType.zrle.rawValue.rawValue))) + #expect(!values.contains(Int32(VNCFrameEncodingType.openH264.rawValue.rawValue))) + #expect(values.contains(Int32(VNCFrameEncodingType.copyRect.rawValue.rawValue))) + #expect(values.contains(Int32(VNCFrameEncodingType.raw.rawValue.rawValue))) + + connection.cancelFramebufferUpdateScheduling() + } + + @Test + func resetsZRLECompressionAtPixelFormatProbeAndTransitionBoundaries() async throws { + let connection = VNCConnection( + settings: makeSettings(frameEncodings: [.zrle, .raw]), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + let zrle = try #require( + connection.encodings[VNCFrameEncodingType.zrle.rawValue] as? VNCProtocol.ZRLEEncoding + ) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + + try primeZRLEStream(zrle.zStream, byte: 0x41) + try await capabilityProbe.message.send(connection: AuditWritingConnection()) + try verifyFreshZRLEStream(zrle.zStream, byte: 0x42) + + let capabilityPayload = try #require(connection.pixelFormatFenceCapabilityProbePayload) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: capabilityPayload + ) + ) + + connection.framebufferUpdateRequestOutstanding = true + connection.updateColorDepth(.depth8Bit) + let transition = try #require(connection.clientToServerMessageQueue.dequeue()) + try await transition.message.send(connection: AuditWritingConnection()) + try verifyFreshZRLEStream(zrle.zStream, byte: 0x44) + + connection.cancelFramebufferUpdateScheduling() + } + @Test func rejectsPixelFormatFenceResponseBeforeRequestIsSent() async throws { let connection = try await makeFenceCapableConnection() @@ -624,11 +707,11 @@ struct AuditFindingsTests { } try await transition.message.send(connection: transitionWriter) #expect(connection.pixelFormatTransitionDeadlineTask == nil) - #expect(transitionWriter.data.count == 20) #expect( transitionWriter.data[0] == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType ) + #expect(transitionWriter.data[20] == VNCProtocol.SetEncodings(encodingTypes: []).messageType) #expect(connection.state.pixelFormat?.depth == 8) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) } @@ -818,7 +901,9 @@ struct AuditFindingsTests { ) } - private func makeSettings() -> VNCConnection.Settings { + private func makeSettings( + frameEncodings: [VNCFrameEncodingType] = [.raw] + ) -> VNCConnection.Settings { VNCConnection.Settings( isDebugLoggingEnabled: false, hostname: "127.0.0.1", @@ -829,13 +914,15 @@ struct AuditFindingsTests { inputMode: .none, isClipboardRedirectionEnabled: false, colorDepth: .depth24Bit, - frameEncodings: [.raw] + frameEncodings: frameEncodings ) } - private func makeFenceCapableConnection() async throws -> VNCConnection { + private func makeFenceCapableConnection( + settings: VNCConnection.Settings? = nil + ) async throws -> VNCConnection { let connection = VNCConnection( - settings: makeSettings(), + settings: settings ?? makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() ) let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) @@ -867,6 +954,32 @@ struct AuditFindingsTests { ) return connection } + + private func setEncodingValues(in data: Data, at offset: Int) -> [Int32] { + let count = Int(data[offset + 2]) << 8 | Int(data[offset + 3]) + return (0.. Date: Sun, 12 Jul 2026 17:02:18 +0200 Subject: [PATCH 193/242] fix(terminal): harden input completion races --- internal/terminalws/client.go | 38 +++- internal/terminalws/client_test.go | 274 ++++++++++++++++++++++++++++- 2 files changed, 301 insertions(+), 11 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 134b7312..5167fbba 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -25,6 +25,7 @@ const ( defaultInputConfirmationTimeout = 5 * time.Second defaultAttachmentShutdownTimeout = 250 * time.Millisecond + defaultOutputAckTimeout = 5 * time.Second messageHello = 1 messageWelcome = 2 @@ -52,6 +53,10 @@ const ( subscribeOutputAcknowledgements = 1 << 3 ) +var ErrInputDeliveryUnknown = errors.New( + "terminal input delivery outcome is unknown; the runner may still complete it", +) + type Options struct { HTTPClient *http.Client Header http.Header @@ -356,11 +361,21 @@ func (c *Client) waitForInputConfirmation(ctx context.Context, waiter chan error return readerUnavailableError(c.readerError()) case <-ctx.Done(): c.clearInputWaiter(waiter) - _ = c.Close() + c.closeNow() return ctx.Err() } } +func (c *Client) closeNow() { + if c.readCancel != nil { + c.readCancel() + } + _ = c.conn.CloseNow() + if c.cancel != nil { + c.cancel() + } +} + func (c *Client) acquireConfirmation(ctx context.Context) error { c.confirmOnce.Do(func() { c.confirmGate = make(chan struct{}, 1) @@ -493,11 +508,17 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c errCh <- err return } - if err := c.write(ctx, frame{ + ackCtx, ackCancel := context.WithTimeout( + context.Background(), + defaultOutputAckTimeout, + ) + err := c.write(ackCtx, frame{ messageType: messageAck, sessionID: c.sessionID, payload: ackPayload(uint32(len(current.payload))), - }); err != nil { + }) + ackCancel() + if err != nil { errCh <- err return } @@ -610,7 +631,6 @@ func (c *Client) handleFrame(ctx context.Context, current frame) error { return err case messageControlRevoked: c.canInput.Store(false) - c.completeInput(frameError(current, "terminal input rejected")) c.deliverAttachment(ctx, current) case messageControlGranted: c.canInput.Store(true) @@ -627,6 +647,8 @@ func (c *Client) handleFrame(ctx context.Context, current frame) error { c.completeInput(nil) case "input-rejected": c.completeInput(frameError(current, "terminal input rejected")) + case "input-delivery-unknown": + c.completeInput(inputDeliveryUnknownError(current)) case "closed": c.canInput.Store(false) c.markTerminalClosed(errors.New("terminal closed")) @@ -916,6 +938,14 @@ func frameError(current frame, fallback string) error { return errors.New(fallback) } +func inputDeliveryUnknownError(current frame) error { + detail := frameError(current, ErrInputDeliveryUnknown.Error()) + if detail.Error() == ErrInputDeliveryUnknown.Error() { + return ErrInputDeliveryUnknown + } + return fmt.Errorf("%w: %s", ErrInputDeliveryUnknown, detail) +} + func normalizeCloseError(err error) error { if err == nil { return nil diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index be66c16c..e6c07139 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -358,7 +359,9 @@ func TestClientDefersOutputAcknowledgementUntilAttach(t *testing.T) { } } -func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { +func TestSendInputConfirmedDoesNotTreatControlRevocationAsInputRejection(t *testing.T) { + revokedSent := make(chan struct{}) + releaseAcceptance := make(chan struct{}) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) if err != nil { @@ -394,10 +397,25 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { return } revoked, _ := json.Marshal(eventPayload{Error: "terminal control revoked"}) - _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ messageType: messageControlRevoked, sessionID: "IS-revoked", payload: revoked, + })); err != nil { + t.Error(err) + return + } + close(revokedSent) + select { + case <-releaseAcceptance: + case <-r.Context().Done(): + return + } + accepted, _ := json.Marshal(eventPayload{Type: "input-accepted"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-revoked", + payload: accepted, })) })) defer server.Close() @@ -411,10 +429,92 @@ func TestSendInputConfirmedReturnsControlRevocation(t *testing.T) { t.Fatal(err) } defer client.Close() - err = client.SendInputConfirmed(context.Background(), []byte("blocked\n")) - if err == nil || !strings.Contains(err.Error(), "control revoked") { + done := make(chan error, 1) + go func() { + done <- client.SendInputConfirmed(context.Background(), []byte("forwarded\n")) + }() + <-revokedSent + select { + case err := <-done: + t.Fatalf("control revocation completed forwarded input: %v", err) + case <-time.After(50 * time.Millisecond): + } + if client.canInput.Load() { + t.Fatal("control revocation did not remove input capability") + } + close(releaseAcceptance) + if err := <-done; err != nil { + t.Fatalf("later input acceptance = %v", err) + } +} + +func TestSendInputConfirmedReportsUnknownDelivery(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-delivery-unknown", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + unknown, _ := json.Marshal(eventPayload{ + Type: "input-delivery-unknown", + Error: ErrInputDeliveryUnknown.Error(), + }) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-delivery-unknown", + payload: unknown, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-delivery-unknown", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + err = client.SendInputConfirmed(context.Background(), []byte("possibly-delivered\n")) + if !errors.Is(err, ErrInputDeliveryUnknown) { t.Fatalf("error = %v", err) } + if err.Error() != ErrInputDeliveryUnknown.Error() { + t.Fatalf("error text = %q", err) + } + if !client.canInput.Load() { + t.Fatal("ambiguous delivery revoked input capability") + } } func TestSendInputConfirmedAcknowledgesOutputWithoutAttachment(t *testing.T) { @@ -1362,6 +1462,7 @@ func TestSendInputConfirmedReturnsImmediatelyForEmptyInput(t *testing.T) { func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { inputReceived := make(chan struct{}) + releaseServer := make(chan struct{}) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) if err != nil { @@ -1397,9 +1498,12 @@ func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { return } close(inputReceived) - _, _, _ = conn.Read(r.Context()) + <-releaseServer })) - defer server.Close() + defer func() { + close(releaseServer) + server.Close() + }() endpoint, err := Endpoint(server.URL) if err != nil { @@ -1413,10 +1517,14 @@ func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() + started := time.Now() err = client.SendInputConfirmed(ctx, []byte("first\n")) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("error = %v", err) } + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + t.Fatalf("confirmation deadline took %s", elapsed) + } <-inputReceived if err := client.SendInputConfirmed(context.Background(), []byte("second\n")); err == nil { t.Fatal("timed-out client accepted another input") @@ -1821,6 +1929,122 @@ func TestAttachBoundsBlockedFrameConsumerShutdown(t *testing.T) { close(terminal.releaseRead) } +func TestRetiredBlockedAttachmentAcknowledgementKeepsReplacementConnectionOpen(t *testing.T) { + firstAcknowledged := make(chan struct{}) + secondAcknowledged := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-stale-ack", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-stale-ack", + payload: []byte("old output\n"), + })); err != nil { + t.Error(err) + return + } + if err := readOutputAcknowledgement(r.Context(), conn, len("old output\n")); err != nil { + t.Error(err) + return + } + close(firstAcknowledged) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-stale-ack", + payload: []byte("replacement output\n"), + })); err != nil { + t.Error(err) + return + } + if err := readOutputAcknowledgement(r.Context(), conn, len("replacement output\n")); err != nil { + t.Error(err) + return + } + close(secondAcknowledged) + <-r.Context().Done() + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-stale-ack", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + client.attachmentShutdownTimeout = 10 * time.Millisecond + + oldTerminal := newUncancelableReadBlockingSuccessfulWriteTerminal() + oldCtx, oldCancel := context.WithCancel(context.Background()) + oldDone := make(chan error, 1) + go func() { + oldDone <- client.Attach(oldCtx, oldTerminal, nil) + }() + <-oldTerminal.readStarted + <-oldTerminal.writeStarted + oldCancel() + if err := <-oldDone; !errors.Is(err, context.Canceled) { + t.Fatalf("old attachment error = %v", err) + } + + replacement := newBlockingTerminal() + replacementCtx, replacementCancel := context.WithCancel(context.Background()) + replacementDone := make(chan error, 1) + go func() { + replacementDone <- client.Attach(replacementCtx, replacement, nil) + }() + <-replacement.started + + close(oldTerminal.releaseWrite) + select { + case <-firstAcknowledged: + case <-time.After(time.Second): + t.Fatal("retired attachment did not acknowledge completed output") + } + select { + case <-secondAcknowledged: + case <-time.After(time.Second): + t.Fatal("replacement connection did not acknowledge subsequent output") + } + replacementCancel() + if err := <-replacementDone; !errors.Is(err, context.Canceled) { + t.Fatalf("replacement error = %v", err) + } + if got := replacement.String(); got != "replacement output\n" { + t.Fatalf("replacement output = %q", got) + } + close(oldTerminal.releaseRead) +} + func TestClientSubscribesReadOnlyAndSuppressesInput(t *testing.T) { acknowledged := make(chan uint32, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -2076,6 +2300,31 @@ func TestClientContinuesReadOnlyAndResumesControl(t *testing.T) { } } +func readOutputAcknowledgement( + ctx context.Context, + conn *websocket.Conn, + expectedBytes int, +) error { + _, payload, err := conn.Read(ctx) + if err != nil { + return err + } + current, err := decodeFrame(payload) + if err != nil { + return err + } + if current.messageType != messageAck { + return fmt.Errorf("acknowledgement message type = %d", current.messageType) + } + if len(current.payload) != 4 { + return fmt.Errorf("acknowledgement payload length = %d", len(current.payload)) + } + if acknowledged := binary.LittleEndian.Uint32(current.payload); acknowledged != uint32(expectedBytes) { + return fmt.Errorf("acknowledged bytes = %d, want %d", acknowledged, expectedBytes) + } + return nil +} + type readWriter struct { reader io.Reader closer io.Closer @@ -2118,6 +2367,7 @@ type uncancelableReadBlockingWriteTerminal struct { writeOnce sync.Once releaseWrite chan struct{} writeDone chan struct{} + writeErr error } func newUncancelableReadBlockingWriteTerminal() *uncancelableReadBlockingWriteTerminal { @@ -2127,9 +2377,16 @@ func newUncancelableReadBlockingWriteTerminal() *uncancelableReadBlockingWriteTe writeStarted: make(chan struct{}), releaseWrite: make(chan struct{}), writeDone: make(chan struct{}), + writeErr: errBlockedTerminalWrite, } } +func newUncancelableReadBlockingSuccessfulWriteTerminal() *uncancelableReadBlockingWriteTerminal { + terminal := newUncancelableReadBlockingWriteTerminal() + terminal.writeErr = nil + return terminal +} + func (terminal *uncancelableReadBlockingWriteTerminal) Read(_ []byte) (int, error) { terminal.readOnce.Do(func() { close(terminal.readStarted) @@ -2144,7 +2401,10 @@ func (terminal *uncancelableReadBlockingWriteTerminal) Write(payload []byte) (in close(terminal.writeStarted) }) <-terminal.releaseWrite - return 0, errBlockedTerminalWrite + if terminal.writeErr != nil { + return 0, terminal.writeErr + } + return len(payload), nil } func newUncancelableTerminal() *uncancelableTerminal { From 39495b6968567cb9e9b2b3ed36a2ccc8f786a631 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:04:22 +0200 Subject: [PATCH 194/242] fix(macos): persist desktop cleanup recovery --- CHANGELOG.md | 2 +- .../CrabfleetMac/CrabfleetMacApp.swift | 4 +- .../PrivateMacShareController.swift | 269 +++++++++++++++--- .../PrivateMacShareTests.swift | 182 +++++++++++- 4 files changed, 418 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96c9dcf0..b806bad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Make terminal input delivery durable across multiplex subscribers, apply backpressure until an attachment owns initial output while waking blocked readers to discard and acknowledge output for one-shot confirmed messages, retain pre-attach closure, prefer completed input over later transport shutdown, reject denied or unacknowledged input explicitly, prevent delivery into retired attachments and wait for their frame consumers even when input reads cannot be canceled, bound serialized browser input backlog by frame count and bytes while preserving one ordered completion per dropped frame, enforce relay-owned runner generations before forwarding GitHub Actions input and acknowledgements, snapshot SSH connection limits before launching handlers, make confirmation serialization cancelable, bound attachment confirmation waits, serialize every acknowledgement-aware input source and per-subscription completion event, wake uncancelable attachments when their context ends, retire connections after ambiguous confirmation timeouts, negotiate bounded one-shot input acknowledgements across rolling upgrades without waiting on empty payloads, keep configured HTTP timeouts from canceling established sockets, scope rejected writes without dropping live subscriptions, and send attributed commands atomically to prevent interleaving. - Add connection-query-negotiated `CFR1` input, output, and acknowledgement frames across GitHub Actions runner and internal viewer relay boundaries, confirm viewer negotiation before leaving raw fallback during mixed deployments, document the independent legacy viewer fallback, buffer split UTF-8 within byte, frame, and age bounds until the string-only Node adapter delivers it to the PTY before acknowledging every contributing frame, define that adapter's UTF-8-only output contract while preserving opaque bytes for byte-oriented adapters, close the runner socket when its PTY exits, prevent binary PTY output from colliding with relay control frames, and keep multiplex dispatch responsive while framed input acknowledgements are pending. - Preserve bounded opaque profile IDs for fixed runtime adapters while rejecting unroutable or ambiguous adapter routes only when provisioning depends on them, so mixed migration configuration cannot break unrelated control-plane reads; durably claim and retry superseded workspace cleanup without touching the replacement workspace; also reject malformed encoded session routes, numeric literals that become integers only after precision loss, and invalid-Unicode JSON event values, and reconcile browser history drawers and focus on back/forward navigation. -- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure, ambiguous committed publication reconciled by stable publication identity, and application-termination races with retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown with bounded retries and no retry when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. +- Harden Share This Mac against stale starts and responses, canceled starts stranded in transition, self-connections, leaked subprocess environment, stalled Tailscale and RFB handshakes, successful commands whose descendants retain output pipes, stale desktop registrations including listener-failure, ambiguous committed publication reconciled by stable publication identity, and application-termination races with durably retained cleanup retries, legacy publishers mutating or deleting token-owned registrations, concurrent teardown calls that could outpace application termination, completed teardown operations coalescing a later stop, dropped auto-starts, stuck remote input including releases retained through revoked Accessibility trust and teardown with bounded retries and no retry when no input is held, and destructive non-text or empty clipboard changes while preserving repeated remote text, X11 Unicode input, proxy, custom-CA networking, and validated Crabbox config/state paths. - Fence Share This Mac registry cleanup with explicitly negotiated per-registration ownership tokens, require a valid publication identity before selecting token ownership, and return the exact atomically written registration row so delayed or overlapping current publishers cannot displace cleanup authority, while preserving tokenless registration and cleanup for rolling upgrades with legacy clients or servers. - Harden the bundled RoyalVNCKit fork across ARD and UltraVNC authentication and parameter validation, composed Unicode keysyms with legacy ASCII scalars, Tight and ZRLE parsing, non-trapping bounded zlib streams, RFB Fence-synchronized color-depth transitions with atomic capability publication, premature-response rejection, and fail-closed legacy handling, CopyRect, EOF handling, reconnects, credential cancellation and release after handoff, and cursor channel preservation. - Pin Crabbox coordinator deployment to an immutable source revision and verify downloaded Crabbox release archives before image assembly, always enforcing the repository digest for the default version and requiring an explicit architecture checksum for non-default versions. diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift index 5afca1bf..c856d204 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift @@ -86,9 +86,9 @@ final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { guard terminationTask == nil else { return .terminateLater } terminationTask = Task { [weak self] in guard let self else { return } - await shareController.stopAndWaitForCleanup() + let cleanupCanRecover = await shareController.stopAndWaitForCleanup() terminationTask = nil - replyToTerminationRequest(true) + replyToTerminationRequest(cleanupCanRecover) } return .terminateLater } diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 153e3efa..90eb1c47 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -36,72 +36,211 @@ final class PrivateMacShareStopCoordinator { } } +@MainActor +protocol DesktopHostRegistrationStateStoring: AnyObject { + func load() throws -> Data? + func save(_ data: Data?) throws +} + +enum DesktopHostRegistrationPersistenceError: LocalizedError { + case unreadableState + case writeFailed + + var errorDescription: String? { + switch self { + case .unreadableState: + "The saved desktop publication recovery state is unreadable." + case .writeFailed: + "The desktop publication recovery state could not be saved." + } + } +} + +@MainActor +final class UserDefaultsDesktopHostRegistrationStateStore: + DesktopHostRegistrationStateStoring +{ + nonisolated static let defaultKey = "org.openclaw.crabfleet.share.desktop-publications" + + private let defaults: UserDefaults + private let key: String + + init(defaults: UserDefaults = .standard, key: String = defaultKey) { + self.defaults = defaults + self.key = key + } + + func load() throws -> Data? { + defaults.data(forKey: key) + } + + func save(_ data: Data?) throws { + if let data { + defaults.set(data, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + guard defaults.synchronize() else { + throw DesktopHostRegistrationPersistenceError.writeFailed + } + } +} + @MainActor final class DesktopHostRegistrationLifecycle { - private struct RegistrationTarget: Equatable { - let identity: TailnetIdentity + private struct PersistedIdentity: Codable, Equatable { + let tailnetName: String + let loginName: String + let dnsName: String + let hostName: String + let ipv4Address: String + let userID: Int64 + + init(_ identity: TailnetIdentity) { + tailnetName = identity.tailnetName + loginName = identity.loginName + dnsName = identity.dnsName + hostName = identity.hostName + ipv4Address = identity.ipv4Address + userID = identity.userID + } + + var identity: TailnetIdentity { + TailnetIdentity( + tailnetName: tailnetName, + loginName: loginName, + dnsName: dnsName, + hostName: hostName, + ipv4Address: ipv4Address, + userID: userID + ) + } + } + + private struct RegistrationTarget: Codable, Equatable { + private let persistedIdentity: PersistedIdentity let hostID: String let port: UInt16 let publicationID: String + + init(identity: TailnetIdentity, hostID: String, port: UInt16, publicationID: String) { + persistedIdentity = PersistedIdentity(identity) + self.hostID = hostID + self.port = port + self.publicationID = publicationID + } + + var identity: TailnetIdentity { persistedIdentity.identity } } private struct PublishedRegistration: Equatable { let identity: TailnetIdentity let hostID: String + let publicationID: String let ownershipToken: String? + let usesLegacyCleanup: Bool + } + + private struct PersistedPublishedRegistration: Codable { + private let persistedIdentity: PersistedIdentity + let hostID: String + let publicationID: String + let usesLegacyCleanup: Bool + + init(_ registration: PublishedRegistration) { + persistedIdentity = PersistedIdentity(registration.identity) + hostID = registration.hostID + publicationID = registration.publicationID + usesLegacyCleanup = registration.usesLegacyCleanup + } + + var registration: PublishedRegistration { + PublishedRegistration( + identity: persistedIdentity.identity, + hostID: hostID, + publicationID: publicationID, + ownershipToken: nil, + usesLegacyCleanup: usesLegacyCleanup + ) + } + } + + private struct PersistedState: Codable { + var uncertainRegistrations: [RegistrationTarget] + var publishedRegistration: PersistedPublishedRegistration? + var pendingRemovals: [PersistedPublishedRegistration] } private let coordinator: DesktopHostRegistrationCoordinator private let createPublicationID: () -> String + private let stateStore: (any DesktopHostRegistrationStateStoring)? private var publishedRegistration: PublishedRegistration? private var uncertainRegistrations: [RegistrationTarget] = [] private var pendingRemovals: [PublishedRegistration] = [] + private var stateLoadError: Error? + private var lastPersistenceError: Error? init( registration: any DesktopHostRegistering, - createPublicationID: @escaping () -> String = { UUID().uuidString } + createPublicationID: @escaping () -> String = { UUID().uuidString }, + stateStore: (any DesktopHostRegistrationStateStoring)? = nil ) { coordinator = DesktopHostRegistrationCoordinator(registration: registration) self.createPublicationID = createPublicationID + self.stateStore = stateStore + guard let stateStore else { return } + do { + guard let data = try stateStore.load() else { return } + let state = try JSONDecoder().decode(PersistedState.self, from: data) + uncertainRegistrations = state.uncertainRegistrations + publishedRegistration = state.publishedRegistration?.registration + pendingRemovals = state.pendingRemovals.map(\.registration) + } catch { + stateLoadError = DesktopHostRegistrationPersistenceError.unreadableState + } + } + + var hasDurableRecoveryState: Bool { + stateStore != nil && stateLoadError == nil && lastPersistenceError == nil + && (!uncertainRegistrations.isEmpty || publishedRegistration != nil + || !pendingRemovals.isEmpty) } func publish(identity: TailnetIdentity, port: UInt16) async throws { + try ensureStateIsReadable() let hostID = CrabfleetDesktopRegistration.hostID(identity: identity) + let existingTarget = uncertainRegistrations.first { + $0.hostID == hostID && $0.identity == identity && $0.port == port + } let target = - uncertainRegistrations.first { - $0.hostID == hostID && $0.identity == identity && $0.port == port - } + existingTarget ?? RegistrationTarget( identity: identity, hostID: hostID, port: port, publicationID: createPublicationID() ) + if existingTarget == nil { + uncertainRegistrations.append(target) + try persistState() + } let ownershipToken: String? - do { - if uncertainRegistrations.contains(target) { - ownershipToken = try await coordinator.recover( - identity: identity, - publicationID: target.publicationID - ) - guard ownershipToken != nil else { - uncertainRegistrations.removeAll { $0 == target } - throw DesktopHostRegistrationSupersededError() - } - } else { - ownershipToken = try await coordinator.register( - identity: identity, - port: port, - publicationID: target.publicationID - ) - } - } catch { - if error is DesktopHostRegistrationResultUncertainError, - !uncertainRegistrations.contains(target) - { - uncertainRegistrations.append(target) + if existingTarget != nil { + ownershipToken = try await coordinator.recover( + identity: identity, + publicationID: target.publicationID + ) + guard ownershipToken != nil else { + uncertainRegistrations.removeAll { $0 == target } + try persistState() + throw DesktopHostRegistrationSupersededError() } - throw error + } else { + ownershipToken = try await coordinator.register( + identity: identity, + port: port, + publicationID: target.publicationID + ) } uncertainRegistrations.removeAll { $0 == target } if let publishedRegistration, publishedRegistration.hostID != hostID, @@ -113,11 +252,15 @@ final class DesktopHostRegistrationLifecycle { publishedRegistration = PublishedRegistration( identity: identity, hostID: hostID, - ownershipToken: ownershipToken + publicationID: target.publicationID, + ownershipToken: ownershipToken, + usesLegacyCleanup: ownershipToken == nil ) + try persistState() } func removePublishedIdentities() async throws { + try ensureStateIsReadable() var firstError: Error? let uncertainRegistrations = uncertainRegistrations for target in uncertainRegistrations { @@ -130,13 +273,16 @@ final class DesktopHostRegistrationLifecycle { let recovered = PublishedRegistration( identity: target.identity, hostID: target.hostID, - ownershipToken: ownershipToken + publicationID: target.publicationID, + ownershipToken: ownershipToken, + usesLegacyCleanup: false ) if !pendingRemovals.contains(recovered) { pendingRemovals.append(recovered) } } self.uncertainRegistrations.removeAll { $0 == target } + try persistState() } catch { firstError = firstError ?? error } @@ -147,22 +293,67 @@ final class DesktopHostRegistrationLifecycle { pendingRemovals.append(publishedRegistration) } self.publishedRegistration = nil + try persistState() } let removals = pendingRemovals for removal in removals { + var ownershipToken = removal.ownershipToken + if ownershipToken == nil, !removal.usesLegacyCleanup { + do { + guard + let recoveredToken = try await coordinator.recover( + identity: removal.identity, + publicationID: removal.publicationID + ) + else { + pendingRemovals.removeAll { $0 == removal } + try persistState() + continue + } + ownershipToken = recoveredToken + } catch { + firstError = firstError ?? error + continue + } + } do { try await coordinator.unregister( identity: removal.identity, - ownershipToken: removal.ownershipToken + ownershipToken: ownershipToken ) pendingRemovals.removeAll { $0 == removal } + try persistState() } catch { firstError = firstError ?? error } } if let firstError { throw firstError } } + + private func ensureStateIsReadable() throws { + if let stateLoadError { throw stateLoadError } + } + + private func persistState() throws { + guard let stateStore else { return } + let state = PersistedState( + uncertainRegistrations: uncertainRegistrations, + publishedRegistration: publishedRegistration.map(PersistedPublishedRegistration.init), + pendingRemovals: pendingRemovals.map(PersistedPublishedRegistration.init) + ) + let hasState = + !state.uncertainRegistrations.isEmpty || state.publishedRegistration != nil + || !state.pendingRemovals.isEmpty + do { + let data = hasState ? try JSONEncoder().encode(state) : nil + try stateStore.save(data) + lastPersistenceError = nil + } catch { + lastPersistenceError = error + throw error + } + } } @MainActor @@ -275,9 +466,15 @@ final class PrivateMacShareController: ObservableObject { defaults: UserDefaults = .standard ) { self.desktopRegistration = desktopRegistration + let registrationStateStore = UserDefaultsDesktopHostRegistrationStateStore(defaults: defaults) desktopRegistrationLifecycle = registrationLifecycle - ?? desktopRegistration.map { DesktopHostRegistrationLifecycle(registration: $0) } + ?? desktopRegistration.map { + DesktopHostRegistrationLifecycle( + registration: $0, + stateStore: registrationStateStore + ) + } self.defaults = defaults registryPhase = desktopRegistration == nil ? .notConfigured : .notPublished let savedDisplayID = defaults.object(forKey: Self.selectedDisplayDefaultsKey) as? Int @@ -491,17 +688,19 @@ final class PrivateMacShareController: ObservableObject { phase = .idle } - func stopAndWaitForCleanup() async { + func stopAndWaitForCleanup() async -> Bool { await stop() let cleanupTask = registrationTask await cleanupTask?.value - guard let desktopRegistrationLifecycle else { return } + guard let desktopRegistrationLifecycle else { return true } do { try await desktopRegistrationLifecycle.removePublishedIdentities() registryPhase = .notPublished + return true } catch { registryPhase = .failed(error.localizedDescription) notice = error.localizedDescription + return desktopRegistrationLifecycle.hasDurableRecoveryState } } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 2d5ac2b0..4cc89ee0 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -686,7 +686,7 @@ struct PrivateMacShareTests { defaults: defaults ) - await controller.stopAndWaitForCleanup() + #expect(await controller.stopAndWaitForCleanup()) #expect(controller.registryPhase == .notPublished) #expect( @@ -755,6 +755,130 @@ struct PrivateMacShareTests { #expect(controller.registryPhase == .notPublished) } + @Test @MainActor + func applicationTerminationRetainsAmbiguousPublicationForRelaunchCleanup() async throws { + let identity = desktopIdentity(name: "durable-cleanup", address: "100.64.12.54") + let registration = RecoverableAmbiguousDesktopRegistration(recoverFailures: 1) + let suiteName = "CrabfleetMacTests.\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suiteName)) + defer { defaults.removePersistentDomain(forName: suiteName) } + let stateStore = UserDefaultsDesktopHostRegistrationStateStore(defaults: defaults) + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "durable-publication" }, + stateStore: stateStore + ) + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + let controller = PrivateMacShareController( + runner: StaticTailscaleRunner(output: statusJSON()), + desktopRegistration: registration, + registrationLifecycle: lifecycle, + defaults: defaults + ) + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) } + ) + + #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) + #expect(await waitUntilAsync { replies == [true] }) + if case .failed = controller.registryPhase { + // Expected: cleanup failed, but its exact retry identity was persisted. + } else { + Issue.record("expected failed registry cleanup state") + } + + let reloadedLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore + ) + try await reloadedLifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register("durable-publication"), + .recover("durable-publication"), + .recover("durable-publication"), + .unregister("recovered:durable-publication"), + ] + ) + } + + @Test @MainActor + func persistedCleanupRecoversOwnershipWithoutStoringTheToken() async throws { + let identity = desktopIdentity(name: "persisted-cleanup", address: "100.64.12.56") + let registration = IdentityAwareAmbiguousDesktopRegistration(uncertainPublicationIDs: []) + let stateStore = ToggleDesktopRegistrationStateStore() + do { + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "persisted-publication" }, + stateStore: stateStore + ) + try await lifecycle.publish(identity: identity, port: 5_901) + } + + let persistedText = String(decoding: try #require(stateStore.data), as: UTF8.self) + #expect(!persistedText.contains("token:persisted-publication")) + + let reloadedLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore + ) + try await reloadedLifecycle.removePublishedIdentities() + + #expect( + await registration.events + == [ + .register(identity.ipv4Address, 5_901, "persisted-publication"), + .recover(identity.ipv4Address, "persisted-publication"), + .unregister(identity.ipv4Address, "recovered:persisted-publication"), + ] + ) + } + + @Test @MainActor + func applicationTerminationIsCancelledWhenRecoveryStateCannotBeSaved() async throws { + let identity = desktopIdentity(name: "unsaved-cleanup", address: "100.64.12.55") + let registration = RecoverableAmbiguousDesktopRegistration() + let stateStore = ToggleDesktopRegistrationStateStore() + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "unsaved-publication" }, + stateStore: stateStore + ) + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + stateStore.failsWrites = true + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: StaticTailscaleRunner(output: statusJSON()), + desktopRegistration: registration, + registrationLifecycle: lifecycle, + defaults: defaults + ) + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) } + ) + + #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) + #expect(await waitUntilAsync { replies == [false] }) + if case .failed = controller.registryPhase { + // Expected: the application remains alive because recovery was not persisted. + } else { + Issue.record("expected failed registry persistence state") + } + } + @Test func privateShareCanStartViewOnlyWithoutAccessibility() { #expect( @@ -2051,6 +2175,62 @@ private actor AmbiguousDesktopRegistration: DesktopHostRegistering { } } +private actor RecoverableAmbiguousDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register(String) + case recover(String) + case unregister(String) + } + + private var recoverFailures: Int + private(set) var events: [Event] = [] + + init(recoverFailures: Int = 0) { + self.recoverFailures = recoverFailures + } + + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + events.append(.register(publicationID)) + throw DesktopHostRegistrationResultUncertainError(message: "response lost") + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + events.append(.recover(publicationID)) + if recoverFailures > 0 { + recoverFailures -= 1 + throw DesktopRegistrationTestError.failed + } + return "recovered:\(publicationID)" + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + events.append(.unregister(ownershipToken ?? "")) + } +} + +@MainActor +private final class ToggleDesktopRegistrationStateStore: + DesktopHostRegistrationStateStoring +{ + var failsWrites = false + private(set) var data: Data? + + func load() throws -> Data? { + data + } + + func save(_ data: Data?) throws { + if failsWrites { + throw DesktopRegistrationTestError.failed + } + self.data = data + } +} + private actor IdentityAwareAmbiguousDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String, UInt16, String) From 85f097629524de0b9b93855d6f2d42d334458aa9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:18:05 +0200 Subject: [PATCH 195/242] fix(actions): retain runner generation authority --- src/github-actions-runner.ts | 16 +---------- tests/github-actions-runner.test.ts | 43 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index 7d6d1552..bc0c6a1c 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -113,11 +113,7 @@ export function acceptGitHubActionsRunnerInput( queue.bytes -= input.payload.byteLength; }); queue.tail = queued; - return queued - .finally(() => { - deleteIdleRunnerInputQueue(socket, queue, queued); - }) - .then(() => true); + return queued.then(() => true); } function sendRunnerInputAcknowledgement( @@ -158,13 +154,3 @@ function isActiveRunnerInputQueue( } return true; } - -function deleteIdleRunnerInputQueue( - socket: GitHubActionsRelaySocket, - queue: RunnerInputQueue, - tail: Promise, -): void { - if (runnerInputQueues.get(socket) === queue && queue.tail === tail && queue.frames === 0) { - runnerInputQueues.delete(socket); - } -} diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index 06b7d4a1..b6442cd6 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -403,6 +403,49 @@ test("runner copies the relay generation into its acknowledgement", async () => }); }); +test("runner retains its relay generation after the input queue drains", async () => { + const socket = relaySocket(); + const writes: string[] = []; + + assert.equal( + await acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("first", "first", "generation-one"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + ), + true, + ); + assert.equal( + await acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("replacement", "replacement", "generation-two"), + async () => { + assert.fail("replacement input must not reach the PTY after the queue drains"); + }, + ), + true, + ); + + assert.deepEqual(writes, ["first"]); + assert.deepEqual(socket.closes, [ + { code: 1012, reason: "GitHub Actions runner generation changed" }, + ]); + assert.deepEqual( + socket.sent.map((message) => parseGitHubActionsRelayInputAcknowledgement(message)), + [ + { inputId: "first", accepted: true, generation: "generation-one" }, + { + inputId: "replacement", + accepted: false, + error: "GitHub Actions runner generation changed", + generation: "generation-two", + }, + ], + ); +}); + test("runner rejects failed writes and ignores unframed terminal data", async () => { const socket = relaySocket(); From 006757b7d271a83c6142e12b5f200fc575629fdd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:18:19 +0200 Subject: [PATCH 196/242] fix(credentials): preserve legacy lookup recovery --- ...dential_policy_registration_lookup_ids.sql | 30 ++----------------- ...ndbox-credential-policy-repository.test.ts | 21 +++++++++++-- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/migrations/0037_credential_policy_registration_lookup_ids.sql b/migrations/0037_credential_policy_registration_lookup_ids.sql index 1bad836f..ddd15f17 100644 --- a/migrations/0037_credential_policy_registration_lookup_ids.sql +++ b/migrations/0037_credential_policy_registration_lookup_ids.sql @@ -1,33 +1,9 @@ ALTER TABLE interactive_session_credential_policy_registrations ADD COLUMN lookup_ids_json TEXT; -UPDATE interactive_session_credential_policy_registrations AS registration -SET lookup_ids_json = ( - SELECT json_group_array(lookup_id) - FROM ( - SELECT DISTINCT lookup_id - FROM ( - SELECT policy.lookup_id - FROM interactive_session_credential_policies AS policy - WHERE policy.session_id = registration.session_id - AND policy.sandbox_id = registration.sandbox_id - UNION ALL - SELECT json_extract(rollback.value, '$.policy.sandboxId') - FROM json_each( - CASE - WHEN json_valid(registration.rollback_policies_json) - THEN registration.rollback_policies_json - ELSE '[]' - END - ) AS rollback - WHERE json_type(rollback.value, '$.policy.sandboxId') = 'text' - UNION ALL - SELECT registration.sandbox_id - ) - WHERE typeof(lookup_id) = 'text' AND length(lookup_id) > 0 - ORDER BY lookup_id - ) -); +-- Pre-migration staging rows may already have registered a new-generation +-- Durable Object lookup that D1 cannot reconstruct. Keep those rows nullable +-- so recovery derives the current compatibility lookup set at runtime. DROP TRIGGER IF EXISTS fence_staged_credential_policy_delete; diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 3bcae3b9..b5309dca 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -535,7 +535,7 @@ test("migration leaves live legacy registrations unstaged while old workers rene ); }); -test("lookup identity migration backfills the exact staged legacy lookup set", () => { +test("lookup identity migration preserves compatibility recovery for pre-0037 staging", () => { const sqlite = credentialPolicyDatabase({ applyMigrations: false }); sqlite.exec( readFileSync( @@ -555,6 +555,14 @@ test("lookup identity migration backfills the exact staged legacy lookup set", ( "utf8", ), ); + sqlite + .prepare(` + DELETE FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' + AND sandbox_id = 'sandbox-1' + AND lookup_id = 'do-1' + `) + .run(); sqlite .prepare(` INSERT INTO interactive_session_credential_policy_registrations ( @@ -603,7 +611,16 @@ test("lookup identity migration backfills the exact staged legacy lookup set", ( WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' `) .get(); - assert.deepEqual(JSON.parse(String(row?.lookup_ids_json)), ["do-1", "do-old", "sandbox-1"]); + const env = sqliteRuntimeEnv(sqlite); + assert.equal(row?.lookup_ids_json, null); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds( + row?.lookup_ids_json as string | null, + "sandbox-1", + sandboxLookupIds(env, "sandbox-1"), + ), + ["sandbox-1", "do-1"], + ); assert.equal(row?.repair_generation, null); }); From 67de7e3f803b26c437b08b40e00564367299d626 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:19:47 +0200 Subject: [PATCH 197/242] fix(vnc): synchronize encoding transitions --- .../SDK/Connection/VNCConnection+API.swift | 63 +++++++++------- .../SDK/Connection/VNCConnection.swift | 4 - .../RoyalVNCKitTests/AuditFindingsTests.swift | 74 +++++++++++++++---- 3 files changed, 93 insertions(+), 48 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index d87ebf5a..713d75b9 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -5,24 +5,28 @@ import Foundation #endif private struct PixelFormatTransitionMessage: VNCSendableMessage { - let fenceMessage: VNCProtocol.ClientFence? + let synchronizationFenceMessage: VNCProtocol.ClientFence? let pixelFormatMessage: VNCProtocol.SetPixelFormat let encodingsMessage: VNCProtocol.SetEncodings - let willSendPixelFormat: () throws -> Void + let completionFenceMessage: VNCProtocol.ClientFence? let willSend: () -> Void let didSend: () -> Void let willSendFence: () -> Void let didSendFence: () -> Void let didFailFence: () -> Void - var messageType: UInt8 { fenceMessage?.messageType ?? pixelFormatMessage.messageType } + var messageType: UInt8 { + synchronizationFenceMessage?.messageType ?? pixelFormatMessage.messageType + } var data: Data { - (fenceMessage?.data ?? Data()) + pixelFormatMessage.data + encodingsMessage.data + (synchronizationFenceMessage?.data ?? Data()) + + pixelFormatMessage.data + + encodingsMessage.data + + (completionFenceMessage?.data ?? Data()) } func send(connection: NetworkConnectionWriting) async throws { - try willSendPixelFormat() - if fenceMessage == nil { + if synchronizationFenceMessage == nil { willSend() } else { willSendFence() @@ -30,12 +34,12 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { do { try await connection.write(data: data) } catch { - if fenceMessage != nil { + if synchronizationFenceMessage != nil { didFailFence() } throw error } - if fenceMessage == nil { + if synchronizationFenceMessage == nil { didSend() } else { didSendFence() @@ -46,20 +50,19 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { private struct PixelFormatTransition { let pixelFormat: VNCProtocol.PixelFormat let fenceFlags: VNCProtocol.FenceFlags - let fencePayload: Data? + let synchronizationFencePayload: Data? + let completionFencePayload: Data? } private struct FenceCapabilityProbeMessage: VNCSendableMessage { let fenceMessage: VNCProtocol.ClientFence let pixelFormatMessage: VNCProtocol.SetPixelFormat - let willSendPixelFormat: () throws -> Void let didSend: () -> Void var messageType: UInt8 { fenceMessage.messageType } var data: Data { fenceMessage.data + pixelFormatMessage.data } func send(connection: NetworkConnectionWriting) async throws { - try willSendPixelFormat() try await connection.write(data: data) didSend() } @@ -182,23 +185,29 @@ extension VNCConnection { pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = true pixelFormatTransitionInFlight = pixelFormat - let fencePayload: Data? + let synchronizationFencePayload: Data? + let completionFencePayload: Data? if !fenceFlags.isEmpty { pixelFormatTransitionFenceSequence &+= 1 var sequence = pixelFormatTransitionFenceSequence.bigEndian - fencePayload = withUnsafeBytes(of: &sequence) { Data($0) } - pixelFormatTransitionFencePayload = fencePayload - pixelFormatTransitionRequiredFenceFlags = [.blockBefore, .syncNext] + synchronizationFencePayload = withUnsafeBytes(of: &sequence) { Data($0) } + pixelFormatTransitionFenceSequence &+= 1 + sequence = pixelFormatTransitionFenceSequence.bigEndian + completionFencePayload = withUnsafeBytes(of: &sequence) { Data($0) } + pixelFormatTransitionFencePayload = completionFencePayload + pixelFormatTransitionRequiredFenceFlags = [.blockBefore] pixelFormatTransitionFenceWasSent = false } else { - fencePayload = nil + synchronizationFencePayload = nil + completionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] pixelFormatTransitionFenceWasSent = false } return PixelFormatTransition( pixelFormat: pixelFormat, fenceFlags: fenceFlags, - fencePayload: fencePayload + synchronizationFencePayload: synchronizationFencePayload, + completionFencePayload: completionFencePayload ) } @@ -210,16 +219,17 @@ extension VNCConnection { handleBreakingError(error) return } - let fenceMessage = transition.fencePayload.map { + let synchronizationFenceMessage = transition.synchronizationFencePayload.map { VNCProtocol.ClientFence(flags: transition.fenceFlags, payload: $0) } + let completionFenceMessage = transition.completionFencePayload.map { + VNCProtocol.ClientFence(flags: [.request, .blockBefore], payload: $0) + } let message = PixelFormatTransitionMessage( - fenceMessage: fenceMessage, + synchronizationFenceMessage: synchronizationFenceMessage, pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: transition.pixelFormat), encodingsMessage: VNCProtocol.SetEncodings(encodingTypes: encodingTypes), - willSendPixelFormat: { [weak self] in - try self?.resetZRLECompressionState() - }, + completionFenceMessage: completionFenceMessage, willSend: { [weak self] in self?.beginPixelFormatTransition(transition.pixelFormat) }, @@ -227,15 +237,15 @@ extension VNCConnection { self?.completePixelFormatTransition() }, willSendFence: { [weak self] in - guard let payload = transition.fencePayload else { return } + guard let payload = transition.completionFencePayload else { return } self?.beginPixelFormatTransitionFenceWrite(payload: payload) }, didSendFence: { [weak self] in - guard let payload = transition.fencePayload else { return } + guard let payload = transition.completionFencePayload else { return } self?.schedulePixelFormatTransitionDeadline(payload: payload) }, didFailFence: { [weak self] in - guard let payload = transition.fencePayload else { return } + guard let payload = transition.completionFencePayload else { return } self?.cancelPixelFormatTransitionFenceWrite(payload: payload) } ) @@ -339,9 +349,6 @@ extension VNCConnection { payload: payload ), pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat), - willSendPixelFormat: { [weak self] in - try self?.resetZRLECompressionState() - }, didSend: { [weak self] in self?.didSendPixelFormatFenceCapabilityProbe(payload: payload) } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index ba1c27c2..2216aadc 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -306,10 +306,6 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { return uniqueEncs } - func resetZRLECompressionState() throws { - try sharedZRLEZStream.reset() - } - // MARK: - Public Initializers public init(settings: Settings, logger: VNCLogger, diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index c491c2e7..1a3b0fb8 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -428,20 +428,39 @@ struct AuditFindingsTests { #expect(writer.data[8] == 8) #expect(writer.data[17] == VNCProtocol.SetPixelFormat(pixelFormat: framebuffer.sourcePixelFormat).messageType) #expect(writer.data[37] == VNCProtocol.SetEncodings(encodingTypes: []).messageType) - #expect(setEncodingValues(in: writer.data, at: 37).contains(Int32(VNCFrameEncodingType.raw.rawValue.rawValue))) + let encodingValues = setEncodingValues(in: writer.data, at: 37) + #expect(encodingValues.contains(Int32(VNCFrameEncodingType.raw.rawValue.rawValue))) + let completionFenceOffset = setEncodingsEndOffset(in: writer.data, at: 37) + #expect(writer.data[completionFenceOffset] == VNCProtocol.ClientFence.messageType) + #expect( + writer.data[(completionFenceOffset + 4)..<(completionFenceOffset + 8)] + == Data([0x80, 0, 0, 1]) + ) #expect(connection.state.pixelFormat?.depth == 24) - let payload = Data(writer.data[9..<17]) + let synchronizationPayload = Data(writer.data[9..<17]) + let completionPayload = Data( + writer.data[(completionFenceOffset + 9)..<(completionFenceOffset + 17)] + ) connection.completeFramebufferUpdateRequest() #expect(connection.pixelFormatTransitionDeadlineTask != nil) try connection.handleServerFence( VNCProtocol.ServerFence( messageType: VNCProtocol.ServerFence.messageType, flags: [.blockBefore, .syncNext], - payload: payload + payload: synchronizationPayload ) ) + #expect(connection.state.pixelFormat?.depth == 24) + #expect(connection.pixelFormatTransitionDeadlineTask != nil) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore], + payload: completionPayload + ) + ) #expect(connection.state.pixelFormat?.depth == 8) #expect(connection.state.pixelFormat?.depth == connection.framebuffer?.sourcePixelFormat.depth) #expect(connection.pixelFormatTransitionDeadlineTask == nil) @@ -479,12 +498,14 @@ struct AuditFindingsTests { #expect(!values.contains(Int32(VNCFrameEncodingType.openH264.rawValue.rawValue))) #expect(values.contains(Int32(VNCFrameEncodingType.copyRect.rawValue.rawValue))) #expect(values.contains(Int32(VNCFrameEncodingType.raw.rawValue.rawValue))) + let completionFenceOffset = setEncodingsEndOffset(in: writer.data, at: 37) + #expect(writer.data[completionFenceOffset] == VNCProtocol.ClientFence.messageType) connection.cancelFramebufferUpdateScheduling() } @Test - func resetsZRLECompressionAtPixelFormatProbeAndTransitionBoundaries() async throws { + func preservesZRLECompressionAcrossPixelFormatProbeAndTransitionBoundaries() async throws { let connection = VNCConnection( settings: makeSettings(frameEncodings: [.zrle, .raw]), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -508,9 +529,18 @@ struct AuditFindingsTests { _ = try #require(connection.clientToServerMessageQueue.dequeue()) let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) - try primeZRLEStream(zrle.zStream, byte: 0x41) + let compressedChunks = continuousZlibChunks() + let first = try zrle.zStream.decompressedData( + compressedData: compressedChunks[0], + uncompressedSize: 1_000 + ) + #expect(first == Data(repeating: 0x41, count: 1_000)) try await capabilityProbe.message.send(connection: AuditWritingConnection()) - try verifyFreshZRLEStream(zrle.zStream, byte: 0x42) + let second = try zrle.zStream.decompressedData( + compressedData: compressedChunks[1], + uncompressedSize: 1_000 + ) + #expect(second == Data(repeating: 0x42, count: 1_000)) let capabilityPayload = try #require(connection.pixelFormatFenceCapabilityProbePayload) try connection.handleServerFence( @@ -525,7 +555,11 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) let transition = try #require(connection.clientToServerMessageQueue.dequeue()) try await transition.message.send(connection: AuditWritingConnection()) - try verifyFreshZRLEStream(zrle.zStream, byte: 0x44) + let third = try zrle.zStream.decompressedData( + compressedData: compressedChunks[2], + uncompressedSize: 1_000 + ) + #expect(third == Data(repeating: 0x43, count: 1_000)) connection.cancelFramebufferUpdateScheduling() } @@ -968,17 +1002,25 @@ struct AuditFindingsTests { } } - private func primeZRLEStream(_ stream: ZlibStream, byte: UInt8) throws { - try verifyFreshZRLEStream(stream, byte: byte) + private func setEncodingsEndOffset(in data: Data, at offset: Int) -> Int { + offset + 4 + setEncodingValues(in: data, at: offset).count * 4 } - private func verifyFreshZRLEStream(_ stream: ZlibStream, byte: UInt8) throws { - let expected = Data(repeating: byte, count: 64) - let actual = try stream.decompressedData( - compressedData: ZlibOneShot.deflate(expected), - maximumOutputSize: expected.count - ) - #expect(actual == expected) + private func continuousZlibChunks() -> [Data] { + [ + Data([ + 0x78, 0x9C, 0x72, 0x74, 0x1C, 0x05, 0xA3, 0x60, 0x14, + 0x0C, 0x77, 0, 0, 0, 0, 0xFF, 0xFF, + ]), + Data([ + 0x72, 0x1A, 0x05, 0xA3, 0x60, 0x14, 0x0C, + 0x7B, 0, 0, 0, 0, 0xFF, 0xFF, + ]), + Data([ + 0x72, 0x1E, 0x05, 0xA3, 0x60, 0x14, 0x0C, + 0x7B, 0, 0, 0, 0, 0xFF, 0xFF, + ]), + ] } } From 4877ee525918bf563847100ead58aa1590652cae Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:21:56 +0200 Subject: [PATCH 198/242] fix(macos): quiesce remote input teardown --- .../Sources/CrabfleetMac/MacRemoteInput.swift | 51 ++++++++++++ .../CrabfleetMac/TailnetRFBServer.swift | 32 +++----- .../PrivateMacShareTests.swift | 80 +++++++++++++++++++ 3 files changed, 141 insertions(+), 22 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift index b216a27a..60c00cc7 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/MacRemoteInput.swift @@ -14,6 +14,57 @@ extension RemoteInputForwarding { func releaseAllInput() {} } +final class RemoteInputSessionGate: @unchecked Sendable { + private let input: any RemoteInputForwarding + private let lock = NSLock() + private var acceptingInput = true + private var viewOnly: Bool + + init(input: any RemoteInputForwarding, viewOnly: Bool) { + self.input = input + self.viewOnly = viewOnly + } + + func setViewOnly(_ enabled: Bool) { + withLock { + guard acceptingInput else { return } + let shouldReleaseInput = enabled && !viewOnly + viewOnly = enabled + if shouldReleaseInput { + input.releaseAllInput() + } + } + } + + func keyEvent(down: Bool, keysym: UInt32) { + withLock { + guard acceptingInput, !viewOnly else { return } + input.keyEvent(down: down, keysym: keysym) + } + } + + func pointerEvent(buttonMask: UInt8, x: UInt16, y: UInt16) { + withLock { + guard acceptingInput, !viewOnly else { return } + input.pointerEvent(buttonMask: buttonMask, x: x, y: y) + } + } + + func finish() { + withLock { + guard acceptingInput else { return } + acceptingInput = false + input.releaseAllInput() + } + } + + private func withLock(_ body: () -> T) -> T { + lock.lock() + defer { lock.unlock() } + return body() + } +} + final class MacRemoteInputController: RemoteInputForwarding, @unchecked Sendable { private static let releaseRetryDelay: DispatchTimeInterval = .milliseconds(250) private static let releaseRetryLimit = 120 diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift index 70b291fa..3480cfb7 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetRFBServer.swift @@ -174,6 +174,7 @@ private final class RFBHostSession: @unchecked Sendable { private let capture: MacScreenCapture private let descriptor: CapturedDisplayDescriptor private let input: any RemoteInputForwarding + private let inputGate: RemoteInputSessionGate private let clipboard: (any HostClipboardSyncing)? private let requiredLocalAddress: String private let desktopName: String @@ -200,7 +201,6 @@ private final class RFBHostSession: @unchecked Sendable { private var clientClipboardCaps: VNCExtendedClipboardCaps? private var currentWidth: Int private var currentHeight: Int - private var viewOnly: Bool private var videoEncoder: MacVideoEncoder? private var videoFrameMailbox: VideoMailbox? private var videoPixelMailbox: VideoMailbox? @@ -231,11 +231,11 @@ private final class RFBHostSession: @unchecked Sendable { self.capture = capture self.descriptor = descriptor self.input = input + inputGate = RemoteInputSessionGate(input: input, viewOnly: viewOnly) self.clipboard = clipboard self.requiredLocalAddress = requiredLocalAddress self.desktopName = desktopName self.handshakeTimeout = handshakeTimeout - self.viewOnly = viewOnly self.didAuthorize = didAuthorize self.eventHandler = eventHandler self.didFinish = didFinish @@ -267,11 +267,7 @@ private final class RFBHostSession: @unchecked Sendable { } func setViewOnly(_ enabled: Bool) { - let shouldReleaseInput = withLock { () -> Bool in - defer { viewOnly = enabled } - return enabled && !viewOnly - } - if shouldReleaseInput { input.releaseAllInput() } + inputGate.setViewOnly(enabled) } private func beginProtocolIfNeeded() { @@ -524,23 +520,15 @@ private final class RFBHostSession: @unchecked Sendable { case 4: // KeyEvent let payload = try await io.readExactly(7) - withLock { - if !viewOnly { - input.keyEvent(down: payload[0] != 0, keysym: payload.readUInt32(at: 3)) - } - } + inputGate.keyEvent(down: payload[0] != 0, keysym: payload.readUInt32(at: 3)) case 5: // PointerEvent let payload = try await io.readExactly(5) - withLock { - if !viewOnly { - input.pointerEvent( - buttonMask: payload[0], - x: payload.readUInt16(at: 1), - y: payload.readUInt16(at: 3) - ) - } - } + inputGate.pointerEvent( + buttonMask: payload[0], + x: payload.readUInt16(at: 1), + y: payload.readUInt16(at: 3) + ) case 6: // ClientCutText try await receiveClientCutText(io: io) @@ -1037,7 +1025,7 @@ private final class RFBHostSession: @unchecked Sendable { finishPixelMailbox() let encoder = replaceVideoEncoder(with: nil) encoder?.invalidate() - input.releaseAllInput() + inputGate.finish() clipboard?.detach() connection.cancel() guard encoder != nil else { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 4cc89ee0..fe9e1b6f 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -1521,6 +1521,39 @@ struct PrivateMacShareTests { #expect(MacRemoteInputController.keyCode(for: 0x1F980) == nil) } + @Test + func inputSessionFinishWaitsForProducersAndRejectsLateInput() { + let input = BlockingRemoteInputRecorder() + let gate = RemoteInputSessionGate(input: input, viewOnly: false) + let producerFinished = DispatchSemaphore(value: 0) + let finishAttempted = DispatchSemaphore(value: 0) + let finishCompleted = DispatchSemaphore(value: 0) + + DispatchQueue.global().async { + gate.keyEvent(down: true, keysym: 0x61) + producerFinished.signal() + } + #expect(input.waitForKeyEntry()) + + DispatchQueue.global().async { + finishAttempted.signal() + gate.finish() + finishCompleted.signal() + } + #expect(finishAttempted.wait(timeout: .now() + 1) == .success) + #expect(input.events == [.key(down: true, keysym: 0x61)]) + #expect(finishCompleted.wait(timeout: .now() + 0.01) == .timedOut) + + input.allowKeyReturn() + #expect(producerFinished.wait(timeout: .now() + 1) == .success) + #expect(finishCompleted.wait(timeout: .now() + 1) == .success) + #expect(input.events == [.key(down: true, keysym: 0x61), .release]) + + gate.keyEvent(down: false, keysym: 0x61) + gate.pointerEvent(buttonMask: 0x01, x: 1, y: 1) + #expect(input.events == [.key(down: true, keysym: 0x61), .release]) + } + @Test func retriesHeldInputReleaseAfterAccessibilityReturns() async { let trust = AccessibilityTrust(granted: true) @@ -2364,6 +2397,53 @@ private final class RemoteInputRecorder: RemoteInputForwarding, @unchecked Senda } } +private final class BlockingRemoteInputRecorder: RemoteInputForwarding, @unchecked Sendable { + enum Event: Equatable { + case key(down: Bool, keysym: UInt32) + case pointer + case release + } + + private let lock = NSLock() + private let keyEntered = DispatchSemaphore(value: 0) + private let keyMayReturn = DispatchSemaphore(value: 0) + private var storage: [Event] = [] + + var events: [Event] { + lock.lock() + defer { lock.unlock() } + return storage + } + + func keyEvent(down: Bool, keysym: UInt32) { + lock.lock() + storage.append(.key(down: down, keysym: keysym)) + lock.unlock() + keyEntered.signal() + keyMayReturn.wait() + } + + func pointerEvent(buttonMask: UInt8, x: UInt16, y: UInt16) { + lock.lock() + storage.append(.pointer) + lock.unlock() + } + + func releaseAllInput() { + lock.lock() + storage.append(.release) + lock.unlock() + } + + func waitForKeyEntry() -> Bool { + keyEntered.wait(timeout: .now() + 1) == .success + } + + func allowKeyReturn() { + keyMayReturn.signal() + } +} + private final class RFBEventRecorder: @unchecked Sendable { private let lock = NSLock() private var storage: [TailnetRFBServerEvent] = [] From 3837413e28079588d8e0fcac0499340a9b772265 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:22:19 +0200 Subject: [PATCH 199/242] fix(macos): scope desktop recovery authority --- .../CrabfleetDesktopRegistration.swift | 75 +++++- .../PrivateMacShareController.swift | 103 +++++--- .../PrivateMacShareTests.swift | 234 +++++++++++++++++- 3 files changed, 371 insertions(+), 41 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift index 6a9b1445..6fec9aaa 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetDesktopRegistration.swift @@ -1,5 +1,14 @@ import Foundation +struct DesktopHostRegistrationRecoveryScope: Equatable, Hashable, Sendable { + let apiOrigin: String + let ownerSubject: String +} + +protocol DesktopHostRegistrationRecoveryScoping: Sendable { + func recoveryScope() async throws -> DesktopHostRegistrationRecoveryScope +} + protocol DesktopHostRegistering: Sendable { func register( identity: TailnetIdentity, @@ -77,7 +86,11 @@ actor DesktopHostRegistrationCoordinator { } } -struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable { +struct CrabfleetDesktopRegistration: + DesktopHostRegistering, + DesktopHostRegistrationRecoveryScoping, + @unchecked Sendable +{ private struct RegistrationResponse: Decodable { private enum CodingKeys: String, CodingKey { case host @@ -115,7 +128,16 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable let ownershipToken: String? } + private struct NativeSessionResponse: Decodable { + struct User: Decodable { + let subject: String + } + + let user: User + } + private let baseURL: URL + private let apiOrigin: String private let sessionCookie: String private let transport: any HTTPDataTransport static let ownershipModeHeader = "X-Crabfleet-Ownership-Mode" @@ -142,11 +164,31 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable normalizedURL.deleteLastPathComponent() } guard normalizedURL.path.isEmpty || normalizedURL.path == "/" else { return nil } + guard let apiOrigin = Self.normalizedAPIOrigin(normalizedURL) else { return nil } self.baseURL = normalizedURL + self.apiOrigin = apiOrigin self.sessionCookie = cookie self.transport = transport } + func recoveryScope() async throws -> DesktopHostRegistrationRecoveryScope { + let request = nativeSessionRequest() + let data: Data + let http: HTTPURLResponse + (data, http) = try await transport.data(for: request) + try validate(response: http, for: request, acceptingNotFound: false) + guard + let response = try? JSONDecoder().decode(NativeSessionResponse.self, from: data), + Self.isValidOwnerSubject(response.user.subject) + else { + throw DesktopHostRegistrationError.invalidResponse + } + return DesktopHostRegistrationRecoveryScope( + apiOrigin: apiOrigin, + ownerSubject: response.user.subject + ) + } + func register( identity: TailnetIdentity, port: UInt16, @@ -330,6 +372,21 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable return request } + private func nativeSessionRequest() -> URLRequest { + let url = + baseURL + .appending(path: "api") + .appending(path: "native") + .appending(path: "v1") + .appending(path: "session") + var request = URLRequest(url: url) + request.httpMethod = "GET" + request.timeoutInterval = 15 + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.setValue(sessionCookie, forHTTPHeaderField: "Cookie") + return request + } + static func hostID(identity: TailnetIdentity) -> String { let dnsLabel = identity.dnsName.split(separator: ".").first.map(String.init) ?? "" let normalized = dnsLabel.lowercased().filter { @@ -353,6 +410,22 @@ struct CrabfleetDesktopRegistration: DesktopHostRegistering, @unchecked Sendable return scheme == "http" && (host == "127.0.0.1" || host == "::1") } + private static func normalizedAPIOrigin(_ url: URL) -> String? { + let scheme = url.scheme?.lowercased() + var components = URLComponents() + components.scheme = scheme + components.host = url.host?.lowercased() + if !((scheme == "https" && url.port == 443) || (scheme == "http" && url.port == 80)) { + components.port = url.port + } + return components.url?.absoluteString + } + + private static func isValidOwnerSubject(_ value: String) -> Bool { + !value.isEmpty && value.utf8.count <= 512 + && !value.unicodeScalars.contains(where: CharacterSet.controlCharacters.contains) + } + private static func isValidOwnershipToken(_ value: String) -> Bool { !value.isEmpty && value.utf8.count <= 200 && !value.unicodeScalars.contains { diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 90eb1c47..f69b88cb 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -1,5 +1,6 @@ import AppKit import CoreGraphics +import CryptoKit import Foundation import ServiceManagement @@ -38,16 +39,19 @@ final class PrivateMacShareStopCoordinator { @MainActor protocol DesktopHostRegistrationStateStoring: AnyObject { - func load() throws -> Data? - func save(_ data: Data?) throws + func load(scope: DesktopHostRegistrationRecoveryScope) throws -> Data? + func save(_ data: Data?, scope: DesktopHostRegistrationRecoveryScope) throws } enum DesktopHostRegistrationPersistenceError: LocalizedError { + case missingScope case unreadableState case writeFailed var errorDescription: String? { switch self { + case .missingScope: + "The desktop publication recovery scope is unavailable." case .unreadableState: "The saved desktop publication recovery state is unreadable." case .writeFailed: @@ -70,20 +74,27 @@ final class UserDefaultsDesktopHostRegistrationStateStore: self.key = key } - func load() throws -> Data? { - defaults.data(forKey: key) + func load(scope: DesktopHostRegistrationRecoveryScope) throws -> Data? { + defaults.data(forKey: scopedKey(scope)) } - func save(_ data: Data?) throws { + func save(_ data: Data?, scope: DesktopHostRegistrationRecoveryScope) throws { + let scopedKey = scopedKey(scope) if let data { - defaults.set(data, forKey: key) + defaults.set(data, forKey: scopedKey) } else { - defaults.removeObject(forKey: key) + defaults.removeObject(forKey: scopedKey) } guard defaults.synchronize() else { throw DesktopHostRegistrationPersistenceError.writeFailed } } + + private func scopedKey(_ scope: DesktopHostRegistrationRecoveryScope) -> String { + let scopeData = Data("\(scope.apiOrigin)\u{0}\(scope.ownerSubject)".utf8) + let digest = SHA256.hash(data: scopeData).map { String(format: "%02x", $0) }.joined() + return "\(key).v2.\(digest)" + } } @MainActor @@ -174,6 +185,9 @@ final class DesktopHostRegistrationLifecycle { private let coordinator: DesktopHostRegistrationCoordinator private let createPublicationID: () -> String private let stateStore: (any DesktopHostRegistrationStateStoring)? + private let recoveryScopeProvider: (() async throws -> DesktopHostRegistrationRecoveryScope)? + private var recoveryScope: DesktopHostRegistrationRecoveryScope? + private var stateLoaded = false private var publishedRegistration: PublishedRegistration? private var uncertainRegistrations: [RegistrationTarget] = [] private var pendingRemovals: [PublishedRegistration] = [] @@ -183,31 +197,23 @@ final class DesktopHostRegistrationLifecycle { init( registration: any DesktopHostRegistering, createPublicationID: @escaping () -> String = { UUID().uuidString }, - stateStore: (any DesktopHostRegistrationStateStoring)? = nil + stateStore: (any DesktopHostRegistrationStateStoring)? = nil, + recoveryScopeProvider: (() async throws -> DesktopHostRegistrationRecoveryScope)? = nil ) { coordinator = DesktopHostRegistrationCoordinator(registration: registration) self.createPublicationID = createPublicationID self.stateStore = stateStore - guard let stateStore else { return } - do { - guard let data = try stateStore.load() else { return } - let state = try JSONDecoder().decode(PersistedState.self, from: data) - uncertainRegistrations = state.uncertainRegistrations - publishedRegistration = state.publishedRegistration?.registration - pendingRemovals = state.pendingRemovals.map(\.registration) - } catch { - stateLoadError = DesktopHostRegistrationPersistenceError.unreadableState - } + self.recoveryScopeProvider = recoveryScopeProvider } var hasDurableRecoveryState: Bool { - stateStore != nil && stateLoadError == nil && lastPersistenceError == nil + stateStore != nil && stateLoaded && stateLoadError == nil && lastPersistenceError == nil && (!uncertainRegistrations.isEmpty || publishedRegistration != nil || !pendingRemovals.isEmpty) } func publish(identity: TailnetIdentity, port: UInt16) async throws { - try ensureStateIsReadable() + try await loadStateIfNeeded() let hostID = CrabfleetDesktopRegistration.hostID(identity: identity) let existingTarget = uncertainRegistrations.first { $0.hostID == hostID && $0.identity == identity && $0.port == port @@ -236,11 +242,19 @@ final class DesktopHostRegistrationLifecycle { throw DesktopHostRegistrationSupersededError() } } else { - ownershipToken = try await coordinator.register( - identity: identity, - port: port, - publicationID: target.publicationID - ) + do { + ownershipToken = try await coordinator.register( + identity: identity, + port: port, + publicationID: target.publicationID + ) + } catch { + if !(error is DesktopHostRegistrationResultUncertainError) { + uncertainRegistrations.removeAll { $0 == target } + try persistState() + } + throw error + } } uncertainRegistrations.removeAll { $0 == target } if let publishedRegistration, publishedRegistration.hostID != hostID, @@ -260,7 +274,7 @@ final class DesktopHostRegistrationLifecycle { } func removePublishedIdentities() async throws { - try ensureStateIsReadable() + try await loadStateIfNeeded() var firstError: Error? let uncertainRegistrations = uncertainRegistrations for target in uncertainRegistrations { @@ -331,12 +345,36 @@ final class DesktopHostRegistrationLifecycle { if let firstError { throw firstError } } - private func ensureStateIsReadable() throws { - if let stateLoadError { throw stateLoadError } + private func loadStateIfNeeded() async throws { + guard stateStore != nil, !stateLoaded else { + if let stateLoadError { throw stateLoadError } + return + } + guard let recoveryScopeProvider else { + throw DesktopHostRegistrationPersistenceError.missingScope + } + let scope = try await recoveryScopeProvider() + recoveryScope = scope + do { + if let data = try stateStore?.load(scope: scope) { + let state = try JSONDecoder().decode(PersistedState.self, from: data) + uncertainRegistrations = state.uncertainRegistrations + publishedRegistration = state.publishedRegistration?.registration + pendingRemovals = state.pendingRemovals.map(\.registration) + } + stateLoaded = true + } catch { + stateLoadError = DesktopHostRegistrationPersistenceError.unreadableState + stateLoaded = true + throw DesktopHostRegistrationPersistenceError.unreadableState + } } private func persistState() throws { guard let stateStore else { return } + guard stateLoaded, let recoveryScope else { + throw DesktopHostRegistrationPersistenceError.missingScope + } let state = PersistedState( uncertainRegistrations: uncertainRegistrations, publishedRegistration: publishedRegistration.map(PersistedPublishedRegistration.init), @@ -347,7 +385,7 @@ final class DesktopHostRegistrationLifecycle { || !state.pendingRemovals.isEmpty do { let data = hasState ? try JSONEncoder().encode(state) : nil - try stateStore.save(data) + try stateStore.save(data, scope: recoveryScope) lastPersistenceError = nil } catch { lastPersistenceError = error @@ -467,12 +505,17 @@ final class PrivateMacShareController: ObservableObject { ) { self.desktopRegistration = desktopRegistration let registrationStateStore = UserDefaultsDesktopHostRegistrationStateStore(defaults: defaults) + let recoveryScopeProvider = (desktopRegistration as? any DesktopHostRegistrationRecoveryScoping) + .map { registration in + { try await registration.recoveryScope() } + } desktopRegistrationLifecycle = registrationLifecycle ?? desktopRegistration.map { DesktopHostRegistrationLifecycle( registration: $0, - stateStore: registrationStateStore + stateStore: registrationStateStore, + recoveryScopeProvider: recoveryScopeProvider ) } self.defaults = defaults diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index fe9e1b6f..a605352c 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -763,10 +763,12 @@ struct PrivateMacShareTests { let defaults = try #require(UserDefaults(suiteName: suiteName)) defer { defaults.removePersistentDomain(forName: suiteName) } let stateStore = UserDefaultsDesktopHostRegistrationStateStore(defaults: defaults) + let recoveryScope = desktopRecoveryScope() let lifecycle = DesktopHostRegistrationLifecycle( registration: registration, createPublicationID: { "durable-publication" }, - stateStore: stateStore + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } ) await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { try await lifecycle.publish(identity: identity, port: 5_901) @@ -793,7 +795,8 @@ struct PrivateMacShareTests { let reloadedLifecycle = DesktopHostRegistrationLifecycle( registration: registration, - stateStore: stateStore + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } ) try await reloadedLifecycle.removePublishedIdentities() @@ -813,11 +816,13 @@ struct PrivateMacShareTests { let identity = desktopIdentity(name: "persisted-cleanup", address: "100.64.12.56") let registration = IdentityAwareAmbiguousDesktopRegistration(uncertainPublicationIDs: []) let stateStore = ToggleDesktopRegistrationStateStore() + let recoveryScope = desktopRecoveryScope() do { let lifecycle = DesktopHostRegistrationLifecycle( registration: registration, createPublicationID: { "persisted-publication" }, - stateStore: stateStore + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } ) try await lifecycle.publish(identity: identity, port: 5_901) } @@ -827,7 +832,8 @@ struct PrivateMacShareTests { let reloadedLifecycle = DesktopHostRegistrationLifecycle( registration: registration, - stateStore: stateStore + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } ) try await reloadedLifecycle.removePublishedIdentities() @@ -846,10 +852,12 @@ struct PrivateMacShareTests { let identity = desktopIdentity(name: "unsaved-cleanup", address: "100.64.12.55") let registration = RecoverableAmbiguousDesktopRegistration() let stateStore = ToggleDesktopRegistrationStateStore() + let recoveryScope = desktopRecoveryScope() let lifecycle = DesktopHostRegistrationLifecycle( registration: registration, createPublicationID: { "unsaved-publication" }, - stateStore: stateStore + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } ) await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { try await lifecycle.publish(identity: identity, port: 5_901) @@ -879,6 +887,32 @@ struct PrivateMacShareTests { } } + @Test @MainActor + func persistedRecoveryDoesNotCrossDeployments() async throws { + try await assertPersistedRecoveryIsScoped( + originalScope: desktopRecoveryScope(), + otherScope: desktopRecoveryScope(origin: "https://other.example") + ) + } + + @Test @MainActor + func persistedRecoveryDoesNotCrossAccounts() async throws { + try await assertPersistedRecoveryIsScoped( + originalScope: desktopRecoveryScope(), + otherScope: desktopRecoveryScope(ownerSubject: "github:other") + ) + } + + @Test @MainActor + func definitiveRegistrationHTTPFailureDoesNotBecomeRecoveryIntent() async throws { + try await assertDefinitiveRegistrationFailureClearsIntent(.httpStatus(403)) + } + + @Test @MainActor + func redirectedRegistrationDoesNotBecomeRecoveryIntent() async throws { + try await assertDefinitiveRegistrationFailureClearsIntent(.redirect) + } + @Test func privateShareCanStartViewOnlyWithoutAccessibility() { #expect( @@ -1052,6 +1086,43 @@ struct PrivateMacShareTests { #expect(removal.httpBody == nil) } + @Test + func desktopRegistrationScopesRecoveryToNormalizedOriginAndStableOwner() async throws { + let transport = DesktopRegistrationTransport { request in + let responseURL = try #require(request.url) + #expect(responseURL.host?.lowercased() == "fleet.example") + #expect(responseURL.path == "/api/native/v1/session") + #expect(request.httpMethod == "GET") + #expect(request.value(forHTTPHeaderField: "Cookie") == "crabbox_session=secret") + return ( + Data(#"{"user":{"subject":"github:123"}}"#.utf8), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: nil + )) + ) + } + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://FLEET.EXAMPLE:443/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + + #expect( + try await registration.recoveryScope() + == DesktopHostRegistrationRecoveryScope( + apiOrigin: "https://fleet.example", + ownerSubject: "github:123" + ) + ) + } + @Test func desktopRegistrationReturnsTheServerOwnershipToken() async throws { let transport = DesktopRegistrationTransport { request in @@ -1973,6 +2044,90 @@ struct PrivateMacShareTests { let assembly = try #require(contents.range(of: "mkdir -p \"$macos_dir\" \"$resources_dir\"")) #expect(removal.lowerBound < assembly.lowerBound) } + + @MainActor + private func assertPersistedRecoveryIsScoped( + originalScope: DesktopHostRegistrationRecoveryScope, + otherScope: DesktopHostRegistrationRecoveryScope + ) async throws { + let identity = desktopIdentity(name: "scoped-recovery", address: "100.64.12.57") + let registration = RecoverableAmbiguousDesktopRegistration() + let stateStore = ToggleDesktopRegistrationStateStore() + let originalLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { "scoped-publication" }, + stateStore: stateStore, + recoveryScopeProvider: { originalScope } + ) + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await originalLifecycle.publish(identity: identity, port: 5_901) + } + + let otherLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore, + recoveryScopeProvider: { otherScope } + ) + try await otherLifecycle.removePublishedIdentities() + #expect(await registration.events == [.register("scoped-publication")]) + + let reloadedLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore, + recoveryScopeProvider: { originalScope } + ) + try await reloadedLifecycle.removePublishedIdentities() + #expect( + await registration.events + == [ + .register("scoped-publication"), + .recover("scoped-publication"), + .unregister("recovered:scoped-publication"), + ] + ) + } + + @MainActor + private func assertDefinitiveRegistrationFailureClearsIntent( + _ failure: DefinitiveRegistrationFailureTransport.Failure + ) async throws { + let transport = DefinitiveRegistrationFailureTransport(failure: failure) + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: transport + )) + let stateStore = ToggleDesktopRegistrationStateStore() + let recoveryScope = desktopRecoveryScope() + var publicationIDs = ["publication-a", "publication-b"] + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + createPublicationID: { publicationIDs.removeFirst() }, + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } + ) + let identity = desktopIdentity(name: "definitive-failure", address: "100.64.12.58") + + await #expect(throws: DesktopHostRegistrationError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + await #expect(throws: DesktopHostRegistrationError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + #expect(await transport.publicationIDs == ["publication-a", "publication-b"]) + #expect(stateStore.data(for: recoveryScope) == nil) + + let reloadedLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } + ) + try await reloadedLifecycle.removePublishedIdentities() + #expect(await transport.publicationIDs == ["publication-a", "publication-b"]) + } } private struct StaticTailscaleRunner: TailscaleCommandRunning { @@ -2250,17 +2405,22 @@ private final class ToggleDesktopRegistrationStateStore: DesktopHostRegistrationStateStoring { var failsWrites = false - private(set) var data: Data? + private var dataByScope: [DesktopHostRegistrationRecoveryScope: Data] = [:] + var data: Data? { dataByScope.values.first } - func load() throws -> Data? { - data + func load(scope: DesktopHostRegistrationRecoveryScope) throws -> Data? { + dataByScope[scope] } - func save(_ data: Data?) throws { + func save(_ data: Data?, scope: DesktopHostRegistrationRecoveryScope) throws { if failsWrites { throw DesktopRegistrationTestError.failed } - self.data = data + dataByScope[scope] = data + } + + func data(for scope: DesktopHostRegistrationRecoveryScope) -> Data? { + dataByScope[scope] } } @@ -2526,6 +2686,50 @@ private final class DesktopRegistrationTransport: HTTPDataTransport { func close() {} } +private actor DefinitiveRegistrationFailureTransport: HTTPDataTransport { + enum Failure { + case httpStatus(Int) + case redirect + } + + let failure: Failure + private(set) var publicationIDs: [String] = [] + + init(failure: Failure) { + self.failure = failure + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let requestURL = try #require(request.url) + let publicationID = try #require( + request.value(forHTTPHeaderField: CrabfleetDesktopRegistration.publicationIDHeader) + ) + publicationIDs.append(publicationID) + let responseURL: URL + let statusCode: Int + switch failure { + case .httpStatus(let status): + responseURL = requestURL + statusCode = status + case .redirect: + responseURL = try #require(URL(string: "https://login.example.test/desktop-host")) + statusCode = 200 + } + return ( + Data(), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: statusCode, + httpVersion: nil, + headerFields: nil + )) + ) + } + + nonisolated func close() {} +} + private actor LegacyDesktopServerTransport: HTTPDataTransport { enum Event: Equatable { case register @@ -2593,3 +2797,13 @@ private func waitUntilAsync( } return await condition() } + +private func desktopRecoveryScope( + origin: String = "https://fleet.example", + ownerSubject: String = "github:123" +) -> DesktopHostRegistrationRecoveryScope { + DesktopHostRegistrationRecoveryScope( + apiOrigin: origin, + ownerSubject: ownerSubject + ) +} From 51c712b948bb82a2ce5e25eb9e8abf4b852bce37 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:22:47 +0200 Subject: [PATCH 200/242] docs(changelog): record final protocol fences --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b806bad0..e1e0dea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, clearing only definitive failed publication intent, quiescing remote-input producers before final release, and synchronizing complete RFB pixel-format and encoding transitions without resetting the peer-owned ZRLE stream. - Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, requiring replayable runtime-adapter deletion tombstones, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes with exact current-identity fallback for mixed-version rows, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. From 653df8f2819bc4abea8de213c5b216298da3b68e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:32:13 +0200 Subject: [PATCH 201/242] fix(vnc): reject weak UltraVNC DH elements --- ...NCMSLogonIIDiffieHellmanKeyAgreement.swift | 4 +- .../SecurityAndInputTests.swift | 48 ++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift index 8fa45cd4..79c6716a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/UltraVNCMSLogonII/UltraVNCMSLogonIIDiffieHellmanKeyAgreement.swift @@ -48,7 +48,7 @@ private extension VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAg guard modulusNum > 3, modulusNum < maxNum, generatorNum > 1, - generatorNum < modulusNum else { + generatorNum <= modulusNum - 2 else { return nil } @@ -80,7 +80,7 @@ private extension VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAg privNum > 1, privNum < modulusNum, respNum > 1, - respNum < modulusNum else { + respNum <= modulusNum - 2 else { return nil } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift index f118b78a..ab31787a 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -5,8 +5,10 @@ import Testing struct SecurityAndInputTests { typealias ARDKeyAgreement = VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement + typealias UltraVNCKeyAgreement = + VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAgreement typealias UltraVNCBigNum = - VNCProtocol.UltraVNCMSLogonIIAuthentication.DiffieHellmanKeyAgreement.UltraVNCBigNum + UltraVNCKeyAgreement.UltraVNCBigNum @Test func rejectsWeakAppleRemoteDesktopModuli() { @@ -129,6 +131,46 @@ struct SecurityAndInputTests { } } + @Test + func rejectsUltraVNCPMinusOneKeyAgreementElements() { + let modulus = ultraVNCValue(17) + + #expect( + UltraVNCKeyAgreement( + generator: ultraVNCValue(16), + modulus: modulus, + resp: ultraVNCValue(3) + ) == nil + ) + #expect( + UltraVNCKeyAgreement( + generator: ultraVNCValue(3), + modulus: modulus, + resp: ultraVNCValue(16) + ) == nil + ) + } + + @Test + func acceptsUltraVNCPMinusTwoKeyAgreementElements() { + let modulus = ultraVNCValue(17) + + #expect( + UltraVNCKeyAgreement( + generator: ultraVNCValue(15), + modulus: modulus, + resp: ultraVNCValue(3) + ) != nil + ) + #expect( + UltraVNCKeyAgreement( + generator: ultraVNCValue(3), + modulus: modulus, + resp: ultraVNCValue(15) + ) != nil + ) + } + @Test func encodesCharactersAsX11KeySyms() { #expect(VNCKeyCode.withCharacter("A").map(\.rawValue) == [0x41]) @@ -153,4 +195,8 @@ struct SecurityAndInputTests { private func paddedARDValue(_ value: UInt8) -> Data { Data(repeating: 0, count: 127) + Data([value]) } + + private func ultraVNCValue(_ value: UInt64) -> Data { + withUnsafeBytes(of: value.bigEndian) { Data($0) } + } } From a0bb61d9126ba81cdaba2086d5354cc95c55c36c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:34:04 +0200 Subject: [PATCH 202/242] fix(macos): allow idle quit after recovery lookup failure --- .../PrivateMacShareController.swift | 8 +- .../PrivateMacShareTests.swift | 99 ++++++++++++++- ...ndbox-credential-policy-cleanup-service.ts | 23 +++- .../sandbox-credential-policy-repository.ts | 15 +++ .../sandbox-credential-policy-scanner.ts | 35 ++++-- src/worker/session-control-policy.ts | 54 +++++++- .../sandbox-credential-policy-cleanup.test.ts | 4 +- ...ndbox-credential-policy-repository.test.ts | 117 ++++++++++++++---- .../sandbox-credential-policy-scanner.test.ts | 2 + 9 files changed, 308 insertions(+), 49 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index f69b88cb..fc486e7f 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -212,6 +212,12 @@ final class DesktopHostRegistrationLifecycle { || !pendingRemovals.isEmpty) } + var canTerminateAfterCleanupFailure: Bool { + let hasActiveState = + !uncertainRegistrations.isEmpty || publishedRegistration != nil || !pendingRemovals.isEmpty + return !hasActiveState || hasDurableRecoveryState + } + func publish(identity: TailnetIdentity, port: UInt16) async throws { try await loadStateIfNeeded() let hostID = CrabfleetDesktopRegistration.hostID(identity: identity) @@ -743,7 +749,7 @@ final class PrivateMacShareController: ObservableObject { } catch { registryPhase = .failed(error.localizedDescription) notice = error.localizedDescription - return desktopRegistrationLifecycle.hasDurableRecoveryState + return desktopRegistrationLifecycle.canTerminateAfterCleanupFailure } } diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index a605352c..17b3f94c 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -755,6 +755,30 @@ struct PrivateMacShareTests { #expect(controller.registryPhase == .notPublished) } + @Test @MainActor + func idleApplicationTerminationContinuesWhenRecoveryServerIsUnavailable() async throws { + try await assertIdleApplicationTerminationContinues { _ in + throw URLError(.cannotConnectToHost) + } + } + + @Test @MainActor + func idleApplicationTerminationContinuesWhenRecoverySessionHasExpired() async throws { + try await assertIdleApplicationTerminationContinues { request in + let responseURL = try #require(request.url) + return ( + Data(), + try #require( + HTTPURLResponse( + url: responseURL, + statusCode: 401, + httpVersion: nil, + headerFields: nil + )) + ) + } + } + @Test @MainActor func applicationTerminationRetainsAmbiguousPublicationForRelaunchCleanup() async throws { let identity = desktopIdentity(name: "durable-cleanup", address: "100.64.12.54") @@ -848,7 +872,7 @@ struct PrivateMacShareTests { } @Test @MainActor - func applicationTerminationIsCancelledWhenRecoveryStateCannotBeSaved() async throws { + func applicationTerminationContinuesWhenDurableRecoveryCannotBeUpdated() async throws { let identity = desktopIdentity(name: "unsaved-cleanup", address: "100.64.12.55") let registration = RecoverableAmbiguousDesktopRegistration() let stateStore = ToggleDesktopRegistrationStateStore() @@ -879,14 +903,47 @@ struct PrivateMacShareTests { ) #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) - #expect(await waitUntilAsync { replies == [false] }) + #expect(await waitUntilAsync { replies == [true] }) + #expect(stateStore.data != nil) if case .failed = controller.registryPhase { - // Expected: the application remains alive because recovery was not persisted. + // Expected: the existing durable retry identity remains available after relaunch. } else { Issue.record("expected failed registry persistence state") } } + @Test @MainActor + func applicationTerminationIsCancelledForUnpersistedActiveCleanup() async throws { + let identity = desktopIdentity(name: "active-cleanup", address: "100.64.12.59") + let registration = RecordingDesktopRegistration( + unregisterFailures: [identity.dnsName: 2] + ) + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + try await lifecycle.publish(identity: identity, port: 5_901) + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: StaticTailscaleRunner(output: statusJSON()), + desktopRegistration: registration, + registrationLifecycle: lifecycle, + defaults: defaults + ) + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) } + ) + + #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) + #expect(await waitUntilAsync { replies == [false] }) + if case .failed = controller.registryPhase { + // Expected: no durable retry state exists for the active registration. + } else { + Issue.record("expected failed active cleanup state") + } + } + @Test @MainActor func persistedRecoveryDoesNotCrossDeployments() async throws { try await assertPersistedRecoveryIsScoped( @@ -2087,6 +2144,42 @@ struct PrivateMacShareTests { ) } + @MainActor + private func assertIdleApplicationTerminationContinues( + transportHandler: @escaping (URLRequest) throws -> (Data, HTTPURLResponse) + ) async throws { + let registration = try #require( + CrabfleetDesktopRegistration( + environment: [ + "CRABFLEET_API_URL": "https://fleet.example/api/fleet", + "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", + ], + transport: DesktopRegistrationTransport(handler: transportHandler) + )) + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: StaticTailscaleRunner(output: statusJSON()), + desktopRegistration: registration, + defaults: defaults + ) + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) } + ) + + #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) + #expect(await waitUntilAsync { replies == [true] }) + #expect(controller.phase == .idle) + if case .failed = controller.registryPhase { + // Recovery remains available for a later launch without blocking this idle quit. + } else { + Issue.record("expected recovery lookup failure") + } + } + @MainActor private func assertDefinitiveRegistrationFailureClearsIntent( _ failure: DefinitiveRegistrationFailureTransport.Failure diff --git a/src/worker/sandbox-credential-policy-cleanup-service.ts b/src/worker/sandbox-credential-policy-cleanup-service.ts index 0772527b..00834637 100644 --- a/src/worker/sandbox-credential-policy-cleanup-service.ts +++ b/src/worker/sandbox-credential-policy-cleanup-service.ts @@ -16,6 +16,7 @@ import { safeProviderError } from "./provisioning/result.ts"; import { queueSandboxCredentialPolicyCleanup, sandboxCredentialPolicyCleanupAuthorizedCondition, + sandboxCredentialPolicyPersistedLookupIds, sandboxLookupIds, } from "./sandbox-credential-policy-repository.ts"; import { restoreSandboxCredentialPolicyRollback } from "./sandbox-credential-policy-rollback.ts"; @@ -23,7 +24,10 @@ import { scanCredentialPolicyCleanupPage } from "./sandbox-credential-policy-sca import { isCurrentSandboxLease, sandboxLeaseInfo } from "./sandbox-lease.ts"; import { isSandboxSessionAlreadyGone } from "./sandbox-session-errors.ts"; import { sandboxControlStub } from "./session-control-do.ts"; -import { sandboxCredentialPolicyRegistrationLookupIds } from "./session-control-policy.ts"; +import { + sandboxCredentialPolicyRegistrationLookupIds, + sandboxCredentialPolicyRollbackLookupIds, +} from "./session-control-policy.ts"; import { finalizeTerminalInteractiveSession } from "./session-terminal-finalization.ts"; const credentialPolicyCleanupLimit = 8; @@ -130,11 +134,28 @@ async function reconcileStagedCredentialPolicyRegistration( .executeTakeFirst(); if ((claimed.numUpdatedRows ?? 0n) === 0n) return; try { + const persistedLookupIds = await sandboxCredentialPolicyPersistedLookupIds( + env, + registration.session_id, + registration.sandbox_id, + ); + let rollbackLookupIds: string[] = []; + if (registration.rollback_policies_json !== null) { + try { + rollbackLookupIds = sandboxCredentialPolicyRollbackLookupIds( + registration.rollback_policies_json, + registration.session_id, + ); + } catch { + // Malformed rollback state cannot authorize additional cleanup identities. + } + } await Promise.all( sandboxCredentialPolicyRegistrationLookupIds( registration.lookup_ids_json, registration.sandbox_id, sandboxLookupIds(env, registration.sandbox_id), + [...persistedLookupIds, ...rollbackLookupIds], ).map((lookupId) => unregisterSandboxCredentialPolicyLookup( env, diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index a198c701..24f9829d 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -156,6 +156,21 @@ export async function sandboxCredentialPolicyLookupIdsForGeneration( return rows.map((row) => row.lookup_id); } +export async function sandboxCredentialPolicyPersistedLookupIds( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, +): Promise { + const rows = await database(env) + .selectFrom("interactive_session_credential_policies") + .select("lookup_id") + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .orderBy("lookup_id") + .execute(); + return rows.map((row) => row.lookup_id); +} + export async function sandboxCredentialPolicyHasDurableOwner( env: RuntimeEnv, lookupId: string, diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index 35c63c3a..d43b3d57 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -14,12 +14,14 @@ import { finishSandboxCredentialPolicyRegistration, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyCleanupAuthorizedCondition, + sandboxCredentialPolicyPersistedLookupIds, sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; import { sandboxLeaseInfo, sandboxLeasePrefix } from "./sandbox-lease.ts"; import { sandboxCredentialPolicyRegistrationLookupIds, + sandboxCredentialPolicyRollbackLookupIds, type SandboxCredentialPolicyRegistration, } from "./session-control-policy.ts"; @@ -339,16 +341,26 @@ async function scanStagedCredentialPolicyRegistrations( LIMIT ${credentialPolicyScanLimit} `.execute(db); for (const row of result.rows) { - const registration: SandboxCredentialPolicyRegistration = { - generation: row.registration_generation, - claim: row.registration_claim, - lookupIds: sandboxCredentialPolicyRegistrationLookupIds( - row.lookup_ids_json, - row.sandbox_id, - sandboxLookupIds(env, row.sandbox_id), - ), - }; try { + const rollbackLookupIds = + row.rollback_policies_json === null + ? [] + : sandboxCredentialPolicyRollbackLookupIds(row.rollback_policies_json, row.session_id); + const persistedLookupIds = await sandboxCredentialPolicyPersistedLookupIds( + env, + row.session_id, + row.sandbox_id, + ); + const registration: SandboxCredentialPolicyRegistration = { + generation: row.registration_generation, + claim: row.registration_claim, + lookupIds: sandboxCredentialPolicyRegistrationLookupIds( + row.lookup_ids_json, + row.sandbox_id, + sandboxLookupIds(env, row.sandbox_id), + [...persistedLookupIds, ...rollbackLookupIds], + ), + }; const ownershipFence = credentialPolicyScanOwnershipFence(row, now); if (!ownershipFence) { await abandonSandboxCredentialPolicyRegistration( @@ -389,7 +401,10 @@ async function scanStagedCredentialPolicyRegistrations( if (row.rollback_policies_json !== null) { if (!restoreRollback) throw new Error("sandbox credential policy rollback is unavailable"); await restoreRollback({ - registration: recovery.registration, + registration: { + ...recovery.registration, + lookupIds: rollbackLookupIds, + }, registrationExpiresAt: recovery.registrationExpiresAt, rollbackJson: row.rollback_policies_json, sessionId: row.session_id, diff --git a/src/worker/session-control-policy.ts b/src/worker/session-control-policy.ts index 26596aa4..a8acf392 100644 --- a/src/worker/session-control-policy.ts +++ b/src/worker/session-control-policy.ts @@ -1,5 +1,6 @@ import { credentialPolicyCleanupMatches, + credentialPolicyRollbackRecord, isCurrentCredentialPolicyGeneration, type CredentialPolicyGenerationRecord, type CredentialPolicyGenerationTombstone, @@ -33,6 +34,7 @@ export function sandboxCredentialPolicyRegistrationLookupIds( value: string | null | undefined, sandboxId: string, expectedLookupIds: readonly string[], + historicalLookupIds: readonly string[] = [], ): string[] { if (value !== null && value !== undefined) { try { @@ -40,10 +42,7 @@ export function sandboxCredentialPolicyRegistrationLookupIds( if ( Array.isArray(parsed) && parsed.length > 0 && - parsed.every( - (lookupId) => - typeof lookupId === "string" && lookupId.length > 0 && lookupId.length <= 200, - ) + parsed.every(validSandboxCredentialPolicyLookupId) ) { const lookupIds = [...new Set(parsed)]; if (lookupIds.includes(sandboxId)) return lookupIds; @@ -53,8 +52,51 @@ export function sandboxCredentialPolicyRegistrationLookupIds( } return [sandboxId]; } - const fallbackLookupIds = [...new Set(expectedLookupIds)]; - return fallbackLookupIds.includes(sandboxId) ? fallbackLookupIds : [sandboxId]; + const currentLookupIds = expectedLookupIds.includes(sandboxId) ? expectedLookupIds : []; + return [ + ...new Set( + [sandboxId, ...currentLookupIds, ...historicalLookupIds].filter( + validSandboxCredentialPolicyLookupId, + ), + ), + ]; +} + +export function sandboxCredentialPolicyRollbackLookupIds( + value: string, + sessionId: string, +): string[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error("sandbox credential policy rollback snapshot is invalid"); + } + if (!Array.isArray(parsed)) { + throw new Error("sandbox credential policy rollback snapshot is invalid"); + } + const generations = new Set(); + const lookupIds = parsed.map((item) => { + const record = credentialPolicyRollbackRecord(item); + const lookupId = record?.policy.sandboxId; + if ( + !record || + record.policy.sessionId !== sessionId || + !validSandboxCredentialPolicyLookupId(lookupId) + ) { + throw new Error("sandbox credential policy rollback snapshot is invalid"); + } + generations.add(record.generation); + return lookupId; + }); + if (new Set(lookupIds).size !== lookupIds.length || generations.size > 1) { + throw new Error("sandbox credential policy rollback snapshot is inconsistent"); + } + return lookupIds; +} + +function validSandboxCredentialPolicyLookupId(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 200; } export function storedSandboxCredentialPolicy( diff --git a/tests/sandbox-credential-policy-cleanup.test.ts b/tests/sandbox-credential-policy-cleanup.test.ts index ea7d2f4b..2874c635 100644 --- a/tests/sandbox-credential-policy-cleanup.test.ts +++ b/tests/sandbox-credential-policy-cleanup.test.ts @@ -214,7 +214,7 @@ test("terminal cleanup atomically stages the session and credential-policy refs" assert.ok(parameters.includes(leaseId)); }); -test("staged cleanup uses the persisted lookup set with an exact current fallback", async () => { +test("staged cleanup recovers durable historical lookup identities", async () => { const source = await readFile( new URL("../src/worker/sandbox-credential-policy-cleanup-service.ts", import.meta.url), "utf8", @@ -224,6 +224,8 @@ test("staged cleanup uses the persisted lookup set with an exact current fallbac const stagedCleanup = source.slice(start, end); assert.match(stagedCleanup, /sandboxCredentialPolicyRegistrationLookupIds/); + assert.match(stagedCleanup, /sandboxCredentialPolicyPersistedLookupIds/); + assert.match(stagedCleanup, /sandboxCredentialPolicyRollbackLookupIds/); assert.match(stagedCleanup, /registration\.lookup_ids_json/); assert.match(stagedCleanup, /sandboxLookupIds\(env, registration\.sandbox_id\)/); }); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index b5309dca..8915c699 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -18,6 +18,7 @@ import { retireObsoleteSandboxCredentialPolicyReference, renewSandboxCredentialPolicyRegistration, sandboxCredentialPolicyLookupIdsForGeneration, + sandboxCredentialPolicyPersistedLookupIds, sandboxCredentialPolicyRegistrationQueries, sandboxLookupIds, stageSandboxCredentialPolicyReferenceRepair, @@ -28,7 +29,10 @@ import { credentialPolicyRegistrationAccepted } from "../src/credential-policy-f import { database } from "../src/worker/database.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import type { SandboxCredentialPolicyRegistration } from "../src/worker/session-control-policy.ts"; -import { sandboxCredentialPolicyRegistrationLookupIds } from "../src/worker/session-control-policy.ts"; +import { + sandboxCredentialPolicyRegistrationLookupIds, + sandboxCredentialPolicyRollbackLookupIds, +} from "../src/worker/session-control-policy.ts"; import type { StoredSandboxCredentialPolicy } from "../src/worker/session-control-policy.ts"; type PreparedStatement = { @@ -194,6 +198,7 @@ function credentialPolicyDatabase(options: { applyMigrations?: boolean } = {}): function sqliteRuntimeEnv( sqlite: DatabaseSync, options: { + durableObjectId?: string; interruptAfterStatement?: number; throwAfterCommit?: boolean; failNextReadAfterBatch?: boolean; @@ -265,7 +270,7 @@ function sqliteRuntimeEnv( } as unknown as D1Database, SANDBOX: { idFromName() { - return { toString: () => "do-1" }; + return { toString: () => options.durableObjectId ?? "do-1" }; }, } as unknown as DurableObjectNamespace, } as RuntimeEnv; @@ -345,6 +350,15 @@ test("staged lookup identity decoder requires the stable sandbox lookup", () => sandboxCredentialPolicyRegistrationLookupIds(null, "sandbox-1", ["sandbox-1", "do-current"]), ["sandbox-1", "do-current"], ); + assert.deepEqual( + sandboxCredentialPolicyRegistrationLookupIds( + null, + "sandbox-1", + ["sandbox-1", "do-current"], + ["do-persisted", "do-rollback", "do-persisted"], + ), + ["sandbox-1", "do-current", "do-persisted", "do-rollback"], + ); assert.deepEqual( sandboxCredentialPolicyRegistrationLookupIds(null, "sandbox-1", ["do-current"]), ["sandbox-1"], @@ -355,6 +369,39 @@ test("staged lookup identity decoder requires the stable sandbox lookup", () => ); }); +test("rollback lookup decoder preserves exact valid historical identities", () => { + const rollbackJson = JSON.stringify([ + { + generation: "generation:existing", + policy: { + allowedHosts: [], + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: "sandbox-1", + sessionId: "IS-42", + }, + }, + { + generation: "generation:existing", + policy: { + allowedHosts: [], + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: "do-old", + sessionId: "IS-42", + }, + }, + ]); + assert.deepEqual(sandboxCredentialPolicyRollbackLookupIds(rollbackJson, "IS-42"), [ + "sandbox-1", + "do-old", + ]); + assert.throws( + () => sandboxCredentialPolicyRollbackLookupIds(rollbackJson, "IS-other"), + /rollback snapshot is invalid/, + ); +}); + test("credential-policy generations reuse exactly one current identity", () => { assert.equal(currentSandboxCredentialPolicyGeneration([]), null); assert.equal( @@ -535,7 +582,7 @@ test("migration leaves live legacy registrations unstaged while old workers rene ); }); -test("lookup identity migration preserves compatibility recovery for pre-0037 staging", () => { +test("pre-0037 recovery preserves current, persisted, and rollback lookup identities", async () => { const sqlite = credentialPolicyDatabase({ applyMigrations: false }); sqlite.exec( readFileSync( @@ -555,14 +602,30 @@ test("lookup identity migration preserves compatibility recovery for pre-0037 st "utf8", ), ); - sqlite - .prepare(` - DELETE FROM interactive_session_credential_policies - WHERE session_id = 'IS-42' - AND sandbox_id = 'sandbox-1' - AND lookup_id = 'do-1' - `) - .run(); + const rollback = [ + { + generation: "generation:existing", + policy: { + allowedHosts: [], + githubCredentialSource: "none", + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: "sandbox-1", + sessionId: "IS-42", + }, + }, + { + generation: "generation:existing", + policy: { + allowedHosts: [], + githubCredentialSource: "none", + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: "do-rollback-old", + sessionId: "IS-42", + }, + }, + ]; sqlite .prepare(` INSERT INTO interactive_session_credential_policy_registrations ( @@ -583,19 +646,7 @@ test("lookup identity migration preserves compatibility recovery for pre-0037 st "generation:staged", "registration:staged", Number.MAX_SAFE_INTEGER, - JSON.stringify([ - { - generation: "generation:existing", - policy: { - allowedHosts: [], - githubCredentialSource: "none", - githubRepo: "openclaw/crabfleet", - owner: "operator", - sandboxId: "do-old", - sessionId: "IS-42", - }, - }, - ]), + JSON.stringify(rollback), ); sqlite.exec( readFileSync( @@ -606,20 +657,32 @@ test("lookup identity migration preserves compatibility recovery for pre-0037 st const row = sqlite .prepare(` - SELECT lookup_ids_json, repair_generation + SELECT lookup_ids_json, repair_generation, rollback_policies_json FROM interactive_session_credential_policy_registrations WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' `) .get(); - const env = sqliteRuntimeEnv(sqlite); + const env = sqliteRuntimeEnv(sqlite, { durableObjectId: "do-current" }); + const persistedLookupIds = await sandboxCredentialPolicyPersistedLookupIds( + env, + "IS-42", + "sandbox-1", + ); + const rollbackLookupIds = sandboxCredentialPolicyRollbackLookupIds( + String(row?.rollback_policies_json), + "IS-42", + ); assert.equal(row?.lookup_ids_json, null); + assert.deepEqual(persistedLookupIds, ["do-1", "sandbox-1"]); + assert.deepEqual(rollbackLookupIds, ["sandbox-1", "do-rollback-old"]); assert.deepEqual( sandboxCredentialPolicyRegistrationLookupIds( row?.lookup_ids_json as string | null, "sandbox-1", sandboxLookupIds(env, "sandbox-1"), + [...persistedLookupIds, ...rollbackLookupIds], ), - ["sandbox-1", "do-1"], + ["sandbox-1", "do-current", "do-1", "do-rollback-old"], ); assert.equal(row?.repair_generation, null); }); diff --git a/tests/sandbox-credential-policy-scanner.test.ts b/tests/sandbox-credential-policy-scanner.test.ts index 22cd4565..222ffa02 100644 --- a/tests/sandbox-credential-policy-scanner.test.ts +++ b/tests/sandbox-credential-policy-scanner.test.ts @@ -144,6 +144,8 @@ test("staged recovery takes a fresh exclusive claim before promotion or rollback stagedRecovery.indexOf("restoreRollback({"), ); assert.match(stagedRecovery, /sandboxCredentialPolicyRegistrationLookupIds/); + assert.match(stagedRecovery, /sandboxCredentialPolicyPersistedLookupIds/); + assert.match(stagedRecovery, /sandboxCredentialPolicyRollbackLookupIds/); assert.match(stagedRecovery, /registration\.lookupIds/); assert.doesNotMatch(stagedRecovery, /renewSandboxCredentialPolicyRegistration/); }); From d74852cb628ac2623c176fef549abb15bde1cad9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:35:31 +0200 Subject: [PATCH 203/242] fix(macos): terminate Tailscale process groups --- .../CrabfleetMac/TailnetIdentity.swift | 146 +++++++++++++++--- .../PrivateMacShareTests.swift | 45 +++++- 2 files changed, 164 insertions(+), 27 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift index 9097e791..82f162ce 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/TailnetIdentity.swift @@ -89,6 +89,7 @@ struct SystemTailscaleCommandRunner: TailscaleCommandRunning { private final class TailscaleCommandExecution: @unchecked Sendable { private static let processDrainTimeout: DispatchTimeInterval = .milliseconds(250) + private static let terminationGracePeriod: TimeInterval = 0.5 private enum StopReason { case cancelled @@ -97,12 +98,15 @@ private final class TailscaleCommandExecution: @unchecked Sendable { } private let lock = NSLock() - private let process = Process() private let outputPipe = Pipe() private let errorPipe = Pipe() private let readGroup = DispatchGroup() + private let executableURL: URL + private let arguments: [String] + private let environment: [String: String] private let timeout: TimeInterval private let maximumOutputBytes: Int + private var processID: pid_t? private var stopReason: StopReason? private var captureShouldStop = false private var standardOutput = Data() @@ -115,14 +119,11 @@ private final class TailscaleCommandExecution: @unchecked Sendable { timeout: TimeInterval, maximumOutputBytes: Int ) { + self.executableURL = executableURL + self.arguments = arguments + self.environment = environment self.timeout = max(0.1, timeout) self.maximumOutputBytes = maximumOutputBytes - process.executableURL = executableURL - process.arguments = arguments - process.environment = environment - process.standardOutput = outputPipe - process.standardError = errorPipe - process.qualityOfService = .userInitiated } func run() throws -> TailscaleCommandResult { @@ -130,8 +131,9 @@ private final class TailscaleCommandExecution: @unchecked Sendable { startCapture(pipe: outputPipe, isStandardOutput: true) startCapture(pipe: errorPipe, isStandardOutput: false) + let pid: pid_t do { - try process.run() + pid = try spawn() } catch { outputPipe.fileHandleForWriting.closeFile() errorPipe.fileHandleForWriting.closeFile() @@ -139,18 +141,40 @@ private final class TailscaleCommandExecution: @unchecked Sendable { throw error } - if currentStopReason() != nil { terminate() } + setProcessID(pid) + outputPipe.fileHandleForWriting.closeFile() + errorPipe.fileHandleForWriting.closeFile() + if currentStopReason() != nil { signalProcessGroup(pid, signal: SIGTERM) } + let clock = ContinuousClock() let deadline = clock.now.advanced(by: .seconds(timeout)) - while process.isRunning { + var terminationDeadline: ContinuousClock.Instant? + var waitStatus: Int32 = 0 + while true { + let waitResult = Darwin.waitpid(pid, &waitStatus, WNOHANG) + if waitResult == pid { break } + if waitResult == -1 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .ECHILD) + } + if currentStopReason() != nil { - terminate() + if terminationDeadline == nil { + signalProcessGroup(pid, signal: SIGTERM) + terminationDeadline = clock.now.advanced(by: .seconds(Self.terminationGracePeriod)) + } else if clock.now >= terminationDeadline! { + signalProcessGroup(pid, signal: SIGKILL) + } } else if clock.now >= deadline { stop(.timedOut) } Thread.sleep(forTimeInterval: 0.01) } - process.waitUntilExit() + + if currentStopReason() != nil { + signalProcessGroup(pid, signal: SIGKILL) + } + clearProcessID(pid) finishCapture() switch currentStopReason() { @@ -165,10 +189,11 @@ private final class TailscaleCommandExecution: @unchecked Sendable { } let result = values() - guard process.terminationStatus == 0 else { + let terminationStatus = Self.terminationStatus(from: waitStatus) + guard terminationStatus == 0 else { let message = result.standardError.trimmingCharacters(in: .whitespacesAndNewlines) throw PrivateMacShareError.commandFailed( - status: process.terminationStatus, + status: terminationStatus, message: String(message.prefix(500)) ) } @@ -253,18 +278,95 @@ private final class TailscaleCommandExecution: @unchecked Sendable { private func stop(_ reason: StopReason) { lock.lock() if stopReason == nil { stopReason = reason } + let pid = processID lock.unlock() - terminate() + if let pid { + signalProcessGroup(pid, signal: SIGTERM) + } + } + + private func spawn() throws -> pid_t { + var fileActions: posix_spawn_file_actions_t? + var attributes: posix_spawnattr_t? + guard posix_spawn_file_actions_init(&fileActions) == 0 else { + throw POSIXError(.ENOMEM) + } + defer { posix_spawn_file_actions_destroy(&fileActions) } + guard posix_spawnattr_init(&attributes) == 0 else { + throw POSIXError(.ENOMEM) + } + defer { posix_spawnattr_destroy(&attributes) } + + let outputRead = outputPipe.fileHandleForReading.fileDescriptor + let outputWrite = outputPipe.fileHandleForWriting.fileDescriptor + let errorRead = errorPipe.fileHandleForReading.fileDescriptor + let errorWrite = errorPipe.fileHandleForWriting.fileDescriptor + posix_spawn_file_actions_addclose(&fileActions, outputRead) + posix_spawn_file_actions_addclose(&fileActions, errorRead) + posix_spawn_file_actions_adddup2(&fileActions, outputWrite, STDOUT_FILENO) + posix_spawn_file_actions_adddup2(&fileActions, errorWrite, STDERR_FILENO) + posix_spawn_file_actions_addclose(&fileActions, outputWrite) + posix_spawn_file_actions_addclose(&fileActions, errorWrite) + + let flags = Int16(POSIX_SPAWN_SETPGROUP) + posix_spawnattr_setflags(&attributes, flags) + posix_spawnattr_setpgroup(&attributes, 0) + + let argv = [executableURL.path] + arguments + let env = environment.map { "\($0.key)=\($0.value)" } + return try withCStringArray(argv) { argumentPointers in + try withCStringArray(env) { environmentPointers in + var pid: pid_t = 0 + let result = posix_spawn( + &pid, + executableURL.path, + &fileActions, + &attributes, + argumentPointers, + environmentPointers + ) + guard result == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: result) ?? .EINVAL) + } + return pid + } + } + } + + private func withCStringArray( + _ strings: [String], + body: ([UnsafeMutablePointer?]) throws -> Result + ) rethrows -> Result { + let pointers = strings.map { strdup($0) } + defer { pointers.forEach { free($0) } } + return try body(pointers + [nil]) + } + + private func setProcessID(_ pid: pid_t) { + lock.lock() + processID = pid + lock.unlock() + } + + private func clearProcessID(_ pid: pid_t) { + lock.lock() + if processID == pid { processID = nil } + lock.unlock() + } + + private func signalProcessGroup(_ pid: pid_t, signal: Int32) { + guard pid > 0 else { return } + if Darwin.kill(-pid, signal) != 0, errno != ESRCH { + return + } } - private func terminate() { - guard process.isRunning else { return } - process.terminate() - let pid = process.processIdentifier - DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 0.5) { [process] in - guard process.isRunning else { return } - _ = Darwin.kill(pid, SIGKILL) + private static func terminationStatus(from waitStatus: Int32) -> Int32 { + let signal = waitStatus & 0x7f + if signal == 0 { + return (waitStatus >> 8) & 0xff } + return 128 + signal } private func finishCapture() { diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 17b3f94c..f8c185e6 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -100,7 +100,7 @@ struct PrivateMacShareTests { } @Test - func tailscaleCommandTimeoutDoesNotWaitForDescendantPipeEOF() async throws { + func tailscaleCommandTimeoutTerminatesDescendantProcessGroup() async throws { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("CrabfleetMacTests.\(UUID().uuidString)") try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -132,10 +132,7 @@ struct PrivateMacShareTests { let descendantPID = try #require( Int32(String(contentsOf: descendantPIDFile, encoding: .utf8)) ) - defer { - _ = Darwin.kill(descendantPID, SIGKILL) - } - #expect(Darwin.kill(descendantPID, 0) == 0) + #expect(await waitUntilAsync { Darwin.kill(descendantPID, 0) != 0 }) #expect(elapsed < .seconds(4)) } @@ -178,6 +175,44 @@ struct PrivateMacShareTests { #expect(elapsed < .seconds(2)) } + @Test + func tailscaleCommandCancellationTerminatesDescendantProcessGroup() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("CrabfleetMacTests.\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let executable = directory.appendingPathComponent("tailscale") + let descendantPIDFile = directory.appendingPathComponent("descendant-pid") + try Data( + """ + #!/bin/sh + ( + trap '' HUP TERM + exec sleep 30 + ) & + printf '%s' "$!" > '\(descendantPIDFile.path)' + exec sleep 30 + """.utf8 + ).write(to: executable) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: executable.path) + + let runner = SystemTailscaleCommandRunner(executableURL: executable, timeout: 30) + let task = Task { + try await runner.run(arguments: ["status"]) + } + #expect(await waitUntilAsync { + FileManager.default.fileExists(atPath: descendantPIDFile.path) + }) + let descendantPID = try #require( + Int32(String(contentsOf: descendantPIDFile, encoding: .utf8)) + ) + task.cancel() + await #expect(throws: CancellationError.self) { + try await task.value + } + #expect(await waitUntilAsync { Darwin.kill(descendantPID, 0) != 0 }) + } + @Test @MainActor func stopInvalidatesAnInFlightPrivateShareStart() async throws { let runner = SuspendedTailscaleRunner() From e472f64b9b25af47aac1177bf8a44e7123d378f4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:35:48 +0200 Subject: [PATCH 204/242] docs(changelog): record final teardown hardening --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e0dea7..5b81aff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, clearing only definitive failed publication intent, quiescing remote-input producers before final release, and synchronizing complete RFB pixel-format and encoding transitions without resetting the peer-owned ZRLE stream. +- Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions without resetting the peer-owned ZRLE stream. - Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, requiring replayable runtime-adapter deletion tombstones, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes with exact current-identity fallback for mixed-version rows, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. From 420acb6906215f33da8f200f285bc5252517cf9d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:46:32 +0200 Subject: [PATCH 205/242] test(assets): generate ignored fixtures before import --- tests/generated-assets.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/generated-assets.test.ts b/tests/generated-assets.test.ts index 041baf12..a6d0af72 100644 --- a/tests/generated-assets.test.ts +++ b/tests/generated-assets.test.ts @@ -1,10 +1,14 @@ import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; import { readFile } from "node:fs/promises"; import test from "node:test"; +import { promisify } from "node:util"; -import { SPEC_MARKDOWN } from "../src/generated.ts"; +const execFileAsync = promisify(execFile); test("generated embedded specification matches the canonical markdown", async () => { + await execFileAsync(process.execPath, ["scripts/generate-assets.mjs"]); + const { SPEC_MARKDOWN } = await import(`../src/generated.ts?spec-assets=${Date.now()}`); const source = await readFile(new URL("../docs/spec.md", import.meta.url), "utf8"); const markdown = source.replace(/^---\n[\s\S]*?\n---\n+/, ""); From 0b813a4f45636dbe75e4447af63b87236c80a5e1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:47:29 +0200 Subject: [PATCH 206/242] fix(terminal): keep viewer timeouts passive --- src/worker/terminal-hub.ts | 6 +++++- tests/terminal-hub.test.ts | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index f14127f8..39fb0e0c 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -835,6 +835,7 @@ function beginTerminalInputAcknowledgement( return; } if (subscription.upstream.readyState === WebSocket.OPEN) { + subscription.markClosing("input acknowledgement timed out"); subscription.upstream.close(1011, "input acknowledgement timed out"); } }, timeoutMs), @@ -1048,7 +1049,10 @@ function terminalCloseMessage(code: number, reason: string): string { function isPassiveTerminalClose(reason: string | undefined): boolean { return ( - reason === "unsubscribed" || reason === "client closed" || reason === "no terminals mounted" + reason === "unsubscribed" || + reason === "client closed" || + reason === "no terminals mounted" || + reason === "input acknowledgement timed out" ); } diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index b66937db..37a9b0c2 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -653,11 +653,15 @@ test("GitHub Actions acknowledgement timeout reports an ambiguous delivery outco const client = socket(); const server = socket(); const upstream = socket(); + const detachedSessions: string[] = []; const hub = new TerminalHub( dependencies(client, server, upstream, { async readSession() { return githubActionsSession; }, + async markDetached(_user, sessionId) { + detachedSessions.push(sessionId); + }, inputAcknowledgementTimeoutMs: 1, }), ); @@ -705,6 +709,9 @@ test("GitHub Actions acknowledgement timeout reports an ambiguous delivery outco error: "terminal input delivery outcome is unknown; the runner may still complete it", }, ]); + upstream.emit("close", { code: 1011, reason: "input acknowledgement timed out" }); + await flushQueues(); + assert.deepEqual(detachedSessions, []); server.emit("close"); }); From e3b0e4bc8027e422dcc5e7f7c5af1cc3a078eead Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:47:29 +0200 Subject: [PATCH 207/242] docs(actions): fence queued runner steering --- docs/github-actions-sessions.md | 45 ++++++++++++++++++++++++++----- tests/github-actions-docs.test.ts | 26 ++++++++++++++---- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index bfd53b6c..97a6b836 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -299,6 +299,8 @@ let pendingInputs = []; let pendingInputBytes = 0; let pendingInputTimer; let inputQueue = Promise.resolve(); +let terminalClosed = false; +let activeGeneration; const terminal = new WebSocket(runnerPtyUrl, "cfr1-framed-io-v2"); terminal.binaryType = "arraybuffer"; @@ -320,7 +322,13 @@ subscribeSteeringExit(() => { terminal.addEventListener("message", (event) => { const input = admitInput(event.data); if (!input) return; - inputQueue = inputQueue.then(() => acceptInput(input)); + inputQueue = inputQueue.then(() => { + if (!inputIsActive(input)) { + releaseInputs([input]); + return; + } + return acceptInput(input); + }); }); async function acceptInput(input) { @@ -361,6 +369,14 @@ function admitInput(data) { } const input = framed ? decodeInput(data) : decodeRawInput(data); if (!input) return null; + if (framed) { + if (activeGeneration === undefined) { + activeGeneration = input.generation; + } else if (input.generation !== activeGeneration) { + sendAck(input, false); + return null; + } + } const nextBytes = admittedInputBytes + input.payload.byteLength; const nextFrames = admittedInputFrames + 1; if (nextBytes > maxAdmittedInputBytes || nextFrames > maxAdmittedInputFrames) { @@ -376,6 +392,14 @@ function admitInput(data) { return input; } +function inputIsActive(input) { + return ( + !terminalClosed && + terminal.readyState === WebSocket.OPEN && + (!framed || input.generation === activeGeneration) + ); +} + function decodeRawInput(data) { if (typeof data === "string") return { payload: encoder.encode(data) }; if (data instanceof ArrayBuffer) return { payload: new Uint8Array(data) }; @@ -499,13 +523,16 @@ function encodeUtf8Output(outputText) { return frame; } -terminal.addEventListener("close", () => { +function deactivateTerminal() { + if (terminalClosed) return; + terminalClosed = true; + activeGeneration = undefined; + rejectInputs(takePendingInputs(), 1001, "terminal closed"); closeSteering(); -}); +} -terminal.addEventListener("error", () => { - closeSteering(); -}); +terminal.addEventListener("close", deactivateTerminal); +terminal.addEventListener("error", deactivateTerminal); ``` Set `CRABFLEET_RUNNER_PTY_URL` to the `runnerPtyUrl` returned by registration. @@ -525,7 +552,11 @@ pending, queued, or blocked in `deliverSteeringInput`, so a stalled steering call cannot retain an unbounded sequence of `MessageEvent` payloads. Framed overflow receives a negative acknowledgement; raw overflow closes the socket because legacy mode has no acknowledgement channel. An incomplete UTF-8 group -expires after one second. +expires after one second. The first framed input pins the relay-owned generation +for that socket. Every admitted input rechecks both that generation and socket +liveness before entering the restricted steering handler; close or error +invalidates the generation and releases buffered or queued input instead of +delivering it through a replacement runner. WebSocket subprotocol selection is fixed during the opening handshake. There is no capability message or mode transition after the socket opens. Older relays diff --git a/tests/github-actions-docs.test.ts b/tests/github-actions-docs.test.ts index f41ece97..10fea060 100644 --- a/tests/github-actions-docs.test.ts +++ b/tests/github-actions-docs.test.ts @@ -26,11 +26,26 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async assert.equal(guide.match(/runnerProtocol/g)?.length, 1); assert.match(guide, /let pendingInputs = \[\]/); assert.match(guide, /let inputQueue = Promise\.resolve\(\)/); + assert.match(guide, /let terminalClosed = false;\s+let activeGeneration;/); assert.match( guide, - /const input = admitInput\(event\.data\);\s+if \(!input\) return;\s+inputQueue = inputQueue\.then\(\(\) => acceptInput\(input\)\)/, + /const input = admitInput\(event\.data\);\s+if \(!input\) return;\s+inputQueue = inputQueue\.then\(\(\) => \{\s+if \(!inputIsActive\(input\)\) \{\s+releaseInputs\(\[input\]\);\s+return;\s+\}\s+return acceptInput\(input\);\s+\}\)/, ); assert.doesNotMatch(guide, /\.then\(\(\) => acceptInput\(event\.data\)\)/); + assert.match( + guide, + /if \(framed\) \{\s+if \(activeGeneration === undefined\) \{\s+activeGeneration = input\.generation;\s+\} else if \(input\.generation !== activeGeneration\) \{\s+sendAck\(input, false\);\s+return null;/, + ); + assert.match( + guide, + /function inputIsActive\(input\) \{\s+return \(\s+!terminalClosed &&\s+terminal\.readyState === WebSocket\.OPEN &&\s+\(!framed \|\| input\.generation === activeGeneration\)/, + ); + assert.match( + guide, + /function deactivateTerminal\(\) \{\s+if \(terminalClosed\) return;\s+terminalClosed = true;\s+activeGeneration = undefined;\s+rejectInputs\(takePendingInputs\(\), 1001, "terminal closed"\);\s+closeSteering\(\);/, + ); + assert.match(guide, /terminal\.addEventListener\("close", deactivateTerminal\)/); + assert.match(guide, /terminal\.addEventListener\("error", deactivateTerminal\)/); assert.match(guide, /pendingInputs\.push\(input\)/); assert.match(guide, /text = decodeCompleteUtf8\(payload\)/); assert.match(guide, /if \(text === null\) \{\s+armPendingInputTimer\(\);\s+return;/); @@ -83,13 +98,14 @@ test("the documented Node runner acknowledges only delivered UTF-8 input", async const messageHandler = guide.indexOf('terminal.addEventListener("message"'); const admission = guide.indexOf("const input = admitInput(event.data);", messageHandler); - const serialization = guide.indexOf( - "inputQueue = inputQueue.then(() => acceptInput(input));", - messageHandler, - ); + const serialization = guide.indexOf("inputQueue = inputQueue.then(() => {", messageHandler); + const activeCheck = guide.indexOf("if (!inputIsActive(input))", serialization); + const deliveryAdmission = guide.indexOf("return acceptInput(input);", activeCheck); assert.ok(messageHandler >= 0); assert.ok(admission > messageHandler); assert.ok(serialization > admission); + assert.ok(activeCheck > serialization); + assert.ok(deliveryAdmission > activeCheck); const timerArm = guide.indexOf("armPendingInputTimer();"); const batchSnapshot = guide.indexOf("const inputs = takePendingInputs();"); From 997d00456e17b16c7a31e791ea1eb378390f9ea1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:49:58 +0200 Subject: [PATCH 208/242] fix(vnc): reset ZRLE on pixel format changes --- .../SDK/Connection/VNCConnection+API.swift | 40 +++++-- .../SDK/Connection/VNCConnection.swift | 18 ++- .../RoyalVNCKitTests/AuditFindingsTests.swift | 109 +++++++++++++++++- 3 files changed, 153 insertions(+), 14 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index 713d75b9..f6978231 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -9,7 +9,7 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { let pixelFormatMessage: VNCProtocol.SetPixelFormat let encodingsMessage: VNCProtocol.SetEncodings let completionFenceMessage: VNCProtocol.ClientFence? - let willSend: () -> Void + let willSend: () throws -> Void let didSend: () -> Void let willSendFence: () -> Void let didSendFence: () -> Void @@ -27,7 +27,7 @@ private struct PixelFormatTransitionMessage: VNCSendableMessage { func send(connection: NetworkConnectionWriting) async throws { if synchronizationFenceMessage == nil { - willSend() + try willSend() } else { willSendFence() } @@ -52,6 +52,7 @@ private struct PixelFormatTransition { let fenceFlags: VNCProtocol.FenceFlags let synchronizationFencePayload: Data? let completionFencePayload: Data? + let sequence: UInt64 } private struct FenceCapabilityProbeMessage: VNCSendableMessage { @@ -185,6 +186,9 @@ extension VNCConnection { pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = true pixelFormatTransitionInFlight = pixelFormat + nextPixelFormatChangeSequence &+= 1 + let sequence = nextPixelFormatChangeSequence + pixelFormatTransitionInFlightSequence = sequence let synchronizationFencePayload: Data? let completionFencePayload: Data? if !fenceFlags.isEmpty { @@ -207,7 +211,8 @@ extension VNCConnection { pixelFormat: pixelFormat, fenceFlags: fenceFlags, synchronizationFencePayload: synchronizationFencePayload, - completionFencePayload: completionFencePayload + completionFencePayload: completionFencePayload, + sequence: sequence ) } @@ -231,7 +236,10 @@ extension VNCConnection { encodingsMessage: VNCProtocol.SetEncodings(encodingTypes: encodingTypes), completionFenceMessage: completionFenceMessage, willSend: { [weak self] in - self?.beginPixelFormatTransition(transition.pixelFormat) + try self?.beginPixelFormatTransition( + transition.pixelFormat, + sequence: transition.sequence + ) }, didSend: { [weak self] in self?.completePixelFormatTransition() @@ -340,6 +348,8 @@ extension VNCConnection { framebufferRequestLock.unlock() return } + nextPixelFormatChangeSequence &+= 1 + pixelFormatFenceCapabilityProbeSequence = nextPixelFormatChangeSequence framebufferRequestLock.unlock() enqueueClientToServerMessage( @@ -424,9 +434,14 @@ extension VNCConnection { framebufferRequestLock.lock() if fence.payload == pixelFormatFenceCapabilityProbePayload || fence.payload == expiredPixelFormatFenceCapabilityProbePayload { + guard let sequence = pixelFormatFenceCapabilityProbeSequence else { + framebufferRequestLock.unlock() + throw VNCError.protocol(.invalidData) + } cancelPixelFormatFenceNegotiationTimeoutLocked() pixelFormatFenceCapabilityProbePayload = nil expiredPixelFormatFenceCapabilityProbePayload = nil + pixelFormatFenceCapabilityProbeSequence = nil state.pixelFormatTransitionFenceFlags = fence.flags.intersection([ .blockBefore, .blockAfter, @@ -435,6 +450,7 @@ extension VNCConnection { let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() + try resetZRLECompressionState(for: sequence) if let transition { enqueuePixelFormatTransition(transition) } @@ -453,7 +469,8 @@ extension VNCConnection { framebufferRequestLock.unlock() throw VNCError.protocol(.invalidData) } - guard let pixelFormat = pixelFormatTransitionInFlight else { + guard let pixelFormat = pixelFormatTransitionInFlight, + let sequence = pixelFormatTransitionInFlightSequence else { framebufferRequestLock.unlock() throw VNCError.protocol(.invalidData) } @@ -463,17 +480,21 @@ extension VNCConnection { pixelFormatTransitionRequiredFenceFlags = [] framebufferRequestLock.unlock() - beginPixelFormatTransition(pixelFormat) + try beginPixelFormatTransition(pixelFormat, sequence: sequence) completePixelFormatTransition() } - private func beginPixelFormatTransition(_ pixelFormat: VNCProtocol.PixelFormat) { - withLifecycleLock { + private func beginPixelFormatTransition( + _ pixelFormat: VNCProtocol.PixelFormat, + sequence: UInt64 + ) throws { + try withLifecycleLock { guard connectionState.status == .connected, let framebuffer = framebuffer else { return } + try resetZRLECompressionState(for: sequence) state.pixelFormat = pixelFormat recreateFramebuffer(size: framebuffer.size, screens: framebuffer.screens, @@ -485,6 +506,7 @@ extension VNCConnection { framebufferRequestLock.lock() isPixelFormatTransitionInFlight = false pixelFormatTransitionInFlight = nil + pixelFormatTransitionInFlightSequence = nil let nextTransition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() @@ -740,12 +762,14 @@ extension VNCConnection { pendingPixelFormatTransition = nil isPixelFormatTransitionInFlight = false pixelFormatTransitionInFlight = nil + pixelFormatTransitionInFlightSequence = nil pixelFormatTransitionFencePayload = nil pixelFormatTransitionRequiredFenceFlags = [] pixelFormatTransitionFenceWasSent = false cancelPixelFormatTransitionDeadlineLocked() pixelFormatFenceCapabilityProbePayload = nil expiredPixelFormatFenceCapabilityProbePayload = nil + pixelFormatFenceCapabilityProbeSequence = nil cancelPixelFormatFenceNegotiationTimeoutLocked() framebufferRequestLock.unlock() } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift index 2216aadc..f666aa3f 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection.swift @@ -113,6 +113,7 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pendingPixelFormatTransition: VNCProtocol.PixelFormat? var isPixelFormatTransitionInFlight = false var pixelFormatTransitionInFlight: VNCProtocol.PixelFormat? + var pixelFormatTransitionInFlightSequence: UInt64? var pixelFormatTransitionFenceSequence: UInt64 = 0 var pixelFormatTransitionFencePayload: Data? var pixelFormatTransitionRequiredFenceFlags: VNCProtocol.FenceFlags = [] @@ -120,7 +121,9 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { var pixelFormatTransitionDeadlineTask: Task? var pixelFormatFenceCapabilityProbePayload: Data? var expiredPixelFormatFenceCapabilityProbePayload: Data? + var pixelFormatFenceCapabilityProbeSequence: UInt64? var pixelFormatFenceNegotiationTask: Task? + var nextPixelFormatChangeSequence: UInt64 = 0 private let queue = DispatchQueue(label: "com.royalapps.royalvnc.connectionqueue", attributes: .concurrent) private let lifecycleLock = NSRecursiveLock() @@ -131,6 +134,8 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { private let sharedZStream: ZlibStream private let sharedZRLEZStream: ZlibStream + private let zrleCompressionStateLock = NSLock() + private var lastZRLECompressionResetSequence: UInt64 = 0 // MARK: - Internal Properties let taskPriority = TaskPriority.high @@ -306,6 +311,15 @@ public final class VNCConnection: NSObjectOrAnyObject, @unchecked Sendable { return uniqueEncs } + func resetZRLECompressionState(for sequence: UInt64) throws { + zrleCompressionStateLock.lock() + defer { zrleCompressionStateLock.unlock() } + guard sequence > lastZRLECompressionResetSequence else { return } + + try sharedZRLEZStream.reset() + lastZRLECompressionResetSequence = sequence + } + // MARK: - Public Initializers public init(settings: Settings, logger: VNCLogger, @@ -442,10 +456,10 @@ extension VNCConnection { beginDisconnecting(error: error) } - func withLifecycleLock(_ operation: () -> T) -> T { + func withLifecycleLock(_ operation: () throws -> T) rethrows -> T { lifecycleLock.lock() defer { lifecycleLock.unlock() } - return operation() + return try operation() } func registerCredentialRequest( diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 1a3b0fb8..13a89336 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -505,7 +505,7 @@ struct AuditFindingsTests { } @Test - func preservesZRLECompressionAcrossPixelFormatProbeAndTransitionBoundaries() async throws { + func resetsZRLECompressionAtSynchronizedPixelFormatBoundaries() async throws { let connection = VNCConnection( settings: makeSettings(frameEncodings: [.zrle, .raw]), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -550,16 +550,108 @@ struct AuditFindingsTests { payload: capabilityPayload ) ) + let restartedAfterProbe = try zrle.zStream.decompressedData( + compressedData: compressedChunks[0], + uncompressedSize: 1_000 + ) + #expect(restartedAfterProbe == Data(repeating: 0x41, count: 1_000)) connection.framebufferUpdateRequestOutstanding = true connection.updateColorDepth(.depth8Bit) let transition = try #require(connection.clientToServerMessageQueue.dequeue()) - try await transition.message.send(connection: AuditWritingConnection()) - let third = try zrle.zStream.decompressedData( + let transitionWriter = AuditWritingConnection() + try await transition.message.send(connection: transitionWriter) + let continuedBeforeBoundary = try zrle.zStream.decompressedData( + compressedData: compressedChunks[1], + uncompressedSize: 1_000 + ) + #expect(continuedBeforeBoundary == Data(repeating: 0x42, count: 1_000)) + + let completionFenceOffset = setEncodingsEndOffset(in: transitionWriter.data, at: 37) + let synchronizationPayload = Data(transitionWriter.data[9..<17]) + let completionPayload = Data( + transitionWriter.data[(completionFenceOffset + 9)..<(completionFenceOffset + 17)] + ) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: synchronizationPayload + ) + ) + let continuedAfterSynchronizationFence = try zrle.zStream.decompressedData( compressedData: compressedChunks[2], uncompressedSize: 1_000 ) - #expect(third == Data(repeating: 0x43, count: 1_000)) + #expect(continuedAfterSynchronizationFence == Data(repeating: 0x43, count: 1_000)) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore], + payload: completionPayload + ) + ) + try verifyFreshZRLEStream(zrle.zStream, byte: 0x44) + + connection.cancelFramebufferUpdateScheduling() + } + + @Test + func ignoresStaleZRLEResetFromLateCapabilityProbeResponse() async throws { + let connection = VNCConnection( + settings: makeSettings(frameEncodings: [.zrle, .raw]), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + let zrle = try #require( + connection.encodings[VNCFrameEncodingType.zrle.rawValue] as? VNCProtocol.ZRLEEncoding + ) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .syncNext], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + try await capabilityProbe.message.send(connection: AuditWritingConnection()) + let capabilityPayload = try #require(connection.pixelFormatFenceCapabilityProbePayload) + + let compressedChunks = continuousZlibChunks() + _ = try zrle.zStream.decompressedData( + compressedData: compressedChunks[0], + uncompressedSize: 1_000 + ) + connection.updateColorDepth(.depth8Bit) + connection.expirePixelFormatFenceNegotiation() + let transition = try #require(connection.clientToServerMessageQueue.dequeue()) + try await transition.message.send(connection: AuditWritingConnection()) + + let restartedAfterTransition = try zrle.zStream.decompressedData( + compressedData: compressedChunks[0], + uncompressedSize: 1_000 + ) + #expect(restartedAfterTransition == Data(repeating: 0x41, count: 1_000)) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .syncNext], + payload: capabilityPayload + ) + ) + let continuedAfterLateProbe = try zrle.zStream.decompressedData( + compressedData: compressedChunks[1], + uncompressedSize: 1_000 + ) + #expect(continuedAfterLateProbe == Data(repeating: 0x42, count: 1_000)) connection.cancelFramebufferUpdateScheduling() } @@ -1006,6 +1098,15 @@ struct AuditFindingsTests { offset + 4 + setEncodingValues(in: data, at: offset).count * 4 } + private func verifyFreshZRLEStream(_ stream: ZlibStream, byte: UInt8) throws { + let expected = Data(repeating: byte, count: 64) + let actual = try stream.decompressedData( + compressedData: ZlibOneShot.deflate(expected), + maximumOutputSize: expected.count + ) + #expect(actual == expected) + } + private func continuousZlibChunks() -> [Data] { [ Data([ From 8a6a6da4662b7112cd95d9c29d00bd796dea616b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:50:39 +0200 Subject: [PATCH 209/242] fix(credentials): fence stale rollback recovery --- .../sandbox-credential-policy-scanner.ts | 50 +++- .../sandbox-credential-policy-scanner.test.ts | 215 ++++++++++++++++++ 2 files changed, 264 insertions(+), 1 deletion(-) diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index d43b3d57..44cdfe27 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -18,6 +18,7 @@ import { sandboxLookupIds, type SandboxCredentialPolicyOwnershipFence, } from "./sandbox-credential-policy-repository.ts"; +import { parseSandboxCredentialPolicyRollback } from "./sandbox-credential-policy-rollback.ts"; import { sandboxLeaseInfo, sandboxLeasePrefix } from "./sandbox-lease.ts"; import { sandboxCredentialPolicyRegistrationLookupIds, @@ -301,6 +302,29 @@ export async function scanCredentialPolicyCleanupPage( } } +async function sandboxCredentialPolicyRollbackIsSuperseded( + db: Kysely, + sessionId: string, + sandboxId: string, + lookupIds: readonly string[], + rollbackGeneration: string | null, +): Promise { + const rows = await db + .selectFrom("interactive_session_credential_policies") + .select("registration_generation") + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", [...new Set(lookupIds)]) + .where("state", "=", "active") + .where("registration_claim", "is", null) + .execute(); + return rows.some( + (row) => + isCurrentCredentialPolicyGeneration(row.registration_generation) && + row.registration_generation !== rollbackGeneration, + ); +} + async function scanStagedCredentialPolicyRegistrations( env: RuntimeEnv, db: Kysely, @@ -351,13 +375,14 @@ async function scanStagedCredentialPolicyRegistrations( row.session_id, row.sandbox_id, ); + const currentLookupIds = sandboxLookupIds(env, row.sandbox_id); const registration: SandboxCredentialPolicyRegistration = { generation: row.registration_generation, claim: row.registration_claim, lookupIds: sandboxCredentialPolicyRegistrationLookupIds( row.lookup_ids_json, row.sandbox_id, - sandboxLookupIds(env, row.sandbox_id), + currentLookupIds, [...persistedLookupIds, ...rollbackLookupIds], ), }; @@ -400,6 +425,29 @@ async function scanStagedCredentialPolicyRegistrations( } if (row.rollback_policies_json !== null) { if (!restoreRollback) throw new Error("sandbox credential policy rollback is unavailable"); + const rollbackGeneration = + parseSandboxCredentialPolicyRollback( + row.rollback_policies_json, + rollbackLookupIds, + row.session_id, + )[0]?.generation ?? null; + const rollbackSuperseded = await sandboxCredentialPolicyRollbackIsSuperseded( + db, + row.session_id, + row.sandbox_id, + [...persistedLookupIds, ...currentLookupIds, ...rollbackLookupIds], + rollbackGeneration, + ); + if (rollbackSuperseded) { + await abandonSandboxCredentialPolicyRegistration( + env, + row.session_id, + row.sandbox_id, + recovery.registration, + "sandbox credential policy generation advanced before rollback", + ); + continue; + } await restoreRollback({ registration: { ...recovery.registration, diff --git a/tests/sandbox-credential-policy-scanner.test.ts b/tests/sandbox-credential-policy-scanner.test.ts index 222ffa02..643bc506 100644 --- a/tests/sandbox-credential-policy-scanner.test.ts +++ b/tests/sandbox-credential-policy-scanner.test.ts @@ -1,16 +1,120 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; import test from "node:test"; import { credentialPolicyProvisioningStaleMs, credentialPolicyScanOwnershipFence, credentialPolicyScanRequiresCleanup, + scanCredentialPolicyCleanupPage, type CredentialPolicyScanRow, } from "../src/worker/sandbox-credential-policy-scanner.ts"; +import type { RuntimeEnv } from "../src/worker/env.ts"; +import { activeSandboxCredentialPolicyGeneration } from "../src/worker/sandbox-credential-policy-repository.ts"; const now = 2_000_000; +type SqliteStatement = { + all(...parameters: unknown[]): Record[]; + run(...parameters: unknown[]): { changes: number | bigint; lastInsertRowid: number | bigint }; +}; + +function scannerDatabase(): DatabaseSync { + const db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE interactive_sessions ( + id TEXT PRIMARY KEY, + adapter TEXT, + status TEXT NOT NULL, + lease_id TEXT, + credential_cleanup_terminal_status TEXT, + sandbox_refresh_sandbox_id TEXT, + sandbox_refresh_claim TEXT, + sandbox_refresh_claim_expires_at INTEGER, + agent_token_hash TEXT, + updated_at INTEGER NOT NULL + ); + CREATE TABLE standalone_sandbox_provisions ( + id TEXT PRIMARY KEY, + sandbox_id TEXT NOT NULL, + state TEXT NOT NULL, + ownership_claim TEXT, + ownership_claim_expires_at INTEGER, + updated_at INTEGER NOT NULL + ); + CREATE TABLE interactive_session_credential_policies ( + session_id TEXT NOT NULL, + sandbox_id TEXT NOT NULL, + lookup_id TEXT NOT NULL, + state TEXT NOT NULL, + registration_generation TEXT NOT NULL, + registration_claim TEXT, + registration_claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, sandbox_id, lookup_id) + ); + CREATE TABLE interactive_session_credential_policy_registrations ( + session_id TEXT NOT NULL, + sandbox_id TEXT NOT NULL, + state TEXT NOT NULL, + registration_generation TEXT NOT NULL, + registration_claim TEXT, + registration_claim_expires_at INTEGER, + lookup_ids_json TEXT, + rollback_policies_json TEXT, + last_error TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, sandbox_id) + ); + `); + return db; +} + +function scannerRuntimeEnv(sqlite: DatabaseSync): RuntimeEnv { + function execute(sql: string, parameters: unknown[]) { + const statement = sqlite.prepare(sql) as unknown as SqliteStatement; + if (/^\s*(?:select|pragma|with)\b|\breturning\b/i.test(sql)) { + const results = statement.all(...parameters).map((row) => ({ ...row })); + const changes = Number(sqlite.prepare("SELECT changes() AS changes").get()?.changes ?? 0); + return { results, success: true as const, meta: { changes } }; + } + const result = statement.run(...parameters); + return { + results: [], + success: true as const, + meta: { + changes: Number(result.changes), + last_row_id: Number(result.lastInsertRowid), + }, + }; + } + return { + DB: { + prepare(sql: string) { + return { + bind(...parameters: unknown[]) { + return { + async all() { + return execute(sql, parameters); + }, + async run() { + return execute(sql, parameters); + }, + }; + }, + }; + }, + } as unknown as D1Database, + SANDBOX: { + idFromName() { + return { toString: () => "do-current" }; + }, + } as unknown as DurableObjectNamespace, + } as RuntimeEnv; +} + function scanRow(values: Partial = {}): CredentialPolicyScanRow { return { scan_rowid: 1, @@ -143,6 +247,14 @@ test("staged recovery takes a fresh exclusive claim before promotion or rollback stagedRecovery.indexOf("claimSandboxCredentialPolicyRegistrationRecovery(") < stagedRecovery.indexOf("restoreRollback({"), ); + assert.ok( + stagedRecovery.indexOf("claimSandboxCredentialPolicyRegistrationRecovery(") < + stagedRecovery.indexOf("sandboxCredentialPolicyRollbackIsSuperseded("), + ); + assert.ok( + stagedRecovery.indexOf("sandboxCredentialPolicyRollbackIsSuperseded(") < + stagedRecovery.indexOf("restoreRollback({"), + ); assert.match(stagedRecovery, /sandboxCredentialPolicyRegistrationLookupIds/); assert.match(stagedRecovery, /sandboxCredentialPolicyPersistedLookupIds/); assert.match(stagedRecovery, /sandboxCredentialPolicyRollbackLookupIds/); @@ -150,6 +262,109 @@ test("staged recovery takes a fresh exclusive claim before promotion or rollback assert.doesNotMatch(stagedRecovery, /renewSandboxCredentialPolicyRegistration/); }); +test("staged recovery refuses rollback when one historical legacy lookup advances", async () => { + const sqlite = scannerDatabase(); + const env = scannerRuntimeEnv(sqlite); + const rollback = ["sandbox-1", "do-rollback"].map((lookupId) => ({ + generation: "generation:rollback", + policy: { + allowedHosts: [], + githubCredentialSource: "none", + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: lookupId, + sessionId: "IS-42", + }, + })); + sqlite + .prepare(` + INSERT INTO interactive_sessions ( + id, + adapter, + status, + lease_id, + credential_cleanup_terminal_status, + sandbox_refresh_sandbox_id, + sandbox_refresh_claim, + sandbox_refresh_claim_expires_at, + agent_token_hash, + updated_at + ) VALUES (?, NULL, 'ready', ?, NULL, NULL, NULL, NULL, 'agent-token', ?) + `) + .run("IS-42", "sandbox:sandbox-1:terminal-1:autostart-v4", now); + sqlite + .prepare(` + INSERT INTO interactive_session_credential_policy_registrations ( + session_id, + sandbox_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + lookup_ids_json, + rollback_policies_json, + last_error, + updated_at + ) VALUES (?, ?, 'registering', ?, ?, ?, ?, ?, NULL, ?) + `) + .run( + "IS-42", + "sandbox-1", + "generation:stale", + "registration:stale", + now - 1, + JSON.stringify(["sandbox-1", "do-current"]), + JSON.stringify(rollback), + now - 1, + ); + const activeInsert = sqlite.prepare(` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at + ) VALUES (?, ?, ?, 'active', ?, NULL, NULL, ?, ?) + `); + activeInsert.run("IS-42", "sandbox-1", "sandbox-1", "generation:rollback", now, now); + activeInsert.run("IS-42", "sandbox-1", "do-legacy", "generation:advanced", now, now); + let rollbackCalls = 0; + + assert.equal(await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), null); + await scanCredentialPolicyCleanupPage( + env, + now, + async () => false, + "IS-42", + async () => { + rollbackCalls += 1; + }, + ); + + assert.equal(rollbackCalls, 0); + assert.deepEqual( + { + ...sqlite + .prepare(` + SELECT state, registration_claim, registration_claim_expires_at, last_error + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(), + }, + { + state: "cleanup_pending", + registration_claim: null, + registration_claim_expires_at: null, + last_error: "sandbox credential policy generation advanced before rollback", + }, + ); +}); + test("foreground rollback rechecks its exact claim before restoring policy", async () => { const source = await readFile( new URL("../src/worker/sandbox-credential-policy-registration-service.ts", import.meta.url), From f32532625c4b3ac1c5330aef16c6f51903b43a99 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:52:08 +0200 Subject: [PATCH 210/242] docs(changelog): record final audit follow-up --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b81aff3..956ddea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -- Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions without resetting the peer-owned ZRLE stream. +- Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. +- Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets. - Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, requiring replayable runtime-adapter deletion tombstones, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. - Harden session lifecycle concurrency with atomic card claims and duplicate-first claim results, single-winner GitHub Actions credential rotation, monotonic exact authenticated-revision fences on runner writes, terminal session-status fences, revision-fenced lifecycle updates and grant revocation, exclusively claimed rollback recovery for Sandbox credential rotation, rejection of live legacy registration claims during migration without staging renewable legacy claims, staged-rotation fences that block legacy policy mutation while new-worker claims are live but release abandoned rows for rollback compatibility, persisted staged lookup identities across namespace changes with exact current-identity fallback for mixed-version rows, ownership-fenced repair of incomplete and rotated lookup sets before credential rotation, explicit retirement of obsolete durable identities, idempotent recovery after ambiguous committed promotion, R2-clean reservation rollback, preserved Sandbox attachment state, retained registration data for superseded runtime workspace cleanup, and durable observed-deletion markers that terminate cleanup after post-delete crashes. - Close final terminal and desktop publication race windows by carrying the initial GitHub Actions runner generation through viewer authorization, translating generation-fenced acknowledgements for legacy framed viewers, serializing and bounding per-runner PTY input by frames, bytes, and age, matching generation-fenced local send failures, ordering raw and confirmed Go client acknowledgements, bounding shutdown when terminal writers block, rejecting malformed desktop recovery IDs as client errors, preserving idempotent publication retries across mixed worker versions, and retaining uncertain Share This Mac publications when older servers lack the recovery route. From d2347e55d4121660370c42e68ebbea727e745fd4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:53:47 +0200 Subject: [PATCH 211/242] fix(macos): harden share shutdown recovery --- .../CrabfleetMac/CrabfleetMacApp.swift | 37 ++- .../PrivateMacShareController.swift | 68 +++++- .../PrivateMacShareTests.swift | 216 +++++++++++++++++- 3 files changed, 300 insertions(+), 21 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift index c856d204..4c4b6b2e 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/CrabfleetMacApp.swift @@ -49,6 +49,8 @@ enum VNCConnectionLaunchMode { final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { let shareController: PrivateMacShareController private let replyToTerminationRequest: @MainActor (Bool) -> Void + private let isAutoShareRequested: @MainActor () -> Bool + private let autoShareDelay: Duration private var autoShareTask: Task? private var terminationTask: Task? @@ -57,6 +59,8 @@ final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { replyToTerminationRequest = { shouldTerminate in NSApp.reply(toApplicationShouldTerminate: shouldTerminate) } + isAutoShareRequested = { PrivateMacShareLaunchMode.isRequested() } + autoShareDelay = .milliseconds(500) super.init() } @@ -64,19 +68,30 @@ final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { shareController: PrivateMacShareController, replyToTerminationRequest: @escaping @MainActor (Bool) -> Void = { NSApp.reply(toApplicationShouldTerminate: $0) - } + }, + isAutoShareRequested: @escaping @MainActor () -> Bool = { + PrivateMacShareLaunchMode.isRequested() + }, + autoShareDelay: Duration = .milliseconds(500) ) { self.shareController = shareController self.replyToTerminationRequest = replyToTerminationRequest + self.isAutoShareRequested = isAutoShareRequested + self.autoShareDelay = autoShareDelay super.init() } func applicationDidFinishLaunching(_ notification: Notification) { - guard PrivateMacShareLaunchMode.isRequested() else { return } + guard isAutoShareRequested() else { return } NSApp.activate(ignoringOtherApps: true) autoShareTask = Task { [weak self] in guard let self else { return } - try? await Task.sleep(for: .milliseconds(500)) + do { + try await Task.sleep(for: autoShareDelay) + } catch { + return + } + guard !Task.isCancelled else { return } await self.startPrivateShare(shareController) } } @@ -99,6 +114,7 @@ final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { private func startPrivateShare(_ controller: PrivateMacShareController) async { await controller.refresh() + guard !Task.isCancelled else { return } report( "private share prerequisites: tailnet \(controller.identity == nil ? "unavailable" : "ready"), " + "Screen Recording \(controller.screenRecordingGranted ? "allowed" : "denied")" @@ -106,23 +122,34 @@ final class CrabfleetApplicationDelegate: NSObject, NSApplicationDelegate { ) if !controller.screenRecordingGranted { await controller.requestScreenRecordingPermission() + guard !Task.isCancelled else { return } } let clock = ContinuousClock() let deadline = clock.now.advanced(by: .seconds(300)) while !Task.isCancelled, clock.now < deadline { await controller.refresh() + guard !Task.isCancelled else { return } if controller.canStart { await controller.start() + guard !Task.isCancelled else { return } for _ in 0..<50 where controller.phase == .starting { - try? await Task.sleep(for: .milliseconds(100)) + do { + try await Task.sleep(for: .milliseconds(100)) + } catch { + return + } } let address = controller.connectionAddress.map { " at \($0)" } ?? "" let notice = controller.notice.map { ": \($0)" } ?? "" report("private share \(controller.phase.title.lowercased())\(address)\(notice)") return } - try? await Task.sleep(for: .seconds(2)) + do { + try await Task.sleep(for: .seconds(2)) + } catch { + return + } } let missing = [ diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index fc486e7f..c6ed239f 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -39,10 +39,15 @@ final class PrivateMacShareStopCoordinator { @MainActor protocol DesktopHostRegistrationStateStoring: AnyObject { + func containsState() -> Bool func load(scope: DesktopHostRegistrationRecoveryScope) throws -> Data? func save(_ data: Data?, scope: DesktopHostRegistrationRecoveryScope) throws } +extension DesktopHostRegistrationStateStoring { + func containsState() -> Bool { true } +} + enum DesktopHostRegistrationPersistenceError: LocalizedError { case missingScope case unreadableState @@ -74,6 +79,11 @@ final class UserDefaultsDesktopHostRegistrationStateStore: self.key = key } + func containsState() -> Bool { + let prefix = "\(key).v2." + return defaults.dictionaryRepresentation().keys.contains { $0.hasPrefix(prefix) } + } + func load(scope: DesktopHostRegistrationRecoveryScope) throws -> Data? { defaults.data(forKey: scopedKey(scope)) } @@ -280,7 +290,7 @@ final class DesktopHostRegistrationLifecycle { } func removePublishedIdentities() async throws { - try await loadStateIfNeeded() + try await loadStateIfNeeded(requireRecoveryScope: false) var firstError: Error? let uncertainRegistrations = uncertainRegistrations for target in uncertainRegistrations { @@ -346,27 +356,52 @@ final class DesktopHostRegistrationLifecycle { try persistState() } catch { firstError = firstError ?? error + if removal.usesLegacyCleanup { + // A tokenless legacy DELETE cannot identify its publication. Try it + // only while stopping the process that published it; never retain it + // for a delayed retry that could delete a replacement publisher. + pendingRemovals.removeAll { $0 == removal } + do { + try persistState() + } catch { + firstError = firstError ?? error + } + } } } if let firstError { throw firstError } } - private func loadStateIfNeeded() async throws { - guard stateStore != nil, !stateLoaded else { + private func loadStateIfNeeded(requireRecoveryScope: Bool = true) async throws { + guard let stateStore else { return } + if stateLoaded { if let stateLoadError { throw stateLoadError } + if requireRecoveryScope, recoveryScope == nil { + try await loadRecoveryScope() + } return } - guard let recoveryScopeProvider else { + if !requireRecoveryScope, !stateStore.containsState() { + stateLoaded = true + return + } + try await loadRecoveryScope() + guard let recoveryScope else { throw DesktopHostRegistrationPersistenceError.missingScope } - let scope = try await recoveryScopeProvider() - recoveryScope = scope + var filteredLegacyState = false do { - if let data = try stateStore?.load(scope: scope) { + if let data = try stateStore.load(scope: recoveryScope) { let state = try JSONDecoder().decode(PersistedState.self, from: data) + filteredLegacyState = + state.publishedRegistration?.usesLegacyCleanup == true + || state.pendingRemovals.contains { $0.usesLegacyCleanup } uncertainRegistrations = state.uncertainRegistrations - publishedRegistration = state.publishedRegistration?.registration + publishedRegistration = state.publishedRegistration + .map(\.registration) + .flatMap { $0.usesLegacyCleanup ? nil : $0 } pendingRemovals = state.pendingRemovals.map(\.registration) + .filter { !$0.usesLegacyCleanup } } stateLoaded = true } catch { @@ -374,6 +409,16 @@ final class DesktopHostRegistrationLifecycle { stateLoaded = true throw DesktopHostRegistrationPersistenceError.unreadableState } + if filteredLegacyState { + try persistState() + } + } + + private func loadRecoveryScope() async throws { + guard let recoveryScopeProvider else { + throw DesktopHostRegistrationPersistenceError.missingScope + } + recoveryScope = try await recoveryScopeProvider() } private func persistState() throws { @@ -383,8 +428,11 @@ final class DesktopHostRegistrationLifecycle { } let state = PersistedState( uncertainRegistrations: uncertainRegistrations, - publishedRegistration: publishedRegistration.map(PersistedPublishedRegistration.init), - pendingRemovals: pendingRemovals.map(PersistedPublishedRegistration.init) + publishedRegistration: publishedRegistration + .flatMap { $0.usesLegacyCleanup ? nil : PersistedPublishedRegistration($0) }, + pendingRemovals: pendingRemovals + .filter { !$0.usesLegacyCleanup } + .map(PersistedPublishedRegistration.init) ) let hasState = !state.uncertainRegistrations.isEmpty || state.publishedRegistration != nil diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index f8c185e6..12518da9 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -633,6 +633,24 @@ struct PrivateMacShareTests { #expect(await registration.activePublicationID == nil) } + @Test @MainActor + func retainedLegacyCleanupDoesNotDeleteANewerPublisher() async throws { + let identity = desktopIdentity(name: "legacy-host", address: "100.64.12.60") + let registration = RetainedLegacyDesktopRegistration() + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + + try await lifecycle.publish(identity: identity, port: 5_901) + await #expect(throws: DesktopRegistrationTestError.failed) { + try await lifecycle.removePublishedIdentities() + } + await registration.publishNewerEndpoint() + + try await lifecycle.removePublishedIdentities() + + #expect(await registration.activeEndpoint == "newer-publisher") + #expect(await registration.events == [.register, .unregister]) + } + @Test @MainActor func desktopPublicationReplacementUsesSanitizedHostIDAcrossIdentityChanges() async throws { let first = desktopIdentity(name: "shared-host", address: "100.64.12.48") @@ -749,6 +767,79 @@ struct PrivateMacShareTests { #expect(delegate.shareController === controller) } + @Test @MainActor + func applicationTerminationCancelsPendingAutoShareStartup() async throws { + let runner = CountingTailscaleRunner(output: statusJSON()) + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: nil, + defaults: defaults + ) + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) }, + isAutoShareRequested: { true }, + autoShareDelay: .seconds(30) + ) + let application = NSApplication.shared + + delegate.applicationDidFinishLaunching( + Notification(name: NSApplication.didFinishLaunchingNotification) + ) + #expect(delegate.applicationShouldTerminate(application) == .terminateLater) + #expect(await waitUntilAsync { replies == [true] }) + try await Task.sleep(for: .milliseconds(50)) + + #expect(await runner.callCount == 0) + #expect(controller.phase == .idle) + } + + @Test @MainActor + func applicationTerminationCancelsAutoShareDuringInitialRefresh() async throws { + let runner = SequencedTailscaleRunner() + let defaults = try #require( + UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") + ) + let controller = PrivateMacShareController( + runner: runner, + desktopRegistration: nil, + defaults: defaults + ) + var replies: [Bool] = [] + let delegate = CrabfleetApplicationDelegate( + shareController: controller, + replyToTerminationRequest: { replies.append($0) }, + isAutoShareRequested: { true }, + autoShareDelay: .zero + ) + let application = NSApplication.shared + + delegate.applicationDidFinishLaunching( + Notification(name: NSApplication.didFinishLaunchingNotification) + ) + #expect(await waitUntilAsync { await runner.callCount == 1 }) + #expect(delegate.applicationShouldTerminate(application) == .terminateLater) + #expect(await waitUntilAsync { replies == [true] }) + + await runner.resumeNext( + .success(.init(standardOutput: statusJSON(), standardError: "")) + ) + let continuedPreflight = await waitUntilAsync(timeout: .milliseconds(200)) { + await runner.callCount > 1 + } + if continuedPreflight { + await runner.resumeNext(.failure(CancellationError())) + } + + #expect(!continuedPreflight) + #expect(!controller.isRefreshing) + #expect(controller.phase == .idle) + } + @Test @MainActor func applicationTerminationWaitsForPrivateShareCleanup() async throws { let registration = SuspendedDesktopCleanupRegistration() @@ -906,6 +997,61 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func loadingPersistedLegacyCleanupClearsUnsafeState() async throws { + let identity = desktopIdentity(name: "persisted-legacy", address: "100.64.12.61") + let stateStore = ToggleDesktopRegistrationStateStore() + let recoveryScope = desktopRecoveryScope() + let persistedIdentity: [String: Any] = [ + "tailnetName": identity.tailnetName, + "loginName": identity.loginName, + "dnsName": identity.dnsName, + "hostName": identity.hostName, + "ipv4Address": identity.ipv4Address, + "userID": identity.userID, + ] + let legacyRegistration: [String: Any] = [ + "persistedIdentity": persistedIdentity, + "hostID": CrabfleetDesktopRegistration.hostID(identity: identity), + "publicationID": "legacy-publication", + "usesLegacyCleanup": true, + ] + let data = try JSONSerialization.data(withJSONObject: [ + "uncertainRegistrations": [], + "publishedRegistration": legacyRegistration, + "pendingRemovals": [legacyRegistration], + ]) + try stateStore.save(data, scope: recoveryScope) + let registration = RecordingDesktopRegistration() + var recoveryScopeRequests = 0 + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore, + recoveryScopeProvider: { + recoveryScopeRequests += 1 + return recoveryScope + } + ) + + try await lifecycle.removePublishedIdentities() + + #expect(stateStore.data(for: recoveryScope) == nil) + #expect(await registration.events.isEmpty) + + let reloadedLifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore, + recoveryScopeProvider: { + recoveryScopeRequests += 1 + return recoveryScope + } + ) + try await reloadedLifecycle.removePublishedIdentities() + + #expect(recoveryScopeRequests == 1) + #expect(await registration.events.isEmpty) + } + @Test @MainActor func applicationTerminationContinuesWhenDurableRecoveryCannotBeUpdated() async throws { let identity = desktopIdentity(name: "unsaved-cleanup", address: "100.64.12.55") @@ -2183,13 +2329,17 @@ struct PrivateMacShareTests { private func assertIdleApplicationTerminationContinues( transportHandler: @escaping (URLRequest) throws -> (Data, HTTPURLResponse) ) async throws { + var recoveryScopeRequests = 0 let registration = try #require( CrabfleetDesktopRegistration( environment: [ "CRABFLEET_API_URL": "https://fleet.example/api/fleet", "CRABFLEET_SESSION_COOKIE": "crabbox_session=secret", ], - transport: DesktopRegistrationTransport(handler: transportHandler) + transport: DesktopRegistrationTransport { request in + recoveryScopeRequests += 1 + return try transportHandler(request) + } )) let defaults = try #require( UserDefaults(suiteName: "CrabfleetMacTests.\(UUID().uuidString)") @@ -2207,12 +2357,9 @@ struct PrivateMacShareTests { #expect(delegate.applicationShouldTerminate(NSApplication.shared) == .terminateLater) #expect(await waitUntilAsync { replies == [true] }) + #expect(recoveryScopeRequests == 0) #expect(controller.phase == .idle) - if case .failed = controller.registryPhase { - // Recovery remains available for a later launch without blocking this idle quit. - } else { - Issue.record("expected recovery lookup failure") - } + #expect(controller.registryPhase == .notPublished) } @MainActor @@ -2266,6 +2413,20 @@ private struct StaticTailscaleRunner: TailscaleCommandRunning { } } +private actor CountingTailscaleRunner: TailscaleCommandRunning { + let output: String + private(set) var callCount = 0 + + init(output: String) { + self.output = output + } + + func run(arguments: [String]) async throws -> TailscaleCommandResult { + callCount += 1 + return .init(standardOutput: output, standardError: "") + } +} + private struct NoopRemoteInput: RemoteInputForwarding { func keyEvent(down: Bool, keysym: UInt32) {} func pointerEvent(buttonMask: UInt8, x: UInt16, y: UInt16) {} @@ -2536,6 +2697,10 @@ private final class ToggleDesktopRegistrationStateStore: private var dataByScope: [DesktopHostRegistrationRecoveryScope: Data] = [:] var data: Data? { dataByScope.values.first } + func containsState() -> Bool { + !dataByScope.isEmpty + } + func load(scope: DesktopHostRegistrationRecoveryScope) throws -> Data? { dataByScope[scope] } @@ -2633,6 +2798,45 @@ private actor TwoProcessDesktopRegistration: DesktopHostRegistering { } } +private actor RetainedLegacyDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register + case unregister + } + + private var failNextUnregister = true + private(set) var activeEndpoint: String? + private(set) var events: [Event] = [] + + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + events.append(.register) + activeEndpoint = "legacy-publisher" + return nil + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + nil + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + #expect(ownershipToken == nil) + events.append(.unregister) + if failNextUnregister { + failNextUnregister = false + throw DesktopRegistrationTestError.failed + } + activeEndpoint = nil + } + + func publishNewerEndpoint() { + activeEndpoint = "newer-publisher" + } +} + private actor SuspendedDesktopCleanupRegistration: DesktopHostRegistering { private var unregistrationContinuation: CheckedContinuation? From c971c6a0e943d7f7383debe4be01dd1776b437ba Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:08:33 +0200 Subject: [PATCH 212/242] fix(http): classify body stream failures --- src/worker/http.ts | 7 ++++++- tests/http.test.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/worker/http.ts b/src/worker/http.ts index c35bc98a..a14258c2 100644 --- a/src/worker/http.ts +++ b/src/worker/http.ts @@ -54,7 +54,12 @@ export function wantsMarkdown(request: Request): boolean { } export async function readJson(request: Request): Promise { - const source = await request.text(); + let source: string; + try { + source = await request.text(); + } catch { + throw badRequest("invalid json"); + } let parsed: unknown; try { parsed = JSON.parse(source) as unknown; diff --git a/tests/http.test.ts b/tests/http.test.ts index bdf39c84..5034d44d 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -67,6 +67,25 @@ test("JSON parsing and status errors retain stable messages and status codes", a "status" in error && error.status === 400, ); + const rejectedBody = new ReadableStream({ + start(controller) { + controller.error(new Error("request body aborted")); + }, + }); + await assert.rejects( + readJson( + new Request("https://fleet.example", { + method: "POST", + body: rejectedBody, + duplex: "half", + } as RequestInit & { duplex: "half" }), + ), + (error: unknown) => + error instanceof Error && + error.message === "invalid json" && + "status" in error && + error.status === 400, + ); for (const [error, status, message] of [ [unauthorized(), 401, "unauthorized"], From d75e8c2f8a6f9dfacd3314cdb34f3776b5b527c9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:08:33 +0200 Subject: [PATCH 213/242] fix(app): preserve session grid terminals --- src/app/app-navigation.js | 6 +++++- tests/app-navigation.test.ts | 36 +++++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/app/app-navigation.js b/src/app/app-navigation.js index 64dd3799..403345d8 100644 --- a/src/app/app-navigation.js +++ b/src/app/app-navigation.js @@ -38,6 +38,10 @@ export function appNavigationLocationState(locationLike = location) { }; } +export function shouldDisposeTerminalsForNavigation(state) { + return !state.focusedSessionId && !state.drawers.sessions; +} + export function useAppNavigation({ initialSessionLink, sessionItemByIdRef }) { const [appView, setAppViewState] = useState(initialAppView); const [drawers, setDrawers] = useState(initialSessionLink.route ? { sessions: true } : {}); @@ -65,7 +69,7 @@ export function useAppNavigation({ initialSessionLink, sessionItemByIdRef }) { setSharedSessionId(next.sharedSessionId); setSharedToken(next.sharedToken); if (next.focusedSessionId) warmGhosttyModule(); - else disposeAllTerminals(); + else if (shouldDisposeTerminalsForNavigation(next)) disposeAllTerminals(); }; window.addEventListener("popstate", onPopState); return () => window.removeEventListener("popstate", onPopState); diff --git a/tests/app-navigation.test.ts b/tests/app-navigation.test.ts index 5ddb92d5..b2ba86ab 100644 --- a/tests/app-navigation.test.ts +++ b/tests/app-navigation.test.ts @@ -5,6 +5,7 @@ import { appNavigationLocationState, normalizedAppView, sessionOpenTarget, + shouldDisposeTerminalsForNavigation, topOpenDrawer, } from "../src/app/app-navigation.js"; @@ -55,31 +56,36 @@ test("session navigation derives focus and durable route targets", () => { }); test("browser history locations reconcile view, drawers, and session focus", () => { - assert.deepEqual( - appNavigationLocationState({ - pathname: "/sessions/IS-2", - search: "?token=shared", - }), - { - appView: "fleet", - drawers: { sessions: true }, - focusedSessionId: "IS-2", - sharedSessionId: "IS-2", - sharedToken: "shared", - }, - ); - assert.deepEqual(appNavigationLocationState({ pathname: "/sessions", search: "" }), { + const focusedSession = appNavigationLocationState({ + pathname: "/sessions/IS-2", + search: "?token=shared", + }); + assert.deepEqual(focusedSession, { + appView: "fleet", + drawers: { sessions: true }, + focusedSessionId: "IS-2", + sharedSessionId: "IS-2", + sharedToken: "shared", + }); + assert.equal(shouldDisposeTerminalsForNavigation(focusedSession), false); + + const sessionGrid = appNavigationLocationState({ pathname: "/sessions", search: "" }); + assert.deepEqual(sessionGrid, { appView: "fleet", drawers: { sessions: true }, focusedSessionId: null, sharedSessionId: null, sharedToken: null, }); - assert.deepEqual(appNavigationLocationState({ pathname: "/app/board", search: "" }), { + assert.equal(shouldDisposeTerminalsForNavigation(sessionGrid), false); + + const board = appNavigationLocationState({ pathname: "/app/board", search: "" }); + assert.deepEqual(board, { appView: "board", drawers: {}, focusedSessionId: null, sharedSessionId: null, sharedToken: null, }); + assert.equal(shouldDisposeTerminalsForNavigation(board), true); }); From 12688ba8f6947754e49097c3895cb5659ec0cd8d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:08:55 +0200 Subject: [PATCH 214/242] fix(credentials): fence final policy promotion --- .../sandbox-credential-policy-repository.ts | 14 ++++++++ ...ndbox-credential-policy-repository.test.ts | 35 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 24f9829d..32163085 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -1024,6 +1024,14 @@ export async function finishSandboxCredentialPolicyRegistration( registration: SandboxCredentialPolicyRegistration, ownershipFence: SandboxCredentialPolicyOwnershipFence, ): Promise { + const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + ); + if (!registrationExpiresAt) return false; const now = Date.now(); let batchError: unknown; try { @@ -1034,6 +1042,7 @@ export async function finishSandboxCredentialPolicyRegistration( sessionId, sandboxId, registration, + registrationExpiresAt, ownershipFence, now, ), @@ -1123,6 +1132,7 @@ export function sandboxCredentialPolicyPromotionQueries( sessionId: string, sandboxId: string, registration: SandboxCredentialPolicyRegistration, + registrationExpiresAt: number, ownershipFence: SandboxCredentialPolicyOwnershipFence, now: number, ): CompilableQuery[] { @@ -1135,6 +1145,8 @@ export function sandboxCredentialPolicyPromotionQueries( AND state = 'registering' AND registration_generation = ${registration.generation} AND registration_claim = ${registration.claim} + AND registration_claim_expires_at = ${registrationExpiresAt} + AND registration_claim_expires_at > ${now} ) AND ${noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)} AND NOT EXISTS ( @@ -1214,6 +1226,8 @@ export function sandboxCredentialPolicyPromotionQueries( AND state = 'registering' AND registration_generation = ${registration.generation} AND registration_claim = ${registration.claim} + AND registration_claim_expires_at = ${registrationExpiresAt} + AND registration_claim_expires_at > ${now} AND ${promotionComplete} `, ]; diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 8915c699..7a5050aa 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -1379,6 +1379,41 @@ test("completed credential-policy rotation atomically promotes every active look ); }); +test("expired credential-policy claims cannot promote active authority", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = 0 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + + assert.equal( + await finishSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + false, + ); + assert.equal( + activeCredentialPolicyRows(sqlite).some( + (row) => row.registration_generation === staged.generation && row.state === "active", + ), + false, + ); +}); + test("completed credential-policy rotation tolerates an ambiguous committed batch", async () => { const sqlite = credentialPolicyDatabase(); const staged = await beginSandboxCredentialPolicyRegistration( From 90259d700daf63da8071e83e0deac4f9c87e8ca2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:09:28 +0200 Subject: [PATCH 215/242] fix(runtime): negotiate delete tombstones --- docs/api.md | 2 +- src/worker/runtime-adapter-workspaces.ts | 22 +++++++++- tests/runtime-adapter-workspaces.test.ts | 55 +++++++++++++++++++++++- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/docs/api.md b/docs/api.md index a6798f8a..43c302eb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -507,7 +507,7 @@ Crabfleet authenticates every adapter request with `Authorization: Bearer CRABBO - `POST /v1/workspaces`: idempotent create. Crabfleet persists the deterministic adapter identity, TTL, idle timeout, requested capabilities, and exact serialized create payload before the request, then sends the same namespaced DNS-safe lowercase `id` and `Idempotency-Key`, plus repo, branch, runtime, opaque profile, command, prompt, ownership/lineage, and lifecycle settings. A definitive non-2xx response to the initial request is read once, sanitized, and durably recorded as the failure reason before provider release begins. After an ambiguous result, a bounded reconciliation pass retries only that immutable payload and key before any inspect; later edits to session metadata do not alter it. Replay-time authentication, routing, validation, or other non-success responses cannot prove the original request failed and therefore keep create ambiguity pending. - An adapter that finds the requested ID already bound to a different immutable request returns `409` with `error.code = "workspace_id_conflict"`. Crabfleet marks only its local session failed and atomically drops that adapter identity when the exact pending create attempt still owns the lifecycle revision and reconciliation claim; a stale conflict response is ignored. It never adopts, inspects, or deletes the pre-existing workspace. Other `409` responses remain ambiguous and retryable. - `GET /v1/workspaces/:id`: inspect current status, capabilities, terminal URL, expiry, and provider resource identity. Status-only responses preserve previously stored capabilities and expiry; explicit `null` clears those fields. Active external sessions are reconciled in bounded batches; state responses wait only for a short foreground budget while remaining work continues in the Worker background. -- `DELETE /v1/workspaces/:id`: idempotent stop/release. Before provider release, the adapter must durably retain the exact workspace identity and stopping intent. Once a DELETE is accepted, every retry for that immutable ID must return `204` or a valid exact-ID `stopping`, `stopped`, or `expired` response, including after provider deletion or adapter restart; it must not collapse that lifecycle tombstone into `404`. This lets a caller recover when deletion commits but the response is lost without treating a pre-visibility `404` from an ambiguous create as release proof. Crabfleet enters `stopping` before calling the adapter and marks the session stopped only after `204`, a `404` when create ambiguity is absent or prior deletion evidence is durable, or a valid exact-ID terminal response confirms release; malformed successful bodies remain `stopping`. Plain-text and malformed-JSON responses are read once and sanitized before their evidence is retained. An explicit stop whose ownership claim loses returns success only when the exact workspace is already stopping or terminal; otherwise it returns a lifecycle conflict. +- `DELETE /v1/workspaces/:id`: idempotent stop/release. Crabfleet advertises `delete-tombstone-v1` in `X-Crabfleet-Runtime-Adapter-Capabilities`; adapters that support the stricter contract echo that token in the response header. Before provider release, a supporting adapter must durably retain the exact workspace identity and stopping intent. Once a DELETE is accepted, every retry for that immutable ID must return `204` or a valid exact-ID `stopping`, `stopped`, or `expired` response, including after provider deletion or adapter restart; it must not collapse that lifecycle tombstone into `404`. This lets a caller recover when deletion commits but the response is lost without treating a pre-visibility `404` from an ambiguous create as release proof. Crabfleet enters `stopping` before calling the adapter and marks the session stopped only after `204`, a `404` when create ambiguity is absent, prior deletion evidence is durable, or the adapter does not echo the capability during a rolling upgrade, or a valid exact-ID terminal response confirms release; malformed successful bodies remain `stopping`. Plain-text and malformed-JSON responses are read once and sanitized before their evidence is retained. An explicit stop whose ownership claim loses returns success only when the exact workspace is already stopping or terminal; otherwise it returns a lifecycle conflict. - `POST /v1/workspaces/:id/connections/desktop`: mint a current transient desktop URL. The request has no body. `expiresAt` is optional; when present it must be in the future and no more than 15 minutes away. Accepted HTTPS URLs are treated as opaque signed connection material and redirected byte-for-byte without URL normalization. After minting, Crabfleet re-reads the exact current session status, control grant, capabilities, and registered adapter identity before redirecting; a concurrent stop, revocation, capability withdrawal, or lifecycle replacement discards the URL and denies access. - `POST /v1/workspaces/:id/connections/native-vnc`: mint a short-lived, single-use native VNC grant. The response must use the `crabbox/native-vnc-grant/v1` schema, an HTTPS broker URL (literal loopback HTTP is allowed for development), the exact opaque lease ID, a 32-byte-hex `native_vnc_` ticket, and an expiry no more than two minutes away. Crabfleet never exposes the provider lease ID in Fleet state and requests this grant only after revalidating current session control and the persisted adapter identity. diff --git a/src/worker/runtime-adapter-workspaces.ts b/src/worker/runtime-adapter-workspaces.ts index f25f0631..123c64a8 100644 --- a/src/worker/runtime-adapter-workspaces.ts +++ b/src/worker/runtime-adapter-workspaces.ts @@ -36,6 +36,9 @@ import { } from "./runtime-adapter-preflight.ts"; import type { RuntimeAdapterWorkspaceStopResult } from "./session-runtime-adapter-stop.ts"; +export const runtimeAdapterCapabilitiesHeader = "x-crabfleet-runtime-adapter-capabilities"; +export const runtimeAdapterDeleteTombstoneCapability = "delete-tombstone-v1"; + export type RuntimeAdapterWorkspaceLifecycleDependencies = { now(): number; fetch(input: string, init: RequestInit): Promise; @@ -538,7 +541,12 @@ export class RuntimeAdapterWorkspaceLifecycle { ); const response = await this.dependencies.fetch( runtimeAdapterWorkspaceUrl(controlPlane, adapterWorkspaceId), - { method: "DELETE" }, + { + method: "DELETE", + headers: { + [runtimeAdapterCapabilitiesHeader]: runtimeAdapterDeleteTombstoneCapability, + }, + }, ); const body = response.status === 204 ? null : await this.dependencies.readResponseBody(response); @@ -553,7 +561,11 @@ export class RuntimeAdapterWorkspaceLifecycle { const message = parsed?.message ?? redactedAdapterResponseMessage(body, fallbackMessage, [adapterWorkspaceId]); - if (response.status === 404 && retryMissing) { + if ( + response.status === 404 && + retryMissing && + adapterAdvertisesCapability(response, runtimeAdapterDeleteTombstoneCapability) + ) { // An ambiguous create may still appear. Accepted DELETE retries must replay // the adapter's retained stopping or terminal tombstone instead. return { @@ -573,6 +585,12 @@ export class RuntimeAdapterWorkspaceLifecycle { } } +function adapterAdvertisesCapability(response: Response, capability: string): boolean { + return (response.headers.get(runtimeAdapterCapabilitiesHeader) ?? "") + .split(/[\s,]+/u) + .includes(capability); +} + export function runtimeAdapterProviderConfigured(env: RuntimeEnv): boolean { return Boolean( configuredRuntimeAdapterControlPlane(env, "profile-route") && runtimeAdapterToken(env), diff --git a/tests/runtime-adapter-workspaces.test.ts b/tests/runtime-adapter-workspaces.test.ts index 631109a9..4f76a1f4 100644 --- a/tests/runtime-adapter-workspaces.test.ts +++ b/tests/runtime-adapter-workspaces.test.ts @@ -5,6 +5,8 @@ import { containerCapabilities } from "../src/worker/session-model.ts"; import type { RuntimeEnv } from "../src/worker/env.ts"; import { RuntimeAdapterWorkspaceLifecycle, + runtimeAdapterCapabilitiesHeader, + runtimeAdapterDeleteTombstoneCapability, type RuntimeAdapterWorkspaceLifecycleDependencies, } from "../src/worker/runtime-adapter-workspaces.ts"; import type { InteractiveProvisionResult } from "../src/worker/provisioning/types.ts"; @@ -411,7 +413,15 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi requests.push({ url: input, method: init.method }); return responseStatus === 204 ? new Response(null, { status: 204 }) - : Response.json({ message: "workspace not found" }, { status: responseStatus }); + : Response.json( + { message: "workspace not found" }, + { + status: responseStatus, + headers: { + [runtimeAdapterCapabilitiesHeader]: runtimeAdapterDeleteTombstoneCapability, + }, + }, + ); }, }), ); @@ -448,6 +458,39 @@ test("superseded pending creates retry DELETE until the old workspace becomes vi ]); }); +test("create-pending cleanup preserves legacy 404 release semantics", async () => { + let requestHeaders = new Headers(); + const service = new RuntimeAdapterWorkspaceLifecycle( + runtimeEnv(), + dependencies({ + async fetch(_input, init) { + requestHeaders = new Headers(init.headers); + return Response.json({ message: "workspace not found" }, { status: 404 }); + }, + }), + ); + + assert.deepEqual( + await service.stopForSession( + "IS-42", + "workspace-superseded", + { + profile: "default", + controlPlane: "https://adapter.example.test/", + }, + true, + ), + { + status: "stopped", + message: "workspace not found", + }, + ); + assert.equal( + requestHeaders.get(runtimeAdapterCapabilitiesHeader), + runtimeAdapterDeleteTombstoneCapability, + ); +}); + test("create-pending cleanup recovers a lost DELETE response from the terminal tombstone", async () => { let attempt = 0; const service = new RuntimeAdapterWorkspaceLifecycle( @@ -456,7 +499,15 @@ test("create-pending cleanup recovers a lost DELETE response from the terminal t async fetch() { attempt += 1; if (attempt === 1) { - return Response.json({ message: "workspace not found" }, { status: 404 }); + return Response.json( + { message: "workspace not found" }, + { + status: 404, + headers: { + [runtimeAdapterCapabilitiesHeader]: runtimeAdapterDeleteTombstoneCapability, + }, + }, + ); } if (attempt === 2) { throw new Error("response lost after delete commit"); From 2b9e2d492db9d96f00a2e8a358c5e19077347454 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:10:18 +0200 Subject: [PATCH 216/242] fix(vnc): fail closed on probe timeout --- .../SDK/Connection/VNCConnection+API.swift | 32 +++++++++--------- .../RoyalVNCKitTests/AuditFindingsTests.swift | 33 +++++++------------ 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index f6978231..a1e0e0b0 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -403,10 +403,14 @@ extension VNCConnection { framebufferRequestLock.unlock() return } - if probeTimedOut { - expiredPixelFormatFenceCapabilityProbePayload = pixelFormatFenceCapabilityProbePayload - pixelFormatFenceCapabilityProbePayload = nil - } + if probeTimedOut { + expiredPixelFormatFenceCapabilityProbePayload = pixelFormatFenceCapabilityProbePayload + pixelFormatFenceCapabilityProbePayload = nil + framebufferRequestLock.unlock() + handleBreakingError(VNCError.protocol(.pixelFormatTransitionTimedOut)) + logger.logDebug("Fence capability negotiation timed out") + return + } let isWaitingForLegacyFramebufferBoundary = negotiationTimedOut && !state.areContinuousUpdatesEnabled @@ -414,19 +418,15 @@ extension VNCConnection { if negotiationTimedOut && !isWaitingForLegacyFramebufferBoundary { pendingPixelFormatTransition = nil } - let transition = probeTimedOut ? takePendingPixelFormatTransitionLocked() : nil - let shouldResumeUpdates = - transition == nil - && pendingPixelFormatTransition == nil - && !framebufferUpdateRequestOutstanding - && !isPixelFormatTransitionInFlight - framebufferRequestLock.unlock() + let shouldResumeUpdates = + pendingPixelFormatTransition == nil + && !framebufferUpdateRequestOutstanding + && !isPixelFormatTransitionInFlight + framebufferRequestLock.unlock() - if let transition { - enqueuePixelFormatTransition(transition) - } else if shouldResumeUpdates { - scheduleNextFramebufferUpdate() - } + if shouldResumeUpdates { + scheduleNextFramebufferUpdate() + } logger.logDebug("Fence capability negotiation timed out") } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 13a89336..414bf60d 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -271,7 +271,7 @@ struct AuditFindingsTests { } @Test - func expiresUnansweredFenceCapabilityProbeWithoutStallingUpdates() async throws { + func disconnectsWhenFenceCapabilityProbeIsUnanswered() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -298,15 +298,15 @@ struct AuditFindingsTests { connection.expirePixelFormatFenceNegotiation() #expect(connection.pixelFormatFenceCapabilityProbePayload == nil) - #expect(connection.expiredPixelFormatFenceCapabilityProbePayload != nil) - let transition = try #require(connection.clientToServerMessageQueue.dequeue()) - try await transition.message.send(connection: AuditWritingConnection()) + #expect(connection.expiredPixelFormatFenceCapabilityProbePayload == nil) + #expect(connection.connectionState.status == .disconnected) #expect(connection.pendingPixelFormatTransition == nil) - #expect(connection.state.pixelFormat?.depth == 8) + #expect(connection.state.pixelFormat?.depth == 24) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) } @Test - func acceptsLateFenceCapabilityResponseAfterProbeTimeout() async throws { + func lateFenceCapabilityResponseCannotReviveTimedOutConnection() async throws { let connection = VNCConnection( settings: makeSettings(), framebufferAllocator: VNCFramebufferMallocAllocator() @@ -330,8 +330,7 @@ struct AuditFindingsTests { connection.updateColorDepth(.depth8Bit) connection.expirePixelFormatFenceNegotiation() - let fallback = try #require(connection.clientToServerMessageQueue.dequeue()) - try await fallback.message.send(connection: AuditWritingConnection()) + #expect(connection.connectionState.status == .disconnected) try connection.handleServerFence( VNCProtocol.ServerFence( @@ -342,12 +341,9 @@ struct AuditFindingsTests { ) #expect(connection.expiredPixelFormatFenceCapabilityProbePayload == nil) - #expect(connection.state.pixelFormatTransitionFenceFlags.contains(.blockBefore)) - #expect(connection.state.pixelFormatTransitionFenceFlags.contains(.syncNext)) - - connection.state.areContinuousUpdatesEnabled = true - connection.updateColorDepth(.depth16Bit) - #expect(connection.clientToServerMessageQueue.dequeue() != nil) + #expect(connection.connectionState.status == .disconnected) + #expect(connection.state.pixelFormatTransitionFenceFlags.isEmpty) + #expect(connection.clientToServerMessageQueue.dequeue() == nil) } @Test @@ -631,14 +627,7 @@ struct AuditFindingsTests { ) connection.updateColorDepth(.depth8Bit) connection.expirePixelFormatFenceNegotiation() - let transition = try #require(connection.clientToServerMessageQueue.dequeue()) - try await transition.message.send(connection: AuditWritingConnection()) - - let restartedAfterTransition = try zrle.zStream.decompressedData( - compressedData: compressedChunks[0], - uncompressedSize: 1_000 - ) - #expect(restartedAfterTransition == Data(repeating: 0x41, count: 1_000)) + #expect(connection.connectionState.status == .disconnected) try connection.handleServerFence( VNCProtocol.ServerFence( From d2f8b1e7f05434f74a5dd12e4f5367e3f83020ed Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:10:31 +0200 Subject: [PATCH 217/242] docs(changelog): record review blocker fixes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 956ddea5..f7e80bfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Close the final review blockers by requiring an exact live credential-policy lease at promotion, negotiating strict runtime-adapter deletion tombstones without breaking legacy `404` release semantics, disconnecting VNC sessions when pixel-format capability probes cannot establish a safe ZRLE boundary, classifying aborted JSON body streams as bad requests, and preserving live terminal subscriptions when browser history restores the sessions grid. - Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. - Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets. - Finish the audited terminal, credential, runtime, and native-app lifecycle boundaries by explicitly negotiating the generation-fenced GitHub Actions runner protocol, retiring stale and overflowing runner input queues, capturing relay replacement during viewer authorization, fencing retired Go terminal attachments, repairing credential lookup namespaces with rollback-compatible staging, requiring replayable runtime-adapter deletion tombstones, validating Apple Remote Desktop Diffie-Hellman groups, keying desktop publication cleanup by the API host ID, and requiring exact retained identity before uncertain publication recovery; the runner guide now preserves raw fallback, bounds admission before serialized restricted steering, and distinguishes unknown delivery from rejection. From dcbf92be1c1f8fb438c47431f319bc3c5d43c460 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:14:25 +0200 Subject: [PATCH 218/242] test(assets): serialize generated fixtures --- tests/embedded-terminal-assets.test.ts | 6 +-- tests/generated-assets.test.ts | 6 +-- tests/helpers/generated-assets.ts | 51 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 tests/helpers/generated-assets.ts diff --git a/tests/embedded-terminal-assets.test.ts b/tests/embedded-terminal-assets.test.ts index ff50477f..c62dd74f 100644 --- a/tests/embedded-terminal-assets.test.ts +++ b/tests/embedded-terminal-assets.test.ts @@ -1,17 +1,15 @@ import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { createServer } from "node:http"; import { test } from "node:test"; -import { promisify } from "node:util"; import { loadGhosttyRuntime } from "@openclaw/libterminal/browser"; import { GHOSTTY_ASSET_PATHS, readGhosttyAsset } from "@openclaw/libterminal/node"; import { readGhosttyWorkerAsset } from "@openclaw/libterminal/worker-assets"; -const execFileAsync = promisify(execFile); +import { generateAssetsForTest } from "./helpers/generated-assets.ts"; test("libterminal Worker Ghostty assets are byte-exact and keep Crabfleet response policy", async () => { - await execFileAsync(process.execPath, ["scripts/generate-assets.mjs"]); + await generateAssetsForTest(); const generated = await import(`../src/generated.ts?terminal-assets=${Date.now()}`); const { terminalAssetResponse } = await import( `../src/worker/terminal-assets.ts?terminal-assets=${Date.now()}` diff --git a/tests/generated-assets.test.ts b/tests/generated-assets.test.ts index a6d0af72..c4513266 100644 --- a/tests/generated-assets.test.ts +++ b/tests/generated-assets.test.ts @@ -1,13 +1,11 @@ import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { readFile } from "node:fs/promises"; import test from "node:test"; -import { promisify } from "node:util"; -const execFileAsync = promisify(execFile); +import { generateAssetsForTest } from "./helpers/generated-assets.ts"; test("generated embedded specification matches the canonical markdown", async () => { - await execFileAsync(process.execPath, ["scripts/generate-assets.mjs"]); + await generateAssetsForTest(); const { SPEC_MARKDOWN } = await import(`../src/generated.ts?spec-assets=${Date.now()}`); const source = await readFile(new URL("../docs/spec.md", import.meta.url), "utf8"); const markdown = source.replace(/^---\n[\s\S]*?\n---\n+/, ""); diff --git a/tests/helpers/generated-assets.ts b/tests/helpers/generated-assets.ts new file mode 100644 index 00000000..488873c4 --- /dev/null +++ b/tests/helpers/generated-assets.ts @@ -0,0 +1,51 @@ +import { execFile } from "node:child_process"; +import { mkdir, rm, stat } from "node:fs/promises"; +import { setTimeout as delay } from "node:timers/promises"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const lockParent = new URL("../../dist/", import.meta.url); +const lockPath = new URL("../../dist/.generated-assets-test.lock/", import.meta.url); +const lockWaitMs = 120_000; +const staleLockMs = 300_000; + +export async function generateAssetsForTest(): Promise { + await mkdir(lockParent, { recursive: true }); + const deadline = Date.now() + lockWaitMs; + while (!(await tryAcquireLock())) { + if (Date.now() >= deadline) throw new Error("timed out waiting for generated asset test lock"); + await delay(50); + } + + try { + await execFileAsync(process.execPath, ["scripts/generate-assets.mjs"]); + } finally { + await rm(lockPath, { recursive: true, force: true }); + } +} + +async function tryAcquireLock(): Promise { + try { + await mkdir(lockPath); + return true; + } catch (error) { + if (!isAlreadyExists(error)) throw error; + try { + const lock = await stat(lockPath); + if (Date.now() - lock.mtimeMs > staleLockMs) { + await rm(lockPath, { recursive: true, force: true }); + } + } catch (lockError) { + if (!isMissing(lockError)) throw lockError; + } + return false; + } +} + +function isAlreadyExists(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "EEXIST"; +} + +function isMissing(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "ENOENT"; +} From ac74a0bcd529492e368d7587c281eb79301e89e8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:24:42 +0200 Subject: [PATCH 219/242] test(assets): avoid unsafe stale lock recovery --- tests/helpers/generated-assets.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/helpers/generated-assets.ts b/tests/helpers/generated-assets.ts index 488873c4..19a32fa9 100644 --- a/tests/helpers/generated-assets.ts +++ b/tests/helpers/generated-assets.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir, rm, stat } from "node:fs/promises"; +import { mkdir, rm } from "node:fs/promises"; import { setTimeout as delay } from "node:timers/promises"; import { promisify } from "node:util"; @@ -7,7 +7,6 @@ const execFileAsync = promisify(execFile); const lockParent = new URL("../../dist/", import.meta.url); const lockPath = new URL("../../dist/.generated-assets-test.lock/", import.meta.url); const lockWaitMs = 120_000; -const staleLockMs = 300_000; export async function generateAssetsForTest(): Promise { await mkdir(lockParent, { recursive: true }); @@ -30,14 +29,6 @@ async function tryAcquireLock(): Promise { return true; } catch (error) { if (!isAlreadyExists(error)) throw error; - try { - const lock = await stat(lockPath); - if (Date.now() - lock.mtimeMs > staleLockMs) { - await rm(lockPath, { recursive: true, force: true }); - } - } catch (lockError) { - if (!isMissing(lockError)) throw lockError; - } return false; } } @@ -45,7 +36,3 @@ async function tryAcquireLock(): Promise { function isAlreadyExists(error: unknown): boolean { return error instanceof Error && "code" in error && error.code === "EEXIST"; } - -function isMissing(error: unknown): boolean { - return error instanceof Error && "code" in error && error.code === "ENOENT"; -} From 98a444b113e791157e4490160b1208ea58090fbb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:37:07 +0200 Subject: [PATCH 220/242] test(assets): hold a crash-released generation lock --- tests/embedded-terminal-assets.test.ts | 51 +++++++++---------- tests/generated-assets.test.ts | 15 +++--- tests/helpers/generated-assets.ts | 69 ++++++++++++++++++-------- 3 files changed, 82 insertions(+), 53 deletions(-) diff --git a/tests/embedded-terminal-assets.test.ts b/tests/embedded-terminal-assets.test.ts index c62dd74f..e6324ff7 100644 --- a/tests/embedded-terminal-assets.test.ts +++ b/tests/embedded-terminal-assets.test.ts @@ -6,34 +6,35 @@ import { loadGhosttyRuntime } from "@openclaw/libterminal/browser"; import { GHOSTTY_ASSET_PATHS, readGhosttyAsset } from "@openclaw/libterminal/node"; import { readGhosttyWorkerAsset } from "@openclaw/libterminal/worker-assets"; -import { generateAssetsForTest } from "./helpers/generated-assets.ts"; +import { withGeneratedAssetsForTest } from "./helpers/generated-assets.ts"; test("libterminal Worker Ghostty assets are byte-exact and keep Crabfleet response policy", async () => { - await generateAssetsForTest(); - const generated = await import(`../src/generated.ts?terminal-assets=${Date.now()}`); - const { terminalAssetResponse } = await import( - `../src/worker/terminal-assets.ts?terminal-assets=${Date.now()}` - ); - assert.equal(generated.APP_HTML.includes("__GHOSTTY_WASM_PATH__"), false); - assert.equal(generated.APP_HTML.includes(GHOSTTY_ASSET_PATHS.wasm), true); - assert.equal("GHOSTTY_VT_WASM_BASE64" in generated, false); - - for (const pathname of Object.values(GHOSTTY_ASSET_PATHS)) { - const expected = await readGhosttyAsset(pathname); - const workerAsset = readGhosttyWorkerAsset(pathname); - const response = terminalAssetResponse(pathname); - assert.ok(expected); - assert.equal(workerAsset?.contentType, expected.contentType); - assert.equal( - Buffer.compare(Buffer.from(workerAsset?.body ?? []), Buffer.from(expected.body)), - 0, + await withGeneratedAssetsForTest(async () => { + const generated = await import(`../src/generated.ts?terminal-assets=${Date.now()}`); + const { terminalAssetResponse } = await import( + `../src/worker/terminal-assets.ts?terminal-assets=${Date.now()}` ); - assert.equal(response?.status, 200); - assert.equal(response?.headers.get("content-type"), expected.contentType); - assert.equal(response?.headers.get("cache-control"), "no-store"); - assert.equal(Buffer.compare(Buffer.from(await response!.arrayBuffer()), expected.body), 0); - } - assert.equal(terminalAssetResponse("/vendor/unknown.js"), null); + assert.equal(generated.APP_HTML.includes("__GHOSTTY_WASM_PATH__"), false); + assert.equal(generated.APP_HTML.includes(GHOSTTY_ASSET_PATHS.wasm), true); + assert.equal("GHOSTTY_VT_WASM_BASE64" in generated, false); + + for (const pathname of Object.values(GHOSTTY_ASSET_PATHS)) { + const expected = await readGhosttyAsset(pathname); + const workerAsset = readGhosttyWorkerAsset(pathname); + const response = terminalAssetResponse(pathname); + assert.ok(expected); + assert.equal(workerAsset?.contentType, expected.contentType); + assert.equal( + Buffer.compare(Buffer.from(workerAsset?.body ?? []), Buffer.from(expected.body)), + 0, + ); + assert.equal(response?.status, 200); + assert.equal(response?.headers.get("content-type"), expected.contentType); + assert.equal(response?.headers.get("cache-control"), "no-store"); + assert.equal(Buffer.compare(Buffer.from(await response!.arrayBuffer()), expected.body), 0); + } + assert.equal(terminalAssetResponse("/vendor/unknown.js"), null); + }); }); test("Ghostty loader injects the explicit WASM runtime into terminal modules", async () => { diff --git a/tests/generated-assets.test.ts b/tests/generated-assets.test.ts index c4513266..15142249 100644 --- a/tests/generated-assets.test.ts +++ b/tests/generated-assets.test.ts @@ -2,14 +2,15 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import test from "node:test"; -import { generateAssetsForTest } from "./helpers/generated-assets.ts"; +import { withGeneratedAssetsForTest } from "./helpers/generated-assets.ts"; test("generated embedded specification matches the canonical markdown", async () => { - await generateAssetsForTest(); - const { SPEC_MARKDOWN } = await import(`../src/generated.ts?spec-assets=${Date.now()}`); - const source = await readFile(new URL("../docs/spec.md", import.meta.url), "utf8"); - const markdown = source.replace(/^---\n[\s\S]*?\n---\n+/, ""); + await withGeneratedAssetsForTest(async () => { + const { SPEC_MARKDOWN } = await import(`../src/generated.ts?spec-assets=${Date.now()}`); + const source = await readFile(new URL("../docs/spec.md", import.meta.url), "utf8"); + const markdown = source.replace(/^---\n[\s\S]*?\n---\n+/, ""); - assert.equal(SPEC_MARKDOWN, markdown); - assert.match(SPEC_MARKDOWN, /relay-generation-fenced binary `CFR1` input/); + assert.equal(SPEC_MARKDOWN, markdown); + assert.match(SPEC_MARKDOWN, /relay-generation-fenced binary `CFR1` input/); + }); }); diff --git a/tests/helpers/generated-assets.ts b/tests/helpers/generated-assets.ts index 19a32fa9..cce15c67 100644 --- a/tests/helpers/generated-assets.ts +++ b/tests/helpers/generated-assets.ts @@ -1,38 +1,65 @@ import { execFile } from "node:child_process"; -import { mkdir, rm } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { createServer, type Server } from "node:net"; import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; const execFileAsync = promisify(execFile); -const lockParent = new URL("../../dist/", import.meta.url); -const lockPath = new URL("../../dist/.generated-assets-test.lock/", import.meta.url); const lockWaitMs = 120_000; +const lockPortBase = 49_152; +const lockPortRange = 65_535 - lockPortBase + 1; +const lockPort = + lockPortBase + + (createHash("sha256") + .update(fileURLToPath(new URL("../../", import.meta.url))) + .digest() + .readUInt16BE(0) % + lockPortRange); -export async function generateAssetsForTest(): Promise { - await mkdir(lockParent, { recursive: true }); - const deadline = Date.now() + lockWaitMs; - while (!(await tryAcquireLock())) { - if (Date.now() >= deadline) throw new Error("timed out waiting for generated asset test lock"); - await delay(50); - } - +export async function withGeneratedAssetsForTest(consume: () => T | Promise): Promise { + const lock = await acquireLock(); try { await execFileAsync(process.execPath, ["scripts/generate-assets.mjs"]); + return await consume(); } finally { - await rm(lockPath, { recursive: true, force: true }); + await closeServer(lock); } } -async function tryAcquireLock(): Promise { - try { - await mkdir(lockPath); - return true; - } catch (error) { - if (!isAlreadyExists(error)) throw error; - return false; +async function acquireLock(): Promise { + const deadline = Date.now() + lockWaitMs; + while (true) { + const server = createServer(); + try { + await listen(server); + return server; + } catch (error) { + if (!isAddressInUse(error)) throw error; + if (Date.now() >= deadline) + throw new Error("timed out waiting for generated asset test lock"); + await delay(50); + } } } -function isAlreadyExists(error: unknown): boolean { - return error instanceof Error && "code" in error && error.code === "EEXIST"; +function listen(server: Server): Promise { + return new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + server.once("error", onError); + server.listen({ host: "127.0.0.1", port: lockPort, exclusive: true }, () => { + server.off("error", onError); + resolve(); + }); + }); +} + +function closeServer(server: Server): Promise { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +function isAddressInUse(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "EADDRINUSE"; } From 88c2c25e7b8d71931b3b52600dcf7edc7befd8eb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:39:01 +0200 Subject: [PATCH 221/242] fix(terminal): preserve ambiguous input outcomes --- internal/terminalws/client.go | 17 ++++++++-- internal/terminalws/client_test.go | 28 ++++++++++++++++ src/github-actions-runner.ts | 44 +++++++++++++++++++++++-- src/worker/terminal-hub.ts | 8 +++-- tests/github-actions-runner.test.ts | 51 +++++++++++++++++++++++++++++ tests/terminal-hub.test.ts | 26 ++++++++------- 6 files changed, 154 insertions(+), 20 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 5167fbba..f906dddd 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -357,10 +357,19 @@ func (c *Client) waitForInputConfirmation(ctx context.Context, waiter chan error return err default: } - c.clearInputWaiter(waiter) + if !c.clearInputWaiter(waiter) { + return <-waiter + } return readerUnavailableError(c.readerError()) case <-ctx.Done(): - c.clearInputWaiter(waiter) + select { + case err := <-waiter: + return err + default: + } + if !c.clearInputWaiter(waiter) { + return <-waiter + } c.closeNow() return ctx.Err() } @@ -682,12 +691,14 @@ func (c *Client) registerInputWaiter(waiter chan error) error { return nil } -func (c *Client) clearInputWaiter(waiter chan error) { +func (c *Client) clearInputWaiter(waiter chan error) bool { c.stateMu.Lock() defer c.stateMu.Unlock() if c.inputWaiter == waiter { c.inputWaiter = nil + return true } + return false } func (c *Client) completeInput(err error) { diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index e6c07139..74256103 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -1460,6 +1460,34 @@ func TestSendInputConfirmedReturnsImmediatelyForEmptyInput(t *testing.T) { } } +func TestWaitForInputConfirmationPrefersDetachedAcceptanceOverTimeout(t *testing.T) { + client := &Client{readerDone: make(chan struct{})} + waiter := make(chan error, 1) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + time.AfterFunc(time.Millisecond, func() { + waiter <- nil + }) + + if err := client.waitForInputConfirmation(ctx, waiter); err != nil { + t.Fatalf("confirmation = %v", err) + } +} + +func TestWaitForInputConfirmationPrefersDetachedAcceptanceOverReaderShutdown(t *testing.T) { + readerDone := make(chan struct{}) + close(readerDone) + client := &Client{readerDone: readerDone} + waiter := make(chan error, 1) + time.AfterFunc(time.Millisecond, func() { + waiter <- nil + }) + + if err := client.waitForInputConfirmation(context.Background(), waiter); err != nil { + t.Fatalf("confirmation = %v", err) + } +} + func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { inputReceived := make(chan struct{}) releaseServer := make(chan struct{}) diff --git a/src/github-actions-runner.ts b/src/github-actions-runner.ts index bc0c6a1c..48dca9dd 100644 --- a/src/github-actions-runner.ts +++ b/src/github-actions-runner.ts @@ -8,9 +8,11 @@ import { const runnerInputQueueMaxBytes = 16 * 1024 * 1024; const runnerInputQueueMaxFrames = 32; const runnerInputQueueMaxAgeMs = 5_000; +const runnerInputWriteTimeoutMs = 5_000; const runnerInputBacklogError = "GitHub Actions runner input backlog exceeded"; const runnerInputExpiredError = "GitHub Actions runner input expired"; const runnerInputGenerationError = "GitHub Actions runner generation changed"; +const runnerInputWriteTimeoutError = "GitHub Actions runner input write timed out"; type RunnerInputQueue = { bytes: number; @@ -34,6 +36,7 @@ export function acceptGitHubActionsRunnerInput( message: string | ArrayBuffer, writeToPty: (payload: ArrayBuffer) => void | Promise, now: () => number = Date.now, + writeTimeoutMs: number = runnerInputWriteTimeoutMs, ): Promise { const input = parseGitHubActionsRelayInput(message); if (!input) return Promise.resolve(false); @@ -97,12 +100,20 @@ export function acceptGitHubActionsRunnerInput( ); return; } - try { - await writeToPty(input.payload); + const writeResult = await writeRunnerInputWithTimeout( + () => writeToPty(input.payload), + writeTimeoutMs, + ); + if (writeResult === "timed-out") { + queue.retired = true; + closeRunnerInputSocket(socket, runnerInputWriteTimeoutError); + return; + } + if (writeResult === "accepted") { if (isActiveRunnerInputQueue(socket, queue)) { sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, true); } - } catch { + } else { if (isActiveRunnerInputQueue(socket, queue)) { sendRunnerInputAcknowledgement(socket, input.inputId, input.generation, false); } @@ -116,6 +127,33 @@ export function acceptGitHubActionsRunnerInput( return queued.then(() => true); } +function writeRunnerInputWithTimeout( + write: () => void | Promise, + timeoutMs: number, +): Promise<"accepted" | "rejected" | "timed-out"> { + return new Promise((resolve) => { + let settled = false; + const finish = (result: "accepted" | "rejected" | "timed-out") => { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolve(result); + }; + const timeout = setTimeout(() => finish("timed-out"), timeoutMs); + let result: void | Promise; + try { + result = write(); + } catch { + finish("rejected"); + return; + } + Promise.resolve(result).then( + () => finish("accepted"), + () => finish("rejected"), + ); + }); +} + function sendRunnerInputAcknowledgement( socket: GitHubActionsRelaySocket, inputId: string, diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 39fb0e0c..36cbf88c 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -679,7 +679,9 @@ export class TerminalHub { (pending) => pending.runnerGeneration !== relayEvent.generation, { accepted: false, - error: "GitHub Actions runner was replaced before accepting input", + deliveryUnknown: true, + error: + "terminal input delivery outcome is unknown; the runner may still complete it", }, ); } else { @@ -691,7 +693,9 @@ export class TerminalHub { runnerGenerationAtReceipt as number, { accepted: false, - error: "GitHub Actions runner was replaced before accepting input", + deliveryUnknown: true, + error: + "terminal input delivery outcome is unknown; the runner may still complete it", }, ); } diff --git a/tests/github-actions-runner.test.ts b/tests/github-actions-runner.test.ts index b6442cd6..90f3c9e8 100644 --- a/tests/github-actions-runner.test.ts +++ b/tests/github-actions-runner.test.ts @@ -240,6 +240,57 @@ test("runner bounds queued bytes while a PTY write is stalled", async () => { ); }); +test("runner times out a stalled PTY write and retires its socket queue", async () => { + const socket = relaySocket(); + let completeWrite!: () => void; + const blockedWrite = new Promise((resolve) => { + completeWrite = resolve; + }); + const writes: string[] = []; + const first = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-first", "first"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + await blockedWrite; + }, + Date.now, + 10, + ); + const queued = acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-queued", "queued"), + async (payload) => { + writes.push(new TextDecoder().decode(payload)); + }, + Date.now, + 10, + ); + + assert.deepEqual(await Promise.all([first, queued]), [true, true]); + assert.deepEqual(writes, ["first"]); + assert.deepEqual(socket.sent, []); + assert.deepEqual(socket.closes, [ + { code: 1012, reason: "GitHub Actions runner input write timed out" }, + ]); + + assert.equal( + await acceptGitHubActionsRunnerInput( + socket, + encodeGitHubActionsRelayInput("input-late", "late"), + async () => { + assert.fail("retired input must not reach the PTY"); + }, + Date.now, + 10, + ), + true, + ); + completeWrite(); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(socket.sent, []); +}); + test("runner does not execute queued input after its socket is replaced", async () => { const replaced = relaySocket(); const replacement = relaySocket(); diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 37a9b0c2..b686cae8 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -1511,7 +1511,7 @@ test("GitHub Actions runner disconnect rejects pending input without closing the server.emit("close"); }); -test("GitHub Actions runner replacement rejects old input and accepts new input", async () => { +test("GitHub Actions runner replacement reports old input as unknown and accepts new input", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -1559,9 +1559,9 @@ test("GitHub Actions runner replacement rejects old input and accepts new input" assert.equal( replacementEvents.some( (event) => - (event as { type?: string }).type === "input-rejected" && + (event as { type?: string }).type === "input-delivery-unknown" && (event as { error?: string }).error === - "GitHub Actions runner was replaced before accepting input", + "terminal input delivery outcome is unknown; the runner may still complete it", ), true, JSON.stringify(replacementEvents), @@ -1587,16 +1587,18 @@ test("GitHub Actions runner replacement rejects old input and accepts new input" .map((payload) => frame(payload)) .filter((message) => message.type === TerminalMessageType.Event) .map((message) => decodeJsonPayload(message.payload) as { type?: string }) - .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + .filter( + (message) => message.type === "input-accepted" || message.type === "input-delivery-unknown", + ); assert.deepEqual( completions.map((message) => message.type), - ["input-rejected", "input-accepted"], + ["input-delivery-unknown", "input-accepted"], ); assert.deepEqual(upstream.closed, []); server.emit("close"); }); -test("queued runner replacement rejects only acknowledgements sent to the old generation", async () => { +test("queued runner replacement marks only old-generation acknowledgements unknown", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -1672,11 +1674,11 @@ test("queued runner replacement rejects only acknowledgements sent to the old ge .map((payload) => frame(payload)) .filter((message) => message.type === TerminalMessageType.Event) .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) - .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + .filter((message) => message.type === "input-delivery-unknown"); assert.deepEqual(completions, [ { - type: "input-rejected", - error: "GitHub Actions runner was replaced before accepting input", + type: "input-delivery-unknown", + error: "terminal input delivery outcome is unknown; the runner may still complete it", }, ]); assert.deepEqual(upstream.closed, []); @@ -1773,11 +1775,11 @@ test("relay generations bind interleaved replacement input before lifecycle proc .map((payload) => frame(payload)) .filter((message) => message.type === TerminalMessageType.Event) .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) - .filter((message) => message.type === "input-accepted" || message.type === "input-rejected"); + .filter((message) => message.type === "input-delivery-unknown"); assert.deepEqual(completions, [ { - type: "input-rejected", - error: "GitHub Actions runner was replaced before accepting input", + type: "input-delivery-unknown", + error: "terminal input delivery outcome is unknown; the runner may still complete it", }, ]); assert.deepEqual(upstream.closed, []); From 73076abe4e51d2aec0c38a6355d16264e61972de Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:41:06 +0200 Subject: [PATCH 222/242] fix(credentials): preserve rollback recovery fences --- ...ential_policy_registration_write_fence.sql | 156 ++++++++++++++++ .../sandbox-credential-policy-cleanup.ts | 30 ++-- ...-credential-policy-registration-service.ts | 21 ++- .../sandbox-credential-policy-repository.ts | 30 ++++ .../sandbox-credential-policy-scanner.ts | 13 +- src/worker/session-cleanup.ts | 11 +- src/worker/session-terminal-finalization.ts | 93 +++++----- .../sandbox-credential-policy-cleanup.test.ts | 6 + ...ndbox-credential-policy-repository.test.ts | 167 +++++++++++++++++- .../sandbox-credential-policy-scanner.test.ts | 1 + tests/session-cleanup.test.ts | 2 + tests/session-terminal-finalization.test.ts | 16 +- 12 files changed, 480 insertions(+), 66 deletions(-) create mode 100644 migrations/0040_credential_policy_registration_write_fence.sql diff --git a/migrations/0040_credential_policy_registration_write_fence.sql b/migrations/0040_credential_policy_registration_write_fence.sql new file mode 100644 index 00000000..1f2e0715 --- /dev/null +++ b/migrations/0040_credential_policy_registration_write_fence.sql @@ -0,0 +1,156 @@ +ALTER TABLE interactive_session_credential_policy_registrations + ADD COLUMN registration_write_started INTEGER NOT NULL DEFAULT 0 + CHECK (registration_write_started IN (0, 1)); + +-- Existing staged rows may already have written a replacement generation to +-- the Durable Object. Conservatively retain their legacy-writer fence until +-- current recovery either completes or removes the staged registration. +UPDATE interactive_session_credential_policy_registrations +SET registration_write_started = 1; + +DROP TRIGGER IF EXISTS fence_staged_credential_policy_insert; +DROP TRIGGER IF EXISTS fence_staged_credential_policy_update; +DROP TRIGGER IF EXISTS fence_staged_credential_policy_delete; + +CREATE TRIGGER fence_staged_credential_policy_insert +BEFORE INSERT ON interactive_session_credential_policies +WHEN EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = NEW.session_id + AND staged.sandbox_id = NEW.sandbox_id + AND staged.registration_generation != NEW.registration_generation + AND ( + ( + staged.registration_write_started = 1 + AND NOT ( + staged.state = 'cleanup_pending' + AND NEW.state = 'cleanup_pending' + ) + ) + OR ( + NEW.state != 'cleanup_pending' + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) + ) + ) + AND NOT ( + staged.state = 'registering' + AND staged.repair_generation = NEW.registration_generation + AND staged.registration_claim = NEW.registration_claim + AND staged.registration_claim_expires_at = NEW.registration_claim_expires_at + AND NEW.state = 'registering' + ) +) +BEGIN + SELECT RAISE(IGNORE); +END; + +CREATE TRIGGER fence_staged_credential_policy_update +BEFORE UPDATE ON interactive_session_credential_policies +WHEN EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = NEW.session_id + AND staged.sandbox_id = NEW.sandbox_id + AND staged.registration_generation != NEW.registration_generation + AND ( + ( + staged.registration_write_started = 1 + AND NOT ( + staged.state = 'cleanup_pending' + AND OLD.state != 'cleanup_pending' + AND NEW.state = 'cleanup_pending' + ) + ) + OR ( + NOT (OLD.state != 'cleanup_pending' AND NEW.state = 'cleanup_pending') + AND ( + ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) + ) + ) + AND NOT ( + staged.state = 'registering' + AND staged.repair_generation = NEW.registration_generation + AND staged.registration_claim = OLD.registration_claim + AND staged.registration_claim_expires_at = OLD.registration_claim_expires_at + AND OLD.state = 'registering' + AND NEW.state = 'active' + AND NEW.registration_claim IS NULL + AND NEW.registration_claim_expires_at IS NULL + ) +) +BEGIN + SELECT RAISE(IGNORE); +END; + +CREATE TRIGGER fence_staged_credential_policy_delete +BEFORE DELETE ON interactive_session_credential_policies +WHEN EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations AS staged + WHERE staged.session_id = OLD.session_id + AND staged.sandbox_id = OLD.sandbox_id + AND ( + staged.registration_write_started = 1 + OR ( + staged.state = 'registering' + AND staged.registration_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + ) + OR ( + staged.state = 'cleanup_pending' + AND ( + staged.cleanup_claim_expires_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 + OR staged.updated_at > + CAST(strftime('%s', 'now') AS INTEGER) * 1000 - 300000 + ) + ) + ) + AND NOT ( + staged.state = 'registering' + AND staged.repair_generation = OLD.registration_generation + AND staged.registration_claim = OLD.cleanup_claim + AND staged.registration_claim_expires_at = OLD.cleanup_claim_expires_at + AND OLD.state = 'cleanup_pending' + AND json_valid(staged.lookup_ids_json) + AND NOT EXISTS ( + SELECT 1 + FROM json_each(staged.lookup_ids_json) AS current_lookup + WHERE current_lookup.type = 'text' + AND current_lookup.value = OLD.lookup_id + ) + ) +) +BEGIN + SELECT RAISE(IGNORE); +END; diff --git a/src/worker/sandbox-credential-policy-cleanup.ts b/src/worker/sandbox-credential-policy-cleanup.ts index cd884cac..62e4368b 100644 --- a/src/worker/sandbox-credential-policy-cleanup.ts +++ b/src/worker/sandbox-credential-policy-cleanup.ts @@ -187,6 +187,21 @@ export async function stageTerminalCredentialPolicyCleanup( ]), ) .where(sandboxManagedStoredOwnershipCondition(ownership.fence)); + const registrationTransitions = generations.map(({ sandboxId }) => + db + .updateTable("interactive_session_credential_policy_registrations") + .set({ + state: "cleanup_pending", + registration_claim: null, + registration_claim_expires_at: null, + updated_at: stageRevision, + }) + .where("session_id", "=", session.id) + .where("sandbox_id", "=", sandboxId) + .where( + sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), + ), + ); const policyTransitions = generations.flatMap(({ generation, sandboxId }) => [ ...sandboxCredentialPolicyRefQueries( env, @@ -208,21 +223,8 @@ export async function stageTerminalCredentialPolicyCleanup( .where( sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), ), - db - .updateTable("interactive_session_credential_policy_registrations") - .set({ - state: "cleanup_pending", - registration_claim: null, - registration_claim_expires_at: null, - updated_at: stageRevision, - }) - .where("session_id", "=", session.id) - .where("sandbox_id", "=", sandboxId) - .where( - sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), - ), ]); - await executeBatch(env, [sessionTransition, ...policyTransitions]); + await executeBatch(env, [sessionTransition, ...registrationTransitions, ...policyTransitions]); const staged = await db .selectFrom("interactive_sessions") .select([ diff --git a/src/worker/sandbox-credential-policy-registration-service.ts b/src/worker/sandbox-credential-policy-registration-service.ts index 9f9d0734..ff5c6060 100644 --- a/src/worker/sandbox-credential-policy-registration-service.ts +++ b/src/worker/sandbox-credential-policy-registration-service.ts @@ -10,6 +10,7 @@ import { existingSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, incompleteSandboxCredentialPolicyGeneration, + markSandboxCredentialPolicyRegistrationWriteStarted, recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, repairSandboxCredentialPolicyReferences, @@ -115,9 +116,21 @@ async function repairIncompleteSandboxCredentialPolicyLookupSet( ) { throw new Error("sandbox credential policy registration claim was revoked"); } + const missingLookupIds = registration.lookupIds.filter((lookupId) => !records.get(lookupId)); + if (missingLookupIds.length > 0) { + registrationExpiresAt = await markSandboxCredentialPolicyRegistrationWriteStarted( + env, + sessionId, + sandboxId, + registration, + ownershipFence, + ); + if (!registrationExpiresAt) { + throw new Error("sandbox credential policy registration claim was revoked"); + } + } const repairExpiresAt = registrationExpiresAt - 1; - for (const lookupId of registration.lookupIds) { - if (records.get(lookupId)) continue; + for (const lookupId of missingLookupIds) { const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { method: "POST", body: JSON.stringify({ @@ -232,7 +245,7 @@ export async function restoreSandboxCredentialPolicyRollbackIfOwned( ownershipFence: SandboxCredentialPolicyOwnershipFence, restoreRollback: RestoreSandboxCredentialPolicyRollback = restoreSandboxCredentialPolicyRollback, ): Promise { - const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + const registrationExpiresAt = await markSandboxCredentialPolicyRegistrationWriteStarted( env, sessionId, sandboxId, @@ -327,7 +340,7 @@ export async function registerSandboxCredentialPolicy( ...(env.OPENAI_ORG_ID ? { openAIOrgId: env.OPENAI_ORG_ID } : {}), }; for (const lookupId of registration.lookupIds) { - const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + const registrationExpiresAt = await markSandboxCredentialPolicyRegistrationWriteStarted( env, session.id, sandboxId, diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 32163085..0ad5bf33 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -687,6 +687,36 @@ export async function renewSandboxCredentialPolicyRegistration( return Number(renewed.numUpdatedRows ?? 0n) === 1 ? registrationExpiresAt : null; } +export async function markSandboxCredentialPolicyRegistrationWriteStarted( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const now = Date.now(); + const registrationExpiresAt = now + credentialPolicyRegistrationClaimMs; + const marked = await sql<{ registration_claim_expires_at: number }>` + UPDATE interactive_session_credential_policy_registrations + SET + registration_claim_expires_at = ${registrationExpiresAt}, + registration_write_started = 1, + updated_at = ${now} + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND state = 'registering' + AND registration_generation = ${registration.generation} + AND registration_claim = ${registration.claim} + AND registration_claim_expires_at > ${now} + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + AND ${noLivePolicyTableRegistrationCondition(sessionId, sandboxId, now)} + RETURNING registration_claim_expires_at + `.execute(database(env)); + return marked.rows[0]?.registration_claim_expires_at === registrationExpiresAt + ? registrationExpiresAt + : null; +} + export async function claimSandboxCredentialPolicyRegistrationRecovery( env: RuntimeEnv, sessionId: string, diff --git a/src/worker/sandbox-credential-policy-scanner.ts b/src/worker/sandbox-credential-policy-scanner.ts index 44cdfe27..579bb77b 100644 --- a/src/worker/sandbox-credential-policy-scanner.ts +++ b/src/worker/sandbox-credential-policy-scanner.ts @@ -12,6 +12,7 @@ import { abandonSandboxCredentialPolicyRegistration, claimSandboxCredentialPolicyRegistrationRecovery, finishSandboxCredentialPolicyRegistration, + markSandboxCredentialPolicyRegistrationWriteStarted, recordSandboxCredentialPolicyRefs, sandboxCredentialPolicyCleanupAuthorizedCondition, sandboxCredentialPolicyPersistedLookupIds, @@ -425,6 +426,16 @@ async function scanStagedCredentialPolicyRegistrations( } if (row.rollback_policies_json !== null) { if (!restoreRollback) throw new Error("sandbox credential policy rollback is unavailable"); + const rollbackExpiresAt = await markSandboxCredentialPolicyRegistrationWriteStarted( + env, + row.session_id, + row.sandbox_id, + recovery.registration, + ownershipFence, + ); + if (!rollbackExpiresAt) { + throw new Error("sandbox credential policy registration claim was revoked"); + } const rollbackGeneration = parseSandboxCredentialPolicyRollback( row.rollback_policies_json, @@ -453,7 +464,7 @@ async function scanStagedCredentialPolicyRegistrations( ...recovery.registration, lookupIds: rollbackLookupIds, }, - registrationExpiresAt: recovery.registrationExpiresAt, + registrationExpiresAt: rollbackExpiresAt, rollbackJson: row.rollback_policies_json, sessionId: row.session_id, }); diff --git a/src/worker/session-cleanup.ts b/src/worker/session-cleanup.ts index 48f8e9e8..542926e4 100644 --- a/src/worker/session-cleanup.ts +++ b/src/worker/session-cleanup.ts @@ -13,13 +13,18 @@ import { cleanupSessionLogArchiveObjects } from "./session-log-archive.ts"; const terminalCleanupDeletePending = 2; type SessionReference = string | RawBuilder; -function hasNoCredentialPolicy(sessionId: SessionReference): RawBuilder { +export function hasNoCredentialPolicyLifecycle(sessionId: SessionReference): RawBuilder { return sql` NOT EXISTS ( SELECT 1 FROM interactive_session_credential_policies WHERE session_id = ${sessionId} ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policy_registrations + WHERE session_id = ${sessionId} + ) `; } @@ -140,7 +145,7 @@ export async function readInteractiveSessionCleanupCandidates( .selectAll() .where("status", "in", deadInteractiveSessionStatuses) .where("terminal_finalize_pending", "=", 0) - .where(hasNoCredentialPolicy(sql.ref("interactive_sessions.id"))) + .where(hasNoCredentialPolicyLifecycle(sql.ref("interactive_sessions.id"))) .where(archiveCoversAllEvents(sql.ref("interactive_sessions.id"))) .where(sql` EXISTS ( @@ -185,7 +190,7 @@ export async function deleteFinalizedInteractiveSession( .where("status", "=", row.status) .where("updated_at", "=", row.updated_at) .where("terminal_finalize_pending", "=", 0) - .where(hasNoCredentialPolicy(row.id)) + .where(hasNoCredentialPolicyLifecycle(row.id)) .where(hasNoActiveDescendants(row.id)) .where(sql` ${archive ? 1 : 0} = 1 diff --git a/src/worker/session-terminal-finalization.ts b/src/worker/session-terminal-finalization.ts index 3887ce8f..82609de8 100644 --- a/src/worker/session-terminal-finalization.ts +++ b/src/worker/session-terminal-finalization.ts @@ -1,4 +1,4 @@ -import { sql, type Kysely } from "kysely"; +import { sql, type Kysely, type RawBuilder } from "kysely"; import { retainedRuntimeAdapterFailureMessage } from "../runtime-adapter.ts"; import { completeTerminalFinalization } from "../terminal-finalization.ts"; @@ -6,6 +6,7 @@ import { database, executeBatch, type CompilableQuery, type Database } from "./d import type { RuntimeEnv } from "./env.ts"; import { deadInteractiveSessionStatuses } from "./models.ts"; import { archiveInteractiveSessionLogs } from "./session-log-archive.ts"; +import { hasNoCredentialPolicyLifecycle } from "./session-cleanup.ts"; import { countInteractiveSessionEvents } from "./session-repository.ts"; export type TerminalInteractiveSessionStatus = "stopped" | "expired" | "failed"; @@ -41,6 +42,50 @@ export function terminalInteractiveSessionFinalizationMessage( return status === "expired" ? "interactive workspace expired" : "interactive workspace stopped"; } +export function terminalFinalizationClearPendingQuery( + id: string, + status: TerminalInteractiveSessionStatus, + sessionLogsEnabled: boolean, +): RawBuilder { + return sql` + UPDATE interactive_sessions + SET terminal_finalize_pending = 0 + WHERE id = ${id} + AND status = ${status} + AND terminal_finalize_pending > 0 + AND EXISTS ( + SELECT 1 + FROM interactive_session_log_archives AS archive + WHERE archive.session_id = interactive_sessions.id + AND archive.session_updated_at = interactive_sessions.updated_at + ) + AND ${hasNoCredentialPolicyLifecycle(id)} + AND COALESCE( + ( + SELECT event_count + FROM interactive_session_log_archives + WHERE session_id = ${id} + ), + -1 + ) >= ( + SELECT count(*) + FROM interactive_session_events + WHERE session_id = ${id} + ) + AND ( + ${sessionLogsEnabled ? 1 : 0} = 0 + OR EXISTS ( + SELECT 1 + FROM interactive_session_log_archives + WHERE session_id = ${id} + AND events_key IS NOT NULL + AND transcript_key IS NOT NULL + AND summary_key IS NOT NULL + ) + ) + `; +} + export async function finalizeTerminalInteractiveSession( env: RuntimeEnv, id: string, @@ -105,47 +150,11 @@ export async function finalizeTerminalInteractiveSession( }, archive: () => archiveInteractiveSessionLogs(env, id, now, { force: true }), clearPending: async () => { - const cleared = await sql` - UPDATE interactive_sessions - SET terminal_finalize_pending = 0 - WHERE id = ${id} - AND status = ${status} - AND terminal_finalize_pending > 0 - AND EXISTS ( - SELECT 1 - FROM interactive_session_log_archives AS archive - WHERE archive.session_id = interactive_sessions.id - AND archive.session_updated_at = interactive_sessions.updated_at - ) - AND NOT EXISTS ( - SELECT 1 - FROM interactive_session_credential_policies - WHERE session_id = ${id} - ) - AND COALESCE( - ( - SELECT event_count - FROM interactive_session_log_archives - WHERE session_id = ${id} - ), - -1 - ) >= ( - SELECT count(*) - FROM interactive_session_events - WHERE session_id = ${id} - ) - AND ( - ${env.SESSION_LOGS ? 1 : 0} = 0 - OR EXISTS ( - SELECT 1 - FROM interactive_session_log_archives - WHERE session_id = ${id} - AND events_key IS NOT NULL - AND transcript_key IS NOT NULL - AND summary_key IS NOT NULL - ) - ) - `.execute(db); + const cleared = await terminalFinalizationClearPendingQuery( + id, + status, + Boolean(env.SESSION_LOGS), + ).execute(db); if ((cleared.numAffectedRows ?? 0n) > 0n) return true; const current = await db .selectFrom("interactive_sessions") diff --git a/tests/sandbox-credential-policy-cleanup.test.ts b/tests/sandbox-credential-policy-cleanup.test.ts index 2874c635..7b1d4d16 100644 --- a/tests/sandbox-credential-policy-cleanup.test.ts +++ b/tests/sandbox-credential-policy-cleanup.test.ts @@ -197,6 +197,12 @@ test("terminal cleanup atomically stages the session and credential-policy refs" ); assert.equal(batch.length, 4); + assert.match( + batch[1]?.sql ?? "", + /update "interactive_session_credential_policy_registrations"/i, + ); + assert.match(batch[2]?.sql ?? "", /interactive_session_credential_policies/i); + assert.match(batch[3]?.sql ?? "", /update "interactive_session_credential_policies"/i); const sql = batch.map((statement) => statement.sql).join("\n"); const parameters = batch.flatMap((statement) => statement.parameters); assert.match(sql, /update "interactive_sessions"/i); diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 7a5050aa..a3a6a9f2 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -12,6 +12,7 @@ import { currentSandboxCredentialPolicyGeneration, finishSandboxCredentialPolicyRegistration, incompleteSandboxCredentialPolicyGeneration, + markSandboxCredentialPolicyRegistrationWriteStarted, recordSandboxCredentialPolicyRefs, recordSandboxCredentialPolicyRollback, repairSandboxCredentialPolicyReferences, @@ -191,6 +192,15 @@ function credentialPolicyDatabase(options: { applyMigrations?: boolean } = {}): "utf8", ), ); + db.exec( + readFileSync( + new URL( + "../migrations/0040_credential_policy_registration_write_fence.sql", + import.meta.url, + ), + "utf8", + ), + ); } return db; } @@ -905,7 +915,7 @@ test("staged rotations fence old-worker writes until the staged row is removed", assert.equal(postFenceClaim.changes, 2); }); -test("expired staged rotations release the legacy worker compatibility fence", async () => { +test("expired pre-write staged rotations release the legacy worker compatibility fence", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); await beginSandboxCredentialPolicyRegistration(env, "IS-42", "sandbox-1", ownershipFence); @@ -939,6 +949,161 @@ test("expired staged rotations release the legacy worker compatibility fence", a ); }); +test("started staged rotations retain the legacy fence after claim expiry", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + assert.ok( + await markSandboxCredentialPolicyRegistrationWriteStarted( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + ); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = 0, updated_at = 0 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + + const legacyClaim = sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'registering', + registration_generation = 'generation:legacy-rollback', + registration_claim = 'legacy-rollback-claim', + registration_claim_expires_at = ?, + updated_at = 1 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(Number.MAX_SAFE_INTEGER); + const legacyCleanupInsert = sqlite + .prepare(` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at + ) VALUES ( + 'IS-42', + 'sandbox-1', + 'legacy-cleanup', + 'cleanup_pending', + 'generation:existing', + NULL, + NULL, + 1, + 1 + ) + `) + .run(); + const legacyDelete = sqlite + .prepare(` + DELETE FROM interactive_session_credential_policies + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + + assert.equal(legacyClaim.changes, 0); + assert.equal(legacyCleanupInsert.changes, 0); + assert.equal(legacyDelete.changes, 0); + assert.equal( + sqlite + .prepare(` + SELECT registration_write_started + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get()?.registration_write_started, + 1, + ); + assert.ok( + await claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + staged, + 0, + ownershipFence, + ), + ); +}); + +test("write-fence migration conservatively protects existing staged rotations", () => { + const sqlite = credentialPolicyDatabase({ applyMigrations: false }); + for (const migration of [ + "0034_credential_policy_registration_staging.sql", + "0035_credential_policy_registration_rollback.sql", + "0036_credential_policy_lookup_repair.sql", + "0037_credential_policy_registration_lookup_ids.sql", + ]) { + sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); + } + sqlite + .prepare(` + INSERT INTO interactive_session_credential_policy_registrations ( + session_id, + sandbox_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + lookup_ids_json, + created_at, + updated_at + ) VALUES (?, ?, 'registering', ?, ?, 0, ?, 1, 0) + `) + .run( + "IS-42", + "sandbox-1", + "generation:pre-write-fence", + "registration:pre-write-fence", + JSON.stringify(["sandbox-1", "do-1"]), + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0040_credential_policy_registration_write_fence.sql", import.meta.url), + "utf8", + ), + ); + + assert.equal( + sqlite + .prepare(` + SELECT registration_write_started + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get()?.registration_write_started, + 1, + ); + assert.equal( + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET registration_generation = 'generation:legacy-after-rollback' + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run().changes, + 0, + ); +}); + test("stale staged cleanup releases legacy deletion but an active cleanup claim stays fenced", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); diff --git a/tests/sandbox-credential-policy-scanner.test.ts b/tests/sandbox-credential-policy-scanner.test.ts index 643bc506..59bd7009 100644 --- a/tests/sandbox-credential-policy-scanner.test.ts +++ b/tests/sandbox-credential-policy-scanner.test.ts @@ -62,6 +62,7 @@ function scannerDatabase(): DatabaseSync { registration_generation TEXT NOT NULL, registration_claim TEXT, registration_claim_expires_at INTEGER, + registration_write_started INTEGER NOT NULL DEFAULT 0, lookup_ids_json TEXT, rollback_policies_json TEXT, last_error TEXT, diff --git a/tests/session-cleanup.test.ts b/tests/session-cleanup.test.ts index 770a0a11..8c9b9c51 100644 --- a/tests/session-cleanup.test.ts +++ b/tests/session-cleanup.test.ts @@ -118,6 +118,7 @@ test("cleanup candidates require finalized archives, no credentials, and no acti sessionReads += 1; assert.match(sql, /"terminal_finalize_pending" =/i); assert.match(sql, /interactive_session_credential_policies/i); + assert.match(sql, /interactive_session_credential_policy_registrations/i); assert.match(sql, /WITH RECURSIVE active_ancestor\(id\)/i); assert.match(sql, /archive\.session_updated_at = interactive_sessions\.updated_at/i); assert.match(sql, /events_key IS NOT NULL/i); @@ -163,6 +164,7 @@ test("finalized deletion claims and removes events, archive metadata, and sessio assert.equal(batch.length, 5); assert.match(batch[0]?.sql ?? "", /update "interactive_sessions"/i); assert.match(batch[0]?.sql ?? "", /interactive_session_credential_policies/i); + assert.match(batch[0]?.sql ?? "", /interactive_session_credential_policy_registrations/i); assert.match(batch[0]?.sql ?? "", /WITH RECURSIVE active_ancestor\(id\)/i); assert.match(batch[0]?.sql ?? "", /event_count =/i); assert.match(batch[0]?.sql ?? "", /events_key IS/i); diff --git a/tests/session-terminal-finalization.test.ts b/tests/session-terminal-finalization.test.ts index 9a76588f..0eca38a8 100644 --- a/tests/session-terminal-finalization.test.ts +++ b/tests/session-terminal-finalization.test.ts @@ -1,7 +1,12 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { terminalInteractiveSessionFinalizationMessage } from "../src/worker/session-terminal-finalization.ts"; +import { database } from "../src/worker/database.ts"; +import type { RuntimeEnv } from "../src/worker/env.ts"; +import { + terminalFinalizationClearPendingQuery, + terminalInteractiveSessionFinalizationMessage, +} from "../src/worker/session-terminal-finalization.ts"; test("terminal finalization messages preserve lifecycle and failure evidence", () => { assert.equal( @@ -29,3 +34,12 @@ test("terminal finalization messages preserve lifecycle and failure evidence", ( "interactive workspace failed after release", ); }); + +test("terminal finalization remains pending while any credential lifecycle row exists", () => { + const db = database({ DB: {} as D1Database } as RuntimeEnv); + const compiled = terminalFinalizationClearPendingQuery("IS-42", "stopped", true).compile(db); + + assert.match(compiled.sql, /interactive_session_credential_policies/i); + assert.match(compiled.sql, /interactive_session_credential_policy_registrations/i); + assert.match(compiled.sql, /NOT EXISTS/i); +}); From 44c6315aba12db1621d0a720d9560da7000d0a43 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:45:42 +0200 Subject: [PATCH 223/242] fix(vnc): validate ARD safe-prime groups --- .../ARDDiffieHellmanKeyAgreement.swift | 49 +++++++++++++-- .../SecurityAndInputTests.swift | 60 +++++++++++++++---- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift index fa6ddc0d..dce4a8b3 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift @@ -3,6 +3,7 @@ import FoundationEssentials #else import Foundation #endif +import CryptoSwift extension VNCProtocol.ARDAuthentication { struct DiffieHellmanKeyAgreement { @@ -50,6 +51,11 @@ extension VNCProtocol.ARDAuthentication { } private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { + static let safePrimeCondition = NSCondition() + static var validatedSafePrimes = Set() + static var rejectedSafePrimes = Set() + static var safePrimeValidations = Set() + struct KeyPair { let publicKey: Data let privateKey: Data @@ -64,17 +70,52 @@ private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { prime.count == keyLength, peerKey.count == keyLength, let bigPrime = BigNum(data: prime), - bigPrime.bitsCount >= 1_024, + bigPrime.bitsCount == keyLength * 8, let bigGenerator = BigNum(data: generator), bigGenerator.isValidDiffieHellmanElement(modulus: bigPrime), let bigPeerKey = BigNum(data: peerKey), - bigPeerKey.isValidDiffieHellmanElement(modulus: bigPrime) else { + bigPeerKey.isValidDiffieHellmanElement(modulus: bigPrime), + Self.isSafePrime(prime) else { return false } return true } + static func isSafePrime(_ data: Data) -> Bool { + safePrimeCondition.lock() + while safePrimeValidations.contains(data) { + safePrimeCondition.wait() + } + if validatedSafePrimes.contains(data) { + safePrimeCondition.unlock() + return true + } + if rejectedSafePrimes.contains(data) { + safePrimeCondition.unlock() + return false + } + safePrimeValidations.insert(data) + safePrimeCondition.unlock() + + let prime = CS.BigUInt(data) + let valid = prime.isPrime(rounds: 16) && ((prime - 1) >> 1).isPrime(rounds: 16) + + safePrimeCondition.lock() + safePrimeValidations.remove(data) + if valid { + validatedSafePrimes.insert(data) + } else { + if rejectedSafePrimes.count >= 32, let evicted = rejectedSafePrimes.first { + rejectedSafePrimes.remove(evicted) + } + rejectedSafePrimes.insert(data) + } + safePrimeCondition.broadcast() + safePrimeCondition.unlock() + return valid + } + static func generateKeyPair(generator: Data, prime: Data, keyLength: Int) -> KeyPair? { @@ -101,7 +142,7 @@ private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { x: bigPrivKey, p: bigPrime) - guard modSuccess else { + guard modSuccess, !bigPubKey.isZero else { return nil } @@ -139,7 +180,7 @@ private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { x: bigPrivKey, p: bigPrime) - guard modSuccess else { + guard modSuccess, !bigSharedKey.isZero else { return nil } diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift index ab31787a..dd8b0d3c 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -16,6 +16,8 @@ struct SecurityAndInputTests { Data(repeating: 0, count: 128), Data(repeating: 1, count: 128), Data([0]) + Data(repeating: 0xFF, count: 127), + Data([0x80]) + Data(repeating: 0, count: 127), + appleRemoteDesktopUnsafePrime, ] { let agreement = ARDKeyAgreement( prime: prime, @@ -30,8 +32,8 @@ struct SecurityAndInputTests { @Test func rejectsAppleRemoteDesktopElementsOutsideTheSafeRange() { - let prime = Data(repeating: 0xFF, count: 128) - let primeMinusOne = Data(repeating: 0xFF, count: 127) + Data([0xFE]) + let prime = appleRemoteDesktopSafePrime + let primeMinusOne = prime.dropLast() + Data([0xFE]) for generator in [ Data([0]), @@ -67,18 +69,18 @@ struct SecurityAndInputTests { } @Test - func acceptsAppleRemoteDesktopKeyMaterialAtAndAboveTheMinimum() { - for keyLength in [128, 256] { - let agreement = ARDKeyAgreement( - prime: Data(repeating: 0xFF, count: keyLength), - generator: Data([2]), - peerKey: Data(repeating: 0, count: keyLength - 1) + Data([2]), - keyLength: keyLength - ) + func acceptsSafeAppleRemoteDesktopKeyMaterial() { + let agreement = ARDKeyAgreement( + prime: appleRemoteDesktopSafePrime, + generator: Data([2]), + peerKey: paddedARDValue(2), + keyLength: 128 + ) - #expect(agreement?.publicKey.count == keyLength) - #expect(agreement?.secretKey.count == keyLength) - } + #expect(agreement?.publicKey.count == 128) + #expect(agreement?.publicKey.contains { $0 != 0 } == true) + #expect(agreement?.secretKey.count == 128) + #expect(agreement?.secretKey.contains { $0 != 0 } == true) } @Test @@ -196,6 +198,38 @@ struct SecurityAndInputTests { Data(repeating: 0, count: 127) + Data([value]) } + private var appleRemoteDesktopSafePrime: Data { + hexadecimalData( + """ + C692B0343A9FC77AB54DD8F0912F24E657BACB3D4272E6525E624DCBAB26A479 + 904118111CCE782B6709522BD201F15C38EDF1B3E94DEAA7DEE91B4B4619607B + 3B76E1A1F9B65F6F545D42982FEE07F1F78D5855E9C490CAD9B45855F6BDEA7 + 5BF549643A572571B9F8073EE56A36DD1B9EAD50DCF444406BFDFD851DE76E51B + """ + ) + } + + private var appleRemoteDesktopUnsafePrime: Data { + hexadecimalData( + """ + F1EEAEF06F42BDFEF9524C7A03A6B26F074DC39F74F8C160BD15BA3869F54450 + CE55FD8DA6415AF88CEF7FFE7768BB1A061B7A3C0BCE0023B2C15C0A095D416B + E103EB8EE3BE0EE5874ADFE2BF7270B8719CC8F99B38BFFC126D6005DBEABAB + EE0037C10BAFB4D9CC864259DA28E1F5ECB949DCAC308512F9FA3E911F1E36061 + """ + ) + } + + private func hexadecimalData(_ value: String) -> Data { + let hex = value.filter(\.isHexDigit) + return Data( + stride(from: 0, to: hex.count, by: 2).compactMap { offset in + let start = hex.index(hex.startIndex, offsetBy: offset) + let end = hex.index(start, offsetBy: 2) + return UInt8(hex[start.. Data { withUnsafeBytes(of: value.bigEndian) { Data($0) } } From 916a60c3537be2c8e355fabd820ef563818d80c5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:45:42 +0200 Subject: [PATCH 224/242] fix(macos): complete ambiguous desktop cleanup --- .../PrivateMacShareController.swift | 26 ++++--- .../PrivateMacShareTests.swift | 74 +++++++++++++++++++ 2 files changed, 88 insertions(+), 12 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index c6ed239f..9fa90d98 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -299,17 +299,15 @@ final class DesktopHostRegistrationLifecycle { identity: target.identity, publicationID: target.publicationID ) - if let ownershipToken { - let recovered = PublishedRegistration( - identity: target.identity, - hostID: target.hostID, - publicationID: target.publicationID, - ownershipToken: ownershipToken, - usesLegacyCleanup: false - ) - if !pendingRemovals.contains(recovered) { - pendingRemovals.append(recovered) - } + let recovered = PublishedRegistration( + identity: target.identity, + hostID: target.hostID, + publicationID: target.publicationID, + ownershipToken: ownershipToken, + usesLegacyCleanup: ownershipToken == nil + ) + if !pendingRemovals.contains(recovered) { + pendingRemovals.append(recovered) } self.uncertainRegistrations.removeAll { $0 == target } try persistState() @@ -323,7 +321,11 @@ final class DesktopHostRegistrationLifecycle { pendingRemovals.append(publishedRegistration) } self.publishedRegistration = nil - try persistState() + do { + try persistState() + } catch { + firstError = firstError ?? error + } } let removals = pendingRemovals diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 12518da9..0a9c5aa0 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -500,6 +500,21 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func ambiguousLegacyDesktopPublicationAttemptsGuardedCleanup() async throws { + let identity = desktopIdentity(name: "ambiguous-legacy", address: "100.64.12.62") + let registration = AmbiguousLegacyDesktopRegistration() + let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) + + await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { + try await lifecycle.publish(identity: identity, port: 5_901) + } + try await lifecycle.removePublishedIdentities() + + #expect(!(await registration.isPublished)) + #expect(await registration.events == [.register, .recover, .unregister]) + } + @Test @MainActor func ambiguousDesktopPublicationRetryRecoversOnlyTheExactIdentity() async throws { let identity = desktopIdentity(name: "retry-publish", address: "100.64.12.50") @@ -997,6 +1012,33 @@ struct PrivateMacShareTests { ) } + @Test @MainActor + func persistenceFailureDoesNotAbortDesktopUnregister() async throws { + let identity = desktopIdentity(name: "persist-failure", address: "100.64.12.63") + let registration = RecordingDesktopRegistration() + let stateStore = ToggleDesktopRegistrationStateStore() + let recoveryScope = desktopRecoveryScope() + let lifecycle = DesktopHostRegistrationLifecycle( + registration: registration, + stateStore: stateStore, + recoveryScopeProvider: { recoveryScope } + ) + try await lifecycle.publish(identity: identity, port: 5_901) + stateStore.failsWrites = true + + await #expect(throws: DesktopRegistrationTestError.failed) { + try await lifecycle.removePublishedIdentities() + } + + #expect( + await registration.events + == [ + .register(identity.dnsName), + .unregister(identity.dnsName, "token:\(identity.dnsName)"), + ] + ) + } + @Test @MainActor func loadingPersistedLegacyCleanupClearsUnsafeState() async throws { let identity = desktopIdentity(name: "persisted-legacy", address: "100.64.12.61") @@ -2652,6 +2694,38 @@ private actor AmbiguousDesktopRegistration: DesktopHostRegistering { } } +private actor AmbiguousLegacyDesktopRegistration: DesktopHostRegistering { + enum Event: Equatable { + case register + case recover + case unregister + } + + private(set) var isPublished = false + private(set) var events: [Event] = [] + + func register( + identity: TailnetIdentity, + port: UInt16, + publicationID: String + ) async throws -> String? { + events.append(.register) + isPublished = true + throw DesktopHostRegistrationResultUncertainError(message: "legacy response lost") + } + + func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { + events.append(.recover) + return nil + } + + func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { + #expect(ownershipToken == nil) + events.append(.unregister) + isPublished = false + } +} + private actor RecoverableAmbiguousDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register(String) From 1bc9a2fa910344234b3007f6766aee2c7ad5daf7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:45:42 +0200 Subject: [PATCH 225/242] fix(database): surface desktop ownership conflicts --- .../0041_desktop_host_ownership_errors.sql | 28 +++++++++++++ tests/desktop-host-migration.test.ts | 41 ++++++++++++++----- tests/desktop-host-repository.test.ts | 8 ++++ 3 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 migrations/0041_desktop_host_ownership_errors.sql diff --git a/migrations/0041_desktop_host_ownership_errors.sql b/migrations/0041_desktop_host_ownership_errors.sql new file mode 100644 index 00000000..590da19e --- /dev/null +++ b/migrations/0041_desktop_host_ownership_errors.sql @@ -0,0 +1,28 @@ +DROP TRIGGER IF EXISTS protect_token_owned_desktop_host_update; +DROP TRIGGER IF EXISTS protect_token_owned_desktop_host_delete; + +CREATE TRIGGER protect_token_owned_desktop_host_update +BEFORE UPDATE ON desktop_hosts +WHEN OLD.ownership_token <> '' + AND NEW.ownership_token = OLD.ownership_token + AND ( + NEW.owner_subject IS NOT OLD.owner_subject + OR NEW.id IS NOT OLD.id + OR NEW.owner IS NOT OLD.owner + OR NEW.name IS NOT OLD.name + OR NEW.address IS NOT OLD.address + OR NEW.port IS NOT OLD.port + OR NEW.created_at IS NOT OLD.created_at + OR NEW.updated_at IS NOT OLD.updated_at + ) +BEGIN + SELECT RAISE(ABORT, 'token-owned desktop host update requires ownership token'); +END; + +CREATE TRIGGER protect_token_owned_desktop_host_delete +BEFORE DELETE ON desktop_hosts +WHEN OLD.ownership_token <> '' + AND OLD.ownership_token NOT GLOB 'delete-authorized:*' +BEGIN + SELECT RAISE(ABORT, 'token-owned desktop host delete requires ownership token'); +END; diff --git a/tests/desktop-host-migration.test.ts b/tests/desktop-host-migration.test.ts index 2476ef9a..85aaae97 100644 --- a/tests/desktop-host-migration.test.ts +++ b/tests/desktop-host-migration.test.ts @@ -22,6 +22,12 @@ test("desktop host migration creates an owner-scoped registry with bounded ports "utf8", ), ); + database.exec( + readFileSync( + new URL("../migrations/0041_desktop_host_ownership_errors.sql", import.meta.url), + "utf8", + ), + ); const insert = database.prepare(` INSERT INTO desktop_hosts @@ -66,6 +72,7 @@ test("desktop host publication migration clears identities rotated by old worker "0030_desktop_hosts.sql", "0033_desktop_host_ownership.sql", "0038_desktop_host_publication_identity.sql", + "0041_desktop_host_ownership_errors.sql", ]) { database.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); } @@ -104,6 +111,12 @@ test("desktop host ownership migration blocks old-worker mutations of token-owne database.exec( readFileSync(new URL("../migrations/0033_desktop_host_ownership.sql", import.meta.url), "utf8"), ); + database.exec( + readFileSync( + new URL("../migrations/0041_desktop_host_ownership_errors.sql", import.meta.url), + "utf8", + ), + ); database.exec(` INSERT INTO desktop_hosts ( owner_subject, id, owner, name, address, port, ownership_token, created_at, updated_at @@ -123,15 +136,19 @@ test("desktop host ownership migration blocks old-worker mutations of token-owne port = excluded.port, updated_at = excluded.updated_at `); - oldWorkerUpsert.run( - "github:1", - "owned", - "old-worker", - "Overwritten", - "100.64.1.99", - 5902, - 10, - 20, + assert.throws( + () => + oldWorkerUpsert.run( + "github:1", + "owned", + "old-worker", + "Overwritten", + "100.64.1.99", + 5902, + 10, + 20, + ), + /token-owned desktop host update requires ownership token/, ); oldWorkerUpsert.run( "github:1", @@ -169,7 +186,11 @@ test("desktop host ownership migration blocks old-worker mutations of token-owne "Updated Legacy", ); - database.exec("DELETE FROM desktop_hosts WHERE owner_subject = 'github:1' AND id = 'owned'"); + assert.throws( + () => + database.exec("DELETE FROM desktop_hosts WHERE owner_subject = 'github:1' AND id = 'owned'"), + /token-owned desktop host delete requires ownership token/, + ); database.exec("DELETE FROM desktop_hosts WHERE owner_subject = 'github:1' AND id = 'legacy'"); assert.equal( database.prepare("SELECT count(*) AS count FROM desktop_hosts WHERE id = 'owned'").get()?.count, diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index dbe296a2..e9af8f86 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -288,6 +288,12 @@ test("legacy desktop host writes and cleanup cannot mutate token-owned rows", as "utf8", ), ); + sqlite.exec( + readFileSync( + new URL("../migrations/0041_desktop_host_ownership_errors.sql", import.meta.url), + "utf8", + ), + ); sqlite.exec(` INSERT INTO desktop_hosts ( owner_subject, id, owner, name, address, port, ownership_token, publication_id, @@ -342,6 +348,7 @@ test("desktop host publication recovery matches only the current publication", a "0030_desktop_hosts.sql", "0033_desktop_host_ownership.sql", "0038_desktop_host_publication_identity.sql", + "0041_desktop_host_ownership_errors.sql", ]) { sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); } @@ -376,6 +383,7 @@ test("same-publication retries remain recoverable after the publication migratio "0030_desktop_hosts.sql", "0033_desktop_host_ownership.sql", "0038_desktop_host_publication_identity.sql", + "0041_desktop_host_ownership_errors.sql", ]) { sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); } From 9f8234090bc52d65a0aa7929543acd8a1ca86be1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:46:03 +0200 Subject: [PATCH 226/242] docs(changelog): record final audit hardening --- CHANGELOG.md | 1 + docs/api.md | 4 +++- docs/github-actions-sessions.md | 5 +++++ docs/macos-native-client.md | 6 ++++-- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7e80bfc..180c41f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement and bounded PTY-write failure, making terminal confirmation races prefer completed delivery, retaining rollback recovery after a staged credential write begins, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material, surfacing legacy mutations of token-owned desktop registrations, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. - Close the final review blockers by requiring an exact live credential-policy lease at promotion, negotiating strict runtime-adapter deletion tombstones without breaking legacy `404` release semantics, disconnecting VNC sessions when pixel-format capability probes cannot establish a safe ZRLE boundary, classifying aborted JSON body streams as bad requests, and preserving live terminal subscriptions when browser history restores the sessions grid. - Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. - Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets. diff --git a/docs/api.md b/docs/api.md index 43c302eb..b49faf42 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1164,7 +1164,9 @@ Removes one registered desktop owned by the signed-in viewer. The route cannot remove another user's record with the same ID. Fenced registrations require the exact `X-Crabfleet-Ownership-Token` returned by `PUT`. Legacy clients may omit the header only to remove a registration whose stored ownership token is empty; -omission never removes a tokenized registration. +omission never removes a tokenized registration. Legacy writes or deletes that +target a tokenized registration fail explicitly instead of reporting success +without changing the row. ## Static Routes diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index 97a6b836..a7e20370 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -621,6 +621,11 @@ Properties: `input-rejected`, because that write may still complete. Legacy input reports acceptance after relay delivery. The unknown-delivery JSON control event carries `{"type":"input-delivery-unknown","error":"terminal input delivery outcome is unknown; the runner may still complete it"}`. +- Runner replacement also marks unresolved old-generation input as + `input-delivery-unknown`; a write that already entered the old PTY cannot be + proven absent. A runner-side PTY write that exceeds the bounded write deadline + retires that runner socket, so queued frames cannot execute behind a wedged + write. - Framed viewer lifecycle events remain typed binary frames while no runner is connected. Legacy viewers receive the JSON fallback. - When runner and viewer modes differ, the relay wraps or unwraps terminal diff --git a/docs/macos-native-client.md b/docs/macos-native-client.md index 2bea01f6..b3d5b070 100644 --- a/docs/macos-native-client.md +++ b/docs/macos-native-client.md @@ -161,8 +161,10 @@ from the build, or obtain written provenance approval. bound to loopback behind an authenticated SSH tunnel; Share This Mac is the identity-gated tailnet exception. - The hardened prototype negotiates standard VNC password or no-auth security - only. ARD Diffie-Hellman, UltraVNC MS Logon II, Tight security, and TLS remain - disabled until their parsers and cryptography are replaced or fully tested. + only. The bundled ARD Diffie-Hellman path now requires a full-width + probabilistic safe-prime group and nonzero public/shared results, but ARD, + UltraVNC MS Logon II, Tight security, and TLS remain disabled in the app until + their complete interoperability surfaces are enabled and tested. - Password authentication uses a process-global DES key schedule. The fork serializes that path; replace it before concurrent password-auth sessions. - App-owned hosting shares one selected display at a time to a single client. From 17447abf7753ad7c15b2ec22120fab702499fe3c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:57:07 +0200 Subject: [PATCH 227/242] fix(credentials): permit staged namespace retirement --- ...ential_policy_registration_write_fence.sql | 24 +++++++++++++++--- ...ndbox-credential-policy-repository.test.ts | 25 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/migrations/0040_credential_policy_registration_write_fence.sql b/migrations/0040_credential_policy_registration_write_fence.sql index 1f2e0715..694384f1 100644 --- a/migrations/0040_credential_policy_registration_write_fence.sql +++ b/migrations/0040_credential_policy_registration_write_fence.sql @@ -72,9 +72,27 @@ WHEN EXISTS ( ( staged.registration_write_started = 1 AND NOT ( - staged.state = 'cleanup_pending' - AND OLD.state != 'cleanup_pending' - AND NEW.state = 'cleanup_pending' + ( + staged.state = 'cleanup_pending' + AND OLD.state != 'cleanup_pending' + AND NEW.state = 'cleanup_pending' + ) + OR ( + staged.state = 'registering' + AND staged.repair_generation = OLD.registration_generation + AND NEW.registration_generation = OLD.registration_generation + AND staged.registration_claim = NEW.cleanup_claim + AND staged.registration_claim_expires_at = NEW.cleanup_claim_expires_at + AND OLD.state = 'active' + AND NEW.state = 'cleanup_pending' + AND json_valid(staged.lookup_ids_json) + AND NOT EXISTS ( + SELECT 1 + FROM json_each(staged.lookup_ids_json) AS current_lookup + WHERE current_lookup.type = 'text' + AND current_lookup.value = OLD.lookup_id + ) + ) ) ) OR ( diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index a3a6a9f2..9e93f91c 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -2046,6 +2046,14 @@ test("credential refresh replaces an obsolete durable namespace without losing r ), true, ); + registrationExpiresAt = await markSandboxCredentialPolicyRegistrationWriteStarted( + env, + "IS-42", + "sandbox-1", + registration, + ownershipFence, + ); + assert.ok(registrationExpiresAt); assert.equal( ( await stub.fetch("https://crabfleet.internal/api/session-control/register", { @@ -2088,6 +2096,23 @@ test("credential refresh replaces an obsolete durable namespace without losing r ownershipFence, ); assert.ok(registrationExpiresAt); + assert.equal( + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET + state = 'cleanup_pending', + cleanup_claim = 'registration:wrong', + cleanup_claim_expires_at = ?, + updated_at = ? + WHERE session_id = 'IS-42' + AND sandbox_id = 'sandbox-1' + AND lookup_id = 'do-old' + AND state = 'active' + `) + .run(registrationExpiresAt, Date.now()).changes, + 0, + ); assert.deepEqual( await claimObsoleteSandboxCredentialPolicyReferences( env, From 4da3119906527fb048ab8a87a5cfec4dc99bc000 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:57:36 +0200 Subject: [PATCH 228/242] test(credentials): prove interrupted upgrade recovery --- ...ndbox-credential-policy-repository.test.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 9e93f91c..4597221e 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -1104,6 +1104,82 @@ test("write-fence migration conservatively protects existing staged rotations", ); }); +test("write-fence migration recovers a crashed namespace repair", async () => { + const sqlite = credentialPolicyDatabase({ applyMigrations: false }); + for (const migration of [ + "0034_credential_policy_registration_staging.sql", + "0035_credential_policy_registration_rollback.sql", + "0036_credential_policy_lookup_repair.sql", + "0037_credential_policy_registration_lookup_ids.sql", + ]) { + sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); + } + sqlite + .prepare(` + UPDATE interactive_session_credential_policies + SET lookup_id = 'do-old' + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' AND lookup_id = 'do-1' + `) + .run(); + const expiredRegistration = { + generation: "generation:interrupted", + claim: "registration:interrupted", + lookupIds: ["sandbox-1", "do-1"], + }; + sqlite + .prepare(` + INSERT INTO interactive_session_credential_policy_registrations ( + session_id, + sandbox_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + lookup_ids_json, + repair_generation, + created_at, + updated_at + ) VALUES (?, ?, 'registering', ?, ?, 0, ?, 'generation:existing', 1, 0) + `) + .run( + "IS-42", + "sandbox-1", + expiredRegistration.generation, + expiredRegistration.claim, + JSON.stringify(expiredRegistration.lookupIds), + ); + sqlite.exec( + readFileSync( + new URL("../migrations/0040_credential_policy_registration_write_fence.sql", import.meta.url), + "utf8", + ), + ); + + const env = sqliteRuntimeEnv(sqlite); + const recovered = await claimSandboxCredentialPolicyRegistrationRecovery( + env, + "IS-42", + "sandbox-1", + expiredRegistration, + 0, + ownershipFence, + ); + assert.ok(recovered); + assert.deepEqual( + await claimObsoleteSandboxCredentialPolicyReferences( + env, + "IS-42", + "sandbox-1", + recovered.registration, + "generation:existing", + ["do-old"], + ownershipFence, + recovered.registrationExpiresAt, + ), + ["do-old"], + ); +}); + test("stale staged cleanup releases legacy deletion but an active cleanup claim stays fenced", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); From 8569b6f151d0cecb05ba4529a4160a7942372aa2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:00:08 +0200 Subject: [PATCH 229/242] fix(terminal): preserve attachment handoff state --- internal/terminalws/client.go | 108 ++++++++++++++--- internal/terminalws/client_test.go | 185 +++++++++++++++++++++++++++++ 2 files changed, 276 insertions(+), 17 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index f906dddd..945a57a0 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -99,6 +99,8 @@ type Client struct { inputWaiter chan error attachment *terminalAttachment attachmentReady chan struct{} + controlGrantGeneration uint64 + handledControlGrant uint64 terminalErr error readerDone chan struct{} readerErr error @@ -111,13 +113,15 @@ type frame struct { } type terminalAttachment struct { - frames chan attachmentDelivery - done chan struct{} + frames chan attachmentDelivery + done chan struct{} + controlGrantGeneration uint64 } type attachmentDelivery struct { - frame frame - accepted chan bool + frame frame + accepted chan bool + controlGrantGeneration uint64 } type eventPayload struct { @@ -379,7 +383,9 @@ func (c *Client) closeNow() { if c.readCancel != nil { c.readCancel() } - _ = c.conn.CloseNow() + if c.conn != nil { + _ = c.conn.CloseNow() + } if c.cancel != nil { c.cancel() } @@ -499,6 +505,12 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c go func() { defer wg.Done() defer close(frameConsumerDone) + if generation := attachment.controlGrantGeneration; generation > 0 { + if err := c.resendRememberedSize(ctx, generation); err != nil { + errCh <- err + return + } + } for { select { case <-ctx.Done(): @@ -514,6 +526,7 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c switch current.messageType { case messageOutput: if _, err := terminal.Write(current.payload); err != nil { + c.retireConnection(err) errCh <- err return } @@ -535,15 +548,12 @@ func (c *Client) Attach(ctx context.Context, terminal io.ReadWriter, resizes <-c errCh <- frameError(current, "terminal connection failed") return case messageControlGranted: - if size := c.rememberedSize(); size.Cols > 0 && size.Rows > 0 { - if err := c.write(ctx, frame{ - messageType: messageResize, - sessionID: c.sessionID, - payload: resizePayload(size), - }); err != nil { - errCh <- err - return - } + if err := c.resendRememberedSize( + ctx, + delivery.controlGrantGeneration, + ); err != nil { + errCh <- err + return } case messageEvent: var event eventPayload @@ -596,6 +606,25 @@ func (c *Client) rememberedSize() Size { return Size{Cols: uint32(value >> 32), Rows: uint32(value)} } +func (c *Client) resendRememberedSize(ctx context.Context, generation uint64) error { + size := c.rememberedSize() + if size.Cols > 0 && size.Rows > 0 { + if err := c.write(ctx, frame{ + messageType: messageResize, + sessionID: c.sessionID, + payload: resizePayload(size), + }); err != nil { + return err + } + } + c.stateMu.Lock() + if generation > c.handledControlGrant { + c.handledControlGrant = generation + } + c.stateMu.Unlock() + return nil +} + func (c *Client) write(ctx context.Context, current frame) error { c.writeMu.Lock() defer c.writeMu.Unlock() @@ -640,10 +669,20 @@ func (c *Client) handleFrame(ctx context.Context, current frame) error { return err case messageControlRevoked: c.canInput.Store(false) + c.stateMu.Lock() + c.handledControlGrant = c.controlGrantGeneration + c.stateMu.Unlock() c.deliverAttachment(ctx, current) case messageControlGranted: c.canInput.Store(true) - c.deliverAttachment(ctx, current) + c.stateMu.Lock() + c.controlGrantGeneration++ + generation := c.controlGrantGeneration + attachment := c.attachment + c.stateMu.Unlock() + if attachment != nil { + c.deliverControlGranted(ctx, attachment, current, generation) + } case messageEvent: var event eventPayload if err := json.Unmarshal(current.payload, &event); err != nil { @@ -729,6 +768,9 @@ func (c *Client) registerAttachment() (*terminalAttachment, error) { frames: make(chan attachmentDelivery), done: make(chan struct{}), } + if c.handledControlGrant < c.controlGrantGeneration { + attachment.controlGrantGeneration = c.controlGrantGeneration + } c.attachment = attachment if c.attachmentReady != nil { close(c.attachmentReady) @@ -759,6 +801,11 @@ func (c *Client) markTerminalClosed(err error) { c.stateMu.Unlock() } +func (c *Client) retireConnection(err error) { + c.markTerminalClosed(err) + c.closeNow() +} + func (c *Client) deliverOrQueueOutput(ctx context.Context, current frame) error { for { c.stateMu.Lock() @@ -807,10 +854,34 @@ func (c *Client) deliverToAttachment( ctx context.Context, attachment *terminalAttachment, current frame, +) bool { + return c.deliverToAttachmentWithControlGrant(ctx, attachment, current, 0) +} + +func (c *Client) deliverControlGranted( + ctx context.Context, + attachment *terminalAttachment, + current frame, + generation uint64, +) bool { + return c.deliverToAttachmentWithControlGrant( + ctx, + attachment, + current, + generation, + ) +} + +func (c *Client) deliverToAttachmentWithControlGrant( + ctx context.Context, + attachment *terminalAttachment, + current frame, + generation uint64, ) bool { delivery := attachmentDelivery{ - frame: current, - accepted: make(chan bool, 1), + frame: current, + accepted: make(chan bool, 1), + controlGrantGeneration: generation, } select { case attachment.frames <- delivery: @@ -841,6 +912,9 @@ func (c *Client) acceptAttachmentDelivery( func (c *Client) finishReader(err error) { c.stateMu.Lock() c.readerErr = normalizeCloseError(err) + if c.terminalErr != nil { + c.readerErr = c.terminalErr + } waiter := c.inputWaiter c.inputWaiter = nil if waiter != nil { diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 74256103..5fa03c7b 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -1957,6 +1957,94 @@ func TestAttachBoundsBlockedFrameConsumerShutdown(t *testing.T) { close(terminal.releaseRead) } +func TestLateRetiredAttachmentWriteFailureClosesConnection(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-late-write-failure", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageOutput, + sessionID: "IS-late-write-failure", + payload: []byte("blocked output\n"), + })); err != nil { + t.Error(err) + return + } + <-r.Context().Done() + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-late-write-failure", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + client.attachmentShutdownTimeout = 10 * time.Millisecond + + oldTerminal := newUncancelableReadBlockingWriteTerminal() + oldCtx, oldCancel := context.WithCancel(context.Background()) + oldDone := make(chan error, 1) + go func() { + oldDone <- client.Attach(oldCtx, oldTerminal, nil) + }() + <-oldTerminal.readStarted + <-oldTerminal.writeStarted + oldCancel() + if err := <-oldDone; !errors.Is(err, context.Canceled) { + t.Fatalf("old attachment error = %v", err) + } + + replacement := newBlockingTerminal() + replacementDone := make(chan error, 1) + go func() { + replacementDone <- client.Attach(context.Background(), replacement, nil) + }() + <-replacement.started + + close(oldTerminal.releaseWrite) + select { + case <-oldTerminal.writeDone: + case <-time.After(time.Second): + t.Fatal("retired attachment write did not finish") + } + select { + case err := <-replacementDone: + if !errors.Is(err, errBlockedTerminalWrite) { + t.Fatalf("replacement attachment error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("late write failure did not retire the replacement attachment") + } + select { + case <-client.readerDone: + case <-time.After(time.Second): + t.Fatal("late write failure did not close the connection") + } + close(oldTerminal.releaseRead) +} + func TestRetiredBlockedAttachmentAcknowledgementKeepsReplacementConnectionOpen(t *testing.T) { firstAcknowledged := make(chan struct{}) secondAcknowledged := make(chan struct{}) @@ -2328,6 +2416,103 @@ func TestClientContinuesReadOnlyAndResumesControl(t *testing.T) { } } +func TestClientReplaysControlGrantedToNextAttachment(t *testing.T) { + grantControl := make(chan struct{}) + resumedSize := make(chan Size, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: false}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-between-attachments", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + <-grantControl + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageControlGranted, + sessionID: "IS-between-attachments", + })); err != nil { + t.Error(err) + return + } + _, payload, err := conn.Read(r.Context()) + if err != nil { + t.Error(err) + return + } + resize, err := decodeFrame(payload) + if err != nil || resize.messageType != messageResize { + t.Errorf("resumed resize = %#v, %v", resize, err) + return + } + resumedSize <- Size{ + Cols: binary.LittleEndian.Uint32(resize.payload[0:4]), + Rows: binary.LittleEndian.Uint32(resize.payload[4:8]), + } + closed, _ := json.Marshal(eventPayload{Type: "closed"}) + _ = conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-between-attachments", + payload: closed, + })) + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-between-attachments", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + if err := client.Attach( + context.Background(), + &readWriter{reader: bytes.NewReader(nil)}, + nil, + ); err != nil { + t.Fatal(err) + } + expectedSize := Size{Cols: 132, Rows: 43} + if err := client.Resize(context.Background(), expectedSize); err != nil { + t.Fatal(err) + } + close(grantControl) + deadline := time.Now().Add(time.Second) + for !client.canInput.Load() { + if time.Now().After(deadline) { + t.Fatal("control grant was not received") + } + time.Sleep(time.Millisecond) + } + + terminal := newBlockingTerminal() + attachCtx, attachCancel := context.WithTimeout(context.Background(), time.Second) + defer attachCancel() + if err := client.Attach(attachCtx, terminal, nil); err != nil { + t.Fatal(err) + } + if size := <-resumedSize; size != expectedSize { + t.Fatalf("resumed size = %#v", size) + } +} + func readOutputAcknowledgement( ctx context.Context, conn *websocket.Conn, From 6bf5b198f2617f820fc9e302c05e740509eb6f0a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:00:34 +0200 Subject: [PATCH 230/242] fix(vnc): defer ZRLE reset for partial fences --- .../SDK/Connection/VNCConnection+API.swift | 52 +++++++++-- .../RoyalVNCKitTests/AuditFindingsTests.swift | 89 +++++++++++++++++++ 2 files changed, 136 insertions(+), 5 deletions(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift index a1e0e0b0..5a25e3a4 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/SDK/Connection/VNCConnection+API.swift @@ -58,10 +58,13 @@ private struct PixelFormatTransition { private struct FenceCapabilityProbeMessage: VNCSendableMessage { let fenceMessage: VNCProtocol.ClientFence let pixelFormatMessage: VNCProtocol.SetPixelFormat + let completionFenceMessage: VNCProtocol.ClientFence let didSend: () -> Void var messageType: UInt8 { fenceMessage.messageType } - var data: Data { fenceMessage.data + pixelFormatMessage.data } + var data: Data { + fenceMessage.data + pixelFormatMessage.data + completionFenceMessage.data + } func send(connection: NetworkConnectionWriting) async throws { try await connection.write(data: data) @@ -69,6 +72,11 @@ private struct FenceCapabilityProbeMessage: VNCSendableMessage { } } +private enum PixelFormatFenceProbePayload { + static let capability = Data("royalvnc-pixel-format".utf8) + static let completion = Data("royalvnc-pixel-format-complete".utf8) +} + // MARK: - Connect/Disconnect public extension VNCConnection { #if canImport(ObjectiveC) @@ -335,7 +343,7 @@ extension VNCConnection { cancelPixelFormatFenceNegotiationTimeoutLocked() if pixelFormatFenceCapabilityProbePayload == nil, state.pixelFormat != nil { - pixelFormatFenceCapabilityProbePayload = Data("royalvnc-pixel-format".utf8) + pixelFormatFenceCapabilityProbePayload = PixelFormatFenceProbePayload.capability } state.areFencesSupported = true return true @@ -359,6 +367,10 @@ extension VNCConnection { payload: payload ), pixelFormatMessage: VNCProtocol.SetPixelFormat(pixelFormat: pixelFormat), + completionFenceMessage: VNCProtocol.ClientFence( + flags: [.request, .blockBefore], + payload: PixelFormatFenceProbePayload.completion + ), didSend: { [weak self] in self?.didSendPixelFormatFenceCapabilityProbe(payload: payload) } @@ -439,14 +451,44 @@ extension VNCConnection { throw VNCError.protocol(.invalidData) } cancelPixelFormatFenceNegotiationTimeoutLocked() - pixelFormatFenceCapabilityProbePayload = nil - expiredPixelFormatFenceCapabilityProbePayload = nil - pixelFormatFenceCapabilityProbeSequence = nil + if fence.payload == PixelFormatFenceProbePayload.completion { + guard fence.flags.contains(.blockBefore) else { + framebufferRequestLock.unlock() + throw VNCError.protocol(.invalidData) + } + pixelFormatFenceCapabilityProbePayload = nil + expiredPixelFormatFenceCapabilityProbePayload = nil + pixelFormatFenceCapabilityProbeSequence = nil + let transition = takePendingPixelFormatTransitionLocked() + framebufferRequestLock.unlock() + + try resetZRLECompressionState(for: sequence) + if let transition { + enqueuePixelFormatTransition(transition) + } + return + } state.pixelFormatTransitionFenceFlags = fence.flags.intersection([ .blockBefore, .blockAfter, .syncNext ]) + guard state.pixelFormatTransitionFenceFlags.contains(.syncNext) else { + let requiredFlags: VNCProtocol.FenceFlags = [.blockBefore, .blockAfter] + guard state.pixelFormatTransitionFenceFlags.intersection(requiredFlags) + == requiredFlags else { + framebufferRequestLock.unlock() + throw VNCError.protocol(.invalidData) + } + pixelFormatFenceCapabilityProbePayload = PixelFormatFenceProbePayload.completion + expiredPixelFormatFenceCapabilityProbePayload = nil + schedulePixelFormatFenceNegotiationTimeoutLocked() + framebufferRequestLock.unlock() + return + } + pixelFormatFenceCapabilityProbePayload = nil + expiredPixelFormatFenceCapabilityProbePayload = nil + pixelFormatFenceCapabilityProbeSequence = nil let transition = takePendingPixelFormatTransitionLocked() framebufferRequestLock.unlock() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift index 414bf60d..b577e7fa 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/AuditFindingsTests.swift @@ -593,6 +593,85 @@ struct AuditFindingsTests { connection.cancelFramebufferUpdateScheduling() } + @Test + func defersZRLEResetUntilTrailingFenceWhenSyncNextIsUnsupported() async throws { + let connection = VNCConnection( + settings: makeSettings(frameEncodings: [.zrle, .raw]), + framebufferAllocator: VNCFramebufferMallocAllocator() + ) + let framebuffer = try makeFramebuffer(width: 2, height: 2, depth: 24) + connection.framebuffer = framebuffer + connection.state.pixelFormat = framebuffer.sourcePixelFormat + connection.connectionState = .connected + connection._framebufferUpdatePolicy = .paused + let zrle = try #require( + connection.encodings[VNCFrameEncodingType.zrle.rawValue] as? VNCProtocol.ZRLEEncoding + ) + + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.request, .blockBefore, .blockAfter], + payload: Data("support".utf8) + ) + ) + _ = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityProbe = try #require(connection.clientToServerMessageQueue.dequeue()) + let capabilityWriter = AuditWritingConnection() + try await capabilityProbe.message.send(connection: capabilityWriter) + + let compressedChunks = continuousZlibChunks() + let first = try zrle.zStream.decompressedData( + compressedData: compressedChunks[0], + uncompressedSize: 1_000 + ) + #expect(first == Data(repeating: 0x41, count: 1_000)) + let second = try zrle.zStream.decompressedData( + compressedData: compressedChunks[1], + uncompressedSize: 1_000 + ) + #expect(second == Data(repeating: 0x42, count: 1_000)) + + let capabilityLength = Int(capabilityWriter.data[8]) + let capabilityPayload = Data(capabilityWriter.data[9..<(9 + capabilityLength)]) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore, .blockAfter], + payload: capabilityPayload + ) + ) + + let continuedAfterPartialResponse = try zrle.zStream.decompressedData( + compressedData: compressedChunks[2], + uncompressedSize: 1_000 + ) + #expect(continuedAfterPartialResponse == Data(repeating: 0x43, count: 1_000)) + + let trailingFenceOffset = 9 + capabilityLength + 20 + #expect(capabilityWriter.data[trailingFenceOffset] == VNCProtocol.ClientFence.messageType) + #expect( + capabilityWriter.data[(trailingFenceOffset + 4)..<(trailingFenceOffset + 8)] + == Data([0x80, 0, 0, 1]) + ) + let trailingPayloadLength = Int(capabilityWriter.data[trailingFenceOffset + 8]) + let trailingPayload = Data( + capabilityWriter.data[ + (trailingFenceOffset + 9)..<(trailingFenceOffset + 9 + trailingPayloadLength) + ] + ) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore], + payload: trailingPayload + ) + ) + + try verifyFreshZRLEStream(zrle.zStream, byte: 0x44) + connection.cancelFramebufferUpdateScheduling() + } + @Test func ignoresStaleZRLEResetFromLateCapabilityProbeResponse() async throws { let connection = VNCConnection( @@ -816,6 +895,16 @@ struct AuditFindingsTests { ) connection.completeFramebufferUpdateRequest() + #expect(connection.clientToServerMessageQueue.dequeue() == nil) + let trailingPayload = try #require(connection.pixelFormatFenceCapabilityProbePayload) + try connection.handleServerFence( + VNCProtocol.ServerFence( + messageType: VNCProtocol.ServerFence.messageType, + flags: [.blockBefore], + payload: trailingPayload + ) + ) + let transition = try #require(connection.clientToServerMessageQueue.dequeue()) let transitionWriter = AuditWritingConnection { #expect(connection.state.pixelFormat?.depth == 8) From 92c0e9368c559f71edea3230b109868518e4c5e9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:01:21 +0200 Subject: [PATCH 231/242] docs(changelog): record final review fixes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 180c41f0..42327edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement and bounded PTY-write failure, making terminal confirmation races prefer completed delivery, retaining rollback recovery after a staged credential write begins, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material, surfacing legacy mutations of token-owned desktop registrations, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. +- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement and bounded PTY-write failure, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material, deferring partial-fence ZRLE resets to a trailing synchronization boundary, surfacing legacy mutations of token-owned desktop registrations, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. - Close the final review blockers by requiring an exact live credential-policy lease at promotion, negotiating strict runtime-adapter deletion tombstones without breaking legacy `404` release semantics, disconnecting VNC sessions when pixel-format capability probes cannot establish a safe ZRLE boundary, classifying aborted JSON body streams as bad requests, and preserving live terminal subscriptions when browser history restores the sessions grid. - Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. - Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets. From 414eb463d5965d49c5fa6e8a91a325d9ac423c79 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:14:02 +0200 Subject: [PATCH 232/242] docs(api): clarify publication id contract --- docs/api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index b49faf42..651cb784 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1131,8 +1131,8 @@ address, port, and timestamp while preserving its creation time. Clients opt into fenced registration by sending `X-Crabfleet-Ownership-Mode: token-v1` and a stable `X-Crabfleet-Publication-ID`. The publication ID identifies one client's -attempt across retries and restarts; it must satisfy the same 1-80 character -identifier rules as the host ID. The response includes an `ownershipToken` +attempt across retries and restarts; it is an opaque 1-200 byte value that +cannot contain whitespace or control characters. The response includes an `ownershipToken` required for deletion. Omitting the ownership-mode header preserves the legacy `{ "host": ... }` response and stores a tokenless registration so older clients can still clean up during rolling upgrades. Current clients tolerate a legacy From 21f6e8f2c42f9caf13405c85cf4bed993b38404f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:14:47 +0200 Subject: [PATCH 233/242] fix(terminal): preserve disconnect delivery ambiguity --- src/worker/terminal-hub.ts | 8 +++- tests/terminal-hub.test.ts | 84 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 36cbf88c..1c68607f 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -661,13 +661,17 @@ export class TerminalHub { (pending) => pending.runnerGeneration === relayEvent.generation, { accepted: false, - error: "GitHub Actions runner disconnected before accepting input", + deliveryUnknown: true, + error: + "terminal input delivery outcome is unknown; the runner may still complete it", }, ); } else { completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, - error: "GitHub Actions runner disconnected before accepting input", + deliveryUnknown: true, + error: + "terminal input delivery outcome is unknown; the runner may still complete it", }); } } else if (relayEvent.type === "runner_connected") { diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index b686cae8..260f85bf 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -1453,7 +1453,7 @@ test("GitHub Actions close rejects every pending input acknowledgement", async ( server.emit("close"); }); -test("GitHub Actions runner disconnect rejects pending input without closing the viewer relay", async () => { +test("GitHub Actions runner disconnect reports pending input as delivery unknown", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -1501,9 +1501,9 @@ test("GitHub Actions runner disconnect rejects pending input without closing the assert.equal( disconnectEvents.some( (event) => - (event as { type?: string }).type === "input-rejected" && + (event as { type?: string }).type === "input-delivery-unknown" && (event as { error?: string }).error === - "GitHub Actions runner disconnected before accepting input", + "terminal input delivery outcome is unknown; the runner may still complete it", ), true, JSON.stringify(disconnectEvents), @@ -1511,6 +1511,84 @@ test("GitHub Actions runner disconnect rejects pending input without closing the server.emit("close"); }); +test("generation-fenced runner disconnect marks only matching input as delivery unknown", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + async openUpstream() { + return { + socket: upstream, + inputAcknowledgements: true, + inputGenerations: true, + initialRunnerGeneration: "generation-current", + outputAcknowledgements: false, + async markConnected() {}, + }; + }, + }), + ); + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, + ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("possibly delivered"), + }), + }); + await flushQueues(); + assert.equal(relayInput(upstream.sent.at(-1)!).generation, "generation-current"); + + emitRelayEvent(upstream, "runner_disconnected", "generation-stale"); + await flushQueues(); + await flushQueues(); + assert.equal( + server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string }) + .some((message) => message.type?.startsWith("input-")), + false, + ); + + emitRelayEvent(upstream, "runner_disconnected", "generation-current"); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type?.startsWith("input-")); + assert.deepEqual(completions, [ + { + type: "input-delivery-unknown", + error: "terminal input delivery outcome is unknown; the runner may still complete it", + }, + ]); + assert.deepEqual(upstream.closed, []); + server.emit("close"); +}); + test("GitHub Actions runner replacement reports old input as unknown and accepts new input", async () => { const client = socket(); const server = socket(); From b1a358f7b2790ac6ccbb114bee59669fb1df0047 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:15:13 +0200 Subject: [PATCH 234/242] fix(terminal): mark post-write input ambiguity --- internal/terminalws/client.go | 36 +++++++++++++-- internal/terminalws/client_test.go | 74 ++++++++++++++++++++++++++++-- 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/internal/terminalws/client.go b/internal/terminalws/client.go index 945a57a0..afe79210 100644 --- a/internal/terminalws/client.go +++ b/internal/terminalws/client.go @@ -124,6 +124,18 @@ type attachmentDelivery struct { controlGrantGeneration uint64 } +type inputConfirmationInterruptedError struct { + cause error +} + +func (e *inputConfirmationInterruptedError) Error() string { + return e.cause.Error() +} + +func (e *inputConfirmationInterruptedError) Unwrap() error { + return e.cause +} + type eventPayload struct { Type string `json:"type"` Error string `json:"error"` @@ -341,7 +353,12 @@ func (c *Client) SendInputConfirmed(ctx context.Context, payload []byte) error { c.clearInputWaiter(waiter) return err } - return c.waitForInputConfirmation(ctx, waiter) + err := c.waitForInputConfirmation(ctx, waiter) + var interrupted *inputConfirmationInterruptedError + if errors.As(err, &interrupted) { + return inputDeliveryUnknownCause(interrupted.cause) + } + return err } func (c *Client) drainInputConfirmation(waiter chan error) { @@ -364,7 +381,9 @@ func (c *Client) waitForInputConfirmation(ctx context.Context, waiter chan error if !c.clearInputWaiter(waiter) { return <-waiter } - return readerUnavailableError(c.readerError()) + return &inputConfirmationInterruptedError{ + cause: readerUnavailableError(c.readerError()), + } case <-ctx.Done(): select { case err := <-waiter: @@ -375,7 +394,7 @@ func (c *Client) waitForInputConfirmation(ctx context.Context, waiter chan error return <-waiter } c.closeNow() - return ctx.Err() + return &inputConfirmationInterruptedError{cause: ctx.Err()} } } @@ -918,7 +937,9 @@ func (c *Client) finishReader(err error) { waiter := c.inputWaiter c.inputWaiter = nil if waiter != nil { - waiter <- readerUnavailableError(c.readerErr) + waiter <- &inputConfirmationInterruptedError{ + cause: readerUnavailableError(c.readerErr), + } } close(c.readerDone) c.stateMu.Unlock() @@ -1031,6 +1052,13 @@ func inputDeliveryUnknownError(current frame) error { return fmt.Errorf("%w: %s", ErrInputDeliveryUnknown, detail) } +func inputDeliveryUnknownCause(cause error) error { + if cause == nil { + return ErrInputDeliveryUnknown + } + return fmt.Errorf("%w: %w", ErrInputDeliveryUnknown, cause) +} + func normalizeCloseError(err error) error { if err == nil { return nil diff --git a/internal/terminalws/client_test.go b/internal/terminalws/client_test.go index 5fa03c7b..4fc37e94 100644 --- a/internal/terminalws/client_test.go +++ b/internal/terminalws/client_test.go @@ -727,7 +727,7 @@ func TestSendInputConfirmedFailsWhenConnectionClosesBeforeAcknowledgement(t *tes t.Error(err) return } - _ = conn.Close(websocket.StatusNormalClosure, "") + _ = conn.Close(websocket.StatusInternalError, "reader lost after input write") })) defer server.Close() @@ -744,8 +744,73 @@ func TestSendInputConfirmedFailsWhenConnectionClosesBeforeAcknowledgement(t *tes ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() err = client.SendInputConfirmed(ctx, []byte("echo ready\n")) - if err == nil { - t.Fatal("normal close before acknowledgement reported success") + if !errors.Is(err, ErrInputDeliveryUnknown) { + t.Fatalf("error = %v", err) + } + var closeError websocket.CloseError + if !errors.As(err, &closeError) { + t.Fatalf("error does not preserve websocket close: %v", err) + } + if closeError.Code != websocket.StatusInternalError { + t.Fatalf("close code = %v", closeError.Code) + } +} + +func TestSendInputConfirmedKeepsPreWriteReaderLossOrdinary(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + for range 2 { + if _, _, err := conn.Read(r.Context()); err != nil { + t.Error(err) + return + } + } + welcome, _ := json.Marshal(welcomePayload{InputAcknowledgements: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageWelcome, + payload: welcome, + })); err != nil { + t.Error(err) + return + } + subscribed, _ := json.Marshal(eventPayload{Type: "subscribed", CanInput: true}) + if err := conn.Write(r.Context(), websocket.MessageBinary, encodeFrame(frame{ + messageType: messageEvent, + sessionID: "IS-close-before-write", + payload: subscribed, + })); err != nil { + t.Error(err) + return + } + _ = conn.Close(websocket.StatusPolicyViolation, "reader lost before input write") + })) + defer server.Close() + + endpoint, err := Endpoint(server.URL) + if err != nil { + t.Fatal(err) + } + client, err := Dial(context.Background(), endpoint, "IS-close-before-write", Options{}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + <-client.readerDone + + err = client.SendInputConfirmed(context.Background(), []byte("never-written\n")) + if errors.Is(err, ErrInputDeliveryUnknown) { + t.Fatalf("pre-write error became ambiguous: %v", err) + } + var closeError websocket.CloseError + if !errors.As(err, &closeError) { + t.Fatalf("error does not preserve websocket close: %v", err) + } + if closeError.Code != websocket.StatusPolicyViolation { + t.Fatalf("close code = %v", closeError.Code) } } @@ -1547,6 +1612,9 @@ func TestSendInputConfirmedClosesAfterConfirmationTimeout(t *testing.T) { defer cancel() started := time.Now() err = client.SendInputConfirmed(ctx, []byte("first\n")) + if !errors.Is(err, ErrInputDeliveryUnknown) { + t.Fatalf("error = %v", err) + } if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("error = %v", err) } From af362b1c389989c7e6de3764dd3e690546061f20 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:15:13 +0200 Subject: [PATCH 235/242] fix(desktop): reject legacy ownership conflicts --- src/worker/desktop-host-repository.ts | 53 +++++------ tests/control-plane-routes.test.ts | 39 ++++++++ tests/desktop-host-repository.test.ts | 132 +++++++++++++++++--------- 3 files changed, 150 insertions(+), 74 deletions(-) diff --git a/src/worker/desktop-host-repository.ts b/src/worker/desktop-host-repository.ts index 6e6b5499..8746e4d9 100644 --- a/src/worker/desktop-host-repository.ts +++ b/src/worker/desktop-host-repository.ts @@ -1,7 +1,6 @@ -import { sql } from "kysely"; - import { database, executeBatch } from "./database.ts"; import type { RuntimeEnv } from "./env.ts"; +import { conflict } from "./http.ts"; export type DesktopHostRow = { ownerSubject: string; @@ -87,31 +86,19 @@ export class DesktopHostRepository implements DesktopHostStore { publication_write_token: host.ownershipToken, updated_at: host.updatedAt, }) - : update.doUpdateSet({ - owner: sql`CASE - WHEN desktop_hosts.ownership_token = '' THEN excluded.owner - ELSE desktop_hosts.owner - END`, - name: sql`CASE - WHEN desktop_hosts.ownership_token = '' THEN excluded.name - ELSE desktop_hosts.name - END`, - address: sql`CASE - WHEN desktop_hosts.ownership_token = '' THEN excluded.address - ELSE desktop_hosts.address - END`, - port: sql`CASE - WHEN desktop_hosts.ownership_token = '' THEN excluded.port - ELSE desktop_hosts.port - END`, - updated_at: sql`CASE - WHEN desktop_hosts.ownership_token = '' THEN excluded.updated_at - ELSE desktop_hosts.updated_at - END`, - }); + : update + .doUpdateSet({ + owner: host.owner, + name: host.name, + address: host.address, + port: host.port, + updated_at: host.updatedAt, + }) + .where("desktop_hosts.ownership_token", "=", ""); }) .returningAll() - .executeTakeFirstOrThrow(); + .executeTakeFirst(); + if (!row) throw desktopHostOwnershipConflict(); return { ownerSubject: row.owner_subject, id: row.id, @@ -145,12 +132,20 @@ export class DesktopHostRepository implements DesktopHostStore { async remove(ownerSubject: string, id: string, ownershipToken: string | null): Promise { const db = database(this.env); if (!ownershipToken) { - await db + const deleted = await db .deleteFrom("desktop_hosts") .where("owner_subject", "=", ownerSubject) .where("id", "=", id) .where("ownership_token", "=", "") - .execute(); + .executeTakeFirst(); + if ((deleted.numDeletedRows ?? 0n) > 0n) return; + const existing = await db + .selectFrom("desktop_hosts") + .select("ownership_token") + .where("owner_subject", "=", ownerSubject) + .where("id", "=", id) + .executeTakeFirst(); + if (existing?.ownership_token) throw desktopHostOwnershipConflict(); return; } const deleteMarker = `delete-authorized:${crypto.randomUUID()}`; @@ -171,3 +166,7 @@ export class DesktopHostRepository implements DesktopHostStore { ]); } } + +function desktopHostOwnershipConflict(): ReturnType { + return conflict("desktop host is owned by a token-aware registration"); +} diff --git a/tests/control-plane-routes.test.ts b/tests/control-plane-routes.test.ts index 7ad1c3cd..8086924b 100644 --- a/tests/control-plane-routes.test.ts +++ b/tests/control-plane-routes.test.ts @@ -10,6 +10,7 @@ import { desktopHostOwnershipModeHeader, desktopHostTokenOwnershipMode, } from "../src/worker/desktop-host-service.ts"; +import { conflict } from "../src/worker/http.ts"; const viewer: User = { subject: "github:1", @@ -254,6 +255,44 @@ test("desktop host routes register and remove only the authenticated user's host ]); }); +test("desktop host routes expose legacy ownership conflicts", async () => { + for (const method of ["PUT", "DELETE"]) { + const calls: string[] = []; + await assert.rejects( + dispatch( + request( + method, + "/api/desktop-hosts/token-owned", + method === "PUT" + ? { + name: "Legacy Studio", + address: "100.64.1.3", + port: 5901, + } + : undefined, + ), + viewer, + calls, + method === "PUT" + ? { + async registerDesktopHost() { + throw conflict("desktop host is owned by a token-aware registration"); + }, + } + : { + async removeDesktopHost() { + throw conflict("desktop host is owned by a token-aware registration"); + }, + }, + ), + (error) => { + assert.equal(status(error), 409); + return true; + }, + ); + } +}); + test("desktop host recovery rejects malformed encoded ids with a client error", async () => { const calls: string[] = []; await assert.rejects( diff --git a/tests/desktop-host-repository.test.ts b/tests/desktop-host-repository.test.ts index e9af8f86..197c277a 100644 --- a/tests/desktop-host-repository.test.ts +++ b/tests/desktop-host-repository.test.ts @@ -217,7 +217,7 @@ test("desktop host upsert returns the row written by the same atomic statement", assert.equal(row.ownershipToken, "token-a"); }); -test("legacy desktop host upserts preserve token ownership", async () => { +test("legacy desktop host upserts reject token ownership", async () => { let statement = ""; const stored = { owner_subject: "github:1", @@ -239,7 +239,7 @@ test("legacy desktop host upserts preserve token ownership", async () => { bind() { return { async all() { - return { results: [stored], meta: { changes: 1 } }; + return { results: [], meta: { changes: 0 } }; }, }; }, @@ -248,30 +248,27 @@ test("legacy desktop host upserts preserve token ownership", async () => { } as unknown as D1Database, } as RuntimeEnv; - const row = await new DesktopHostRepository(env).upsert({ - ownerSubject: stored.owner_subject, - id: stored.id, - owner: stored.owner, - name: stored.name, - address: stored.address, - port: stored.port, - ownershipToken: "", - publicationID: "", - createdAt: stored.created_at, - updatedAt: stored.updated_at, - }); + await assert.rejects( + new DesktopHostRepository(env).upsert({ + ownerSubject: stored.owner_subject, + id: stored.id, + owner: stored.owner, + name: stored.name, + address: stored.address, + port: stored.port, + ownershipToken: "", + publicationID: "", + createdAt: stored.created_at, + updatedAt: stored.updated_at, + }), + (error) => { + assert.equal(status(error), 409); + return true; + }, + ); const updateClause = statement.split(/do update set/i)[1] ?? ""; - for (const column of ["owner", "name", "address", "port", "updated_at"]) { - assert.match( - updateClause, - new RegExp( - `"${column}" = CASE\\s+WHEN desktop_hosts\\.ownership_token = '' THEN excluded\\.${column}\\s+ELSE desktop_hosts\\.${column}\\s+END`, - "i", - ), - ); - } - assert.equal(row.ownershipToken, "current-token"); + assert.match(updateClause, /where "desktop_hosts"\."ownership_token" = \?/i); }); test("legacy desktop host writes and cleanup cannot mutate token-owned rows", async () => { @@ -305,40 +302,75 @@ test("legacy desktop host writes and cleanup cannot mutate token-owned rows", as `); const repository = new DesktopHostRepository(sqliteRuntimeEnv(sqlite)); - const preserved = await repository.upsert({ - ownerSubject: "github:1", - id: "studio", - owner: "legacy-worker", - name: "Overwritten Studio", - address: "100.64.1.99", - port: 5902, - ownershipToken: "", - publicationID: "", - createdAt: 10, - updatedAt: 20, + await assert.rejects( + repository.upsert({ + ownerSubject: "github:1", + id: "studio", + owner: "legacy-worker", + name: "Overwritten Studio", + address: "100.64.1.99", + port: 5902, + ownershipToken: "", + publicationID: "", + createdAt: 10, + updatedAt: 20, + }), + (error) => { + assert.equal(status(error), 409); + return true; + }, + ); + await assert.rejects(repository.remove("github:1", "studio", null), (error) => { + assert.equal(status(error), 409); + return true; }); - assert.deepEqual(preserved, { + await repository.remove("github:1", "studio", "stale-token"); + assert.equal( + sqlite.prepare("SELECT ownership_token FROM desktop_hosts WHERE id = 'studio'").get() + ?.ownership_token, + "current-token", + ); + + await repository.remove("github:1", "studio", "current-token"); + assert.equal(sqlite.prepare("SELECT count(*) AS count FROM desktop_hosts").get()?.count, 0); +}); + +test("legacy desktop host writes and cleanup preserve legacy rows", async () => { + const sqlite = new DatabaseSync(":memory:"); + for (const migration of [ + "0030_desktop_hosts.sql", + "0033_desktop_host_ownership.sql", + "0038_desktop_host_publication_identity.sql", + "0041_desktop_host_ownership_errors.sql", + ]) { + sqlite.exec(readFileSync(new URL(`../migrations/${migration}`, import.meta.url), "utf8")); + } + const repository = new DesktopHostRepository(sqliteRuntimeEnv(sqlite)); + const host = { ownerSubject: "github:1", id: "studio", owner: "alice", - name: "Token Studio", + name: "Legacy Studio", address: "100.64.1.2", port: 5901, - ownershipToken: "current-token", - publicationID: "current-publication", + ownershipToken: "", + publicationID: "", createdAt: 1, updatedAt: 2, - }); + }; - await repository.remove("github:1", "studio", null); - await repository.remove("github:1", "studio", "stale-token"); + await repository.upsert(host); assert.equal( - sqlite.prepare("SELECT ownership_token FROM desktop_hosts WHERE id = 'studio'").get() - ?.ownership_token, - "current-token", + ( + await repository.upsert({ + ...host, + name: "Updated Legacy Studio", + updatedAt: 3, + }) + ).name, + "Updated Legacy Studio", ); - - await repository.remove("github:1", "studio", "current-token"); + await repository.remove(host.ownerSubject, host.id, null); assert.equal(sqlite.prepare("SELECT count(*) AS count FROM desktop_hosts").get()?.count, 0); }); @@ -441,3 +473,9 @@ test("same-publication retries remain recoverable after the publication migratio null, ); }); + +function status(error: unknown): number | undefined { + return typeof error === "object" && error !== null && "status" in error + ? Number(error.status) + : undefined; +} From 4cc0ce78abbf77f311e78cd072beda512507c1dd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:15:23 +0200 Subject: [PATCH 236/242] fix(macos): discard negative publication recovery --- .../PrivateMacShareController.swift | 8 +++++-- .../PrivateMacShareTests.swift | 24 +++++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift index 9fa90d98..1347087e 100644 --- a/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift +++ b/macos/CrabfleetMac/Sources/CrabfleetMac/PrivateMacShareController.swift @@ -299,17 +299,21 @@ final class DesktopHostRegistrationLifecycle { identity: target.identity, publicationID: target.publicationID ) + self.uncertainRegistrations.removeAll { $0 == target } + guard let ownershipToken else { + try persistState() + continue + } let recovered = PublishedRegistration( identity: target.identity, hostID: target.hostID, publicationID: target.publicationID, ownershipToken: ownershipToken, - usesLegacyCleanup: ownershipToken == nil + usesLegacyCleanup: false ) if !pendingRemovals.contains(recovered) { pendingRemovals.append(recovered) } - self.uncertainRegistrations.removeAll { $0 == target } try persistState() } catch { firstError = firstError ?? error diff --git a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift index 0a9c5aa0..9061351d 100644 --- a/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift +++ b/macos/CrabfleetMac/Tests/CrabfleetMacTests/PrivateMacShareTests.swift @@ -501,18 +501,19 @@ struct PrivateMacShareTests { } @Test @MainActor - func ambiguousLegacyDesktopPublicationAttemptsGuardedCleanup() async throws { - let identity = desktopIdentity(name: "ambiguous-legacy", address: "100.64.12.62") - let registration = AmbiguousLegacyDesktopRegistration() + func negativeRecoveryDoesNotDeleteANewerTokenlessPublisher() async throws { + let identity = desktopIdentity(name: "negative-recovery", address: "100.64.12.62") + let registration = NegativeRecoveryDesktopRegistration() let lifecycle = DesktopHostRegistrationLifecycle(registration: registration) await #expect(throws: DesktopHostRegistrationResultUncertainError.self) { try await lifecycle.publish(identity: identity, port: 5_901) } + await registration.publishNewerEndpoint() try await lifecycle.removePublishedIdentities() - #expect(!(await registration.isPublished)) - #expect(await registration.events == [.register, .recover, .unregister]) + #expect(await registration.activeEndpoint == "newer-publisher") + #expect(await registration.events == [.register, .recover]) } @Test @MainActor @@ -2694,14 +2695,14 @@ private actor AmbiguousDesktopRegistration: DesktopHostRegistering { } } -private actor AmbiguousLegacyDesktopRegistration: DesktopHostRegistering { +private actor NegativeRecoveryDesktopRegistration: DesktopHostRegistering { enum Event: Equatable { case register case recover case unregister } - private(set) var isPublished = false + private(set) var activeEndpoint: String? private(set) var events: [Event] = [] func register( @@ -2710,8 +2711,7 @@ private actor AmbiguousLegacyDesktopRegistration: DesktopHostRegistering { publicationID: String ) async throws -> String? { events.append(.register) - isPublished = true - throw DesktopHostRegistrationResultUncertainError(message: "legacy response lost") + throw DesktopHostRegistrationResultUncertainError(message: "response lost") } func recover(identity: TailnetIdentity, publicationID: String) async throws -> String? { @@ -2722,7 +2722,11 @@ private actor AmbiguousLegacyDesktopRegistration: DesktopHostRegistering { func unregister(identity: TailnetIdentity, ownershipToken: String?) async throws { #expect(ownershipToken == nil) events.append(.unregister) - isPublished = false + activeEndpoint = nil + } + + func publishNewerEndpoint() { + activeEndpoint = "newer-publisher" } } From 7b5bcd22b48b4a6a790bfcf9a414e496374f3d0e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:15:55 +0200 Subject: [PATCH 237/242] docs(changelog): record delivery and ownership fixes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42327edc..2a25eee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement and bounded PTY-write failure, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material, deferring partial-fence ZRLE resets to a trailing synchronization boundary, surfacing legacy mutations of token-owned desktop registrations, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. +- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement, disconnect, bounded PTY-write failure, and post-write confirmation loss, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material, deferring partial-fence ZRLE resets to a trailing synchronization boundary, rejecting legacy mutations and deletions of token-owned desktop registrations, discarding definitively unowned publication recovery without tokenless cleanup, documenting the opaque publication-ID contract, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. - Close the final review blockers by requiring an exact live credential-policy lease at promotion, negotiating strict runtime-adapter deletion tombstones without breaking legacy `404` release semantics, disconnecting VNC sessions when pixel-format capability probes cannot establish a safe ZRLE boundary, classifying aborted JSON body streams as bad requests, and preserving live terminal subscriptions when browser history restores the sessions grid. - Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. - Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets. From 4fa0be33482ce208ad367206c73dfd8d5760e16d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:26:30 +0200 Subject: [PATCH 238/242] fix(terminal): preserve upstream delivery ambiguity --- src/worker/terminal-hub.ts | 6 ++- tests/terminal-hub.test.ts | 83 +++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/src/worker/terminal-hub.ts b/src/worker/terminal-hub.ts index 1c68607f..f1fb5d3d 100644 --- a/src/worker/terminal-hub.ts +++ b/src/worker/terminal-hub.ts @@ -759,7 +759,8 @@ export class TerminalHub { upstream.addEventListener("close", (event) => { completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, - error: "terminal upstream closed before accepting input", + deliveryUnknown: true, + error: "terminal input delivery outcome is unknown; the runner may still complete it", }); const closeReason = consumeCloseReason(); const safeUpstreamReason = event.reason @@ -787,7 +788,8 @@ export class TerminalHub { upstream.addEventListener("error", () => { completeAllTerminalInputAcknowledgements(activeSubscription, { accepted: false, - error: "terminal upstream failed before accepting input", + deliveryUnknown: true, + error: "terminal input delivery outcome is unknown; the runner may still complete it", }); const closeReason = closingReason; if (subscriptions.delete(id)) this.dependencies.releaseInputState(id); diff --git a/tests/terminal-hub.test.ts b/tests/terminal-hub.test.ts index 260f85bf..3c620fd0 100644 --- a/tests/terminal-hub.test.ts +++ b/tests/terminal-hub.test.ts @@ -1389,7 +1389,7 @@ test("generation-fenced send failure completes its matching acknowledgement", as server.emit("close"); }); -test("GitHub Actions close rejects every pending input acknowledgement", async () => { +test("GitHub Actions close reports forwarded pending input as delivery unknown", async () => { const client = socket(); const server = socket(); const upstream = socket(); @@ -1426,30 +1426,75 @@ test("GitHub Actions close rejects every pending input acknowledgement", async ( }), }); await waitForInputPayloads(); + assert.equal(upstream.sent.length, 2); upstream.emit("close", { code: 1011, reason: "runner disconnected" }); await flushQueues(); await flushQueues(); - const messages = server.sent.map((payload) => frame(payload)); - assert.equal( - messages.some( - (message) => - message.type === TerminalMessageType.Event && - (decodeJsonPayload(message.payload) as { type?: string }).type === "input-accepted", - ), - false, + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type?.startsWith("input-")); + assert.deepEqual(completions, [ + { + type: "input-delivery-unknown", + error: "terminal input delivery outcome is unknown; the runner may still complete it", + }, + ]); + server.emit("close"); +}); + +test("GitHub Actions error reports forwarded pending input as delivery unknown", async () => { + const client = socket(); + const server = socket(); + const upstream = socket(); + const hub = new TerminalHub( + dependencies(client, server, upstream, { + async readSession() { + return githubActionsSession; + }, + }), ); - assert.equal( - messages.some( - (message) => - message.type === TerminalMessageType.Event && - (decodeJsonPayload(message.payload) as { type?: string; error?: string }).type === - "input-rejected" && - (decodeJsonPayload(message.payload) as { error?: string }).error === - "terminal upstream closed before accepting input", - ), - true, + await hub.open( + new Request("https://fleet.example/api/terminal/ws", { + headers: { upgrade: "websocket" }, + }), + user, ); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Subscribe, + sessionId: githubActionsSession.id, + payload: encodeSubscribePayload({ flags: 0, columns: 120, rows: 34 }), + }), + }); + await flushQueues(); + await flushQueues(); + server.emit("message", { + data: encodeTerminalFrame({ + type: TerminalMessageType.Input, + sessionId: githubActionsSession.id, + payload: new TextEncoder().encode("input"), + }), + }); + await waitForInputPayloads(); + assert.equal(upstream.sent.length, 1); + upstream.emit("error"); + await flushQueues(); + await flushQueues(); + + const completions = server.sent + .map((payload) => frame(payload)) + .filter((message) => message.type === TerminalMessageType.Event) + .map((message) => decodeJsonPayload(message.payload) as { type?: string; error?: string }) + .filter((message) => message.type?.startsWith("input-")); + assert.deepEqual(completions, [ + { + type: "input-delivery-unknown", + error: "terminal input delivery outcome is unknown; the runner may still complete it", + }, + ]); server.emit("close"); }); From 8dff7f11c87a6e181eb2927de16c743e7d91d0d1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:26:55 +0200 Subject: [PATCH 239/242] fix(vnc): bound validated safe-prime cache --- .../ARDDiffieHellmanKeyAgreement.swift | 30 ++++++++++++++++++- .../SecurityAndInputTests.swift | 26 ++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift index dce4a8b3..7b7a5be7 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Sources/RoyalVNCKit/Protocol/SecurityTypes/AppleRemoteDesktop/ARDDiffieHellmanKeyAgreement.swift @@ -7,6 +7,34 @@ import CryptoSwift extension VNCProtocol.ARDAuthentication { struct DiffieHellmanKeyAgreement { + struct ValidatedSafePrimeCache { + let capacity: Int + private var insertionOrder = [Data]() + private var values = Set() + + init(capacity: Int) { + precondition(capacity > 0) + self.capacity = capacity + } + + var count: Int { + values.count + } + + func contains(_ value: Data) -> Bool { + values.contains(value) + } + + mutating func insert(_ value: Data) { + guard values.insert(value).inserted else { return } + + insertionOrder.append(value) + if values.count > capacity { + values.remove(insertionOrder.removeFirst()) + } + } + } + let publicKey: Data let privateKey: Data let secretKey: Data @@ -52,7 +80,7 @@ extension VNCProtocol.ARDAuthentication { private extension VNCProtocol.ARDAuthentication.DiffieHellmanKeyAgreement { static let safePrimeCondition = NSCondition() - static var validatedSafePrimes = Set() + static var validatedSafePrimes = ValidatedSafePrimeCache(capacity: 32) static var rejectedSafePrimes = Set() static var safePrimeValidations = Set() diff --git a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift index dd8b0d3c..91701073 100644 --- a/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift +++ b/macos/CrabfleetMac/Vendor/RoyalVNCKit/Tests/RoyalVNCKitTests/SecurityAndInputTests.swift @@ -83,6 +83,32 @@ struct SecurityAndInputTests { #expect(agreement?.secretKey.contains { $0 != 0 } == true) } + @Test + func evictsValidatedAppleRemoteDesktopSafePrimesInInsertionOrder() { + var cache = ARDKeyAgreement.ValidatedSafePrimeCache(capacity: 3) + let values = (0..<5).map { Data([$0]) } + + for value in values.prefix(3) { + cache.insert(value) + } + cache.insert(values[0]) + cache.insert(values[3]) + + #expect(cache.count == 3) + #expect(!cache.contains(values[0])) + #expect(cache.contains(values[1])) + #expect(cache.contains(values[2])) + #expect(cache.contains(values[3])) + + cache.insert(values[4]) + + #expect(cache.count == 3) + #expect(!cache.contains(values[1])) + #expect(cache.contains(values[2])) + #expect(cache.contains(values[3])) + #expect(cache.contains(values[4])) + } + @Test func computesUltraVNCModularArithmeticKnownAnswers() { #expect( From ed6b9e10cf3e5c0257dbe11fc1540b6929bad0bb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:27:17 +0200 Subject: [PATCH 240/242] docs(changelog): record final cache and delivery fixes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a25eee5..8bfae700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement, disconnect, bounded PTY-write failure, and post-write confirmation loss, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material, deferring partial-fence ZRLE resets to a trailing synchronization boundary, rejecting legacy mutations and deletions of token-owned desktop registrations, discarding definitively unowned publication recovery without tokenless cleanup, documenting the opaque publication-ID contract, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. +- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement, disconnect, upstream close or error, bounded PTY-write failure, and post-write confirmation loss, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material with bounded accepted-prime caching, deferring partial-fence ZRLE resets to a trailing synchronization boundary, rejecting legacy mutations and deletions of token-owned desktop registrations, discarding definitively unowned publication recovery without tokenless cleanup, documenting the opaque publication-ID contract, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. - Close the final review blockers by requiring an exact live credential-policy lease at promotion, negotiating strict runtime-adapter deletion tombstones without breaking legacy `404` release semantics, disconnecting VNC sessions when pixel-format capability probes cannot establish a safe ZRLE boundary, classifying aborted JSON body streams as bad requests, and preserving live terminal subscriptions when browser history restores the sessions grid. - Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. - Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets. From 41140f5f2a9a84c3d81f5fe94ac1d94a5578c5f4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:37:22 +0200 Subject: [PATCH 241/242] fix(credentials): preserve rotation recovery invariants --- .../sandbox-credential-policy-repository.ts | 8 +- ...ndbox-credential-policy-repository.test.ts | 86 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/worker/sandbox-credential-policy-repository.ts b/src/worker/sandbox-credential-policy-repository.ts index 0ad5bf33..aad47e16 100644 --- a/src/worker/sandbox-credential-policy-repository.ts +++ b/src/worker/sandbox-credential-policy-repository.ts @@ -65,6 +65,7 @@ export function activeSandboxCredentialPolicyCondition( state != 'active' OR registration_generation != ${generation} OR registration_claim IS NOT NULL + OR lookup_id NOT IN (${sql.join(lookupIds)}) OR NOT (${updatedAtCondition}) ) ) @@ -82,11 +83,12 @@ export async function activeSandboxCredentialPolicyGeneration( .where("session_id", "=", sessionId) .where("sandbox_id", "=", sandboxId) .execute(); - const expected = sandboxLookupIds(env, sandboxId); + const expected = new Set(sandboxLookupIds(env, sandboxId)); const generation = rows[0]?.registration_generation; if ( !isCurrentCredentialPolicyGeneration(generation) || - !expected.every((lookupId) => + rows.length !== expected.size || + ![...expected].every((lookupId) => rows.some( (row) => row.lookup_id === lookupId && @@ -97,6 +99,7 @@ export async function activeSandboxCredentialPolicyGeneration( ) || rows.some( (row) => + !expected.has(row.lookup_id) || row.state !== "active" || row.registration_generation !== generation || row.registration_claim !== null, @@ -586,6 +589,7 @@ export function sandboxCredentialPolicyRegistrationQueries( cleanup_claim_expires_at = NULL, updated_at = excluded.updated_at WHERE interactive_session_credential_policy_registrations.state != 'cleanup_pending' + AND interactive_session_credential_policy_registrations.registration_write_started = 0 AND ( interactive_session_credential_policy_registrations.registration_claim IS NULL OR interactive_session_credential_policy_registrations.registration_claim_expires_at <= ${now} diff --git a/tests/sandbox-credential-policy-repository.test.ts b/tests/sandbox-credential-policy-repository.test.ts index 4597221e..1484c51a 100644 --- a/tests/sandbox-credential-policy-repository.test.ts +++ b/tests/sandbox-credential-policy-repository.test.ts @@ -442,6 +442,7 @@ test("credential-policy registration SQL proves every supported ownership fence" assert.match(current.sql, /sandbox_refresh_claim is null/i); assert.match(current.sql, /state = 'registering'/i); assert.match(current.sql, /registration_claim_expires_at >/i); + assert.match(current.sql, /registration_write_started = 0/i); assert.ok(current.parameters.includes("sandbox:sandbox-1:terminal-1:autostart-v4")); assert.doesNotMatch(current.sql, /1 = 1/); @@ -915,6 +916,82 @@ test("staged rotations fence old-worker writes until the staged row is removed", assert.equal(postFenceClaim.changes, 2); }); +test("expired write-started registrations remain reserved for recovery", async () => { + const sqlite = credentialPolicyDatabase(); + const env = sqliteRuntimeEnv(sqlite); + const staged = await beginSandboxCredentialPolicyRegistration( + env, + "IS-42", + "sandbox-1", + ownershipFence, + ); + const rollback = ["sandbox-1", "do-1"].map((lookupId) => ({ + generation: "generation:existing", + policy: { + allowedHosts: [], + githubCredentialSource: "none" as const, + githubRepo: "openclaw/crabfleet", + owner: "operator", + sandboxId: lookupId, + sessionId: "IS-42", + }, + })); + assert.equal( + await recordSandboxCredentialPolicyRollback( + env, + "IS-42", + "sandbox-1", + staged, + rollback, + ownershipFence, + ), + true, + ); + assert.ok( + await markSandboxCredentialPolicyRegistrationWriteStarted( + env, + "IS-42", + "sandbox-1", + staged, + ownershipFence, + ), + ); + sqlite + .prepare(` + UPDATE interactive_session_credential_policy_registrations + SET registration_claim_expires_at = 0 + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .run(); + + await assert.rejects( + beginSandboxCredentialPolicyRegistration(env, "IS-42", "sandbox-1", ownershipFence), + { message: "sandbox credential policy registration is unavailable" }, + ); + + assert.deepEqual( + { + ...sqlite + .prepare(` + SELECT + registration_generation, + registration_claim, + registration_write_started, + rollback_policies_json + FROM interactive_session_credential_policy_registrations + WHERE session_id = 'IS-42' AND sandbox_id = 'sandbox-1' + `) + .get(), + }, + { + registration_generation: staged.generation, + registration_claim: staged.claim, + registration_write_started: 1, + rollback_policies_json: JSON.stringify(rollback), + }, + ); +}); + test("expired pre-write staged rotations release the legacy worker compatibility fence", async () => { const sqlite = credentialPolicyDatabase(); const env = sqliteRuntimeEnv(sqlite); @@ -1761,6 +1838,15 @@ test("active credential-policy generation requires every exact lookup row", asyn rows[1] = { ...rows[1]!, registration_generation: "generation:test-1" }; rows[1] = { ...rows[1]!, registration_claim: "stale" }; assert.equal(await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), null); + + rows[1] = { ...rows[1]!, registration_claim: null }; + rows.push({ + lookup_id: "do-obsolete", + state: "active", + registration_generation: "generation:test-1", + registration_claim: null, + }); + assert.equal(await activeSandboxCredentialPolicyGeneration(env, "IS-42", "sandbox-1"), null); }); test("credential refresh repairs an incomplete legacy lookup set before rotation", async () => { From 75f76bf30973f4e66c94373fbdcb0d6a5f7b1c7e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:37:28 +0200 Subject: [PATCH 242/242] docs(changelog): record credential recovery invariants --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bfae700..6b816cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement, disconnect, upstream close or error, bounded PTY-write failure, and post-write confirmation loss, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material with bounded accepted-prime caching, deferring partial-fence ZRLE resets to a trailing synchronization boundary, rejecting legacy mutations and deletions of token-owned desktop registrations, discarding definitively unowned publication recovery without tokenless cleanup, documenting the opaque publication-ID contract, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. +- Resolve final audit blockers by preserving unknown GitHub Actions input outcomes across runner replacement, disconnect, upstream close or error, bounded PTY-write failure, and post-write confirmation loss, making terminal confirmation races prefer completed delivery, retiring terminal connections after late detached-writer failures, replaying control grants across attachment gaps, retaining rollback recovery and authorized namespace retirement after a staged credential write begins, reserving expired write-started credential rows for recovery, requiring active credential generations to match the exact current lookup set, proving interrupted pre-fence migration recovery, blocking session deletion while staged credential rows remain, validating ARD safe-prime groups and nonzero key material with bounded accepted-prime caching, deferring partial-fence ZRLE resets to a trailing synchronization boundary, rejecting legacy mutations and deletions of token-owned desktop registrations, discarding definitively unowned publication recovery without tokenless cleanup, documenting the opaque publication-ID contract, completing ambiguous legacy publication cleanup despite persistence failures, and holding generated-asset tests under a crash-released lock through module consumption. - Close the final review blockers by requiring an exact live credential-policy lease at promotion, negotiating strict runtime-adapter deletion tombstones without breaking legacy `404` release semantics, disconnecting VNC sessions when pixel-format capability probes cannot establish a safe ZRLE boundary, classifying aborted JSON body streams as bad requests, and preserving live terminal subscriptions when browser history restores the sessions grid. - Complete the final audit follow-up by fencing rollback against newer legacy credential generations, keeping viewer acknowledgement timeouts from detaching live GitHub Actions sessions, fencing queued documented-runner input after relay replacement, generating ignored embedded assets before parity tests import them, resetting ZRLE exactly at pixel-format boundaries, preventing delayed tokenless desktop cleanup from deleting replacement publishers, skipping idle recovery I/O when no local state exists, and stopping canceled auto-share preflight. - Preserve upgrade and teardown authority by leaving pre-lookup-migration credential registrations recoverable from current and historical runtime identities, retaining GitHub Actions runner generations after queues drain, scoping Share This Mac recovery state to the normalized API origin and stable owner, allowing idle termination after unavailable recovery lookup while retaining active cleanup vetoes, terminating timed-out or canceled Tailscale descendant process groups, clearing only definitive failed publication intent, quiescing remote-input producers before final release, rejecting UltraVNC Diffie-Hellman elements at `p - 1`, and synchronizing complete RFB pixel-format and encoding transitions with protocol-required ZRLE resets.