From 44a7cb1057b6ef4aedc89760575c2e506669f8a5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 20:51:44 -0700 Subject: [PATCH 001/113] fix(relay): tunnel recovery no longer drives the Cloudflare API into rate limits (#22059) Co-authored-by: Claude Opus 5.5 --- .../migration.sql | 4 + .../snapshot.json | 1634 +++++++++++++++++ infra/relay/package.json | 1 + .../environments/CloudflareApiPause.test.ts | 59 + .../src/environments/CloudflareApiPause.ts | 85 + .../ManagedEndpointProvider.test.ts | 196 ++ .../environments/ManagedEndpointProvider.ts | 479 +++-- .../ManagedEndpointReaper.test.ts | 14 +- .../src/environments/ManagedEndpointReaper.ts | 15 +- infra/relay/src/http/Api.test.ts | 96 + infra/relay/src/http/Api.ts | 15 + infra/relay/src/monitoring.ts | 17 +- infra/relay/src/persistence/schema.ts | 8 + infra/relay/src/worker.ts | 2 + pnpm-lock.yaml | 6 + pnpm-workspace.yaml | 3 + 16 files changed, 2468 insertions(+), 166 deletions(-) create mode 100644 infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/migration.sql create mode 100644 infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/snapshot.json create mode 100644 infra/relay/src/environments/CloudflareApiPause.test.ts create mode 100644 infra/relay/src/environments/CloudflareApiPause.ts diff --git a/infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/migration.sql b/infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/migration.sql new file mode 100644 index 0000000000000..175f886bd3c16 --- /dev/null +++ b/infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/migration.sql @@ -0,0 +1,4 @@ +CREATE TABLE "relay_upstream_api_pauses" ( + "api" varchar(64) PRIMARY KEY, + "paused_until_ms" bigint NOT NULL +); diff --git a/infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/snapshot.json b/infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/snapshot.json new file mode 100644 index 0000000000000..84acbfa6fd44e --- /dev/null +++ b/infra/relay/migrations/postgres/20261020013101_upstream_api_pauses/snapshot.json @@ -0,0 +1,1634 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "f8594baa-6f4a-4a0d-a7f0-0aa717dfe785", + "prevIds": ["7b10c420-1566-4dfc-8e9c-c73efa561d3b"], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_upstream_api_pauses", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hold_webhooks_while_offline", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_released_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "android_api_level", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "api", + "entityType": "columns", + "schema": "public", + "table": "relay_upstream_api_pauses" + }, + { + "type": "bigint", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "paused_until_ms", + "entityType": "columns", + "schema": "public", + "table": "relay_upstream_api_pauses" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + }, + { + "columns": ["api"], + "nameExplicit": false, + "name": "relay_upstream_api_pauses_pkey", + "schema": "public", + "table": "relay_upstream_api_pauses", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/package.json b/infra/relay/package.json index 2992bb56a12ad..dd2265aafb0d5 100644 --- a/infra/relay/package.json +++ b/infra/relay/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@clerk/backend": "catalog:", + "@distilled.cloud/cloudflare": "catalog:", "@effect/sql-pg": "catalog:", "@effect/sql-sqlite-do": "catalog:", "@noble/curves": "catalog:", diff --git a/infra/relay/src/environments/CloudflareApiPause.test.ts b/infra/relay/src/environments/CloudflareApiPause.test.ts new file mode 100644 index 0000000000000..c844a679c106a --- /dev/null +++ b/infra/relay/src/environments/CloudflareApiPause.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as SqlClient from "effect/sql/SqlClient"; + +import * as RelayDb from "../db.ts"; +import * as CloudflareApiPause from "./CloudflareApiPause.ts"; + +const TransactionConnection = SqlClient.TransactionConnection(7); + +// Like the Worker's alchemy database proxy: every property read returns +// another proxy, never the client's real transaction key. +const lazyProxy = (): unknown => new Proxy(() => {}, { get: () => lazyProxy() }); + +// Records whether each statement would have joined the ambient transaction. +function makeFakeDb(joinedTransaction: Array) { + const statement = (result: A) => + Effect.serviceOption(TransactionConnection).pipe( + Effect.tap((connection) => + Effect.sync(() => joinedTransaction.push(Option.isSome(connection))), + ), + Effect.as(result), + ); + return { + $client: lazyProxy(), + select: () => ({ + from: () => ({ where: () => ({ limit: () => statement([{ pausedUntilMs: 42 }]) }) }), + }), + insert: () => ({ values: () => ({ onConflictDoUpdate: () => statement([]) }) }), + } as unknown as RelayDb.RelayDb["Service"]; +} + +describe("CloudflareApiPause", () => { + it.effect("reads and records the pause outside a surrounding transaction", () => { + const joinedTransaction: Array = []; + return Effect.gen(function* () { + const pause = yield* CloudflareApiPause.CloudflareApiPause; + const inTransaction = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(TransactionConnection, [ + {} as never, + 0, + ] as unknown as SqlClient.TransactionConnection.Service), + ); + + expect(yield* inTransaction(pause.pausedUntilMillis)).toBe(42); + yield* inTransaction(pause.pauseUntil(1_000)); + + expect(joinedTransaction).toEqual([false, false]); + }).pipe( + Effect.provide( + CloudflareApiPause.layer.pipe( + Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb(joinedTransaction))), + ), + ), + ); + }); +}); diff --git a/infra/relay/src/environments/CloudflareApiPause.ts b/infra/relay/src/environments/CloudflareApiPause.ts new file mode 100644 index 0000000000000..00a076e4d6219 --- /dev/null +++ b/infra/relay/src/environments/CloudflareApiPause.ts @@ -0,0 +1,85 @@ +import { eq, sql } from "drizzle-orm"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as RelayDb from "../db.ts"; +import { relayUpstreamApiPauses } from "../persistence/schema.ts"; + +const CLOUDFLARE_API = "cloudflare"; + +// Every SqlClient's transaction connection lives under a key with this prefix. +// Matching by key name instead of `db.$client.transactionService` because the +// relay's database is an alchemy proxy, whose properties are proxies too, not +// the real key. +const SQL_TRANSACTION_KEY_PREFIX = "effect/sql/SqlClient/TransactionConnection/"; + +const withoutSqlTransaction = (context: Context.Context): Context.Context => + Context.makeUnsafe( + new Map( + Array.from(context.mapUnsafe).filter(([key]) => !key.startsWith(SQL_TRANSACTION_KEY_PREFIX)), + ), + ); + +/** + * Cloudflare's API rate limit is per account, so a 429 seen by any isolate + * applies to all of them. Both operations fail open: a pause that cannot be + * read or recorded must not block provisioning. + * + * Both also run outside any ambient transaction. Provisioning calls Cloudflare + * while holding an allocation row lock, and a 429 there rolls that transaction + * back, which would drop the recorded pause; a failed read inside it would + * abort the provision's own statements. + */ +export class CloudflareApiPause extends Context.Service< + CloudflareApiPause, + { + /** Epoch milliseconds until which calls are paused, or 0. */ + readonly pausedUntilMillis: Effect.Effect; + readonly pauseUntil: (untilMillis: number) => Effect.Effect; + } +>()("t3code-relay/environments/CloudflareApiPause") {} + +export const make = Effect.gen(function* () { + const db = yield* RelayDb.RelayDb; + const outsideTransaction = (effect: Effect.Effect) => + Effect.updateContext(effect, withoutSqlTransaction); + + return CloudflareApiPause.of({ + pausedUntilMillis: db + .select({ pausedUntilMs: relayUpstreamApiPauses.pausedUntilMs }) + .from(relayUpstreamApiPauses) + .where(eq(relayUpstreamApiPauses.api, CLOUDFLARE_API)) + .limit(1) + .pipe( + Effect.map((rows) => rows[0]?.pausedUntilMs ?? 0), + Effect.catch((cause) => + Effect.logWarning("Could not read the shared Cloudflare API pause", { cause }).pipe( + Effect.as(0), + ), + ), + outsideTransaction, + Effect.withSpan("relay.cloudflare_api_pause.read"), + ), + pauseUntil: (untilMillis) => + db + .insert(relayUpstreamApiPauses) + .values({ api: CLOUDFLARE_API, pausedUntilMs: untilMillis }) + .onConflictDoUpdate({ + target: relayUpstreamApiPauses.api, + set: { + pausedUntilMs: sql`greatest(${relayUpstreamApiPauses.pausedUntilMs}, excluded.paused_until_ms)`, + }, + }) + .pipe( + Effect.asVoid, + Effect.catch((cause) => + Effect.logWarning("Could not record the shared Cloudflare API pause", { cause }), + ), + outsideTransaction, + Effect.withSpan("relay.cloudflare_api_pause.record"), + ), + }); +}); + +export const layer = Layer.effect(CloudflareApiPause, make); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 2e8c895d11005..9717382444fb8 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -3,11 +3,15 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; +import * as TestClock from "effect/testing/TestClock"; import * as RelayConfiguration from "../Config.ts"; +import * as CloudflareApiPause from "./CloudflareApiPause.ts"; import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; import * as ManagedTunnelLimits from "./ManagedTunnelLimits.ts"; @@ -364,6 +368,17 @@ function layerProvider( ); } +function makeSharedCloudflareApiPause() { + let pausedUntilMillis = 0; + return CloudflareApiPause.CloudflareApiPause.of({ + pausedUntilMillis: Effect.sync(() => pausedUntilMillis), + pauseUntil: (untilMillis) => + Effect.sync(() => { + pausedUntilMillis = Math.max(pausedUntilMillis, untilMillis); + }), + }); +} + const MANAGED_ENDPOINT_HASHES: Record = { "user_ABC:env_ABC": "d101ac68108a423e", "user_ABC:env_shared": "d7e6356aaf8863aa", @@ -413,6 +428,9 @@ describe("ManagedEndpointProvider", () => { Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()), ), Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())), + Layer.provide( + Layer.succeed(CloudflareApiPause.CloudflareApiPause, makeSharedCloudflareApiPause()), + ), ); return Effect.gen(function* () { @@ -754,6 +772,184 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("recovers a ready tunnel for the same origin with only reads", () => { + const tunnelCalls: TunnelCall[] = []; + const dnsCalls: DnsCall[] = []; + const allocationCalls: AllocationCall[] = []; + const layer = layerProvider( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(dnsCalls), + makeAllocations(allocationCalls), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const request = { + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + } as const; + const provisioned = yield* provider.provision(request); + tunnelCalls.length = 0; + dnsCalls.length = 0; + allocationCalls.length = 0; + + const recovered = yield* provider.provision({ ...request, reuseReadyTunnel: true }); + + expect(recovered).toEqual(provisioned); + expect(tunnelCalls.map((call) => call.operation)).toEqual(["get", "getToken"]); + expect(dnsCalls).toEqual([]); + expect(allocationCalls.map((call) => call.operation)).toEqual(["reserve"]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("fully provisions a recovery when the ready tunnel cannot be reused", () => { + const tunnelCalls: TunnelCall[] = []; + const tunnelClient = makePersistentTunnelClient(tunnelCalls); + const layer = layerProvider(tunnelClient); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const request = { + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + reuseReadyTunnel: true, + } as const; + yield* provider.provision(request); + + tunnelCalls.length = 0; + yield* provider.provision({ + ...request, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3774 }, + }); + expect(tunnelCalls.map((call) => call.operation)).toEqual([ + "list", + "putConfiguration", + "getToken", + ]); + + yield* tunnelClient.delete("tunnel-id"); + tunnelCalls.length = 0; + yield* provider.provision({ + ...request, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3774 }, + }); + expect(tunnelCalls.map((call) => call.operation)).toEqual([ + "get", + "list", + "create", + "putConfiguration", + "getToken", + ]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a pause recorded while a shared pause read was in flight", () => { + let listCalls = 0; + const tunnelClient = { + list: () => + Effect.suspend(() => { + listCalls += 1; + return Effect.fail({ _tag: "TooManyRequests", message: "Rate limited" }); + }), + } as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient; + + return Effect.gen(function* () { + const readStarted = yield* Deferred.make(); + const releaseRead = yield* Deferred.make(); + const layer = ManagedEndpointProvider.layerCloudflareBindings( + tunnelClient, + {} as Cloudflare.DNS.ReadWriteDnsClient, + {} as Alchemy.BaseRuntimeContext, + ).pipe( + Layer.provideMerge(NodeServices.layer), + Layer.provide(RelayConfiguration.layer(config)), + Layer.provide( + Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()), + ), + Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())), + Layer.provide( + Layer.succeed( + CloudflareApiPause.CloudflareApiPause, + CloudflareApiPause.CloudflareApiPause.of({ + pausedUntilMillis: Deferred.succeed(readStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseRead)), + Effect.as(0), + ), + pauseUntil: () => Effect.void, + }), + ), + ), + ); + + yield* Effect.gen(function* () { + const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; + const list = tunnels.list({ isDeleted: false }).pipe(Effect.flip); + // The first call starts the shared read; the second hits a 429 meanwhile. + const reading = yield* Effect.forkChild(list); + yield* Deferred.await(readStarted); + yield* list; + yield* Deferred.succeed(releaseRead, undefined); + yield* Fiber.join(reading); + yield* list; + expect(listCalls).toBe(1); + }).pipe(Effect.provide(layer)); + }); + }); + + it.effect("pauses Cloudflare calls in every isolate until a rate limit has passed", () => { + let listCalls = 0; + const tunnelClient = { + list: () => + Effect.suspend(() => { + listCalls += 1; + return Effect.fail({ _tag: "TooManyRequests", message: "Rate limited" }); + }), + } as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient; + const sharedPause = makeSharedCloudflareApiPause(); + const isolate = () => + ManagedEndpointProvider.layerCloudflareBindings( + tunnelClient, + {} as Cloudflare.DNS.ReadWriteDnsClient, + {} as Alchemy.BaseRuntimeContext, + ).pipe( + Layer.provideMerge(NodeServices.layer), + Layer.provide(RelayConfiguration.layer(config)), + Layer.provide( + Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()), + ), + Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())), + Layer.provide(Layer.succeed(CloudflareApiPause.CloudflareApiPause, sharedPause)), + ); + const provision = ManagedEndpointProvider.ManagedEndpointProvider.pipe( + Effect.flatMap((provider) => + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ), + Effect.flip, + ); + + return Effect.gen(function* () { + const rateLimited = yield* provision.pipe(Effect.provide(isolate())); + const pausedElsewhere = yield* provision.pipe(Effect.provide(isolate())); + expect(listCalls).toBe(1); + expect(ManagedEndpointProvider.isCloudflareRateLimited(rateLimited)).toBe(true); + expect(pausedElsewhere).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "ensure-tunnel", + cause: { cause: { _tag: "CloudflareApiPaused" } }, + }); + + yield* TestClock.adjust("30 seconds"); + yield* provision.pipe(Effect.provide(isolate())); + expect(listCalls).toBe(2); + }); + }); + it.effect("recreates a checkpointed DNS record when it was removed externally", () => { const dnsCalls: DnsCall[] = []; const allocationCalls: AllocationCall[] = []; diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index bb906db16a8fa..6a5d1f8bc3c9a 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -1,9 +1,13 @@ +import * as CloudflareRetry from "@distilled.cloud/cloudflare/Retry"; import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Arr from "effect/Array"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Hex from "effect/encoding/Hex"; import * as Layer from "effect/Layer"; @@ -23,6 +27,7 @@ import { managedEndpointHostname, managedEndpointTunnelName, } from "../deploymentConfig.ts"; +import * as CloudflareApiPause from "./CloudflareApiPause.ts"; import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; import * as ManagedTunnelLimits from "./ManagedTunnelLimits.ts"; @@ -148,6 +153,11 @@ export class ManagedEndpointProvider extends Context.Service< readonly userId: string; readonly environmentId: string; readonly origin: RelayManagedEndpointOrigin; + /** + * Return a ready tunnel that already serves this origin with only a + * liveness check and a fresh token, skipping ingress and DNS repair. + */ + readonly reuseReadyTunnel?: boolean; }) => Effect.Effect; readonly reconcileOrigin: (input: { readonly userId: string; @@ -389,6 +399,132 @@ export function isManagedEndpointNotFound(cause: unknown): boolean { return "cause" in cause && isManagedEndpointNotFound(cause.cause); } +export class CloudflareApiPaused extends Schema.TaggedError()( + "CloudflareApiPaused", + { retryAfterMillis: Schema.Number }, +) { + override get message(): string { + return `Cloudflare API calls are paused for ${this.retryAfterMillis}ms after a rate limit`; + } +} + +export function isCloudflareRateLimited(cause: unknown): boolean { + if (typeof cause !== "object" || cause === null) { + return false; + } + if ( + "_tag" in cause && + (cause._tag === "TooManyRequests" || + cause._tag === "CloudflareRateLimited" || + cause._tag === "CloudflareApiPaused") + ) { + return true; + } + if ("status" in cause && cause.status === 429) { + return true; + } + return "cause" in cause && isCloudflareRateLimited(cause.cause); +} + +const CLOUDFLARE_DEFAULT_RATE_LIMIT_PAUSE = Duration.seconds(30); +const CLOUDFLARE_MAX_RATE_LIMIT_PAUSE = Duration.minutes(2); + +function cloudflareRateLimitPauseMillis(cause: unknown): number { + const retryAfter = + typeof cause === "object" && cause !== null && "retryAfter" in cause + ? cause.retryAfter + : undefined; + const pause = Duration.isDuration(retryAfter) ? retryAfter : CLOUDFLARE_DEFAULT_RATE_LIMIT_PAUSE; + return Duration.toMillis(Duration.min(pause, CLOUDFLARE_MAX_RATE_LIMIT_PAUSE)); +} + +/** + * How long a caller should wait before retrying, when `cause` is a Cloudflare + * rate limit or a call refused during the pause that follows one. + */ +export function cloudflareRetryAfterMillis(cause: unknown): number | undefined { + for (let current = cause, depth = 0; depth < 8; depth++) { + if (typeof current !== "object" || current === null) return undefined; + if ("_tag" in current && current._tag === "CloudflareApiPaused") { + const retryAfterMillis: unknown = Reflect.get(current, "retryAfterMillis"); + return typeof retryAfterMillis === "number" ? retryAfterMillis : undefined; + } + if ( + ("_tag" in current && + (current._tag === "TooManyRequests" || current._tag === "CloudflareRateLimited")) || + ("status" in current && current.status === 429) + ) { + return cloudflareRateLimitPauseMillis(current); + } + if (!("cause" in current)) return undefined; + current = current.cause; + } + return undefined; +} + +// The SDK's default policy sleeps through Retry-After inside the request, which +// held every caller until its timeout and kept the account over its limit. +// Rate limits surface at once instead; other transient failures still retry. +const withoutRateLimitRetries: CloudflareRetry.Factory = (lastError) => { + const defaults = CloudflareRetry.makeDefault(lastError); + return { + ...defaults, + while: (error) => !isCloudflareRateLimited(error) && (defaults.while?.(error) ?? false), + }; +}; + +// Every isolate rereads the shared pause at most this often, so a quiet +// period costs one query per isolate rather than one per Cloudflare call. +const CLOUDFLARE_PAUSE_REFRESH_MILLIS = 5_000; + +/** + * The rate limit is account-wide, so once one call is rejected every other + * call is too, from any isolate. Failing them without a request leaves the + * account room to recover instead of extending the limit. + */ +const makeCloudflareApiGate = (alchemyRuntimeContext: Alchemy.BaseRuntimeContext) => { + let pausedUntilMillis = 0; + let checkedAtMillis = Number.NEGATIVE_INFINITY; + return (pause: CloudflareApiPause.CloudflareApiPause["Service"]) => { + const recordRateLimit = Effect.fn("relay.cloudflare_api_pause.rate_limited")(function* ( + cause: unknown, + ) { + const pauseMillis = cloudflareRateLimitPauseMillis(cause); + const until = (yield* Clock.currentTimeMillis) + pauseMillis; + pausedUntilMillis = Math.max(pausedUntilMillis, until); + yield* Effect.annotateCurrentSpan({ "relay.cloudflare.pause_ms": pauseMillis }); + yield* pause.pauseUntil(until); + }); + return ( + call: Effect.Effect, + ): Effect.Effect => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (now >= pausedUntilMillis && now - checkedAtMillis >= CLOUDFLARE_PAUSE_REFRESH_MILLIS) { + checkedAtMillis = now; + // Read first: a 429 recorded while the read is in flight must win. + const shared = yield* pause.pausedUntilMillis; + pausedUntilMillis = Math.max(pausedUntilMillis, shared); + } + // A paused call never reaches Cloudflare, so it leaves no 429 behind. + // Its own span keeps a storm visible to the rate-limit monitors. + if (now < pausedUntilMillis) { + return yield* new CloudflareApiPaused({ retryAfterMillis: pausedUntilMillis - now }).pipe( + Effect.withSpan("relay.cloudflare_api_pause.rejected"), + ); + } + return yield* call.pipe( + Effect.provideService(CloudflareRetry.Retry, withoutRateLimitRetries), + Effect.timeout("8 seconds"), + Effect.tapError((cause) => + isCloudflareRateLimited(cause) ? recordRateLimit(cause) : Effect.void, + ), + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + ); + }); + }; +}; + /** * Cloudflare refuses to delete a tunnel while a connector is still attached, * either one that has not finished draining or another runtime still serving @@ -503,6 +639,65 @@ export const make = Effect.gen(function* () { ); }); + // Hosts ask for recovery whenever their connector cannot register, mostly + // for local reasons a fresh provision cannot fix. Those requests outnumber + // every other Cloudflare caller, so a tunnel that is ready for the same + // origin costs two reads instead of a full provision. + const reuseReadyTunnel = Effect.fnUntraced(function* ( + allocation: ManagedEndpointAllocations.ManagedEndpointAllocation, + origin: RelayManagedEndpointOrigin, + ) { + const { tunnelId } = allocation; + if ( + tunnelId === null || + allocation.readyAt === null || + allocation.dnsRecordId === null || + allocation.tunnelReleasedAt !== null || + allocation.origin?.localHttpHost !== origin.localHttpHost || + allocation.origin.localHttpPort !== origin.localHttpPort + ) { + return Option.none(); + } + const failed = ( + stage: "verify-tunnel" | "get-tunnel-token", + cause: ManagedEndpointTunnelClientError, + ) => + new ManagedEndpointProvisioningFailed({ + userId: allocation.userId, + environmentId: allocation.environmentId, + stage, + hostname: allocation.hostname, + tunnelName: allocation.tunnelName, + tunnelId, + cause, + }); + const recorded = yield* tunnels.get(tunnelId).pipe( + Effect.asSome, + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeedNone + : Effect.fail(failed("verify-tunnel", cause)), + }), + ); + if (Option.isNone(recorded) || recorded.value.deletedAt) { + return Option.none(); + } + const connectorToken = yield* tunnels + .getToken(tunnelId) + .pipe(Effect.mapError((cause) => failed("get-tunnel-token", cause))); + yield* Effect.annotateCurrentSpan({ "relay.managed_endpoint.reused_ready_tunnel": true }); + return Option.some({ + endpoint: managedEndpointForHostname(allocation.hostname), + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken, + tunnelId, + tunnelName: allocation.tunnelName, + }, + }); + }); + const prepareDeprovision = Effect.fn("relay.managed_endpoint_provider.prepare_deprovision")( function* (input: { readonly userId: string; readonly environmentId: string }) { return yield* allocations.get(input).pipe( @@ -1039,6 +1234,13 @@ export const make = Effect.gen(function* () { ); const { hostname, tunnelName } = allocation; + if (input.reuseReadyTunnel === true) { + const reused = yield* reuseReadyTunnel(allocation, input.origin); + if (Option.isSome(reused)) { + return reused.value; + } + } + const tunnelResponse = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( Effect.map((tunnels) => tunnels.result), Effect.map(Arr.findFirst((tunnel) => tunnel.name === tunnelName)), @@ -1344,153 +1546,142 @@ export const layerCloudflareBindings = ( tunnelClient: Cloudflare.Tunnel.ReadWriteTunnelClient, dnsClient: Cloudflare.DNS.ReadWriteDnsClient, alchemyRuntimeContext: Alchemy.BaseRuntimeContext, -) => - layer.pipe( +) => { + // Created once per isolate so its cached pause survives layer rebuilds. + const cloudflare = makeCloudflareApiGate(alchemyRuntimeContext); + return layer.pipe( Layer.provideMerge( - Layer.mergeAll( - layerTunnelClient({ - get: (tunnelId) => - tunnelClient.get(tunnelId).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointTunnelClientError({ - operation: "get", - tunnelId, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - list: (request) => - tunnelClient.list(request).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointTunnelClientError({ - operation: "list", - ...(request.name === undefined ? {} : { tunnelName: request.name }), - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - create: (request) => - tunnelClient.create(request).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointTunnelClientError({ - operation: "create", - tunnelName: request.name, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - putConfiguration: (tunnelId, config) => - tunnelClient.putConfiguration(tunnelId, config).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointTunnelClientError({ - operation: "put-configuration", - tunnelId, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - getToken: (tunnelId) => - tunnelClient.getToken(tunnelId).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointTunnelClientError({ - operation: "get-token", - tunnelId, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - delete: (tunnelId) => - tunnelClient.delete(tunnelId).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointTunnelClientError({ - operation: "delete", - tunnelId, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - }), - layerDnsClient({ - listRecords: (hostname) => - dnsClient.listDnsRecords({ search: hostname }).pipe( - Effect.timeout("8 seconds"), - Effect.map((response) => - response.result.filter( - (record): record is typeof record & { readonly id: string } => - typeof record.id === "string" && - normalizeHostname(record.name) === normalizeHostname(hostname), + Layer.unwrap( + Effect.gen(function* () { + const pause = yield* CloudflareApiPause.CloudflareApiPause; + const gate = cloudflare(pause); + return Layer.mergeAll( + layerTunnelClient({ + get: (tunnelId) => + gate(tunnelClient.get(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause, + }), + ), ), - ), - Effect.mapError( - (cause) => - new ManagedEndpointDnsClientError({ - operation: "list-records", - hostname, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - createRecord: (request) => - dnsClient.createDnsRecord(request).pipe( - Effect.timeout("8 seconds"), - Effect.map((response) => ({ id: response.id })), - Effect.mapError( - (cause) => - new ManagedEndpointDnsClientError({ - operation: "create-record", - hostname: request.name, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - updateRecord: (dnsRecordId, request) => - dnsClient.updateDnsRecord(dnsRecordId, request).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointDnsClientError({ - operation: "update-record", - hostname: request.name, - dnsRecordId, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), - deleteRecord: (dnsRecordId) => - dnsClient.deleteDnsRecord(dnsRecordId).pipe( - Effect.timeout("8 seconds"), - Effect.mapError( - (cause) => - new ManagedEndpointDnsClientError({ - operation: "delete-record", - dnsRecordId, - cause, - }), - ), - Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), - ), + list: (request) => + gate(tunnelClient.list(request)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "list", + ...(request.name === undefined ? {} : { tunnelName: request.name }), + cause, + }), + ), + ), + create: (request) => + gate(tunnelClient.create(request)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "create", + tunnelName: request.name, + cause, + }), + ), + ), + putConfiguration: (tunnelId, config) => + gate(tunnelClient.putConfiguration(tunnelId, config)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "put-configuration", + tunnelId, + cause, + }), + ), + ), + getToken: (tunnelId) => + gate(tunnelClient.getToken(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "get-token", + tunnelId, + cause, + }), + ), + ), + delete: (tunnelId) => + gate(tunnelClient.delete(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId, + cause, + }), + ), + ), + }), + layerDnsClient({ + listRecords: (hostname) => + gate(dnsClient.listDnsRecords({ search: hostname })).pipe( + Effect.map((response) => + response.result.filter( + (record): record is typeof record & { readonly id: string } => + typeof record.id === "string" && + normalizeHostname(record.name) === normalizeHostname(hostname), + ), + ), + Effect.mapError( + (cause) => + new ManagedEndpointDnsClientError({ + operation: "list-records", + hostname, + cause, + }), + ), + ), + createRecord: (request) => + gate(dnsClient.createDnsRecord(request)).pipe( + Effect.map((response) => ({ id: response.id })), + Effect.mapError( + (cause) => + new ManagedEndpointDnsClientError({ + operation: "create-record", + hostname: request.name, + cause, + }), + ), + ), + updateRecord: (dnsRecordId, request) => + gate(dnsClient.updateDnsRecord(dnsRecordId, request)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDnsClientError({ + operation: "update-record", + hostname: request.name, + dnsRecordId, + cause, + }), + ), + ), + deleteRecord: (dnsRecordId) => + gate(dnsClient.deleteDnsRecord(dnsRecordId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDnsClientError({ + operation: "delete-record", + dnsRecordId, + cause, + }), + ), + ), + }), + ); }), ), ), ); +}; diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 252b468e7cc79..26ce55a58a738 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -66,6 +66,7 @@ function harness(input?: { readonly namespace?: string; readonly failTunnelId?: string; readonly rateLimitedTunnelId?: string; + readonly rateLimitCause?: unknown; readonly failAllDeletes?: boolean; readonly failDeleteWhen?: (tunnelId: string) => boolean; readonly missingOnDeleteTunnelId?: string; @@ -155,13 +156,13 @@ function harness(input?: { tunnelId === input?.missingOnDeleteTunnelId ? { _tag: "NotFound" } : tunnelId === input?.rateLimitedTunnelId - ? { + ? (input.rateLimitCause ?? { cause: { _tag: "TooManyRequests", message: "Cloudflare rate limit exceeded", retryAfter: 60, }, - } + }) : "Cloudflare refused the deletion", }), ) @@ -653,7 +654,13 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); - it.effect("stops the sweep after a structured Cloudflare rate limit error", () => { + it.effect.each([ + ["a structured Cloudflare rate limit error", undefined], + [ + "a Cloudflare API pause recorded by another isolate", + new ManagedEndpointProvider.CloudflareApiPaused({ retryAfterMillis: 30_000 }), + ], + ] as const)("stops the sweep after %s", ([, rateLimitCause]) => { const state = harness({ tunnels: [ tunnel({ @@ -674,6 +681,7 @@ describe("ManagedEndpointReaper", () => { allocation({ tunnelId: "next", recoveryEnabled: true }), ], rateLimitedTunnelId: "limited", + rateLimitCause, }); return Effect.gen(function* () { diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 27c3574ff47a1..cbaf63577e739 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -110,19 +110,6 @@ function inactiveDaysAt( return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000; } -function isRateLimited(cause: unknown): boolean { - if (typeof cause !== "object" || cause === null) { - return false; - } - if ("_tag" in cause && cause._tag === "TooManyRequests") { - return true; - } - if ("status" in cause && cause.status === 429) { - return true; - } - return "cause" in cause && isRateLimited(cause.cause); -} - function rotatedPages(input: { readonly totalCount: number | undefined; readonly slot: number; @@ -383,7 +370,7 @@ export const make = Effect.gen(function* () { legacy: candidate.legacy, cause: result.failure, }); - if (isRateLimited(result.failure)) { + if (ManagedEndpointProvider.isCloudflareRateLimited(result.failure)) { stopDeleting = true; truncated = true; } diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 81574b0392248..9455cf36ca4b4 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -187,6 +187,102 @@ describe("device listing compatibility", () => { }); }); +describe("Cloudflare rate-limit retry hint", () => { + const provisioningFailed = (cause?: unknown) => + new ManagedEndpointProvider.ManagedEndpointProvisioningFailed({ + userId: "user-1", + environmentId: "env-1", + stage: "ensure-tunnel", + ...(cause === undefined + ? {} + : { + cause: new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "list", + cause, + }), + }), + }); + + const linkResponse = (failure: ManagedEndpointProvider.ManagedEndpointProvisioningFailed) => { + const layerHandlers = RelayHttpApi.layerClientApi.pipe( + HttpRouter.provideRequest( + Layer.mergeAll( + Layer.mock(EnvironmentCredentials.EnvironmentCredentials, {}), + Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), + Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), + Layer.mock(RelayDb.RelayTransactions, {}), + Layer.mock(HookInbox.HookInbox, {}), + ), + ), + Layer.provide( + Layer.mergeAll( + Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings), + NodeCryptoLayer.layer, + Layer.mock(RelayTokens.RelayTokens, { resolveDpopAccessTokenScopes: () => null }), + Layer.mock(EnvironmentLinker.EnvironmentLinker, { link: () => Effect.fail(failure) }), + Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), + Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), + Layer.mock(Devices.Devices, {}), + ), + ), + Layer.provideMerge( + Layer.succeed(RelayClientAuth, { + clientBearer: (effect) => + Effect.provideService(effect, RelayClientPrincipal, { + userId: "user-1", + token: "test-token", + }), + }), + ), + ); + return Effect.acquireRelease( + Effect.sync(() => + HttpRouter.toWebHandler( + HttpApiBuilder.layer(HttpApi.make("RelayApi").add(RelayApi.groups.client)).pipe( + Layer.provide(layerHandlers), + Layer.provide(HttpServer.layerServices), + ), + { disableLogger: true }, + ), + ), + (app) => Effect.promise(() => app.dispose()), + ).pipe( + Effect.flatMap((app) => + Effect.promise(() => + app.handler( + new Request("https://relay.example.test/v1/client/environment-links", { + method: "POST", + headers: { authorization: "Bearer test-token", "content-type": "application/json" }, + body: JSON.stringify({ + proof: "signed-proof", + notificationsEnabled: true, + liveActivitiesEnabled: true, + managedTunnelsEnabled: true, + }), + }), + ), + ), + ), + ); + }; + + it.effect("tells a link refused during a Cloudflare pause when to retry", () => + Effect.gen(function* () { + const paused = yield* linkResponse( + provisioningFailed( + new ManagedEndpointProvider.CloudflareApiPaused({ retryAfterMillis: 24_100 }), + ), + ); + expect(paused.status).toBe(503); + expect(paused.headers.get("retry-after")).toBe("25"); + + const other = yield* linkResponse(provisioningFailed()); + expect(other.status).toBe(503); + expect(other.headers.get("retry-after")).toBeNull(); + }).pipe(Effect.scoped), + ); +}); + describe("relay client authentication", () => { it.effect("preserves the existing Clerk session JWT path", () => Effect.gen(function* () { diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 8db3a4054d641..69925a3645311 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -102,6 +102,17 @@ const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler ), ); +// T3 Code hosts never retry sooner than Retry-After, so during a Cloudflare +// rate limit they wait out the pause instead of retrying into it. +const appendCloudflareRetryAfter = (error: unknown) => { + const retryAfterMillis = ManagedEndpointProvider.cloudflareRetryAfterMillis(error); + if (retryAfterMillis === undefined) return Effect.void; + const seconds = String(Math.max(1, Math.ceil(retryAfterMillis / 1000))); + return HttpEffect.appendPreResponseHandler((_request, response) => + Effect.succeed(HttpServerResponse.setHeader(response, "retry-after", seconds)), + ); +}; + const appendRelayDpopChallengeHeader = HttpEffect.appendPreResponseHandler((_request, response) => Effect.succeed( response.status === 401 @@ -693,6 +704,7 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( userId: input.userId, environmentId: input.environmentId, origin: input.origin, + reuseReadyTunnel: true, }); const recoveredTunnelId = recovered.runtime.tunnelId; if ( @@ -873,6 +885,7 @@ export const layerClientApi = HttpApiBuilder.group( cloudMintPublicKey: config.cloudMintPublicKey, }; }, + Effect.tapError(appendCloudflareRetryAfter), mapErrorTags({ EnvironmentLinkProofExpired: (_error, traceId) => new RelayEnvironmentLinkProofExpiredError({ @@ -1334,6 +1347,7 @@ export const layerServerApi = HttpApiBuilder.group( origin: payload.origin, }); }, + Effect.tapError(appendCloudflareRetryAfter), Effect.catchTags({ ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), ManagedEndpointProvisioningNotConfigured: () => @@ -1369,6 +1383,7 @@ export const layerServerApi = HttpApiBuilder.group( origin: payload.origin, }); }, + Effect.tapError(appendCloudflareRetryAfter), Effect.catchTags({ ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), ManagedEndpointProvisioningNotConfigured: () => diff --git a/infra/relay/src/monitoring.ts b/infra/relay/src/monitoring.ts index 6577dab61cf95..3ed521de7e2de 100644 --- a/infra/relay/src/monitoring.ts +++ b/infra/relay/src/monitoring.ts @@ -119,9 +119,9 @@ const relayMonitors: ReadonlyArray = [ severity: "page", name: "T3 Connect relay: Cloudflare API rate limited", description: - "Cloudflare API 429s per 10 min from tunnel and DNS provisioning. Sustained at this level, links and recoveries stall. Peak before #16810: 1,286; during it: ~11k.", + "Cloudflare API 429s, plus calls the relay refused while paused after one, per 10 min. Sustained at this level, links and recoveries stall. Peak 429s before #16810: 1,286; during it: ~11k.", query: (dataset) => - `['${dataset}']\n| where ${CLOUDFLARE_API} and ${STATUS} == 429\n| summarize rate_limited = count()`, + `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`, operator: "AboveOrEqual", threshold: 500, rangeMinutes: 10, @@ -145,9 +145,9 @@ const relayMonitors: ReadonlyArray = [ severity: "warn", name: "T3 Connect relay: Cloudflare API near its limit", description: - "Any sustained Cloudflare API 429s. These bursts ran for days before #16810; raise the account limit or cut call volume before they grow.", + "Any sustained Cloudflare API 429s or paused calls. 429 bursts ran for days before #16810; raise the account limit or cut call volume before they grow.", query: (dataset) => - `['${dataset}']\n| where ${CLOUDFLARE_API} and ${STATUS} == 429\n| summarize rate_limited = count()`, + `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`, operator: "AboveOrEqual", threshold: 20, rangeMinutes: 10, @@ -250,7 +250,14 @@ const relayDashboardCharts = (dataset: string): ReadonlyArray => { timeSeries( "cloudflare-api", "Cloudflare API calls by status", - `${source}\n| where ${CLOUDFLARE_API}\n| summarize count() by bin_auto(_time), status = ${STATUS}`, + // Calls refused during a rate-limit pause never reach Cloudflare, so + // they are their own series or a storm would look like a quiet period. + `${source}\n| where ${CLOUDFLARE_API} or name == "relay.cloudflare_api_pause.rejected"\n| summarize count() by bin_auto(_time), status = case(name == "relay.cloudflare_api_pause.rejected", "paused", isnull(${STATUS}), "no response", tostring(${STATUS}))`, + ), + timeSeries( + "provision-paths", + "Tunnel provisions by path", + `${source}\n| where name == "relay.managed_endpoint_provider.provision"\n| summarize reused_ready_tunnel = countif(error != true and tobool(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel']) == true), full_provision = countif(error != true and isnull(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel'])), failed = countif(error == true) by bin_auto(_time)`, ), timeSeries( "postgres", diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 3ed6a75a5f6d6..132ceba16a900 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -5,6 +5,7 @@ import type { RelayManagedEndpointOrigin, } from "@t3tools/contracts/relay"; import { + bigint, boolean, index, integer, @@ -111,6 +112,13 @@ export const relayManagedEndpointAllocations = pgTable( ], ); +// One row per upstream API whose rate limit is shared by every Worker +// isolate, so a 429 seen by one isolate pauses the rest. +export const relayUpstreamApiPauses = pgTable("relay_upstream_api_pauses", { + api: varchar("api", { length: 64 }).primaryKey(), + pausedUntilMs: bigint("paused_until_ms", { mode: "number" }).notNull(), +}); + export const relayManagedTunnelLimits = pgTable("relay_managed_tunnel_limits", { userId: varchar("user_id", { length: 191 }).primaryKey(), maxTunnels: integer("max_tunnels").notNull(), diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 7e381b23f7d20..2d3e388ceb10b 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -57,6 +57,7 @@ import * as ApnsClient from "./agentActivity/ApnsClient.ts"; import * as ApnsProviderTokens from "./agentActivity/ApnsProviderTokens.ts"; import * as ApnsDeliveryQueue from "./agentActivity/ApnsDeliveryQueue.ts"; import * as ApnsDeliveries from "./agentActivity/ApnsDeliveries.ts"; +import * as CloudflareApiPause from "./environments/CloudflareApiPause.ts"; import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts"; import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts"; import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts"; @@ -291,6 +292,7 @@ export const layer = Api.make( Layer.provideMerge(EnvironmentCredentials.layer), Layer.provideMerge( Layer.mergeAll( + CloudflareApiPause.layer, EnvironmentLinks.layer, ManagedEndpointAllocations.layer, ManagedTunnelLimits.layer, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed2566a4e2294..3744c9e1e38a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,9 @@ settings: catalogs: default: + '@distilled.cloud/cloudflare': + specifier: 1.0.0-rc.13 + version: 1.0.0-rc.13 '@effect/graphql-generator': specifier: 4.0.5 version: 4.0.5 @@ -1064,6 +1067,9 @@ importers: '@clerk/backend': specifier: 3.25.0 version: 3.25.0(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + '@distilled.cloud/cloudflare': + specifier: 'catalog:' + version: 1.0.0-rc.13(effect@4.0.5) '@effect/sql-pg': specifier: 4.0.5 version: 4.0.5(effect@4.0.5) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ac0a089fb9504..12ec4bc1b6e1c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,6 +34,9 @@ catalog: "@clerk/expo": 4.13.0 "@clerk/react": 6.17.8 "@clerk/shared": 4.40.0 + # Must match the version alchemy depends on: the relay overrides its + # Cloudflare retry policy through this package's Retry service. + "@distilled.cloud/cloudflare": 1.0.0-rc.13 "@effect/atom-react": 4.0.5 "@effect/graphql-generator": 4.0.5 "@effect/openapi-generator": 4.0.5 From 143ab1606dc6d9fbbd91869904a4c19fe3b694e8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 20:57:59 -0700 Subject: [PATCH 002/113] fix(acp): settle native foreground work independently (#21616) Co-authored-by: Wout Stiens <71498452+StiensWout@users.noreply.github.com> --- packages/provider-acp/src/server/adapter.ts | 402 ++++++++++++++++---- 1 file changed, 330 insertions(+), 72 deletions(-) diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 5aaac47c943bb..70951b1ef312b 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -327,6 +327,8 @@ export interface AcpAdapterV2Flavor { readonly terminateRuntimeProcessGroupOnInterrupt?: boolean; readonly preserveRuntimeOnSettledInterrupt?: boolean; readonly enablePostSettleContinuation?: boolean; + /** Settle on native idle after the accepted user echo, only on negotiated ACP v2. */ + readonly followAgentWorkState?: boolean; /** * Grok CLI currently accepts and vision-processes image blocks while still * reporting the capability as false; without this override, screenshot turns @@ -1039,6 +1041,7 @@ interface ActiveAcpTurn { finalizedStatus: "completed" | "interrupted" | "failed" | "cancelled" | null; promptSettled: boolean; promptSettledStatus: "completed" | "interrupted" | "failed" | "cancelled" | null; + workAccepted: boolean; readonly promptWireSettled: Deferred.Deferred; backgroundFinalizeGeneration: number; } @@ -1493,6 +1496,14 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( input.runtimePolicy; const events = yield* Queue.unbounded(); const activeTurn = yield* Ref.make(null); + let pendingStopCloseTurn: ActiveAcpTurn | null = null; + let followsAgentWorkState = false; + let agentWork: Extract< + EffectAcpSchema.SessionUpdate, + { sessionUpdate: "state_update" } + > | null = null; + const agentRequests = new Set(); + let turnAttached = yield* Deferred.make(); const activeSessionId = yield* Ref.make(null); const contextUsageBySessionId = yield* Ref.make( new Map(), @@ -1510,6 +1521,17 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } | null>(null); const activeSessionSetup = yield* Ref.make(null); + const setActiveSessionSetup = Effect.fnUntraced(function* ( + setup: AcpSessionRuntime.AcpSessionRuntimeStartResult, + ) { + followsAgentWorkState = + flavor.followAgentWorkState === true && + continuationRequests !== undefined && + setup.initializeResult.protocolVersion === 2; + agentWork = null; + agentRequests.clear(); + yield* Ref.set(activeSessionSetup, setup); + }); const activeSelection = yield* Ref.make(null); const activeInteractionMode = yield* Ref.make(null); const promptInstructionStates = yield* Ref.make(new Map()); @@ -1879,7 +1901,18 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const handleRuntimeTerminationAtGeneration = (runtimeGeneration: number) => runRuntimeCallbackAtGeneration( runtimeGeneration, - Ref.set(runtimeRestartRequired, true), + Effect.gen(function* () { + yield* Ref.set(runtimeRestartRequired, true); + if (!followsAgentWorkState) return; + agentWork = { sessionUpdate: "state_update", state: "idle", stopReason: "error" }; + const context = yield* Ref.get(activeTurn); + if (context !== null) + yield* finalizeTurn( + context, + "failed", + makeProviderFailure({ class: "transport_error" }), + ); + }), ).pipe(Effect.asVoid); const makeRuntimeInput = ( runtimeGeneration: number, @@ -3816,6 +3849,89 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return true; }); + const settleIdleWork = Effect.fnUntraced(function* (context: ActiveAcpTurn) { + if ( + !followsAgentWorkState || + context.finalized || + (!context.workAccepted && !context.interrupted) || + agentWork?.state !== "idle" || + agentRequests.size > 0 + ) + return; + const work = agentWork; + // Native close sends cancelled idle before acknowledging the user's Stop. + if (work.stopReason === "cancelled" && pendingStopCloseTurn === context) { + context.interrupted = true; + } + const failure = + work.stopReason === "error" + ? (flavor.promptFailure?.( + new EffectAcpErrors.AcpRequestError({ + code: work.error?.code ?? -32603, + errorMessage: work.error?.message ?? "Provider turn failed.", + ...(work.error?.data === undefined ? {} : { data: work.error.data }), + }), + ) ?? makeProviderFailure({ message: work.error?.message, class: "provider_error" })) + : undefined; + yield* finalizeTurn( + context, + failure !== undefined + ? "failed" + : work.stopReason === "cancelled" + ? "cancelled" + : "completed", + failure, + ); + }); + + const awaitRequestTurn = Effect.fnUntraced(function* (generation: number) { + if (!followsAgentWorkState) return true; + while (true) { + const waiting = yield* runRuntimeCallbackAtGeneration( + generation, + Effect.gen(function* () { + if (yield* Ref.get(stoppedRunQuarantine)) return false; + if ((yield* Ref.get(activeTurn)) !== null) return null; + const sessionId = yield* Ref.get(activeSessionId); + if (sessionId !== null) yield* offerContinuationRun(sessionId); + return turnAttached; + }), + ); + if (Option.isNone(waiting)) return yield* Effect.never; + if (waiting.value === false) return false; + if (waiting.value === null) return true; + if (!(yield* Deferred.await(waiting.value))) return false; + } + }); + + const holdingAgentRequest = + (generation: number) => + (effect: Effect.Effect) => + Effect.suspend(() => + !followsAgentWorkState + ? effect + : Effect.acquireUseRelease( + runRuntimeCallbackAtGeneration( + generation, + Effect.sync(() => { + const token = {}; + agentRequests.add(token); + return token; + }), + ), + (token) => (Option.isNone(token) ? Effect.never : effect), + (token) => + runRuntimeCallbackAtGeneration( + generation, + Effect.gen(function* () { + if (Option.isSome(token)) agentRequests.delete(token.value); + const context = yield* Ref.get(activeTurn); + if (context !== null) yield* settleIdleWork(context); + }), + ).pipe(Effect.asVoid), + ), + ); + const handleSessionUpdate = Effect.fnUntraced(function* ( notification: EffectAcpSchema.SessionNotification, ) { @@ -3878,7 +3994,42 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( providerThread, }); } - return; + if (update.sessionUpdate !== "state_update") return; + } + if ( + followsAgentWorkState && + !(yield* Ref.get(stoppedRunQuarantine)) && + notification.sessionId === (yield* Ref.get(activeSessionId)) + ) { + if (update.sessionUpdate === "state_update") agentWork = update; + if ( + update.sessionUpdate === "user_message" || + update.sessionUpdate === "user_message_chunk" + ) { + agentWork = { sessionUpdate: "state_update", state: "running" }; + if (context !== null) { + context.workAccepted = true; + if (!context.finalized) return; + } + } + if ( + context !== null && + update.sessionUpdate === "state_update" && + update.state === "idle" + ) { + yield* settleIdleWork(context); + } else if ( + context === null && + !(yield* Ref.get(stoppedRunQuarantine)) && + (acpRootSessionUpdateIngestsOutput(notification) || + update.sessionUpdate === "user_message" || + update.sessionUpdate === "user_message_chunk" || + (update.sessionUpdate === "state_update" && update.state === "running")) + ) { + yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + yield* offerContinuationRun(notification.sessionId); + return; + } } if ( context?.finalized === true && @@ -4664,62 +4815,73 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } as const; }); - const requestUserInputWithAdmission = ( - generation: number, - request: Effect.Effect, - transportRequestId: string, - ) => - runRuntimeCallbackAtGeneration( - generation, - request.pipe( - Effect.flatMap((value) => - beginUserInputRequest(value, generation, transportRequestId), + const requestUserInputWithAdmission = Effect.fn( + "AcpAdapterV2.requestUserInputWithAdmission", + )( + function* ( + generation: number, + request: Effect.Effect, + transportRequestId: string, + ) { + if (!(yield* awaitRequestTurn(generation))) { + return { answers: null, acknowledgeNativeResponse: Effect.void }; + } + return yield* runRuntimeCallbackAtGeneration( + generation, + request.pipe( + Effect.flatMap((value) => + beginUserInputRequest(value, generation, transportRequestId), + ), ), - ), - ).pipe( - Effect.flatMap((pending) => { - if (Option.isNone(pending)) return Effect.never; - const { answers, requestId, transportRequestId } = pending.value; - return Deferred.await(answers).pipe( - Effect.flatMap((result) => - runRuntimeCallbackAtGeneration(generation, Effect.succeed(result)).pipe( - Effect.flatMap((checked) => - Option.isSome(checked) - ? Effect.succeed({ - acknowledgeNativeResponse: acknowledgeNativeResponse( - generation, - transportRequestId, - ), - answers: checked.value, - }) - : Effect.never, + ).pipe( + Effect.flatMap((pending) => { + if (Option.isNone(pending)) return Effect.never; + const { answers, requestId, transportRequestId } = pending.value; + return Deferred.await(answers).pipe( + Effect.flatMap((result) => + runRuntimeCallbackAtGeneration(generation, Effect.succeed(result)).pipe( + Effect.flatMap((checked) => + Option.isSome(checked) + ? Effect.succeed({ + acknowledgeNativeResponse: acknowledgeNativeResponse( + generation, + transportRequestId, + ), + answers: checked.value, + }) + : Effect.never, + ), ), ), - ), - Effect.ensuring( - runRuntimeCallbackAtGeneration( - generation, - Effect.gen(function* () { - yield* Ref.update(pendingRuntimeRequests, (current) => { - const updated = new Map(current); - updated.delete(String(requestId)); - return updated; - }); - }), - ).pipe(Effect.asVoid), - ), - ); - }), - Effect.mapError( - (cause) => - new EffectAcpErrors.AcpTransportError({ - detail: "Failed to handle ACP user input request", - cause, - }), - ), - ); + Effect.ensuring( + runRuntimeCallbackAtGeneration( + generation, + Effect.gen(function* () { + yield* Ref.update(pendingRuntimeRequests, (current) => { + const updated = new Map(current); + updated.delete(String(requestId)); + return updated; + }); + }), + ).pipe(Effect.asVoid), + ), + ); + }), + ); + }, + Effect.mapError( + (cause) => + new EffectAcpErrors.AcpTransportError({ + detail: "Failed to handle ACP user input request", + cause, + }), + ), + ); const cancelPendingRuntimeRequests = Effect.fnUntraced(function* () { + const waiting = turnAttached; + turnAttached = yield* Deferred.make(); + yield* Deferred.succeed(waiting, false); const requests = yield* Ref.modify(pendingRuntimeRequests, (current) => [ [...current.values()], new Map(), @@ -5310,13 +5472,20 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* userInput.acknowledgeNativeResponse; return response ?? ({ outcome: { outcome: "cancelled" } } as const); } + const dispositionFor = (policy: ProviderAdapter.ProviderAdapterV2RuntimePolicy) => + (flavor.permissionDisposition ?? acpPermissionDisposition)(policy, params); + if (followsAgentWorkState && dispositionFor(latestRuntimePolicy) === "ask") { + if (!(yield* awaitRequestTurn(handlerGeneration))) + return { outcome: { outcome: "cancelled" } } as const; + } const admitted = yield* runRuntimeCallbackAtGeneration( handlerGeneration, Effect.gen(function* () { - const context = yield* activeContext; - const disposition = (flavor.permissionDisposition ?? acpPermissionDisposition)( - context.input.runtimePolicy, - params, + const context = followsAgentWorkState + ? yield* Ref.get(activeTurn) + : yield* activeContext; + const disposition = dispositionFor( + context?.input.runtimePolicy ?? latestRuntimePolicy, ); if (disposition === "allow") { const optionId = selectAutoApprovedPermissionOption(params); @@ -5341,7 +5510,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return { _tag: "Pending" as const, pending: yield* beginApprovalRequest( - context, + yield* activeContext, params, handlerGeneration, transportRequestId, @@ -5426,6 +5595,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( cause, }), ), + holdingAgentRequest(handlerGeneration), ), ); yield* targetRuntime.handleElicitation((params, requestContext) => @@ -5561,7 +5731,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } as const); yield* userInput.acknowledgeNativeResponse; return response; - }), + }).pipe(holdingAgentRequest(handlerGeneration)), ); if (flavor.registerExtensions !== undefined) { yield* flavor.registerExtensions({ @@ -5857,7 +6027,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return yield* startAcpRuntime(input.threadId); }); yield* Ref.set(activeSessionId, started.sessionId); - yield* Ref.set(activeSessionSetup, started); + yield* setActiveSessionSetup(started); rememberTerminalEnvironment(started.sessionId, yield* readMcpContext(input.threadId)); const capabilities = negotiatedCapabilities(flavor.capabilities, started); const canLoadSession = started.initializeResult.agentCapabilities?.loadSession === true; @@ -6513,7 +6683,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (needsSessionActivation) { const activated = yield* activateSession(requestedSessionId, turnInput.threadId); yield* Ref.set(activeSessionId, activated.sessionId); - yield* Ref.set(activeSessionSetup, activated); + yield* setActiveSessionSetup(activated); yield* configureSession(activated, turnInput.modelSelection, turnInput.runtimePolicy); yield* Ref.set(activeSelection, turnInput.modelSelection); yield* Ref.set(activeInteractionMode, turnInput.runtimePolicy.interactionMode); @@ -6550,7 +6720,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( // Continuation turns attach to wake traffic the agent already produced // after the prior root turn settled; do not re-prompt the ACP session. const isContinuationTurn = - postSettleContinuationEnabled && acpIsProviderContinuationMessage(turnInput.message); + (postSettleContinuationEnabled || followsAgentWorkState) && + acpIsProviderContinuationMessage(turnInput.message); const isAppOwnedWakeTurn = acpIsAppOwnedWakeTurn(turnInput.message); if (!isAppOwnedWakeTurn) { yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); @@ -6629,6 +6800,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( finalizedStatus: null, promptSettled: false, promptSettledStatus: null, + workAccepted: false, promptWireSettled, backgroundFinalizeGeneration: 0, }; @@ -6696,6 +6868,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( createdAt: startedAt, updatedAt: startedAt, }); + const attached = turnAttached; + turnAttached = yield* Deferred.make(); + yield* Deferred.succeed(attached, true); // Every attach ends the deferred-terminal contract, but only the // queued provider continuation owns wake traffic. A user turn may // start before that continuation and must leave its buffer intact. @@ -6708,13 +6883,28 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( reports: new Map([...reports].filter(([key]) => !offered.has(key))), offered: noWakeReports.offered, })); - const drainedWakeCount = yield* Ref.modify(wakeBuffer, (current) => { + const takeWakeBuffer = Ref.modify(wakeBuffer, (current) => { const next: Array = []; return [ current.filter((notification) => notification.sessionId === requestedSessionId), next, ] as const; - }).pipe( + }); + if (followsAgentWorkState) { + yield* runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + const liveWork = agentWork; + yield* Effect.forEach(yield* takeWakeBuffer, handleSessionUpdate, { + discard: true, + }); + agentWork = liveWork; + context.workAccepted = true; + yield* settleIdleWork(context); + }), + ); + return; + } + const drainedWakeCount = yield* takeWakeBuffer.pipe( Effect.tap((drained) => Effect.forEach(drained, handleSessionUpdate, { concurrency: 1, @@ -6766,7 +6956,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( runRuntimeCallbackAtGeneration( promptGeneration, Effect.gen(function* () { - if (context.finalized) return; + if (context.finalized || followsAgentWorkState) return; const status = result.stopReason === "cancelled" ? context.interrupted @@ -6795,9 +6985,17 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( promptGeneration, Effect.gen(function* () { if (context.finalized) return; + const error = Cause.findError(cause); + // Native close rejects prompts cancelled before insertion with -32800. + const interrupted = + context.interrupted || + (pendingStopCloseTurn === context && + Result.isSuccess(error) && + error.success._tag === "AcpRequestError" && + error.success.code === -32800); yield* finalizeTurn( context, - context.interrupted ? "interrupted" : "failed", + interrupted ? "interrupted" : "failed", flavor.promptFailure?.(Cause.squash(cause)) ?? makeProviderFailure({ cause: Cause.squash(cause), @@ -6911,9 +7109,16 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( : flavor.getModelContextWindow?.(selection, session); }, }), - ...(postSettleContinuationEnabled + ...(postSettleContinuationEnabled || followsAgentWorkState ? { hasPendingBackgroundWork: Effect.gen(function* () { + if ( + followsAgentWorkState && + (agentWork?.state === "running" || + agentWork?.state === "requires_action" || + agentRequests.size > 0) + ) + return true; if ((yield* Ref.get(wakeBuffer)).length > 0) return true; if (yield* Ref.get(continuationRequested)) return true; if ((yield* Ref.get(runningBackgroundTaskIds)).size > 0) return true; @@ -7003,7 +7208,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( threadInput.providerThread.appThreadId, ).pipe(Effect.tapError(() => restorePreviousItemIdentity)); yield* Ref.set(activeSessionId, activated.sessionId); - yield* Ref.set(activeSessionSetup, activated); + yield* setActiveSessionSetup(activated); const nextSelection = threadInput.modelSelection ?? input.modelSelection; const nextRuntimePolicy = threadInput.runtimePolicy ?? input.runtimePolicy; yield* configureSession(activated, nextSelection, nextRuntimePolicy); @@ -7058,6 +7263,53 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return yield* Effect.gen(function* () { const transition = yield* Effect.gen(function* () { const interruptContext = yield* Ref.get(activeTurn); + if ( + followsAgentWorkState && + turnInput.requestRuntimeRestart === true && + (interruptContext === null || + interruptContext.providerTurnId === turnInput.providerTurnId) + ) { + // ACP cancel leaves native children alive; close confirms session-wide Stop. + pendingStopCloseTurn = interruptContext; + const closed = yield* runtime.closeSession().pipe( + Effect.timeoutOption("10 seconds"), + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: + "The agent refused to close its session; background work may still run", + cause, + }), + ), + ); + if (Option.isNone(closed)) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: + "The agent did not confirm session close; background work may still run", + }); + } + yield* runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + // Callbacks can settle the turn while close is awaiting confirmation. + const context = yield* Ref.get(activeTurn); + const carryover = yield* Ref.get(carryoverSubagents); + yield* Ref.set(stoppedRunQuarantine, true); + yield* Ref.set(runtimeRestartRequired, true); + agentWork = null; + agentRequests.clear(); + yield* quarantineStoppedRun(); + if (context !== null) context.interrupted = true; + yield* cancelPendingRuntimeRequests(); + yield* terminalizeCarryoverSubagents(carryover); + if (context !== null) { + yield* finalizeTurn(context, "interrupted"); + } + }), + ); + return undefined; + } const softSteerCandidate = flavor.preserveRuntimeOnSettledInterrupt === true && turnInput.requestRuntimeRestart !== true && @@ -7209,7 +7461,13 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( teardownBarrier, settledSoftInterrupt, }; - }); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + pendingStopCloseTurn = null; + }), + ), + ); if (transition === undefined) return; const { context, @@ -7407,7 +7665,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* readMcpContext(snapshotInput.providerThread.appThreadId), ); yield* Ref.set(activeSessionId, activated.sessionId); - yield* Ref.set(activeSessionSetup, activated); + yield* setActiveSessionSetup(activated); yield* Ref.set(activeSelection, null); yield* Ref.set(activeInteractionMode, null); } @@ -7469,7 +7727,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ); yield* Ref.set(runtimeRestartRequired, false); yield* Ref.set(activeSessionId, candidate.sessionId); - yield* Ref.set(activeSessionSetup, candidate); + yield* setActiveSessionSetup(candidate); yield* Ref.set(activeSelection, null); yield* Ref.set(activeInteractionMode, null); yield* Ref.set(promptInstructionStates, new Map()); @@ -7569,7 +7827,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* readMcpContext(forkInput.targetThreadId), ); yield* Ref.set(activeSessionId, forked.sessionId); - yield* Ref.set(activeSessionSetup, forked); + yield* setActiveSessionSetup(forked); yield* Ref.set(activeSelection, null); yield* Ref.set(activeInteractionMode, null); itemIdentityVersion = 2; From 64be3613011eaf38e9938b26fd9b6586f15c3438 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 21:05:19 -0700 Subject: [PATCH 003/113] fix(usage): retain transcript diagnostics across scans (#21862) --- apps/server/src/usage/UsageService.test.ts | 47 +++++++++++++ apps/server/src/usage/UsageService.ts | 59 +++++++++++++--- apps/server/src/usage/usageScanCache.test.ts | 31 +++++++++ apps/server/src/usage/usageScanCache.ts | 7 ++ .../src/usage/usageTranscriptReader.test.ts | 68 +++++++++++++++++++ .../server/src/usage/usageTranscriptReader.ts | 10 +-- packages/provider-core/src/server/usage.ts | 2 + 7 files changed, 210 insertions(+), 14 deletions(-) diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index 667a71dd0c398..a4b0a8b30cf97 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -19,6 +19,7 @@ import { type UsageSummaryInput, } from "@t3tools/contracts"; import * as Duration from "effect/Duration"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -42,6 +43,7 @@ import * as AntigravityUsage from "../provider/Drivers/AntigravityUsage.ts"; import * as ProviderHostLive from "../provider/ProviderHostLive.ts"; import { totalTokens, type UsageRecord } from "@t3tools/provider-core/server/usage"; import * as UsageService from "./UsageService.ts"; +import { encodeScanCache } from "./usageScanCache.ts"; const encodeUnknownJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const encodeUnknownJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); @@ -1881,6 +1883,51 @@ describe("UsageService", () => { }).pipe(Effect.scoped), ); + it.live("keeps an absent directory missing when its cached files only contain diagnostics", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => + NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }), + ); + yield* Effect.gen(function* () { + const { stateDir } = yield* ServerConfig.ServerConfig; + const mtimeMs = yield* Clock.currentTimeMillis; + const cache = encodeScanCache( + new Map([ + [ + transcript, + { + size: 0, + mtimeMs, + provider: "claude" as const, + records: [], + tailRecords: [], + malformedRecords: 2, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }, + ], + ]), + ); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(stateDir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(stateDir, "usage-scan-cache-v5.json"), + encodeUnknownJsonString(cache), + ); + }); + const summary = yield* (yield* UsageService.make).readSummary(WINDOW); + const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); + assert.strictEqual(source?.status, "missing"); + assert.strictEqual(source?.scannedFiles, 0); + assert.strictEqual(source?.malformedRecords, 2); + }).pipe( + Effect.provide( + layerService({ prefix: "usage-service-missing-diagnostics", home, settings }), + ), + ); + }).pipe(Effect.scoped), + ); + it.live("reports saved usage of a removed directory only for windows it reaches", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index c27321b51777f..b17d5cb306473 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -353,7 +353,7 @@ export const make = Effect.gen(function* () { ([filePath, entry]) => entry.provider === provider && entry.mtimeMs >= retentionCutoffMs && - entry.records.length + entry.tailRecords.length > 0 && + (entry.records.length + entry.tailRecords.length > 0 || entry.malformedRecords > 0) && isWithinDirectory(filePath, dir), ); const volumeId = @@ -459,6 +459,7 @@ export const make = Effect.gen(function* () { format: TranscriptUsageFormat, ): Effect.fn.Return<{ readonly records: readonly UsageRecord[]; + readonly malformedRecords: number; readonly failed?: true; readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined }; }> { @@ -476,6 +477,7 @@ export const make = Effect.gen(function* () { cached.tailRecords.length === 0 ? cached.records : [...cached.records, ...cached.tailRecords], + malformedRecords: cached.malformedRecords, }; } @@ -492,6 +494,7 @@ export const make = Effect.gen(function* () { if (parsed === null) return { records: cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : [], + malformedRecords: cached?.provider === provider ? cached.malformedRecords : 0, failed: true, }; @@ -502,8 +505,17 @@ export const make = Effect.gen(function* () { return { records: tailRecords.length === 0 ? records : [...records, ...tailRecords], + malformedRecords: parsed.malformedRecords, update: { - entry: { size, mtimeMs, provider, records, tailRecords, position: parsed.position }, + entry: { + size, + mtimeMs, + provider, + records, + tailRecords, + malformedRecords: parsed.malformedRecords, + position: parsed.position, + }, replaces: cached, }, }; @@ -518,7 +530,11 @@ export const make = Effect.gen(function* () { readonly message?: string; readonly action?: UsageSource["action"]; readonly files: - | readonly { readonly path: string; readonly records: readonly UsageRecord[] }[] + | readonly { + readonly path: string; + readonly records: readonly UsageRecord[]; + readonly malformedRecords: number; + }[] | null; readonly refreshing?: true; } @@ -543,8 +559,8 @@ export const make = Effect.gen(function* () { ), { concurrency: TRANSCRIPT_READ_CONCURRENCY }, ); - const parsedFiles = read.map(({ path, records, update }) => { - if (update === undefined) return { path, records }; + const parsedFiles = read.map(({ path, records, malformedRecords, update }) => { + if (update === undefined) return { path, records, malformedRecords }; // A scan of another window may have cached its own read of this file // meanwhile. Then keep whichever read saw the later file, so a slower // scan never replaces newer usage with older. @@ -557,7 +573,7 @@ export const make = Effect.gen(function* () { fileCache.set(path, update.entry); cacheDirty = true; } - return { path, records }; + return { path, records, malformedRecords }; }); const unread = failedPaths + read.filter((file) => file.failed).length; return { @@ -602,6 +618,7 @@ export const make = Effect.gen(function* () { : Effect.succeed(volumeId), (resolved): ScannedDir => ({ ...source, + files: source.files?.map((file) => ({ ...file, malformedRecords: 0 })) ?? null, provider: reader.provider, volumeId: resolved, }), @@ -710,7 +727,11 @@ export const make = Effect.gen(function* () { !isWithinDirectory(filePath, dir) ) continue; - retainedFiles.push({ path: filePath, records: [...entry.records, ...entry.tailRecords] }); + retainedFiles.push({ + path: filePath, + records: [...entry.records, ...entry.tailRecords], + malformedRecords: entry.malformedRecords, + }); } return retainedFiles; }); @@ -722,11 +743,13 @@ export const make = Effect.gen(function* () { ] of scannedDirs.entries()) { let scannedFiles = 0; let skippedFiles = 0; + let malformedRecords = 0; // Distinct per directory. Buckets carry per-cell session counts, but a // session spans days and models, so clients total this figure instead. const sessionIds = new Set(); for (const file of filesByDir[index] ?? []) { + malformedRecords += file.malformedRecords; if (file.records.length === 0) { skippedFiles += 1; continue; @@ -758,15 +781,31 @@ export const make = Effect.gen(function* () { } } + const sourceMessage = + message ?? (files === null ? "No transcript directory on this environment." : null); sources.push({ fingerprint: { hostId: sourceHostId ?? hostId, provider, resolvedHomePath: dir, volumeId }, - status: files === null && scannedFiles === 0 ? "missing" : (status ?? "ok"), + status: + files === null && scannedFiles === 0 + ? "missing" + : malformedRecords > 0 + ? status === undefined || status === "ok" + ? "partial" + : status + : (status ?? "ok"), scannedFiles, skippedFiles, - malformedRecords: 0, + malformedRecords, distinctSessions: sessionIds.size, message: - message ?? (files === null ? "No transcript directory on this environment." : null), + malformedRecords > 0 + ? [ + sourceMessage, + `${malformedRecords} malformed or inconsistent usage record(s) were skipped; usage may be incomplete.`, + ] + .filter(Boolean) + .join(" ") + : sourceMessage, ...(action ? { action } : {}), ...(refreshing ? { refreshing } : {}), }); diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index f7edc7f86b441..e58293a02a613 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -52,6 +52,7 @@ function cacheWith(entries: readonly [string, number, readonly UsageRecord[]][]) provider: "claude", records, tailRecords: [], + malformedRecords: 0, position: position(), }); } @@ -86,6 +87,7 @@ describe("scan cache round trip", () => { ]); original.set("/grok.jsonl", { size: 40, + malformedRecords: 2, mtimeMs: 300, provider: "grok", records: [ @@ -97,6 +99,7 @@ describe("scan cache round trip", () => { }); original.set("/codex.jsonl", { size: 80, + malformedRecords: 3, mtimeMs: 400, provider: "codex", records: [ @@ -135,9 +138,35 @@ describe("scan cache round trip", () => { TEST_FORMAT_MAP, ); + expect(restored.size).toBe(4); expect(restored).toEqual(original); }); + it("defaults missing v5 diagnostic counts to zero without losing cached records", () => { + const original = cacheWith([["/gone.jsonl", 100, [record({ speed: "fast" })]]]); + const encoded = encodeScanCache(original); + const entry = encoded.files["/gone.jsonl"]!; + const legacy = { + ...encoded, + version: 5, + files: { + "/gone.jsonl": { + s: entry.s, + m: entry.m, + p: entry.p, + r: entry.r, + t: entry.t, + o: entry.o, + gl: entry.gl, + gh: entry.gh, + cs: entry.cs, + }, + }, + }; + + expect(decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP)).toEqual(original); + }); + it("drops an entry whose persisted parse state is corrupt", () => { const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]])); const poisoned = { @@ -190,12 +219,14 @@ describe("scan cache round trip", () => { ["/a.jsonl", 100, [record()]], ["/b.jsonl", 200, [record({ sessionId: "session-b" })]], ]); + cache.set("/b.jsonl", { ...cache.get("/b.jsonl")!, malformedRecords: 2 }); const sources = { "claude\u0000/projects": { dir: "/projects", volumeId: "1:2" } }; expect(decodeScanCache(JSON.parse(write(cache, { sources })), TEST_FORMAT_MAP)).toEqual(cache); cache.set("/b.jsonl", { ...cache.get("/b.jsonl")!, size: 30, + malformedRecords: 4, records: [record({ sessionId: "session-c", model: "claude-opus-5-5", dedupeKey: "msg_3:" })], }); const document = JSON.parse(write(cache, { sources })); diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 43b3a548a519c..5d9df36a1d9f9 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -33,6 +33,8 @@ export interface CachedFile { * re-reads that segment and would otherwise double count it. */ readonly tailRecords: readonly UsageRecord[]; + /** Whole-file count, including diagnostics from the tentative tail. */ + readonly malformedRecords: number; readonly position: TranscriptParsePosition; } @@ -59,6 +61,7 @@ interface SerializedFile { readonly p: UsageProviderKind; readonly r: readonly SerializedRecord[]; readonly t: readonly SerializedRecord[]; + readonly mr: number; readonly o: number; readonly gl: number; readonly gh: number; @@ -115,6 +118,7 @@ function serializeFile(entry: CachedFile, tables: InternTables): SerializedFile p: entry.provider, r: entry.records.map(serializeRecord), t: entry.tailRecords.map(serializeRecord), + mr: entry.malformedRecords, o: entry.position.resumeOffset, gl: entry.position.guardLength, gh: entry.position.guardHash, @@ -271,6 +275,8 @@ export function decodeScanCache( const format = formats.get(entry.p); if (format === undefined) continue; if (!isRecordArray(entry.r) || !isRecordArray(entry.t)) continue; + const malformedRecords = entry.mr === undefined ? 0 : entry.mr; + if (!Number.isSafeInteger(malformedRecords) || malformedRecords < 0) continue; // Position fields feed byte offsets and a Buffer allocation in the reader, // so anything outside their real ranges must reject the entry: a bogus // guard length would otherwise fail every parse of the file, silently @@ -309,6 +315,7 @@ export function decodeScanCache( provider, records, tailRecords, + malformedRecords, position: legacy ? { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null } : { diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 3cd49d1cd1c98..79014bfc222dd 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -6,6 +6,8 @@ import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest"; +import type { TranscriptUsageFormat } from "@t3tools/provider-core/server/usage"; +import * as Schema from "effect/Schema"; import { TEST_FORMATS } from "./usageTestFormats.ts"; import { readTranscriptRecords, transcriptLineGate } from "./usageTranscriptReader.ts"; @@ -70,6 +72,71 @@ function codexUsageLine(outputTokens: number, secondsOffset: number): string { } describe("readTranscriptRecords resume", () => { + it.each([ + { mode: "buffered", streamingThresholdBytes: Infinity }, + { mode: "streaming", streamingThresholdBytes: 0 }, + ])("counts malformed tails once across $mode reads and resume", async (options) => { + const format: TranscriptUsageFormat<{ errors: number }> = { + ...TEST_FORMATS.claude, + selectFields: { ...TEST_FORMATS.claude.selectFields, usage: true }, + state: { + initial: () => ({ errors: 0 }), + schema: Schema.Struct({ errors: Schema.Number }), + }, + parseLine(line, state) { + return this.parseProjected(JSON.parse(line), state); + }, + parseProjected(value, state) { + if ( + typeof value === "object" && + value !== null && + "usage" in value && + value.usage === "malformed" + ) { + state.errors += 1; + return []; + } + return TEST_FORMATS.claude.parseProjected(value, undefined); + }, + readMalformedRecords: (state: { errors: number }) => state.errors, + }; + const path = NodePath.join(dir, "diagnostics.jsonl"); + const malformed = JSON.stringify({ usage: "malformed" }); + // Completed lines enter the projected parser only when they cross a stream chunk. + const committed = `${JSON.stringify({ usage: "malformed", padding: "x".repeat(256 * 1024) })}\n${claudeLine(1, 5)}`; + await NodeFSP.writeFile(path, committed + malformed); + + const first = await readTranscriptRecords(path, format, undefined, options); + assert.isNotNull(first); + assert.deepStrictEqual( + first.records.map((record) => record.totals.outputTokens), + [5], + ); + assert.deepStrictEqual(first.tailRecords, []); + assert.strictEqual(first.position.resumeOffset, Buffer.byteLength(committed)); + assert.deepStrictEqual(first.position.state, { errors: 1 }); + assert.strictEqual(first.malformedRecords, 2); + + const savedPosition = JSON.parse(JSON.stringify(first.position)); + await NodeFSP.appendFile(path, `\n${claudeLine(2, 7)}${malformed}\n`); + const resumed = await readTranscriptRecords(path, format, savedPosition, options); + assert.isNotNull(resumed); + assert.isTrue(resumed.resumed); + assert.deepStrictEqual( + resumed.records.map((record) => record.totals.outputTokens), + [7], + ); + assert.deepStrictEqual(resumed.position.state, { errors: 3 }); + assert.strictEqual(resumed.malformedRecords, 3); + + const full = await readTranscriptRecords(path, format, undefined, options); + assert.isNotNull(full); + assert.deepStrictEqual([...first.records, ...resumed.records], full.records); + assert.deepStrictEqual(resumed.tailRecords, full.tailRecords); + assert.deepStrictEqual(resumed.position, full.position); + assert.strictEqual(resumed.malformedRecords, full.malformedRecords); + }); + it("parses only appended lines when resuming a grown file", async () => { const path = NodePath.join(dir, "claude.jsonl"); await NodeFSP.writeFile(path, claudeLine(1, 5) + claudeLine(2, 7)); @@ -77,6 +144,7 @@ describe("readTranscriptRecords resume", () => { assert.isNotNull(first); assert.strictEqual(first.records.length, 2); assert.isFalse(first.resumed); + assert.strictEqual(first.malformedRecords, 0); await NodeFSP.appendFile(path, claudeLine(3, 11)); const second = await readTranscriptRecords(path, TEST_FORMATS.claude, first.position); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index d3f19afd20a94..1131b95bc7e91 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -45,6 +45,8 @@ export interface TranscriptParseResult { * segment: the next scan re-reads it once the writer finishes the line. */ readonly tailRecords: readonly UsageRecord[]; + /** Includes tentative tail diagnostics; position.state only includes committed lines. */ + readonly malformedRecords: number; readonly position: TranscriptParsePosition; readonly resumed: boolean; } @@ -306,10 +308,9 @@ export async function readTranscriptRecords( // `resumeOffset` and the next scan replays the tail from it. const encodedState = encodeState === undefined ? null : encodeState(state); const tailRecords: UsageRecord[] = []; - finish( - decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState)), - tailRecords, - ); + const tailState = + decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState)); + finish(tailState, tailRecords); const guardLength = Math.min(GUARD_LENGTH, resumeOffset); let guardHash = 0; @@ -322,6 +323,7 @@ export async function readTranscriptRecords( return { records, tailRecords, + malformedRecords: format.readMalformedRecords?.(tailState) ?? 0, position: { resumeOffset, guardLength, diff --git a/packages/provider-core/src/server/usage.ts b/packages/provider-core/src/server/usage.ts index 68cd6571307c9..89c627e5fa162 100644 --- a/packages/provider-core/src/server/usage.ts +++ b/packages/provider-core/src/server/usage.ts @@ -62,6 +62,8 @@ export interface TranscriptUsageFormat { readonly usageMarkers: readonly [string, ...string[]]; parseLine(line: string, state: State): readonly UsageRecord[]; parseProjected(projected: unknown, state: State): readonly UsageRecord[]; + /** Cumulative malformed or inconsistent usage records, including skipped records. */ + readMalformedRecords?(state: State): number; /** * Present for formats whose records depend on earlier lines. The state at a * resume point is persisted in the scan cache through `schema`, so a grown From a09b9ca28256c483e6412ac1dd580c1543444a0c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 21:05:40 -0700 Subject: [PATCH 004/113] fix(claude): retain approvals when bypass is disabled (#21805) Co-authored-by: shivam <91240327+shivamhwp@users.noreply.github.com> --- .../Adapters/ClaudeAdapterV2.test.ts | 24 +++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 24 ++++++++++++++++--- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 256dff5b8358f..e4704ae412fdc 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -471,6 +471,30 @@ describe("ClaudeAdapterV2 runtime query policy", () => { }); }); + it("keeps approval checks when Full access must fall back to an allowed Claude mode", () => { + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: "/workspace", + approvalPolicy: "never", + }); + + assert.deepEqual( + ClaudeAdapterV2.claudeRuntimeQueryPolicyForRuntimePolicy(runtimePolicy, { + bypassDisabled: true, + autoDisabled: false, + }), + { permissionMode: "auto", installPermissionCallback: true }, + ); + assert.deepEqual( + ClaudeAdapterV2.claudeRuntimeQueryPolicyForRuntimePolicy(runtimePolicy, { + bypassDisabled: true, + autoDisabled: true, + }), + { permissionMode: "default", installPermissionCallback: true }, + ); + }); + it("maps Auto runtime mode to Claude's AI-reviewed permission mode", () => { const queryPolicy = ClaudeAdapterV2.claudeRuntimeQueryPolicyForRuntimePolicy( ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 2aaff3b573e79..6f0ab121c68a0 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -1927,10 +1927,27 @@ export interface ClaudeRuntimeQueryPolicy { readonly installPermissionCallback: boolean; } +interface ClaudeBypassAvailability { + readonly bypassDisabled: boolean; + readonly autoDisabled: boolean; +} + +const CLAUDE_BYPASS_FULLY_AVAILABLE: ClaudeBypassAvailability = { + bypassDisabled: false, + autoDisabled: false, +}; + export function claudeRuntimeQueryPolicyForRuntimePolicy( runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy, + availability: ClaudeBypassAvailability = CLAUDE_BYPASS_FULLY_AVAILABLE, ): ClaudeRuntimeQueryPolicy { - const permissionMode = permissionModeForClaudeRuntimePolicy(runtimePolicy); + const requestedMode = permissionModeForClaudeRuntimePolicy(runtimePolicy); + const downgraded = requestedMode === "bypassPermissions" && availability.bypassDisabled; + const permissionMode = downgraded + ? availability.autoDisabled + ? "default" + : "auto" + : requestedMode; const readOnlyTools = sandboxPolicyKindForClaudeRuntimePolicy(runtimePolicy) === "readOnly" ? CLAUDE_READ_ONLY_ALLOWED_TOOLS @@ -1942,10 +1959,11 @@ export function claudeRuntimeQueryPolicyForRuntimePolicy( // acceptEdits approves edits before the callback runs; everything else it // leaves to the callback, which must ask rather than allow. const installPermissionCallback = - runtimePolicy.approvalPolicy === undefined + downgraded || + (runtimePolicy.approvalPolicy === undefined ? runtimePolicy.runtimeMode === "approval-required" || runtimePolicy.runtimeMode === "auto-accept-edits" - : runtimePolicy.approvalPolicy !== "never"; + : runtimePolicy.approvalPolicy !== "never"); if (permissionMode === "plan") { return { From 1327e7bdfb454594eb8308961691d1b7419156f3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 21:06:26 -0700 Subject: [PATCH 005/113] fix(server): preserve Claude question option previews (#21932) --- .../orchestration-v2/Adapters/ClaudeAdapterV2.test.ts | 11 +++++++++-- .../src/orchestration-v2/Adapters/ClaudeAdapterV2.ts | 10 +++++++++- packages/contracts/src/orchestrationV2.ts | 1 + 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index e4704ae412fdc..19311662d756d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -302,13 +302,17 @@ describe("ClaudeAdapterV2 runtime query policy", () => { }); it("projects AskUserQuestion input with question text as the answer key", () => { + const preview = "\n **Proposed change**\n\n```ts\n\tconst enabled = true;\n```\n "; assert.deepEqual( ClaudeAdapterV2.claudeUserInputQuestions({ questions: [ { header: "Approach", question: "Which approach?", - options: [{ label: "Simple", description: "Use fewer moving parts" }], + options: [ + { label: "Simple", description: "Use fewer moving parts", preview }, + { label: "Safe", description: "Keep the current setup" }, + ], multiSelect: true, }, ], @@ -318,7 +322,10 @@ describe("ClaudeAdapterV2 runtime query policy", () => { id: "Which approach?", header: "Approach", question: "Which approach?", - options: [{ label: "Simple", description: "Use fewer moving parts" }], + options: [ + { label: "Simple", description: "Use fewer moving parts", preview }, + { label: "Safe", description: "Keep the current setup" }, + ], multiSelect: true, }, ], diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6f0ab121c68a0..d6aada46a9025 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -3404,7 +3404,15 @@ export function claudeUserInputQuestions( if (label.length === 0) return []; const description = typeof optionRecord.description === "string" ? optionRecord.description.trim() : ""; - return [{ label, description: description || label }]; + return [ + { + label, + description: description || label, + ...(typeof optionRecord.preview === "string" + ? { preview: optionRecord.preview } + : {}), + }, + ]; }) : []; return [ diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index d5364ea2fa8f4..6a957dc2ba890 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1132,6 +1132,7 @@ export const OrchestrationV2UserInputQuestion = Schema.Struct({ label: TrimmedNonEmptyString, description: TrimmedNonEmptyString, value: Schema.optional(Schema.String), + preview: Schema.optional(Schema.String), }), ), multiSelect: Schema.optional(Schema.Boolean), From fb1486901f7d07e9e394b6061323d07967405f4f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 21:07:06 -0700 Subject: [PATCH 006/113] fix(server): expose turns blocked only on T3 thread waits (#21499) --- .../Adapters/AcpAdapterV2.test.ts | 135 +++++++++++++ .../Adapters/OpenCode2AdapterV2.test.ts | 44 ++++ .../orchestration-v2/ProjectionStore.test.ts | 189 ++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 110 ++++++---- packages/contracts/src/orchestrationV2.ts | 1 + .../src/server/adapter.test.ts | 6 +- .../provider-cursor/src/server/adapter.ts | 2 +- .../provider-opencode/src/server/toolItems.ts | 2 + .../orchestrationV2PendingBackgroundWork.ts | 44 ++++ 9 files changed, 488 insertions(+), 45 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 92bb99e4de16e..cbc0ee419ea0d 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -22,10 +22,12 @@ import { ProviderThreadId, ThreadId, type OrchestrationV2ProviderThread, + type OrchestrationV2TurnItem, type ProviderOptionDescriptor, } from "@t3tools/contracts"; import { GrokSettings } from "@t3tools/provider-grok/settings"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { deriveBlockedOnThreadWait } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { resolveSelfInvocation } from "@t3tools/shared/nodeRuntime"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; @@ -913,6 +915,139 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.effect.each(["agent_message_chunk", "agent_thought_chunk"] as const)( + "keeps named ACP %s open while deriving thread wait execution state", + (sessionUpdate) => + Effect.gen(function* () { + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let handler: Parameters[0] | undefined; + const instanceId = ProviderInstanceId.make("acp-named-stream-wait"); + const path = yield* Path.Path; + const adapter = yield* makeAcpAdapterV2({ + instanceId, + selfInvocation: yield* resolveSelfInvocation(), + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner: yield* ChildProcessSpawner.ChildProcessSpawner, + mockAgentPath: yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ), + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (next) => + Effect.sync(() => { + handler = next; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(next))), + prompt: () => + Effect.gen(function* () { + assert.isDefined(handler); + const updates = [ + { + sessionUpdate, + messageId: "named-stream", + content: { type: "text", text: "Waiting for the child result" }, + }, + { + sessionUpdate: "tool_call", + toolCallId: "wait", + title: "Wait for the child", + status: "in_progress", + rawInput: { + server: "t3-code", + tool: "t3_thread_wait", + arguments: { threadId: "child" }, + }, + _meta: { is_mcp_tool_call: true }, + }, + { + sessionUpdate: "tool_call", + toolCallId: "command", + kind: "execute", + title: "Run command", + status: "in_progress", + rawInput: { command: "pwd" }, + }, + { + sessionUpdate: "tool_call_update", + toolCallId: "command", + status: "completed", + }, + { + sessionUpdate: "tool_call_update", + toolCallId: "wait", + status: "completed", + }, + ] satisfies Array; + for (const update of updates) + yield* handler!({ sessionId: "mock-session-1", update }); + return { stopReason: "end_turn" as const }; + }), + }), + }), + }, + }); + const threadId = ThreadId.make("thread-acp-named-stream-wait"); + const modelSelection = { instanceId, model: "default" }; + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("session-acp-named-stream-wait"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const input = makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }); + yield* runtime.startTurn(input); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ); + const retained = new Map(); + const states: Array = []; + for (const event of events) { + if (event.type !== "turn_item.updated") continue; + const item = event.turnItem; + retained.set(item.id, item); + if (item.type !== "dynamic_tool" && item.type !== "command_execution") continue; + const streams = [...retained.values()].filter( + (entry) => entry.type === "assistant_message" || entry.type === "reasoning", + ); + assert.deepEqual( + streams.map((entry) => [entry.type, entry.status]), + [ + [ + sessionUpdate === "agent_message_chunk" ? "assistant_message" : "reasoning", + "running", + ], + ], + ); + states.push( + deriveBlockedOnThreadWait({ + activeRun: { id: input.runId, status: "running" }, + turnItems: [...retained.values()], + }), + ); + } + assert.deepEqual(states, [true, false, true, false]); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.effect("starts a new replay message after ACP v2 plan boundaries", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 2c6c62446bc1a..a7212a95a572c 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -3333,6 +3333,50 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); + it.effect("keeps thread-scoped T3 waits as dynamic tools through their lifecycle", () => + Effect.gen(function* () { + const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" }; + const name = "t3-code-web_delegate_task"; + const input = { task: "Review", mode: "wait" }; + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.tool.input.started", { ...tool, name }), + event("session.tool.called", { ...tool, name, input, executed: true }), + event("session.tool.success", { + ...tool, + content: [{ type: "text", text: "Reviewed" }], + metadata: {}, + executed: true, + }), + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn(turnInput(thread)); + const items = (yield* Fiber.join(events)).flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id) + ? [event.turnItem] + : [], + ); + assert.deepEqual( + items.map((item) => item.type), + ["dynamic_tool", "dynamic_tool", "dynamic_tool"], + ); + assert.deepEqual( + items.map((item) => item.status), + ["running", "running", "completed"], + ); + const last = items.at(-1); + assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name); + assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input); + }).pipe(Effect.scoped), + ); + it.effect( "registers T3's MCP server for the thread alone and removes it when the thread unloads", () => diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 35581cde5409a..32c83629c6652 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -9,6 +9,7 @@ import { MessageId, type ModelSelection, type OrchestrationV2ProviderThread, + type OrchestrationV2TurnItem, type OrchestrationV2ThreadShellSnapshot, NodeId, PlanId, @@ -185,6 +186,194 @@ const addRolledBackRecoveryCandidate = Effect.fn("addRolledBackRecoveryCandidate return threadId; }); +it.effect.each([ + ["memory", ProjectionStore.layerMemory], + ["SQL", layerTest], +] as const)("%s shells flag only an active turn blocked solely on T3 thread waits", ([, layer]) => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = yield* addRolledBackRecoveryCandidate("thread-wait"); + const initial = yield* store.getThreadProjection(threadId); + const at = initial.updatedAt; + const previousCommand = initial.turnItems.find((item) => item.type === "command_execution")!; + const itemBase = { + ...previousCommand, + providerThreadId: previousCommand.providerThreadId ?? null, + providerTurnId: previousCommand.providerTurnId ?? null, + nativeItemRef: previousCommand.nativeItemRef ?? null, + parentItemId: previousCommand.parentItemId ?? null, + }; + yield* store.apply({ + id: EventId.make("event:thread-wait:previous-completed"), + type: "run.updated", + threadId, + occurredAt: at, + payload: { ...initial.runs[0]!, status: "completed", completedAt: at }, + }); + const run = { + ...initial.runs[0]!, + id: RunId.make("run:thread-wait:active"), + ordinal: 2, + status: "running" as const, + completedAt: null, + }; + yield* store.apply({ + id: EventId.make("event:thread-wait:active"), + type: "run.created", + threadId, + occurredAt: at, + payload: run, + }); + let sequence = 0; + const updateItem = (payload: OrchestrationV2TurnItem) => + store.apply({ + id: EventId.make(`event:thread-wait:item:${sequence++}`), + type: "turn-item.updated", + threadId, + runId: payload.runId ?? undefined, + driver, + occurredAt: at, + payload, + }); + const assertBlocked = Effect.fnUntraced(function* (expected: boolean) { + const listed = (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId); + for (const shell of [yield* store.getThreadShell(threadId), listed]) { + assert.equal(shell?.activeRunId, run.id); + assert.equal(shell?.status, "running"); + assert.equal(shell?.blockedOnThreadWait ?? false, expected); + } + }); + const wait = { + ...itemBase, + id: TurnItemId.make("item:thread-wait:active"), + runId: run.id, + type: "dynamic_tool" as const, + toolName: "mcp__t3_code__t3_thread_wait", + input: { threadId: "child-thread" }, + status: "running" as const, + }; + + yield* updateItem({ + ...wait, + id: TurnItemId.make("item:thread-wait:stale"), + runId: initial.runs[0]!.id, + }); + yield* assertBlocked(false); + const foreign = { + ...wait, + id: TurnItemId.make("item:thread-wait:foreign"), + toolName: "weather.t3_thread_wait", + }; + yield* updateItem(foreign); + yield* assertBlocked(false); + yield* updateItem({ ...foreign, status: "completed", completedAt: at }); + yield* updateItem({ ...wait, status: "pending", startedAt: null }); + yield* assertBlocked(false); + yield* updateItem(wait); + yield* assertBlocked(true); + + yield* updateItem({ + ...wait, + id: TurnItemId.make("item:thread-wait:plan"), + type: "todo_list", + planId: PlanId.make("plan:thread-wait"), + steps: [{ id: "review", text: "Review the child result", status: "running" }], + }); + yield* assertBlocked(true); + yield* updateItem({ + ...wait, + id: TurnItemId.make("item:thread-wait:proposed-plan"), + type: "proposed_plan", + planId: PlanId.make("plan:thread-wait:proposed"), + markdown: "Review the child result", + streaming: true, + }); + yield* updateItem({ + ...wait, + id: TurnItemId.make("item:thread-wait:named-message"), + nativeItemRef: { driver, nativeId: "named-message", strength: "weak" }, + type: "assistant_message", + messageId: MessageId.make("message:thread-wait:named"), + text: "Waiting for the child result", + streaming: true, + }); + yield* updateItem({ + ...wait, + id: TurnItemId.make("item:thread-wait:named-thought"), + nativeItemRef: { driver, nativeId: "named-thought", strength: "weak" }, + type: "reasoning", + text: "I need the child result", + streaming: true, + }); + yield* assertBlocked(true); + + const command = { + ...itemBase, + id: TurnItemId.make("item:thread-wait:concurrent"), + runId: run.id, + }; + yield* updateItem(command); + yield* assertBlocked(false); + yield* updateItem({ ...wait, status: "waiting" }); + yield* assertBlocked(false); + yield* updateItem({ ...command, status: "completed", completedAt: at }); + yield* assertBlocked(true); + const fileChange = { + ...command, + id: TurnItemId.make("item:thread-wait:concurrent-edit"), + type: "file_change" as const, + fileName: "example.ts", + }; + yield* updateItem(fileChange); + yield* assertBlocked(false); + yield* updateItem({ ...fileChange, status: "completed", completedAt: at }); + yield* assertBlocked(true); + const compaction = { + ...command, + id: TurnItemId.make("item:thread-wait:compaction"), + type: "compaction" as const, + driver, + }; + yield* updateItem(compaction); + yield* assertBlocked(false); + yield* updateItem({ ...compaction, status: "completed", completedAt: at }); + yield* assertBlocked(true); + yield* updateItem({ ...wait, status: "completed", completedAt: at }); + yield* assertBlocked(false); + + const delegation = { + ...wait, + id: TurnItemId.make("item:thread-wait:delegation"), + toolName: "t3-code-thread_opencode2-adapter_delegate_task", + input: { task: "Review", mode: "async" }, + }; + yield* updateItem(delegation); + yield* assertBlocked(false); + yield* updateItem({ ...delegation, status: "completed", completedAt: at }); + const blockingDelegation = { + ...delegation, + id: TurnItemId.make("item:thread-wait:blocking-delegation"), + input: { task: "Review", mode: "wait" }, + }; + yield* updateItem({ + ...command, + id: TurnItemId.make("item:thread-wait:child"), + type: "subagent", + subagentId: NodeId.make("node:thread-wait:child"), + origin: "app_owned", + driver, + providerInstanceId, + childThreadId: ThreadId.make("child-thread"), + prompt: "Review", + result: null, + }); + yield* updateItem(blockingDelegation); + yield* assertBlocked(true); + yield* updateItem({ ...blockingDelegation, status: "completed", completedAt: at }); + yield* assertBlocked(false); + }).pipe(Effect.provide(layer)), +); + it.effect.each([ ["memory", ProjectionStore.layerMemory], ["SQL", layerTest], diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8448ff8268e1e..b349ff667f5f5 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -72,7 +72,10 @@ import { createOrchestrationV2TurnItemVisibility, isOrchestrationV2SupersededInterrupt, } from "@t3tools/shared/orchestrationV2Timeline"; -import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; +import { + deriveBlockedOnThreadWait, + derivePendingBackgroundWork, +} from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -1056,6 +1059,7 @@ type ShellThreadRow = { readonly latest_run_started_at: string | null; readonly latest_run_completed_at: string | null; readonly active_run_id: string | null; + readonly active_run_status: string | null; readonly activity_run_status: string | null; readonly activity_run_started_at: string | null; readonly last_error: string | null; @@ -1595,6 +1599,7 @@ export function threadShellFromProjection( latestRunStartedAt: latestRun?.startedAt ?? null, latestRunCompletedAt: latestRun?.completedAt ?? null, activeRunId: activeRun?.id ?? null, + blockedOnThreadWait: deriveBlockedOnThreadWait({ activeRun, turnItems: projection.turnItems }), activityRunStatus: activityRun?.status ?? null, activityRunStartedAt: activityRun === null ? null : orchestrationV2RunWorkStartedAt(activityRun), @@ -1693,6 +1698,7 @@ type ShellThreadState = { readonly latestRunStartedAt: DateTime.Utc | null; readonly latestRunCompletedAt: DateTime.Utc | null; readonly activeRunId: RunId | null; + readonly blockedOnThreadWait: boolean; readonly activityRunStatus: ShellActivityRunStatus | null; readonly activityRunStartedAt: DateTime.Utc | null; readonly lastError: string | null; @@ -1845,6 +1851,7 @@ function shellFromState(input: { latestRunStartedAt: input.state.latestRunStartedAt, latestRunCompletedAt: input.state.latestRunCompletedAt, activeRunId: input.state.activeRunId, + blockedOnThreadWait: input.state.blockedOnThreadWait, activityRunStatus: input.state.activityRunStatus, activityRunStartedAt: input.state.activityRunStartedAt, status: input.state.latestRunStatus, @@ -5565,14 +5572,8 @@ export const layer: Layer.Layer = presented.requested_at AS latest_run_requested_at, json_extract(presented.payload_json, '$.startedAt') AS latest_run_started_at, presented.completed_at AS latest_run_completed_at, - ( - SELECT r.run_id - FROM orchestration_v2_projection_runs r - WHERE r.thread_id = t.thread_id - AND r.status IN ('preparing', 'starting', 'running') - ORDER BY r.ordinal DESC, r.run_id DESC - LIMIT 1 - ) AS active_run_id, + active.run_id AS active_run_id, + active.status AS active_run_status, ( SELECT r.status FROM orchestration_v2_projection_runs r @@ -5697,6 +5698,14 @@ export const layer: Layer.Layer = AND i.run_id IS NULL ) AS runless_item_count FROM orchestration_v2_projection_threads t + LEFT JOIN orchestration_v2_projection_runs active ON active.run_id = ( + SELECT r.run_id + FROM orchestration_v2_projection_runs r + WHERE r.thread_id = t.thread_id + AND r.status IN ('preparing', 'starting', 'running') + ORDER BY r.ordinal DESC, r.run_id DESC + LIMIT 1 + ) -- The newest run not waiting in a held queue, matching latestUnheldRun. LEFT JOIN orchestration_v2_projection_runs presented ON presented.run_id = ( SELECT candidate.run_id @@ -5780,34 +5789,34 @@ export const layer: Layer.Layer = WHERE thread_id IN ${sql.in(threadIds)} `; - const selectShellPendingTurnItemRows = (threadIds?: ReadonlyArray) => - threadIds === undefined - ? sql` - SELECT i.thread_id, i.payload_json, p.provider_instance_id AS owning_provider_instance_id - FROM orchestration_v2_projection_turn_items i - LEFT JOIN orchestration_v2_projection_provider_threads p - ON p.provider_thread_id = i.provider_thread_id - LEFT JOIN orchestration_v2_projection_runs r - ON r.run_id = i.run_id - WHERE i.type IN ('command_execution', 'dynamic_tool', 'subagent') - AND i.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') - -- A rolled-back run's items are abandoned, not pending. Without - -- this the shell reports Waiting for work no one will finish, - -- matching the item_count query's exclusion above. - AND (i.run_id IS NULL OR r.status <> 'rolled_back') - ` - : sql` - SELECT i.thread_id, i.payload_json, p.provider_instance_id AS owning_provider_instance_id - FROM orchestration_v2_projection_turn_items i - LEFT JOIN orchestration_v2_projection_provider_threads p - ON p.provider_thread_id = i.provider_thread_id - LEFT JOIN orchestration_v2_projection_runs r - ON r.run_id = i.run_id - WHERE i.type IN ('command_execution', 'dynamic_tool', 'subagent') - AND i.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') - AND (i.run_id IS NULL OR r.status <> 'rolled_back') - AND i.thread_id IN ${sql.in(threadIds)} - `; + const selectShellPendingTurnItemRows = ( + threadIds?: ReadonlyArray, + activeRunIds: ReadonlyArray = [], + ) => sql` + SELECT i.thread_id, i.payload_json, p.provider_instance_id AS owning_provider_instance_id + FROM orchestration_v2_projection_turn_items i + LEFT JOIN orchestration_v2_projection_provider_threads p + ON p.provider_thread_id = i.provider_thread_id + LEFT JOIN orchestration_v2_projection_runs r + ON r.run_id = i.run_id + WHERE i.type IN ('command_execution', 'dynamic_tool', 'subagent') + AND i.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') + AND (i.run_id IS NULL OR r.status <> 'rolled_back') + ${threadIds === undefined ? sql`` : sql`AND i.thread_id IN ${sql.in(threadIds)}`} + ${ + activeRunIds.length === 0 + ? sql`` + : sql` + -- Preserve the pending-item partial index; read other work only from running runs. + UNION ALL + SELECT i.thread_id, i.payload_json, NULL AS owning_provider_instance_id + FROM orchestration_v2_projection_turn_items i + WHERE i.run_id IN ${sql.in(activeRunIds)} + AND i.type IN ('file_change', 'file_search', 'web_search', 'compaction') + AND i.status IN ('running', 'waiting') + ` + } + `; const runMapsByThreadId = (input: { readonly runRows: ReadonlyArray; @@ -6137,6 +6146,13 @@ export const layer: Layer.Layer = latestRunStartedAt, latestRunCompletedAt, activeRunId: row.active_run_id === null ? null : RunId.make(row.active_run_id), + blockedOnThreadWait: deriveBlockedOnThreadWait({ + activeRun: + row.active_run_id === null || row.active_run_status !== "running" + ? null + : { id: RunId.make(row.active_run_id), status: "running" }, + turnItems: pendingTurnItemsByThreadId.get(thread.id) ?? [], + }), activityRunStartedAt: row.activity_run_started_at === null ? null @@ -6223,7 +6239,16 @@ export const layer: Layer.Layer = AND aggregate_kind = 'thread' `, readForThreadIds(selectShellProviderThreadRows), - readForThreadIds(selectShellPendingTurnItemRows), + readForThreadIds((ids) => + selectShellPendingTurnItemRows( + ids, + threadRows.flatMap((row) => + row.active_run_id !== null && row.active_run_status === "running" + ? [row.active_run_id] + : [], + ), + ), + ), ]); return { targetThreadIds, @@ -6329,7 +6354,14 @@ export const layer: Layer.Layer = ? Effect.succeed([] as ReadonlyArray) : selectShellRunItemCounts(forkSourceIds), selectShellProviderThreadRows(threadIds), - selectShellPendingTurnItemRows(threadIds), + selectShellPendingTurnItemRows( + threadIds, + [...rowsByThreadId.values()].flatMap((row) => + row.active_run_id !== null && row.active_run_status === "running" + ? [row.active_run_id] + : [], + ), + ), ]); const { runOrdinalsByThreadId, itemCountsByThreadId } = runMapsByThreadId({ runRows, diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 6a957dc2ba890..60c05ab7ede38 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1826,6 +1826,7 @@ export const OrchestrationV2ThreadShell = Schema.Struct({ activityRunStatus: Schema.optional( Schema.NullOr(Schema.Literals(["preparing", "starting", "running", "waiting"])), ), + blockedOnThreadWait: Schema.optional(Schema.Boolean), status: OrchestrationV2ShellThreadStatus, lastError: Schema.optional(Schema.NullOr(Schema.String)), lastErrorClass: Schema.optional(Schema.NullOr(OrchestrationV2ProviderFailureClass)), diff --git a/packages/provider-cursor/src/server/adapter.test.ts b/packages/provider-cursor/src/server/adapter.test.ts index be87e4a481968..8aecb9f92dfe4 100644 --- a/packages/provider-cursor/src/server/adapter.test.ts +++ b/packages/provider-cursor/src/server/adapter.test.ts @@ -830,11 +830,7 @@ describe("CursorAdapterV2", () => { name: "weather", kind: "integration", }); - assert.deepEqual(item.input, { - providerIdentifier: "weather", - toolName: "get_weather", - args: { city: "Berlin" }, - }); + assert.deepEqual(item.input, { city: "Berlin" }); } const fileSearchItems = events.flatMap((event) => event.type === "turn_item.updated" && diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.ts index b69ba4449b82d..765ec16fdec57 100644 --- a/packages/provider-cursor/src/server/adapter.ts +++ b/packages/provider-cursor/src/server/adapter.ts @@ -1321,7 +1321,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( }) : {}), toolName: cursorToolName(toolCall), - input: toolCall.args, + input: toolCall.type === "mcp" ? (toolCall.args.args ?? {}) : toolCall.args, ...(cursorToolOutput(toolCall) === undefined ? {} : { output: cursorToolOutput(toolCall) }), diff --git a/packages/provider-opencode/src/server/toolItems.ts b/packages/provider-opencode/src/server/toolItems.ts index c222c223bf892..a0f7e2c6bfe37 100644 --- a/packages/provider-opencode/src/server/toolItems.ts +++ b/packages/provider-opencode/src/server/toolItems.ts @@ -1,4 +1,5 @@ import type { OrchestrationV2TurnItem } from "@t3tools/contracts"; +import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation"; import { formatReadToolLabel, formatSearchToolLabel } from "@t3tools/shared/toolActivity"; // Search results stay on the timeline wire, so keep their text a preview. @@ -12,6 +13,7 @@ type ToolItemBase = Omit< export function openCodeToolProjectionKind( toolName: string, ): "command_execution" | "file_change" | "file_search" | "web_search" | "dynamic_tool" { + if (resolveT3McpToolId(toolName) !== null) return "dynamic_tool"; const normalized = toolName.toLowerCase(); if (normalized === "todowrite") { return "dynamic_tool"; diff --git a/packages/shared/src/orchestrationV2PendingBackgroundWork.ts b/packages/shared/src/orchestrationV2PendingBackgroundWork.ts index 15453ae01850c..8073bcccd0bb5 100644 --- a/packages/shared/src/orchestrationV2PendingBackgroundWork.ts +++ b/packages/shared/src/orchestrationV2PendingBackgroundWork.ts @@ -7,7 +7,9 @@ import type { ThreadPullRequestLink, } from "@t3tools/contracts"; import { isOrchestrationV2WorkActive } from "@t3tools/contracts"; +import * as Predicate from "effect/Predicate"; +import { resolveT3McpToolId } from "./t3McpToolPresentation.ts"; import { threadPullRequestKeyOf } from "./threadPullRequests.ts"; const BACKGROUND_TURN_ITEM_TYPES = new Set([ @@ -16,6 +18,15 @@ const BACKGROUND_TURN_ITEM_TYPES = new Set([ "subagent", ]); +// Plans and text streams can stay open across tool calls without executing work. +const EXECUTION_TURN_ITEM_TYPES = new Set([ + ...BACKGROUND_TURN_ITEM_TYPES, + "file_change", + "file_search", + "web_search", + "compaction", +]); + const TERMINAL_RUN_STATUSES = new Set([ "cancelled", "completed", @@ -48,6 +59,39 @@ export function turnItemUpdateCanEndBackgroundWork( return BACKGROUND_TURN_ITEM_TYPES.has(item.type) && !isOrchestrationV2WorkActive(item.status); } +export function deriveBlockedOnThreadWait(input: { + readonly activeRun: Pick | null; + readonly turnItems: ReadonlyArray; +}): boolean { + if (input.activeRun?.status !== "running") return false; + let hasWait = false; + for (const item of input.turnItems) { + if ( + item.runId !== input.activeRun.id || + !EXECUTION_TURN_ITEM_TYPES.has(item.type) || + (item.status !== "running" && item.status !== "waiting") || + // App-owned rows track the child, including the child a blocking delegate is waiting for. + (item.type === "subagent" && item.origin === "app_owned") + ) { + continue; + } + if (item.type !== "dynamic_tool") return false; + const tool = resolveT3McpToolId(item.toolName); + if ( + tool !== "t3_thread_wait" && + !( + tool === "delegate_task" && + Predicate.isObject(item.input) && + Reflect.get(item.input, "mode") === "wait" + ) + ) { + return false; + } + hasWait = true; + } + return hasWait; +} + export function backgroundWorkHoldsCompletion( tasks: ReadonlyArray>, ): boolean { From 7ee6428aa8f5279bf6df07719b294111504438cb Mon Sep 17 00:00:00 2001 From: Scott Norteman Date: Tue, 20 Oct 2026 00:07:44 -0400 Subject: [PATCH 007/113] fix(web): deleted answer chips no longer return after picking an option (#19483) Co-authored-by: Claude Opus 5.5 Co-authored-by: Julius Marminge --- .../components/chat/usePendingThreadRequests.ts | 6 +++++- apps/web/src/pendingUserInput.test.ts | 17 +++++++++++++++++ apps/web/src/pendingUserInput.ts | 11 +++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/chat/usePendingThreadRequests.ts b/apps/web/src/components/chat/usePendingThreadRequests.ts index 88e8f82dc0bd8..9ce6172a28d59 100644 --- a/apps/web/src/components/chat/usePendingThreadRequests.ts +++ b/apps/web/src/components/chat/usePendingThreadRequests.ts @@ -37,6 +37,7 @@ import { pendingUserInputRequestKey, pruneResolvedUserInputDrafts, setPendingUserInputCustomAnswer, + threadContextsReferencedByAnswer, togglePendingUserInputOptionSelection, type PendingUserInputDraftAnswer, } from "../../pendingUserInput"; @@ -452,7 +453,10 @@ export function usePendingThreadRequests(input: { ); store.addThreadContexts( composerDraftTarget, - store.getComposerDraft(questionTarget)?.threadContexts ?? [], + threadContextsReferencedByAnswer( + displacedAnswer ?? "", + store.getComposerDraft(questionTarget)?.threadContexts ?? [], + ), { appendReference: false }, ); } diff --git a/apps/web/src/pendingUserInput.test.ts b/apps/web/src/pendingUserInput.test.ts index d6d14b0c4cacb..407b84be2af39 100644 --- a/apps/web/src/pendingUserInput.test.ts +++ b/apps/web/src/pendingUserInput.test.ts @@ -12,6 +12,7 @@ import { projectPendingUserInputAnswer, resolvePendingUserInputAnswer, setPendingUserInputCustomAnswer, + threadContextsReferencedByAnswer, togglePendingUserInputOptionSelection, } from "./pendingUserInput"; @@ -42,6 +43,22 @@ describe("projectPendingUserInputAnswer", () => { }); }); +describe("threadContextsReferencedByAnswer", () => { + const kept = threadContextRecord( + { environmentId: EnvironmentId.make("env-answer"), threadId: ThreadId.make("kept-thread") }, + "Kept work", + ); + const deleted = threadContextRecord( + { environmentId: EnvironmentId.make("env-answer"), threadId: ThreadId.make("deleted-thread") }, + "Deleted work", + ); + + it("preserves remaining chips without restoring deleted thread records", () => { + const answer = `Compare ${formatInlineContextReference(threadContextReference(kept))}`; + expect(threadContextsReferencedByAnswer(answer, [deleted, kept])).toEqual([kept]); + }); +}); + const singleSelectQuestion = { id: "scope", header: "Scope", diff --git a/apps/web/src/pendingUserInput.ts b/apps/web/src/pendingUserInput.ts index 2b37af07b3e35..fb159426b5f40 100644 --- a/apps/web/src/pendingUserInput.ts +++ b/apps/web/src/pendingUserInput.ts @@ -2,6 +2,8 @@ import type { ThreadContextRecord, UserInputQuestion } from "@t3tools/contracts" import { replaceComposerContextReferences } from "@t3tools/shared/composerContextReferences"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import { collectInlineContextIds } from "./lib/composerContextReferences"; + export function projectPendingUserInputAnswer( answer: unknown, threadContexts: ReadonlyArray, @@ -14,6 +16,15 @@ export function projectPendingUserInputAnswer( }); } +// Question drafts retain deleted chip records for undo; only transfer referenced records. +export function threadContextsReferencedByAnswer( + answer: string, + threadContexts: ReadonlyArray, +) { + const referencedIds = new Set(collectInlineContextIds(answer)); + return threadContexts.filter((record) => referencedIds.has(record.contextId)); +} + export interface PendingUserInputDraftAnswer { selectedOptionValues?: string[]; customAnswer?: string; From 651fe94dc94553f6fd3c19a73d2b3377de6c7f15 Mon Sep 17 00:00:00 2001 From: Quy Ngoc Nguyen Date: Tue, 20 Oct 2026 11:08:05 +0700 Subject: [PATCH 008/113] fix(mobile): composer buttons stay visible with animations removed (#19239) Co-authored-by: Julius Marminge --- .../features/voice-input/ComposerDictationControl.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index 6e7977e928a8a..3e7d852be7554 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -13,6 +13,7 @@ import Animated, { LinearTransition, ReduceMotion, useAnimatedStyle, + useReducedMotion, useSharedValue, withTiming, type EntryExitAnimationFunction, @@ -135,14 +136,19 @@ export function ComposerDictationToolbar(props: { readonly showsDictation: boolean; readonly visible?: boolean; }) { + // Android can drop the reduced-motion flip's final frame, leaving the toolbar invisible. + const reducedMotion = useReducedMotion(); + const entering = props.showsDictation ? DICTATION_TOOLBAR_ENTERING : DRAFT_TOOLBAR_ENTERING; + const exiting = props.showsDictation ? DICTATION_TOOLBAR_EXITING : DRAFT_TOOLBAR_EXITING; + return ( {props.visible !== false ? ( {props.children} From da26f1ef70eb86a2f088a2ec0cf997ae4c1c6890 Mon Sep 17 00:00:00 2001 From: Rohan Patnaik Date: Tue, 20 Oct 2026 09:38:49 +0530 Subject: [PATCH 009/113] fix(web): decode timeline image attachments asynchronously (#19923) Co-authored-by: Rohan Patnaik Co-authored-by: Julius Marminge --- apps/web/src/components/chat/MessagesTimeline.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 195e6cf061fe4..2105454784721 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -2353,6 +2353,7 @@ function UserTimelineRow({ row }: { row: Extract From dc52893873a1e35b387283b1893cc4d2f200ff15 Mon Sep 17 00:00:00 2001 From: Thalles Passos Date: Tue, 20 Oct 2026 01:09:58 -0300 Subject: [PATCH 010/113] fix(mobile): the Android composer no longer lags the keyboard or drops under the nav bar (#19980) Co-authored-by: Claude Opus 5.5 Co-authored-by: Julius Marminge --- .../features/threads/ThreadDetailScreen.tsx | 25 ++++++++----------- .../src/features/threads/ThreadFeed.tsx | 2 ++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index ac22a8b7d9e10..2b36b8b095947 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -493,16 +493,13 @@ export const ThreadDetailScreen = function ThreadDetailScreen(props: ThreadDetai const [anchorMessageId, setAnchorMessageId] = useState(null); const [submittedMessageId, setSubmittedMessageId] = useState(null); const [endFollowEnabled, setEndFollowEnabled] = useState(true); - // Android keys the safe-area padding on keyboard visibility (#5988): the back gesture closes - // the keyboard while the editor stays focused, and a focus-keyed inset would leave the toolbar - // under the gesture bar. iOS must NOT use visibility — it only flips on keyboardDidHide, after - // the hide animation, so the composer would ride down flush to the screen edge and then snap up - // into the inset. - const composerBottomInset = ( - Platform.OS === "android" ? isKeyboardVisible : composerExpanded || composerFocused - ) - ? 0 - : Math.max(insets.bottom, 12); + // Android Back hides the keyboard without blurring. Keep its safe-area padding + // fixed through the animation and cancel it with matching composer/list offsets. + const composerBottomInset = + Platform.OS === "ios" && (composerExpanded || composerFocused) + ? 0 + : Math.max(insets.bottom, 12); + const composerKeyboardOffset = Platform.OS === "android" ? composerBottomInset : 0; const contentPresentationKind = props.contentPresentation.kind; const realThreadSyncLabel = (() => { switch (props.threadSyncStatus) { @@ -1145,6 +1142,7 @@ export const ThreadDetailScreen = function ThreadDetailScreen(props: ThreadDetai anchorMessageId={anchorMessageId} submittedMessageId={submittedMessageId} contentInsetEndAdjustment={combinedContentInsetEndAdjustment} + keyboardOffset={composerKeyboardOffset} contentTopInset={0} contentBottomInset={ estimatedOverlayHeight + @@ -1180,12 +1178,11 @@ export const ThreadDetailScreen = function ThreadDetailScreen(props: ThreadDetai {showContent ? (