From 0deafe6ac114e7cb41ddf23c3da9f11a1f3eda84 Mon Sep 17 00:00:00 2001 From: lynnswap <65545348+lynnswap@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:21:29 +0900 Subject: [PATCH 1/5] fix(proxy): explain Xcode MCP access timeouts Document the global Xcode MCP access switch separately from per-connection approval. Add actionable, non-causal recovery metadata to tools/list timeout logs so operators can diagnose silent attachment failures. --- Docs/troubleshooting.md | 34 ++++++++++++++++++- README.md | 19 +++++++++-- .../Runtime/RuntimeCoordinator+Health.swift | 20 +++++++---- ...rdinator+XcodeProcessRouteActivation.swift | 4 +++ .../Session/Support/ProxyLogging.swift | 10 ++++++ .../ProxyLoggingTests.swift | 16 +++++++++ 6 files changed, 93 insertions(+), 10 deletions(-) diff --git a/Docs/troubleshooting.md b/Docs/troubleshooting.md index c74a8064..14850204 100644 --- a/Docs/troubleshooting.md +++ b/Docs/troubleshooting.md @@ -8,7 +8,12 @@ If it fails: - Confirm `xcrun mcpbridge -h` works in Terminal. ## `MCP client ... timed out` -Ensure the proxy server is running. Increase `startup_timeout_sec` in the client config if needed. +Ensure the proxy server is running. Before increasing any timeout, confirm that +Xcode MCP access is enabled as described in +[Set Up Your MCP Client](../README.md#1-enable-xcode-mcp-access). If the proxy +logs `route_activation_timeout` or `attach_probe_timeout`, follow the dedicated +section below. Increase `startup_timeout_sec` only when setup is correct and the +upstream is still starting too slowly. If you see an error like: @@ -20,6 +25,31 @@ it’s usually because the upstream (`xcrun mcpbridge` / Xcode) was slow on the - The tool list cache is **not persisted to disk**. It survives repeated Codex restarts as long as the proxy server stays running. - `tools/list` is intentionally treated as stable for the lifetime of the proxy process (no background refresh), to avoid upstream churn and surprise Xcode permission dialogs. +## `route_activation_timeout` / `attach_probe_timeout` +These logs mean `mcpbridge` initialized, but Xcode did not return a usable +`tools/list` response before the route or bridge-attachment deadline. + +First, open your project in Xcode, choose **Xcode > Settings > Intelligence**, +and turn on **Allow external agents to use Xcode tools** under +**Model Context Protocol**. This global switch is required by +[Xcode's external-agent setup][apple-xcode-mcp-access]. + +`--auto-approve` only handles the per-connection **Allow** dialog; it does not +enable the global Xcode setting. If the setting is already on: + +- Approve any pending Xcode connection dialog. +- When using `--auto-approve`, allow the app that launched the proxy (for + example, Terminal or iTerm) in + **System Settings > Privacy & Security > Accessibility**. +- Wait for the proxy's automatic retry. A recovered route logs + `route_activation_cataloged`; a recovered secondary bridge logs + `bridge_pool_attach_verification_completed` with `success=true`. + +When Xcode does not send a JSON-RPC response for `tools/list`, the proxy cannot +recover Xcode's internal error from the stdio transport. The timeout log +therefore includes a `recovery_action` instead of claiming a specific upstream +failure. + ## Streamable HTTP client cannot connect - Ensure `xcode-mcp-proxy-server` is running. - Confirm the URL is correct (default: `http://localhost:8765/mcp`). @@ -90,3 +120,5 @@ Ensure the client is using the server-issued `MCP-Session-Id`. Initialize reques ## `protocol version required` / `protocol version mismatch` The proxy only accepts `MCP-Protocol-Version: 2025-06-18` after initialize. Reinitialize the client session if it cached an older protocol version or omitted the header. + +[apple-xcode-mcp-access]: https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode diff --git a/README.md b/README.md index ea21fe3a..e149fff3 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,24 @@ source ~/.zshrc ## Set Up Your MCP Client -### 1. Start the Proxy Server +### 1. Enable Xcode MCP Access + +Open your project in Xcode, choose **Xcode > Settings > Intelligence**, and turn +on **Allow external agents to use Xcode tools** under **Model Context Protocol**. +See [Giving external agents access to Xcode][apple-xcode-mcp-access]. + +This global Xcode setting is separate from the per-connection **Allow** dialog. +`--auto-approve` handles the dialog; it does not enable Xcode MCP access. + +### 2. Start the Proxy Server ```bash xcode-mcp-proxy-server --auto-approve ``` -`--auto-approve` clicks the Xcode **Allow** button automatically. It requires macOS Accessibility permission. +`--auto-approve` clicks the Xcode **Allow** button automatically. In +**System Settings > Privacy & Security > Accessibility**, allow the app that +launches the proxy (for example, Terminal or iTerm). Without Accessibility permission, omit `--auto-approve` and click **Allow** yourself: @@ -71,7 +82,7 @@ Without Accessibility permission, omit `--auto-approve` and click **Allow** your xcode-mcp-proxy-server ``` -### 2. Register the Client +### 3. Register the Client Replace `xcrun mcpbridge` with the proxy endpoint. @@ -234,3 +245,5 @@ Edit the draft release notes, then publish the release manually. ## License [LICENSE](LICENSE) + +[apple-xcode-mcp-access]: https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift index dafa3772..ae326bc2 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift @@ -1,4 +1,5 @@ import Foundation +import Logging import NIO import NIOFoundationCompat import XcodeMCPKit @@ -298,14 +299,21 @@ extension RuntimeCoordinator { case .processBridgeAttachment(let verification) = probe.purpose else { return } + var metadata: Logger.Metadata = [ + "pid": .string("\(verification.recovery.routeID.processID)"), + "upstream": .string("\(probe.upstreamIndex)"), + "success": .string(success ? "true" : "false"), + "reason": .string(reason), + "method": .string("tools/list"), + ] + if let recoveryAction = XcodeMCPToolsAvailabilityDiagnostic.action( + forAttachProbeFailureReason: reason + ) { + metadata["recovery_action"] = .string(recoveryAction) + } logger.info( "bridge_pool_attach_verification_completed", - metadata: [ - "pid": .string("\(verification.recovery.routeID.processID)"), - "upstream": .string("\(probe.upstreamIndex)"), - "success": .string(success ? "true" : "false"), - "reason": .string(reason), - ] + metadata: metadata ) } diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift index 64194fda..79e567e3 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift @@ -495,10 +495,14 @@ extension RuntimeCoordinator { "upstream": .string("\(lease.upstreamIndex)"), "attempt": .string("\(lease.attempt)"), "phase": .string("catalog"), + "method": .string("tools/list"), "timeout_ms": .string( processRouteActivationCatalogTimeoutMillisecondsDescription() ), "retry_delay_ms": .string("\(retry.delayMilliseconds)"), + "recovery_action": .string( + XcodeMCPToolsAvailabilityDiagnostic.enableToolsAction + ), ] ) diff --git a/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift b/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift index a78b62bc..a83d48d0 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift @@ -69,3 +69,13 @@ enum LogLevelParser { } } } + +enum XcodeMCPToolsAvailabilityDiagnostic { + static let enableToolsAction = + "Check that \"Allow external agents to use Xcode tools\" is enabled in " + + "Xcode > Settings > Intelligence. If Xcode shows a connection dialog, approve it." + + static func action(forAttachProbeFailureReason reason: String) -> String? { + reason == "timeout" ? enableToolsAction : nil + } +} diff --git a/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift b/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift index 7a91c047..339023fd 100644 --- a/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift +++ b/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift @@ -24,4 +24,20 @@ struct ProxyLoggingTests { ) #expect(level == .error) } + + @Test func toolsAvailabilityDiagnosticExplainsAttachTimeout() { + let expectedAction = + "Check that \"Allow external agents to use Xcode tools\" is enabled in " + + "Xcode > Settings > Intelligence. If Xcode shows a connection dialog, approve it." + #expect( + XcodeMCPToolsAvailabilityDiagnostic.action( + forAttachProbeFailureReason: "timeout" + ) == expectedAction + ) + #expect( + XcodeMCPToolsAvailabilityDiagnostic.action( + forAttachProbeFailureReason: "invalid_response" + ) == nil + ) + } } From 884b19ae7b4ece36c93f022ff1ee6b224abb4e14 Mon Sep 17 00:00:00 2001 From: lynnswap <65545348+lynnswap@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:39:07 +0900 Subject: [PATCH 2/5] fix(proxy): make Xcode timeout guidance readable Keep retry events concise and emit a multiline warning only for the first tools/list timeout. Use control-plane retry ordinals for deduplication and document the actual bridge attachment event name. --- Docs/troubleshooting.md | 22 +++++++------ .../ProcessControlPlaneAuthority.swift | 4 ++- .../Runtime/RuntimeCoordinator+Health.swift | 20 +++++------- ...ordinator+XcodeProcessReconciliation.swift | 10 ++++++ ...rdinator+XcodeProcessRouteActivation.swift | 12 +++++-- .../Session/Support/ProxyLogging.swift | 31 ++++++++++++++++--- .../ControlPlaneAuthorityTests.swift | 5 +++ .../ProxyLoggingTests.swift | 24 +++++++------- 8 files changed, 85 insertions(+), 43 deletions(-) diff --git a/Docs/troubleshooting.md b/Docs/troubleshooting.md index 14850204..cdd2526a 100644 --- a/Docs/troubleshooting.md +++ b/Docs/troubleshooting.md @@ -11,9 +11,10 @@ If it fails: Ensure the proxy server is running. Before increasing any timeout, confirm that Xcode MCP access is enabled as described in [Set Up Your MCP Client](../README.md#1-enable-xcode-mcp-access). If the proxy -logs `route_activation_timeout` or `attach_probe_timeout`, follow the dedicated -section below. Increase `startup_timeout_sec` only when setup is correct and the -upstream is still starting too slowly. +logs `route_activation_timeout`, or `bridge_pool_attach_verification_completed` +with `success=false reason=timeout`, follow the dedicated section below. +Increase `startup_timeout_sec` only when setup is correct and the upstream is +still starting too slowly. If you see an error like: @@ -25,9 +26,11 @@ it’s usually because the upstream (`xcrun mcpbridge` / Xcode) was slow on the - The tool list cache is **not persisted to disk**. It survives repeated Codex restarts as long as the proxy server stays running. - `tools/list` is intentionally treated as stable for the lifetime of the proxy process (no background refresh), to avoid upstream churn and surprise Xcode permission dialogs. -## `route_activation_timeout` / `attach_probe_timeout` -These logs mean `mcpbridge` initialized, but Xcode did not return a usable -`tools/list` response before the route or bridge-attachment deadline. +## Xcode tools are unavailable +`route_activation_timeout`, or `bridge_pool_attach_verification_completed` with +`success=false reason=timeout`, means `mcpbridge` initialized but Xcode did not +return a usable `tools/list` response before the route or bridge-attachment +deadline. First, open your project in Xcode, choose **Xcode > Settings > Intelligence**, and turn on **Allow external agents to use Xcode tools** under @@ -46,9 +49,10 @@ enable the global Xcode setting. If the setting is already on: `bridge_pool_attach_verification_completed` with `success=true`. When Xcode does not send a JSON-RPC response for `tools/list`, the proxy cannot -recover Xcode's internal error from the stdio transport. The timeout log -therefore includes a `recovery_action` instead of claiming a specific upstream -failure. +recover Xcode's internal error from the stdio transport. The first timeout +therefore prints an **Xcode tools are unavailable** warning with recovery steps +and continues retrying automatically. Later retries keep the structured event +but do not repeat the warning. ## Streamable HTTP client cannot connect - Ensure `xcode-mcp-proxy-server` is running. diff --git a/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift b/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift index 06c40013..a13ea5d9 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift @@ -101,6 +101,7 @@ struct ProcessBridgeRecovery: Sendable, Hashable { struct ProcessBridgeRecoveryRetry: Sendable { let reservation: ProcessBridgePoolRecovery let delay: TimeAmount + let consecutiveFailureCount: Int } struct ProcessControlPlaneTransition: Sendable { @@ -2609,7 +2610,8 @@ final class ProcessControlPlaneAuthority: Sendable { reservation: recovery, delay: owner.bridgeRecovery.consecutiveFailureCount == 1 ? .seconds(1) - : .seconds(10) + : .seconds(10), + consecutiveFailureCount: owner.bridgeRecovery.consecutiveFailureCount ) } diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift index ae326bc2..b799c4fd 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+Health.swift @@ -299,21 +299,15 @@ extension RuntimeCoordinator { case .processBridgeAttachment(let verification) = probe.purpose else { return } - var metadata: Logger.Metadata = [ - "pid": .string("\(verification.recovery.routeID.processID)"), - "upstream": .string("\(probe.upstreamIndex)"), - "success": .string(success ? "true" : "false"), - "reason": .string(reason), - "method": .string("tools/list"), - ] - if let recoveryAction = XcodeMCPToolsAvailabilityDiagnostic.action( - forAttachProbeFailureReason: reason - ) { - metadata["recovery_action"] = .string(recoveryAction) - } logger.info( "bridge_pool_attach_verification_completed", - metadata: metadata + metadata: [ + "pid": .string("\(verification.recovery.routeID.processID)"), + "upstream": .string("\(probe.upstreamIndex)"), + "success": .string(success ? "true" : "false"), + "reason": .string(reason), + "method": .string("tools/list"), + ] ) } diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift index 381085f2..6a90bfb2 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift @@ -404,8 +404,18 @@ extension RuntimeCoordinator { "upstream": .string("\(retry.reservation.upstreamID.rawValue)"), "reason": .string(reason), "delay_ms": .string("\(retry.delay.nanoseconds / 1_000_000)"), + "consecutive_failures": .string("\(retry.consecutiveFailureCount)"), ] ) + if reason == "attach_probe_timeout", + retry.consecutiveFailureCount == 1 { + XcodeMCPToolsAvailabilityDiagnostic.logTimeout( + logger: logger, + processID: retry.reservation.routeID.processID, + upstreamIndex: retry.reservation.upstreamID.rawValue, + retryDelayMilliseconds: retry.delay.nanoseconds / 1_000_000 + ) + } let timeout = scheduleRuntimeTimeout(retry.delay) { [weak self] in guard let self else { return } self.applyProcessControlPlaneTransition( diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift index 79e567e3..443303cb 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift @@ -499,12 +499,18 @@ extension RuntimeCoordinator { "timeout_ms": .string( processRouteActivationCatalogTimeoutMillisecondsDescription() ), + "catalog_timeout_count": .string("\(retry.attempt)"), "retry_delay_ms": .string("\(retry.delayMilliseconds)"), - "recovery_action": .string( - XcodeMCPToolsAvailabilityDiagnostic.enableToolsAction - ), ] ) + if retry.attempt == 1 { + XcodeMCPToolsAvailabilityDiagnostic.logTimeout( + logger: logger, + processID: lease.processID, + upstreamIndex: lease.upstreamIndex, + retryDelayMilliseconds: retry.delayMilliseconds + ) + } scheduleMissingProcessToolsCatalogRetry( processID: lease.processID, diff --git a/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift b/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift index a83d48d0..b80812a4 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Support/ProxyLogging.swift @@ -71,11 +71,32 @@ enum LogLevelParser { } enum XcodeMCPToolsAvailabilityDiagnostic { - static let enableToolsAction = - "Check that \"Allow external agents to use Xcode tools\" is enabled in " - + "Xcode > Settings > Intelligence. If Xcode shows a connection dialog, approve it." + static let timeoutSummary = """ + Xcode tools are unavailable - static func action(forAttachProbeFailureReason reason: String) -> String? { - reason == "timeout" ? enableToolsAction : nil + The proxy timed out waiting for tools/list. + Recovery: + 1. Open a project in Xcode. + 2. Check that "Allow external agents to use Xcode tools" is enabled in + Xcode > Settings > Intelligence. + 3. If Xcode shows a connection dialog, approve it. + The proxy will retry automatically. + """ + + static func logTimeout( + logger: Logger, + processID: pid_t, + upstreamIndex: Int, + retryDelayMilliseconds: Int64 + ) { + logger.warning( + "\(timeoutSummary)", + metadata: [ + "pid": .string("\(processID)"), + "upstream": .string("\(upstreamIndex)"), + "method": .string("tools/list"), + "retry_delay_ms": .string("\(retryDelayMilliseconds)"), + ] + ) } } diff --git a/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift b/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift index 57a7745e..d0efe89a 100644 --- a/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift +++ b/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift @@ -134,6 +134,7 @@ struct ControlPlaneAuthorityTests { let firstRetry = try #require(authority.prepareBridgeRecoveryRetry(firstAttempt)) #expect(firstRetry.delay.nanoseconds == TimeAmount.seconds(1).nanoseconds) + #expect(firstRetry.consecutiveFailureCount == 1) guard case .restoreBridgePool(let secondAttempt) = authority .handleBridgeRecoveryRetryFired(firstRetry.reservation).effects.first else { Issue.record("expected early bridge recovery retry") @@ -160,6 +161,7 @@ struct ControlPlaneAuthorityTests { let periodicRetry = try #require(authority.prepareBridgeRecoveryRetry(secondAttempt)) #expect(periodicRetry.delay.nanoseconds == TimeAmount.seconds(10).nanoseconds) + #expect(periodicRetry.consecutiveFailureCount == 2) guard case .restoreBridgePool(let thirdAttempt) = authority .handleBridgeRecoveryRetryFired(periodicRetry.reservation).effects.first else { Issue.record("expected periodic bridge recovery retry") @@ -173,6 +175,7 @@ struct ControlPlaneAuthorityTests { #expect(nextSlot.upstreamID == UpstreamSlotID(rawValue: 2)) let resetRetry = try #require(authority.prepareBridgeRecoveryRetry(nextSlot)) #expect(resetRetry.delay.nanoseconds == TimeAmount.seconds(1).nanoseconds) + #expect(resetRetry.consecutiveFailureCount == 1) } @Test func bridgeRecoveryRetryContinuesWhenCatalogDisappears() throws { @@ -1534,6 +1537,7 @@ struct ControlPlaneAuthorityTests { } #expect(firstRetryLease.attempt == firstLease.attempt) + #expect(firstRetry.attempt == 1) #expect(firstRetry.delay == .milliseconds(250)) #expect(authority.beginCatalogAttempt( routeID: route.id, @@ -1562,6 +1566,7 @@ struct ControlPlaneAuthorityTests { } #expect(secondLease.attempt == firstLease.attempt) + #expect(secondRetry.attempt == 2) #expect(secondRetry.delay == .milliseconds(500)) } diff --git a/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift b/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift index 339023fd..52fc79f2 100644 --- a/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift +++ b/Tests/XcodeMCPProxyRuntimeTests/ProxyLoggingTests.swift @@ -25,19 +25,19 @@ struct ProxyLoggingTests { #expect(level == .error) } - @Test func toolsAvailabilityDiagnosticExplainsAttachTimeout() { - let expectedAction = - "Check that \"Allow external agents to use Xcode tools\" is enabled in " - + "Xcode > Settings > Intelligence. If Xcode shows a connection dialog, approve it." + @Test func toolsAvailabilityDiagnosticExplainsTimeoutAndRetry() { #expect( - XcodeMCPToolsAvailabilityDiagnostic.action( - forAttachProbeFailureReason: "timeout" - ) == expectedAction - ) - #expect( - XcodeMCPToolsAvailabilityDiagnostic.action( - forAttachProbeFailureReason: "invalid_response" - ) == nil + XcodeMCPToolsAvailabilityDiagnostic.timeoutSummary == """ + Xcode tools are unavailable + + The proxy timed out waiting for tools/list. + Recovery: + 1. Open a project in Xcode. + 2. Check that "Allow external agents to use Xcode tools" is enabled in + Xcode > Settings > Intelligence. + 3. If Xcode shows a connection dialog, approve it. + The proxy will retry automatically. + """ ) } } From 5844f04f381fe66fbf0f20543b21c5cf1c3581a0 Mon Sep 17 00:00:00 2001 From: lynnswap <65545348+lynnswap@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:13:27 +0900 Subject: [PATCH 3/5] fix(proxy): track timeout diagnostics per incident --- .../ProcessControlPlaneAuthority.swift | 38 +++++++-- ...ordinator+XcodeProcessReconciliation.swift | 11 ++- ...rdinator+XcodeProcessRouteActivation.swift | 7 +- .../ControlPlaneAuthorityTests.swift | 79 ++++++++++++++++--- 4 files changed, 110 insertions(+), 25 deletions(-) diff --git a/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift b/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift index a13ea5d9..633a7974 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/ControlPlane/ProcessControlPlaneAuthority.swift @@ -98,10 +98,16 @@ struct ProcessBridgeRecovery: Sendable, Hashable { var upstreamID: UpstreamSlotID { reservation.upstreamID } } +enum ProcessBridgeRecoveryFailure: Sendable, Equatable { + case toolsListTimeout + case other +} + struct ProcessBridgeRecoveryRetry: Sendable { let reservation: ProcessBridgePoolRecovery let delay: TimeAmount let consecutiveFailureCount: Int + let shouldLogToolsUnavailableWarning: Bool } struct ProcessControlPlaneTransition: Sendable { @@ -307,13 +313,14 @@ final class ProcessControlPlaneAuthority: Sendable { case retryRequired( transition: ProcessControlPlaneTransition, retry: Retry, - catalogLease: CatalogLease + catalogLease: CatalogLease, + timeoutCount: Int ) var transition: ProcessControlPlaneTransition { switch self { case .loadTimedOut(let transition), - .retryRequired(let transition, _, _): + .retryRequired(let transition, _, _, _): return transition } } @@ -407,6 +414,7 @@ final class ProcessControlPlaneAuthority: Sendable { var readinessToken: UpstreamReadinessWaiterToken? var retryTimeout: RuntimeScheduledTimeout? var retryKind: RetryKind? + var catalogTimeoutCount: Int var nextLoadID: Int var loads: [CatalogLoadID: Load] @@ -570,6 +578,7 @@ final class ProcessControlPlaneAuthority: Sendable { var phase: Phase = .idle var generation: UInt64 = 0 var consecutiveFailureCount = 0 + var didLogToolsUnavailableWarning = false mutating func reset(pendingUpstreamIDs: Set) -> [ProcessControlPlaneEffect] @@ -584,6 +593,7 @@ final class ProcessControlPlaneAuthority: Sendable { self.pendingUpstreamIDs = pendingUpstreamIDs phase = .idle consecutiveFailureCount = 0 + didLogToolsUnavailableWarning = false return effects } } @@ -694,6 +704,7 @@ final class ProcessControlPlaneAuthority: Sendable { let previousOwner = owner owner.bridgeRecovery.phase = .idle owner.bridgeRecovery.consecutiveFailureCount = 0 + owner.bridgeRecovery.didLogToolsUnavailableWarning = false let effects = Self.takeBridgePoolRecoveryEffects(owner: &owner) state.recordsByKey[key] = owner guard commit() else { @@ -719,13 +730,15 @@ final class ProcessControlPlaneAuthority: Sendable { } func prepareBridgeRecoveryRetry( - _ recovery: ProcessBridgePoolRecovery + _ recovery: ProcessBridgePoolRecovery, + failure: ProcessBridgeRecoveryFailure ) -> ProcessBridgeRecoveryRetry? { state.withLockedValue { state in guard let key = Self.key(routeID: recovery.routeID, in: state), var owner = state.recordsByKey[key], let retry = Self.prepareBridgeRecoveryRetry( recovery, + failure: failure, owner: &owner ) else { return nil } state.recordsByKey[key] = owner @@ -1298,6 +1311,7 @@ final class ProcessControlPlaneAuthority: Sendable { readinessToken: readinessToken, retryTimeout: nil, retryKind: nil, + catalogTimeoutCount: 0, nextLoadID: 0, loads: [:] ) @@ -1541,6 +1555,7 @@ final class ProcessControlPlaneAuthority: Sendable { readinessToken: nil, retryTimeout: nil, retryKind: nil, + catalogTimeoutCount: 0, nextLoadID: 0, loads: [:] ) @@ -1593,6 +1608,7 @@ final class ProcessControlPlaneAuthority: Sendable { readinessToken: nil, retryTimeout: nil, retryKind: nil, + catalogTimeoutCount: 0, nextLoadID: 0, loads: [:] ) @@ -2222,6 +2238,7 @@ final class ProcessControlPlaneAuthority: Sendable { attempt.retryTimeout = nil attempt.retryKind = .catalog attempt.phase = .backoff + attempt.catalogTimeoutCount &+= 1 record.catalogRetryCount &+= 1 record.attempt = attempt state.recordsByKey[key] = record @@ -2233,7 +2250,8 @@ final class ProcessControlPlaneAuthority: Sendable { publishesToolsListChanged: false ), retry: Self.retry(forAttempt: record.catalogRetryCount), - catalogLease: lease + catalogLease: lease, + timeoutCount: attempt.catalogTimeoutCount ) } } @@ -2296,6 +2314,7 @@ final class ProcessControlPlaneAuthority: Sendable { rejectedBridgeRecovery.upstreamID == failedProof.slotID, let retry = Self.prepareBridgeRecoveryRetry( rejectedBridgeRecovery, + failure: .other, owner: &record ) { state.recordsByKey[key] = record @@ -2600,18 +2619,26 @@ final class ProcessControlPlaneAuthority: Sendable { private static func prepareBridgeRecoveryRetry( _ recovery: ProcessBridgePoolRecovery, + failure: ProcessBridgeRecoveryFailure, owner: inout XcodeProcessOwner ) -> ProcessBridgeRecoveryRetry? { guard case .attempting(let current) = owner.bridgeRecovery.phase, current == recovery else { return nil } owner.bridgeRecovery.consecutiveFailureCount += 1 + let shouldLogToolsUnavailableWarning = + failure == .toolsListTimeout + && owner.bridgeRecovery.didLogToolsUnavailableWarning == false + if shouldLogToolsUnavailableWarning { + owner.bridgeRecovery.didLogToolsUnavailableWarning = true + } owner.bridgeRecovery.phase = .waitingRetry(recovery, nil) return ProcessBridgeRecoveryRetry( reservation: recovery, delay: owner.bridgeRecovery.consecutiveFailureCount == 1 ? .seconds(1) : .seconds(10), - consecutiveFailureCount: owner.bridgeRecovery.consecutiveFailureCount + consecutiveFailureCount: owner.bridgeRecovery.consecutiveFailureCount, + shouldLogToolsUnavailableWarning: shouldLogToolsUnavailableWarning ) } @@ -2667,6 +2694,7 @@ final class ProcessControlPlaneAuthority: Sendable { readinessToken: nil, retryTimeout: nil, retryKind: .activation, + catalogTimeoutCount: 0, nextLoadID: 0, loads: [:] ) diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift index 6a90bfb2..dc9982b8 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessReconciliation.swift @@ -386,7 +386,10 @@ extension RuntimeCoordinator { ) { var retry: ProcessBridgeRecoveryRetry? guard initializeManager.performIfRunning({ - retry = processControlPlane.prepareBridgeRecoveryRetry(recovery) + retry = processControlPlane.prepareBridgeRecoveryRetry( + recovery, + failure: reason == "attach_probe_timeout" ? .toolsListTimeout : .other + ) }), let retry else { return } @@ -407,8 +410,7 @@ extension RuntimeCoordinator { "consecutive_failures": .string("\(retry.consecutiveFailureCount)"), ] ) - if reason == "attach_probe_timeout", - retry.consecutiveFailureCount == 1 { + if retry.shouldLogToolsUnavailableWarning { XcodeMCPToolsAvailabilityDiagnostic.logTimeout( logger: logger, processID: retry.reservation.routeID.processID, @@ -449,7 +451,8 @@ extension RuntimeCoordinator { var retry: ProcessBridgeRecoveryRetry? guard initializeManager.performIfRunning({ retry = processControlPlane.prepareBridgeRecoveryRetry( - recovery.reservation + recovery.reservation, + failure: reason == "attach_probe_timeout" ? .toolsListTimeout : .other ) }), let retry else { return } guard let replacement else { diff --git a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift index 443303cb..f7b8120e 100644 --- a/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift +++ b/Sources/XcodeMCPProxyRuntime/Session/Runtime/RuntimeCoordinator+XcodeProcessRouteActivation.swift @@ -483,7 +483,8 @@ extension RuntimeCoordinator { guard case .retryRequired( _, let retry, - let retryLease + let retryLease, + let catalogTimeoutCount ) = timeout else { return } @@ -499,11 +500,11 @@ extension RuntimeCoordinator { "timeout_ms": .string( processRouteActivationCatalogTimeoutMillisecondsDescription() ), - "catalog_timeout_count": .string("\(retry.attempt)"), + "catalog_timeout_count": .string("\(catalogTimeoutCount)"), "retry_delay_ms": .string("\(retry.delayMilliseconds)"), ] ) - if retry.attempt == 1 { + if catalogTimeoutCount == 1 { XcodeMCPToolsAvailabilityDiagnostic.logTimeout( logger: logger, processID: lease.processID, diff --git a/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift b/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift index d0efe89a..d86c3b5c 100644 --- a/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift +++ b/Tests/XcodeMCPProxyRuntimeTests/ControlPlaneAuthorityTests.swift @@ -132,9 +132,13 @@ struct ControlPlaneAuthorityTests { } #expect(firstAttempt.upstreamID == UpstreamSlotID(rawValue: 1)) - let firstRetry = try #require(authority.prepareBridgeRecoveryRetry(firstAttempt)) + let firstRetry = try #require(authority.prepareBridgeRecoveryRetry( + firstAttempt, + failure: .other + )) #expect(firstRetry.delay.nanoseconds == TimeAmount.seconds(1).nanoseconds) #expect(firstRetry.consecutiveFailureCount == 1) + #expect(firstRetry.shouldLogToolsUnavailableWarning == false) guard case .restoreBridgePool(let secondAttempt) = authority .handleBridgeRecoveryRetryFired(firstRetry.reservation).effects.first else { Issue.record("expected early bridge recovery retry") @@ -159,23 +163,41 @@ struct ControlPlaneAuthorityTests { #expect(cancelled.withLockedValue { $0 }) #expect(authority.handleBridgeRecoveryRetryFired(firstRetry.reservation).effects.isEmpty) - let periodicRetry = try #require(authority.prepareBridgeRecoveryRetry(secondAttempt)) + let periodicRetry = try #require(authority.prepareBridgeRecoveryRetry( + secondAttempt, + failure: .toolsListTimeout + )) #expect(periodicRetry.delay.nanoseconds == TimeAmount.seconds(10).nanoseconds) #expect(periodicRetry.consecutiveFailureCount == 2) + #expect(periodicRetry.shouldLogToolsUnavailableWarning) guard case .restoreBridgePool(let thirdAttempt) = authority .handleBridgeRecoveryRetryFired(periodicRetry.reservation).effects.first else { Issue.record("expected periodic bridge recovery retry") return } + let repeatedTimeoutRetry = try #require(authority.prepareBridgeRecoveryRetry( + thirdAttempt, + failure: .toolsListTimeout + )) + #expect(repeatedTimeoutRetry.shouldLogToolsUnavailableWarning == false) + guard case .restoreBridgePool(let recoveredAttempt) = authority + .handleBridgeRecoveryRetryFired(repeatedTimeoutRetry.reservation).effects.first else { + Issue.record("expected repeated timeout recovery retry") + return + } guard case .restoreBridgePool(let nextSlot) = authority - .completeBridgeRecovery(thirdAttempt).effects.first else { + .completeBridgeRecovery(recoveredAttempt).effects.first else { Issue.record("expected next serialized bridge slot") return } #expect(nextSlot.upstreamID == UpstreamSlotID(rawValue: 2)) - let resetRetry = try #require(authority.prepareBridgeRecoveryRetry(nextSlot)) + let resetRetry = try #require(authority.prepareBridgeRecoveryRetry( + nextSlot, + failure: .toolsListTimeout + )) #expect(resetRetry.delay.nanoseconds == TimeAmount.seconds(1).nanoseconds) #expect(resetRetry.consecutiveFailureCount == 1) + #expect(resetRetry.shouldLogToolsUnavailableWarning) } @Test func bridgeRecoveryRetryContinuesWhenCatalogDisappears() throws { @@ -196,7 +218,10 @@ struct ControlPlaneAuthorityTests { Issue.record("expected bridge recovery after catalog commit") return } - let retry = try #require(authority.prepareBridgeRecoveryRetry(firstAttempt)) + let retry = try #require(authority.prepareBridgeRecoveryRetry( + firstAttempt, + failure: .other + )) _ = authority.invalidateCatalogSource( processID: target.processID, @@ -209,7 +234,10 @@ struct ControlPlaneAuthorityTests { return } #expect(resumedAttempt.upstreamID == firstAttempt.upstreamID) - let periodicRetry = try #require(authority.prepareBridgeRecoveryRetry(resumedAttempt)) + let periodicRetry = try #require(authority.prepareBridgeRecoveryRetry( + resumedAttempt, + failure: .other + )) #expect(periodicRetry.delay.nanoseconds == TimeAmount.seconds(10).nanoseconds) } @@ -455,7 +483,10 @@ struct ControlPlaneAuthorityTests { #expect(retry.reservation == rejectedRecovery) #expect(retry.delay == .seconds(1)) #expect(authority.attemptSnapshot(processID: target.processID) == nil) - #expect(authority.prepareBridgeRecoveryRetry(rejectedRecovery) == nil) + #expect(authority.prepareBridgeRecoveryRetry( + rejectedRecovery, + failure: .other + ) == nil) guard case .restoreBridgePool(let retried) = authority .handleBridgeRecoveryRetryFired(retry.reservation).effects.first else { Issue.record("expected the atomic retry to remain schedulable") @@ -474,7 +505,10 @@ struct ControlPlaneAuthorityTests { Issue.record("expected initial bridge recovery") return } - let staleRetry = try #require(authority.prepareBridgeRecoveryRetry(staleRecovery)) + let staleRetry = try #require(authority.prepareBridgeRecoveryRetry( + staleRecovery, + failure: .other + )) guard case .restoreBridgePool(let currentRecovery) = authority .handleBridgeRecoveryRetryFired(staleRetry.reservation).effects.first else { Issue.record("expected a newer bridge recovery") @@ -495,7 +529,10 @@ struct ControlPlaneAuthorityTests { #expect(attemptingTransition.effects.isEmpty) #expect(authority.validateBridgeRecovery(currentRecovery)) - let currentRetry = try #require(authority.prepareBridgeRecoveryRetry(currentRecovery)) + let currentRetry = try #require(authority.prepareBridgeRecoveryRetry( + currentRecovery, + failure: .other + )) let waitingResult = try #require(authority.recoverAfterRejectedCancellation( routeID: route.id, failedProof: testTopologyProof(1), @@ -773,7 +810,10 @@ struct ControlPlaneAuthorityTests { Issue.record("expected bridge recovery") return } - let retry = try #require(authority.prepareBridgeRecoveryRetry(recovery)) + let retry = try #require(authority.prepareBridgeRecoveryRetry( + recovery, + failure: .other + )) let cancelled = NIOLockedValueBox(false) let timeout = RuntimeScheduledTimeout { cancelled.withLockedValue { $0 = true } @@ -1529,7 +1569,7 @@ struct ControlPlaneAuthorityTests { to: firstTimeoutReservation ) - guard case .retryRequired(_, let firstRetry, let firstRetryLease) = + guard case .retryRequired(_, let firstRetry, let firstRetryLease, let firstTimeoutCount) = authority.handleCatalogRequestTimeout(firstTimeoutReservation, nowUptimeNs: 2) else { Issue.record("expected the final load timeout to require retry") @@ -1539,6 +1579,7 @@ struct ControlPlaneAuthorityTests { #expect(firstRetryLease.attempt == firstLease.attempt) #expect(firstRetry.attempt == 1) #expect(firstRetry.delay == .milliseconds(250)) + #expect(firstTimeoutCount == 1) #expect(authority.beginCatalogAttempt( routeID: route.id, preferredUpstreamProof: proof, @@ -1558,7 +1599,7 @@ struct ControlPlaneAuthorityTests { RuntimeScheduledTimeout {}, to: secondTimeoutReservation ) - guard case .retryRequired(_, let secondRetry, _) = + guard case .retryRequired(_, let secondRetry, _, let secondTimeoutCount) = authority.handleCatalogRequestTimeout(secondTimeoutReservation, nowUptimeNs: 5) else { Issue.record("expected the retry load timeout to require another retry") @@ -1568,6 +1609,7 @@ struct ControlPlaneAuthorityTests { #expect(secondLease.attempt == firstLease.attempt) #expect(secondRetry.attempt == 2) #expect(secondRetry.delay == .milliseconds(500)) + #expect(secondTimeoutCount == 2) } @Test func catalogTimeoutFiringBeforeAttachmentConsumesReservation() throws { @@ -1602,7 +1644,7 @@ struct ControlPlaneAuthorityTests { #expect(authority.validateCatalogLoad(lease) == false) } - @Test func staleCatalogTimeoutCannotTerminateNewerRetryLoad() throws { + @Test func catalogTimeoutCountIgnoresNonTimeoutRetryAndRejectsStaleReservation() throws { let target = xcodeProcessTarget(processID: 41033, xcodeVersion: "27.0") let authority = makeAuthority([(target, [0])]) let route = try #require(authority.route(forProcessID: target.processID)) @@ -1644,6 +1686,17 @@ struct ControlPlaneAuthorityTests { authority.attemptSnapshot(processID: target.processID)?.phase == .loadingCatalog ) + let retryTimeoutReservation = try #require( + authority.reserveCatalogTimeout(for: retryLease) + ) + guard case .retryRequired(_, let retry, _, let timeoutCount) = + authority.handleCatalogRequestTimeout(retryTimeoutReservation, nowUptimeNs: 5) + else { + Issue.record("expected the current load timeout to require retry") + return + } + #expect(retry.attempt == 2) + #expect(timeoutCount == 1) } @Test func catalogLoadTimeoutPreservesSiblingLoad() throws { From 057142c7831ad923589906ca51022b9de4df0465 Mon Sep 17 00:00:00 2001 From: lynnswap <65545348+lynnswap@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:13:37 +0900 Subject: [PATCH 4/5] test(proxy): drive queued request after timeout --- Tests/XcodeMCPProxyRuntimeTests/HTTPConcurrencyTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/XcodeMCPProxyRuntimeTests/HTTPConcurrencyTests.swift b/Tests/XcodeMCPProxyRuntimeTests/HTTPConcurrencyTests.swift index f1d953de..b30c8b06 100644 --- a/Tests/XcodeMCPProxyRuntimeTests/HTTPConcurrencyTests.swift +++ b/Tests/XcodeMCPProxyRuntimeTests/HTTPConcurrencyTests.swift @@ -354,6 +354,7 @@ struct HTTPConcurrencyTests { let firstObject = try jsonObject(from: firstResponse.body) #expect((firstObject["error"] as? [String: Any])?["message"] as? String == "upstream timeout") + await sessionManager.drainRuntimeTasksForTesting() secondChannel.embeddedEventLoop.run() await sessionManager.drainRuntimeTasksForTesting() let secondRequestLabels = try await waitForUpstreamRequestCount(upstream, count: 3) From 07269756160dfd701b68c76a16249022686169da Mon Sep 17 00:00:00 2001 From: lynnswap <65545348+lynnswap@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:06:57 +0900 Subject: [PATCH 5/5] test(proxy): synchronize cancellation delivery assertions --- .../RuntimeCoordinatorTests.swift | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/Tests/XcodeMCPProxyRuntimeTests/RuntimeCoordinatorTests.swift b/Tests/XcodeMCPProxyRuntimeTests/RuntimeCoordinatorTests.swift index 21391810..aeb67303 100644 --- a/Tests/XcodeMCPProxyRuntimeTests/RuntimeCoordinatorTests.swift +++ b/Tests/XcodeMCPProxyRuntimeTests/RuntimeCoordinatorTests.swift @@ -6808,7 +6808,11 @@ struct RuntimeCoordinatorRecoveryTests { ) } try await waitForSentCount(upstream, count: 3, timeoutSeconds: 2) - let firstRequest = try await sentValue(from: upstream, at: 2, timeout: .seconds(2)) + let firstRequest = try await sentValue( + from: upstream, + at: 2, + timeout: .seconds(2) + ) #expect(methodName(from: firstRequest) == "tools/list") try await upstream.waitForBlockedSend() try await advanceRuntimeCoordinatorTimeout( @@ -7073,6 +7077,7 @@ struct RuntimeCoordinatorRecoveryTests { let sessionID = "session-tools-shared-no-starvation" _ = manager.session(id: sessionID) + await upstream.blockNextCancellation() let firstTask = Task { try await manager.sharedToolsList( @@ -7080,7 +7085,8 @@ struct RuntimeCoordinatorRecoveryTests { requestTimeoutOverride: .seconds(5) ) } - _ = try await sentValue(from: upstream, at: 2, timeout: .seconds(2)) + let firstRequest = try await sentValue(from: upstream, at: 2, timeout: .seconds(2)) + #expect(methodName(from: firstRequest) == "tools/list") uptimeClock.advance(by: .nanoseconds(120_000_001)) @@ -7090,7 +7096,16 @@ struct RuntimeCoordinatorRecoveryTests { requestTimeoutOverride: .seconds(5) ) } - _ = try await sentValue(from: upstream, at: 3, timeout: .seconds(2)) + try await upstream.waitForBlockedCancellation() + let firstCancellation = try await sentValue( + from: upstream, + at: 3, + timeout: .seconds(2) + ) + #expect( + try extractCancellationRequestID(from: firstCancellation) + == extractUpstreamID(from: firstRequest) + ) uptimeClock.advance(by: .nanoseconds(120_000_001)) @@ -7106,7 +7121,9 @@ struct RuntimeCoordinatorRecoveryTests { $0.waiterCounts.toolsCatalog == 3 } } - #expect(await upstream.sentCount() == 4) + let sentCount = await upstream.sentCount() + #expect(sentCount == 4) + await upstream.releaseBlockedCancellation() firstTask.cancel() secondTask.cancel() @@ -7224,6 +7241,7 @@ struct RuntimeCoordinatorRecoveryTests { manager.refreshToolsListIfNeeded() let prewarmRequest = try await sentValue(from: upstream, at: 2, timeout: .seconds(2)) #expect(methodName(from: prewarmRequest) == "tools/list") + await upstream.blockNextCancellation() let sessionID = "session-tools-prewarm-timeout" _ = manager.session(id: sessionID) @@ -7248,6 +7266,7 @@ struct RuntimeCoordinatorRecoveryTests { await #expect(throws: TimeoutError.self) { _ = try await firstTask.value } + try await upstream.waitForBlockedCancellation() let prewarmCancellation = try await sentValue( from: upstream, at: 3, @@ -7257,6 +7276,7 @@ struct RuntimeCoordinatorRecoveryTests { try extractCancellationRequestID(from: prewarmCancellation) == extractUpstreamID(from: prewarmRequest) ) + await upstream.releaseBlockedCancellation() await manager.drainRuntimeTasksForTesting() #expect(manager.debugSnapshot().upstreams[0].activeCorrelatedRequestCount == 0)