diff --git a/apps/desktop/src/main/__tests__/runtime-host-collaboration-ipc-main.test.ts b/apps/desktop/src/main/__tests__/runtime-host-collaboration-ipc-main.test.ts index c6c3d190d9..3da977f0a2 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-collaboration-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-collaboration-ipc-main.test.ts @@ -129,11 +129,7 @@ test('requires plaintext confirmation and reports the issued invitation routes', name: 'Peer Lab', transport: { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.1/udp/41000/quic-v1'], - coordinationRelays: [ - '/dns4/relay.example/udp/443/quic-v1/p2p/12D3KooWrelay', - ], + reachability: peerReachability(), }, }), ); @@ -151,6 +147,22 @@ test('requires plaintext confirmation and reports the issued invitation routes', ); }); +function peerReachability() { + return { + lease: { + version: 1 as const, + peerId: '12D3KooWpeer', + revision: 1, + issuedAt: 1, + expiresAt: 2, + directRoutes: ['/ip4/192.0.2.1/udp/41000/quic-v1'], + coordinationRoutes: ['/dns4/relay.example/udp/443/quic-v1/p2p/12D3KooWrelay'], + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), + }; +} + test('treats an unavailable collaboration authority as an empty background inbox', async () => { const handlers = new Map(); registerRuntimeHostCollaborationIpc( diff --git a/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts b/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts index 6869b50584..f8573f5b22 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts @@ -22,6 +22,7 @@ import test from 'node:test'; import type { BotIncomingMessage } from '@maka/runtime/bots'; import { RuntimeHostOperationError, + RuntimeHostPermanentReconnectError, RuntimeHostRequestInterruptedError, type RuntimeHostSpawnedProcess, } from '@maka/runtime-host/client'; @@ -1018,6 +1019,88 @@ test('keeps Local explicitly usable without routing default work away from an un await manager.close(); }); +test('keeps an initially unavailable Direct target live and wakes it on new routes', async () => { + const local = candidateHarness(); + const remote = candidateHarness({ hostId: 'a'.repeat(64), ownership: 'external' }); + let starts = 0; + let routeListener: (() => void) | undefined; + let reportBackoff!: () => void; + const waitingForBackoff = new Promise((resolve) => { + reportBackoff = resolve; + }); + const manager = await startRuntimeHostDesktopManager( + { + peerClient: { + subscribeRoutes: (peerId: string, listener: () => void) => { + assert.equal(peerId, '12D3KooWpeer'); + routeListener = listener; + return () => undefined; + }, + }, + } as DesktopRuntimeHostCandidateStartInput, + { + startCandidate: async () => { + starts += 1; + if (starts === 1) return ready(local.candidate); + if (starts <= 3) return { kind: 'failed', reason: 'host_unresponsive' }; + return ready(remote.candidate); + }, + reconnectBackoff: { + minMs: 30_000, + maxMs: 30_000, + wait: (_delayMs, signal) => + new Promise((_resolve, reject) => { + reportBackoff(); + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }), + }, + }, + ); + + await manager.enable(peerTarget('office')); + await waitingForBackoff; + assert.equal( + manager.entries().find((state) => state.target.profile.id === 'office')?.readiness, + 'reconnecting', + ); + assert.equal(manager.current('office')?.readiness, 'reconnecting'); + assert.equal(manager.current('office')?.candidate, undefined); + assert.ok(routeListener); + + routeListener(); + await manager.waitUntilReady('office'); + assert.equal(manager.current('office')?.candidate, remote.candidate); + assert.equal(starts, 4); + await manager.close(); +}); + +test('does not activate a Direct target whose immediate retry fails permanently', async () => { + const local = candidateHarness(); + const permanent = new RuntimeHostPermanentReconnectError('credential rejected'); + let starts = 0; + const manager = await startRuntimeHostDesktopManager( + {} as DesktopRuntimeHostCandidateStartInput, + { + startCandidate: async () => { + starts += 1; + if (starts === 1) return ready(local.candidate); + if (starts === 2) throw new Error('route is temporarily unavailable'); + throw permanent; + }, + }, + ); + + await assert.rejects(manager.enable(peerTarget('office')), (error: unknown) => error === permanent); + assert.equal(starts, 3); + assert.equal(manager.current('office'), undefined); + const state = manager.entries().find((entry) => entry.target.profile.id === 'office'); + assert.equal(state?.readiness, 'unavailable'); + if (state?.readiness === 'unavailable') assert.equal(state.error, permanent); + await manager.close(); +}); + test('keeps reconnecting through transient startup failures until the Desktop adapter is restored', async () => { const first = candidateHarness(); const replacement = candidateHarness(); @@ -1682,6 +1765,13 @@ function remoteTarget( function peerGuestTarget( id: string, +): NonNullable { + return peerTarget(id, 'session_guest'); +} + +function peerTarget( + id: string, + access?: 'session_guest', ): NonNullable { return { profile: { @@ -1690,13 +1780,27 @@ function peerGuestTarget( kind: 'remote', transport: { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.1/udp/41000/quic-v1'], - coordinationRelays: [], + reachability: testPeerReachability('12D3KooWpeer'), }, rootId: 'a'.repeat(64), - access: 'session_guest', + ...(access ? { access } : {}), }, credential: 'credential-peer', }; } + +function testPeerReachability(peerId: string) { + return { + lease: { + version: 1 as const, + peerId, + revision: 1, + issuedAt: 1, + expiresAt: 2, + directRoutes: ['/ip4/192.0.2.1/udp/41000/quic-v1'], + coordinationRoutes: [], + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), + }; +} diff --git a/apps/desktop/src/main/__tests__/runtime-host-guest-session-mounts.test.ts b/apps/desktop/src/main/__tests__/runtime-host-guest-session-mounts.test.ts index a6546c70f4..39a7948302 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-guest-session-mounts.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-guest-session-mounts.test.ts @@ -19,7 +19,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import type { ResolvedRuntimeHostProfile } from '@maka/runtime-host/client'; +import { + RuntimeHostPermanentReconnectError, + type ResolvedRuntimeHostProfile, +} from '@maka/runtime-host/client'; import { encodeCollaborationInvitationCode, type HostPeerEndpoint, @@ -106,9 +109,7 @@ test('persists authenticated route rotation for reconnect and restart', async () target.profile.kind === 'remote' ? target.profile.transport : undefined, { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.1/udp/41000/quic-v1'], - coordinationRelays: ['/memory/stale-relay'], + reachability: guestPeerReachability(), }, ); assert.ok(onPeerEndpoint); @@ -118,19 +119,12 @@ test('persists authenticated route rotation for reconnect and restart', async () const imported = await first.importInvitation(peerInvitation('guest-routes'), false, 'routes'); assert.equal(imported.kind, 'connected'); - observePeerEndpoint({ - lease: { - version: 1, - peerId: '12D3KooWpeer', - revision: 2, - issuedAt: 1, - expiresAt: 2, - directRoutes: ['/ip4/198.51.100.2/udp/42000/quic-v1'], - coordinationRoutes: ['/memory/fresh-relay'], - }, - publicKey: 'AA', - signature: 'AA', - }); + const rotated = guestPeerReachability( + 2, + ['/ip4/198.51.100.2/udp/42000/quic-v1'], + ['/memory/fresh-relay'], + ); + observePeerEndpoint(rotated); await first.close(); let restarted!: ReturnType; @@ -141,9 +135,7 @@ test('persists authenticated route rotation for reconnect and restart', async () const target = await restartedTarget; assert.deepEqual(target.profile.kind === 'remote' ? target.profile.transport : undefined, { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: ['/ip4/198.51.100.2/udp/42000/quic-v1'], - coordinationRelays: ['/memory/fresh-relay'], + reachability: rotated, }); await restarted.close(); }); @@ -153,7 +145,9 @@ test('removes failed activation desire instead of creating recoverable profile s const unmounted: string[] = []; const mounts = service(store, { mount: async () => { - throw Object.assign(new Error('route missing'), { code: 'direct_path_unavailable' }); + throw Object.assign(new Error('route missing'), { + code: 'peer_reachability_needs_repair', + }); }, unmount: async (mountId) => { unmounted.push(mountId); @@ -166,6 +160,34 @@ test('removes failed activation desire instead of creating recoverable profile s assert.equal(unmounted.length, 1); }); +test('does not retry a startup mount whose reachability recovery is exhausted', async () => { + const store = memoryStore(); + await store.write([retainedMount('shared-needs-repair')]); + let attempts = 0; + let waits = 0; + let reportFailure!: () => void; + const failureReported = new Promise((resolve) => { + reportFailure = resolve; + }); + const mounts = service(store, { + mount: async () => { + attempts += 1; + throw new RuntimeHostPermanentReconnectError('reachability recovery exhausted'); + }, + wait: async () => { + waits += 1; + }, + onError: () => reportFailure(), + }); + + await mounts.start(); + await failureReported; + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(attempts, 1); + assert.equal(waits, 0); + await mounts.close(); +}); + test('settles admitted finalization before committing unmount desire', async () => { const store = memoryStore(); let started!: () => void; @@ -417,6 +439,8 @@ function service( readonly mount?: Parameters[0]['mount']; readonly finalizeAccess?: Parameters[0]['finalizeAccess']; readonly unmount?: Parameters[0]['unmount']; + readonly wait?: Parameters[0]['wait']; + readonly onError?: Parameters[0]['onError']; } = {}, ) { return createDesktopGuestSessionMountService({ @@ -424,7 +448,8 @@ function service( mount: overrides.mount ?? (async () => undefined), finalizeAccess: overrides.finalizeAccess ?? (async () => undefined), unmount: overrides.unmount ?? (async () => undefined), - onError: () => undefined, + ...(overrides.wait ? { wait: overrides.wait } : {}), + onError: overrides.onError ?? (() => undefined), }); } @@ -463,14 +488,32 @@ function peerInvitation(credential: string): string { name: 'Shared Host', transport: { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.1/udp/41000/quic-v1'], - coordinationRelays: ['/memory/stale-relay'], + reachability: guestPeerReachability(), }, }, }); } +function guestPeerReachability( + revision = 1, + directRoutes: readonly string[] = ['/ip4/192.0.2.1/udp/41000/quic-v1'], + coordinationRoutes: readonly string[] = ['/memory/stale-relay'], +) { + return { + lease: { + version: 1 as const, + peerId: '12D3KooWpeer', + revision, + issuedAt: 1, + expiresAt: 2, + directRoutes, + coordinationRoutes, + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), + }; +} + function retainedMount(mountId: string): GuestSessionMount { return { mountId, diff --git a/apps/desktop/src/main/__tests__/runtime-host-local-remote-access.test.ts b/apps/desktop/src/main/__tests__/runtime-host-local-remote-access.test.ts index 08db786017..5f48a4c904 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-local-remote-access.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-local-remote-access.test.ts @@ -143,12 +143,7 @@ test('enabling remote access hands the same root to one managed service before D assert.deepEqual(decodeRuntimeHostOwnerConnectionCode(result.connectionCode), { name: decodeRuntimeHostOwnerConnectionCode(result.connectionCode).name, rootId: 'a'.repeat(64), - transport: { - kind: 'libp2p-direct', - peerId: livePeer.lease.peerId, - routeHints: livePeer.lease.directRoutes, - coordinationRelays: livePeer.lease.coordinationRoutes, - }, + transport: { kind: 'libp2p-direct', reachability: livePeer }, credential: 'pending-credential', }); const lifecycle = JSON.parse( @@ -169,12 +164,12 @@ test('shares the running Local Host endpoint instead of its persisted startup ro await writeManagedLifecycle(clientDataRoot, rootPath, rootId); const configuredPeer = { peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.1/udp/41000/quic-v1'], + routeHints: [], coordinationRelays: [], }; const livePeer = peerReachability( configuredPeer.peerId, - configuredPeer.routeHints, + ['/ip4/192.0.2.1/udp/41000/quic-v1'], ['/dns4/relay.example/udp/443/quic-v1/p2p/12D3KooWrelay'], ); const service = createDesktopLocalRuntimeHostRemoteAccess({ @@ -219,12 +214,7 @@ test('shares the running Local Host endpoint instead of its persisted startup ro const target = await service.createCollaborationConnectionTarget(); assert.deepEqual(target, { name: target.name, - transport: { - kind: 'libp2p-direct', - peerId: livePeer.lease.peerId, - routeHints: livePeer.lease.directRoutes, - coordinationRelays: livePeer.lease.coordinationRoutes, - }, + transport: { kind: 'libp2p-direct', reachability: livePeer }, }); }); diff --git a/apps/desktop/src/main/__tests__/runtime-host-management.test.ts b/apps/desktop/src/main/__tests__/runtime-host-management.test.ts index c3fb08c973..572dfe59da 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-management.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-management.test.ts @@ -307,9 +307,7 @@ test('identifies, rotates, and revokes managed credentials without exposing secr rootId: profile.rootId, transport: { kind: 'libp2p-direct', - peerId: '12D3KooWoffice', - routeHints: ['/ip4/192.0.2.8/udp/44001/quic-v1'], - coordinationRelays: [], + reachability: testPeerReachability('12D3KooWoffice'), }, credential: 'pending-credential', }); @@ -429,9 +427,7 @@ test('identifies, rotates, and revokes managed credentials without exposing secr rootId: 'c'.repeat(64), transport: { kind: 'libp2p-direct', - peerId: '12D3KooWoffice', - routeHints: ['/ip4/192.0.2.8/udp/44001/quic-v1'], - coordinationRelays: [], + reachability: testPeerReachability('12D3KooWoffice'), }, credential: 'pending-credential', }); @@ -444,9 +440,7 @@ test('identifies, rotates, and revokes managed credentials without exposing secr rootId: profile.rootId, transport: { kind: 'libp2p-direct', - peerId: '12D3KooWunexpected', - routeHints: ['/ip4/192.0.2.8/udp/44001/quic-v1'], - coordinationRelays: [], + reachability: testPeerReachability('12D3KooWunexpected'), }, credential: 'pending-credential', }); @@ -1230,8 +1224,8 @@ test('keeps the SSH profile while adding and removing its managed Direct peer', exists: peerProfileExists, enabled: false, }), - upsertManagedDirectPeerProfile: async (_profileId, descriptor) => { - assert.deepEqual(descriptor.routeHints, ['/ip4/192.0.2.8/udp/44001/quic-v1']); + upsertManagedDirectPeerProfile: async (_profileId, peerId) => { + assert.equal(peerId, '12D3KooWpeer'); peerProfileExists = true; }, removeManagedDirectPeerProfile: async () => { @@ -1535,3 +1529,19 @@ function accessCredential( createdAt: '2026-08-21T01:00:00.000Z', }; } + +function testPeerReachability(peerId: string) { + return { + lease: { + version: 1 as const, + peerId, + revision: 1, + issuedAt: 1, + expiresAt: 2, + directRoutes: ['/ip4/192.0.2.8/udp/44001/quic-v1'], + coordinationRoutes: [], + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), + }; +} diff --git a/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts b/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts index 5c5819a838..cd71890006 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts @@ -668,9 +668,7 @@ test('classifies connection-code failures without exposing transport errors to t rootId: ROOT_ID, transport: { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.8/udp/44001/quic-v1'], - coordinationRelays: [], + reachability: peerReachability('12D3KooWpeer'), }, credential: 'pending-credential', }), @@ -685,12 +683,15 @@ test('persists authenticated Owner routes imported through a connection code', a const startup = await resolveDesktopRuntimeHostStartup(root, { catalog }); const staleTransport = { kind: 'libp2p-direct' as const, - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.8/udp/44001/quic-v1'], - coordinationRelays: ['/memory/stale-relay'], + reachability: peerReachability( + '12D3KooWpeer', + 1, + ['/ip4/192.0.2.8/udp/44001/quic-v1'], + ['/memory/stale-relay'], + ), }; const freshEndpoint = peerReachability( - staleTransport.peerId, + '12D3KooWpeer', 2, ['/ip4/198.51.100.9/udp/44002/quic-v1'], ['/memory/fresh-relay'], @@ -731,12 +732,7 @@ test('persists authenticated Owner routes imported through a connection code', a assert.equal(persisted.profileIncarnationId, observedIncarnation); assert.deepEqual( persisted.profile.kind === 'remote' ? persisted.profile.transport : undefined, - { - kind: 'libp2p-direct', - peerId: freshEndpoint.lease.peerId, - routeHints: freshEndpoint.lease.directRoutes, - coordinationRelays: freshEndpoint.lease.coordinationRoutes, - }, + { kind: 'libp2p-direct', reachability: freshEndpoint }, ); const restarted = await resolveDesktopRuntimeHostStartup(root, { catalog }); assert.deepEqual(restarted.remotes, [persisted]); @@ -814,11 +810,8 @@ test("keeps a managed Direct route on the SSH profile credential authority", asy /Enable Direct peer access/u, ); - await service.upsertManagedDirectPeerProfile(MANAGED_PROFILE.id, { - peerId: "12D3KooWpeer", - routeHints: ["/ip4/192.0.2.8/udp/44001/quic-v1"], - coordinationRelays: [], - }); + exposeReadyState = true; + await service.upsertManagedDirectPeerProfile(MANAGED_PROFILE.id, '12D3KooWpeer'); const directId = (await catalog.read()).profiles.find( (profile) => profile.kind === 'remote' && profile.transport.kind === 'libp2p-direct', @@ -830,21 +823,13 @@ test("keeps a managed Direct route on the SSH profile credential authority", asy if (direct.profile.kind !== "remote") assert.fail("expected a remote Direct profile"); assert.deepEqual(direct.profile.transport, { kind: "libp2p-direct", - peerId: "12D3KooWpeer", - routeHints: ["/ip4/192.0.2.8/udp/44001/quic-v1"], - coordinationRelays: [], + reachability: livePeer, }); - exposeReadyState = true; assert.deepEqual( await service.resolveCollaborationConnectionTarget(MANAGED_PROFILE), { name: MANAGED_PROFILE.name, - transport: { - kind: 'libp2p-direct', - peerId: livePeer.lease.peerId, - routeHints: livePeer.lease.directRoutes, - coordinationRelays: livePeer.lease.coordinationRoutes, - }, + transport: { kind: 'libp2p-direct', reachability: livePeer }, }, ); exposeReadyState = false; diff --git a/apps/desktop/src/main/runtime-host-boot.ts b/apps/desktop/src/main/runtime-host-boot.ts index c436e862ec..f19f38ded0 100644 --- a/apps/desktop/src/main/runtime-host-boot.ts +++ b/apps/desktop/src/main/runtime-host-boot.ts @@ -23,6 +23,7 @@ import { clipboard, ipcMain, nativeTheme, + powerMonitor, powerSaveBlocker, shell, type MessageBoxOptions, @@ -1912,6 +1913,7 @@ function wireLifecycle(): void { if (process.platform !== "darwin" && !isBrowserMessageBoxPresentationActive()) app.quit(); }); app.on("before-quit", quitCoordinator.handleBeforeQuit); + powerMonitor.on("resume", wakePeerRecoveryAfterResume); quitCoordinator.focusOrCreateWindow(); } @@ -1931,6 +1933,7 @@ async function showRuntimeHostQuitFailure(error: unknown): Promise { } async function closeRuntimeHostDesktop(): Promise { + powerMonitor.off("resume", wakePeerRecoveryAfterResume); clientSettingsWatcher.stop(); updateService.dispose(); settingsBotsIpc?.dispose(); @@ -1976,6 +1979,10 @@ async function closeRuntimeHostDesktop(): Promise { } } +function wakePeerRecoveryAfterResume(): void { + runtimeHostManager?.wakePeerRecovery(); +} + function resolveDesktopE2eFixture(): ReturnType { try { return resolveE2eFixture( diff --git a/apps/desktop/src/main/runtime-host-collaboration-ipc-main.ts b/apps/desktop/src/main/runtime-host-collaboration-ipc-main.ts index b55fb3a302..c025db70b8 100644 --- a/apps/desktop/src/main/runtime-host-collaboration-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-collaboration-ipc-main.ts @@ -75,7 +75,8 @@ export function registerRuntimeHostCollaborationIpc( target.transport.kind === 'libp2p-direct' ? { kind: 'peer' as const, - coordinationRelayCount: target.transport.coordinationRelays.length, + coordinationRelayCount: + target.transport.reachability.lease.coordinationRoutes.length, } : { kind: 'configured' as const }, }, diff --git a/apps/desktop/src/main/runtime-host-desktop-manager.ts b/apps/desktop/src/main/runtime-host-desktop-manager.ts index 1818ca130d..b531d0b3a7 100644 --- a/apps/desktop/src/main/runtime-host-desktop-manager.ts +++ b/apps/desktop/src/main/runtime-host-desktop-manager.ts @@ -85,6 +85,7 @@ export interface RuntimeHostDesktopManager { onAccessActivated?: () => void, ): Promise; unmountGuest(mountId: string): Promise; + wakePeerRecovery(): void; disable(profileId: string): Promise; waitUntilReady( profileId: string, @@ -212,6 +213,7 @@ interface DesktopRuntimeHostTargetGeneration { hostId?: string; lifecycle?: RuntimeHostReconnectLifecycle; unsubscribeLifecycle?: () => void; + unsubscribeRoutes?: () => void; skipPeerRouteRefreshOnce?: boolean; lastCandidate?: { readonly hostId: string; @@ -591,18 +593,27 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { await target.lifecycle.close(); throw new Error('Desktop Runtime Host manager is closed'); } + if (!target.valid) { + await target.lifecycle.close(); + throw target.state.readiness === 'unavailable' + ? target.state.error + : new Error('Desktop Runtime Host target became unavailable during startup'); + } this.#activate(target); } catch (error) { + const alreadyUnavailable = target.state.readiness === 'unavailable'; target.valid = false; this.#ipcMain.deactivate(target.epoch); await this.#closeObservations(target.observations); - this.#publishState(target, { - epoch: target.epoch, - target: target.target, - readiness: 'unavailable', - ...(target.hostId ? { hostId: target.hostId } : {}), - error: error instanceof Error ? error : new Error(String(error)), - }); + if (!alreadyUnavailable) { + this.#publishState(target, { + epoch: target.epoch, + target: target.target, + readiness: 'unavailable', + ...(target.hostId ? { hostId: target.hostId } : {}), + error: error instanceof Error ? error : new Error(String(error)), + }); + } throw error; } } @@ -621,6 +632,18 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { }); } + wakePeerRecovery(): void { + for (const target of this.#targets.values()) { + if ( + target.valid && + target.target.profile.kind === 'remote' && + target.target.profile.transport.kind === 'libp2p-direct' + ) { + target.lifecycle?.wake(); + } + } + } + async waitUntilReady( profileId: string, previousHostEpoch?: string, @@ -839,24 +862,35 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { initialSignal?: AbortSignal, ): Promise> { let starting = true; + let initialAttempt = true; + let fatalDuringStart: Error | undefined; + const retryInitialFailure = + target.target.profile.kind === 'remote' && + target.target.profile.transport.kind === 'libp2p-direct'; try { - return await startRuntimeHostReconnectLifecycle({ - connect: (signal) => - this.connect( + const lifecycle = await startRuntimeHostReconnectLifecycle({ + connect: (signal) => { + const first = initialAttempt; + initialAttempt = false; + return this.connect( target, - starting && initialSignal - ? AbortSignal.any([signal, initialSignal]) - : signal, - starting ? target.input.profileTarget?.sshInteraction : 'batch', - starting ? target.input.onConnectionPhase : undefined, - ), + signal, + first ? target.input.profileTarget?.sshInteraction : 'batch', + first ? target.input.onConnectionPhase : undefined, + ); + }, + retryInitialFailure, + ...(initialSignal ? { initialSignal } : {}), onReconnectError: (error) => { console.warn('[runtime-host] reconnect attempt failed:', error); }, onFatalError: (error) => { - if (!starting && target.valid) { + if (starting) { + fatalDuringStart = error; + } else if (target.valid) { target.valid = false; target.unsubscribeLifecycle?.(); + target.unsubscribeRoutes?.(); this.#ipcMain.deactivate(target.epoch); this.#publishState(target, { epoch: target.epoch, @@ -870,6 +904,11 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { }, ...(this.reconnectBackoff ? { backoff: this.reconnectBackoff } : {}), }); + if (fatalDuringStart) { + await lifecycle.close(); + throw fatalDuringStart; + } + return lifecycle; } finally { starting = false; } @@ -1129,6 +1168,13 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { #activate(target: DesktopRuntimeHostTargetGeneration): void { this.#ipcMain.activate(target.epoch); + const profile = target.target.profile; + if (profile.kind === 'remote' && profile.transport.kind === 'libp2p-direct') { + target.unsubscribeRoutes = this.#baseInput.peerClient?.subscribeRoutes( + profile.transport.reachability.lease.peerId, + () => target.lifecycle?.wake(), + ); + } target.unsubscribeLifecycle = target.lifecycle?.subscribe((candidate) => { if (!target.valid) return; this.#publishState( @@ -1174,6 +1220,7 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { target.valid = false; this.onTargetRemoved?.(target.state); target.unsubscribeLifecycle?.(); + target.unsubscribeRoutes?.(); this.#ipcMain.deactivate(target.epoch); try { await target.lifecycle?.close(); diff --git a/apps/desktop/src/main/runtime-host-guest-session-mounts.ts b/apps/desktop/src/main/runtime-host-guest-session-mounts.ts index 5c1a61780f..44d0a94e5d 100644 --- a/apps/desktop/src/main/runtime-host-guest-session-mounts.ts +++ b/apps/desktop/src/main/runtime-host-guest-session-mounts.ts @@ -21,6 +21,7 @@ import { randomUUID } from 'node:crypto'; import { decodeRemoteRuntimeHostProfile, RUNTIME_HOST_ACCESS_CREDENTIAL_MAX_BYTES, + RuntimeHostPermanentReconnectError, type ResolvedRuntimeHostProfile, type RuntimeHostConnectionPhase, type RuntimeHostRemoteTransport, @@ -177,7 +178,7 @@ export function createDesktopGuestSessionMountService(input: { if ( closed || mount.transport.kind !== 'libp2p-direct' || - endpoint.lease.peerId !== mount.transport.peerId + endpoint.lease.peerId !== mount.transport.reachability.lease.peerId ) return; void mutate(async () => { if (removingMounts.has(mount.mountId)) return; @@ -185,19 +186,14 @@ export function createDesktopGuestSessionMountService(input: { const retained = current.get(mount.mountId); if ( retained?.transport.kind !== 'libp2p-direct' || - retained.transport.peerId !== endpoint.lease.peerId || - ( - sameStrings(retained.transport.routeHints, endpoint.lease.directRoutes) && - sameStrings(retained.transport.coordinationRelays, endpoint.lease.coordinationRoutes) - ) + retained.transport.reachability.lease.peerId !== endpoint.lease.peerId || + retained.transport.reachability.lease.revision >= endpoint.lease.revision ) return; const updated = decodeMount({ ...retained, transport: { kind: 'libp2p-direct', - peerId: endpoint.lease.peerId, - routeHints: endpoint.lease.directRoutes, - coordinationRelays: endpoint.lease.coordinationRoutes, + reachability: endpoint, }, }); await persist(new Map(current).set(mount.mountId, updated)); @@ -272,7 +268,9 @@ export function createDesktopGuestSessionMountService(input: { removingMounts.has(mount.mountId) ) return; activation.stage = 'connecting'; - onError(asError(error), mount); + const failure = asError(error); + onError(failure, mount); + if (failure instanceof RuntimeHostPermanentReconnectError) return; await wait(delayMs, activation.controller.signal); delayMs = Math.min(delayMs * 2, STARTUP_RETRY_MAX_MS); } @@ -467,10 +465,6 @@ export function createDesktopGuestSessionMountService(input: { }; } -function sameStrings(left: readonly string[], right: readonly string[]): boolean { - return left.length === right.length && left.every((value, index) => value === right[index]); -} - export function registerDesktopGuestSessionMountIpc( ipcMain: Pick, service: DesktopGuestSessionMountService, @@ -573,10 +567,13 @@ function decodeMount(value: unknown): GuestSessionMount { function isPeerPathUnavailable(error: unknown): boolean { if (!isRecord(error) || typeof error.code !== 'string') return false; - return error.code === 'direct_path_unavailable' || error.code === 'transit_unavailable'; + return ( + error.code === 'direct_path_unavailable' || + error.code === 'transit_unavailable' || + error.code === 'peer_reachability_needs_repair' + ); } - function collaborationProgressForConnectionPhase( phase: RuntimeHostConnectionPhase, ): SessionCollaborationImportPhase { diff --git a/apps/desktop/src/main/runtime-host-local-remote-access.ts b/apps/desktop/src/main/runtime-host-local-remote-access.ts index c507042e2e..c0d8926795 100644 --- a/apps/desktop/src/main/runtime-host-local-remote-access.ts +++ b/apps/desktop/src/main/runtime-host-local-remote-access.ts @@ -27,7 +27,7 @@ import { issueRuntimeHostOwnerConnectionCode, } from '@maka/runtime-host/client'; import { resolveRuntimeHostManagedDeploymentAuthority } from '@maka/runtime-host/operator'; -import type { HostRegistration } from '@maka/runtime-host/protocol'; +import type { HostPeerEndpoint, HostRegistration } from '@maka/runtime-host/protocol'; import type { DesktopLocalRuntimeHostRemoteAccessEnableResult, DesktopLocalRuntimeHostRemoteAccessSnapshot, @@ -107,9 +107,7 @@ export interface DesktopLocalRuntimeHostRemoteAccess { readonly name: string; readonly transport: { readonly kind: 'libp2p-direct'; - readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; + readonly reachability: HostPeerEndpoint; }; }>; enable(value: unknown): Promise; @@ -156,8 +154,6 @@ type LocalServiceLifecycle = interface LocalPeerDescriptor { readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; } type DesktopRuntimeHostLocalOperator = ReturnType< @@ -349,7 +345,7 @@ export function createDesktopLocalRuntimeHostRemoteAccess(input: { encodeRuntimeHostOwnerConnectionCode({ name: hostName(), rootId: completed.managed.rootId, - transport: { kind: 'libp2p-direct', ...livePeer }, + transport: { kind: 'libp2p-direct', reachability: livePeer }, credential: completed.credential, }), ); @@ -524,7 +520,7 @@ export function createDesktopLocalRuntimeHostRemoteAccess(input: { const livePeer = await readLivePeer(localClient(input.manager), peer); return { name: hostName(), - transport: { kind: 'libp2p-direct' as const, ...livePeer }, + transport: { kind: 'libp2p-direct' as const, reachability: livePeer }, }; }); @@ -948,15 +944,7 @@ function requireEnabledPeer(value: unknown): LocalPeerDescriptor { if (typeof value.peerId !== 'string' || value.peerId.length === 0 || value.peerId.length > 160) { throw new Error('Runtime Host returned an invalid peer identity'); } - const peer = { - peerId: value.peerId, - routeHints: requireAddresses(value.routeHints), - coordinationRelays: requireAddresses(value.coordinationRelays), - }; - if (peer.routeHints.length === 0 && peer.coordinationRelays.length === 0) { - throw new Error('Runtime Host Direct peer has no reachable route'); - } - return peer; + return { peerId: value.peerId }; } function onSnapshot(sharedAccess: boolean): Extract< @@ -993,7 +981,7 @@ async function issueConnectionCode( async function readLivePeer( client: DesktopRuntimeHostClient, configured: LocalPeerDescriptor, -): Promise { +): Promise { const endpoint = (await client.status()).peerEndpoint; if (!endpoint) { throw new Error('Runtime Host Direct peer is not available'); @@ -1001,12 +989,7 @@ async function readLivePeer( if (endpoint.lease.peerId !== configured.peerId) { throw new Error('Runtime Host Direct peer identity changed'); } - return requireEnabledPeer({ - state: 'enabled', - peerId: endpoint.lease.peerId, - routeHints: endpoint.lease.directRoutes, - coordinationRelays: endpoint.lease.coordinationRoutes, - }); + return endpoint; } async function hasSharedAccess( diff --git a/apps/desktop/src/main/runtime-host-management.ts b/apps/desktop/src/main/runtime-host-management.ts index 3b07068d7f..d31fcad04e 100644 --- a/apps/desktop/src/main/runtime-host-management.ts +++ b/apps/desktop/src/main/runtime-host-management.ts @@ -556,6 +556,7 @@ export function createDesktopRuntimeHostManagement(input: { if (peerProfile.enabled) { throw new Error('Disable the Direct peer profile before changing its listener'); } + const previousHostEpoch = input.currentHostEpoch(profileId); const response = await input.runPeerManagement({ destination: transport.destination, ...(transport.sshPort === undefined ? {} : { sshPort: transport.sshPort }), @@ -574,6 +575,9 @@ export function createDesktopRuntimeHostManagement(input: { : 'Runtime Host returned an unrelated direct-peer result', ); } + if (response.action !== (enabledValue ? 'enable' : 'disable')) { + throw new Error('Runtime Host returned an unrelated direct-peer result'); + } const status = response.status; if (enabledValue) { try { @@ -584,11 +588,13 @@ export function createDesktopRuntimeHostManagement(input: { ) { throw new Error('Runtime Host did not return a usable direct-peer descriptor'); } - await input.profiles.upsertManagedDirectPeerProfile(profileId, { - peerId: status.peerId, - routeHints: status.routeHints, - coordinationRelays: status.coordinationRelays, - }); + await input.awaitUpdatedConnection( + profileId, + managed.profile.rootId, + previousHostEpoch, + response.restarted, + ); + await input.profiles.upsertManagedDirectPeerProfile(profileId, status.peerId); } catch (failure) { try { const rollback = await input.runPeerManagement({ @@ -919,7 +925,10 @@ export function createDesktopRuntimeHostManagement(input: { const peerProfile = await input.profiles.resolveManagedDirectPeerProfile( access.managed.profile.id, ); - if (peerProfile.peerId && decoded.transport.peerId !== peerProfile.peerId) { + if ( + peerProfile.peerId && + decoded.transport.reachability.lease.peerId !== peerProfile.peerId + ) { throw new Error('Remote Runtime Host returned a connection code for a different Direct peer'); } return response.connectionCode; diff --git a/apps/desktop/src/main/runtime-host-profile-service.ts b/apps/desktop/src/main/runtime-host-profile-service.ts index 3d1aee5b40..8fe56cc7d6 100644 --- a/apps/desktop/src/main/runtime-host-profile-service.ts +++ b/apps/desktop/src/main/runtime-host-profile-service.ts @@ -129,11 +129,7 @@ export interface DesktopRuntimeHostProfileService { }>; upsertManagedDirectPeerProfile( profileId: string, - peer: { - readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; - }, + expectedPeerId: string, ): Promise; removeManagedDirectPeerProfile(profileId: string): Promise; clearManagedServiceBinding(expected: DesktopRuntimeHostManagedSshServiceBinding): Promise; @@ -930,7 +926,7 @@ export function createDesktopRuntimeHostProfileService(input: { } let configuredPeerId: string | undefined; if (profile.transport.kind === 'libp2p-direct') { - configuredPeerId = profile.transport.peerId; + configuredPeerId = profile.transport.reachability.lease.peerId; } else { const direct = (await catalog.read()).profiles.find( (candidate) => candidate.id === managedDirectPeerProfileId(profile.id), @@ -940,7 +936,7 @@ export function createDesktopRuntimeHostProfileService(input: { direct.rootId === profile.rootId && direct.transport.kind === 'libp2p-direct' ) { - configuredPeerId = direct.transport.peerId; + configuredPeerId = direct.transport.reachability.lease.peerId; } } if (!configuredPeerId) { @@ -963,9 +959,7 @@ export function createDesktopRuntimeHostProfileService(input: { name: profile.name, transport: { kind: 'libp2p-direct' as const, - peerId: endpoint.lease.peerId, - routeHints: endpoint.lease.directRoutes, - coordinationRelays: endpoint.lease.coordinationRoutes, + reachability: endpoint, }, }; }); @@ -1005,12 +999,21 @@ export function createDesktopRuntimeHostProfileService(input: { exists: profile !== undefined, enabled: preferences.enabledRemoteProfileIds.includes(peerProfileId), ...(profile?.kind === 'remote' && profile.transport.kind === 'libp2p-direct' - ? { peerId: profile.transport.peerId } + ? { peerId: profile.transport.reachability.lease.peerId } : {}), }; }); }, - upsertManagedDirectPeerProfile(profileId, peer) { + async upsertManagedDirectPeerProfile(profileId, expectedPeerId) { + requirePairingComplete(profileId); + const active = input.states().find((state) => state.target.profile.id === profileId); + if (!active || active.readiness !== 'ready') { + throw new Error('Connect this Runtime Host before enabling Direct peer access'); + } + const reachability = (await active.candidate.client.status()).peerEndpoint; + if (!reachability || reachability.lease.peerId !== expectedPeerId) { + throw new Error('Runtime Host Direct peer identity changed'); + } return mutateProfiles(async () => { requirePairingComplete(profileId); const source = await catalog.resolve(profileId); @@ -1028,7 +1031,10 @@ export function createDesktopRuntimeHostProfileService(input: { if (!managed || managed.state !== 'active') { throw new Error('This Runtime Host profile is not bound to an active managed service'); } - if (peer.routeHints.length === 0 && peer.coordinationRelays.length === 0) { + if ( + reachability.lease.directRoutes.length === 0 && + reachability.lease.coordinationRoutes.length === 0 + ) { throw new Error('Runtime Host returned an invalid direct-peer descriptor'); } const peerProfileId = managedDirectPeerProfileId(profileId); @@ -1042,9 +1048,7 @@ export function createDesktopRuntimeHostProfileService(input: { rootId: source.profile.rootId, transport: { kind: 'libp2p-direct', - peerId: peer.peerId, - routeHints: peer.routeHints, - coordinationRelays: peer.coordinationRelays, + reachability, }, }; const existing = (await catalog.read()).profiles.find( @@ -1350,7 +1354,7 @@ function createAuthenticatedPeerRouteObserver( target.profile.transport.kind !== 'libp2p-direct' || !target.profileIncarnationId ) return undefined; - const expectedPeerId = target.profile.transport.peerId; + const expectedPeerId = target.profile.transport.reachability.lease.peerId; const incarnation = { profile: target.profile, profileIncarnationId: target.profileIncarnationId, @@ -1362,17 +1366,14 @@ function createAuthenticatedPeerRouteObserver( .then(async () => { await catalog.updateRemoteProfileIfCurrent(incarnation, (current) => { if (current.transport.kind !== 'libp2p-direct') return current; - if ( - sameStrings(current.transport.routeHints, endpoint.lease.directRoutes) && - sameStrings(current.transport.coordinationRelays, endpoint.lease.coordinationRoutes) - ) return current; + if (current.transport.reachability.lease.revision >= endpoint.lease.revision) { + return current; + } return { ...current, transport: { kind: 'libp2p-direct', - peerId: expectedPeerId, - routeHints: endpoint.lease.directRoutes, - coordinationRelays: endpoint.lease.coordinationRoutes, + reachability: endpoint, }, }; }); @@ -1387,11 +1388,6 @@ function createAuthenticatedPeerRouteObserver( return { observe, flush: () => pending }; } -function sameStrings(left: readonly string[], right: readonly string[]): boolean { - return left.length === right.length && left.every((value, index) => value === right[index]); -} - - function managedDirectPeerProfileId(sourceProfileId: string): string { const digest = createHash('sha256').update(sourceProfileId).digest('hex').slice(0, 32); return `direct-${digest}`; @@ -1563,6 +1559,7 @@ function connectionCodeImportFailure( if ( error.code === 'direct_path_unavailable' || error.code === 'coordination_unavailable' || + error.code === 'peer_reachability_needs_repair' || error.code === 'peer_connect_in_progress' ) { return 'host_unreachable'; diff --git a/apps/desktop/src/renderer/settings/runtime-host-profiles-section.tsx b/apps/desktop/src/renderer/settings/runtime-host-profiles-section.tsx index eef3be890d..db8f30a05f 100644 --- a/apps/desktop/src/renderer/settings/runtime-host-profiles-section.tsx +++ b/apps/desktop/src/renderer/settings/runtime-host-profiles-section.tsx @@ -658,11 +658,11 @@ export function RuntimeHostProfilesSection(props: { : profile.transport.kind === "libp2p-direct" ? ( settingsActionErrorMessage(error, locale)} diff --git a/docs/architecture/peer-reachability-recovery-plan.md b/docs/architecture/peer-reachability-recovery-plan.md index a7fb71bd0c..de27e15e72 100644 --- a/docs/architecture/peer-reachability-recovery-plan.md +++ b/docs/architecture/peer-reachability-recovery-plan.md @@ -21,7 +21,7 @@ - Status: implementation plan - Tracking issue: [#4554](https://github.com/apache/maka/issues/4554) -- Baseline: `main` at `ad18da42c803607117762c3e7e0a2e4d9bc74fea` +- Baseline: `main` at `eacfb46aa7ec93273bf468335f4270bba62d35a5` - Delivery: four stacked pull requests ## Review charter @@ -132,8 +132,10 @@ The publisher atomically persists the exact next signed lease before exposing it Skipped revisions are valid; publishing different facts at one revision is not. A receiver validates signature, expected peer, bounds, signed lifetime, and revision. Runtime freshness uses a local monotonic receipt deadline capped by the signed lifetime -so modest clock skew cannot turn an old lease into current truth. Restarted processes -conservatively revalidate persisted records against wall time. +so wall-clock rollback cannot turn an old lease into current truth. A publisher restart +authenticates its own persisted record without treating the local wall clock as remote +freshness authority, then immediately publishes a higher revision when its timestamps +are no longer usable under the current clock. ### MeshMemberAdvertisement @@ -275,7 +277,53 @@ charter. Their evidence is adjudicated into this ledger: | ID | Source | Decision | Evidence or resolution | | --- | --- | --- | --- | -| — | — | open | No findings recorded yet | +| R1-C1 | correctness | confirmed | Route resolution now distinguishes `available`, bounded `recovering`, and `exhausted`; Mesh presentation, native waiting, direct profiles, and Session startup consume the same state. | +| R1-C2 | correctness | confirmed | Runtime lease freshness uses a local monotonic receipt deadline capped by the signed lifetime; wall-clock rollback cannot extend current truth in-process. | +| R1-C3 | correctness | confirmed | The strict Peer Mesh wire/storage replacement advances the compatibility epoch to 98. | +| R1-C4 | correctness | confirmed | Windows recovery source closure follows the new reachability owner and publisher instead of the removed Mesh owner path. | +| R1-S1 | simplification | confirmed | The endpoint owner is the sole peer-client lifetime authority; Desktop no longer closes the same client a second time. | +| R1-S2 | simplification | confirmed | Route resolution has one lifecycle: clients start unattached and Mesh explicitly attaches and detaches the resolver. Constructor/factory injection was removed. | +| R1-S3 | simplification | superseded | A later security review proved that deriving long-lived authority identity from a replaceable locator lets a member rebind the authority. R2-C2 replaces this decision. | +| R2-C1 | correctness | confirmed | Startup verifies all persisted signatures, prunes leases beyond the bounded recovery horizon, and retains roster membership; ordinary offline time can no longer brick Mesh initialization. | +| R2-C2 | correctness | confirmed | The authority PeerId is now part of the authority-signed roster and immutable across roster revisions. Invitations and authenticated streams must match it, while reachability remains only a replaceable locator. | +| R2-S1 | simplification | confirmed | Remembered Relay anchors are persisted regardless of whether public discovery is enabled; discovery selects new anchors, while anchor recovery is a separate concern. | +| R2-S2 | simplification | confirmed | Relay-anchor persistence uses one coalescing watch slot instead of an unbounded snapshot queue. | +| R2-S3 | simplification | confirmed | Mesh presence reads the native Swarm connectivity snapshot instead of maintaining a partial, stale `recentlyReached` cache. | +| R2-S4 | simplification | superseded | The shared revision-only vector was simpler, but it hid equal-revision conflicts between replicas. R11-C1 replaces it with one shared content-bound summary. | +| R2-S5 | simplification | rejected | The one-shot post-finalization refresh suppression remains: it prevents guest credential finalization from becoming a second network acquisition, as required by the frozen collaboration invariant. | +| R2-S6 | simplification | superseded | The target wrapper was removed first; R3-S2 then removed the remaining duplicate authority lease from replica state entirely. | +| R2-CI1 | CI | confirmed | Lower-stack Desktop fixtures retain the flat transport shape until PR4 introduces signed profile reachability, preserving each PR's review boundary. | +| R2-CI2 | CI | confirmed | The Peer Mesh protocol imports the reachability wire decoder directly from its model module, keeping filesystem-backed publisher code out of the Linux preload bundle. | +| R3-C1 | correctness | confirmed | Publisher renewal now uses a monotonic receipt deadline, immediately replaces a lease whose issue time is ahead of the local wall clock, and authenticates persisted local state without applying receiver freshness policy. A rollback therefore neither strands renewal nor prevents restart. | +| R3-S1 | simplification | confirmed | Peer listeners no longer expose an unused `ownsClient` branch; the endpoint owner remains the only client lifetime authority. | +| R3-S2 | simplification | confirmed | Replica state no longer duplicates authority reachability. Invitation and redemption evidence merge into the bounded common lease table, while the signed roster remains the sole authority-identity source. | +| R4-CI1 | CI | confirmed | The installed CLI smoke exposed an untyped client/configuration ambiguity. The service now accepts only endpoint configuration and always owns the resulting endpoint; the smoke uses that production composition and an independent Mesh fixture instead of injecting a borrowed endpoint. | +| R4-S1 | simplification | confirmed | The legacy combined Mesh owner duplicated endpoint and Mesh-component composition solely for release smoke. It was removed; callers now compose and close those two independently owned lifetimes in dependency order. | +| R5-C1 | correctness | confirmed | The top-stack installed CLI smoke now supplies the listener's signed reachability object instead of the removed flat Direct transport fields. | +| R5-C2 | correctness | confirmed | Desktop shutdown settles Mesh cleanup before endpoint cleanup but never lets a Mesh failure skip the endpoint owner; it reports accumulated failures only after both lifetimes have been released. | +| R5-S1 | simplification | confirmed | The test-only borrowed endpoint service mode was removed. Runtime Host service ownership is now invariant, while tests that need independent endpoints compose them outside the service. | +| R5-S2 | simplification | confirmed | An attached route resolver is now one complete snapshot/refresh/subscription capability. The client still supports no resolver, but no longer carries unused partial-capability branches. | +| R6-S1 | simplification | confirmed | Peer identity now exposes only the immutable PeerId. Dynamic addresses flow from the native reachability snapshot into the signed lease, and listener/CLI projections derive from that lease instead of retaining parallel unsigned copies. | +| R7-C1 | correctness | confirmed | Reachability authentication is now side-effect free. A remote lease enters the monotonic receipt cache only after invitation redemption or Mesh synchronization proves that both peers are authorized by the active roster; the cache is pruned and insertion-limited to active remote roster members. | +| R7-CI1 | CI | confirmed | The lower-stack installed CLI smoke now derives route hints from the listener's signed lease. It no longer relies on unsigned listener fields introduced only by a later PR, so every layer remains independently testable. | +| R7-S1 | simplification | partially confirmed | The publisher file no longer repeats an unsigned outer PeerId already authenticated by the signed lease. The local lease remains in the Mesh evidence table because that table is the uniform bounded anti-entropy cache, not a competing authority; excluding self would add special cases to vectors, pages, persistence, and status projection. | +| R7-S2 | simplification | confirmed | The cross-layer `waitForRoutes` option was removed. Application attempts intrinsically accept live route updates until their deadline; Mesh control either uses explicit candidates, reuses an already eligible direct connection, or fails immediately. | +| R8-S1 | simplification | confirmed | The Mesh receipt cache no longer mirrors the publisher's local receipt or retains an unconsumed pending-authority receipt. Its sole authority class is now active remote roster members. | +| R8-C1 | correctness | confirmed | Peer-keyed recovery state now has collection-level bounds. Authenticated route receipts are globally horizon-pruned and LRU-bounded with active observers preferred, while completed Mesh sweeps are retained only for currently visible remote roster members. | +| R9-C1 | correctness | confirmed | Each Mesh synchronization page revalidates the target against the current active roster both before dialing and after asynchronous route preparation, immediately before emitting local reachability. A queued worker therefore cannot disclose refreshed locators to a member removed from its original roster snapshot. | +| R9-C2 | correctness | confirmed | A fenced native connection attempt now records WebRTC signaling attempts per current relay PeerId and keeps at most one active upgrade. Failure retires only that relay attempt; a newly introduced relay can be tried within the same immutable request and deadline without retrying failed candidates or creating a second connection authority. | +| R10-C1 | correctness | confirmed | Persisted recovery evidence is authenticated independently of receiver freshness policy. A lease from before a wall-clock rollback remains a bounded dial hint but receives no currentness receipt until its issue time is credible; the local publisher can immediately replace it with a higher revision. Direct profiles use the same historical-evidence admission path. | +| R10-C2 | correctness | confirmed | Authenticated Mesh input now flows through the common reachability merge without deleting the prior record first. Equal revisions with different signed facts fail closed, preserving the revision vector's convergence contract. | +| R10-S1 | simplification | confirmed | Desktop's managed-service descriptor retains only the immutable PeerId. Startup route arrays no longer form an unsigned, stale availability gate; connection codes and collaboration targets come from the Runtime Host's live signed endpoint. | +| R11-C1 | correctness | confirmed | Reachability and advertisement anti-entropy now summarize each signed fact as `{ peerId, revision, digest }`, where the digest binds the canonical signed payload. Equal-revision disagreement is rejected during summary comparison instead of remaining silently partitioned between replicas. | +| R11-C2 | correctness | confirmed | Each WebRTC Relay upgrade now has its own fenced identity and child cancellation token. Removing its Relay retires the active attempt, a replacement can start in the same connection attempt, and a late success is closed unless both its identity and Relay membership are still current. | +| R12-C1 | correctness | confirmed | The WebRTC Relay-attempt identity now survives through the libp2p dial lifecycle. Candidate replacement retires and closes stale dials, an established connection is admitted only for the exact current attempt, and a late terminal event from an old dial cannot clear its replacement. | +| R13-C1 | correctness | confirmed | A persisted Direct target now keeps one live reconnect lifecycle when its first connection fails transiently. Route and resume wakeups can therefore recover the same target after a cold-start ordering gap; identity, credential, compatibility, caller cancellation, and other permanent failures still terminate it. `needs_repair` is recoverable state rather than a permanent reconnect verdict. | +| R13-S1 | simplification | rejected | The active WebRTC attempt is the logical flow identity, while the dial origin maps an external libp2p `ConnectionId` back to that identity. Replacing the explicit relation with phase-dependent map scans does not remove a state owner and weakens the clarity of the remove/re-add Relay ABA fence. | +| R14-C1 | correctness | confirmed | Direct profile startup now classifies immutable signed-reachability authentication failures, peer identity mismatch, and an unavailable native peer API as permanent. Route exhaustion and ordinary transport failure remain recoverable, so retry state cannot hide a profile that requires repair. | +| R14-C2 | correctness | confirmed | A permanent result from the immediate retry is captured on either side of the lifecycle-factory microtask boundary. Desktop never activates a target already made terminal, while a later fatal result still follows the ordinary unavailable transition. | +| R14-S1 | simplification | confirmed | Remote lease verification no longer belongs to the local durable publisher, and its standalone opener is no longer a public package API. The aggregate endpoint owner is the sole production exclusion lifetime, so the publisher no longer acquires a nested writer lock for a call path that production cannot use independently. | +| R15-C1 | correctness | confirmed | Direct profile startup now classifies an absent peer endpoint client through the same permanent native-capability boundary as an incompatible or unavailable addon. A build that cannot provide peer networking therefore becomes honestly unavailable instead of retaining an in-process reconnect loop that can never acquire the missing dependency. | Only findings that affect the merge bar and have a proportionate root fix enter the stack. Narrow constructed paths and low-value polish do not. A local fix triggers a diff --git a/docs/runtime-host-remote-access.md b/docs/runtime-host-remote-access.md index e94409de50..a43e93a6f1 100644 --- a/docs/runtime-host-remote-access.md +++ b/docs/runtime-host-remote-access.md @@ -124,10 +124,11 @@ maka runtime-host service peer descriptor \ ``` The descriptor contains the PeerId, Root ID, and candidate routes, but never an access credential. -Use those values with `runtime-host profile set --peer-id ... --peer-route ...`; supply the -credential created by setup through `MAKA_RUNTIME_HOST_ACCESS_CREDENTIAL`. Disable and re-enable -preserve the PeerId and listener settings; `peer rotate` intentionally changes the PeerId, and -service uninstall removes its key while retaining the State Root. Pass +Raw descriptor routes are diagnostic output, not a durable Client profile: routes can change and +are not authenticated as a current reachability claim. Use a one-time connection code when adding +a Direct peer to Desktop. Disable and re-enable preserve the PeerId and listener settings; `peer +rotate` intentionally changes the PeerId, and service uninstall removes its key while retaining the +State Root. Pass `peer enable --clear-coordination-relays` to remove every configured coordination relay. This direct-only path is experimental and may fail on restrictive NAT or UDP-blocked networks. It diff --git a/docs/runtime-host-remote-access.zh-CN.md b/docs/runtime-host-remote-access.zh-CN.md index 03db43d0b4..2245539ab9 100644 --- a/docs/runtime-host-remote-access.zh-CN.md +++ b/docs/runtime-host-remote-access.zh-CN.md @@ -110,11 +110,11 @@ maka runtime-host service peer descriptor \ --expected-root-id '' ``` -Descriptor 只包含 PeerId、Root ID 和候选 route,不包含 access credential。使用这些值执行 -`runtime-host profile set --peer-id ... --peer-route ...`,并通过 -`MAKA_RUNTIME_HOST_ACCESS_CREDENTIAL` 提供 setup 创建的 credential。Disable 后重新 enable 会保留 -PeerId 和 listener 配置;`peer rotate` 会明确更换 PeerId;卸载 service 会删除 peer key,但保留 State -Root。执行 `peer enable --clear-coordination-relays` 可以删除所有已配置的 coordination relay。 +Descriptor 只包含 PeerId、Root ID 和候选 route,不包含 access credential。这里的 route 仅用于诊断, +不能作为长期保存的 Client profile:它既可能变化,也不是 Host 对当前可达性的签名声明。要在 Desktop +中添加 Direct peer,请使用一次性 connection code。Disable 后重新 enable 会保留 PeerId 和 listener +配置;`peer rotate` 会明确更换 PeerId;卸载 service 会删除 peer key,但保留 State Root。执行 `peer +enable --clear-coordination-relays` 可以删除所有已配置的 coordination relay。 Direct-only 路径仍是实验能力,在受限 NAT 或禁用 UDP 的网络中可能失败。它不会替代已有的 TLS、SSH 或 overlay network fallback。Host 默认通过公共 IPFS DHT 的有界 client-only 视图发现 Circuit Relay v2 diff --git a/native/runtime-host-peer/src/bindings.rs b/native/runtime-host-peer/src/bindings.rs index 8ef78a3fad..4b65b01997 100644 --- a/native/runtime-host-peer/src/bindings.rs +++ b/native/runtime-host-peer/src/bindings.rs @@ -35,6 +35,7 @@ use crate::engine::{ type IncomingStreamReceiver = mpsc::Receiver, PeerError>>; const IDENTITY_PAYLOAD_MAX_BYTES: usize = 8 * 1024; +const MAX_CONNECT_ROUTES_PER_CLASS: usize = 32; const MAX_TRANSIT_PEERS: usize = 64; const MAX_TRANSIT_RELAY_ADDRESSES: usize = 256; const MAX_WEBRTC_STUN_URLS: usize = 8; @@ -61,6 +62,14 @@ pub struct ConnectPeerOptions { pub direct_deadline_ms: u32, } +#[napi(object)] +pub struct UpdatePeerConnectOptions { + pub request_id: u32, + pub route_hints: Vec, + pub coordination_relays: Option>, + pub transit_relay_peer_ids: Option>, +} + #[napi(object)] pub struct ConfigurePeerTransitOptions { pub allowed_peer_ids: Vec, @@ -101,10 +110,17 @@ pub struct PeerReachabilitySnapshot { pub active_coordination_relays: Vec, } +#[napi(object)] +pub struct PeerConnectivitySnapshot { + pub generation: u32, + pub connected_peer_ids: Vec, +} + #[napi] pub struct PeerEndpoint { peer_id: String, reachability: watch::Receiver, + connectivity: watch::Receiver, transit_snapshot: Arc>, commands: mpsc::Sender, incoming: Arc>>, @@ -152,6 +168,38 @@ impl PeerEndpoint { Ok(reachability_snapshot(&receiver.borrow())) } + #[napi(getter)] + pub fn connectivity_snapshot(&self) -> PeerConnectivitySnapshot { + connectivity_snapshot(&self.connectivity.borrow()) + } + + #[napi] + pub async fn watch_connectivity( + &self, + after_generation: u32, + timeout_ms: u32, + ) -> Result { + if !(1..=300_000).contains(&timeout_ms) { + return Err(Error::new( + Status::InvalidArg, + "connectivity watch timeout must be between 1 and 300000 milliseconds", + )); + } + let mut receiver = self.connectivity.clone(); + if receiver.borrow().generation == after_generation { + match tokio::time::timeout( + Duration::from_millis(u64::from(timeout_ms)), + receiver.changed(), + ) + .await + { + Ok(Ok(())) | Err(_) => {} + Ok(Err(_)) => return Err(native_closed_error()), + } + } + Ok(connectivity_snapshot(&receiver.borrow())) + } + #[napi(getter)] pub fn transit_snapshot(&self) -> PeerTransitSnapshot { let snapshot = self @@ -239,6 +287,34 @@ impl PeerEndpoint { result_rx.await.map_err(|_| native_closed_error()) } + #[napi] + pub async fn update_connect(&self, options: UpdatePeerConnectOptions) -> Result { + let route_hints = parse_connect_addresses(options.route_hints, "route hint")?; + let coordination_relays = parse_connect_addresses( + options.coordination_relays.unwrap_or_default(), + "coordination relay", + )?; + let transit_relay_peers = + parse_peer_id_list(options.transit_relay_peer_ids.unwrap_or_default())?; + let (result_tx, result_rx) = oneshot::channel(); + self.commands + .send(EngineCommand::UpdateConnect { + request_id: options.request_id, + candidates: engine::ConnectCandidates { + route_hints, + coordination_relays, + transit_relay_peers, + }, + result: result_tx, + }) + .await + .map_err(|_| native_closed_error())?; + result_rx + .await + .map_err(|_| native_closed_error())? + .map_err(peer_error) + } + #[napi] pub async fn accept(&self) -> Result> { let mut incoming = self.incoming.lock().await; @@ -284,8 +360,8 @@ async fn connect_peer( stream_kind: engine::StreamKind, ) -> Result { let peer_id = parse_peer_id(&options.peer_id)?; - let route_hints = parse_addresses(options.route_hints, "route hint")?; - let coordination_relays = parse_addresses( + let route_hints = parse_connect_addresses(options.route_hints, "route hint")?; + let coordination_relays = parse_connect_addresses( options.coordination_relays.unwrap_or_default(), "coordination relay", )?; @@ -437,6 +513,7 @@ pub fn start_peer_endpoint(options: StartPeerEndpointOptions) -> Result PeerReachab } } +fn connectivity_snapshot(snapshot: &engine::ConnectivitySnapshot) -> PeerConnectivitySnapshot { + PeerConnectivitySnapshot { + generation: snapshot.generation, + connected_peer_ids: snapshot + .connected_peers + .iter() + .map(ToString::to_string) + .collect(), + } +} + #[napi] pub async fn ensure_peer_identity(key_path: String) -> Result { engine::ensure_identity(PathBuf::from(key_path)) @@ -552,6 +640,16 @@ fn parse_addresses(values: Vec, label: &str) -> Result> { .collect() } +fn parse_connect_addresses(values: Vec, label: &str) -> Result> { + if values.len() > MAX_CONNECT_ROUTES_PER_CLASS { + return Err(Error::new( + Status::InvalidArg, + format!("peer connection cannot contain more than 32 {label}s"), + )); + } + parse_addresses(values, label) +} + fn parse_webrtc_stun_urls(values: Vec) -> Result> { if values.len() > MAX_WEBRTC_STUN_URLS { return Err(Error::new( diff --git a/native/runtime-host-peer/src/engine.rs b/native/runtime-host-peer/src/engine.rs index b8aa00be21..d3262845ff 100644 --- a/native/runtime-host-peer/src/engine.rs +++ b/native/runtime-host-peer/src/engine.rs @@ -76,6 +76,8 @@ const MAX_PENDING_OUTGOING_CONNECTIONS: u32 = 1024; const MAX_ESTABLISHED_INCOMING_CONNECTIONS: u32 = 32; const MAX_ESTABLISHED_CONNECTIONS: u32 = 1024; const MAX_CONNECTIONS_PER_PEER: u32 = 4; +const MAX_CONNECT_ROUTES_PER_CLASS: usize = 32; +const MAX_CONNECT_TRANSIT_PEERS: usize = 64; const LISTENER_ADDRESS_QUIET_PERIOD: Duration = Duration::from_millis(250); const COORDINATION_RETRY_INTERVAL: Duration = Duration::from_secs(1); const TRANSIT_HOLE_PUNCH_RETRY_INTERVAL: Duration = Duration::from_secs(30); @@ -107,6 +109,7 @@ pub struct StartOptions { pub struct StartedEndpoint { pub peer_id: PeerId, pub reachability: tokio::sync::watch::Receiver, + pub connectivity: tokio::sync::watch::Receiver, pub transit_snapshot: Arc>, pub commands: mpsc::Sender, pub incoming: mpsc::Receiver, @@ -122,6 +125,17 @@ pub struct ReachabilitySnapshot { pub active_coordination_relays: Vec, } +#[derive(Clone, Default, PartialEq, Eq)] +pub struct ConnectivitySnapshot { + pub generation: u32, + pub connected_peers: Vec, +} + +struct EndpointObservability { + reachability: watch::Sender, + connectivity: watch::Sender, +} + pub struct IdentitySignature { pub public_key: Vec, pub signature: Vec, @@ -136,6 +150,12 @@ pub struct ConnectOptions { pub deadline: Duration, } +pub struct ConnectCandidates { + pub route_hints: Vec, + pub coordination_relays: Vec, + pub transit_relay_peers: Vec, +} + pub enum EngineCommand { Connect { options: ConnectOptions, @@ -146,6 +166,11 @@ pub enum EngineCommand { request_id: u32, result: oneshot::Sender, }, + UpdateConnect { + request_id: u32, + candidates: ConnectCandidates, + result: oneshot::Sender>, + }, ConfigureTransit { policy: TransitPolicy, result: oneshot::Sender<()>, @@ -238,10 +263,23 @@ struct PendingConnect { transit_after: Instant, next_route_attempt: Instant, retry_coordination: bool, - webrtc_attempted: bool, + webrtc_attempted_relays: HashSet, + next_webrtc_attempt_id: u64, + webrtc_active_attempt: Option, + cancellation: CancellationToken, +} + +struct ActiveWebRtcRelayAttempt { + attempt: WebRtcRelayAttempt, cancellation: CancellationToken, } +#[derive(Clone, Copy, PartialEq, Eq)] +struct WebRtcRelayAttempt { + id: u64, + relay_peer_id: PeerId, +} + struct PendingStreamOpen { connection_id: ConnectionId, task: tokio::task::JoinHandle<()>, @@ -257,11 +295,11 @@ pub enum StreamKind { } #[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum DialOrigin { +enum DialOrigin { Direct, Coordination, Transit, - WebRtc, + WebRtc(WebRtcRelayAttempt), } struct StartedConnect { @@ -283,6 +321,7 @@ struct TransitRuntime { struct RouteRuntime<'a> { reachability: Option<&'a tokio::sync::watch::Sender>, + connectivity: Option<&'a tokio::sync::watch::Sender>, relay_anchors: Option<&'a mut RelayAnchorHistory>, transit: &'a mut TransitRuntime, } @@ -465,6 +504,7 @@ struct OutgoingWebRtcUpgrade { request_id: u32, attempt_id: ConnectAttemptId, peer_id: PeerId, + relay_attempt: WebRtcRelayAttempt, result: Result, } @@ -528,6 +568,7 @@ pub fn start(options: StartOptions) -> Result { let (mesh_incoming_tx, mesh_incoming_rx) = mpsc::channel(MESH_INCOMING_STREAM_CAPACITY); let (terminal_tx, terminal_rx) = mpsc::channel(1); let (reachability_tx, reachability_rx) = watch::channel(ReachabilitySnapshot::default()); + let (connectivity_tx, connectivity_rx) = watch::channel(ConnectivitySnapshot::default()); let transit_snapshot = Arc::new(RwLock::new(TransitSnapshot::default())); let transit_snapshot_for_thread = Arc::clone(&transit_snapshot); let thread = thread::Builder::new() @@ -539,7 +580,10 @@ pub fn start(options: StartOptions) -> Result { incoming_tx, mesh_incoming_tx, ready_tx.clone(), - reachability_tx, + EndpointObservability { + reachability: reachability_tx, + connectivity: connectivity_tx, + }, transit_snapshot_for_thread, ); if let Err(error) = result { @@ -554,6 +598,7 @@ pub fn start(options: StartOptions) -> Result { Ok(StartedEndpoint { peer_id: ready, reachability: reachability_rx, + connectivity: connectivity_rx, transit_snapshot, commands: command_tx, incoming: incoming_rx, @@ -569,7 +614,7 @@ fn run_endpoint( incoming_tx: mpsc::Sender, mesh_incoming_tx: mpsc::Sender, ready_tx: std::sync::mpsc::SyncSender>, - reachability: watch::Sender, + observability: EndpointObservability, transit_snapshot: Arc>, ) -> Result<(), PeerError> { let runtime = tokio::runtime::Builder::new_multi_thread() @@ -583,7 +628,7 @@ fn run_endpoint( incoming_tx, mesh_incoming_tx, ready_tx, - reachability, + observability, transit_snapshot, )) } @@ -594,9 +639,13 @@ async fn run_endpoint_async( incoming_tx: mpsc::Sender, mesh_incoming_tx: mpsc::Sender, ready_tx: std::sync::mpsc::SyncSender>, - reachability: watch::Sender, + observability: EndpointObservability, transit_snapshot: Arc>, ) -> Result<(), PeerError> { + let EndpointObservability { + reachability, + connectivity, + } = observability; let key = match options.expected_peer_id { Some(expected) => { let key = identity_store::load_key(&options.key_path).await?; @@ -729,6 +778,7 @@ async fn run_endpoint_async( &mut relay_anchors, &bound_addresses, ); + publish_connectivity(&swarm, &connectivity); let _ = ready_tx.send(Ok(local_peer_id)); let (opened_tx, mut opened_rx) = mpsc::channel::(COMMAND_CAPACITY); @@ -757,12 +807,27 @@ async fn run_endpoint_async( ))); continue; } + if stream_kind == StreamKind::MeshControl + && options.route_hints.is_empty() + && options.coordination_relays.is_empty() + && options.transit_relay_peers.is_empty() + && !mesh_control.has_connection( + options.peer_id, + &direct.retiring_connections, + &HashSet::new(), + ) + { + let _ = result.send(Err(PeerError::new( + "mesh_control_unavailable", + "the peer profile has no direct, coordination, or transit route", + ))); + continue; + } let started = match start_connect( &mut swarm, &mut coordination_relays, &options, local_peer_id, - stream_kind, &direct.active, ) { Ok(peers) => peers, @@ -798,7 +863,9 @@ async fn run_endpoint_async( transit_after, next_route_attempt: Instant::now(), retry_coordination, - webrtc_attempted: false, + webrtc_attempted_relays: HashSet::new(), + next_webrtc_attempt_id: 0, + webrtc_active_attempt: None, cancellation: CancellationToken::new(), }); retry_connect_routes( @@ -842,6 +909,41 @@ async fn run_endpoint_async( }; let _ = result.send(cancelled); } + Some(EngineCommand::UpdateConnect { request_id, candidates, result }) => { + let updated = update_connect_candidates( + &mut swarm, + &mut direct, + &mut coordination_relays, + request_id, + candidates, + local_peer_id, + ); + if matches!(updated, Ok(true)) { + retry_connect_routes( + &mut swarm, + &mut direct, + &coordination_relays, + &stream_control, + Instant::now(), + ); + start_pending_webrtc_upgrades( + &mut direct.pending, + &direct.retiring_connections, + webrtc_signaling_control.clone(), + webrtc_stun_urls.as_deref(), + &mut outgoing_webrtc_upgrades, + ); + maybe_open_peer_stream( + request_id, + &mut direct.pending, + &direct.retiring_connections, + stream_control.clone(), + mesh_control.clone(), + opened_tx.clone(), + ); + } + let _ = result.send(updated); + } Some(EngineCommand::ConfigureTransit { policy, result, @@ -1109,7 +1211,7 @@ async fn run_endpoint_async( continue; } waiter.opening.take(); - if waiter.webrtc_attempted { + if !waiter.webrtc_attempted_relays.is_empty() { webrtc_debug(format_args!( "application stream result peer={} request={} success={}", waiter.peer_id, @@ -1218,7 +1320,7 @@ async fn run_endpoint_async( request_id, )); waiter.rejected_connections.insert(connection_id); - if waiter.dials.remove(&connection_id).is_some() { + if remove_pending_dial(&mut waiter, connection_id).is_some() { direct.retiring_connections.insert(connection_id); let _ = swarm.close_connection(connection_id); } @@ -1250,6 +1352,7 @@ async fn run_endpoint_async( &mut direct, RouteRuntime { reachability: Some(&reachability), + connectivity: Some(&connectivity), relay_anchors: Some(&mut relay_anchors), transit: &mut transit, }, @@ -1491,22 +1594,8 @@ fn start_connect( coordination_relays: &mut HashMap, options: &ConnectOptions, local_peer_id: PeerId, - stream_kind: StreamKind, active_streams: &HashMap, ) -> Result { - if options.route_hints.is_empty() - && options.coordination_relays.is_empty() - && options.transit_relay_peers.is_empty() - { - let code = match stream_kind { - StreamKind::Application => "direct_path_unavailable", - StreamKind::MeshControl => "mesh_control_unavailable", - }; - return Err(PeerError::new( - code, - "the peer profile has no direct, coordination, or transit route", - )); - } let mut relay_peers = Vec::new(); for relay_address in &options.coordination_relays { let relay_peer = coordination_relay_peer_id(relay_address)?; @@ -1555,6 +1644,151 @@ fn start_connect( }) } +fn update_connect_candidates( + swarm: &mut Swarm, + direct: &mut DirectConnectState, + coordination_relays: &mut HashMap, + request_id: u32, + candidates: ConnectCandidates, + local_peer_id: PeerId, +) -> Result { + let Some(waiter) = direct.pending.get(&request_id) else { + return Ok(false); + }; + let peer_id = waiter.peer_id; + let direct_routes = bounded_unique( + candidates + .route_hints + .iter() + .map(|address| address_with_expected_peer(address, peer_id)) + .collect::, _>>()?, + MAX_CONNECT_ROUTES_PER_CLASS, + ); + let coordination_routes = + bounded_unique(candidates.coordination_relays, MAX_CONNECT_ROUTES_PER_CLASS); + let mut relay_peers = Vec::new(); + for relay_address in &coordination_routes { + let relay_peer = coordination_relay_peer_id(relay_address)?; + validate_relay_target( + relay_peer, + peer_id, + local_peer_id, + "coordination_unavailable", + "coordination relay", + )?; + if !relay_peers.contains(&relay_peer) { + relay_peers.push(relay_peer); + } + } + for relay_peer in &candidates.transit_relay_peers { + validate_relay_target( + *relay_peer, + peer_id, + local_peer_id, + "transit_unavailable", + "transit relay", + )?; + } + + let old_coordination_peers = waiter + .coordination_relay_peers + .iter() + .copied() + .collect::>(); + let mut referenced = HashSet::new(); + for relay_address in &coordination_routes { + let relay_peer = coordination_relay_peer_id(relay_address) + .expect("coordination relay was validated before registration"); + register_coordination_relay( + coordination_relays, + relay_address, + local_peer_id, + false, + !old_coordination_peers.contains(&relay_peer) && referenced.insert(relay_peer), + )?; + } + let next_coordination_peers = relay_peers.iter().copied().collect::>(); + let released_coordination_peers = old_coordination_peers + .difference(&next_coordination_peers) + .copied() + .collect::>(); + + let waiter = direct + .pending + .get_mut(&request_id) + .expect("pending connection was retained while candidates were validated"); + let direct_changed = waiter.direct_routes != direct_routes; + waiter.direct_routes = direct_routes; + let coordination_changed = waiter.coordination_relays != coordination_routes; + waiter.coordination_relays = coordination_routes; + waiter.coordination_relay_peers = relay_peers; + let next_transit = candidates + .transit_relay_peers + .into_iter() + .take(MAX_CONNECT_TRANSIT_PEERS) + .collect::>(); + let transit_changed = waiter.transit_relay_peers != next_transit; + waiter.transit_relay_peers = next_transit; + let retired_webrtc_dials = + retain_current_webrtc_relay_attempts(waiter, &mut direct.retiring_connections); + for connection_id in retired_webrtc_dials { + let _ = swarm.close_connection(connection_id); + } + let changed = direct_changed || coordination_changed || transit_changed; + if changed { + waiter.next_route_attempt = Instant::now(); + waiter.transit_after = Instant::now(); + if coordination_changed { + waiter.retry_coordination = true; + } + if direct_changed { + retire_pending_dials_by_origin( + swarm, + &mut direct.retiring_connections, + waiter, + DialOrigin::Direct, + ); + } + if coordination_changed { + retire_pending_dials_by_origin( + swarm, + &mut direct.retiring_connections, + waiter, + DialOrigin::Coordination, + ); + } + if transit_changed { + retire_pending_dials_by_origin( + swarm, + &mut direct.retiring_connections, + waiter, + DialOrigin::Transit, + ); + } + } + release_coordination_relays( + swarm, + coordination_relays, + &released_coordination_peers, + &direct.active, + ); + maintain_coordination_relays(swarm, coordination_relays, &direct.active, Instant::now()); + Ok(true) +} + +fn bounded_unique(values: Vec, limit: usize) -> Vec { + let mut bounded = Vec::with_capacity(limit.min(values.len())); + for value in values { + if !bounded.contains(&value) { + bounded.push(value); + if bounded.len() == limit { + break; + } + } + } + bounded +} + fn validate_relay_target( relay_peer: PeerId, target_peer: PeerId, @@ -1604,29 +1838,30 @@ fn start_pending_webrtc_upgrades( let Some(waiter) = pending.get_mut(&request_id) else { continue; }; - if waiter.stream_kind != StreamKind::Application || waiter.webrtc_attempted { + if waiter.stream_kind != StreamKind::Application || waiter.webrtc_active_attempt.is_some() { continue; } - let relay_peers = waiter - .coordination_relay_peers - .iter() - .copied() - .chain(waiter.transit_relay_peers.iter().copied()) - .collect::>(); - if !signaling_control.has_relayed_connection_via( - waiter.peer_id, - retiring_connections, - &relay_peers, - ) { + let Some(relay_peer_id) = next_webrtc_relay_peer( + &waiter.coordination_relay_peers, + &waiter.transit_relay_peers, + &waiter.webrtc_attempted_relays, + |relay_peer_id| { + signaling_control.has_relayed_connection_via( + waiter.peer_id, + retiring_connections, + &HashSet::from([relay_peer_id]), + ) + }, + ) else { continue; - } + }; - waiter.webrtc_attempted = true; + let (relay_attempt, cancellation) = begin_webrtc_relay_attempt(waiter, relay_peer_id); let attempt_id = waiter.attempt_id; let peer_id = waiter.peer_id; - let cancellation = waiter.cancellation.clone(); let excluded_connections = retiring_connections.clone(); let stun_urls = stun_urls.to_vec(); + let relay_peers = HashSet::from([relay_peer_id]); let mut signaling_control = signaling_control.clone(); upgrades.spawn(async move { let signaling = tokio::select! { @@ -1635,6 +1870,7 @@ fn start_pending_webrtc_upgrades( request_id, attempt_id, peer_id, + relay_attempt, result: Err("WebRTC direct upgrade was cancelled".to_owned()), }; } @@ -1666,12 +1902,117 @@ fn start_pending_webrtc_upgrades( request_id, attempt_id, peer_id, + relay_attempt, result, } }); } } +fn next_webrtc_relay_peer( + coordination_relay_peers: &[PeerId], + transit_relay_peers: &HashSet, + attempted_relays: &HashSet, + mut is_available: impl FnMut(PeerId) -> bool, +) -> Option { + let mut considered = HashSet::new(); + coordination_relay_peers + .iter() + .copied() + .chain(transit_relay_peers.iter().copied()) + .find(|relay_peer_id| { + considered.insert(*relay_peer_id) + && !attempted_relays.contains(relay_peer_id) + && is_available(*relay_peer_id) + }) +} + +fn retain_current_webrtc_relay_attempts( + waiter: &mut PendingConnect, + retiring_connections: &mut HashSet, +) -> Vec { + let current = waiter + .coordination_relay_peers + .iter() + .copied() + .chain(waiter.transit_relay_peers.iter().copied()) + .collect::>(); + waiter + .webrtc_attempted_relays + .retain(|relay_peer_id| current.contains(relay_peer_id)); + if waiter + .webrtc_active_attempt + .as_ref() + .is_some_and(|active| !current.contains(&active.attempt.relay_peer_id)) + && let Some(active) = waiter.webrtc_active_attempt.take() + { + active.cancellation.cancel(); + } + let stale_dials = waiter + .dials + .iter() + .filter_map(|(connection_id, origin)| match origin { + DialOrigin::WebRtc(attempt) if !is_current_webrtc_relay_attempt(waiter, *attempt) => { + Some(*connection_id) + } + _ => None, + }) + .collect::>(); + if waiter + .opening + .as_ref() + .is_some_and(|opening| stale_dials.contains(&opening.connection_id)) + && let Some(opening) = waiter.opening.take() + { + opening.task.abort(); + } + for connection_id in &stale_dials { + remove_pending_dial(waiter, *connection_id); + retiring_connections.insert(*connection_id); + } + stale_dials +} + +fn begin_webrtc_relay_attempt( + waiter: &mut PendingConnect, + relay_peer_id: PeerId, +) -> (WebRtcRelayAttempt, CancellationToken) { + debug_assert!(waiter.webrtc_active_attempt.is_none()); + let attempt = WebRtcRelayAttempt { + id: waiter.next_webrtc_attempt_id, + relay_peer_id, + }; + waiter.next_webrtc_attempt_id += 1; + let cancellation = waiter.cancellation.child_token(); + waiter.webrtc_attempted_relays.insert(relay_peer_id); + waiter.webrtc_active_attempt = Some(ActiveWebRtcRelayAttempt { + attempt, + cancellation: cancellation.clone(), + }); + (attempt, cancellation) +} + +fn finish_webrtc_relay_attempt(waiter: &mut PendingConnect, attempt: WebRtcRelayAttempt) { + if waiter + .webrtc_active_attempt + .as_ref() + .is_some_and(|active| active.attempt == attempt) + { + waiter.webrtc_active_attempt = None; + } +} + +fn is_current_webrtc_relay_attempt(waiter: &PendingConnect, attempt: WebRtcRelayAttempt) -> bool { + waiter + .webrtc_active_attempt + .as_ref() + .is_some_and(|active| active.attempt == attempt) + && (waiter + .coordination_relay_peers + .contains(&attempt.relay_peer_id) + || waiter.transit_relay_peers.contains(&attempt.relay_peer_id)) +} + fn complete_outgoing_webrtc_upgrade( swarm: &mut Swarm, transport: &WebRtcTransportControl, @@ -1682,12 +2023,14 @@ fn complete_outgoing_webrtc_upgrade( request_id, attempt_id, peer_id, + relay_attempt, result, } = completed; - let current = direct - .pending - .get(&request_id) - .is_some_and(|waiter| waiter.attempt_id == attempt_id && waiter.peer_id == peer_id); + let current = direct.pending.get(&request_id).is_some_and(|waiter| { + waiter.attempt_id == attempt_id + && waiter.peer_id == peer_id + && is_current_webrtc_relay_attempt(waiter, relay_attempt) + }); if !current { if let Ok(connection) = result { connection.close_in_background(); @@ -1697,9 +2040,16 @@ fn complete_outgoing_webrtc_upgrade( let connection = match result { Ok(connection) => connection, Err(error) => { + finish_webrtc_relay_attempt( + direct + .pending + .get_mut(&request_id) + .expect("current WebRTC upgrade has a pending connect"), + relay_attempt, + ); webrtc_debug(format_args!( - "outbound upgrade to {} failed: {error}", - peer_id, + "outbound upgrade to {peer_id} via {} failed: {error}", + relay_attempt.relay_peer_id, )); return; } @@ -1707,9 +2057,16 @@ fn complete_outgoing_webrtc_upgrade( let address = match transport.register_outbound(peer_id, connection) { Ok(address) => address, Err(error) => { + finish_webrtc_relay_attempt( + direct + .pending + .get_mut(&request_id) + .expect("current WebRTC upgrade has a pending connect"), + relay_attempt, + ); webrtc_debug(format_args!( - "could not register outbound upgrade to {}: {error}", - peer_id, + "could not register outbound upgrade to {peer_id} via {}: {error}", + relay_attempt.relay_peer_id, )); return; } @@ -1724,12 +2081,21 @@ fn complete_outgoing_webrtc_upgrade( .pending .get_mut(&request_id) .expect("current WebRTC upgrade has a pending connect"); - waiter.dials.insert(connection_id, DialOrigin::WebRtc); + waiter + .dials + .insert(connection_id, DialOrigin::WebRtc(relay_attempt)); webrtc_debug(format_args!( "direct dial submitted peer={} request={} connection={connection_id}", peer_id, request_id, )); } else { + finish_webrtc_relay_attempt( + direct + .pending + .get_mut(&request_id) + .expect("current WebRTC upgrade has a pending connect"), + relay_attempt, + ); transport.discard_outbound(peer_id); } } @@ -1779,7 +2145,7 @@ fn maybe_open_peer_stream( }; let attempt_id = waiter.attempt_id; let cancellation = waiter.cancellation.clone(); - if stream_kind == StreamKind::Application && waiter.webrtc_attempted { + if stream_kind == StreamKind::Application && !waiter.webrtc_attempted_relays.is_empty() { webrtc_debug(format_args!( "opening application stream peer={peer_id} request={request_id} connection={connection_id}" )); @@ -1841,6 +2207,10 @@ fn handle_swarm_event( direct: &mut DirectConnectState, mut route_runtime: RouteRuntime<'_>, ) { + let connectivity_changed = matches!( + &event, + SwarmEvent::ConnectionEstablished { .. } | SwarmEvent::ConnectionClosed { .. } + ); match event { SwarmEvent::ConnectionEstablished { peer_id, @@ -1848,11 +2218,24 @@ fn handle_swarm_event( endpoint, .. } => { - if direct - .pending - .values() - .any(|connect| connect.dials.get(&connection_id) == Some(&DialOrigin::WebRtc)) - { + let stale_webrtc_dial = direct.pending.values().any(|connect| { + matches!( + connect.dials.get(&connection_id), + Some(DialOrigin::WebRtc(attempt)) + if !is_current_webrtc_relay_attempt(connect, *attempt) + ) + }); + if stale_webrtc_dial { + direct.retiring_connections.insert(connection_id); + let _ = swarm.close_connection(connection_id); + return; + } + if direct.pending.values().any(|connect| { + matches!( + connect.dials.get(&connection_id), + Some(DialOrigin::WebRtc(_)) + ) + }) { webrtc_debug(format_args!( "direct connection established peer={peer_id} connection={connection_id}" )); @@ -1894,7 +2277,7 @@ fn handle_swarm_event( } => { direct.retiring_connections.remove(&connection_id); for connect in direct.pending.values_mut() { - connect.dials.remove(&connection_id); + remove_pending_dial(connect, connection_id); connect.rejected_connections.remove(&connection_id); if connect .opening @@ -1948,7 +2331,7 @@ fn handle_swarm_event( discovery_debug(format_args!("outgoing connection failed: {error}")); direct.retiring_connections.remove(&connection_id); for connect in direct.pending.values_mut() { - connect.dials.remove(&connection_id); + remove_pending_dial(connect, connection_id); connect.rejected_connections.remove(&connection_id); if connect .opening @@ -2107,6 +2490,9 @@ fn handle_swarm_event( } _ => {} } + if connectivity_changed && let Some(connectivity) = route_runtime.connectivity { + publish_connectivity(swarm, connectivity); + } } fn handle_startup_event( @@ -2122,6 +2508,7 @@ fn handle_startup_event( &mut DirectConnectState::default(), RouteRuntime { reachability: None, + connectivity: None, relay_anchors: None, transit, }, @@ -2248,6 +2635,11 @@ fn reconcile_pending_transit_connects( waiter .transit_relay_peers .retain(|peer| !revoked_relays.contains(peer)); + let retired_webrtc_dials = + retain_current_webrtc_relay_attempts(waiter, &mut direct.retiring_connections); + for connection_id in retired_webrtc_dials { + let _ = swarm.close_connection(connection_id); + } if let Some(opening) = waiter.opening.take() { opening.task.abort(); } @@ -2295,12 +2687,23 @@ fn retire_pending_dials_by_origin( .filter_map(|(connection_id, current)| (*current == origin).then_some(*connection_id)) .collect::>(); for connection_id in connections { - waiter.dials.remove(&connection_id); + remove_pending_dial(waiter, connection_id); retiring.insert(connection_id); let _ = swarm.close_connection(connection_id); } } +fn remove_pending_dial( + waiter: &mut PendingConnect, + connection_id: ConnectionId, +) -> Option { + let origin = waiter.dials.remove(&connection_id); + if let Some(DialOrigin::WebRtc(attempt)) = origin { + finish_webrtc_relay_attempt(waiter, attempt); + } + origin +} + fn fail_pending_connect( swarm: &mut Swarm, direct: &mut DirectConnectState, @@ -2657,6 +3060,22 @@ fn publish_active_coordination_relays( }); } +fn publish_connectivity( + swarm: &Swarm, + connectivity: &watch::Sender, +) { + let mut connected_peers = swarm.connected_peers().copied().collect::>(); + connected_peers.sort_unstable_by_key(ToString::to_string); + let current = connectivity.borrow().clone(); + if current.connected_peers == connected_peers { + return; + } + connectivity.send_replace(ConnectivitySnapshot { + generation: current.generation.wrapping_add(1).max(1), + connected_peers, + }); +} + fn active_coordination_routes(relays: &HashMap) -> Vec { let mut relay_routes = relays .iter() @@ -3276,7 +3695,9 @@ mod tests { transit_after: now, next_route_attempt: now, retry_coordination: false, - webrtc_attempted: false, + webrtc_attempted_relays: HashSet::new(), + next_webrtc_attempt_id: 0, + webrtc_active_attempt: None, cancellation: CancellationToken::new(), }, ); @@ -3288,6 +3709,142 @@ mod tests { assert!(direct.pending.is_empty()); } + #[test] + fn failed_webrtc_upgrade_can_use_a_new_relay_on_the_same_connect_attempt() { + let now = Instant::now(); + let first_relay = PeerId::random(); + let second_relay = PeerId::random(); + let attempt_id = ConnectAttemptId(9); + let (result, _response) = oneshot::channel(); + let mut waiter = PendingConnect { + attempt_id, + peer_id: PeerId::random(), + result, + stream_kind: StreamKind::Application, + deadline: now + Duration::from_secs(30), + opening: None, + rejected_connections: HashSet::new(), + dials: HashMap::new(), + direct_routes: Vec::new(), + coordination_relays: Vec::new(), + coordination_relay_peers: vec![first_relay], + transit_relay_peers: HashSet::new(), + transit_after: now, + next_route_attempt: now, + retry_coordination: false, + webrtc_attempted_relays: HashSet::new(), + next_webrtc_attempt_id: 0, + webrtc_active_attempt: None, + cancellation: CancellationToken::new(), + }; + + let selected = next_webrtc_relay_peer( + &waiter.coordination_relay_peers, + &waiter.transit_relay_peers, + &waiter.webrtc_attempted_relays, + |_| true, + ); + assert_eq!(selected, Some(first_relay)); + let (first_attempt, _) = begin_webrtc_relay_attempt(&mut waiter, first_relay); + finish_webrtc_relay_attempt(&mut waiter, first_attempt); + + waiter.coordination_relay_peers = vec![first_relay, second_relay]; + let mut retiring_connections = HashSet::new(); + assert!( + retain_current_webrtc_relay_attempts(&mut waiter, &mut retiring_connections,) + .is_empty() + ); + let selected = next_webrtc_relay_peer( + &waiter.coordination_relay_peers, + &waiter.transit_relay_peers, + &waiter.webrtc_attempted_relays, + |_| true, + ); + assert_eq!(selected, Some(second_relay)); + assert!(waiter.attempt_id == attempt_id); + } + + #[tokio::test] + async fn removed_webrtc_relay_attempt_is_cancelled_and_cannot_commit() { + let now = Instant::now(); + let removed_relay = PeerId::random(); + let replacement_relay = PeerId::random(); + let (result, _response) = oneshot::channel(); + let mut waiter = PendingConnect { + attempt_id: ConnectAttemptId(11), + peer_id: PeerId::random(), + result, + stream_kind: StreamKind::Application, + deadline: now + Duration::from_secs(30), + opening: None, + rejected_connections: HashSet::new(), + dials: HashMap::new(), + direct_routes: Vec::new(), + coordination_relays: Vec::new(), + coordination_relay_peers: vec![removed_relay], + transit_relay_peers: HashSet::new(), + transit_after: now, + next_route_attempt: now, + retry_coordination: false, + webrtc_attempted_relays: HashSet::new(), + next_webrtc_attempt_id: 0, + webrtc_active_attempt: None, + cancellation: CancellationToken::new(), + }; + + let (removed_attempt, removed_cancellation) = + begin_webrtc_relay_attempt(&mut waiter, removed_relay); + let removed_connection_id = DialOpts::peer_id(waiter.peer_id) + .condition(PeerCondition::Always) + .build() + .connection_id(); + waiter + .dials + .insert(removed_connection_id, DialOrigin::WebRtc(removed_attempt)); + waiter.opening = Some(PendingStreamOpen { + connection_id: removed_connection_id, + task: tokio::spawn(std::future::pending()), + }); + waiter.coordination_relay_peers = vec![replacement_relay]; + let mut retiring_connections = HashSet::new(); + assert_eq!( + retain_current_webrtc_relay_attempts(&mut waiter, &mut retiring_connections), + vec![removed_connection_id], + ); + + assert!(removed_cancellation.is_cancelled()); + assert!(!is_current_webrtc_relay_attempt(&waiter, removed_attempt)); + assert!(waiter.opening.is_none()); + assert!(!waiter.dials.contains_key(&removed_connection_id)); + assert!(retiring_connections.contains(&removed_connection_id)); + let replacement = next_webrtc_relay_peer( + &waiter.coordination_relay_peers, + &waiter.transit_relay_peers, + &waiter.webrtc_attempted_relays, + |_| true, + ) + .expect("replacement relay is immediately eligible"); + let (replacement_attempt, _) = begin_webrtc_relay_attempt(&mut waiter, replacement); + let replacement_connection_id = DialOpts::peer_id(waiter.peer_id) + .condition(PeerCondition::Always) + .build() + .connection_id(); + waiter.dials.insert( + replacement_connection_id, + DialOrigin::WebRtc(replacement_attempt), + ); + assert!(remove_pending_dial(&mut waiter, removed_connection_id).is_none()); + assert!(is_current_webrtc_relay_attempt( + &waiter, + replacement_attempt + )); + assert!(matches!( + waiter.dials.get(&replacement_connection_id), + Some(DialOrigin::WebRtc(attempt)) if *attempt == replacement_attempt + )); + assert_ne!(removed_attempt.id, replacement_attempt.id); + } + #[tokio::test] async fn identity_signature_is_bound_to_peer_and_payload() { let root = std::env::temp_dir().join(format!("maka-peer-signature-{}", PeerId::random())); @@ -3391,6 +3948,158 @@ mod tests { std::fs::remove_dir_all(root).expect("remove test root"); } + #[tokio::test(flavor = "multi_thread")] + async fn pending_connect_accepts_a_new_route_without_restarting_the_attempt() { + let root = std::env::temp_dir().join(format!("maka-peer-live-route-{}", PeerId::random())); + std::fs::create_dir_all(&root).expect("create test root"); + let source = start(test_endpoint_options(root.join("source.key"))).expect("start source"); + let mut target = + start(test_endpoint_options(root.join("target.key"))).expect("start target"); + let response = begin_test_connect( + &source, + ConnectOptions { + request_id: 1, + peer_id: target.peer_id, + route_hints: Vec::new(), + coordination_relays: Vec::new(), + transit_relay_peers: Vec::new(), + deadline: Duration::from_secs(5), + }, + ) + .await; + let route = test_listen_address(&target); + let (updated, update_response) = oneshot::channel(); + source + .commands + .send(EngineCommand::UpdateConnect { + request_id: 1, + candidates: ConnectCandidates { + route_hints: vec![route], + coordination_relays: Vec::new(), + transit_relay_peers: Vec::new(), + }, + result: updated, + }) + .await + .expect("send route update"); + assert!( + update_response + .await + .expect("route update response") + .expect("route update failed"), + ); + let source_stream = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("live-route connect timeout") + .expect("live-route connect response") + .expect("live-route connect failed"); + let target_stream = tokio::time::timeout(Duration::from_secs(5), target.incoming.recv()) + .await + .expect("live-route inbound timeout") + .expect("live-route inbound stream"); + assert!( + source + .connectivity + .borrow() + .connected_peers + .contains(&target.peer_id), + "the established peer connection must wake higher-level recovery", + ); + + close_test_stream(source_stream).await; + close_test_stream(target_stream).await; + stop_test_endpoint(source).await; + stop_test_endpoint(target).await; + std::fs::remove_dir_all(root).expect("remove test root"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn mesh_control_derives_empty_route_behavior_from_live_connectivity() { + let root = std::env::temp_dir().join(format!("maka-peer-mesh-route-{}", PeerId::random())); + std::fs::create_dir_all(&root).expect("create test root"); + let source = start(test_endpoint_options(root.join("source.key"))).expect("start source"); + let mut target = + start(test_endpoint_options(root.join("target.key"))).expect("start target"); + let isolated = + start(test_endpoint_options(root.join("isolated.key"))).expect("start isolated"); + + let application_source = connect_test_stream( + &source, + target.peer_id, + test_listen_address(&target), + 1, + StreamKind::Application, + ) + .await; + let application_target = + tokio::time::timeout(Duration::from_secs(5), target.incoming.recv()) + .await + .expect("application inbound timeout") + .expect("application inbound stream"); + + let (result, response) = oneshot::channel(); + source + .commands + .send(EngineCommand::Connect { + options: ConnectOptions { + request_id: 2, + peer_id: target.peer_id, + route_hints: Vec::new(), + coordination_relays: Vec::new(), + transit_relay_peers: Vec::new(), + deadline: Duration::from_secs(5), + }, + stream_kind: StreamKind::MeshControl, + result, + }) + .await + .expect("send Mesh control connect"); + let mesh_source = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("Mesh control connect timeout") + .expect("Mesh control connect response") + .expect("Mesh control connect failed"); + let mesh_target = tokio::time::timeout(Duration::from_secs(5), target.mesh_incoming.recv()) + .await + .expect("Mesh control inbound timeout") + .expect("Mesh control inbound stream"); + + let (result, response) = oneshot::channel(); + isolated + .commands + .send(EngineCommand::Connect { + options: ConnectOptions { + request_id: 1, + peer_id: target.peer_id, + route_hints: Vec::new(), + coordination_relays: Vec::new(), + transit_relay_peers: Vec::new(), + deadline: Duration::from_secs(5), + }, + stream_kind: StreamKind::MeshControl, + result, + }) + .await + .expect("send unavailable Mesh control connect"); + let response = tokio::time::timeout(Duration::from_secs(1), response) + .await + .expect("unavailable Mesh control response timeout") + .expect("unavailable Mesh control response"); + let Err(error) = response else { + panic!("unavailable Mesh control unexpectedly connected"); + }; + assert_eq!(error.code, "mesh_control_unavailable"); + + close_test_stream(application_source).await; + close_test_stream(application_target).await; + close_test_stream(mesh_source).await; + close_test_stream(mesh_target).await; + stop_test_endpoint(source).await; + stop_test_endpoint(target).await; + stop_test_endpoint(isolated).await; + std::fs::remove_dir_all(root).expect("remove test root"); + } + #[tokio::test(flavor = "multi_thread")] async fn approved_peers_can_exchange_an_application_stream_through_transit() { let root = std::env::temp_dir().join(format!("maka-peer-transit-{}", PeerId::random())); diff --git a/native/runtime-host-peer/src/lib.rs b/native/runtime-host-peer/src/lib.rs index 5d12abf38e..67bb82a3dc 100644 --- a/native/runtime-host-peer/src/lib.rs +++ b/native/runtime-host-peer/src/lib.rs @@ -22,8 +22,8 @@ mod engine; mod webrtc_direct; pub use bindings::{ - ConfigurePeerTransitOptions, ConnectPeerOptions, PeerEndpoint, PeerIdentitySignature, - PeerReachabilitySnapshot, PeerStream, PeerTransitRelayCandidate, PeerTransitSnapshot, - StartPeerEndpointOptions, ensure_peer_identity, sign_peer_identity, start_peer_endpoint, - verify_peer_identity, + ConfigurePeerTransitOptions, ConnectPeerOptions, PeerConnectivitySnapshot, PeerEndpoint, + PeerIdentitySignature, PeerReachabilitySnapshot, PeerStream, PeerTransitRelayCandidate, + PeerTransitSnapshot, StartPeerEndpointOptions, ensure_peer_identity, sign_peer_identity, + start_peer_endpoint, verify_peer_identity, }; diff --git a/packages/cli/src/__tests__/runtime-host-cli-context.test.ts b/packages/cli/src/__tests__/runtime-host-cli-context.test.ts index d2d1abc60b..8c4ca7183f 100644 --- a/packages/cli/src/__tests__/runtime-host-cli-context.test.ts +++ b/packages/cli/src/__tests__/runtime-host-cli-context.test.ts @@ -373,7 +373,7 @@ test('remote CLI profiles pin root identity and resolve credential outside the p }, profileCatalog: { read: async () => ({ - schemaVersion: 3, + schemaVersion: 4, profiles: [ { id: 'office', @@ -677,7 +677,7 @@ function incompatibleRemoteHandshake(overrides: Partial = {}): function singleRemoteProfileCatalog(profile: RemoteRuntimeHostProfile): RuntimeHostProfileCatalog { return { - read: async () => ({ schemaVersion: 3, profiles: [profile] }), + read: async () => ({ schemaVersion: 4, profiles: [profile] }), resolve: async (profileId) => { assert.equal(profileId, profile.id); return { diff --git a/packages/cli/src/__tests__/runtime-host-operator-command.test.ts b/packages/cli/src/__tests__/runtime-host-operator-command.test.ts index 65521960d7..464ecc2085 100644 --- a/packages/cli/src/__tests__/runtime-host-operator-command.test.ts +++ b/packages/cli/src/__tests__/runtime-host-operator-command.test.ts @@ -506,7 +506,6 @@ describe('Runtime Host operator commands', () => { websocketEndpoints: ['wss://runtime.example.com:443/runtime-host'], peerListeners: [ { - peerId: '12D3KooWPeer', reachability: { lease: { version: 1, diff --git a/packages/cli/src/__tests__/runtime-host-profile-command.test.ts b/packages/cli/src/__tests__/runtime-host-profile-command.test.ts index 15fee776b2..68d4b4d21d 100644 --- a/packages/cli/src/__tests__/runtime-host-profile-command.test.ts +++ b/packages/cli/src/__tests__/runtime-host-profile-command.test.ts @@ -88,7 +88,7 @@ describe('Runtime Host profile CLI', () => { expectedRootId: ROOT_ID, }, ); - assert.deepEqual( + assert.equal( parseRuntimeHostCommand([ 'profile', 'set', @@ -102,19 +102,8 @@ describe('Runtime Host profile CLI', () => { '/ip4/192.0.2.10/udp/4001/quic-v1', '--expected-root', ROOT_ID, - ]), - { - kind: 'runtime-host-profile-set', - id: 'peer-lab', - name: 'Peer Lab', - transport: { - kind: 'libp2p-direct', - peerId: '12D3KooWPeer', - routeHints: ['/ip4/192.0.2.10/udp/4001/quic-v1'], - coordinationRelays: [], - }, - expectedRootId: ROOT_ID, - }, + ]).kind, + 'error', ); assert.equal( parseRuntimeHostCommand([ @@ -215,7 +204,7 @@ function createProfileCatalogCapture(): { saved: Array<{ profile: RemoteRuntimeHostProfile; credential?: string }>; } { const state: { document: RuntimeHostProfileDocument } = { - document: { schemaVersion: 3, profiles: [] }, + document: { schemaVersion: 4, profiles: [] }, }; const saved: Array<{ profile: RemoteRuntimeHostProfile; credential?: string }> = []; const catalog: RuntimeHostProfileCatalog = { @@ -225,7 +214,7 @@ function createProfileCatalogCapture(): { save: async (profile: RemoteRuntimeHostProfile, credential?: string) => { saved.push({ profile, credential }); state.document = { - schemaVersion: 3, + schemaVersion: 4, profiles: [ ...state.document.profiles.filter((candidate) => candidate.id !== profile.id), profile, diff --git a/packages/cli/src/__tests__/tui-mcp-remote-publication.test.ts b/packages/cli/src/__tests__/tui-mcp-remote-publication.test.ts index b8c6864226..77b8338f5d 100644 --- a/packages/cli/src/__tests__/tui-mcp-remote-publication.test.ts +++ b/packages/cli/src/__tests__/tui-mcp-remote-publication.test.ts @@ -40,13 +40,32 @@ const PROFILE: RemoteRuntimeHostProfile = { rootId: 'a'.repeat(64), }; +function reachability( + peerId: string, + directRoutes: readonly string[], + coordinationRoutes: readonly string[] = [], + revision = 1, +): Extract['reachability'] { + return { + lease: { + version: 1, + peerId, + revision, + issuedAt: 1, + expiresAt: 2, + directRoutes, + coordinationRoutes, + }, + publicKey: Buffer.from('public-key').toString('base64url'), + signature: Buffer.from(`signature-${revision}`).toString('base64url'), + }; +} + const DIRECT_PROFILE: RemoteRuntimeHostProfile = { ...PROFILE, transport: { kind: 'libp2p-direct', - peerId: 'peer-a', - routeHints: ['/ip4/127.0.0.1/tcp/4001'], - coordinationRelays: [], + reachability: reachability('peer-a', ['/ip4/127.0.0.1/tcp/4001']), }, }; @@ -415,9 +434,12 @@ test('remote TUI publication reconnects through current direct-peer routes', asy ...DIRECT_PROFILE, transport: { kind: 'libp2p-direct', - peerId: 'peer-a', - routeHints: ['/ip6/2001:db8::10/udp/4001/quic-v1'], - coordinationRelays: ['/dns4/relay.example.com/tcp/443/wss/p2p/relay-a'], + reachability: reachability( + 'peer-a', + ['/ip6/2001:db8::10/udp/4001/quic-v1'], + ['/dns4/relay.example.com/tcp/443/wss/p2p/relay-a'], + 2, + ), }, }; profiles.update(moved); diff --git a/packages/cli/src/cli-core.ts b/packages/cli/src/cli-core.ts index 0e314b83fb..7c2fc2d315 100644 --- a/packages/cli/src/cli-core.ts +++ b/packages/cli/src/cli-core.ts @@ -170,7 +170,6 @@ function helpText(cliCommand: string): string { ` ${cliCommand} runtime-host profile set --id --name --tls-url --expected-root [--credential-env ]`, ` ${cliCommand} runtime-host profile set --id --name --ssh-destination --ssh-remote-port --expected-root [--ssh-port ] [--credential-env ]`, ` ${cliCommand} runtime-host profile set --id --name --plaintext-url --acknowledge-plaintext --expected-root [--credential-env ]`, - ` ${cliCommand} runtime-host profile set --id --name --peer-id --peer-route --expected-root [--credential-env ]`, ` ${cliCommand} runtime-host profile remove --id `, ` ${cliCommand} runtime-host capability-provider serve --url --mcp-config --expected-root `, '', diff --git a/packages/cli/src/runtime-host-cli.ts b/packages/cli/src/runtime-host-cli.ts index 7bafac93d4..0fae0b15e8 100644 --- a/packages/cli/src/runtime-host-cli.ts +++ b/packages/cli/src/runtime-host-cli.ts @@ -355,12 +355,6 @@ export type RuntimeHostCliCommand = sshPort?: number; remotePort: number; websocketPath: string; - } - | { - kind: 'libp2p-direct'; - peerId: string; - routeHints: string[]; - coordinationRelays: string[]; }; expectedRootId: string; credentialEnv?: string; @@ -1713,11 +1707,8 @@ function parseProfileCommand(argv: string[]): RuntimeHostCliCommand { let sshRemotePort: number | undefined; let sshWebSocketPath = '/runtime-host'; let sshWebSocketPathConfigured = false; - let peerId: string | undefined; let wslDistribution: string | undefined; let operatorPath: string | undefined; - const peerRouteHints: string[] = []; - const peerCoordinationRelays: string[] = []; let expectedRootId: string | undefined; let credentialEnv: string | undefined; for (let index = 1; index < argv.length; index += 1) { @@ -1731,9 +1722,6 @@ function parseProfileCommand(argv: string[]): RuntimeHostCliCommand { argument !== '--ssh-port' && argument !== '--ssh-remote-port' && argument !== '--ssh-websocket-path' && - argument !== '--peer-id' && - argument !== '--peer-route' && - argument !== '--peer-coordination-relay' && argument !== '--wsl-distribution' && argument !== '--operator-path' && argument !== '--expected-root' && @@ -1759,9 +1747,6 @@ function parseProfileCommand(argv: string[]): RuntimeHostCliCommand { sshWebSocketPath = parsed; sshWebSocketPathConfigured = true; } - if (argument === '--peer-id') peerId = parsed; - if (argument === '--peer-route') peerRouteHints.push(parsed); - if (argument === '--peer-coordination-relay') peerCoordinationRelays.push(parsed); if (argument === '--wsl-distribution') wslDistribution = parsed; if (argument === '--operator-path') operatorPath = parsed; if (argument === '--expected-root') expectedRootId = parsed; @@ -1774,12 +1759,11 @@ function parseProfileCommand(argv: string[]): RuntimeHostCliCommand { (tlsUrl ? 1 : 0) + (plaintextUrl ? 1 : 0) + (sshDestination ? 1 : 0) + - (peerId ? 1 : 0) + (wslDistribution ? 1 : 0) !== 1 ) { return error( - 'exactly one of --tls-url, --plaintext-url, --ssh-destination, --peer-id, or --wsl-distribution is required', + 'exactly one of --tls-url, --plaintext-url, --ssh-destination, or --wsl-distribution is required', ); } if (wslDistribution && !operatorPath) { @@ -1806,12 +1790,6 @@ function parseProfileCommand(argv: string[]): RuntimeHostCliCommand { if (sshDestination && !sshRemotePort) { return error('--ssh-destination requires --ssh-remote-port'); } - if (!peerId && (peerRouteHints.length > 0 || peerCoordinationRelays.length > 0)) { - return error('peer route options require --peer-id'); - } - if (peerId && peerRouteHints.length === 0 && peerCoordinationRelays.length === 0) { - return error('--peer-id requires at least one --peer-route or --peer-coordination-relay'); - } if (sshPort !== undefined && (!Number.isInteger(sshPort) || sshPort < 1 || sshPort > 65_535)) { return error('--ssh-port must be an integer between 1 and 65535'); } @@ -1832,32 +1810,26 @@ function parseProfileCommand(argv: string[]): RuntimeHostCliCommand { expectedRootId, }; } + const transport = tlsUrl + ? ({ kind: 'tls', url: tlsUrl } as const) + : plaintextUrl + ? ({ + kind: 'plaintext', + url: plaintextUrl, + acknowledgement: 'plaintext-bearer-v1', + } as const) + : ({ + kind: 'ssh', + destination: sshDestination!, + ...(sshPort === undefined ? {} : { sshPort }), + remotePort: sshRemotePort!, + websocketPath: sshWebSocketPath, + } as const); return { kind: 'runtime-host-profile-set', id, name, - transport: tlsUrl - ? { kind: 'tls', url: tlsUrl } - : plaintextUrl - ? { - kind: 'plaintext', - url: plaintextUrl, - acknowledgement: 'plaintext-bearer-v1', - } - : sshDestination - ? { - kind: 'ssh', - destination: sshDestination, - ...(sshPort === undefined ? {} : { sshPort }), - remotePort: sshRemotePort!, - websocketPath: sshWebSocketPath, - } - : { - kind: 'libp2p-direct', - peerId: peerId!, - routeHints: peerRouteHints, - coordinationRelays: peerCoordinationRelays, - }, + transport, expectedRootId, ...(credentialEnv ? { credentialEnv } : {}), }; diff --git a/packages/cli/src/runtime-host-service-command.ts b/packages/cli/src/runtime-host-service-command.ts index d95ef043c7..d42007a757 100644 --- a/packages/cli/src/runtime-host-service-command.ts +++ b/packages/cli/src/runtime-host-service-command.ts @@ -102,7 +102,7 @@ export async function runRuntimeHostServiceCli( } for (const peer of host.peerListeners) { process.stdout.write( - `Runtime Host direct peer is ready as ${peer.peerId} at ${peer.reachability.lease.directRoutes.join(', ')}\n`, + `Runtime Host direct peer is ready as ${peer.reachability.lease.peerId} at ${peer.reachability.lease.directRoutes.join(', ')}\n`, ); } }, @@ -150,7 +150,6 @@ export function createRuntimeHostServiceReadyEvent(host: { readonly endpoint: string; readonly websocketEndpoints: readonly string[]; readonly peerListeners: readonly { - readonly peerId: string; readonly reachability: SignedPeerReachabilityLeaseV1; }[]; readonly compositionDescriptor: { readonly id: string; readonly revision: string }; @@ -179,7 +178,7 @@ export function createRuntimeHostServiceReadyEvent(host: { }), ...host.peerListeners.map((peer) => ({ kind: 'libp2p_direct' as const, - peerId: peer.peerId, + peerId: peer.reachability.lease.peerId, listenAddresses: peer.reachability.lease.directRoutes, })), ], diff --git a/packages/runtime-host/src/__tests__/host-profile.test.ts b/packages/runtime-host/src/__tests__/host-profile.test.ts index 0e8c6b05ce..cd36ca4212 100644 --- a/packages/runtime-host/src/__tests__/host-profile.test.ts +++ b/packages/runtime-host/src/__tests__/host-profile.test.ts @@ -28,6 +28,7 @@ import { RuntimeHostRemoteCompatibilityError, } from '../client/index.js'; import { + connectPeerRuntimeHost, connectRemoteRuntimeHostProfile, createFileRuntimeHostProfileCatalog, createRuntimeHostCapabilityProviderCredentialStore, @@ -41,6 +42,7 @@ import { type RuntimeHostProfileCredential, type RuntimeHostProfileCredentialStore, } from '../client/host-profile.js'; +import type { RuntimeHostPeerClient } from '../client/peer-client.js'; import { RuntimeHostPermanentReconnectError } from '../client/reconnect-lifecycle.js'; import { INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, @@ -48,6 +50,7 @@ import { RUNTIME_HOST_PROTOCOL_VERSION, type HostIncompatible, } from '../protocol/index.js'; +import { RuntimeHostPeerError } from '../transport/peer-native.js'; const ROOT_A = 'a'.repeat(64); const ROOT_B = 'b'.repeat(64); @@ -61,7 +64,7 @@ describe('Runtime Host profiles', () => { test('persists WSL environments without projecting a remote credential', async () => { const path = await profilePath(); const catalog = createFileRuntimeHostProfileCatalog(path, memoryCredentials()); - assert.deepEqual(await catalog.read(), { schemaVersion: 3, profiles: [] }); + assert.deepEqual(await catalog.read(), { schemaVersion: 4, profiles: [] }); await catalog.create({ id: 'ubuntu', name: 'Ubuntu', @@ -121,7 +124,7 @@ describe('Runtime Host profiles', () => { ); assert.deepEqual(await catalog.read(), { - schemaVersion: 3, + schemaVersion: 4, profiles: [ { id: 'office', @@ -144,7 +147,7 @@ describe('Runtime Host profiles', () => { if (process.platform !== 'win32') assert.equal((await stat(path)).mode & 0o777, 0o600); assert.deepEqual(await catalog.remove('office'), { - schemaVersion: 3, + schemaVersion: 4, profiles: [ { id: 'backup', @@ -197,7 +200,7 @@ describe('Runtime Host profiles', () => { const catalog = createFileRuntimeHostProfileCatalog(path, memoryCredentials()); const document = await catalog.read(); - assert.equal(document.schemaVersion, 3); + assert.equal(document.schemaVersion, 4); assert.equal( (JSON.parse(await readFile(path, 'utf8')) as { schemaVersion: number }).schemaVersion, 1, @@ -278,7 +281,7 @@ describe('Runtime Host profiles', () => { assert.deepEqual(await desktop.removeIfCurrent(created), { removed: false, document: { - schemaVersion: 3, + schemaVersion: 4, profiles: [ { ...profile, @@ -292,7 +295,7 @@ describe('Runtime Host profiles', () => { assert.equal(rotated.credential, 'rotated-token'); assert.equal(rotated.profileIncarnationId, created.profileIncarnationId); assert.equal((await desktop.removeIfCurrent(rotated)).removed, true); - assert.deepEqual(await desktop.read(), { schemaVersion: 3, profiles: [] }); + assert.deepEqual(await desktop.read(), { schemaVersion: 4, profiles: [] }); }); test('conditionally updates one Host connection and credential', async () => { @@ -646,12 +649,15 @@ describe('Runtime Host profiles', () => { test('pins a direct-peer profile to its PeerId while allowing route discovery to change', () => { const original = directPeerProfile('peer-a', ['/ip4/192.0.2.10/udp/4001/quic-v1']); const moved = directPeerProfile('peer-a', ['/ip6/2001:db8::10/udp/4001/quic-v1']); - const replacement = directPeerProfile('peer-b', moved.transport.routeHints); + const replacement = directPeerProfile( + 'peer-b', + moved.transport.reachability.lease.directRoutes, + ); assert.equal(sameRemoteRuntimeHostProfileTarget(original, moved), true); assert.equal(sameRemoteRuntimeHostProfileTarget(original, replacement), false); assert.deepEqual( - decodeRuntimeHostProfileDocument({ schemaVersion: 1, profiles: [moved] }).profiles[0], + decodeRuntimeHostProfileDocument({ schemaVersion: 4, profiles: [moved] }).profiles[0], moved, ); }); @@ -1113,6 +1119,70 @@ describe('Runtime Host profiles', () => { ); }); + test('treats missing, immutable, and native Direct capability failures as terminal', async () => { + const profile = directPeerProfile('peer-a', ['/memory/peer-a']); + const connect = (peerClient: RuntimeHostPeerClient) => + connectPeerRuntimeHost({ + profileId: profile.id, + transport: profile.transport, + credential: 'opaque-token', + expectedRootId: profile.rootId, + clientInstanceId: 'client-1', + peerClient, + }); + await assert.rejects( + () => + connectRemoteRuntimeHostProfile({ + profile, + credential: 'opaque-token', + clientInstanceId: 'client-1', + }), + (error: unknown) => { + assert.ok(error instanceof RuntimeHostPermanentReconnectError); + assert.ok(error.cause instanceof RuntimeHostPeerError); + assert.equal(error.cause.code, 'peer_native_unavailable'); + return true; + }, + ); + const invalidEvidence = new Error('signature is invalid'); + await assert.rejects( + () => + connect({ + observeAuthenticatedReachability: () => { + throw invalidEvidence; + }, + } as unknown as RuntimeHostPeerClient), + (error: unknown) => { + assert.ok(error instanceof RuntimeHostProfileConnectionError); + assert.equal(error.reason, 'target_mismatch'); + assert.equal(error.cause, invalidEvidence); + return true; + }, + ); + + for (const code of ['peer_identity_mismatch', 'peer_native_unavailable'] as const) { + const failure = new RuntimeHostPeerError(code, code); + await assert.rejects( + () => + connect({ + observeAuthenticatedReachability: () => profile.transport.reachability, + connect: async () => { + throw failure; + }, + } as unknown as RuntimeHostPeerClient), + (error: unknown) => { + assert.ok(error instanceof RuntimeHostPermanentReconnectError); + assert.equal(error.cause, failure); + if (code === 'peer_identity_mismatch') { + assert.ok(error instanceof RuntimeHostProfileConnectionError); + assert.equal(error.reason, 'target_mismatch'); + } + return true; + }, + ); + } + }); + test('reports retryable remote connection failure categories', async () => { const reasons = [ ['tls_failed', /could not verify the TLS connection/], @@ -1161,7 +1231,23 @@ function directPeerProfile( name: 'Peer', kind: 'remote', rootId: ROOT_A, - transport: { kind: 'libp2p-direct', peerId, routeHints, coordinationRelays: [] }, + transport: { kind: 'libp2p-direct', reachability: reachability(peerId, routeHints) }, + }; +} + +function reachability(peerId: string, directRoutes: readonly string[]) { + return { + lease: { + version: 1 as const, + peerId, + revision: 1, + issuedAt: 1, + expiresAt: 2, + directRoutes, + coordinationRoutes: [], + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), }; } diff --git a/packages/runtime-host/src/__tests__/owner-connection-code.test.ts b/packages/runtime-host/src/__tests__/owner-connection-code.test.ts index 98e47da86a..0fd6c4de65 100644 --- a/packages/runtime-host/src/__tests__/owner-connection-code.test.ts +++ b/packages/runtime-host/src/__tests__/owner-connection-code.test.ts @@ -30,9 +30,7 @@ test('owner connection code round-trips its bounded direct-peer pairing payload' rootId: 'a'.repeat(64), transport: { kind: 'libp2p-direct' as const, - peerId: '12D3KooWpeer', - routeHints: ['/ip4/192.0.2.1/udp/41000/quic-v1'], - coordinationRelays: [], + reachability: reachability(['/ip4/192.0.2.1/udp/41000/quic-v1']), }, credential: 'pending-credential', }; @@ -50,11 +48,25 @@ test('owner connection code rejects unversioned and route-less payloads', () => rootId: 'a'.repeat(64), transport: { kind: 'libp2p-direct', - peerId: '12D3KooWpeer', - routeHints: [], - coordinationRelays: [], + reachability: reachability([]), }, credential: 'pending-credential', }), ); }); + +function reachability(directRoutes: readonly string[]) { + return { + lease: { + version: 1 as const, + peerId: '12D3KooWpeer', + revision: 1, + issuedAt: 1, + expiresAt: 2, + directRoutes, + coordinationRoutes: [], + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), + }; +} diff --git a/packages/runtime-host/src/__tests__/peer-listener.test.ts b/packages/runtime-host/src/__tests__/peer-listener.test.ts index 4b3550b795..497e072101 100644 --- a/packages/runtime-host/src/__tests__/peer-listener.test.ts +++ b/packages/runtime-host/src/__tests__/peer-listener.test.ts @@ -197,6 +197,7 @@ function peerWith(streams: RuntimeHostPeerNativeStream[]): RuntimeHostPeerClient throw new Error('not used'); }, verifyIdentity: () => false, + isConnected: () => false, transitSnapshot: () => ({ allowedPeerCount: 0, activeReservationCount: 0, @@ -208,7 +209,11 @@ function peerWith(streams: RuntimeHostPeerNativeStream[]): RuntimeHostPeerClient maxCircuitBytes: 256 * 1024 * 1024, }), configureTransit: async () => undefined, - observeAuthenticatedRoutes: () => undefined, + attachRouteResolver: () => () => undefined, + subscribeRoutes: () => () => undefined, + observeAuthenticatedReachability: () => { + throw new Error('not used'); + }, connect: async () => { throw new Error('not used'); }, diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index 0366b3b4b4..2bbf52212c 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -447,18 +447,26 @@ test('repairs an existing membership with a fresh invitation after every locator await authorityPeer.setCoordinationRelays([]); await authorityPeer.setRouteHints([]); await authority.reconcile(); + authorityPeer.setReachable(false); await member.reconcile(); assert.equal( member.status()[0]?.memberRoutes.find(({ peerId }) => peerId === 'peer-a')?.state, 'reconnecting', ); + const observedStates: string[] = []; + const unsubscribe = member.subscribeRoutes('peer-a', () => { + observedStates.push(member.resolveRoutes('peer-a').state); + }); await member.prepareRoutes('peer-a', AbortSignal.timeout(4_000)); + unsubscribe(); + assert.deepEqual(observedStates, ['exhausted']); assert.equal( member.status()[0]?.memberRoutes.find(({ peerId }) => peerId === 'peer-a')?.state, 'needs_repair', ); const recoveredRoute = '/memory/peer-a-recovered/p2p/peer-a'; + authorityPeer.setReachable(true); await authorityPeer.setRouteHints([recoveredRoute]); const repaired = await member.join(await authority.invite(meshId)); assert.equal(member.status().length, 1); @@ -1381,6 +1389,7 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { #stallNextControl = false; #responseDelayMs = 0; #reachable = true; + readonly #connectedPeerIds = new Set(); #routeHints: readonly string[]; #coordinationRelays: readonly string[]; #reachability: SignedPeerReachabilityLeaseV1 | undefined; @@ -1421,10 +1430,13 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { } identity() { + return { peerId: this.peerId } as const; + } + + reachability() { return { - peerId: this.peerId, listenAddresses: this.#routeHints, - coordinationRelays: this.#coordinationRelays, + activeCoordinationRelays: this.#coordinationRelays, } as const; } @@ -1438,13 +1450,13 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { } async refresh(): Promise { - const identity = this.identity(); + const reachability = this.reachability(); const now = this.#now(); if ( this.#reachability && this.#reachability.lease.issuedAt <= now && this.#reachability.lease.expiresAt > now + PEER_REACHABILITY_REFRESH_LEAD_MS && - samePeerReachabilityRoutes(this.#reachability.lease, identity) + samePeerReachabilityRoutes(this.#reachability.lease, reachability) ) { return this.#reachability; } @@ -1455,8 +1467,8 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { revision: this.#reachabilityRevision, issuedAt: now, expiresAt: now + PEER_REACHABILITY_LEASE_TTL_MS, - directRoutes: identity.listenAddresses, - coordinationRoutes: identity.coordinationRelays, + directRoutes: reachability.listenAddresses, + coordinationRoutes: reachability.activeCoordinationRelays, }); const proof = await this.signIdentity(peerReachabilityLeaseSigningBytes(lease)); const signed = decodeSignedPeerReachabilityLease({ @@ -1501,6 +1513,10 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { setReachable(reachable: boolean): void { this.#reachable = reachable; + if (!reachable) { + this.#connectedPeerIds.clear(); + for (const peer of this.peers.values()) peer.#connectedPeerIds.delete(this.peerId); + } } setResponseDelay(delayMs: number): void { @@ -1573,6 +1589,10 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { ); } + isConnected(peerId: string): boolean { + return this.#connectedPeerIds.has(peerId); + } + transitSnapshot() { return { allowedPeerCount: this.transitPolicy.allowedPeerIds.length, @@ -1627,9 +1647,11 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { } signal?.throwIfAborted(); const remote = this.peers.get(input.peerId); - if (!remote || !remote.#reachable) { + if (!this.#reachable || !remote || !remote.#reachable) { throw new Error('Peer is unavailable'); } + this.#connectedPeerIds.add(input.peerId); + remote.#connectedPeerIds.add(this.peerId); const [localStream, remoteStream] = memoryStreamPair(this.peerId, input.peerId, (bytes) => { const newline = bytes.indexOf(0x0a); if (newline < 0) return; @@ -1675,6 +1697,8 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { close(): Promise { if (this.#closed) return Promise.resolve(); this.#closed = true; + this.#connectedPeerIds.clear(); + for (const peer of this.peers.values()) peer.#connectedPeerIds.delete(this.peerId); this.#reachabilityListeners.clear(); this.#meshServer?.stop(); return Promise.resolve(); diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index ebd0698167..ab75b07478 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -23,7 +23,11 @@ import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; import { setImmediate as waitForImmediate } from 'node:timers/promises'; import { test } from 'node:test'; -import { createRuntimeHostPeerClient } from '../client/peer-client.js'; +import { + createRuntimeHostPeerClient, + RuntimeHostPeerReachabilityUnavailableError, +} from '../client/peer-client.js'; +import { PEER_REACHABILITY_MAX_CLOCK_SKEW_MS } from '../peer-reachability/index.js'; import { ensureRuntimeHostPeerIdentity, normalizePeerError, @@ -48,10 +52,11 @@ test('shares one peer endpoint, serializes same-peer connects, and cancels indep nativePath, `let finishAccept; let finishMeshAccept; +let finishConnectivity; +let connectivity = { generation: 0, connectedPeerIds: [] }; const pending = new Map(); -const stats = { starts: 0, closes: 0, requests: [], cancellations: [] }; +const stats = { starts: 0, closes: 0, requests: [], updates: [], cancellations: [] }; let missFirstCancellation = true; -let selfContainedRoutesPrepared = false; const stream = { read: async () => null, write: async () => {}, close: async () => {}, abort: () => {} }; module.exports = { stats, @@ -59,6 +64,11 @@ module.exports = { pending.get(requestId)?.resolve(stream); pending.delete(requestId); }, + establishPeer: (peerId) => { + connectivity = { generation: connectivity.generation + 1, connectedPeerIds: [peerId] }; + finishConnectivity?.(connectivity); + finishConnectivity = undefined; + }, failEndpoint: () => { finishAccept?.(null); finishMeshAccept?.(null); }, ensurePeerIdentity: async () => 'client', signPeerIdentity: async () => ({ publicKey: Buffer.from('public'), signature: Buffer.from('signature') }), @@ -68,23 +78,33 @@ module.exports = { return { peerId: 'client', reachabilitySnapshot: { generation: 0, listenAddresses: [], activeCoordinationRelays: [] }, + get connectivitySnapshot() { return connectivity; }, transitSnapshot: { allowedPeerCount: 0, activeReservationCount: 0, activeCircuitCount: 0, maxReservationCount: 32, maxCircuitCount: 8, maxCircuitsPerPeer: 2, maxCircuitDurationSeconds: 7_200, maxCircuitBytes: 256 * 1024 * 1024 }, watchReachability: async () => ({ generation: 0, listenAddresses: [], activeCoordinationRelays: [] }), + watchConnectivity: async (afterGeneration) => connectivity.generation === afterGeneration + ? new Promise((resolve) => { finishConnectivity = resolve; }) + : connectivity, connect: ({ requestId, peerId, routeHints, coordinationRelays, transitRelayPeerIds }) => { stats.requests.push({ requestId, peerId, routeHints, coordinationRelays, transitRelayPeerIds }); if (peerId === 'unreachable') return Promise.reject(Object.assign(new Error('transit_unavailable: no approved route'), { code: 'GenericFailure' })); - if (peerId === 'ready' || peerId === 'fallback' || peerId === 'observed' || (peerId === 'self-contained' && selfContainedRoutesPrepared)) return Promise.resolve(stream); - return new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); + if (peerId === 'ready' || peerId === 'fallback' || peerId === 'observed') return Promise.resolve(stream); + return new Promise((resolve, reject) => pending.set(requestId, { peerId, resolve, reject })); }, connectMeshControl: ({ requestId, peerId, routeHints, coordinationRelays, transitRelayPeerIds }) => { stats.requests.push({ requestId, peerId, routeHints, coordinationRelays, transitRelayPeerIds }); - if (peerId === 'ready' || peerId === 'self-contained') { - if (peerId === 'self-contained') selfContainedRoutesPrepared = true; - return Promise.resolve(stream); - } - return new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); + if (peerId === 'ready' || peerId === 'self-contained') return Promise.resolve(stream); + return new Promise((resolve, reject) => pending.set(requestId, { peerId, resolve, reject })); }, configureTransit: async () => {}, + updateConnect: async (options) => { + stats.updates.push(options); + const request = pending.get(options.requestId); + if (request?.peerId === 'self-contained') { + request.resolve(stream); + pending.delete(options.requestId); + } + return Boolean(request); + }, cancelConnect: async (requestId) => { stats.cancellations.push(requestId); if (missFirstCancellation) { @@ -97,29 +117,44 @@ module.exports = { }, accept: () => new Promise((resolve) => { finishAccept = resolve; }), acceptMeshControl: () => new Promise((resolve) => { finishMeshAccept = resolve; }), - close: async () => { stats.closes += 1; finishAccept?.(null); finishMeshAccept?.(null); }, + close: async () => { stats.closes += 1; finishAccept?.(null); finishMeshAccept?.(null); finishConnectivity?.(connectivity); }, }; }, }; `, ); let routesPrepared = false; + let selfContainedRoutesPrepared = false; + let recoveryExhausted = false; const preparedPeerIds: string[] = []; - let client!: ReturnType; - client = createRuntimeHostPeerClient({ + const client = createRuntimeHostPeerClient({ nativePath, keyPath: join(directory, 'peer.key'), - routeResolver: { - prepareRoutes: async (peerId) => { - preparedPeerIds.push(peerId); - routesPrepared = true; - if (peerId === 'fallback') throw new Error('Mesh refresh failed'); - if (peerId === 'self-contained') { - await client.connectMeshControl(peerConnectInput(peerId)); - } - }, - resolveRoutes: (peerId) => - routesPrepared && peerId !== 'observed' + }); + const detachRouteResolver = client.attachRouteResolver({ + prepareRoutes: async (peerId) => { + preparedPeerIds.push(peerId); + if (peerId === 'exhausted') { + recoveryExhausted = true; + return; + } + routesPrepared = true; + if (peerId === 'fallback') throw new Error('Mesh refresh failed'); + if (peerId === 'self-contained') { + await new Promise((resolve) => setImmediate(resolve)); + selfContainedRoutesPrepared = true; + } + }, + resolveRoutes: (peerId) => + peerId === 'exhausted' + ? { + state: recoveryExhausted ? 'exhausted' : 'recovering', + routeHints: [], + coordinationRelays: [], + transitRelayPeerIds: [], + } + : peerId !== 'observed' && + (peerId === 'self-contained' ? selfContainedRoutesPrepared : routesPrepared) ? { state: 'available', routeHints: ['/memory/discovered'], @@ -132,7 +167,7 @@ module.exports = { coordinationRelays: [], transitRelayPeerIds: [], }, - }, + subscribeRoutes: () => () => undefined, }); const native = await import(nativePath); const phases: string[] = []; @@ -181,86 +216,114 @@ module.exports = { }); await selfContained; assert.equal(preparedPeerIds.includes('self-contained'), true); - client.observeAuthenticatedRoutes({ + client.observeAuthenticatedReachability({ + expectedPeerId: 'observed', + value: signedReachability('observed', ['/memory/fresh'], ['/memory/fresh-relay']), + }); + await client.connect({ + ...peerConnectInput('observed'), + routeHints: ['/memory/stale'], + coordinationRelays: ['/memory/stale-relay'], + }); + assert.equal(native.default.stats.starts, 1); + assert.equal(native.default.stats.closes, 0); + assert.equal(native.default.stats.requests.length, 8); + assert.equal( + native.default.stats.requests.filter( + ({ peerId }: { peerId: string }) => peerId === 'self-contained', + ).length, + 1, + ); + assert.deepEqual(native.default.stats.requests[0], { + requestId: 1, + peerId: 'pending', + routeHints: ['/memory/1'], + coordinationRelays: [], + transitRelayPeerIds: [], + }); + assert.deepEqual(native.default.stats.updates[0], { + requestId: 1, + routeHints: ['/memory/discovered', '/memory/1'], + coordinationRelays: ['/memory/relay'], + transitRelayPeerIds: ['transit-peer'], + }); + assert.deepEqual(native.default.stats.requests.at(-1), { + requestId: 8, peerId: 'observed', - routeHints: ['/memory/fresh'], - coordinationRelays: ['/memory/fresh-relay'], + routeHints: ['/memory/fresh', '/memory/stale'], + coordinationRelays: ['/memory/fresh-relay', '/memory/stale-relay'], + transitRelayPeerIds: [], }); + for (let index = 0; index < 160; index += 1) { + const peerId = `remembered-${index}`; + client.observeAuthenticatedReachability({ + expectedPeerId: peerId, + value: signedReachability(peerId, [`/memory/${peerId}`], []), + }); + } await client.connect({ ...peerConnectInput('observed'), routeHints: ['/memory/stale'], coordinationRelays: ['/memory/stale-relay'], + refreshRoutes: false, + }); + assert.deepEqual(native.default.stats.requests.at(-1), { + requestId: 9, + peerId: 'observed', + routeHints: ['/memory/stale'], + coordinationRelays: ['/memory/stale-relay'], + transitRelayPeerIds: [], + }); + assert.deepEqual(native.default.stats.cancellations, [1, 1]); + + const rolledBackIssuedAt = Date.now() + PEER_REACHABILITY_MAX_CLOCK_SKEW_MS + 1; + client.observeAuthenticatedReachability({ + expectedPeerId: 'ready', + value: signedReachability('ready', ['/memory/before-clock-reset'], [], rolledBackIssuedAt), + allowHistorical: true, }); - assert.deepEqual(native.default.stats, { - starts: 1, - closes: 0, - requests: [ - { - requestId: 1, - peerId: 'pending', - routeHints: ['/memory/discovered', '/memory/1'], - coordinationRelays: ['/memory/relay'], - transitRelayPeerIds: ['transit-peer'], - }, - { - requestId: 2, - peerId: 'shared', - routeHints: ['/memory/discovered', '/memory/1'], - coordinationRelays: ['/memory/relay'], - transitRelayPeerIds: ['transit-peer'], - }, - { - requestId: 3, - peerId: 'shared', - routeHints: ['/memory/1'], - coordinationRelays: [], - transitRelayPeerIds: [], - }, - { - requestId: 4, - peerId: 'ready', - routeHints: ['/memory/discovered', '/memory/1'], - coordinationRelays: ['/memory/relay'], - transitRelayPeerIds: ['transit-peer'], - }, - { - requestId: 5, - peerId: 'fallback', - routeHints: ['/memory/discovered', '/memory/1'], - coordinationRelays: ['/memory/relay'], - transitRelayPeerIds: ['transit-peer'], - }, - { - requestId: 6, - peerId: 'unreachable', - routeHints: ['/memory/discovered', '/memory/1'], - coordinationRelays: ['/memory/relay'], - transitRelayPeerIds: ['transit-peer'], - }, - { - requestId: 7, - peerId: 'self-contained', - routeHints: ['/memory/1'], - coordinationRelays: [], - transitRelayPeerIds: [], - }, - { - requestId: 8, - peerId: 'self-contained', - routeHints: ['/memory/discovered', '/memory/1'], - coordinationRelays: ['/memory/relay', '/memory/explicit-relay'], - transitRelayPeerIds: ['transit-peer'], - }, - { - requestId: 9, - peerId: 'observed', - routeHints: ['/memory/fresh', '/memory/stale'], - coordinationRelays: ['/memory/fresh-relay', '/memory/stale-relay'], - transitRelayPeerIds: [], - }, - ], - cancellations: [1, 1], + await client.connect({ + ...peerConnectInput('ready'), + routeHints: [], + refreshRoutes: false, }); + const historicalRequest = native.default.stats.requests.at(-1) as { + readonly peerId: string; + readonly routeHints: readonly string[]; + }; + assert.equal(historicalRequest.peerId, 'ready'); + assert.equal(historicalRequest.routeHints.includes('/memory/before-clock-reset'), true); + + await assert.rejects( + client.connect({ + ...peerConnectInput('exhausted'), + routeHints: [], + }), + (error: unknown) => + error instanceof RuntimeHostPeerReachabilityUnavailableError && + error.code === 'peer_reachability_needs_repair', + ); + + detachRouteResolver(); + const requestCount = native.default.stats.requests.length; + await assert.rejects( + client.connect({ + ...peerConnectInput('needs-repair'), + routeHints: [], + refreshRoutes: false, + }), + (error: unknown) => error instanceof RuntimeHostPeerReachabilityUnavailableError, + ); + assert.equal(native.default.stats.requests.length, requestCount); + + let connectivityWakeups = 0; + const unsubscribeConnectivity = client.subscribeRoutes('restored', () => { + connectivityWakeups += 1; + }); + native.default.establishPeer('restored'); + await waitForImmediate(); + assert.equal(connectivityWakeups, 1); + unsubscribeConnectivity(); native.default.failEndpoint(); await waitForImmediate(); @@ -308,13 +371,16 @@ module.exports = { startPeerEndpoint: (options) => { starts.push(options); return ({ - peerId: 'peer', - reachabilitySnapshot: { generation: 0, listenAddresses: [], activeCoordinationRelays: [] }, + peerId: 'peer', + reachabilitySnapshot: { generation: 0, listenAddresses: [], activeCoordinationRelays: [] }, + connectivitySnapshot: { generation: 0, connectedPeerIds: [] }, transitSnapshot: { allowedPeerCount: 0, activeReservationCount: 0, activeCircuitCount: 0, maxReservationCount: 32, maxCircuitCount: 8, maxCircuitsPerPeer: 2, maxCircuitDurationSeconds: 7_200, maxCircuitBytes: 256 * 1024 * 1024 }, - watchReachability: async () => ({ generation: 0, listenAddresses: [], activeCoordinationRelays: [] }), + watchReachability: async () => ({ generation: 0, listenAddresses: [], activeCoordinationRelays: [] }), + watchConnectivity: async () => ({ generation: 0, connectedPeerIds: [] }), connect: async () => stream, connectMeshControl: async () => stream, configureTransit: async () => {}, + updateConnect: async () => true, cancelConnect: async () => true, accept: async () => null, acceptMeshControl: async () => null, @@ -392,6 +458,27 @@ function streamWith(chunk: Buffer): RuntimeHostPeerNativeStream { }; } +function signedReachability( + peerId: string, + directRoutes: readonly string[], + coordinationRoutes: readonly string[], + issuedAt = Date.now(), +) { + return { + lease: { + version: 1 as const, + peerId, + revision: 1, + issuedAt, + expiresAt: issuedAt + 60_000, + directRoutes, + coordinationRoutes, + }, + publicKey: Buffer.from('public').toString('base64url'), + signature: Buffer.from('signature').toString('base64url'), + }; +} + function peerConnectInput(peerId: string) { return { peerId, diff --git a/packages/runtime-host/src/__tests__/peer-reachability.test.ts b/packages/runtime-host/src/__tests__/peer-reachability.test.ts index 642a382480..fdf03eb812 100644 --- a/packages/runtime-host/src/__tests__/peer-reachability.test.ts +++ b/packages/runtime-host/src/__tests__/peer-reachability.test.ts @@ -27,7 +27,7 @@ import { isPeerReachabilityLeaseCurrent, peerReachabilityLeaseReceipt, verifySignedPeerReachabilityLease, - type PeerReachabilityIdentity, + type PeerReachabilityPeer, } from '../peer-reachability/model.js'; import { openPeerReachabilityPublisher } from '../peer-reachability/publisher.js'; @@ -208,7 +208,7 @@ test('publisher replaces future leases and renews them on monotonic time across } }); -class TestPeerIdentity implements PeerReachabilityIdentity { +class TestPeerIdentity implements PeerReachabilityPeer { readonly #publicKey: KeyObject; readonly #privateKey: KeyObject; listenAddresses: readonly string[]; @@ -223,10 +223,13 @@ class TestPeerIdentity implements PeerReachabilityIdentity { } identity() { + return { peerId: this.peerId }; + } + + reachability() { return { - peerId: this.peerId, listenAddresses: this.listenAddresses, - coordinationRelays: this.coordinationRelays, + activeCoordinationRelays: this.coordinationRelays, }; } diff --git a/packages/runtime-host/src/__tests__/reconnecting-connection.test.ts b/packages/runtime-host/src/__tests__/reconnecting-connection.test.ts index cb1699455e..29857ebee7 100644 --- a/packages/runtime-host/src/__tests__/reconnecting-connection.test.ts +++ b/packages/runtime-host/src/__tests__/reconnecting-connection.test.ts @@ -473,6 +473,151 @@ test('reconnect lifecycle suspension admits recovery while no connection exists' await lifecycle.close(); }); +test('new reachability evidence wakes an existing reconnect loop without starting another lifecycle', async () => { + const first = connectionHarness('first', () => undefined); + const replacement = connectionHarness('replacement', () => undefined); + const waitingForBackoff = deferred(); + let connectCalls = 0; + const lifecycle = await startRuntimeHostReconnectLifecycle({ + initial: first.connection, + connect: async () => { + connectCalls += 1; + if (connectCalls === 1) throw new Error('route is not reachable yet'); + return replacement.connection; + }, + backoff: { + minMs: 30_000, + maxMs: 30_000, + wait: (_delayMs, signal) => + new Promise((_resolve, reject) => { + waitingForBackoff.resolve(); + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }), + }, + }); + + first.disconnect(); + await waitingForBackoff.promise; + assert.equal(connectCalls, 1); + + lifecycle.wake(); + assert.equal(await lifecycle.waitForCurrent(), replacement.connection); + assert.equal(connectCalls, 2); + await lifecycle.close(); +}); + +test('an initial transient failure keeps a wakeable reconnect lifecycle', async () => { + const replacement = connectionHarness('replacement', () => undefined); + const waitingForBackoff = deferred(); + const failures: Error[] = []; + let connectCalls = 0; + const lifecycle = await startRuntimeHostReconnectLifecycle({ + retryInitialFailure: true, + connect: async () => { + connectCalls += 1; + if (connectCalls <= 2) throw new Error('the peer is not reachable yet'); + return replacement.connection; + }, + onReconnectError: (error) => failures.push(error), + backoff: { + minMs: 30_000, + maxMs: 30_000, + wait: (_delayMs, signal) => + new Promise((_resolve, reject) => { + waitingForBackoff.resolve(); + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }), + }, + }); + try { + assert.equal(lifecycle.current, undefined); + assert.equal(connectCalls, 2); + assert.match(failures[0]?.message ?? '', /not reachable yet/u); + await waitingForBackoff.promise; + + lifecycle.wake(); + assert.equal(await lifecycle.waitForCurrent(), replacement.connection); + assert.equal(connectCalls, 3); + } finally { + await lifecycle.close(); + } +}); + +test('initial retry mode does not outlive caller cancellation', async () => { + const controller = new AbortController(); + const connecting = deferred(); + const cancelled = new Error('initial connection cancelled'); + let connectCalls = 0; + const starting = startRuntimeHostReconnectLifecycle({ + retryInitialFailure: true, + initialSignal: controller.signal, + connect: async (signal): Promise => { + connectCalls += 1; + connecting.resolve(); + return new Promise((_resolve, reject) => { + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }); + }, + backoff: { minMs: 0, maxMs: 0 }, + }); + + await connecting.promise; + controller.abort(cancelled); + await assert.rejects(starting, (error: unknown) => error === cancelled); + await yieldToEventLoop(); + assert.equal(connectCalls, 1); +}); + +test('reachability discovered during a failed attempt skips the next reconnect delay', async () => { + const first = connectionHarness('first', () => undefined); + const replacement = connectionHarness('replacement', () => undefined); + const attemptStarted = deferredValue(); + const failAttempt = deferredValue(); + let connectCalls = 0; + let delayCalls = 0; + const lifecycle = await startRuntimeHostReconnectLifecycle({ + initial: first.connection, + connect: async () => { + connectCalls += 1; + if (connectCalls === 1) { + attemptStarted.resolve(undefined); + await failAttempt.promise; + throw new Error('the first route stopped working'); + } + return replacement.connection; + }, + backoff: { + minMs: 30_000, + maxMs: 30_000, + wait: (_delayMs, signal) => + new Promise((_resolve, reject) => { + delayCalls += 1; + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }), + }, + }); + try { + first.disconnect(); + await attemptStarted.promise; + lifecycle.wake(); + failAttempt.resolve(undefined); + + await waitForCondition(() => connectCalls === 2); + assert.equal(await lifecycle.waitForCurrent(), replacement.connection); + assert.equal(delayCalls, 0); + } finally { + await lifecycle.close(); + } +}); + test('reconnect delay escalates past maxMs while the Host never stabilizes', async () => { const first = connectionHarness('first', () => undefined); const delays: number[] = []; diff --git a/packages/runtime-host/src/client/host-profile.ts b/packages/runtime-host/src/client/host-profile.ts index c6cf66666a..632cfdc428 100644 --- a/packages/runtime-host/src/client/host-profile.ts +++ b/packages/runtime-host/src/client/host-profile.ts @@ -47,7 +47,11 @@ import { writeRuntimeHostPeerAuthentication, } from '../transport/peer-native.js'; import type { RuntimeHostPeerClient, RuntimeHostPeerConnectionPhase } from './peer-client.js'; -import { verifySignedPeerReachabilityLease } from '../peer-reachability/model.js'; +import { + decodeSignedPeerReachabilityLease, + isPeerReachabilityLeaseRecoverable, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/model.js'; import { RuntimeHostPermanentReconnectError } from './reconnect-lifecycle.js'; import { RuntimeHostRemoteCompatibilityError } from './remote-compatibility-error.js'; import { @@ -64,15 +68,12 @@ import { type RuntimeHostWslProcessFactory, } from './wsl-environment.js'; -const PROFILE_SCHEMA_VERSION = 3; +const PROFILE_SCHEMA_VERSION = 4; const CLIENT_PROFILE_DOCUMENT_NAME = 'runtime-host-profiles.json'; const PROFILE_DOCUMENT_MAX_BYTES = 64 * 1024; const PROFILE_COUNT_MAX = 32; const PROFILE_NAME_MAX_BYTES = 128; const PROFILE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; -const PEER_ID_MAX_BYTES = 160; -const PEER_ADDRESS_MAX_BYTES = 2 * 1024; -const PEER_ROUTE_MAX = 16; const DEFAULT_PEER_HANDSHAKE_TIMEOUT_MS = 5_000; const PROFILE_CREDENTIAL_RECORD_PREFIX = 'maka-runtime-host-profile-credential-v1:'; const PROFILE_INCARNATION_ID_MAX_BYTES = 128; @@ -149,9 +150,7 @@ export type RuntimeHostRemoteTransport = } | { readonly kind: 'libp2p-direct'; - readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; + readonly reachability: SignedPeerReachabilityLeaseV1; }; export interface RuntimeHostProfileDocument { @@ -276,8 +275,9 @@ export class RuntimeHostProfileConnectionError extends RuntimeHostPermanentRecon constructor( readonly reason: RuntimeHostProfileConnectionFailureReason, message: string, + options?: ErrorOptions, ) { - super(message); + super(message, options); this.name = 'RuntimeHostProfileConnectionError'; } } @@ -608,17 +608,52 @@ export async function connectPeerRuntimeHost(input: { }): Promise { input.signal?.throwIfAborted(); const handshakeTimeoutMs = input.handshakeTimeoutMs ?? DEFAULT_PEER_HANDSHAKE_TIMEOUT_MS; - const stream = await input.peerClient.connect( - { - peerId: input.transport.peerId, - routeHints: input.transport.routeHints, - coordinationRelays: input.transport.coordinationRelays, - directDeadlineMs: Math.min(input.connectTimeoutMs ?? 40_000, 120_000), - ...(input.refreshPeerRoutes === undefined ? {} : { refreshRoutes: input.refreshPeerRoutes }), - }, - input.signal, - input.onConnectionPhase, - ); + const peerId = input.transport.reachability.lease.peerId; + let reachability: SignedPeerReachabilityLeaseV1; + try { + reachability = input.peerClient.observeAuthenticatedReachability({ + expectedPeerId: peerId, + value: input.transport.reachability, + allowHistorical: true, + }); + } catch (cause) { + throw new RuntimeHostProfileConnectionError( + 'target_mismatch', + `Runtime Host profile ${input.profileId} contains invalid peer reachability evidence`, + { cause }, + ); + } + const bootstrap = isPeerReachabilityLeaseRecoverable(reachability.lease, Date.now()) + ? reachability.lease + : undefined; + let stream: Awaited>; + try { + stream = await input.peerClient.connect( + { + peerId, + routeHints: bootstrap?.directRoutes ?? [], + coordinationRelays: bootstrap?.coordinationRoutes ?? [], + directDeadlineMs: Math.min(input.connectTimeoutMs ?? 40_000, 120_000), + ...(input.refreshPeerRoutes === undefined + ? {} + : { refreshRoutes: input.refreshPeerRoutes }), + }, + input.signal, + input.onConnectionPhase, + ); + } catch (cause) { + if (cause instanceof RuntimeHostPeerError && cause.code === 'peer_identity_mismatch') { + throw new RuntimeHostProfileConnectionError( + 'target_mismatch', + `Runtime Host profile ${input.profileId} resolved to a different peer identity`, + { cause }, + ); + } + if (cause instanceof RuntimeHostPeerError && cause.code === 'peer_native_unavailable') { + throw runtimeHostPeerUnavailableError(cause); + } + throw cause; + } const abort = () => stream.abort(); input.signal?.addEventListener('abort', abort, { once: true }); if (input.signal?.aborted) abort(); @@ -653,16 +688,9 @@ export async function connectPeerRuntimeHost(input: { onHostStatus: (status) => { const endpoint = status.peerEndpoint; if (endpoint) { - const verified = verifySignedPeerReachabilityLease({ + input.peerClient.observeAuthenticatedReachability({ value: endpoint, - expectedPeerId: input.transport.peerId, - now: Date.now(), - verifyIdentity: input.peerClient.verifyIdentity.bind(input.peerClient), - }); - input.peerClient.observeAuthenticatedRoutes({ - peerId: verified.lease.peerId, - routeHints: verified.lease.directRoutes, - coordinationRelays: verified.lease.coordinationRoutes, + expectedPeerId: peerId, }); } input.onHostStatus?.(status); @@ -708,9 +736,20 @@ function requireRuntimeHostPeerClient( peerClient: RuntimeHostPeerClient | undefined, ): RuntimeHostPeerClient { if (peerClient) return peerClient; - throw new RuntimeHostPeerError( - 'peer_native_unavailable', - 'Experimental direct peer requires a Client peer endpoint owner', + throw runtimeHostPeerUnavailableError( + new RuntimeHostPeerError( + 'peer_native_unavailable', + 'Experimental direct peer requires a Client peer endpoint owner', + ), + ); +} + +function runtimeHostPeerUnavailableError( + cause: RuntimeHostPeerError, +): RuntimeHostPermanentReconnectError { + return new RuntimeHostPermanentReconnectError( + 'Runtime Host peer networking is unavailable in this Maka build', + { cause }, ); } @@ -753,6 +792,7 @@ export function decodeRuntimeHostProfileDocument(value: unknown): RuntimeHostPro if ( record.schemaVersion !== 1 && record.schemaVersion !== 2 && + record.schemaVersion !== 3 && record.schemaVersion !== PROFILE_SCHEMA_VERSION ) { throw new Error('Runtime Host profile document has an unsupported schema'); @@ -772,11 +812,19 @@ export function decodeRuntimeHostProfileDocument(value: unknown): RuntimeHostPro throw new Error('Runtime Host profile schema 1 cannot contain activation'); } if ( - record.schemaVersion !== PROFILE_SCHEMA_VERSION && + (record.schemaVersion as number) < 3 && profiles.some((profile) => profile.kind === 'remote' && profile.access === 'session_guest') ) { throw new Error('Runtime Host profile schema 3 is required for restricted access'); } + if ( + (record.schemaVersion as number) < 4 && + profiles.some( + (profile) => profile.kind === 'remote' && profile.transport.kind === 'libp2p-direct', + ) + ) { + throw new Error('Runtime Host profile schema 4 is required for Direct peer reachability'); + } const ids = new Set(); for (const profile of profiles) { if (ids.has(profile.id)) throw new Error(`Duplicate Runtime Host profile: ${profile.id}`); @@ -1295,24 +1343,18 @@ export function decodeRuntimeHostRemoteTransport(value: unknown): RuntimeHostRem if (kind === 'libp2p-direct') { const record = requireExactRecord(value, 'Runtime Host direct peer transport', [ 'kind', - 'peerId', - 'routeHints', - 'coordinationRelays', + 'reachability', ]); - const peerId = requireBoundedToken(record.peerId, 'Runtime Host peer id', PEER_ID_MAX_BYTES); - const routeHints = requirePeerAddresses(record.routeHints, 'Runtime Host peer route hints'); - const coordinationRelays = requirePeerAddresses( - record.coordinationRelays, - 'Runtime Host coordination relays', - ); - if (routeHints.length === 0 && coordinationRelays.length === 0) { + const reachability = decodeSignedPeerReachabilityLease(record.reachability); + if ( + reachability.lease.directRoutes.length === 0 && + reachability.lease.coordinationRoutes.length === 0 + ) { throw new Error('Runtime Host direct peer transport requires at least one route'); } return Object.freeze({ kind: 'libp2p-direct', - peerId, - routeHints, - coordinationRelays, + reachability, }); } throw new Error('Runtime Host transport kind is invalid'); @@ -1479,29 +1521,8 @@ function transportCredentialBinding(transport: RuntimeHostRemoteTransport): stri ? `${transport.destination}\0${transport.sshPort ?? ''}\0activate\0${transport.activation.operatorPath}` : `${transport.destination}\0${transport.sshPort ?? ''}\0${transport.remotePort}\0${transport.websocketPath}`; case 'libp2p-direct': - return transport.peerId; - } -} - -function requirePeerAddresses(value: unknown, label: string): readonly string[] { - if (!Array.isArray(value) || value.length > PEER_ROUTE_MAX) { - throw new Error(`${label} must be an array with at most ${PEER_ROUTE_MAX} entries`); - } - const addresses = value.map((entry) => { - const address = requireString(entry, label); - if ( - !address.startsWith('/') || - Buffer.byteLength(address, 'utf8') > PEER_ADDRESS_MAX_BYTES || - /[\s\u0000-\u001f\u007f]/u.test(address) - ) { - throw new Error(`${label} contains an invalid multiaddr`); - } - return address; - }); - if (new Set(addresses).size !== addresses.length) { - throw new Error(`${label} contains duplicates`); + return transport.reachability.lease.peerId; } - return Object.freeze(addresses); } function requireBoundedToken(value: unknown, label: string, maxBytes: number): string { @@ -1646,16 +1667,20 @@ async function writeProfileDocument( document: RuntimeHostProfileDocument, ): Promise { const schemaVersion = document.profiles.some( - (profile) => profile.kind === 'remote' && profile.access === 'session_guest', + (profile) => profile.kind === 'remote' && profile.transport.kind === 'libp2p-direct', ) ? PROFILE_SCHEMA_VERSION : document.profiles.some( - (profile) => - profile.kind === 'environment' || - (profile.transport.kind === 'ssh' && profile.transport.activation !== undefined), + (profile) => profile.kind === 'remote' && profile.access === 'session_guest', ) - ? 2 - : 1; + ? 3 + : document.profiles.some( + (profile) => + profile.kind === 'environment' || + (profile.transport.kind === 'ssh' && profile.transport.activation !== undefined), + ) + ? 2 + : 1; const encoded = `${JSON.stringify({ ...document, schemaVersion }, null, 2)}\n`; if (Buffer.byteLength(encoded, 'utf8') > PROFILE_DOCUMENT_MAX_BYTES) { throw new Error('Runtime Host profile document exceeds its size limit'); diff --git a/packages/runtime-host/src/client/index.ts b/packages/runtime-host/src/client/index.ts index 9359eaccb3..bb5a98bfb9 100644 --- a/packages/runtime-host/src/client/index.ts +++ b/packages/runtime-host/src/client/index.ts @@ -181,6 +181,7 @@ export { ensureRuntimeHostPeerIdentity } from '../transport/peer-native.js'; export { createRuntimeHostPeerClient, createRuntimeHostPeerClientFromEnvironment, + RuntimeHostPeerReachabilityUnavailableError, type RuntimeHostPeerClient, type RuntimeHostPeerConnectInput, } from './peer-client.js'; diff --git a/packages/runtime-host/src/client/owner-connection-code.ts b/packages/runtime-host/src/client/owner-connection-code.ts index b8855c86bd..edc953a5a4 100644 --- a/packages/runtime-host/src/client/owner-connection-code.ts +++ b/packages/runtime-host/src/client/owner-connection-code.ts @@ -27,7 +27,7 @@ import { type RuntimeHostRemoteTransport, } from './host-profile.js'; -const PREFIX = 'maka-runtime-host:connect:v1:'; +const PREFIX = 'maka-runtime-host:connect:v2:'; const ENCODED_MAX_BYTES = 48 * 1024; export const REMOTE_DESKTOP_OWNER_ACCESS_POLICY = Object.freeze({ @@ -45,7 +45,7 @@ const boundedString = (maxBytes: number) => const nameSchema = boundedString(128); const payloadSchema = z .object({ - schemaVersion: z.literal(1), + schemaVersion: z.literal(2), name: boundedString(128), rootId: z.string().refine((value) => { try { @@ -92,9 +92,7 @@ export async function issueRuntimeHostOwnerConnectionCode( } const transport = requireDirectPeerTransport({ kind: 'libp2p-direct', - peerId: endpoint.lease.peerId, - routeHints: endpoint.lease.directRoutes, - coordinationRelays: endpoint.lease.coordinationRoutes, + reachability: endpoint, }); const prepared = await input.client.request('access.credential.prepare', { ...REMOTE_DESKTOP_OWNER_ACCESS_POLICY, @@ -113,7 +111,7 @@ export function encodeRuntimeHostOwnerConnectionCode( input: RuntimeHostOwnerConnectionCode, ): string { const transport = requireDirectPeerTransport(input.transport); - const payload = payloadSchema.parse({ schemaVersion: 1, ...input, transport }); + const payload = payloadSchema.parse({ schemaVersion: 2, ...input, transport }); const encoded = Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url'); if (Buffer.byteLength(encoded, 'utf8') > ENCODED_MAX_BYTES) { throw new RangeError('Runtime Host connection code is too large'); diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index 06ed0fb79d..715678b92b 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -17,6 +17,16 @@ * under the License. */ +import { + authenticateSignedPeerReachabilityLease, + isPeerReachabilityLeaseCurrent, + isPeerReachabilityLeaseRecoverable, + PEER_REACHABILITY_MAX_CLOCK_SKEW_MS, + peerReachabilityLeaseReceipt, + verifySignedPeerReachabilityLease, + type PeerReachabilityLeaseReceipt, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/index.js'; import { normalizePeerError, RuntimeHostPeerError, @@ -32,6 +42,9 @@ import { } from '../transport/peer-native.js'; import { RuntimeHostPermanentReconnectError } from './reconnect-lifecycle.js'; +// One Desktop endpoint can retain 32 Host profiles and 128 guest mounts. +const AUTHENTICATED_REACHABILITY_MAX_ENTRIES = 160; + export interface RuntimeHostPeerConnectInput { readonly peerId: string; readonly routeHints: readonly string[]; @@ -46,7 +59,7 @@ export type RuntimeHostPeerConnectionPhase = 'discovering' | 'connecting'; interface RuntimeHostPeerRouteCandidateSnapshot { readonly routeHints: readonly string[]; readonly coordinationRelays: readonly string[]; - readonly transitRelayPeerIds?: readonly string[]; + readonly transitRelayPeerIds: readonly string[]; } export type RuntimeHostPeerRouteResolution = @@ -56,7 +69,17 @@ export type RuntimeHostPeerRouteResolution = export interface RuntimeHostPeerRouteResolver { resolveRoutes(peerId: string): RuntimeHostPeerRouteResolution; - prepareRoutes?(peerId: string, signal: AbortSignal): Promise; + prepareRoutes(peerId: string, signal: AbortSignal): Promise; + subscribeRoutes(peerId: string, listener: () => void): () => void; +} + +export class RuntimeHostPeerReachabilityUnavailableError extends Error { + readonly code = 'peer_reachability_needs_repair'; + + constructor(peerId: string) { + super(`Peer ${peerId} has no usable or recoverable route`); + this.name = 'RuntimeHostPeerReachabilityUnavailableError'; + } } export interface RuntimeHostPeerClient { @@ -67,22 +90,23 @@ export interface RuntimeHostPeerClient { ): Promise; identity(): Readonly<{ peerId: string; - listenAddresses: readonly string[]; - coordinationRelays: readonly string[]; }>; signIdentity(payload: Buffer): Promise; verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean; + isConnected(peerId: string): boolean; transitSnapshot(): RuntimeHostPeerTransitSnapshot; configureTransit(input: { readonly allowedPeerIds: readonly string[]; readonly approvedRelayPeerIds: readonly string[]; readonly relayCandidates: readonly RuntimeHostPeerTransitRelayCandidate[]; }): Promise; - observeAuthenticatedRoutes(input: { - readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; - }): void; + attachRouteResolver(resolver: RuntimeHostPeerRouteResolver): () => void; + subscribeRoutes(peerId: string, listener: () => void): () => void; + observeAuthenticatedReachability(input: { + readonly expectedPeerId: string; + readonly value: unknown; + readonly allowHistorical?: boolean; + }): SignedPeerReachabilityLeaseV1; connect( input: RuntimeHostPeerConnectInput, signal?: AbortSignal, @@ -111,7 +135,6 @@ export function createRuntimeHostPeerClientFromEnvironment( readonly coordinationRelays?: readonly string[]; readonly automaticRelayDiscovery?: boolean; readonly webRtcStunUrls?: readonly string[]; - readonly routeResolver?: RuntimeHostPeerRouteResolver; } = {}, ): RuntimeHostPeerClient { const nativePath = environment.MAKA_RUNTIME_HOST_PEER_NATIVE_PATH; @@ -134,7 +157,6 @@ export function createRuntimeHostPeerClient(input: { readonly coordinationRelays?: readonly string[]; readonly automaticRelayDiscovery?: boolean; readonly webRtcStunUrls?: readonly string[]; - readonly routeResolver?: RuntimeHostPeerRouteResolver; }): RuntimeHostPeerClient { return new RuntimeHostPeerClientImpl(input); } @@ -148,17 +170,14 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { readonly #coordinationRelays: readonly string[] | undefined; readonly #automaticRelayDiscovery: boolean; readonly #webRtcStunUrls: readonly string[] | undefined; - readonly #routeResolver: RuntimeHostPeerRouteResolver | undefined; - readonly #authenticatedRoutes = new Map< - string, - Readonly<{ - routeHints: readonly string[]; - coordinationRelays: readonly string[]; - }> - >(); + #routeResolver: RuntimeHostPeerRouteResolver | undefined; + readonly #routeListeners = new Map void>>(); + readonly #routeResolverSubscriptions = new Map void>(); + readonly #authenticatedReachability = new Map(); #endpoint: RuntimeHostPeerNativeEndpoint | undefined; #draining: Promise | undefined; #meshDraining: Promise | undefined; + #connectivityDraining: Promise | undefined; #applicationConsumer: InboundConsumer | undefined; #meshConsumer: InboundConsumer | undefined; #terminalError: Error | undefined; @@ -176,7 +195,6 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { readonly coordinationRelays?: readonly string[]; readonly automaticRelayDiscovery?: boolean; readonly webRtcStunUrls?: readonly string[]; - readonly routeResolver?: RuntimeHostPeerRouteResolver; }) { this.#nativePath = input.nativePath; this.#keyPath = input.keyPath; @@ -187,20 +205,12 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { this.#automaticRelayDiscovery = input.automaticRelayDiscovery ?? false; this.#webRtcStunUrls = input.webRtcStunUrls === undefined ? undefined : [...input.webRtcStunUrls]; - this.#routeResolver = input.routeResolver; } identity(): Readonly<{ peerId: string; - listenAddresses: readonly string[]; - coordinationRelays: readonly string[]; }> { - const endpoint = this.reachability(); - return Object.freeze({ - peerId: this.#requireEndpoint().peerId, - listenAddresses: Object.freeze([...endpoint.listenAddresses]), - coordinationRelays: Object.freeze([...endpoint.activeCoordinationRelays]), - }); + return Object.freeze({ peerId: this.#requireEndpoint().peerId }); } reachability(): RuntimeHostPeerNativeReachabilitySnapshot { @@ -240,6 +250,10 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { }); } + isConnected(peerId: string): boolean { + return this.#endpoint?.connectivitySnapshot.connectedPeerIds.includes(peerId) ?? false; + } + transitSnapshot(): RuntimeHostPeerTransitSnapshot { return Object.freeze({ ...this.#requireEndpoint().transitSnapshot }); } @@ -256,19 +270,116 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { }); } - observeAuthenticatedRoutes(input: { - readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; - }): void { - if (input.routeHints.length === 0 && input.coordinationRelays.length === 0) return; - this.#authenticatedRoutes.set( - input.peerId, - Object.freeze({ - routeHints: Object.freeze([...input.routeHints]), - coordinationRelays: Object.freeze([...input.coordinationRelays]), - }), - ); + attachRouteResolver(resolver: RuntimeHostPeerRouteResolver): () => void { + if (this.#routeResolver && this.#routeResolver !== resolver) { + throw new Error('Runtime Host peer client already has a reachability resolver'); + } + if (this.#routeResolver === resolver) return () => undefined; + this.#routeResolver = resolver; + for (const peerId of this.#routeListeners.keys()) { + this.#subscribeResolver(peerId); + this.#notifyRouteChange(peerId); + } + let attached = true; + return () => { + if (!attached) return; + attached = false; + if (this.#routeResolver !== resolver) return; + this.#routeResolver = undefined; + for (const unsubscribe of this.#routeResolverSubscriptions.values()) unsubscribe(); + this.#routeResolverSubscriptions.clear(); + for (const peerId of this.#routeListeners.keys()) this.#notifyRouteChange(peerId); + }; + } + + subscribeRoutes(peerId: string, listener: () => void): () => void { + const listeners = this.#routeListeners.get(peerId) ?? new Set<() => void>(); + const first = listeners.size === 0; + listeners.add(listener); + this.#routeListeners.set(peerId, listeners); + if (first) this.#subscribeResolver(peerId); + let subscribed = true; + return () => { + if (!subscribed) return; + subscribed = false; + listeners.delete(listener); + if (listeners.size > 0) return; + this.#routeListeners.delete(peerId); + this.#routeResolverSubscriptions.get(peerId)?.(); + this.#routeResolverSubscriptions.delete(peerId); + }; + } + + observeAuthenticatedReachability(input: { + readonly expectedPeerId: string; + readonly value: unknown; + readonly allowHistorical?: boolean; + }): SignedPeerReachabilityLeaseV1 { + const now = Date.now(); + const identity = { + value: input.value, + expectedPeerId: input.expectedPeerId, + verifyIdentity: this.verifyIdentity.bind(this), + }; + const next = input.allowHistorical + ? authenticateSignedPeerReachabilityLease(identity) + : verifySignedPeerReachabilityLease({ ...identity, now }); + this.#pruneAuthenticatedReachability(now); + let current = this.#authenticatedReachability.get(input.expectedPeerId); + if (!isPeerReachabilityLeaseRecoverable(next.lease, now)) return current?.signed ?? next; + if (current && current.signed.lease.revision >= next.lease.revision) { + if ( + current.signed.lease.revision === next.lease.revision && + !sameReachability(current.signed, next) + ) { + throw new Error('Peer reachability revision contains conflicting signed facts'); + } + this.#rememberAuthenticatedReachability(input.expectedPeerId, current); + return current.signed; + } + this.#rememberAuthenticatedReachability(input.expectedPeerId, { + signed: next, + ...(next.lease.issuedAt <= now + PEER_REACHABILITY_MAX_CLOCK_SKEW_MS + ? { + receipt: peerReachabilityLeaseReceipt({ + signed: next, + wallNow: now, + monotonicNow: performance.now(), + }), + } + : {}), + }); + this.#notifyRouteChange(input.expectedPeerId); + return next; + } + + #pruneAuthenticatedReachability(now: number): void { + for (const [peerId, authenticated] of this.#authenticatedReachability) { + if (!isPeerReachabilityLeaseRecoverable(authenticated.signed.lease, now)) { + this.#authenticatedReachability.delete(peerId); + this.#notifyRouteChange(peerId); + } + } + } + + #rememberAuthenticatedReachability( + peerId: string, + authenticated: AuthenticatedReachability, + ): void { + this.#authenticatedReachability.delete(peerId); + this.#authenticatedReachability.set(peerId, authenticated); + while (this.#authenticatedReachability.size > AUTHENTICATED_REACHABILITY_MAX_ENTRIES) { + let unobservedPeerId: string | undefined; + for (const candidatePeerId of this.#authenticatedReachability.keys()) { + if (this.#routeListeners.has(candidatePeerId)) continue; + unobservedPeerId = candidatePeerId; + break; + } + const evictedPeerId = unobservedPeerId ?? this.#authenticatedReachability.keys().next().value; + if (evictedPeerId === undefined) break; + this.#authenticatedReachability.delete(evictedPeerId); + this.#notifyRouteChange(evictedPeerId); + } } async connect( @@ -276,9 +387,8 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { signal?: AbortSignal, onPhase?: (phase: RuntimeHostPeerConnectionPhase) => void, ): Promise { - if (input.refreshRoutes !== false) { + if (input.refreshRoutes !== false && this.#routeResolver) { notifyPhase(onPhase, 'discovering'); - await this.#prepareRoutes(input, signal); } notifyPhase(onPhase, 'connecting'); return this.#connect(input, signal, 'application'); @@ -288,11 +398,12 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { input: RuntimeHostPeerConnectInput, signal: AbortSignal | undefined, ): Promise { - if (!this.#routeResolver?.prepareRoutes) return; + const resolver = this.#routeResolver; + if (!resolver) return; const deadline = AbortSignal.timeout(Math.min(10_000, input.directDeadlineMs)); const operationSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; try { - await this.#routeResolver.prepareRoutes(input.peerId, operationSignal); + await resolver.prepareRoutes(input.peerId, operationSignal); } catch { // Route preparation enriches an invitation/profile with fresher Mesh // routes. It must not suppress explicit routes the caller already has. @@ -397,29 +508,69 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { ): Promise { signal?.throwIfAborted(); const endpoint = this.#requireEndpoint(); - const requestId = this.#allocateRequestId(); - const discovered = - kind === 'application' ? this.#routeResolver?.resolveRoutes(input.peerId) : undefined; - const authenticated = - kind === 'application' ? this.#authenticatedRoutes.get(input.peerId) : undefined; - const connection = endpoint[kind === 'application' ? 'connect' : 'connectMeshControl']({ - ...input, - routeHints: mergeAddresses(discovered?.routeHints ?? [], [ - ...(authenticated?.routeHints ?? []), - ...input.routeHints, - ]), - coordinationRelays: mergeAddresses(discovered?.coordinationRelays ?? [], [ - ...(authenticated?.coordinationRelays ?? []), - ...(input.coordinationRelays ?? []), - ]), - transitRelayPeerIds: mergeValues( - discovered?.transitRelayPeerIds ?? [], - input.transitRelayPeerIds, - 64, - ), - requestId, - }); let settled = false; + let preparing = Boolean( + kind === 'application' && input.refreshRoutes !== false && this.#routeResolver, + ); + const snapshot = () => this.#connectionResolution(input, kind, preparing); + let resolution = snapshot(); + if (resolution.state === 'exhausted') { + throw new RuntimeHostPeerReachabilityUnavailableError(input.peerId); + } + const requestId = this.#allocateRequestId(); + const attemptLifetime = new AbortController(); + let reachabilityFailure: RuntimeHostPeerReachabilityUnavailableError | undefined; + let updateTail = Promise.resolve(); + const update = () => { + if (settled) return; + const next = snapshot(); + if (sameConnectionResolution(resolution, next)) return; + const candidatesChanged = !sameCandidates(resolution, next); + resolution = next; + if (next.state === 'exhausted') { + reachabilityFailure ??= new RuntimeHostPeerReachabilityUnavailableError(input.peerId); + void cancelPeerConnect(endpoint, requestId, () => settled); + return; + } + if (!candidatesChanged) return; + const candidates = connectCandidates(next); + updateTail = updateTail.then( + () => updatePeerConnect(endpoint, requestId, candidates, () => settled), + () => updatePeerConnect(endpoint, requestId, candidates, () => settled), + ); + void updateTail.catch(() => undefined); + }; + const unsubscribe = + kind === 'application' ? this.subscribeRoutes(input.peerId, update) : undefined; + let connection: Promise; + try { + connection = endpoint[kind === 'application' ? 'connect' : 'connectMeshControl']({ + ...input, + ...connectCandidates(resolution), + requestId, + }); + } catch (error) { + settled = true; + attemptLifetime.abort(); + unsubscribe?.(); + throw normalizePeerError(error); + } + if (preparing) { + const prepared = this.#prepareRoutes( + input, + signal ? AbortSignal.any([signal, attemptLifetime.signal]) : attemptLifetime.signal, + ); + void prepared.then( + () => { + preparing = false; + update(); + }, + () => { + preparing = false; + update(); + }, + ); + } const cancel = () => { void cancelPeerConnect(endpoint, requestId, () => settled); }; @@ -427,6 +578,10 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { if (signal?.aborted) cancel(); try { const stream = await connection; + if (reachabilityFailure) { + stream.abort(); + throw reachabilityFailure; + } if (signal?.aborted) { stream.abort(); signal.throwIfAborted(); @@ -434,13 +589,86 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { return stream; } catch (error) { signal?.throwIfAborted(); + if (reachabilityFailure) throw reachabilityFailure; throw normalizePeerError(error); } finally { settled = true; + attemptLifetime.abort(); + unsubscribe?.(); signal?.removeEventListener('abort', cancel); } } + #connectionResolution( + input: RuntimeHostPeerConnectInput, + kind: 'application' | 'mesh-control', + preparing: boolean, + ): RuntimeHostPeerConnectResolution { + const discovered = + kind === 'application' ? this.#routeResolver?.resolveRoutes(input.peerId) : undefined; + const rememberedEntry = + kind === 'application' ? this.#authenticatedReachability.get(input.peerId) : undefined; + const remembered = rememberedEntry?.signed; + const authenticated = + remembered && isPeerReachabilityLeaseRecoverable(remembered.lease, Date.now()) + ? remembered + : undefined; + if (remembered && !authenticated) this.#authenticatedReachability.delete(input.peerId); + const currentAuthenticated = Boolean( + authenticated && + rememberedEntry && + isPeerReachabilityLeaseCurrent(authenticated, rememberedEntry.receipt, performance.now()), + ); + const candidates = { + routeHints: mergeAddresses(discovered?.routeHints ?? [], [ + ...(currentAuthenticated ? (authenticated?.lease.directRoutes ?? []) : []), + ...input.routeHints, + ...(!currentAuthenticated ? (authenticated?.lease.directRoutes ?? []) : []), + ]), + coordinationRelays: mergeAddresses(discovered?.coordinationRelays ?? [], [ + ...(currentAuthenticated ? (authenticated?.lease.coordinationRoutes ?? []) : []), + ...(input.coordinationRelays ?? []), + ...(!currentAuthenticated ? (authenticated?.lease.coordinationRoutes ?? []) : []), + ]), + transitRelayPeerIds: mergeValues( + discovered?.transitRelayPeerIds ?? [], + input.transitRelayPeerIds, + 64, + ), + }; + const connected = + this.#endpoint?.connectivitySnapshot.connectedPeerIds.includes(input.peerId) ?? false; + return Object.freeze({ + ...candidates, + state: + hasConnectionCandidates(candidates) || connected + ? 'available' + : preparing + ? 'recovering' + : (discovered?.state ?? 'exhausted'), + }); + } + + #subscribeResolver(peerId: string): void { + if (this.#routeResolverSubscriptions.has(peerId)) return; + const resolver = this.#routeResolver; + if (!resolver) return; + const unsubscribe = resolver.subscribeRoutes(peerId, () => { + this.#notifyRouteChange(peerId); + }); + this.#routeResolverSubscriptions.set(peerId, unsubscribe); + } + + #notifyRouteChange(peerId: string): void { + for (const listener of this.#routeListeners.get(peerId) ?? []) { + try { + listener(); + } catch { + // Reachability evidence cannot let one observer disrupt the others. + } + } + } + close(): Promise { this.#closeTask ??= this.#close(); return this.#closeTask; @@ -470,9 +698,30 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { this.#endpoint = endpoint; this.#draining = this.#drainInbound(endpoint); this.#meshDraining = this.#drainMeshInbound(endpoint); + this.#connectivityDraining = this.#drainConnectivity(endpoint); return endpoint; } + async #drainConnectivity(endpoint: RuntimeHostPeerNativeEndpoint): Promise { + let current = endpoint.connectivitySnapshot; + try { + while (!this.#closed) { + const next = await endpoint.watchConnectivity(current.generation, 300_000); + const previousPeers = new Set(current.connectedPeerIds); + const nextPeers = new Set(next.connectedPeerIds); + current = next; + for (const peerId of new Set([...previousPeers, ...nextPeers])) { + if (previousPeers.has(peerId) !== nextPeers.has(peerId)) this.#notifyRouteChange(peerId); + } + } + } catch (error) { + if (this.#closed) return; + this.#terminalError = error instanceof Error ? error : new Error(String(error)); + this.#finishConsumer('application', this.#terminalError); + this.#finishConsumer('mesh', this.#terminalError); + } + } + async #drainInbound(endpoint: RuntimeHostPeerNativeEndpoint): Promise { try { while (true) { @@ -527,6 +776,10 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { async #close(): Promise { this.#closed = true; + for (const unsubscribe of this.#routeResolverSubscriptions.values()) unsubscribe(); + this.#routeResolverSubscriptions.clear(); + this.#routeListeners.clear(); + this.#authenticatedReachability.clear(); const endpoint = this.#endpoint; this.#endpoint = undefined; if (!endpoint) return; @@ -538,7 +791,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { closeFailed = true; closeError = error; } - await Promise.all([this.#draining, this.#meshDraining]); + await Promise.all([this.#draining, this.#meshDraining, this.#connectivityDraining]); if (closeFailed) throw closeError; } @@ -555,6 +808,21 @@ interface InboundConsumer { readonly reject: (error: Error) => void; } +interface AuthenticatedReachability { + readonly signed: SignedPeerReachabilityLeaseV1; + readonly receipt?: PeerReachabilityLeaseReceipt; +} + +interface RuntimeHostPeerConnectCandidates { + readonly routeHints: readonly string[]; + readonly coordinationRelays: readonly string[]; + readonly transitRelayPeerIds: readonly string[]; +} + +interface RuntimeHostPeerConnectResolution extends RuntimeHostPeerConnectCandidates { + readonly state: RuntimeHostPeerRouteResolution['state']; +} + function notifyPhase( observer: ((phase: RuntimeHostPeerConnectionPhase) => void) | undefined, phase: RuntimeHostPeerConnectionPhase, @@ -607,6 +875,62 @@ function mergeValues( return Object.freeze([...new Set([...primary, ...(secondary ?? [])])].slice(0, limit)); } +function sameCandidates( + left: RuntimeHostPeerConnectCandidates, + right: RuntimeHostPeerConnectCandidates, +): boolean { + return ( + sameValues(left.routeHints, right.routeHints) && + sameValues(left.coordinationRelays, right.coordinationRelays) && + sameValues(left.transitRelayPeerIds, right.transitRelayPeerIds) + ); +} + +function sameConnectionResolution( + left: RuntimeHostPeerConnectResolution, + right: RuntimeHostPeerConnectResolution, +): boolean { + return left.state === right.state && sameCandidates(left, right); +} + +function hasConnectionCandidates(candidates: RuntimeHostPeerConnectCandidates): boolean { + return ( + candidates.routeHints.length > 0 || + candidates.coordinationRelays.length > 0 || + candidates.transitRelayPeerIds.length > 0 + ); +} + +function connectCandidates( + resolution: RuntimeHostPeerConnectResolution, +): RuntimeHostPeerConnectCandidates { + return { + routeHints: resolution.routeHints, + coordinationRelays: resolution.coordinationRelays, + transitRelayPeerIds: resolution.transitRelayPeerIds, + }; +} + +function sameValues(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function sameReachability( + left: SignedPeerReachabilityLeaseV1, + right: SignedPeerReachabilityLeaseV1, +): boolean { + return ( + left.publicKey === right.publicKey && + left.signature === right.signature && + left.lease.peerId === right.lease.peerId && + left.lease.revision === right.lease.revision && + left.lease.issuedAt === right.lease.issuedAt && + left.lease.expiresAt === right.lease.expiresAt && + sameValues(left.lease.directRoutes, right.lease.directRoutes) && + sameValues(left.lease.coordinationRoutes, right.lease.coordinationRoutes) + ); +} + async function cancelPeerConnect( endpoint: RuntimeHostPeerNativeEndpoint, requestId: number, @@ -621,3 +945,19 @@ async function cancelPeerConnect( // The endpoint closing also settles the connect promise. } } + +async function updatePeerConnect( + endpoint: RuntimeHostPeerNativeEndpoint, + requestId: number, + candidates: RuntimeHostPeerConnectCandidates, + isSettled: () => boolean, +): Promise { + try { + while (!isSettled() && !(await endpoint.updateConnect({ requestId, ...candidates }))) { + // N-API schedules connect and updates independently. Retry until the + // engine has observed the request or the connect promise settles. + } + } catch { + // The active connection owns failures; route enrichment is best effort. + } +} diff --git a/packages/runtime-host/src/client/reconnect-lifecycle.ts b/packages/runtime-host/src/client/reconnect-lifecycle.ts index 76fff52165..eedbf6f6c7 100644 --- a/packages/runtime-host/src/client/reconnect-lifecycle.ts +++ b/packages/runtime-host/src/client/reconnect-lifecycle.ts @@ -49,6 +49,7 @@ export interface RuntimeHostReconnectLifecycle; subscribe(listener: (current: T | undefined) => void): () => void; + wake(): void; suspend(): Promise>; quiesce(): Promise>; close(): Promise; @@ -76,6 +77,8 @@ export async function startRuntimeHostReconnectLifecycle< >(input: { readonly initial?: T; readonly connect: (signal: AbortSignal) => Promise; + readonly retryInitialFailure?: boolean; + readonly initialSignal?: AbortSignal; readonly onReconnectError?: (error: Error) => void; readonly onFatalError?: (error: Error) => void; readonly backoff?: RuntimeHostReconnectBackoff; @@ -98,6 +101,8 @@ class RuntimeHostReconnectLifecycleImpl readonly closed: Promise; readonly #initial: T | undefined; readonly #connect: (signal: AbortSignal) => Promise; + readonly #retryInitialFailure: boolean; + readonly #initialSignal: AbortSignal | undefined; readonly #onReconnectError: ((error: Error) => void) | undefined; readonly #onFatalError: ((error: Error) => void) | undefined; readonly #minMs: number; @@ -121,16 +126,22 @@ class RuntimeHostReconnectLifecycleImpl #discardTask: Promise = Promise.resolve(); #closeTask: Promise | undefined; #resolveClosed!: () => void; + #wakeDelay: (() => void) | undefined; + #wakeGeneration = 0; constructor(input: { readonly initial?: T; readonly connect: (signal: AbortSignal) => Promise; + readonly retryInitialFailure?: boolean; + readonly initialSignal?: AbortSignal; readonly onReconnectError?: (error: Error) => void; readonly onFatalError?: (error: Error) => void; readonly backoff?: RuntimeHostReconnectBackoff; }) { this.#connect = input.connect; this.#initial = input.initial; + this.#retryInitialFailure = input.retryInitialFailure ?? false; + this.#initialSignal = input.initialSignal; this.#onReconnectError = input.onReconnectError; this.#onFatalError = input.onFatalError; this.#minMs = requireDelay(input.backoff?.minMs ?? DEFAULT_BACKOFF_MIN_MS, 'minMs'); @@ -162,9 +173,30 @@ class RuntimeHostReconnectLifecycleImpl async start(): Promise { try { - this.#install(this.#initial ?? (await this.#connect(this.#abort.signal))); + if (this.#initial) { + this.#install(this.#initial); + return; + } + this.#initialSignal?.throwIfAborted(); + const signal = this.#initialSignal + ? AbortSignal.any([this.#abort.signal, this.#initialSignal]) + : this.#abort.signal; + this.#install(await this.#connect(signal)); } catch (error) { - this.#failPermanently(asError(error)); + const failure = asError(error); + if ( + this.#retryInitialFailure && + !this.#closed && + !this.#abort.signal.aborted && + !this.#initialSignal?.aborted && + !(failure instanceof RuntimeHostPermanentReconnectError) + ) { + this.#failureCount += 1; + notifyError(this.#onReconnectError, failure); + this.#scheduleReconnect(); + return; + } + this.#failPermanently(failure); throw error; } } @@ -197,6 +229,13 @@ class RuntimeHostReconnectLifecycleImpl return () => this.#listeners.delete(listener); } + wake(): void { + this.#wakeGeneration += 1; + const wake = this.#wakeDelay; + this.#wakeDelay = undefined; + wake?.(); + } + async suspend(): Promise> { if (this.#closed || this.#terminalError) { throw new Error('Runtime Host reconnect lifecycle is closed'); @@ -301,6 +340,7 @@ class RuntimeHostReconnectLifecycleImpl } async #reconnect(signal: AbortSignal): Promise { + let attemptedWakeGeneration = this.#wakeGeneration; while (!this.#closed && !this.#quiesced && !this.#terminalError && !this.#current) { const delayMs = reconnectDelayMs( this.#failureCount - 1, @@ -309,8 +349,12 @@ class RuntimeHostReconnectLifecycleImpl this.#random, this.#unstableMaxMs, ); + const wakeGeneration = this.#wakeGeneration; try { - if (delayMs > 0) await this.#wait(delayMs, signal); + if (delayMs > 0 && wakeGeneration === attemptedWakeGeneration) { + await this.#waitForReconnectDelay(delayMs, signal, wakeGeneration); + } + attemptedWakeGeneration = this.#wakeGeneration; const resource = await this.#connect(signal); this.#install(resource); } catch (error) { @@ -326,6 +370,28 @@ class RuntimeHostReconnectLifecycleImpl } } + async #waitForReconnectDelay( + delayMs: number, + signal: AbortSignal, + observedWakeGeneration: number, + ): Promise { + if (this.#wakeGeneration !== observedWakeGeneration) return; + let wake!: () => void; + const routeAvailable = new Promise((resolve) => { + wake = resolve; + }); + const delayAbort = new AbortController(); + const delaySignal = AbortSignal.any([signal, delayAbort.signal]); + this.#wakeDelay = wake; + if (this.#wakeGeneration !== observedWakeGeneration) wake(); + try { + await Promise.race([this.#wait(delayMs, delaySignal), routeAvailable]); + } finally { + if (this.#wakeDelay === wake) this.#wakeDelay = undefined; + delayAbort.abort(); + } + } + #suspension(current: T | undefined): RuntimeHostReconnectSuspension { let active = true; return { diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 89ce4eae96..1aeca858ab 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -58,6 +58,7 @@ import { PEER_REACHABILITY_MAX_CLOCK_SKEW_MS, peerReachabilityLeaseSigningBytes, peerReachabilityLeaseReceipt, + PEER_REACHABILITY_RECOVERY_HORIZON_MS, verifySignedPeerReachabilityLease, type PeerReachabilityPublisher, type PeerReachabilityLeaseReceipt, @@ -82,7 +83,6 @@ const CONNECT_DEADLINE_MS = 30_000; const CONTROL_REQUEST_DEADLINE_MS = 10_000; const MAX_ACTIVE_CONTROL_STREAMS = 32; const MAX_ACTIVE_CONTROL_STREAMS_PER_PEER = 2; -const REACHABILITY_HISTORY_MS = 24 * 60 * 60 * 1_000; const EVIDENCE_PAGE_SIZE = 2; const RECONCILE_CONCURRENCY = 4; const RECONCILE_DEADLINE_MS = 60 * 1_000; @@ -184,6 +184,7 @@ export interface PeerMeshNode { transitSnapshot(): RuntimeHostPeerTransitSnapshot; resolveRoutes(peerId: string): RuntimeHostPeerRouteResolution; prepareRoutes(peerId: string, signal: AbortSignal): Promise; + subscribeRoutes(peerId: string, listener: () => void): () => void; reconcile(signal?: AbortSignal): Promise; serve(): Promise; close(): Promise; @@ -208,11 +209,10 @@ export interface PeerMeshMemberRouteStatus { export interface PeerMeshTransport { identity(): Readonly<{ peerId: string; - listenAddresses: readonly string[]; - coordinationRelays: readonly string[]; }>; signIdentity(payload: Buffer): Promise; verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean; + isConnected(peerId: string): boolean; transitSnapshot(): RuntimeHostPeerTransitSnapshot; configureTransit(input: { readonly allowedPeerIds: readonly string[]; @@ -274,9 +274,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { #unsubscribeReachability: (() => void) | undefined; #reconcileGeneration = 0; readonly #reconcileWaiters = new Set<() => void>(); - readonly #recentlyReached = new Set(); readonly #reachabilityReceipts = new Map(); readonly #completedRecoverySweeps = new Set(); + readonly #routeResolutionListeners = new Map void>>(); #serveTask: Promise | undefined; #closeTask: Promise | undefined; @@ -407,7 +407,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, stored.advertisements, this.#now(), - this.#recentlyReached, + (peerId) => this.#peer.isConnected(peerId), (peerId) => this.resolveRoutes(peerId), (signed) => this.#isReachabilityCurrent(signed), ); @@ -429,7 +429,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, stored.advertisements, this.#now(), - this.#recentlyReached, + (peerId) => this.#peer.isConnected(peerId), (peerId) => this.resolveRoutes(peerId), (signed) => this.#isReachabilityCurrent(signed), ), @@ -480,7 +480,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, stored.advertisements, now, - this.#recentlyReached, + (peerId) => this.#peer.isConnected(peerId), (peerId) => this.resolveRoutes(peerId), (signed) => this.#isReachabilityCurrent(signed), ); @@ -777,7 +777,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, stored.advertisements, this.#now(), - this.#recentlyReached, + (peerId) => this.#peer.isConnected(peerId), (peerId) => this.resolveRoutes(peerId), (signed) => this.#isReachabilityCurrent(signed), ); @@ -992,7 +992,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { async prepareRoutes(peerId: string, signal: AbortSignal): Promise { this.#assertOpen(); signal.throwIfAborted(); - this.#completedRecoverySweeps.delete(peerId); + this.#setRecoverySweepCompleted(peerId, false); const localPeerId = this.#peer.identity().peerId; const stored = this.#store.read(); const visible = this.#pruneCompletedRecoverySweeps(stored, localPeerId).has(peerId); @@ -1002,9 +1002,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { // asks the Mesh control plane for its newest record. Callers with a // self-contained invitation run this reconciliation in parallel with the // first dial; callers without usable routes wait for it. - await this.reconcile(signal); - if (this.#pruneCompletedRecoverySweeps().has(peerId)) { - this.#completedRecoverySweeps.add(peerId); + try { + await this.#queueReconcile(signal, peerId); + } finally { + if (!signal.aborted && !this.#lifetime.signal.aborted) { + this.#setRecoverySweepCompleted(peerId, true); + } } } @@ -1025,7 +1028,46 @@ class PeerMeshNodeImpl implements PeerMeshNode { return visiblePeerIds; } + subscribeRoutes(peerId: string, listener: () => void): () => void { + this.#assertOpen(); + let current = this.resolveRoutes(peerId); + const observe = () => { + const next = this.resolveRoutes(peerId); + if (sameResolvedRoutes(current, next)) return; + current = next; + try { + listener(); + } catch { + // Route observers cannot control Mesh reconciliation. + } + }; + const listeners = this.#routeResolutionListeners.get(peerId) ?? new Set<() => void>(); + listeners.add(observe); + this.#routeResolutionListeners.set(peerId, listeners); + const unsubscribeStore = this.#store.subscribe(observe); + return () => { + unsubscribeStore(); + listeners.delete(observe); + if (listeners.size === 0) this.#routeResolutionListeners.delete(peerId); + }; + } + + #setRecoverySweepCompleted(peerId: string, completed: boolean): void { + const retain = completed && this.#pruneCompletedRecoverySweeps().has(peerId); + const changed = retain + ? !this.#completedRecoverySweeps.has(peerId) + : this.#completedRecoverySweeps.has(peerId); + if (!changed) return; + if (retain) this.#completedRecoverySweeps.add(peerId); + else this.#completedRecoverySweeps.delete(peerId); + for (const listener of this.#routeResolutionListeners.get(peerId) ?? []) listener(); + } + reconcile(signal?: AbortSignal): Promise { + return this.#queueReconcile(signal); + } + + #queueReconcile(signal?: AbortSignal, excludedPeerId?: string): Promise { if (this.#lifetime.signal.aborted) return Promise.reject(new Error('Peer Mesh node is closed')); const previous = this.#reconcileTail; let release!: () => void; @@ -1034,7 +1076,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { }); this.#reconcileTail = previous.then(() => turn); return waitForTurn(previous, signal) - .then(() => this.#reconcile(signal)) + .then(() => this.#reconcile(signal, excludedPeerId)) .finally(release); } @@ -1078,6 +1120,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { await this.#serveTask?.catch(() => undefined); for (const stream of this.#activeControlStreams) stream.abort(); this.#activeControlStreams.clear(); + this.#routeResolutionListeners.clear(); await Promise.all([this.#admissionTail, this.#reconcileTail, this.#transitTail]); return this.#store.close(); } @@ -1127,7 +1170,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#reconcileWaiters.clear(); } - async #reconcile(signal?: AbortSignal): Promise { + async #reconcile(signal?: AbortSignal, excludedPeerId?: string): Promise { const lifetimeSignal = signal ? AbortSignal.any([signal, this.#lifetime.signal]) : this.#lifetime.signal; @@ -1149,7 +1192,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { readonly desiredMembership: 'active' | 'left'; readonly roster: SignedPeerMeshRosterV1; } - > = stored.pendingJoins.map((join) => ({ kind: 'join', join })); + > = stored.pendingJoins + .filter(({ invitation }) => invitation.reachability.lease.peerId !== excludedPeerId) + .map((join) => ({ kind: 'join', join })); const gossipCursor = this.#gossipCursor; this.#gossipCursor = (this.#gossipCursor + 1) % PEER_MESH_MAX_MEMBERS; const now = this.#now(); @@ -1160,7 +1205,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { state.role === 'replica' ? currentAuthorityTarget(state, stored.reachability, now) : undefined; - if (authority) { + if (authority && authority.lease.peerId !== excludedPeerId) { pending.push({ kind: 'membership', meshId, @@ -1174,6 +1219,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { .filter( (peerId) => peerId !== identity.peerId && + peerId !== excludedPeerId && (state.role === 'authority' || peerId !== state.roster.roster.authorityPeerId), ) .flatMap((peerId) => { @@ -1216,9 +1262,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { } } catch (error) { if (lifetimeSignal.aborted) lifetimeSignal.throwIfAborted(); - if (operation.kind === 'membership') { - this.#recentlyReached.delete(operation.target.lease.peerId); - } if (operation.kind === 'join' || operation.desiredMembership === 'left') { failures.push(error); } @@ -1264,7 +1307,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { throw new Error('Peer Mesh authority rejected the leave request'); } await this.#applySync(meshId, response.roster, [], []); - this.#recentlyReached.add(target.lease.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1336,7 +1378,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { response.reachability, response.advertisements, ); - this.#recentlyReached.add(targetPeerId); if (!response.more) return; } finally { await stream.close().catch(() => undefined); @@ -1459,7 +1500,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { verifyIdentity: this.#peer.verifyIdentity.bind(this.#peer), ...(allowExpired ? { allowExpired: true } : {}), }); - if (allowExpired && signed.lease.expiresAt + REACHABILITY_HISTORY_MS <= this.#now()) { + if ( + allowExpired && + signed.lease.expiresAt <= this.#now() - PEER_REACHABILITY_RECOVERY_HORIZON_MS + ) { throw new Error('Peer Mesh reachability is outside the recovery horizon'); } return signed; @@ -1686,7 +1730,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, stored.advertisements, this.#now(), - this.#recentlyReached, + (peerId) => this.#peer.isConnected(peerId), (peerId) => this.resolveRoutes(peerId), (signed) => this.#isReachabilityCurrent(signed), ); @@ -1721,7 +1765,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (response.kind === 'roster-rejected') { throw new Error('Peer Mesh roster announcement was rejected'); } - this.#recentlyReached.add(target.lease.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1969,7 +2012,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { ).filter(({ lease }) => lease.peerId !== remotePeerId), ); } - this.#recentlyReached.add(remotePeerId); } return response; } @@ -2115,7 +2157,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { response.roster.roster.members.includes(remotePeerId) ) { this.#recordReachabilityReceipt(remoteReachability); - this.#recentlyReached.add(remotePeerId); } await this.#reconcileTransit(); } @@ -2166,6 +2207,22 @@ interface PeerMeshTransitEvidence { readonly lease: SignedPeerReachabilityLeaseV1; } +function sameResolvedRoutes( + left: ReturnType, + right: ReturnType, +): boolean { + return ( + left.state === right.state && + sameStringValues(left.routeHints, right.routeHints) && + sameStringValues(left.coordinationRelays, right.coordinationRelays) && + sameStringValues(left.transitRelayPeerIds, right.transitRelayPeerIds) + ); +} + +function sameStringValues(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + function eligibleTransitEvidence( stored: PeerMeshStoredStateV1, localPeerId: string, @@ -2265,7 +2322,7 @@ function peerMeshStatus( reachability: readonly SignedPeerReachabilityLeaseV1[], advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], now: number, - recentlyReached: ReadonlySet, + isConnected: (peerId: string) => boolean, resolveRoutes: (peerId: string) => RuntimeHostPeerRouteResolution, isCurrent: (signed: SignedPeerReachabilityLeaseV1) => boolean, ): PeerMeshStatus { @@ -2298,18 +2355,17 @@ function peerMeshStatus( const lease = signed?.lease; const resolution = resolveRoutes(peerId); const current = Boolean(signed && isCurrent(signed)); - const memberState = - resolution.state === 'exhausted' + const memberState = isConnected(peerId) + ? ('reachable' as const) + : resolution.state === 'exhausted' ? ('needs_repair' as const) : resolution.state === 'recovering' ? signed ? ('reconnecting' as const) : ('connecting' as const) - : current && recentlyReached.has(peerId) - ? ('reachable' as const) - : current - ? ('connecting' as const) - : ('reconnecting' as const); + : current + ? ('connecting' as const) + : ('reconnecting' as const); return Object.freeze({ peerId, ...(advertisement?.endpointKind ? { endpointKind: advertisement.endpointKind } : {}), @@ -2719,7 +2775,7 @@ function hasReachabilityRoutes(signed: SignedPeerReachabilityLeaseV1): boolean { } function usableHistoricalReachability(signed: SignedPeerReachabilityLeaseV1, now: number): boolean { - return signed.lease.expiresAt + REACHABILITY_HISTORY_MS > now; + return signed.lease.expiresAt > now - PEER_REACHABILITY_RECOVERY_HORIZON_MS; } function peerTarget( diff --git a/packages/runtime-host/src/peer-mesh/owner.ts b/packages/runtime-host/src/peer-mesh/owner.ts index bc7be1494b..7de8796f74 100644 --- a/packages/runtime-host/src/peer-mesh/owner.ts +++ b/packages/runtime-host/src/peer-mesh/owner.ts @@ -49,21 +49,37 @@ export async function openRuntimeHostPeerMeshComponent( ? { onBackgroundReconcileError: input.onBackgroundReconcileError } : {}), }); - const serving = mesh.serve(); + let detachResolver: (() => void) | undefined; + let serving: Promise; + try { + detachResolver = input.endpoint.client.attachRouteResolver(mesh); + serving = mesh.serve(); + } catch (error) { + detachResolver?.(); + await mesh.close().catch(() => undefined); + throw error; + } let closeTask: Promise | undefined; const close = () => { - closeTask ??= closeMesh(mesh, serving); + closeTask ??= closeMesh(mesh, serving, detachResolver!); return closeTask; }; const closed = serving.then( - () => closeTask ?? stopUnexpectedMesh(mesh, new Error('Peer Mesh stopped unexpectedly')), - (error: unknown) => closeTask ?? stopUnexpectedMesh(mesh, error), + () => + closeTask ?? + stopUnexpectedMesh(mesh, detachResolver!, new Error('Peer Mesh stopped unexpectedly')), + (error: unknown) => closeTask ?? stopUnexpectedMesh(mesh, detachResolver!, error), ); void closed.catch(() => undefined); return Object.freeze({ mesh, closed, close }); } -async function stopUnexpectedMesh(mesh: PeerMeshNode, error: unknown): Promise { +async function stopUnexpectedMesh( + mesh: PeerMeshNode, + detachResolver: () => void, + error: unknown, +): Promise { + detachResolver(); try { await mesh.close(); } catch (closeError) { @@ -72,8 +88,13 @@ async function stopUnexpectedMesh(mesh: PeerMeshNode, error: unknown): Promise): Promise { +async function closeMesh( + mesh: PeerMeshNode, + serving: Promise, + detachResolver: () => void, +): Promise { const errors: unknown[] = []; + detachResolver(); await mesh.close().catch((error: unknown) => errors.push(error)); await serving.catch((error: unknown) => errors.push(error)); throwCollected(errors, 'Unable to close Peer Mesh'); diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index 8c3a1a1141..1ffee0fac6 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -95,6 +95,7 @@ export interface PeerMeshStoredStateV1 { export interface PeerMeshStateStore { readonly terminalFailure: Promise; read(): PeerMeshStoredStateV1; + subscribe(listener: () => void): () => void; mutate( operation: (state: PeerMeshStoredStateV1) => | { @@ -169,6 +170,7 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { #failure: Error | undefined; #closeTask: Promise | undefined; #closed = false; + readonly #listeners = new Set<() => void>(); constructor( dataRoot: string, @@ -191,6 +193,12 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { return this.#state; } + subscribe(listener: () => void): () => void { + this.#assertOpen(); + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + mutate( operation: (state: PeerMeshStoredStateV1) => | { @@ -213,6 +221,13 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { try { await writeState(this.#path, this.localPeerId, canonical); this.#state = canonical; + for (const listener of this.#listeners) { + try { + listener(); + } catch { + // Persisted state remains authoritative when an observer fails. + } + } } catch (error) { if (error instanceof PeerMeshPostCommitError) { this.#state = canonical; @@ -238,6 +253,7 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { async #close(): Promise { this.#closed = true; + this.#listeners.clear(); await this.#tail; await this.owner.close(); } diff --git a/packages/runtime-host/src/peer-reachability/index.ts b/packages/runtime-host/src/peer-reachability/index.ts index 20b7fb98c0..1c4432f185 100644 --- a/packages/runtime-host/src/peer-reachability/index.ts +++ b/packages/runtime-host/src/peer-reachability/index.ts @@ -26,13 +26,15 @@ export { PEER_REACHABILITY_MAX_LIFETIME_MS, PEER_REACHABILITY_MAX_RECORD_BYTES, PEER_REACHABILITY_MAX_ROUTES_PER_CLASS, + PEER_REACHABILITY_RECOVERY_HORIZON_MS, PEER_REACHABILITY_REFRESH_LEAD_MS, isPeerReachabilityLeaseCurrent, + isPeerReachabilityLeaseRecoverable, peerReachabilityLeaseReceipt, peerReachabilityLeaseSigningBytes, samePeerReachabilityRoutes, verifySignedPeerReachabilityLease, - type PeerReachabilityIdentity, + type PeerReachabilityPeer, type PeerReachabilityLeaseReceipt, type PeerReachabilityLeaseV1, type SignedPeerReachabilityLeaseV1, diff --git a/packages/runtime-host/src/peer-reachability/model.ts b/packages/runtime-host/src/peer-reachability/model.ts index 08520cf955..6395069e24 100644 --- a/packages/runtime-host/src/peer-reachability/model.ts +++ b/packages/runtime-host/src/peer-reachability/model.ts @@ -23,6 +23,7 @@ export const PEER_REACHABILITY_LEASE_TTL_MS = 5 * 60 * 1_000; export const PEER_REACHABILITY_REFRESH_LEAD_MS = 60 * 1_000; export const PEER_REACHABILITY_MAX_LIFETIME_MS = 10 * 60 * 1_000; export const PEER_REACHABILITY_MAX_CLOCK_SKEW_MS = 2 * 60 * 1_000; +export const PEER_REACHABILITY_RECOVERY_HORIZON_MS = 24 * 60 * 60 * 1_000; export const PEER_REACHABILITY_MAX_ROUTES_PER_CLASS = 16; export const PEER_REACHABILITY_MAX_RECORD_BYTES = 48 * 1_024; @@ -53,11 +54,13 @@ export interface PeerReachabilityLeaseReceipt { readonly currentUntil: number; } -export interface PeerReachabilityIdentity { +export interface PeerReachabilityPeer { identity(): Readonly<{ peerId: string; + }>; + reachability(): Readonly<{ listenAddresses: readonly string[]; - coordinationRelays: readonly string[]; + activeCoordinationRelays: readonly string[]; }>; signIdentity(payload: Buffer): Promise; verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean; @@ -115,7 +118,7 @@ export function verifySignedPeerReachabilityLease(input: { readonly value: unknown; readonly expectedPeerId: string; readonly now: number; - readonly verifyIdentity: PeerReachabilityIdentity['verifyIdentity']; + readonly verifyIdentity: PeerReachabilityPeer['verifyIdentity']; readonly allowExpired?: boolean; }): SignedPeerReachabilityLeaseV1 { const signed = authenticateSignedPeerReachabilityLease(input); @@ -131,7 +134,7 @@ export function verifySignedPeerReachabilityLease(input: { export function authenticateSignedPeerReachabilityLease(input: { readonly value: unknown; readonly expectedPeerId: string; - readonly verifyIdentity: PeerReachabilityIdentity['verifyIdentity']; + readonly verifyIdentity: PeerReachabilityPeer['verifyIdentity']; }): SignedPeerReachabilityLeaseV1 { const signed = decodeSignedPeerReachabilityLease(input.value); if (signed.lease.peerId !== input.expectedPeerId) { @@ -164,11 +167,11 @@ export function peerReachabilityLeaseSigningBytes(lease: PeerReachabilityLeaseV1 export function samePeerReachabilityRoutes( lease: PeerReachabilityLeaseV1, - identity: ReturnType, + reachability: ReturnType, ): boolean { return ( - sameStrings(lease.directRoutes, identity.listenAddresses) && - sameStrings(lease.coordinationRoutes, identity.coordinationRelays) + sameStrings(lease.directRoutes, reachability.listenAddresses) && + sameStrings(lease.coordinationRoutes, reachability.activeCoordinationRelays) ); } @@ -213,6 +216,13 @@ export function isPeerReachabilityLeaseCurrent( ); } +export function isPeerReachabilityLeaseRecoverable( + lease: PeerReachabilityLeaseV1, + now: number, +): boolean { + return lease.expiresAt > now - PEER_REACHABILITY_RECOVERY_HORIZON_MS; +} + function exactRecord( value: unknown, label: string, diff --git a/packages/runtime-host/src/peer-reachability/owner.ts b/packages/runtime-host/src/peer-reachability/owner.ts index f1ce0beaaa..a738d06e36 100644 --- a/packages/runtime-host/src/peer-reachability/owner.ts +++ b/packages/runtime-host/src/peer-reachability/owner.ts @@ -24,11 +24,7 @@ import { acquireFileLifetimeOwner, type FileLifetimeOwner, } from '@maka/storage/file-lifetime-owner'; -import { - createRuntimeHostPeerClient, - type RuntimeHostPeerClient, - type RuntimeHostPeerRouteResolver, -} from '../client/peer-client.js'; +import { createRuntimeHostPeerClient, type RuntimeHostPeerClient } from '../client/peer-client.js'; import { RuntimeHostPermanentReconnectError } from '../client/reconnect-lifecycle.js'; import { openPeerReachabilityPublisher, @@ -55,7 +51,6 @@ export async function openRuntimeHostPeerEndpointOwner(input: { readonly coordinationRelays?: readonly string[]; readonly automaticRelayDiscovery?: boolean; readonly webRtcStunUrls?: readonly string[]; - readonly routeResolver?: RuntimeHostPeerRouteResolver; readonly onBackgroundReachabilityError?: (error: unknown) => void; }): Promise { await mkdir(input.dataRoot, { recursive: true, mode: 0o700 }); @@ -75,7 +70,6 @@ export async function openRuntimeHostPeerEndpointOwner(input: { ? {} : { automaticRelayDiscovery: input.automaticRelayDiscovery }), ...(input.webRtcStunUrls === undefined ? {} : { webRtcStunUrls: input.webRtcStunUrls }), - ...(input.routeResolver ? { routeResolver: input.routeResolver } : {}), }); const peerId = client.identity().peerId; reachability = await openPeerReachabilityPublisher({ diff --git a/packages/runtime-host/src/peer-reachability/publisher.ts b/packages/runtime-host/src/peer-reachability/publisher.ts index c0d49f4ed5..101c7f3fc3 100644 --- a/packages/runtime-host/src/peer-reachability/publisher.ts +++ b/packages/runtime-host/src/peer-reachability/publisher.ts @@ -31,7 +31,7 @@ import { peerReachabilityLeaseSigningBytes, samePeerReachabilityRoutes, verifySignedPeerReachabilityLease, - type PeerReachabilityIdentity, + type PeerReachabilityPeer, type PeerReachabilityLeaseReceipt, type SignedPeerReachabilityLeaseV1, } from './model.js'; @@ -48,7 +48,7 @@ export interface PeerReachabilityPublisher { export async function openPeerReachabilityPublisher(input: { readonly dataRoot: string; - readonly peer: PeerReachabilityIdentity; + readonly peer: PeerReachabilityPeer; readonly now?: () => number; readonly monotonicNow?: () => number; }): Promise { @@ -79,7 +79,7 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { constructor( private readonly path: string, - private readonly peer: PeerReachabilityIdentity, + private readonly peer: PeerReachabilityPeer, private readonly now: () => number, private readonly monotonicNow: () => number, current: SignedPeerReachabilityLeaseV1 | undefined, @@ -111,6 +111,7 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { const task = this.#tail.then(async () => { this.#assertOpen(); const identity = this.peer.identity(); + const reachability = this.peer.reachability(); const now = this.now(); const monotonicNow = this.monotonicNow(); if ( @@ -123,7 +124,7 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { this.#receipt, monotonicNow + PEER_REACHABILITY_REFRESH_LEAD_MS, ) && - samePeerReachabilityRoutes(this.#current.lease, identity) + samePeerReachabilityRoutes(this.#current.lease, reachability) ) { return this.#current; } @@ -134,8 +135,8 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { revision, issuedAt: now, expiresAt: now + PEER_REACHABILITY_LEASE_TTL_MS, - directRoutes: identity.listenAddresses, - coordinationRoutes: identity.coordinationRelays, + directRoutes: reachability.listenAddresses, + coordinationRoutes: reachability.activeCoordinationRelays, }); const identityProof = await this.peer.signIdentity(peerReachabilityLeaseSigningBytes(lease)); const signed = decodeSignedPeerReachabilityLease({ @@ -205,7 +206,7 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { async function readState( path: string, - peer: PeerReachabilityIdentity, + peer: PeerReachabilityPeer, ): Promise { try { const stat = await lstat(path); diff --git a/packages/runtime-host/src/server/listener-set.ts b/packages/runtime-host/src/server/listener-set.ts index 766fcdcb03..5610e3ecea 100644 --- a/packages/runtime-host/src/server/listener-set.ts +++ b/packages/runtime-host/src/server/listener-set.ts @@ -45,13 +45,10 @@ export interface RuntimeHostListener { export interface RuntimeHostPeerListener extends RuntimeHostListener { readonly kind: 'libp2p_direct'; - readonly peerId: string; - readonly listenAddresses: readonly string[]; readonly reachability: SignedPeerReachabilityLeaseV1; } export interface RuntimeHostPeerListenerDescriptor { - readonly peerId: string; readonly reachability: SignedPeerReachabilityLeaseV1; } @@ -132,7 +129,6 @@ export function createRuntimeHostListenerSet( const peerListeners = Object.freeze( additional.filter(isRuntimeHostPeerListener).map((listener) => Object.freeze({ - peerId: listener.peerId, get reachability() { return listener.reachability; }, diff --git a/packages/runtime-host/src/server/peer-listener.ts b/packages/runtime-host/src/server/peer-listener.ts index 254253ee55..97a5b80703 100644 --- a/packages/runtime-host/src/server/peer-listener.ts +++ b/packages/runtime-host/src/server/peer-listener.ts @@ -80,8 +80,6 @@ export function createRuntimeHostPeerListener( class RuntimeHostPeerListener implements RuntimeHostPeerListenerContract { readonly kind = 'libp2p_direct' as const; readonly endpoint: string; - readonly peerId: string; - readonly listenAddresses: readonly string[]; readonly #reachability: PeerReachabilityPublisher; readonly #accessAuthority: RuntimeHostAccessAuthority; readonly #accept: (connection: RuntimeHostListenerConnection) => void; @@ -103,8 +101,6 @@ class RuntimeHostPeerListener implements RuntimeHostPeerListenerContract { ) { const identity = client.identity(); this.endpoint = identity.peerId; - this.peerId = identity.peerId; - this.listenAddresses = Object.freeze([...identity.listenAddresses]); this.#reachability = reachability; this.#accessAuthority = accessAuthority; this.#accept = accept; diff --git a/packages/runtime-host/src/transport/peer-native.ts b/packages/runtime-host/src/transport/peer-native.ts index 29e46a7b9b..88f23bd8e2 100644 --- a/packages/runtime-host/src/transport/peer-native.ts +++ b/packages/runtime-host/src/transport/peer-native.ts @@ -74,11 +74,16 @@ export interface RuntimeHostPeerIdentityProof { export interface RuntimeHostPeerNativeEndpoint { readonly peerId: string; readonly reachabilitySnapshot: RuntimeHostPeerNativeReachabilitySnapshot; + readonly connectivitySnapshot: RuntimeHostPeerNativeConnectivitySnapshot; readonly transitSnapshot: RuntimeHostPeerTransitSnapshot; watchReachability( afterGeneration: number, timeoutMs: number, ): Promise; + watchConnectivity( + afterGeneration: number, + timeoutMs: number, + ): Promise; connect(options: { readonly requestId: number; readonly peerId: string; @@ -95,6 +100,12 @@ export interface RuntimeHostPeerNativeEndpoint { readonly transitRelayPeerIds?: readonly string[]; readonly directDeadlineMs: number; }): Promise; + updateConnect(options: { + readonly requestId: number; + readonly routeHints: readonly string[]; + readonly coordinationRelays?: readonly string[]; + readonly transitRelayPeerIds?: readonly string[]; + }): Promise; configureTransit(options: { readonly allowedPeerIds: readonly string[]; readonly approvedRelayPeerIds: readonly string[]; @@ -112,6 +123,11 @@ export interface RuntimeHostPeerNativeReachabilitySnapshot { readonly activeCoordinationRelays: readonly string[]; } +export interface RuntimeHostPeerNativeConnectivitySnapshot { + readonly generation: number; + readonly connectedPeerIds: readonly string[]; +} + export interface RuntimeHostPeerTransitSnapshot { readonly allowedPeerCount: number; readonly activeReservationCount: number; @@ -499,16 +515,22 @@ function isPeerNativeEndpoint(value: unknown): value is RuntimeHostPeerNativeEnd isPeerId(value.peerId) && 'reachabilitySnapshot' in value && isPeerReachabilitySnapshot(value.reachabilitySnapshot) && + 'connectivitySnapshot' in value && + isPeerConnectivitySnapshot(value.connectivitySnapshot) && 'transitSnapshot' in value && isPeerTransitSnapshot(value.transitSnapshot) && 'connect' in value && typeof value.connect === 'function' && 'connectMeshControl' in value && typeof value.connectMeshControl === 'function' && + 'updateConnect' in value && + typeof value.updateConnect === 'function' && 'configureTransit' in value && typeof value.configureTransit === 'function' && 'watchReachability' in value && typeof value.watchReachability === 'function' && + 'watchConnectivity' in value && + typeof value.watchConnectivity === 'function' && 'cancelConnect' in value && typeof value.cancelConnect === 'function' && 'accept' in value && @@ -520,6 +542,20 @@ function isPeerNativeEndpoint(value: unknown): value is RuntimeHostPeerNativeEnd ); } +function isPeerConnectivitySnapshot( + value: unknown, +): value is RuntimeHostPeerNativeConnectivitySnapshot { + return ( + typeof value === 'object' && + value !== null && + 'generation' in value && + isCount(value.generation) && + 'connectedPeerIds' in value && + Array.isArray(value.connectedPeerIds) && + value.connectedPeerIds.every((peerId) => typeof peerId === 'string') + ); +} + function isPeerReachabilitySnapshot( value: unknown, ): value is RuntimeHostPeerNativeReachabilitySnapshot { diff --git a/scripts/smoke-release-cli-package.mjs b/scripts/smoke-release-cli-package.mjs index dfaf6ddf99..7b23c89af8 100644 --- a/scripts/smoke-release-cli-package.mjs +++ b/scripts/smoke-release-cli-package.mjs @@ -277,7 +277,7 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } const listener = host.peerListeners[0]; if ( !listener || - listener.peerId !== peerId || + listener.reachability.lease.peerId !== peerId || listener.reachability.lease.directRoutes.length === 0 ) { throw new Error('Installed Runtime Host direct-peer listener did not become ready'); @@ -332,9 +332,7 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } rootId: host.rootId, transport: { kind: 'libp2p-direct', - peerId, - routeHints: listener.reachability.lease.directRoutes, - coordinationRelays: listener.reachability.lease.coordinationRoutes, + reachability: listener.reachability, }, }, credential: issued.credential,