Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ jobs:
. != "tests/e2e/ssh-codex-display-artifacts-repro.spec.ts" and
. != "tests/e2e/ssh-cold-activation-restore.spec.ts" and
. != "tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts" and
. != "tests/e2e/ssh-emptied-worktree-reactivation.spec.ts" and
. != "tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts" and
. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
. != "tests/e2e/ssh-docker-half-open-link.spec.ts" and
Expand Down Expand Up @@ -386,6 +387,7 @@ jobs:
contains(inputs.test_files, 'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-cold-activation-restore.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-emptied-worktree-reactivation.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-docker-half-open-link.spec.ts') ||
Expand Down
1 change: 1 addition & 0 deletions config/scripts/ci-e2e-job-selection.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export const DOCKER_SSH_E2E_SPECS = [
'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts',
'tests/e2e/ssh-cold-activation-restore.spec.ts',
'tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts',
'tests/e2e/ssh-emptied-worktree-reactivation.spec.ts',
'tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts',
'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts',
'tests/e2e/ssh-docker-half-open-link.spec.ts',
Expand Down
1 change: 1 addition & 0 deletions config/scripts/run-ssh-docker-e2e.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ const result = spawnSync(
'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts',
'tests/e2e/ssh-cold-activation-restore.spec.ts',
'tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts',
'tests/e2e/ssh-emptied-worktree-reactivation.spec.ts',
'tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts',
'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts',
'tests/e2e/ssh-docker-half-open-link.spec.ts',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import {
/** How long a conflicted target keeps watching the catalog for the rows it could not place. The
* catalog for a remote host often only fills in when the user opens the worktree, which is minutes
* after connect, and `conflict` has no other exit: it suppresses uploads and holds terminal
* authority at `unverifiable` until a reconnect or an unsolicited host push happens to arrive. */
* authority for those paths at `unverifiable` until a reconnect or an unsolicited host push
* happens to arrive. */
const DEFERRED_SNAPSHOT_PLACEMENT_TIMEOUT_MS = 600_000

/** A retry's own apply can report paths it still could not place, which arms the next watch. If the
Expand Down
6 changes: 4 additions & 2 deletions src/renderer/src/hooks/remote-workspace-snapshot-apply.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ export async function applyDirectSshRemoteWorkspaceSnapshot({
// replaces the host snapshot - it would delete the very tabs we failed to adopt;
// - workspace-terminal-host-authority.ts treats `offline`/`error` on an un-hydrated target
// as `none`, its bounded floor, which authorises seeding AND sleeping-agent resume.
// `conflict` is deliberately not in that set, so authority stays `unverifiable`.
// `conflict` is deliberately not in that set: the unplaced paths stay `unverifiable`,
// and only the worktrees this apply did place read `none`.
// Hydration is cleared, not merely withheld: the set is add-only, so a target that synced
// cleanly before would otherwise keep uploading from this incomplete picture (STA-3593).
currentStore.clearRemoteWorkspaceHydrated(authority.targetId)
Expand All @@ -238,7 +239,8 @@ export async function applyDirectSshRemoteWorkspaceSnapshot({
direction: 'pull',
revision: snapshot.revision,
updatedAt: snapshot.updatedAt,
hostObservationToken: snapshot.hostObservationToken
hostObservationToken: snapshot.hostObservationToken,
unplacedTabWorktreePaths: [...unplacedTabWorktreePaths]
})
}
const reconnectAbort = new AbortController()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,19 @@ describe('a host snapshot whose terminal tabs cannot be placed locally', () => {
).toBe('unverifiable')
})

it('answers for the worktrees it placed when the host names another path it cannot place', async () => {
const store = createStore()
// A second machine left a tab on beta; this client only has alpha.
landAlphaLineage(store)
await applySnapshot(store, snapshot(1))

expect(syncPhase(store)).toBe('conflict')
expect(isHydrated(store)).toBe(false)
expect(adoptedTabIds(store)).toEqual(['T1', 'T2'])
expect(resolveWorkspaceTerminalHostAuthority(store.getState(), ALPHA_ID)).toBe('none')
expect(resolveWorkspaceTerminalHostAuthority(store.getState(), BETA_ID)).toBe('unverifiable')
})

it('adopts every host tab and declares the target hydrated once the catalog is present', async () => {
const store = createStore()
landHostLineage(store)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,21 @@ describe('sleeping-agent resume across the direct-SSH hydration gap', () => {
expect(state.sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined()
})

it.each([
['wakes it when the host named tabs only on another path', '/srv/proj/another-machines', 1],
['keeps waiting when the host named tabs here it could not place', PATH, 0]
])('after a snapshot that could not place every tab, %s', (_label, unplacedPath, expected) => {
seedColdDirectSshStart()
useAppStore.getState().setRemoteWorkspaceSyncStatus(TARGET_ID, {
phase: 'conflict',
direction: 'pull',
unplacedTabWorktreePaths: [unplacedPath]
})

expect(resumeSleepingAgentSessionsForWorktree(WORKTREE_ID)).toBe(expected)
expect(resumedTabCount(WORKTREE_ID)).toBe(expected)
})

it('leaves a purely local workspace resuming with no added latency', () => {
seedColdDirectSshStart()
const localRecord = {
Expand Down
78 changes: 78 additions & 0 deletions src/renderer/src/lib/workspace-terminal-host-authority.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,84 @@ describe('workspace terminal seeding authority', () => {
expect(terminalTabCount(store, SSH_WORKTREE_ID)).toBe(0)
})

it('seeds a worktree the conflicting snapshot placed, and only that one', () => {
const store = createTestStore()
seedDirectSsh(store)
store.getState().setRemoteWorkspaceSyncStatus(TARGET_ID, {
phase: 'conflict',
direction: 'pull',
unplacedTabWorktreePaths: ['/srv/proj/another-machines-worktree']
})

expect(resolveWorkspaceTerminalHostAuthority(store.getState(), SSH_WORKTREE_ID)).toBe('none')
expect(ensureWorktreeHasInitialTerminal(store.getState(), SSH_WORKTREE_ID)).toBeTruthy()
expect(terminalTabCount(store, SSH_WORKTREE_ID)).toBe(1)
})

it('leaves a worktree emptied on purpose empty at startup, even once the host answered for it', () => {
const store = createTestStore()
seedDirectSsh(store)
store.getState().setRemoteWorkspaceSyncStatus(TARGET_ID, {
phase: 'conflict',
direction: 'pull',
unplacedTabWorktreePaths: ['/srv/proj/another-machines-worktree']
})
store.setState({
tabsByWorktree: { [SSH_WORKTREE_ID]: [] },
closedTerminalTabTombstonesByTabId: {
closed: { closedAt: Date.now(), worktreeId: SSH_WORKTREE_ID }
}
})

// Startup restore passes no reseed option; only an explicit activation does.
expect(ensureWorktreeHasInitialTerminal(store.getState(), SSH_WORKTREE_ID)).toBeNull()
expect(terminalTabCount(store, SSH_WORKTREE_ID)).toBe(0)
expect(
ensureWorktreeHasInitialTerminal(
store.getState(),
SSH_WORKTREE_ID,
undefined,
undefined,
undefined,
undefined,
{ reseedEmptiedWorkspace: true }
)
).toBeTruthy()
})

it.each([
['its own path', '/srv/proj/feature'],
['another spelling of its path', '/srv/proj//feature/']
])('keeps waiting when the host named tabs it could not place on %s', (_label, unplacedPath) => {
const store = createTestStore()
seedDirectSsh(store)
store.getState().setRemoteWorkspaceSyncStatus(TARGET_ID, {
phase: 'conflict',
direction: 'pull',
unplacedTabWorktreePaths: [unplacedPath]
})

expect(resolveWorkspaceTerminalHostAuthority(store.getState(), SSH_WORKTREE_ID)).toBe(
'unverifiable'
)
})

it('stops trusting a placed worktree once a later sync event replaces the conflict', () => {
const store = createTestStore()
seedDirectSsh(store)
store.getState().setRemoteWorkspaceSyncStatus(TARGET_ID, {
phase: 'conflict',
direction: 'pull',
unplacedTabWorktreePaths: []
})
// What a snapshot that could not be applied, or a rejected upload, writes.
store.getState().setRemoteWorkspaceSyncStatus(TARGET_ID, { phase: 'conflict' })

expect(resolveWorkspaceTerminalHostAuthority(store.getState(), SSH_WORKTREE_ID)).toBe(
'unverifiable'
)
})

it('leaves terminal creation to the host of a paired runtime workspace', () => {
const store = createTestStore()
store.setState({
Expand Down
27 changes: 21 additions & 6 deletions src/renderer/src/lib/workspace-terminal-host-authority.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { FLOATING_TERMINAL_WORKTREE_ID } from '../../../shared/constants'
import { parseExecutionHostId } from '../../../shared/execution-host'
import { normalizeRuntimePathForComparison } from '../../../shared/cross-platform-path'
import { parseWorkspaceKey } from '../../../shared/workspace-scope'
import { splitWorktreeId } from '../../../shared/worktree/id'
import type { HostLiveTerminalProbeVerdict } from '@/runtime/host-live-terminal-probe'
import type { RemoteWorkspaceSyncStatus } from '@/store/slices/ssh'
import { isWebRuntimeSessionActive } from '@/runtime/web-runtime-session'
Expand All @@ -16,9 +18,10 @@ import {
* than restated so the two cannot drift:
*
* - `live` — a remote execution host owns terminal creation here. It supplies the surface itself.
* - `unverifiable` — the workspace has a remote execution host with a sync still in flight or not
* yet attempted. It is bounded: a sync that terminates without an answer resolves `none` rather
* than refusing forever (see resolveDirectSshAuthority).
* - `unverifiable` — the workspace has a remote execution host that has not answered for it: a sync
* still in flight or not yet attempted, or a snapshot naming tabs here that could not be placed.
* It is bounded: a sync that terminates without an answer resolves `none` rather than refusing
* forever (see resolveDirectSshAuthority).
* - `none` — no remote host holds terminals here: either the workspace is local (this client *is*
* the execution host, and its own tab rows are the whole truth) or the host answered and holds
* nothing.
Expand Down Expand Up @@ -57,9 +60,11 @@ const TERMINATED_WITHOUT_ANSWER_PHASES = new Set(['offline', 'error'])

function resolveDirectSshAuthority(
state: WorkspaceTerminalHostAuthorityState,
targetId: string
targetId: string,
worktreeId: string
): WorkspaceTerminalHostAuthority {
const phase = state.remoteWorkspaceSyncStatusByTargetId?.[targetId]?.phase
const status = state.remoteWorkspaceSyncStatusByTargetId?.[targetId]
const phase = status?.phase
if (state.remoteWorkspaceHydratedTargetIds?.has(targetId)) {
// Why: the same pair use-app-session-persistence.ts gates uploads on. A conflicting snapshot
// means the client's picture is not the host's, so it is no basis for deciding the host holds
Expand All @@ -73,6 +78,16 @@ function resolveDirectSshAuthority(
// seed is meant to be a wait, not a permanent refusal.
return 'none'
}
const unplacedPaths = status?.unplacedTabWorktreePaths
const worktreePath = splitWorktreeId(worktreeId)?.worktreePath
if (phase === 'conflict' && unplacedPaths && worktreePath) {
// Why: that snapshot was applied to every worktree it could place, so the host has answered for
// this one. Refusing the whole target strands it on one path this client never had (#22015).
const comparablePath = normalizeRuntimePathForComparison(worktreePath)
return unplacedPaths.some((path) => normalizeRuntimePathForComparison(path) === comparablePath)
? 'unverifiable'
: 'none'
Comment thread
AmoabaKelvin marked this conversation as resolved.
Resolved
}
// Not connected, still pulling, or not yet attempted — "we could not ask", never "nothing there".
return 'unverifiable'
}
Expand Down Expand Up @@ -104,7 +119,7 @@ export function resolveWorkspaceTerminalHostAuthority(
// Why the git-worktree narrowing: the snapshot replaces exactly DirectSshTargetScope.gitWorktreeIds
// (remote-workspace-snapshot-apply.ts). A folder workspace's rows are never replaced by the host,
// so waiting on an answer that will never name them would leave it terminal-less for good.
return resolveDirectSshAuthority(state, host.targetId)
return resolveDirectSshAuthority(state, host.targetId, worktreeId)
}
// Local, or outside the host's replace scope: this client is the execution host and its own rows are
// the whole truth. Absence of a catalog row is not evidence of a remote owner, and refusing to act
Expand Down
3 changes: 3 additions & 0 deletions src/renderer/src/store/slices/ssh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ export type RemoteWorkspaceSyncStatus = {
hostObservationToken?: string
lastSyncedAt?: number
message?: string
/** Set only by an applied pull that ends in `conflict`: the host paths whose tabs this client
* could not place. Every other worktree on the target holds the host's rows from that snapshot. */
unplacedTabWorktreePaths?: readonly string[]
}

export type SshCredentialRequest = {
Expand Down
41 changes: 41 additions & 0 deletions tests/e2e/helpers/docker-ssh-relay-workspace-snapshot.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { expect, type Page } from '@stablyai/playwright-test'

import {
execDockerSshRelayTargetControlCommand,
type DockerSshRelayTarget
} from './docker-ssh-relay-target'

/** Where the relay persists a target's workspace snapshot inside the fixture container. */
const REMOTE_SNAPSHOT_DIR = '/root/.orca/sessions'

export async function readTargetSyncPhase(
page: Page,
targetId: string
): Promise<string | undefined> {
return page.evaluate(
(id) => window.__store?.getState().remoteWorkspaceSyncStatusByTargetId[id]?.phase,
targetId
)
}

function findRemoteSnapshotPath(target: DockerSshRelayTarget): string | null {
const listing = execDockerSshRelayTargetControlCommand(
target,
`ls -1 ${REMOTE_SNAPSHOT_DIR}/*.json 2>/dev/null || true`
).trim()
return listing.split('\n').find((line) => line.endsWith('.json')) ?? null
}

export async function waitForUploadedRemoteSnapshot(target: DockerSshRelayTarget): Promise<string> {
let snapshotPath: string | null = null
await expect
.poll(
() => {
snapshotPath = findRemoteSnapshotPath(target)
return snapshotPath
},
{ timeout: 60_000, message: 'the relay never persisted a workspace snapshot' }
)
.not.toBeNull()
return snapshotPath!
}
35 changes: 4 additions & 31 deletions tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import {
type DockerSshRelayTarget
} from './helpers/docker-ssh-relay-target'
import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection'
import {
readTargetSyncPhase,
waitForUploadedRemoteSnapshot
} from './helpers/docker-ssh-relay-workspace-snapshot'
import { createRestartSession } from './helpers/orca-restart'

const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1'
const BASELINE_TAB_COUNT = 3
/** Where the relay persists a target's workspace snapshot inside the fixture container. */
const REMOTE_SNAPSHOT_DIR = '/root/.orca/sessions'

test.use({ seedTestRepo: false })

Expand All @@ -35,13 +37,6 @@ async function isTargetHydrated(page: Page, targetId: string): Promise<boolean>
)
}

async function readTargetSyncPhase(page: Page, targetId: string): Promise<string | undefined> {
return page.evaluate(
(id) => window.__store?.getState().remoteWorkspaceSyncStatusByTargetId[id]?.phase,
targetId
)
}

/** Poll until the worktree's tabs stop changing, so a late seed cannot slip past the sample. */
async function waitForSettledTabIds(page: Page, worktreeId: string): Promise<string[]> {
let latest: string[] = []
Expand All @@ -62,28 +57,6 @@ async function waitForSettledTabIds(page: Page, worktreeId: string): Promise<str
return latest
}

function findRemoteSnapshotPath(target: DockerSshRelayTarget): string | null {
const listing = execDockerSshRelayTargetControlCommand(
target,
`ls -1 ${REMOTE_SNAPSHOT_DIR}/*.json 2>/dev/null || true`
).trim()
return listing.split('\n').find((line) => line.endsWith('.json')) ?? null
}

async function waitForUploadedRemoteSnapshot(target: DockerSshRelayTarget): Promise<string> {
let snapshotPath: string | null = null
await expect
.poll(
() => {
snapshotPath = findRemoteSnapshotPath(target)
return snapshotPath
},
{ timeout: 60_000, message: 'the relay never persisted a workspace snapshot' }
)
.not.toBeNull()
return snapshotPath!
}

/**
* Replace the relay's snapshot file with a FIFO so `workspace.get` blocks on open.
*
Expand Down
Loading
Loading