Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,8 @@ jobs:
. != "tests/e2e/orcad-serve-mode-switch.spec.ts" and
. != "tests/e2e/ssh-orcad-auto-convert.spec.ts" and
. != "tests/e2e/windows-missing-appdata-startup.spec.ts" and
. != "tests/e2e/ssh-orcad-idle-exit.spec.ts"
. != "tests/e2e/ssh-orcad-idle-exit.spec.ts" and
. != "tests/e2e/ssh-orcad-half-open-input.spec.ts"
)' <<<"$TEST_FILES_JSON" > "$RUNNER_TEMP/general-e2e-specs"
fi
mapfile -t TEST_FILES < "$RUNNER_TEMP/general-e2e-specs"
Expand Down Expand Up @@ -568,7 +569,7 @@ jobs:
orcad-auto-convert-docker:
name: ssh host auto-converts to managed orcad (Docker)
needs: [build, prepare-native-cache]
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-idle-exit.spec.ts')
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-idle-exit.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-half-open-input.spec.ts')
runs-on: ubuntu-latest
timeout-minutes: 45
env:
Expand Down Expand Up @@ -604,7 +605,7 @@ jobs:
ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay
run: |
export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template"
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts tests/e2e/ssh-orcad-idle-exit.spec.ts --project=electron-headless --workers=1
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts tests/e2e/ssh-orcad-idle-exit.spec.ts tests/e2e/ssh-orcad-half-open-input.spec.ts --project=electron-headless --workers=1
- uses: actions/upload-artifact@v7
if: failure()
with:
Expand Down
107 changes: 107 additions & 0 deletions config/reliability-gates.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,113 @@
}
},
"gates": [
{
"id": "terminal-input.remote-delivery-uncertainty",
"title": "Unanswered remote terminal input remains visibly uncertain",
"maturity": "experimental",
"protection": "partial",
"owner": "terminal-runtime",
"layer": "host-stream-renderer-and-real-ssh",
"surfaces": [
"paired terminal keyboard input",
"binary input receipts",
"older-host RPC input fallback",
"reconnect warning"
],
"platforms": [
"macos",
"linux",
"windows"
],
"providers": [
"remote-runtime",
"ssh"
],
"coveredPlatforms": [
"macos"
],
"coveredProviders": [
"remote-runtime",
"ssh"
],
"coverageNotes": "Hidden macOS Electron drives a Linux arm64 Docker SSH host through a controlled TCP proxy. Source-built server and installed pre-receipt server exercise negotiation and fallback. Native Windows and WSL are gaps; no physical packet outage claim.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/26483"
],
"invariant": "Only an execution-host receipt settles an input write. Missing receipts warn without replaying uncertain bytes, replacing the remote process, or treating contact loss as process exit. Parser replies do not create delivery warnings.",
"oracle": "Real keyboard input reaches an independent host ledger. Before and after lines arrive once in the same process; a two-way stall loses the during line and warns, while a reply-only stall delivers it exactly once and still warns. The warning survives reconnect. Unit contracts prove negotiated settlement, older-host fallback, unrelated output, out-of-order receipts, bounded state and timer cleanup.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test src/renderer/src/runtime/remote-terminal-input-receipts.test.ts src/renderer/src/runtime/remote-runtime-terminal-input-receipts.test.ts src/main/runtime/rpc/terminal-multiplex-input-receipts.test.ts src/renderer/src/components/terminal-pane/remote-runtime-input-delivery-warning.test.ts",
"ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_SSH_DOCKER=1 pnpm run test:e2e tests/e2e/ssh-orcad-half-open-input.spec.ts --workers=1"
],
"testFiles": [
"src/renderer/src/runtime/remote-terminal-input-receipts.test.ts",
"src/renderer/src/runtime/remote-runtime-terminal-input-receipts.test.ts",
"src/main/runtime/rpc/terminal-multiplex-input-receipts.test.ts",
"src/renderer/src/components/terminal-pane/remote-runtime-input-delivery-warning.test.ts",
"tests/e2e/ssh-orcad-half-open-input.spec.ts"
],
"assertionRefs": [
{
"file": "src/renderer/src/runtime/remote-terminal-input-receipts.test.ts",
"assertions": [
"does not mistake a later write receipt for receipt of an earlier write",
"bounds pending receipts and uses one timer during an input flood"
]
},
{
"file": "src/renderer/src/runtime/remote-runtime-terminal-input-receipts.test.ts",
"assertions": [
"does not treat unrelated output as an input receipt",
"warns on a lost connection and never replays pending input into its replacement"
]
},
{
"file": "tests/e2e/ssh-orcad-half-open-input.spec.ts",
"assertions": [
"managed host warns about uncertain input when both traffic stalls",
"managed host warns about uncertain input when to-client traffic stalls"
]
}
],
"evidenceRuns": [
{
"date": "2026-10-08",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/renderer/src/runtime/remote-terminal-input-receipts.test.ts src/renderer/src/runtime/remote-runtime-terminal-input-receipts.test.ts src/main/runtime/rpc/terminal-multiplex-input-receipts.test.ts src/renderer/src/components/terminal-pane/remote-runtime-input-delivery-warning.test.ts",
"result": "passed",
"durationSeconds": 1.0,
"summary": "Sixteen focused tests pass: receipt settlement, mixed-version fallback, unrelated output, exact-once transport behavior and bounded timer state."
}
],
"runtimeBudget": {
"p95Seconds": 300,
"scope": "Local Docker SSH exercise; long-term CI p95 has not been established."
},
"flakeHistory": {
"status": "not-started",
"evidence": "Two source-built server fault cases passed locally; no soak claim."
},
"redGreenEvidence": {
"status": "complete",
"evidence": "Unchanged app loses the during line without a warning and fails the new rendered-warning assertion. Fixed source-built server warns around 15 seconds in both fault directions, preserves one host process and never duplicates the delivered line."
},
"performanceBudget": {
"required": true,
"evidence": "Existing 8 ms keyboard batching remains. Negotiated binary writes do not wait for prior round trips. At most 1024 numeric receipts and one deadline timer per active stream; no command text retained, polling, inventory reads, startup waits or subprocesses. Older hosts use existing bounded RPC request deadlines."
},
"knownGaps": [
"Native Windows, WSL and Linux desktop clients are not exercised. Local and daemon transports are unchanged; the Linux daemon is exercised behind the managed host.",
"Older hosts without PTY settlements prove request receipt only. This prevents silent network loss but cannot prove an old provider write or command execution.",
"A warning cannot determine whether unacknowledged input ran. The user must inspect the terminal before retrying.",
"Long-term soak and wide-area typing latency measurements remain unproved."
],
"promotionCriteria": [
"Collect policy soak and native Windows/WSL coverage while retaining exact-once process-ledger assertions."
],
"demotionRule": "Keep experimental until soak. Investigate missing warnings, duplicate writes or retained timers without retries or relaxed assertions."
},
{
"id": "native-chat.transcript-consumed-boundary",
"title": "Transcript replacement remains visible when it races message publication",
Expand Down
7 changes: 5 additions & 2 deletions config/scripts/ci-e2e-job-selection.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export const ORCAD_AUTO_CONVERT_E2E_SPEC = 'tests/e2e/ssh-orcad-auto-convert.spe
export const WINDOWS_MISSING_APPDATA_E2E_SPEC = 'tests/e2e/windows-missing-appdata-startup.spec.ts'
// Runs in the auto-convert job, which builds the template it needs.
export const ORCAD_IDLE_EXIT_E2E_SPEC = 'tests/e2e/ssh-orcad-idle-exit.spec.ts'
export const ORCAD_HALF_OPEN_INPUT_E2E_SPEC = 'tests/e2e/ssh-orcad-half-open-input.spec.ts'
export const DEDICATED_E2E_SPECS = [
...DOCKER_SSH_E2E_SPECS,
NODE_NETWORK_E2E_SPEC,
Expand All @@ -56,7 +57,8 @@ export const DEDICATED_E2E_SPECS = [
ORCAD_SERVE_MODE_SWITCH_E2E_SPEC,
ORCAD_AUTO_CONVERT_E2E_SPEC,
WINDOWS_MISSING_APPDATA_E2E_SPEC,
ORCAD_IDLE_EXIT_E2E_SPEC
ORCAD_IDLE_EXIT_E2E_SPEC,
ORCAD_HALF_OPEN_INPUT_E2E_SPEC
]
const dedicatedSpecs = new Set(DEDICATED_E2E_SPECS)
const dockerSpecs = new Set(DOCKER_SSH_E2E_SPECS)
Expand Down Expand Up @@ -97,7 +99,8 @@ export function classifyE2eJobs(input, sshSourceChanged = 'false') {
spec === LOCALHOST_SSH_E2E_SPEC ||
spec === ORCAD_SERVE_MODE_SWITCH_E2E_SPEC ||
spec === ORCAD_AUTO_CONVERT_E2E_SPEC ||
spec === ORCAD_IDLE_EXIT_E2E_SPEC
spec === ORCAD_IDLE_EXIT_E2E_SPEC ||
spec === ORCAD_HALF_OPEN_INPUT_E2E_SPEC
)
}
}
Expand Down
8 changes: 8 additions & 0 deletions config/scripts/ci-e2e-job-selection.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -145,3 +145,11 @@ it('runs remaining SSH tests after real failures and stops them when a run is ca
expect(step.if).toContain('always()')
}
})

it.each([
'src/main/runtime/rpc/methods/terminal/terminal-multiplex-input-delivery.ts',
'src/renderer/src/runtime/remote-terminal-input-receipts.ts',
'src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts'
])('routes input delivery faults from %s', (path) => {
expect(selectPrE2eSpecs([path])).toContain('tests/e2e/ssh-orcad-half-open-input.spec.ts')
})
76 changes: 76 additions & 0 deletions config/scripts/pr-e2e-docker-lane-contract.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { readdirSync, readFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { expect, it } from 'vitest'
import { parse } from 'yaml'

const projectDir = resolve(import.meta.dirname, '../..')
const e2eWorkflow = parse(readFileSync(join(projectDir, '.github/workflows/e2e.yml'), 'utf8'))

it('gives every Docker-gated SSH spec a lane that runs it', () => {
// Why this shape: the sharded lanes set no ORCA_E2E_SSH_DOCKER, so a Docker-gated spec
// that no runner names runs nowhere and still reports green — the silent skip this file
// exists to prevent. Asserting reachability rather than a literal keeps that true when
// the lanes move.
// The remaining exemption needs performance validation before routine CI, recorded in
// run-ssh-docker-e2e.mjs so the gap stays legible rather than looking like coverage.
const unreachableSpecs = new Set(['tests/e2e/ssh-docker-relay-perf.spec.ts'])
// Why comments are stripped: the runner documents the exempt spec by name in a
// prose comment. A substring scan over raw text would count any spec merely *discussed* in a
// runner as claimed by it -- the silent skip this assertion exists to catch, re-entering
// through the documentation.
const stripComments = (text) => text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '')
const laneRunners = [
'run-ssh-docker-e2e.mjs',
'run-ssh-docker-watcher-isolation-e2e.mjs',
'run-ssh-docker-terminal-parking-e2e.mjs'
].map((file) => stripComments(readFileSync(join(projectDir, 'config/scripts', file), 'utf8')))
const managedLane = e2eWorkflow.jobs['orcad-auto-convert-docker']
const managedRun = managedLane.steps.find(
(step) => step.name === 'Convert a relay-era Docker host'
)
expect(managedRun.env.ORCA_E2E_SSH_DOCKER).toBe('1')
expect(managedRun.run).toContain('ORCA_E2E_ORCAD_CONVERT_TEMPLATE=')
laneRunners.push(stripComments(managedRun.run))

// Why a comparison and not the bare name: preview and demo specs cite the flag in a
// "how to run me" comment without gating on it. Why a regex rather than one literal: an
// equally-valid spelling (double quotes, or a `!==` guard) would escape a fixed-string scan
// and the spec would silently leave the contract.
const dockerGateExpression = /ORCA_E2E_SSH_DOCKER\s*[!=]==\s*['"]1['"]/
const dockerGatedSpecs = readdirSync(join(projectDir, 'tests/e2e'))
.filter((file) => file.endsWith('.spec.ts'))
.map((file) => `tests/e2e/${file}`)
.filter((spec) => dockerGateExpression.test(readFileSync(join(projectDir, spec), 'utf8')))
expect(dockerGatedSpecs.length).toBeGreaterThan(0)

const unclaimed = dockerGatedSpecs.filter(
(spec) => !unreachableSpecs.has(spec) && !laneRunners.some((runner) => runner.includes(spec))
)
expect(
unclaimed,
`Docker-gated specs claimed by no lane runner: ${unclaimed.join(', ')}`
).toEqual([])

// Why: an exemption that outlives its spec would quietly excuse a real gap.
for (const spec of unreachableSpecs) {
expect(dockerGatedSpecs, spec).toContain(spec)
// Why also assert absence from every runner: `unreachableSpecs` short-circuits the
// unclaimed check above, so a spec could be documented as exempt while a runner still
// invokes it -- an exemption that reads as coverage removal but changes nothing, and a
// lane that stays red for a reason the file says it excluded.
for (const runner of laneRunners) {
expect(runner.includes(spec), `${spec} is exempt but still invoked by a lane runner`).toBe(
false
)
}
}

const laneStep = e2eWorkflow.jobs['ssh-docker-watcher-isolation'].steps.find(
(step) => step.name === 'Run remaining Docker SSH E2E'
)
expect(laneStep.run).toContain('test:e2e:ssh-docker')
// Why: the added serial tests, several budgeting 4-10 minutes each, do not fit the old 35.
expect(
e2eWorkflow.jobs['ssh-docker-watcher-isolation']['timeout-minutes']
).toBeGreaterThanOrEqual(60)
})
70 changes: 4 additions & 66 deletions config/scripts/pr-e2e-gate-contract.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -385,69 +385,6 @@ describe('PR E2E gate contract', () => {
expect(filterStep.run).toContain('ssh_source_changed=$SSH_SOURCE_CHANGED')
})

it('gives every Docker-gated SSH spec a lane that runs it', () => {
// Why this shape: the sharded lanes set no ORCA_E2E_SSH_DOCKER, so a Docker-gated spec
// that no runner names runs nowhere and still reports green — the silent skip this file
// exists to prevent. Asserting reachability rather than a literal keeps that true when
// the lanes move.
// The remaining exemption needs performance validation before routine CI, recorded in
// run-ssh-docker-e2e.mjs so the gap stays legible rather than looking like coverage.
const unreachableSpecs = new Set(['tests/e2e/ssh-docker-relay-perf.spec.ts'])
// Why comments are stripped: the runner documents the exempt spec by name in a
// prose comment. A substring scan over raw text would count any spec merely *discussed* in a
// runner as claimed by it -- the silent skip this assertion exists to catch, re-entering
// through the documentation.
const stripComments = (text) =>
text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '')
const laneRunners = [
'run-ssh-docker-e2e.mjs',
'run-ssh-docker-watcher-isolation-e2e.mjs',
'run-ssh-docker-terminal-parking-e2e.mjs'
].map((file) => stripComments(readFileSync(join(projectDir, 'config/scripts', file), 'utf8')))

// Why a comparison and not the bare name: preview and demo specs cite the flag in a
// "how to run me" comment without gating on it. Why a regex rather than one literal: an
// equally-valid spelling (double quotes, or a `!==` guard) would escape a fixed-string scan
// and the spec would silently leave the contract.
const dockerGateExpression = /ORCA_E2E_SSH_DOCKER\s*[!=]==\s*['"]1['"]/
const dockerGatedSpecs = readdirSync(join(projectDir, 'tests/e2e'))
.filter((file) => file.endsWith('.spec.ts'))
.map((file) => `tests/e2e/${file}`)
.filter((spec) => dockerGateExpression.test(readFileSync(join(projectDir, spec), 'utf8')))
expect(dockerGatedSpecs.length).toBeGreaterThan(0)

const unclaimed = dockerGatedSpecs.filter(
(spec) => !unreachableSpecs.has(spec) && !laneRunners.some((runner) => runner.includes(spec))
)
expect(
unclaimed,
`Docker-gated specs claimed by no lane runner: ${unclaimed.join(', ')}`
).toEqual([])

// Why: an exemption that outlives its spec would quietly excuse a real gap.
for (const spec of unreachableSpecs) {
expect(dockerGatedSpecs, spec).toContain(spec)
// Why also assert absence from every runner: `unreachableSpecs` short-circuits the
// unclaimed check above, so a spec could be documented as exempt while a runner still
// invokes it -- an exemption that reads as coverage removal but changes nothing, and a
// lane that stays red for a reason the file says it excluded.
for (const runner of laneRunners) {
expect(runner.includes(spec), `${spec} is exempt but still invoked by a lane runner`).toBe(
false
)
}
}

const laneStep = e2eWorkflow.jobs['ssh-docker-watcher-isolation'].steps.find(
(step) => step.name === 'Run remaining Docker SSH E2E'
)
expect(laneStep.run).toContain('test:e2e:ssh-docker')
// Why: the added serial tests, several budgeting 4-10 minutes each, do not fit the old 35.
expect(
e2eWorkflow.jobs['ssh-docker-watcher-isolation']['timeout-minutes']
).toBeGreaterThanOrEqual(60)
})

it('scopes the VM rollback oracle to the PR range and recipe schema authorities', () => {
expect(rollbackStep.run).toMatch(/diff-base\.mjs "\$BASE_SHA"[\s\S]*"\$DIFF_BASE" HEAD --/)
expect(rollbackStep.run).toContain('src/shared/ephemeral-vm-recipes.ts')
Expand All @@ -470,15 +407,16 @@ describe('PR E2E gate contract', () => {
],
[
'src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts',
'tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts'
'tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts',
['tests/e2e/ssh-orcad-half-open-input.spec.ts']
],
[
'src/renderer/src/components/terminal-pane/remote-pane-layout-push.ts',
'tests/e2e/paired-remote-pane-layout-retry.spec.ts'
]
]
for (const [source, spec] of cases) {
expect(selectPrE2eSpecs([source]), source).toEqual([spec])
for (const [source, spec, additionalSpecs = []] of cases) {
expect(selectPrE2eSpecs([source]), source).toEqual([spec, ...additionalSpecs])
expect(selectPrE2eSpecs([source.replace(/\.tsx?$/, '.test.ts')]), source).toEqual([])
expect(existsSync(join(projectDir, spec)), spec).toBe(true)
}
Expand Down
9 changes: 9 additions & 0 deletions config/scripts/pr-e2e-source-routing.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,15 @@ export const PR_E2E_SOURCE_ROUTES = [
file
))
},
{
id: 'ssh.orcad-input-delivery',
specs: ['tests/e2e/ssh-orcad-half-open-input.spec.ts'],
matches: (file) =>
isProductSource(file) &&
/^src\/(?:main\/runtime\/rpc\/methods\/terminal\/(?:terminal-(?:multiplex|input-delivery|stream)|stream-schemas)|renderer\/src\/(?:runtime\/(?:remote-runtime-terminal-(?:multiplexer|flow-controller|response-controller)|remote-terminal-input-receipts)|components\/terminal-pane\/(?:remote-runtime-pty-transport|remote-terminal-input-delivery-warning)))/.test(
file
)
},
{
id: 'ssh.orcad-idle-exit',
specs: ['tests/e2e/ssh-orcad-idle-exit.spec.ts'],
Expand Down
Loading
Loading