diff --git a/.github/actions/build-agent-device/action.yml b/.github/actions/build-agent-device/action.yml new file mode 100644 index 0000000..eceff0e --- /dev/null +++ b/.github/actions/build-agent-device/action.yml @@ -0,0 +1,30 @@ +name: Build Agent Device +description: Build an immutable upstream revision for manual Apple validation +inputs: + sha: + description: Full commit SHA in callstack/agent-device + required: true +runs: + using: composite + steps: + - name: Require an immutable revision + shell: bash + env: + AGENT_DEVICE_REVISION: ${{ inputs.sha }} + run: '[[ "$AGENT_DEVICE_REVISION" =~ ^[0-9a-f]{40}$ ]]' + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: callstack/agent-device + ref: ${{ inputs.sha }} + path: .agent-device-validation + persist-credentials: false + + - name: Build CLI and use its matching runner source + shell: bash + working-directory: .agent-device-validation + run: | + corepack pnpm install --frozen-lockfile + corepack pnpm build + echo "AGENT_DEVICE_BIN=$PWD/bin/agent-device.mjs" >> "$GITHUB_ENV" + echo "Agent Device source: $(git rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/exact-package-candidate.yml b/.github/workflows/exact-package-candidate.yml index 7abade0..57e2c45 100644 --- a/.github/workflows/exact-package-candidate.yml +++ b/.github/workflows/exact-package-candidate.yml @@ -7,6 +7,10 @@ on: - cron: '17 3 * * *' workflow_dispatch: inputs: + agent_device_sha: + description: Optional full agent-device commit SHA to build for Apple validation + required: false + type: string channel: description: Validation channel recorded with the candidate required: true @@ -488,6 +492,12 @@ jobs: - name: Setup uses: ./.github/actions/setup + - name: Build the requested Agent Device revision + if: runner.os == 'macOS' && inputs.agent_device_sha != '' + uses: ./.github/actions/build-agent-device + with: + sha: ${{ inputs.agent_device_sha }} + - name: Download inspected candidate uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: diff --git a/.github/workflows/repro-cold-simulator-touch.yml b/.github/workflows/repro-cold-simulator-touch.yml new file mode 100644 index 0000000..5e9bc3d --- /dev/null +++ b/.github/workflows/repro-cold-simulator-touch.yml @@ -0,0 +1,127 @@ +name: Repro cold-simulator lost touch stream + +# Reproduction for the agent-device upstream report tracked in issue #84: the +# first `gesture drag` synthesized into a freshly erased, cold-booted iOS 26.5 +# simulator on a hosted macos-26 runner reports ok without the expected app +# outcome. Every job erases and cold-boots the simulator several times and +# records what the app observed. Manual validation can require prompt delivery +# or reproduction; observation runs retain results without enforcing a verdict. + +on: + workflow_dispatch: + inputs: + agent_device_sha: + description: Optional full agent-device commit SHA to build for Apple validation + required: false + type: string + expectation: + description: Required delivery result (observe only records evidence) + type: choice + default: observe + options: [observe, prompt, reproduced] + iterations: + description: Cold boot iterations per job + default: '3' + required: false + samples: + description: Parallel jobs per configuration + default: '4' + required: false + push: + branches: + - docs/issue-84-agent-device-upstream-handoff + paths: + - .github/workflows/repro-cold-simulator-touch.yml + - scripts/repro-cold-simulator-touch.mjs + +concurrency: + group: repro-cold-simulator-touch-${{ github.ref }}-${{ inputs.agent_device_sha || 'release' }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + plan: + name: Plan matrix + runs-on: ubuntu-latest + outputs: + samples: ${{ steps.plan.outputs.samples }} + steps: + - id: plan + run: | + count='${{ inputs.samples || '4' }}' + echo "samples=$(node -e "console.log(JSON.stringify(Array.from({length: Number('$count')}, (_, i) => i + 1)))")" >> "$GITHUB_OUTPUT" + + repro: + name: ${{ matrix.label }} / sample ${{ matrix.sample }} + needs: plan + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + sample: ${{ fromJSON(needs.plan.outputs.samples) }} + include: + - label: ios26-macos-26 + runner: macos-26 + xcode: '26' + runtime: iOS-26-5 + runtime_display: iOS 26.5 + exclude: [] + env: + RCT_USE_RN_DEP: 1 + RCT_USE_PREBUILT_RNCORE: 1 + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Setup + uses: ./.github/actions/setup + + - name: Build the requested Agent Device revision + if: runner.os == 'macOS' && inputs.agent_device_sha != '' + uses: ./.github/actions/build-agent-device + with: + sha: ${{ inputs.agent_device_sha }} + + - name: Use appropriate Xcode version + if: matrix.xcode != '' + uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0 + with: + xcode-version: ${{ matrix.xcode }} + + - name: Verify simulator runtime + run: xcrun simctl list runtimes | grep -F '${{ matrix.runtime_display }}' + + - name: Cache cocoapods + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: example/ios/Pods + key: ${{ runner.os }}-contract-cocoapods-${{ hashFiles('example/ios/Podfile.lock') }} + restore-keys: | + ${{ runner.os }}-contract-cocoapods- + + - name: Install cocoapods + run: | + cd example + bundle install + bundle exec pod install --project-directory=ios + + - name: Build the Scenario Lab (same Release build as the device-contract job) + run: | + cd example/ios + xcodebuild -workspace ReorderableExample.xcworkspace -scheme ReorderableExample -configuration Release -sdk iphonesimulator -derivedDataPath build CODE_SIGNING_ALLOWED=NO | tail -n 40 + test -d build/Build/Products/Release-iphonesimulator/ReorderableExample.app + + - name: Reproduce + run: node scripts/repro-cold-simulator-touch.mjs --runtime '${{ matrix.runtime }}' --iterations '${{ inputs.iterations || '3' }}' --out artifacts/repro-cold-touch --expect '${{ inputs.expectation || 'observe' }}' + + - name: Upload evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: repro-cold-touch-${{ matrix.label }}-sample-${{ matrix.sample }} + if-no-files-found: warn + retention-days: 30 + path: artifacts/repro-cold-touch/ diff --git a/.gitignore b/.gitignore index 751fcac..6d20b69 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,7 @@ artifacts/candidate/ # Local agent worktrees .claude/worktrees/ + +# Source builds and cold-touch validation evidence +.agent-device-validation/ +artifacts/repro-cold-touch*/ diff --git a/docs/adr/0006-automated-publication-confidence.md b/docs/adr/0006-automated-publication-confidence.md index 8be43b7..7fb4022 100644 --- a/docs/adr/0006-automated-publication-confidence.md +++ b/docs/adr/0006-automated-publication-confidence.md @@ -33,3 +33,15 @@ Amended 2026-09-04 ([#65](https://github.com/thiagobrez/react-native-reorderable | `android.fallback` | Detox | The iOS 26 cell moves to Detox because the first pointer gesture the Agent Device XCTest runner synthesizes into a freshly erased, cold-booted iOS 26.5 simulator on hosted runners arrives seconds late or never while injection reports success ([#54](https://github.com/thiagobrez/react-native-reorderable/issues/54)) — the pipeline's dominant failure, and one no widened gesture timing or scenario retry may paper over, because a silent input drop is indistinguishable from a broken contract. That cell's unique responsibility is proving automatic engine selection picks the fallback engine on a runtime without native SwiftUI reordering, and Detox pointer synthesis proves that equally: scenario definitions, terminal public-outcome assertions, and the four-engine parity comparison are unchanged. Agent Device is retained on iOS 27, where its selector-targeted center drag is the only harness that expresses insertion against native SwiftUI rows. + +Amended 2026-09-06: restore Agent Device for all five `ios26.auto-fallback` +pointer scenarios, using the same semantic target plans as `ios27.fallback`. +The restoration is being validated with the cold-start synthesis fix in +[callstack/agent-device#2362](https://github.com/callstack/agent-device/pull/2362). +Detox continues to own accessibility actions and in-flight lifecycle mutations, +Android pointer scenarios, and native iOS multi-selection. The package-candidate +workflow still owns the four-engine contract and parity checks; PR gate placement, +exact-tarball validation, render backstops, and the single retained infrastructure +retry (including runner-cache cleanup) retain their existing responsibilities. +Cold-start delivery and the normal contract must pass before this restoration is +ready to merge; a successful warm-simulator run alone is insufficient. diff --git a/docs/handoffs/agent-device-upstream-reports/README.md b/docs/handoffs/agent-device-upstream-reports/README.md new file mode 100644 index 0000000..125f15a --- /dev/null +++ b/docs/handoffs/agent-device-upstream-reports/README.md @@ -0,0 +1,22 @@ +# Agent Device cold-simulator investigation (issue #84) + +The active report is [callstack/agent-device#2362](https://github.com/callstack/agent-device/pull/2362). [Downstream PR #101](https://github.com/thiagobrez/react-native-reorderable/pull/101) restores Agent Device for the five iOS 26 pointer scenarios and allows validation against an immutable upstream source revision. + +Read the [current proposal and evidence](./fix-proposal.md) and [reproduction rules](./reproduction.md) before reusing earlier claims. The original handoff overstated both the cause and the fix. Missing app outcomes are established in retained runs; a persistent digitizer-attachment defect and a successful warm-up cure are not. + +## Current status + +- The maintainer's structural and regression-test requests are implemented. The PR remains draft pending a controlled cold-start comparison showing improvement. +- iOS 27 fallback and native lanes passed the latest completed full matrix; iOS 26 failed before gestures during toolchain discovery. Earlier iOS 26 runs include both successful gestures and missing outcomes. See the [explicit environment comparison](https://github.com/callstack/agent-device/pull/2362#issuecomment-5581718543). +- The separate Android smoke-positioning correction [#2369](https://github.com/callstack/agent-device/pull/2369) passed its checks and was merged by the maintainer. +- The second proposed report (`dyld` / `libcurl`) was dropped as unconfirmed. Do not reopen it from the original handoff. + +## Evidence and historical material + +The [original report draft](./report-1-ios26-cold-simulator-lost-touch-stream.md) is historical material, not a current paste-ready report. Its transport explanation, public-XCTest control claim and video-derived latency estimates are superseded by the reassessment. + +`stage-evidence.sh` retrieves the original recordings and logs from runs 33488489650 and 33355396042. Their original artifact expiry dates are September 15 and September 14, 2026. Retain downloaded copies before expiry. New reproduction runs retain their evidence for 30 days and are linked from the upstream PR. + +Video frames establish visible outcomes. Similar recording and command durations do not establish clock synchronization: do not subtract a command wall timestamp from video presentation time to estimate physical touch latency. + +Issue [#80](https://github.com/thiagobrez/react-native-reorderable/issues/80) tracks adopting a subsequent release. Maintainer approval and passing downstream validation are still required; this handoff does not authorize merging or releasing. diff --git a/docs/handoffs/agent-device-upstream-reports/fix-proposal.md b/docs/handoffs/agent-device-upstream-reports/fix-proposal.md new file mode 100644 index 0000000..44caad5 --- /dev/null +++ b/docs/handoffs/agent-device-upstream-reports/fix-proposal.md @@ -0,0 +1,41 @@ +# Proposed agent-device synthesized-input preparation + +[Upstream PR #2362](https://github.com/callstack/agent-device/pull/2362) is a proposed fix, not a shipped or confirmed fix. [CI restoration PR #101](https://github.com/thiagobrez/react-native-reorderable/pull/101) restores Agent Device for the five iOS 26 pointer scenarios while preserving the exact-candidate gate, existing whole-job retry, render backstop, parity checks and Pods cache fix. + +## Current implementation + +At upstream `211f232ee85df5742e3be093a008b46f80a6e1a7`, the shared iOS synthesized-event constructor attempts one empty event record before constructing the first real input record. Preparation adds no pointer paths or app-delivering contact, uses the already-resolved orientation and process ID, and adds no accessibility, screenshot or frame lookup. Only successful preparation sets the process-wide flag; a failure does not suppress the real action and allows a later attempt. + +The maintainer requested this shared, contact-free boundary in place of the original gesture-only warm-up tap. The native regression exercises the actual bridge across gesture, scroll, drag, tap, swipe and sequence entry points. It observes an initial preparation failure, a later successful preparation and no further empty records. The real event record is created after preparation so its timestamps are fresh. + +## What the evidence establishes + +- The native ordering regression fails without the change and passes with it. This establishes ordering and retry semantics, not a cure for missing input. +- Local runs of the same Objective-C implementation passed three cold boots, nine gestures and three selector presses. A subsequent one-boot check of the corrected measurement harness also passed. +- [Hosted XCTest nightly](https://github.com/callstack/agent-device/actions/runs/34060558728) passes at `211f232ee`, including the isolated regression and the source-derived full-suite count of 226. [iOS smoke](https://github.com/callstack/agent-device/actions/runs/34058548288/attempts/2) and [CI/coverage](https://github.com/callstack/agent-device/actions/runs/34058548253) pass at that SHA. +- All downstream device, consumer, runtime and parity gates passed in [34056536337](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34056536337), using `1cf7e5f1cfa7ff96ac5dd781b8af90fc6037457e` (identical Objective-C implementation). iOS 26 used its existing whole-job retry after an initial scenario-readiness failure. The workflow's optional final npm dry-run failed because version 1.0.0 was already published; the entire run is therefore not green. +- The later complete matrix [34060592647](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34060592647) at `211f232ee` failed: iOS 27 fallback/native and Android passed, but both iOS 26 preparation attempts hit five-second Xcode toolchain-query timeouts. No iOS 26 gestures were measured in that run. Parity therefore could not pass. + +Comparable hosted cold-start evidence and maintainer approval remain required. The fresh same-harness [base](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34060459388) and [candidate](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34060458003) both failed validation. Base first gestures were lost/error/prompt; candidate first gestures were error/error/prompt. The candidate also had an unobserved post-relaunch outcome after successful preparation. These are not a usable green comparison. [Detailed results](https://github.com/callstack/agent-device/pull/2362#issuecomment-5562425331) retain the setup and observation limitations. + +## Withdrawn claims + +In the candidate's unobserved post-relaunch gesture, UIKit logged touch dispatches after an initial delay of about 2.3 seconds from virtual-digitizer attachment. Passing sampled gestures reached initial app dispatch in roughly 10–33 ms. This contradicts complete non-delivery in that sample and suggests delayed/coalesced delivery may disrupt gesture recognition; touch-phase evidence is still needed. See the [native dispatch comparison](https://github.com/callstack/agent-device/pull/2362#issuecomment-5562449851). + +The original persistent-digitizer explanation is contradicted by traces: digitizers attach and detach per real gesture, and empty preparation does not attach one. Preparation may affect other XCTest state, but that mechanism remains unproven. + +The later `press id="engine-fallback"` probe uses private synthesis in both release 0.20.10 and the candidate (`fallbackAttempted=false`). It is not an independent public-XCTest control. A later successful input also cannot exclude an earlier simulator stall. + +Post-command observation waits are not physical touch-delivery latency. A viewport or runner error cannot establish a missing app outcome. Video presentation timestamps must not be aligned with command wall time without an independently verified synchronization point. + +See the [current measurement rules](./reproduction.md#measurement-limits-and-current-validation), [trace correction](https://github.com/callstack/agent-device/pull/2362#issuecomment-5561886720), and [historical proposal](https://github.com/thiagobrez/react-native-reorderable/blob/afdde2e0867beae448ec9784a641bbdd6cbdbde4/docs/handoffs/agent-device-upstream-reports/fix-proposal.md). The historical contact-based proposal and causal claims are superseded. + +## Independent Android smoke blocker + +Upstream Android smoke failed before input because a fixed scroll moved the press canary above the viewport. [PR #2369](https://github.com/callstack/agent-device/pull/2369) isolates a bounded visibility-based test-positioning correction. It changes no runtime code; [hosted Android smoke](https://github.com/callstack/agent-device/actions/runs/34061497846) and all other checks pass at `6211bfc3f15789fbb8cf5fd5907d399a7dd58d54`. The maintainer approved and merged it on September 6, 2026. + +## iOS 26 versus iOS 27 + +The closest control is iOS 27 **fallback**: it explicitly selects the same engine that iOS 26 selects automatically, with the same app, scenarios, semantic gesture plans and Agent Device revision. The environment combinations differ: iOS 26.5 uses `macos-26` with Xcode 26; iOS 27.0 uses `xcode-27` with Xcode 27. The completed jobs also report different host macOS versions. This is an observed reliability difference between those environments, not proof that the simulator OS alone causes it. + +The subsequent [touch-phase diagnostic run](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34062782494) measured no gestures: two preparation errors and an app setup timeout. Its observation-only green status is not fix validation. diff --git a/docs/handoffs/agent-device-upstream-reports/report-1-ios26-cold-simulator-lost-touch-stream.md b/docs/handoffs/agent-device-upstream-reports/report-1-ios26-cold-simulator-lost-touch-stream.md new file mode 100644 index 0000000..91489b5 --- /dev/null +++ b/docs/handoffs/agent-device-upstream-reports/report-1-ios26-cold-simulator-lost-touch-stream.md @@ -0,0 +1,151 @@ +> Historical report draft. Its causal and timing claims are superseded by the [current reassessment](./fix-proposal.md). Do not file this text verbatim. + +# Upstream report 1 (draft for callstack/agent-device) + +**Suggested title:** `gesture drag` reports `ok=1` but the app receives no touch stream on a freshly erased, cold-booted iOS 26.5 simulator (GitHub-hosted macos-26) + +--- + +## Summary + +On a GitHub-hosted `macos-26-arm64` runner, the first `gesture drag` synthesized into a freshly erased and cold-booted iOS 26.5 simulator completes with `AGENT_DEVICE_RUNNER_COMMAND_COMPLETED ... ok=1` and exactly the scripted duration, while the app under test never receives the touch stream. The screen stays pixel-static through the gesture and for 30 s afterwards; the app's own counters stay at zero. When the runner is restarted a minute later in the same job, the identical gesture on the same screen activates within ~2 s. + +In an earlier run on the same hosted image the touch stream did arrive, but ~3.5–4.5 s after touch-down. So delivery is late on this environment, and when the delay exceeds the gesture's own duration the gesture is silently lost. The runner has no way to tell us, and `ok=1` is misleading. + +Not reproducible locally (3/3 passes with instant activation on an Apple Silicon Mac with the same runtime, agent-device version and app build). + +## Environment + +- agent-device **0.20.10** (npm), invoked via `node_modules/.bin/agent-device` +- Node **24.13.0** (GitHub hosted toolcache) +- GitHub-hosted runner image **macos-26-arm64**, version **20260728.0273.1** +- Xcode **26.6** (`/Applications/Xcode_26.6.app`), iPhoneSimulator SDK 26.5 +- iOS Simulator runtime **iOS 26.5 (23F77)**, device **iPhone 17 Pro** +- App under test: React Native 0.85 Release build; the drag activates a `react-native-gesture-handler` `Pan().activateAfterLongPress(350)` gesture +- Simulator is `xcrun simctl shutdown` → `erase` → `boot` → `bootstatus -b` immediately before the job's first agent-device session (cold, freshly migrated data: `bootstatus` reports ~30 s of "Waiting on Data Migration") + +## Exact commands + +Preflight (once per job, after boot and `simctl install`): + +``` +agent-device daemon stop +agent-device prepare ios-runner --platform ios --session issue39-ios-runner-preflight --udid --timeout 300000 +agent-device open reorderable.example --relaunch --platform ios --session ... --udid +agent-device wait "Scenario Lab" 15000 --depth 100 ... +agent-device open "reorderable://lab/free-form?preset=teaching&engine=auto" ... +agent-device alert accept ... +agent-device close ... +``` + +Scenario replay (`agent-device replay pointer-replay.ad --session issue39-free-form-reorder`), where the script is: + +``` +context platform=ios kind=simulator timeout=60000 +env APP_TARGET="reorderable.example" +env DEEP_LINK="reorderable://lab/free-form?preset=teaching&engine=auto" +open "${APP_TARGET}" --relaunch +wait "Scenario Lab" 15000 +open "${DEEP_LINK}" +wait "Current order: card-0, card-1, card-2, card-3, card-4, card-5" 15000 +wait "Current selection: none" 15000 +wait "Last committed event: None" 15000 +wait "Callback count: 0" 15000 +screenshot ".../baseline.png" +wait 1000 +screenshot ".../gesture-start-marker.png" +gesture drag "id=\"card-card-0\"" "id=\"card-card-3\"" 650 1200 8000 +wait "Current order: card-1, card-2, card-0, card-3, card-4, card-5" 15000 +wait "Current selection: none" 15000 +wait "Last committed event: {\"sourceIds\":[\"card-0\"],\"destination\":{\"sectionId\":null,\"beforeId\":\"card-3\"}}" 15000 +wait "Callback count: 1" 15000 +screenshot ".../terminal.png" +``` + +The recording is a device-scope `record start … --scope device` / `record stop` around the whole replay. All 12 pre-gesture steps pass; the app is on screen, the deep-linked list is rendered, and `findText`/`snapshot` work throughout (the accessibility path is healthy). + +## Observed: failing run (2026-09-01) + +Run: https://github.com/thiagobrez/react-native-reorderable/actions/runs/33488489650 (job `ios26.auto-fallback / RN 0.85`, attempt 1) + +Runner log (`sessions/issue39-ios-runner-preflight/runner.log`, runner PID 17428): + +``` +2026-09-01 09:05:19.026160+0000 AgentDeviceRunnerUITests-Runner[17428:50059] AGENT_DEVICE_RUNNER_COMMAND_ACCEPTED command=gesture commandId=runner-57219b81-e2a7-471a-8c35-a534b247d1fe +2026-09-01 09:05:19.027131+0000 AgentDeviceRunnerUITests-Runner[17428:48965] AGENT_DEVICE_RUNNER_FAST_APP_GUARD command=gesture bundle=reorderable.example state=4 +2026-09-01 09:05:30.329312+0000 AgentDeviceRunnerUITests-Runner[17428:50059] AGENT_DEVICE_RUNNER_COMMAND_COMPLETED command=gesture commandId=runner-57219b81-e2a7-471a-8c35-a534b247d1fe ok=1 +``` + +Daemon events for the session (`events.ndjson`): + +``` +09:05:16.040 request.started gesture +09:05:30.338 action.recorded gesture {"durationMs":9850,"pointerCount":1} +09:05:30.339 request.finished gesture ok durationMs=14299 +09:05:30.340 request.started wait "Current order: card-1, card-2, card-0, ..." +09:05:45.458 request.finished wait error COMMAND_FAILED durationMs=15119 +``` + +CLI output: + +``` +Error (REPLAY_DIVERGENCE): Replay failed at step 12 (wait "Current order: card-1, card-2, card-0, card-3, card-4, card-5" 15000): wait timed out for text: Current order: card-1, card-2, card-0, card-3, card-4, card-5. Current surface: Back to scenario ... +``` + +The app-side evidence says nothing happened: + +- `Callback count: 0`, order unchanged, no lift/border highlight in any screenshot taken during or after the gesture (`sample-1..3.png`). +- The recording (`free-form-reorder.pointer.mp4`, 56.5 s, variable frame rate) contains **no frame between t=13.3 s and t=56.5 s**. Aligned to the daemon timeline (recording starts with the replay at 09:05:04.3), the gesture was accepted at t≈14.7 s and completed at t≈26.0 s, so the screen was pixel-static through the entire 9.85 s gesture and the following 30 s of waits. Frame-to-frame mean pixel difference over that window is exactly zero. + +### Same job, one runner restart later, the identical gesture works + +After the divergence our harness closes the session and the daemon/runner restart (new runner PID 27197). `virtualized-list-reorder` uses the same interaction shape (first row dragged before the fourth row, same `650 1200 8000` timings) on a different screen: + +``` +2026-09-01 09:07:21.447542+0000 AgentDeviceRunnerUITests-Runner[27197:72995] AGENT_DEVICE_RUNNER_COMMAND_ACCEPTED command=gesture ... +2026-09-01 09:07:32.124743+0000 AgentDeviceRunnerUITests-Runner[27197:72995] AGENT_DEVICE_RUNNER_COMMAND_COMPLETED command=gesture ... ok=1 +``` + +Its recording shows continuous frame changes starting ~1.7 s after touch-down (lift + move animation) and a final change at release (commit). `multi-selection-reorder`, which renders the **same component** as the failing scenario, also passed in that job. Screen, component, selectors and expectations are exonerated; only the first gesture into the cold simulator is lost. + +## Observed: late delivery on the same image (2026-08-31) + +Run: https://github.com/thiagobrez/react-native-reorderable/actions/runs/33355396042 (same job name, attempt 2, same image version 20260728.0273.1) + +``` +2026-08-31 04:30:10.748949+0000 AgentDeviceRunnerUITests-Runner[92119:238107] AGENT_DEVICE_RUNNER_COMMAND_ACCEPTED command=gesture ... +2026-08-31 04:30:22.789525+0000 AgentDeviceRunnerUITests-Runner[92119:238107] AGENT_DEVICE_RUNNER_COMMAND_COMPLETED command=gesture ... ok=1 +``` + +Daemon: gesture request 04:30:10.248 → 04:30:22.816 (`durationMs` 12568); the following `wait "Current order: card-1, card-2, card-0, ..."` passed in **46 ms**, so the reorder had already committed by release. + +Recording (`free-form-reorder.attempt-2.pointer.mp4`, 37.1 s, aligned to replay start 04:29:46.5): touch-down at t≈24.2 s; the first visible lift/move frames appear at **t=27.6–28.9 s**, i.e. **~3.4–4.7 s after touch-down**, although the scripted source hold is 650 ms and the long-press threshold is 350 ms. The move (scripted for t≈24.85–26.05) shows up as a single burst, after which the card sits at the destination until release. + +So on this environment the synthesized stream reaches the app seconds late; in the failing run the delay exceeded the whole gesture and nothing arrived. Our earlier CI history has the same shape: two passes on this image on 2026-08-31, then failures in most runs since, always on whichever pointer scenario runs first after the cold boot. + +In this run's attempt 1 (also cold), `wait "Scenario Lab" 15000` timed out at step 2 right after `open --relaunch`, and `bootstatus` had shown ~100 s of data migration. The hosted host is slow to bring the cold simulator to a usable state; the lost touch stream appears to be the input-pipeline flavour of the same slowness. + +## Why we are filing this here + +- The runner reported success (`ok=1`, `durationMs` 9850, `pointerCount` 1) for a gesture that had no observable effect. Whether the loss is in Apple's XCTest/HID injection on a cold simulator or in the runner's synthesis path is not distinguishable from outside, but agent-device is the layer that could detect and report it. +- Nothing in the app can see an event that never arrives; `snapshot`, `findText` and `wait` do not exercise the touch path, so no amount of pre-gesture settling helps (we tried 6 s pre-gesture settles and 1200 ms source holds on `fix/ios26-gesture-cooldown`; still lost). + +## Questions / asks + +1. Is this a known limitation of XCUITest touch synthesis on a freshly booted simulator on slow virtualized hosts? If yes, a note in `help workflow` (Apple CI section) would save others the diagnosis. +2. Could `prepare ios-runner` (or an opt-in flag) run an **input-readiness probe** on cold simulators: one observable throwaway touch with bounded retry, so that the first real gesture is not the probe? We designed this locally (press a chip, wait for its observable effect) but did not ship it because we re-routed iOS 26 pointer tests to another driver instead. +3. Could `gesture` surface *some* delivery signal when available (e.g. the runner observing that the target element did not change state, or timing telemetry for when the first touch event was dispatched vs accepted)? Today `ok=1` with the exact scripted duration is indistinguishable from success. + +## What we did meanwhile + +- Diagnosis with evidence: https://github.com/thiagobrez/react-native-reorderable/issues/54 +- Decision: route iOS 26 pointer scenarios through Detox and keep agent-device on iOS 27 (native and fallback engines): https://github.com/thiagobrez/react-native-reorderable/issues/60 +- Happy to test a probe or a fix on our hosted matrix; the failing job is reproducible in CI within a few runs. + +## Attachments + +- `free-form-reorder.pointer.mp4` (failing run, pixel-static through the gesture) +- `virtualized-list-reorder.pointer.mp4` (same job after runner restart, works) +- `runner.log` (failing run, full runner log), `free-form-reorder.events.ndjson`, `free-form-reorder.replay-request.aed3651e9dc0eaca.ndjson`, `free-form-reorder.pointer-replay.ad` +- `sample-1.png`, `sample-2.png`, `sample-3.png` (failing run, app during/after gesture) +- `free-form-reorder.attempt-2.pointer.mp4`, `free-form-reorder.attempt-2.events.ndjson` (late-delivery run) diff --git a/docs/handoffs/agent-device-upstream-reports/reproduction.md b/docs/handoffs/agent-device-upstream-reports/reproduction.md new file mode 100644 index 0000000..99d23cd --- /dev/null +++ b/docs/handoffs/agent-device-upstream-reports/reproduction.md @@ -0,0 +1,111 @@ +# Reproduction: first synthesized drag lost on a cold hosted iOS 26.5 simulator + +> Proposed fix: [callstack/agent-device#2362](https://github.com/callstack/agent-device/pull/2362). The historical runs demonstrate missing or delayed **app outcomes**, not a proven transport-level cause. The persistent-digitizer explanation was contradicted by later traces; see [the reassessment](./fix-proposal.md). + +Workflow: `.github/workflows/repro-cold-simulator-touch.yml` +Driver: `scripts/repro-cold-simulator-touch.mjs` + +## What one iteration does + +1. `xcrun simctl shutdown` → `erase` → `boot` → `bootstatus -b` on the iPhone 17 Pro + simulator of the requested runtime, then `simctl install` of the Release + Scenario Lab build (same xcodebuild invocation as the device-contract job). +2. `agent-device daemon stop`, then `agent-device prepare ios-runner` with a + per-iteration `AGENT_DEVICE_STATE_DIR`. +3. The device-contract preflight dance: `open --relaunch`, wait for + `Scenario Lab`, open the free-form deep link, `alert accept`, relaunch, deep + link again, wait for the initial order and `Callback count: 0`. +4. Start a `simctl io recordVideo` capture. +5. **First gesture**: `gesture drag 'id="card-card-0"' 'id="card-card-3"' 650 1200 8000 --json`, + then `wait "Callback count: 1" 15000 --json`. A successful gesture followed by + a wait whose structured reason is `wait_target_absent` establishes that the + expected outcome was not observed within that wait. Other observer errors do + not establish a missing outcome. +6. Probes, always run so passing iterations act as controls: + - **second gesture** on the same app instance (private event synthesis again; + it drags the last card to the top so it changes the order whatever the first + drag did); + - **relaunch gesture**: `open --relaunch` + deep link, then the same drag; + - **selector press**: `press 'id="engine-fallback"'` then `wait text 'engine=fallback'`. + This can use the same private synthesis bridge as the drag; it is not an + independent public-XCTest control. +7. Collect `simulator-log.txt` (backboardd, SpringBoard, app), `host-log.txt` + (testmanagerd, CoreSimulatorService), the recording, the agent-device state + directory (runner.log, request logs) and `iteration.json`. + +`summary.json` and the job step summary hold one row per iteration with +the gesture observation classes and `selectorTapDelivered` (historical artifacts +called the last field `xctestTapDelivered`). + +## Measurement limits and current validation + +`waitDurationMs` measures observation after the gesture command returns, not time +from touch-down to input delivery. Historical percentile tables below also included +errored commands; those waits cannot establish touch latency. Current summaries use +`postCommandObservationWaitMs` and exclude commands that failed before synthesis +or could not observe the result. A failed wait only counts as `lost` when its +structured reason is `wait_target_absent`; viewport errors, runner restarts and +other observer failures are `observation-error`, and still fail validation. +Both released 0.20.10 and candidate traces show the selector press using +`kind=coordinateTap` private synthesis with `fallbackAttempted=false`. The original +claim that this was an independent public-XCTest discriminator is withdrawn. +A later successful action also cannot rule out a simulator stall that recovered +before that action. A URL confirmation covering the app is a setup failure; +matching text behind it does not establish readiness. + +The normal mode records observations. `--expect prompt` requires every cold +iteration and follow-up to pass; `--expect reproduced` requires a reproduced +symptom and rejects setup errors. Both workflows accept an optional immutable +`agent_device_sha`; local runs accept `AGENT_DEVICE_BIN`. Build the selected CLI +and runner before starting the loop, and keep those artifacts unchanged until +session cleanup completes. + +## Historical results + +The original [four-sample run](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34025552105) and [setup-retry run](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34027523634) reported missing or delayed first outcomes. Their classifier included observer failures as missing outcomes and their wait distributions included failed commands. Their original counts and percentile labels must not be used as current transport or timing evidence. + +The earlier exact-source [base run](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34056421445) observed two missing first outcomes and one delayed first observation. The [candidate run](https://github.com/thiagobrez/react-native-reorderable/actions/runs/34056673023) failed: it included toolchain/setup failures, an invalid viewport before synthesis, a URL confirmation overlay, and an observer error falsely classified as lost even though the recording showed a committed drop. It is not a green comparison. + +Fresh base and candidate runs use the corrected harness at downstream `afdde2e0867beae448ec9784a641bbdd6cbdbde4`; see [current validation](./fix-proposal.md#what-the-evidence-establishes). Original reports remain in Git history and retained run artifacts. + +## Delivery classes + +Each class describes the command result and post-command observation: + +- **lost**: the gesture command succeeded, but the expected effect was absent when the wait failed with `wait_target_absent`. +- **late**: the wait succeeded after more than 3 seconds (`LATE_DELIVERY_THRESHOLD_MS`). +- **prompt**: the wait succeeded within 3 seconds. +- **errored**: the gesture command failed; not counted as a measured missing outcome. +- **observation-error**: the wait failed without establishing target absence; not counted as a measured missing outcome. + +`summary.json` carries the classes and post-command wait distribution. `--expect prompt` requires every gesture and follow-up to pass promptly, every selector press to be observed, and no setup errors. `--expect reproduced` requires at least one lost or late first observation and a usable first-gesture measurement on every iteration. Neither expectation proves a transport mechanism. + +## Reading the probes + + +| first | second | relaunch | Selector press | Reading | +| --- | --- | --- | --- | --- | +| lost | delivered | delivered | delivered | The first outcome was not observed; later gestures worked. Cause remains unproven. | +| lost | lost | delivered | delivered | Outcomes were observed after relaunch; this does not identify a transport cause. | +| lost | lost | lost | delivered | The later selector press worked after failed drag outcomes; both may use private synthesis. | +| lost | lost | lost | lost | No expected outcomes were observed; inspect setup, overlays, app, runner and simulator evidence. | + +## Running it + +```bash +gh workflow run repro-cold-simulator-touch.yml --ref ci/restore-agent-device \ + -f agent_device_sha="" -f expectation=prompt -f iterations=3 -f samples=1 +gh run list --workflow repro-cold-simulator-touch.yml --limit 5 +gh run download -D /tmp/repro +``` + +Locally (a passing run on one host does not establish a hosted cold-start fix): + +```bash +AGENT_DEVICE_BIN=/absolute/path/to/agent-device/bin/agent-device.mjs \ + node scripts/repro-cold-simulator-touch.mjs --runtime iOS-26-5 \ + --iterations 1 --expect prompt --out /tmp/repro-smoke +``` + +Without `--expect`, the driver records observations without enforcing a verdict. +Use `--expect prompt` or `--expect reproduced` for pass/fail validation. diff --git a/docs/handoffs/agent-device-upstream-reports/stage-evidence.sh b/docs/handoffs/agent-device-upstream-reports/stage-evidence.sh new file mode 100755 index 0000000..ecb0687 --- /dev/null +++ b/docs/handoffs/agent-device-upstream-reports/stage-evidence.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Re-creates the evidence folder for the agent-device upstream report (#84, report 1). +# Usage: bash docs/handoffs/agent-device-upstream-reports/stage-evidence.sh [target-dir] +# Requires: gh (authenticated), ~230 MB of downloads. Artifacts expire ~90 days after +# 2026-08-31 / 2026-09-01. +set -euo pipefail + +REPO=thiagobrez/react-native-reorderable +FAILING_RUN=33488489650 # 2026-09-01, ios26.auto-fallback attempt-1: lost touch stream +PASSING_RUN=33355396042 # 2026-08-31, ios26.auto-fallback attempt-2: late but delivered +TARGET="${1:-/tmp/agent-device-upstream-reports}" +WORK="$(mktemp -d)" + +download() { # run-id dest + mkdir -p "$2" + gh run download "$1" -R "$REPO" -n issue-39-ios26.auto-fallback -D "$2" +} + +echo "Downloading run artifacts into $WORK ..." +download "$FAILING_RUN" "$WORK/failing" +download "$PASSING_RUN" "$WORK/passing" + +R1="$TARGET/report-1-lost-touch-stream" +D1="$R1/failing-run-$FAILING_RUN" +D2="$R1/passing-run-$PASSING_RUN" +rm -rf "$TARGET" +mkdir -p "$D1" "$D2" + +F="$WORK/failing/agent-device/ios26.auto-fallback/attempts/attempt-1" +cp "$F/free-form-reorder/pointer.mp4" "$D1/free-form-reorder.pointer.mp4" +cp "$F/free-form-reorder/pointer-replay.ad" "$D1/free-form-reorder.pointer-replay.ad" +cp "$F/free-form-reorder/gesture-start-marker.png" "$D1/free-form-reorder.gesture-start-marker.png" +cp "$WORK"/failing/feedback/ios26.auto-fallback/attempts/attempt-1/free-form-reorder/sample-{1,2,3}.png "$D1/" +cp "$F/virtualized-list-reorder/pointer.mp4" "$D1/virtualized-list-reorder.pointer.mp4" +cp "$F/replay-daemon-state/sessions/issue39-ios-runner-preflight/runner.log" "$D1/runner.log" +cp "$F/replay-daemon-state/sessions/issue39-free-form-reorder/events.ndjson" "$D1/free-form-reorder.events.ndjson" +cp "$F/replay-daemon-state/sessions/issue39-free-form-reorder/requests/aed3651e9dc0eaca.ndjson" \ + "$D1/free-form-reorder.replay-request.aed3651e9dc0eaca.ndjson" +cp "$WORK/failing/device-tables/ios26.auto-fallback-job.json" "$D1/job-attempts.json" +grep -n "command=gesture" "$D1/runner.log" > "$D1/runner.log.gesture-lines.txt" + +P="$WORK/passing/agent-device/ios26.auto-fallback" +cp "$P/free-form-reorder/pointer.mp4" "$D2/free-form-reorder.attempt-2.pointer.mp4" +cp "$P/replay-daemon-state/sessions/issue39-free-form-reorder/events.ndjson" "$D2/free-form-reorder.attempt-2.events.ndjson" +cp "$WORK/passing/device-tables/ios26.auto-fallback-job.json" "$D2/job-attempts.json" +grep -n "command=gesture" "$P/replay-daemon-state/sessions/issue39-ios-runner-preflight/runner.log" > "$D2/runner.log.gesture-lines.txt" + +rm -rf "$WORK" +echo "Staged under $TARGET:" +find "$TARGET" -type f | sort diff --git a/e2e/contracts/pointer-drivers.json b/e2e/contracts/pointer-drivers.json index 59d69ce..bc5a516 100644 --- a/e2e/contracts/pointer-drivers.json +++ b/e2e/contracts/pointer-drivers.json @@ -2,8 +2,13 @@ "intent": "beforeTarget", "default": { "driver": "detox", - "destinationSelector": { "relation": "namedTarget" }, - "destination": { "normalizedX": 0.5, "normalizedY": 0.25 } + "destinationSelector": { + "relation": "namedTarget" + }, + "destination": { + "normalizedX": 0.5, + "normalizedY": 0.25 + } }, "overrides": { "ios27.native": { @@ -40,7 +45,9 @@ } ], "destinationId": "card-3", - "labelsById": { "card-2": "Card row 3" }, + "labelsById": { + "card-2": "Card row 3" + }, "destinationSelector": { "relation": "predecessorCenter", "testID": "card-card-2" @@ -138,6 +145,48 @@ "label": "Accepting drop zone" } } + }, + "ios26.auto-fallback": { + "free-form-reorder": { + "driver": "agent-device", + "semanticTargetLabel": "Card row 4", + "destinationSelector": { + "relation": "namedTargetCenterAfterSourceLift", + "label": "Card row 4" + } + }, + "virtualized-list-reorder": { + "driver": "agent-device", + "semanticTargetLabel": "List row 4", + "destinationSelector": { + "relation": "namedTargetCenterAfterSourceLift", + "label": "List row 4" + } + }, + "section-list-reorder": { + "driver": "agent-device", + "semanticTargetLabel": "Section 1, row 4", + "destinationSelector": { + "relation": "namedTargetCenterAfterSourceLift", + "label": "Section 1, row 4" + } + }, + "multi-selection-reorder": { + "driver": "agent-device", + "semanticTargetLabel": "Card row 6", + "destinationSelector": { + "relation": "namedTargetCenterAfterSourceLift", + "label": "Card row 6" + } + }, + "scoped-drop": { + "driver": "agent-device", + "semanticTargetLabel": "Accepting drop zone", + "destinationSelector": { + "relation": "namedTargetCenter", + "label": "Accepting drop zone" + } + } } }, "routes": { @@ -156,11 +205,11 @@ "scoped-drop": "agent-device" }, "ios26.auto-fallback": { - "free-form-reorder": "detox", - "virtualized-list-reorder": "detox", - "section-list-reorder": "detox", - "multi-selection-reorder": "detox", - "scoped-drop": "detox" + "free-form-reorder": "agent-device", + "virtualized-list-reorder": "agent-device", + "section-list-reorder": "agent-device", + "multi-selection-reorder": "agent-device", + "scoped-drop": "agent-device" }, "android.fallback": { "free-form-reorder": "detox", diff --git a/e2e/contracts/portable-contract.e2e.cjs b/e2e/contracts/portable-contract.e2e.cjs index f31e4a8..5fdcd9a 100644 --- a/e2e/contracts/portable-contract.e2e.cjs +++ b/e2e/contracts/portable-contract.e2e.cjs @@ -122,28 +122,28 @@ function accessibilityActionForPlatform(actionName) { } async function runIndependentIosSystemAction(...args) { - await execFileAsync('yarn', [ - 'agent-device', - ...args, - '--session', - 'issue39-detox-interruption', - '--udid', - device.id, - ]); + await execFileAsync( + process.env.AGENT_DEVICE_BIN ?? + require('node:path').resolve('node_modules/.bin/agent-device'), + [...args, '--session', 'issue39-detox-interruption', '--udid', device.id] + ); } async function prepareIndependentIosSystemSession() { - await execFileAsync('yarn', [ - 'agent-device', - 'open', - 'reorderable.example', - '--platform', - 'ios', - '--session', - 'issue39-detox-interruption', - '--udid', - device.id, - ]); + await execFileAsync( + process.env.AGENT_DEVICE_BIN ?? + require('node:path').resolve('node_modules/.bin/agent-device'), + [ + 'open', + 'reorderable.example', + '--platform', + 'ios', + '--session', + 'issue39-detox-interruption', + '--udid', + device.id, + ] + ); } async function interruptActiveDrag(scenario) { diff --git a/example/src/App.tsx b/example/src/App.tsx index d32dfad..4aa61f4 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -350,7 +350,7 @@ export default function App() { { setPreset(next); diff --git a/example/src/__tests__/issue38-example-app.test.tsx b/example/src/__tests__/issue38-example-app.test.tsx index 5b5c5ac..2b89d04 100644 --- a/example/src/__tests__/issue38-example-app.test.tsx +++ b/example/src/__tests__/issue38-example-app.test.tsx @@ -1,5 +1,6 @@ import { describe, expect, it, jest } from '@jest/globals'; -import { fireEvent, render, waitFor } from '@testing-library/react-native'; +import { act, fireEvent, render, waitFor } from '@testing-library/react-native'; +import { Linking } from 'react-native'; jest.mock('react', () => jest.requireActual('../../../node_modules/react') ); @@ -45,6 +46,66 @@ import App from '../App'; import { ScenarioFrame } from '../components'; describe('issue 38 example application', () => { + it.each(['reset', 'same deep link', 'reset after reorder'])( + 'reports the initial card order after %s', + async (action) => { + const url = 'reorderable://lab/free-form?preset=teaching&engine=auto'; + jest.spyOn(Linking, 'getInitialURL').mockResolvedValueOnce(url); + const subscribe = jest.spyOn(Linking, 'addEventListener'); + const rendered = await render(); + const order = 'Order: card-0, card-1, card-2, card-3, card-4, card-5'; + expect( + rendered.getByTestId('scenario-free-form-order') + ).toHaveTextContent(order); + + if (action === 'reset after reorder') { + await fireEvent( + rendered.getByTestId('free-form-container'), + 'reorder', + { + sourceIds: ['card-0'], + destination: { sectionId: null, beforeId: 'card-3' }, + nextOrder: [ + { + sectionId: null, + itemIds: [ + 'card-1', + 'card-2', + 'card-0', + 'card-3', + 'card-4', + 'card-5', + ], + }, + ], + } + ); + expect( + rendered.getByTestId('scenario-free-form-callback-count') + ).toHaveTextContent('Callback count: 1'); + expect( + rendered.getByTestId('scenario-free-form-order') + ).not.toHaveTextContent(order); + } + if (action === 'same deep link') { + const onLink = subscribe.mock.calls.at(-1)![1]; + await act(() => onLink({ url })); + } else { + await fireEvent.press(rendered.getByTestId('scenario-free-form-reset')); + } + + expect( + rendered.getByTestId('scenario-free-form-order') + ).toHaveTextContent(order); + expect( + rendered.getByTestId('scenario-free-form-callback-count') + ).toHaveTextContent('Callback count: 0'); + expect( + rendered.getByTestId('scenario-free-form-last-event') + ).toHaveTextContent('Last committed event: None'); + } + ); + it('navigates the three visible catalogs without exposing an engine control', async () => { const rendered = await render(); expect(rendered.getAllByTestId(/^open-/)).toHaveLength(5); diff --git a/package.json b/package.json index 533779e..50788e7 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "@shopify/flash-list": "2.3.2", "@testing-library/react-native": "^14.0.1", "@types/react": "^19.2.0", - "agent-device": "0.20.10", + "agent-device": "0.21.0", "commitlint": "^21.0.2", "del-cli": "^7.0.0", "detox": "20.51.4", diff --git a/scripts/prepare-agent-device-ios-runner.mjs b/scripts/prepare-agent-device-ios-runner.mjs index 0d66a39..dd0aa50 100644 --- a/scripts/prepare-agent-device-ios-runner.mjs +++ b/scripts/prepare-agent-device-ios-runner.mjs @@ -3,9 +3,10 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; const configuration = process.argv[2]; +const deviceName = process.argv[3] ?? 'iPhone 17 Pro'; if (configuration == null || !configuration.startsWith('ios')) throw new Error( - 'Usage: node scripts/prepare-agent-device-ios-runner.mjs ' + 'Usage: node scripts/prepare-agent-device-ios-runner.mjs [device-name]' ); const runtimeVersion = configuration.startsWith('ios26') @@ -22,9 +23,9 @@ const devices = JSON.parse(devicesResult.stdout); const runtime = Object.entries(devices.devices).find(([key]) => key.includes(runtimeVersion) )?.[1]; -const target = runtime?.find(({ name }) => name === 'iPhone 17 Pro'); +const target = runtime?.find(({ name }) => name === deviceName); if (target == null) - throw new Error(`${runtimeVersion} iPhone 17 Pro simulator is unavailable`); + throw new Error(`${runtimeVersion} ${deviceName} simulator is unavailable`); const run = (command, args, options = {}) => spawnSync(command, args, { stdio: 'inherit', ...options }); @@ -41,7 +42,8 @@ const installResult = run('xcrun', [ if (installResult.status !== 0) throw new Error('Failed to install the Scenario Lab before runner preflight'); -const agentDevice = resolve('node_modules/.bin/agent-device'); +const agentDevice = + process.env.AGENT_DEVICE_BIN ?? resolve('node_modules/.bin/agent-device'); const sessionName = 'issue39-ios-runner-preflight'; const stateRoot = resolve( 'artifacts/issue-39/agent-device', @@ -58,20 +60,43 @@ const environment = { const deepLinkConfirmationMarker = resolve(stateRoot, 'deep-link-confirmed'); const runAgentDevice = (...args) => run(agentDevice, args, { env: environment }); +const sessionArguments = [ + '--platform', + 'ios', + '--session', + sessionName, + '--udid', + target.udid, +]; const runSessionCommand = (...args) => - runAgentDevice( - ...args, - '--platform', - 'ios', - '--session', - sessionName, - '--udid', - target.udid - ); + runAgentDevice(...args, ...sessionArguments); const requireSuccess = (result, description) => { if (result.status !== 0) throw new Error(`${description} exited ${result.status ?? result.signal}`); }; +const confirmDeepLink = () => { + const result = run( + agentDevice, + ['alert', 'get', '--json', ...sessionArguments], + { env: environment, encoding: 'utf8', stdio: 'pipe' } + ); + if (result.status === 0) { + requireSuccess( + runSessionCommand('alert', 'accept'), + 'Scenario Lab URL confirmation' + ); + return; + } + let error; + try { + error = JSON.parse(result.stdout).error; + } catch { + // Unreadable output cannot establish that a confirmation is absent. + } + if (error?.details?.runnerErrorCode === 'ALERT_NOT_FOUND') return; + process.stderr.write(`${result.stdout ?? ''}${result.stderr ?? ''}`); + requireSuccess(result, 'Scenario Lab URL confirmation'); +}; const defaultEnvironment = { ...process.env }; delete defaultEnvironment.AGENT_DEVICE_STATE_DIR; const stopDefaultDaemon = () => { @@ -111,23 +136,11 @@ try { runSessionCommand('wait', 'Scenario Lab', '15000', '--depth', '100'), 'Scenario Lab preflight readiness' ); - requireSuccess( - runSessionCommand('open', deepLink), - 'Scenario Lab URL confirmation seed' - ); - runSessionCommand('alert', 'accept'); - requireSuccess( - runSessionCommand('open', 'reorderable.example', '--relaunch'), - 'Scenario Lab post-confirmation relaunch' - ); - requireSuccess( - runSessionCommand('wait', 'Scenario Lab', '15000', '--depth', '100'), - 'Scenario Lab post-confirmation readiness' - ); requireSuccess( runSessionCommand('open', deepLink), 'Scenario Lab preflight deep link' ); + confirmDeepLink(); requireSuccess( runSessionCommand('wait', initialOutcome, '15000', '--depth', '100'), 'Scenario Lab preflight deep-link outcome' diff --git a/scripts/repro-cold-simulator-touch.mjs b/scripts/repro-cold-simulator-touch.mjs new file mode 100644 index 0000000..82f9fb1 --- /dev/null +++ b/scripts/repro-cold-simulator-touch.mjs @@ -0,0 +1,508 @@ +// Reproduction driver for the agent-device "lost touch stream" report (issue #84). +// +// Each iteration erases and cold-boots the target simulator, prepares the +// agent-device XCUITest runner, opens the Scenario Lab free-form scenario and +// issues the same `gesture drag` the device-contract job uses. It then records +// whether the app observed the drop (`Callback count` increments) and runs a +// small set of follow-up probes that tell the failure modes apart: +// +// second-gesture same app instance, same private event synthesis path +// relaunch-gesture fresh app instance on the same booted simulator +// selector-press agent-device `press` (may also use private synthesis) +// +// Usage: +// node scripts/repro-cold-simulator-touch.mjs --runtime iOS-26-5 [--iterations 3] [--out artifacts/repro-cold-touch] +// +// Results are written as JSON plus a table on stdout and $GITHUB_STEP_SUMMARY. +// --expect prompt or reproduced turns observations into an explicit validation gate. + +import { spawn, spawnSync } from 'node:child_process'; +import { + appendFileSync, + existsSync, + mkdirSync, + writeFileSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const args = new Map(); +for (let index = 2; index < process.argv.length; index += 2) + args.set(process.argv[index].replace(/^--/, ''), process.argv[index + 1]); +const runtimeVersion = args.get('runtime'); +if (runtimeVersion == null) + throw new Error( + 'Usage: node scripts/repro-cold-simulator-touch.mjs --runtime [--iterations N] [--out DIR]' + ); +const iterations = Number(args.get('iterations') ?? 3); +const expectation = args.get('expect') ?? 'observe'; +if (!['observe', 'prompt', 'reproduced'].includes(expectation)) + throw new Error('Expected --expect observe, prompt, or reproduced'); +if (!Number.isInteger(iterations) || iterations < 1 || iterations > 20) + throw new Error('Iterations must be an integer from 1 to 20'); +const outRoot = resolve(args.get('out') ?? 'artifacts/repro-cold-touch'); +const deviceName = args.get('device') ?? 'iPhone 17 Pro'; +const appPath = resolve( + 'example/ios/build/Build/Products/Release-iphonesimulator/ReorderableExample.app' +); +const bundleId = 'reorderable.example'; +const deepLink = 'reorderable://lab/free-form?preset=teaching&engine=auto'; +const initialOrder = + 'Current order: card-0, card-1, card-2, card-3, card-4, card-5'; +// The contract drag (card-0 before card-3). The follow-up drag on the same app +// instance moves the last card to the top so it always changes the order +// whatever the first drag did. +const contractDrag = { source: 'id="card-card-0"', destination: 'id="card-card-3"' }; +const followUpDrag = { source: 'id="card-card-5"', destination: 'id="card-card-1"' }; +const timing = { sourceHoldMs: 650, moveMs: 1200, destinationHoldMs: 8000 }; +const agentDevice = + process.env.AGENT_DEVICE_BIN ?? resolve('node_modules/.bin/agent-device'); +const sessionName = 'repro-cold-touch'; +// Classify the observable result after the gesture command returns. This is +// post-command observation wait, not the timestamp of the first delivered touch. +// The separate runner/backboardd logs are needed to attribute an input delay. +const LATE_DELIVERY_THRESHOLD_MS = 3000; + +if (!existsSync(appPath)) throw new Error(`Missing app build at ${appPath}`); +mkdirSync(outRoot, { recursive: true }); + +const now = () => Date.now(); +const iso = () => new Date().toISOString(); +const log = (message) => console.log(`[${iso()}] ${message}`); + +function run(command, commandArgs, options = {}) { + const startedAt = now(); + const result = spawnSync(command, commandArgs, { + encoding: 'utf8', + timeout: options.timeoutMs ?? 120000, + env: options.env ?? process.env, + maxBuffer: 64 * 1024 * 1024, + }); + return { + command: [command, ...commandArgs].join(' '), + status: result.status, + signal: result.signal, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + durationMs: now() - startedAt, + error: result.error?.message, + }; +} + +function resolveTargetUdid() { + const devices = run('xcrun', ['simctl', 'list', 'devices', 'available', '-j']); + if (devices.status !== 0) throw new Error('simctl list failed'); + const runtime = Object.entries(JSON.parse(devices.stdout).devices).find( + ([key]) => key.includes(runtimeVersion) + )?.[1]; + const target = runtime?.find(({ name }) => name === deviceName); + if (target == null) + throw new Error(`${runtimeVersion} ${deviceName} simulator is unavailable`); + return target.udid; +} + +function coldReset(udid) { + const steps = {}; + run('xcrun', ['simctl', 'shutdown', udid], { timeoutMs: 120000 }); + steps.erase = run('xcrun', ['simctl', 'erase', udid], { timeoutMs: 180000 }); + if (steps.erase.status !== 0) throw new Error(`simctl erase failed: ${steps.erase.stderr}`); + steps.boot = run('xcrun', ['simctl', 'boot', udid], { timeoutMs: 180000 }); + steps.bootstatus = run('xcrun', ['simctl', 'bootstatus', udid, '-b'], { + timeoutMs: 480000, + }); + if (steps.bootstatus.status !== 0) + throw new Error(`simctl bootstatus failed: ${steps.bootstatus.stderr}`); + steps.install = run('xcrun', ['simctl', 'install', udid, appPath], { + timeoutMs: 180000, + }); + if (steps.install.status !== 0) + throw new Error(`simctl install failed: ${steps.install.stderr}`); + return Object.fromEntries( + Object.entries(steps).map(([name, result]) => [ + name, + { status: result.status, durationMs: result.durationMs }, + ]) + ); +} + +function startRecording(udid, path) { + const recorder = spawn( + 'xcrun', + ['simctl', 'io', udid, 'recordVideo', '--codec=h264', '--force', path], + { stdio: ['ignore', 'ignore', 'pipe'] } + ); + recorder.stderr.setEncoding('utf8'); + let started = false; + let exited = false; + recorder.stderr.on('data', (chunk) => { + if (chunk.includes('Recording started')) started = true; + }); + recorder.once('close', () => { + exited = true; + }); + const startedAt = now(); + return { + waitStarted: async () => { + while (!started && !exited && now() - startedAt < 30000) + await new Promise((resolveSleep) => setTimeout(resolveSleep, 200)); + return started; + }, + stop: async () => { + if (exited) return; + recorder.kill('SIGINT'); + const deadline = now() + 20000; + while (!exited && now() < deadline) + await new Promise((resolveSleep) => setTimeout(resolveSleep, 200)); + if (!exited) recorder.kill('SIGKILL'); + }, + }; +} + +async function iteration(index, udid) { + const iterationRoot = resolve(outRoot, `iteration-${index}`); + const stateDir = resolve(iterationRoot, 'agent-device-state'); + mkdirSync(stateDir, { recursive: true }); + const environment = { ...process.env, AGENT_DEVICE_STATE_DIR: stateDir }; + const defaultEnvironment = { ...process.env }; + delete defaultEnvironment.AGENT_DEVICE_STATE_DIR; + const record = { index, startedAt: iso(), udid, runtimeVersion, steps: {} }; + const commands = []; + const ad = (commandArgs, options = {}) => { + const result = run(agentDevice, commandArgs, { + env: environment, + timeoutMs: options.timeoutMs ?? 240000, + }); + commands.push({ + at: iso(), + args: commandArgs, + status: result.status, + durationMs: result.durationMs, + stdout: result.stdout.slice(0, 4000), + stderr: result.stderr.slice(0, 4000), + }); + return result; + }; + const session = (commandArgs, options) => + ad( + [...commandArgs, '--platform', 'ios', '--session', sessionName, '--udid', udid], + options + ); + const stopDaemons = () => { + if (process.env.CI === 'true') + run(agentDevice, ['daemon', 'stop'], { env: defaultEnvironment }); + run(agentDevice, ['daemon', 'stop'], { env: environment }); + }; + + let recorder; + try { + log(`iteration ${index}: cold reset of ${udid}`); + record.steps.coldReset = coldReset(udid); + stopDaemons(); + + log(`iteration ${index}: prepare ios-runner`); + const prepare = session( + ['prepare', 'ios-runner', '--timeout', '300000'], + { timeoutMs: 360000 } + ); + record.steps.prepare = { status: prepare.status, durationMs: prepare.durationMs }; + if (prepare.status !== 0) throw new Error(`prepare ios-runner exited ${prepare.status}`); + + // Same alert-seeding dance as the device-contract preflight: the first deep + // link after an erase shows the URL confirmation alert. + log(`iteration ${index}: launch and deep link`); + // `alert accept` can hit RUNNER_BUSY right after the deep link on a slow + // host; retry it, because an unaccepted URL confirmation blocks the scenario. + const acceptAlert = () => { + let result; + for (let attempt = 1; attempt <= 4; attempt += 1) { + result = session(['alert', 'accept']); + if (result.status === 0 || !result.stderr.includes('RUNNER_BUSY')) break; + spawnSync('sleep', ['3']); + } + return result; + }; + // Bring the scenario to its start state. A cold hosted host can hiccup on an + // individual `open`/`wait` (xcrun timeout, unrendered deep link) that a + // relaunch clears; retry the whole approach a few times so a transient setup + // hiccup does not burn an iteration that would otherwise measure the gesture. + const runSetup = () => { + const steps = [ + () => session(['open', bundleId, '--relaunch']), + () => session(['wait', 'Scenario Lab', '30000', '--depth', '100']), + () => session(['open', deepLink]), + acceptAlert, + () => session(['open', bundleId, '--relaunch']), + () => session(['wait', 'Scenario Lab', '30000', '--depth', '100']), + () => session(['open', deepLink]), + () => session(['wait', initialOrder, '30000', '--depth', '100']), + () => session(['wait', 'Callback count: 0', '15000', '--depth', '100']), + ]; + for (const [position, step] of steps.entries()) { + const result = step(); + // Absence is expected after the confirmation has already been accepted. + // A timeout can leave the URL sheet over matching text in the scenario. + const absentAlert = position === 3 && result.stderr.includes('alert not found'); + if (result.status !== 0 && !absentAlert) return result; + } + return undefined; + }; + let setupFailure = runSetup(); + for (let attempt = 2; attempt <= 3 && setupFailure != null; attempt += 1) { + log(`iteration ${index}: scenario setup retry ${attempt}`); + setupFailure = runSetup(); + } + record.steps.setup = { + ok: setupFailure == null, + failure: setupFailure?.stderr.slice(0, 500), + }; + if (setupFailure != null) throw new Error(`scenario setup failed: ${setupFailure.stderr}`); + + recorder = startRecording(udid, resolve(iterationRoot, 'screen.mp4')); + record.steps.recordingStarted = await recorder.waitStarted(); + + let deliveredCount = 0; + const gestureAttempt = (name, drag) => { + const expected = `Callback count: ${deliveredCount + 1}`; + const startedAt = iso(); + const gesture = session( + [ + 'gesture', + 'drag', + drag.source, + drag.destination, + String(timing.sourceHoldMs), + String(timing.moveMs), + String(timing.destinationHoldMs), + '--json', + ], + { timeoutMs: 120000 } + ); + let gestureJson = null; + try { + gestureJson = JSON.parse(gesture.stdout); + } catch {} + const wait = session(['wait', expected, '15000', '--depth', '100', '--json']); + let waitJson = null; + try { + waitJson = JSON.parse(wait.stdout); + } catch {} + const delivered = wait.status === 0; + if (delivered) deliveredCount += 1; + // These are observation classes, not touch-transport measurements: + // lost no expected effect observed before the wait failed + // late effect observed after the post-command wait threshold + // prompt effect observed within that threshold + // errored gesture command failed before its outcome could be measured + // observation-error the observer failed without establishing absence + // Inspect the recording/runner trace to distinguish delayed input from + // slow app or accessibility processing. + const gestureReportedOk = gestureJson?.ok ?? gestureJson?.success ?? null; + const deliveryClass = + gesture.status !== 0 + ? 'errored' + : delivered + ? wait.durationMs > LATE_DELIVERY_THRESHOLD_MS + ? 'late' + : 'prompt' + : waitJson?.error?.details?.reason === 'wait_target_absent' + ? 'lost' + : 'observation-error'; + const attempt = { + name, + startedAt, + gestureExit: gesture.status, + gestureDurationMs: gesture.durationMs, + gestureReportedOk, + gestureReportedDurationMs: + gestureJson?.data?.durationMs ?? gestureJson?.durationMs ?? null, + delivered, + deliveryClass, + waitDurationMs: wait.durationMs, + waitError: waitJson?.error ?? null, + expected, + gestureStderr: gesture.stderr.slice(0, 1000), + }; + log( + `iteration ${index}: ${name} class=${deliveryClass} reportedOk=${gestureReportedOk} appDelivered=${delivered} (${wait.durationMs} ms)` + ); + return attempt; + }; + + record.firstGesture = gestureAttempt('first-gesture-after-cold-boot', contractDrag); + record.probes = {}; + record.probes.secondGesture = gestureAttempt('second-gesture-same-app-instance', followUpDrag); + + const relaunch = [ + session(['open', bundleId, '--relaunch']), + session(['wait', 'Scenario Lab', '30000', '--depth', '100']), + session(['open', deepLink]), + session(['wait', initialOrder, '30000', '--depth', '100']), + ]; + if (relaunch.every((result) => result.status === 0)) { + deliveredCount = 0; + record.probes.relaunchGesture = gestureAttempt('gesture-after-app-relaunch', contractDrag); + } else { + record.probes.relaunchGesture = { skipped: 'relaunch failed' }; + } + + const tap = session(['press', 'id="engine-fallback"']); + // The deep-link text in the scenario header switches to engine=fallback. + const tapWait = session(['wait', 'text', 'engine=fallback', '10000', '--depth', '100']); + record.probes.selectorTap = { + pressExit: tap.status, + delivered: tapWait.status === 0, + waitDurationMs: tapWait.durationMs, + }; + log(`iteration ${index}: selector press delivered=${record.probes.selectorTap.delivered}`); + } catch (error) { + record.error = error.message; + log(`iteration ${index}: ${error.message}`); + } finally { + if (recorder != null) await recorder.stop(); + const simulatorLog = run( + 'xcrun', + [ + 'simctl', + 'spawn', + udid, + 'log', + 'show', + '--last', + '8m', + '--style', + 'compact', + '--predicate', + 'process == "backboardd" OR process == "SpringBoard" OR process == "ReorderableExample"', + ], + { timeoutMs: 120000 } + ); + writeFileSync( + resolve(iterationRoot, 'simulator-log.txt'), + (simulatorLog.stdout || simulatorLog.stderr || '').slice(0, 20 * 1024 * 1024) + ); + const hostLog = run( + 'log', + [ + 'show', + '--last', + '8m', + '--style', + 'compact', + '--predicate', + 'process == "testmanagerd" OR process == "CoreSimulatorService" OR process == "SimulatorTrampoline"', + ], + { timeoutMs: 120000 } + ); + writeFileSync( + resolve(iterationRoot, 'host-log.txt'), + (hostLog.stdout || hostLog.stderr || '').slice(0, 20 * 1024 * 1024) + ); + session(['close']); + stopDaemons(); + record.finishedAt = iso(); + record.commands = commands; + writeFileSync( + resolve(iterationRoot, 'iteration.json'), + `${JSON.stringify(record, null, 2)}\n` + ); + } + return record; +} + +const udid = resolveTargetUdid(); +const records = []; +for (let index = 1; index <= iterations; index += 1) + records.push(await iteration(index, udid)); + +const summaryRows = records.map((record) => ({ + iteration: record.index, + bootMs: record.steps.coldReset?.bootstatus?.durationMs ?? null, + prepareMs: record.steps.prepare?.durationMs ?? null, + firstClass: record.firstGesture?.deliveryClass ?? null, + firstWaitMs: record.firstGesture?.waitDurationMs ?? null, + secondClass: record.probes?.secondGesture?.deliveryClass ?? null, + relaunchClass: record.probes?.relaunchGesture?.deliveryClass ?? null, + // A later selector press may use the same private synthesis bridge. It is + // neither a public-XCTest control nor proof against an earlier input stall. + selectorTapDelivered: record.probes?.selectorTap?.delivered ?? null, + error: record.error ?? null, +})); +// Every measured synthesized-gesture wait across the run (first, second, +// relaunch), for the post-command observation-wait distribution. +const gestureWaits = records + .flatMap((record) => [ + record.firstGesture, + record.probes?.secondGesture, + record.probes?.relaunchGesture, + ]) + .filter((attempt) => attempt != null && !['errored', 'observation-error'].includes(attempt.deliveryClass) && + typeof attempt.waitDurationMs === 'number') + .map((attempt) => attempt.waitDurationMs) + .sort((a, b) => a - b); +const percentile = (values, fraction) => + values.length === 0 + ? null + : values[Math.min(values.length - 1, Math.floor(values.length * fraction))]; +const gesturesMeasured = summaryRows.filter((row) => row.firstClass != null && !['errored', 'observation-error'].includes(row.firstClass)).length; +const lost = summaryRows.filter((row) => row.firstClass === 'lost').length; +const late = summaryRows.filter((row) => row.firstClass === 'late').length; +// Count the observed symptom. This alone does not establish a transport defect. +const reproduced = lost + late; +const summary = { + runtimeVersion, + deviceName, + udid, + iterations, + gesturesMeasured, + firstGestureLost: lost, + firstGestureLate: late, + reproduced, + postCommandObservationWaitMs: { + count: gestureWaits.length, + min: gestureWaits[0] ?? null, + median: percentile(gestureWaits, 0.5), + p90: percentile(gestureWaits, 0.9), + max: gestureWaits[gestureWaits.length - 1] ?? null, + }, + rows: summaryRows, +}; +writeFileSync(resolve(outRoot, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`); +const latency = summary.postCommandObservationWaitMs; +const table = [ + `### Cold-simulator touch reproduction (${runtimeVersion}, ${deviceName})`, + '', + `First gesture outcome unobserved or observed after the wait threshold: **${reproduced}/${gesturesMeasured}** measured iterations (lost ${lost}, late ${late}).`, + '', + 'Classes describe outcome observation, not actual touch-delivery timing. Consult the recording and runner trace.', + '', + `Post-command observation wait across ${latency.count} synthesized gestures: min ${latency.min} ms, median ${latency.median} ms, p90 ${latency.p90} ms, max ${latency.max} ms.`, + '', + '| # | boot ms | prepare ms | 1st gesture | 1st wait ms | 2nd gesture | relaunch gesture | Selector press delivered | error |', + '| --- | --- | --- | --- | --- | --- | --- | --- | --- |', + ...summaryRows.map( + (row) => + `| ${row.iteration} | ${row.bootMs ?? ''} | ${row.prepareMs ?? ''} | ${row.firstClass ?? ''} | ${row.firstWaitMs ?? ''} | ${row.secondClass ?? ''} | ${row.relaunchClass ?? ''} | ${row.selectorTapDelivered} | ${row.error ? row.error.split('\n')[0].slice(0, 80) : ''} |` + ), + '', +].join('\n'); +console.log(table); +if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${table}\n`); + +// An observation run preserves exploratory evidence. Validation must prove all +// iterations reached the gesture and cannot turn setup failures into green. +if (expectation !== 'observe') { + const complete = summaryRows.every( + (row) => row.error == null && row.firstClass != null && !['errored', 'observation-error'].includes(row.firstClass) + ); + const matches = + expectation === 'reproduced' + ? reproduced > 0 + : summaryRows.every( + (row) => + row.firstClass === 'prompt' && + row.secondClass === 'prompt' && + row.relaunchClass === 'prompt' && + row.selectorTapDelivered === true + ); + if (!complete || !matches) process.exitCode = 1; +} diff --git a/scripts/run-agent-device-pointer.mjs b/scripts/run-agent-device-pointer.mjs index 942c299..b5e2a89 100644 --- a/scripts/run-agent-device-pointer.mjs +++ b/scripts/run-agent-device-pointer.mjs @@ -96,7 +96,8 @@ if (platform === 'android') { const sessionName = `issue39-${scenarioId}`; const sessionDeviceArgs = platform === 'ios' ? ['--udid', targetId] : []; -const agentDevice = resolve('node_modules/.bin/agent-device'); +const agentDevice = + process.env.AGENT_DEVICE_BIN ?? resolve('node_modules/.bin/agent-device'); const recordingDirectory = resolve( process.env.ISSUE39_AGENT_DEVICE_DIR ?? `artifacts/issue-39/agent-device/${configuration}/${scenarioId}` diff --git a/scripts/run-device-contract-isolated.mjs b/scripts/run-device-contract-isolated.mjs index ddcdf95..7d721ce 100644 --- a/scripts/run-device-contract-isolated.mjs +++ b/scripts/run-device-contract-isolated.mjs @@ -43,13 +43,17 @@ const replayStateRoot = resolve( 'replay-daemon-state' ); const stopReplayDaemon = () => - spawnSync(resolve('node_modules/.bin/agent-device'), ['daemon', 'stop'], { - env: { - ...process.env, - AGENT_DEVICE_STATE_DIR: replayStateRoot, - }, - stdio: 'inherit', - }); + spawnSync( + process.env.AGENT_DEVICE_BIN ?? resolve('node_modules/.bin/agent-device'), + ['daemon', 'stop'], + { + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: replayStateRoot, + }, + stdio: 'inherit', + } + ); const hasObservedScenarioOutcome = (outcomeFile, scenarioId) => { if (!existsSync(outcomeFile)) return false; try { diff --git a/src/__tests__/issue39-device-contract.test.ts b/src/__tests__/issue39-device-contract.test.ts index ab70795..11876af 100644 --- a/src/__tests__/issue39-device-contract.test.ts +++ b/src/__tests__/issue39-device-contract.test.ts @@ -120,14 +120,12 @@ describe('issue 39 portable device contract', () => { destination: { normalizedX: 0.5, normalizedY: 0.25 }, driver: 'detox', }); - expect(pointerDrivers.routes['ios26.auto-fallback']).toEqual({ - 'free-form-reorder': 'detox', - 'virtualized-list-reorder': 'detox', - 'section-list-reorder': 'detox', - 'multi-selection-reorder': 'detox', - 'scoped-drop': 'detox', - }); - expect(pointerDrivers.overrides['ios26.auto-fallback']).toBeUndefined(); + expect(pointerDrivers.routes['ios26.auto-fallback']).toEqual( + pointerDrivers.routes['ios27.fallback'] + ); + expect(pointerDrivers.overrides['ios26.auto-fallback']).toEqual( + pointerDrivers.overrides['ios27.fallback'] + ); expect(pointerDrivers.routes['android.fallback']).toEqual({ 'free-form-reorder': 'detox', 'virtualized-list-reorder': 'detox', @@ -863,9 +861,6 @@ describe('issue 39 portable device contract', () => { /runAgentDevice\(\s*'prepare',\s*'ios-runner'/ ); expect(iosRunnerPreflight).toMatch(/'--timeout',\s*'300000'/); - expect(iosRunnerPreflight).toContain( - "runSessionCommand('alert', 'accept')" - ); expect(iosRunnerPreflight).toContain( "runSessionCommand('open', 'reorderable.example', '--relaunch')" ); diff --git a/src/__tests__/prepare-agent-device-ios-runner.test.ts b/src/__tests__/prepare-agent-device-ios-runner.test.ts new file mode 100644 index 0000000..c4d54b2 --- /dev/null +++ b/src/__tests__/prepare-agent-device-ios-runner.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from '@jest/globals'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +const preflight = resolve('scripts/prepare-agent-device-ios-runner.mjs'); + +function runPreflight(alert: 'accepted' | 'absent' | 'busy') { + const directory = mkdtempSync(join(tmpdir(), 'ios-preflight-')); + const callsPath = join(directory, 'calls.json'); + const agentPath = join(directory, 'agent-device'); + writeFileSync( + join(directory, 'xcrun'), + `#!/usr/bin/env node +console.log(JSON.stringify({ devices: { 'iOS-26-5': [{ name: 'iPhone 17 Pro', udid: 'fixture-device' }] } })); +`, + { mode: 0o755 } + ); + writeFileSync( + agentPath, + `#!/usr/bin/env node +const fs = require('node:fs'); +const callsPath = process.env.PREFLIGHT_CALLS; +const args = process.argv.slice(2); +const calls = fs.existsSync(callsPath) ? JSON.parse(fs.readFileSync(callsPath, 'utf8')) : []; +calls.push(args); +fs.writeFileSync(callsPath, JSON.stringify(calls)); +if (args[0] === 'alert') { + const busy = process.env.PREFLIGHT_ALERT === 'busy'; + const absent = process.env.PREFLIGHT_ALERT === 'absent'; + if (busy || absent) { + console.log(JSON.stringify({ error: { + code: busy ? 'RUNNER_BUSY' : 'COMMAND_FAILED', + message: 'alert not found', + details: { runnerErrorCode: busy ? 'RUNNER_BUSY' : 'ALERT_NOT_FOUND' } + } })); + process.exit(1); + } + console.log(JSON.stringify({ success: true })); +} +if (args[0] === 'wait' && args[1].startsWith('Current order') && process.env.PREFLIGHT_ALERT === 'accepted') { + const lastLink = calls.findLastIndex(call => call[0] === 'open' && call[1].startsWith('reorderable://')); + if (!calls.slice(lastLink + 1).some(call => call[0] === 'alert' && call[1] === 'accept')) process.exit(1); +} +`, + { mode: 0o755 } + ); + try { + const result = spawnSync( + process.execPath, + [preflight, 'ios26.auto-fallback'], + { + cwd: directory, + encoding: 'utf8', + timeout: 15000, + env: { + ...process.env, + CI: 'false', + PATH: `${directory}:${process.env.PATH}`, + AGENT_DEVICE_BIN: agentPath, + PREFLIGHT_CALLS: callsPath, + PREFLIGHT_ALERT: alert, + }, + } + ); + return { + status: result.status, + calls: JSON.parse(readFileSync(callsPath, 'utf8')) as string[][], + confirmed: existsSync( + join( + directory, + 'artifacts/issue-39/agent-device/ios26.auto-fallback/replay-daemon-state/deep-link-confirmed' + ) + ), + }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +describe('iOS runner preflight confirmation', () => { + it('confirms one deep-link open before verifying the app outcome', () => { + const result = runPreflight('accepted'); + expect(result.status).toBe(0); + expect(result.confirmed).toBe(true); + expect( + result.calls.filter((call) => call[0] === 'alert').map((call) => call[1]) + ).toEqual(['get', 'accept']); + expect( + result.calls.filter( + (call) => call[0] === 'open' && call[1]?.startsWith('reorderable://') + ) + ).toHaveLength(1); + }); + + it('allows a previously confirmed URL when the runner explicitly reports no alert', () => { + const result = runPreflight('absent'); + expect(result.status).toBe(0); + expect(result.confirmed).toBe(true); + expect( + result.calls.filter((call) => call[0] === 'alert').map((call) => call[1]) + ).toEqual(['get']); + }); + + it('stops on runner contention even when the error text says alert not found', () => { + const result = runPreflight('busy'); + expect(result.status).not.toBe(0); + expect(result.confirmed).toBe(false); + expect( + result.calls.some( + (call) => call[0] === 'wait' && call[1]?.startsWith('Current order') + ) + ).toBe(false); + expect(result.calls.at(-1)?.slice(0, 2)).toEqual(['daemon', 'stop']); + }); +}); diff --git a/yarn.lock b/yarn.lock index 571b16f..66a810b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2644,20 +2644,6 @@ __metadata: languageName: node linkType: hard -"@limrun/api@npm:^0.24.5": - version: 0.24.5 - resolution: "@limrun/api@npm:0.24.5" - dependencies: - eventsource-client: "npm:^1.2.0" - https-proxy-agent: "npm:7.0.6" - ignore: "npm:^7.0.5" - proxy-from-env: "npm:^2.1.0" - undici: "npm:7.24.7" - ws: "npm:^8.18.3" - checksum: 10c0/2369e26f9c4aa25c800e011ae86dd450aee944723434832f1342af1e3bc7eb885c2d75f5e6ec48d2dad8f9121ef610b64c2394842008f714f3d9a244b615a6ef - languageName: node - linkType: hard - "@manypkg/find-root@npm:^3.1.0": version: 3.1.0 resolution: "@manypkg/find-root@npm:3.1.0" @@ -4268,16 +4254,9 @@ __metadata: languageName: node linkType: hard -"agent-device@npm:0.20.10": - version: 0.20.10 - resolution: "agent-device@npm:0.20.10" - dependencies: - "@limrun/api": "npm:^0.24.5" - ipaddr.js: "npm:^2.5.0" - tar-stream: "npm:^3.2.0" - undici: "npm:7.29.0" - yaml: "npm:^2.9.0" - yauzl: "npm:^3.4.0" +"agent-device@npm:0.21.0": + version: 0.21.0 + resolution: "agent-device@npm:0.21.0" peerDependencies: ai: ^6.0.0 || ^7.0.0 peerDependenciesMeta: @@ -4285,7 +4264,7 @@ __metadata: optional: true bin: agent-device: bin/agent-device.mjs - checksum: 10c0/bbac1e4c6c445363b7680f24765da3073b6b63657811c8257b6fa79319d520800f1af8baf9a2a2e317a89a1ec7ee3d3178f7a08c4f745b3209c3b66b349be8bd + checksum: 10c0/020fd5cb0b5463a7c1a6238ba1c0a26517371acfd32bb1c45c729684bf71babaf207156197f34062ac75d92354fb1ff48e157b49fee50185c8f98cbba685b2ce languageName: node linkType: hard @@ -4605,18 +4584,6 @@ __metadata: languageName: node linkType: hard -"b4a@npm:^1.6.4, b4a@npm:^1.8.1": - version: 1.8.1 - resolution: "b4a@npm:1.8.1" - peerDependencies: - react-native-b4a: "*" - peerDependenciesMeta: - react-native-b4a: - optional: true - checksum: 10c0/344d8c94b244ec7a9cb516ea43a98216312454cb72478e4b7628a679ee343be237564c53bbe73995ab10ea9bc923b420236081b180b3cf78fd0c945bfc886798 - languageName: node - linkType: hard - "babel-jest@npm:^29.7.0": version: 29.7.0 resolution: "babel-jest@npm:29.7.0" @@ -4792,74 +4759,6 @@ __metadata: languageName: node linkType: hard -"bare-events@npm:^2.5.4, bare-events@npm:^2.7.0": - version: 2.9.2 - resolution: "bare-events@npm:2.9.2" - peerDependencies: - bare-abort-controller: "*" - peerDependenciesMeta: - bare-abort-controller: - optional: true - checksum: 10c0/1aff2704e69a0bde978d6e768ebc13e8c21f36bee436b82e98a15f614d2f3b15ed6c894ffa0cfee2ba31baac17ccf9ea6cbe2c48d112a2fc5f6e42ed84f9d31a - languageName: node - linkType: hard - -"bare-fs@npm:^4.5.5": - version: 4.8.1 - resolution: "bare-fs@npm:4.8.1" - dependencies: - bare-events: "npm:^2.5.4" - bare-path: "npm:^3.0.0" - bare-stream: "npm:^2.6.4" - bare-url: "npm:^2.2.2" - fast-fifo: "npm:^1.3.2" - peerDependencies: - bare-buffer: "*" - peerDependenciesMeta: - bare-buffer: - optional: true - checksum: 10c0/1262941ce4e8853b5c16d9a9b8970eebcf2043fda48d17512b6b697941d82b513ed50b514b956c7a80d938071fc55e2c5b6c4783d4f51b2fb04fc3c6fef0b28c - languageName: node - linkType: hard - -"bare-path@npm:^3.0.0": - version: 3.1.1 - resolution: "bare-path@npm:3.1.1" - checksum: 10c0/833f0fe2df164e518878b9c71646228d7841c2e35bd3fc428e432b512317ce7fdbd2a127649366f34db3312778d12a83f2fff5485004bbbdcdd15975fce73f4c - languageName: node - linkType: hard - -"bare-stream@npm:^2.6.4": - version: 2.13.4 - resolution: "bare-stream@npm:2.13.4" - dependencies: - b4a: "npm:^1.8.1" - streamx: "npm:^2.25.0" - teex: "npm:^1.0.1" - peerDependencies: - bare-abort-controller: "*" - bare-buffer: "*" - bare-events: "*" - peerDependenciesMeta: - bare-abort-controller: - optional: true - bare-buffer: - optional: true - bare-events: - optional: true - checksum: 10c0/df8e4567cb6721b2e743762e82d4a15f0a1c28c0f0ace3b0fd1e9cc835ffee29880e624dcc0f3a32bcaa629d2a4edfad44dd15abbe514dae5511cb194df8f8d7 - languageName: node - linkType: hard - -"bare-url@npm:^2.2.2": - version: 2.5.2 - resolution: "bare-url@npm:2.5.2" - dependencies: - bare-path: "npm:^3.0.0" - checksum: 10c0/f88871df80b733423434e7917f3cd35b1b89e2bf46a2bf305981d4c6627ca4c34ba2147920f217739bd6c01c684ae30a827911c419b640645eadbd91a28881ea - languageName: node - linkType: hard - "base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" @@ -6762,31 +6661,6 @@ __metadata: languageName: node linkType: hard -"events-universal@npm:^1.0.0": - version: 1.0.1 - resolution: "events-universal@npm:1.0.1" - dependencies: - bare-events: "npm:^2.7.0" - checksum: 10c0/a1d9a5e9f95843650f8ec240dd1221454c110189a9813f32cdf7185759b43f1f964367ac7dca4ebc69150b59043f2d77c7e122b0d03abf7c25477ea5494785a5 - languageName: node - linkType: hard - -"eventsource-client@npm:^1.2.0": - version: 1.2.0 - resolution: "eventsource-client@npm:1.2.0" - dependencies: - eventsource-parser: "npm:^3.0.0" - checksum: 10c0/32235ff9b0cb7e3f0f0dc0316a97d543470ad6574acb89a4665ca900e325ff2ca9bb19581d6c3c49a3b91b6ce55024856cfd3b191be872a91f821c924096d5aa - languageName: node - linkType: hard - -"eventsource-parser@npm:^3.0.0": - version: 3.1.0 - resolution: "eventsource-parser@npm:3.1.0" - checksum: 10c0/5ab4c6c9a2a042be0b387b6d03810eb580bac4ce90e299ede56458125a97ffe3af8145b2740089fc898a96cfa5aae792ee79f2a06257fba2776b0e7bce037071 - languageName: node - linkType: hard - "execa@npm:^5.0.0, execa@npm:^5.1.1": version: 5.1.1 resolution: "execa@npm:5.1.1" @@ -6859,13 +6733,6 @@ __metadata: languageName: node linkType: hard -"fast-fifo@npm:^1.2.0, fast-fifo@npm:^1.3.2": - version: 1.3.2 - resolution: "fast-fifo@npm:1.3.2" - checksum: 10c0/d53f6f786875e8b0529f784b59b4b05d4b5c31c651710496440006a398389a579c8dbcd2081311478b5bf77f4b0b21de69109c5a4eabea9d8e8783d1eb864e4c - languageName: node - linkType: hard - "fast-glob@npm:^3.3.2, fast-glob@npm:^3.3.3": version: 3.3.3 resolution: "fast-glob@npm:3.3.3" @@ -7747,7 +7614,7 @@ __metadata: languageName: node linkType: hard -"https-proxy-agent@npm:7.0.6, https-proxy-agent@npm:^7.0.5": +"https-proxy-agent@npm:^7.0.5": version: 7.0.6 resolution: "https-proxy-agent@npm:7.0.6" dependencies: @@ -7915,13 +7782,6 @@ __metadata: languageName: node linkType: hard -"ipaddr.js@npm:^2.5.0": - version: 2.5.0 - resolution: "ipaddr.js@npm:2.5.0" - checksum: 10c0/5cef50d91a14f6388c3b5df3568ee7f8b18e07fa18896debc84d3393b78c1096e1b8e3a4f9fa9020c5ca5b556efc28550432f92a34d23c0cd643bd3194eb439d - languageName: node - linkType: hard - "is-absolute-url@npm:^4.0.0": version: 4.0.1 resolution: "is-absolute-url@npm:4.0.1" @@ -11196,13 +11056,6 @@ __metadata: languageName: node linkType: hard -"pend@npm:~1.2.0": - version: 1.2.0 - resolution: "pend@npm:1.2.0" - checksum: 10c0/8a87e63f7a4afcfb0f9f77b39bb92374afc723418b9cb716ee4257689224171002e07768eeade4ecd0e86f1fa3d8f022994219fb45634f2dbd78c6803e452458 - languageName: node - linkType: hard - "picocolors@npm:^1.1.1": version: 1.1.1 resolution: "picocolors@npm:1.1.1" @@ -11378,13 +11231,6 @@ __metadata: languageName: node linkType: hard -"proxy-from-env@npm:^2.1.0": - version: 2.1.0 - resolution: "proxy-from-env@npm:2.1.0" - checksum: 10c0/ed01729fd4d094eab619cd7e17ce3698b3413b31eb102c4904f9875e677cd207392795d5b4adee9cec359dfd31c44d5ad7595a3a3ad51c40250e141512281c58 - languageName: node - linkType: hard - "punycode@npm:^2.1.0": version: 2.3.1 resolution: "punycode@npm:2.3.1" @@ -11627,7 +11473,7 @@ __metadata: "@shopify/flash-list": "npm:2.3.2" "@testing-library/react-native": "npm:^14.0.1" "@types/react": "npm:^19.2.0" - agent-device: "npm:0.20.10" + agent-device: "npm:0.21.0" commitlint: "npm:^21.0.2" del-cli: "npm:^7.0.0" detox: "npm:20.51.4" @@ -12818,17 +12664,6 @@ __metadata: languageName: node linkType: hard -"streamx@npm:^2.12.5, streamx@npm:^2.15.0, streamx@npm:^2.25.0": - version: 2.28.1 - resolution: "streamx@npm:2.28.1" - dependencies: - events-universal: "npm:^1.0.0" - fast-fifo: "npm:^1.3.2" - text-decoder: "npm:^1.1.0" - checksum: 10c0/90871d76712ad9e00f555ffcb0d33bcbf6823cc2237e8868dcb30666ec76ae34157049042e3cd3251698278ebedf153291026eec7c87ad05c746178a35405f64 - languageName: node - linkType: hard - "string-length@npm:^4.0.1": version: 4.0.2 resolution: "string-length@npm:4.0.2" @@ -13082,18 +12917,6 @@ __metadata: languageName: node linkType: hard -"tar-stream@npm:^3.2.0": - version: 3.2.1 - resolution: "tar-stream@npm:3.2.1" - dependencies: - b4a: "npm:^1.6.4" - bare-fs: "npm:^4.5.5" - fast-fifo: "npm:^1.2.0" - streamx: "npm:^2.15.0" - checksum: 10c0/0fef57b5e8bf15698f9b4bde4125ab690422cd98ff59cddd76addfb23a6999ad32bddbeeea0858e2756f8a0f36ef4e636a51efe5e8ef4c0a3dac4c2df5714858 - languageName: node - linkType: hard - "tar@npm:^7.5.4": version: 7.5.19 resolution: "tar@npm:7.5.19" @@ -13107,15 +12930,6 @@ __metadata: languageName: node linkType: hard -"teex@npm:^1.0.1": - version: 1.0.1 - resolution: "teex@npm:1.0.1" - dependencies: - streamx: "npm:^2.12.5" - checksum: 10c0/8df9166c037ba694b49d32a49858e314c60e513d55ac5e084dbf1ddbb827c5fa43cc389a81e87684419c21283308e9d68bb068798189c767ec4c252f890b8a77 - languageName: node - linkType: hard - "telnet-client@npm:1.2.8": version: 1.2.8 resolution: "telnet-client@npm:1.2.8" @@ -13162,15 +12976,6 @@ __metadata: languageName: node linkType: hard -"text-decoder@npm:^1.1.0": - version: 1.2.7 - resolution: "text-decoder@npm:1.2.7" - dependencies: - b4a: "npm:^1.6.4" - checksum: 10c0/929938ed154fbadb660a7f3d1aca30b7e53649a731af7583168fcfba0c158046325d35d945926e2a512bb62d1a49a7818151c987ea38b48853f01e1615722fc5 - languageName: node - linkType: hard - "text-hex@npm:1.0.x": version: 1.0.0 resolution: "text-hex@npm:1.0.0" @@ -13497,20 +13302,6 @@ __metadata: languageName: node linkType: hard -"undici@npm:7.24.7": - version: 7.24.7 - resolution: "undici@npm:7.24.7" - checksum: 10c0/779c67e81677324763ea00ea547ba74757472ebe2625d046d592434ee19d9d148fe0eaef7006c0185096614249ac0f179e7f559b202b518af8d587e9548559b6 - languageName: node - linkType: hard - -"undici@npm:7.29.0": - version: 7.29.0 - resolution: "undici@npm:7.29.0" - checksum: 10c0/85ea96e91e7f3de24de678cbbc54230fdd641a3b0643005f2aef044af1f1a2db8fbecb88b28e6fa78bb41ffeafda4479b57075b6b6ab92beac4b7b5ab53f9b10 - languageName: node - linkType: hard - "undici@npm:^8.4.1": version: 8.7.0 resolution: "undici@npm:8.7.0" @@ -14045,21 +13836,6 @@ __metadata: languageName: node linkType: hard -"ws@npm:^8.18.3": - version: 8.21.3 - resolution: "ws@npm:8.21.3" - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: ">=5.0.2" - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - checksum: 10c0/7b28dc2863ea0e2cece68d142a3eee90361021b73f750431e6d8076bb7dede5fdfdb75b3d29534b62f411261147b76b1bc80fb5cc63ab0aabd467280e85b22e0 - languageName: node - linkType: hard - "y18n@npm:^4.0.0": version: 4.0.3 resolution: "y18n@npm:4.0.3" @@ -14181,15 +13957,6 @@ __metadata: languageName: node linkType: hard -"yauzl@npm:^3.4.0": - version: 3.4.0 - resolution: "yauzl@npm:3.4.0" - dependencies: - pend: "npm:~1.2.0" - checksum: 10c0/17a98c42c0065e8af429eb8a61f7a0e4562181ed54080366b838f34f741b6829f167f804787c86b7646bb042707f35871739f053de0548285e405a2eae4da025 - languageName: node - linkType: hard - "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"