diff --git a/.changeset/remove-physical-performance-pipeline.md b/.changeset/remove-physical-performance-pipeline.md new file mode 100644 index 0000000..f6e11b0 --- /dev/null +++ b/.changeset/remove-physical-performance-pipeline.md @@ -0,0 +1,7 @@ +--- +--- + +Delete the physical performance CI workflow and the manual device +verification scripts (issue #75). No published-package behavior changes: +only repository tooling entries were removed from `package.json` +`scripts`. diff --git a/.github/workflows/physical-performance.yml b/.github/workflows/physical-performance.yml deleted file mode 100644 index cca63bf..0000000 --- a/.github/workflows/physical-performance.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: Minimum physical-device performance - -on: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -jobs: - ios: - name: Minimum physical iOS - runs-on: [self-hosted, macOS, physical-ios-minimum] - timeout-minutes: 120 - env: - AGENT_DEVICE_IOS_TEAM_ID: ${{ vars.ISSUE40_IOS_TEAM_ID }} - AGENT_DEVICE_IOS_BUNDLE_ID: ${{ vars.ISSUE40_IOS_RUNNER_BUNDLE_ID }} - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup - uses: ./.github/actions/setup - - - name: Build signed Release app - run: | - yarn prepare - ( - cd example - bundle install - bundle exec pod install --project-directory=ios - ) - xcodebuild \ - -workspace example/ios/ReorderableExample.xcworkspace \ - -scheme ReorderableExample \ - -configuration Release \ - -destination 'generic/platform=iOS' \ - -derivedDataPath example/ios/build-issue40 \ - "DEVELOPMENT_TEAM=$AGENT_DEVICE_IOS_TEAM_ID" \ - CODE_SIGN_STYLE=Automatic \ - -allowProvisioningUpdates \ - build - - - name: Install exact Release app - run: | - yarn agent-device reinstall \ - reorderable.example \ - example/ios/build-issue40/Build/Products/Release-iphoneos/ReorderableExample.app \ - --platform ios \ - --device '${{ vars.ISSUE40_IOS_DEVICE }}' - - - name: Collect paired physical measurements - run: | - yarn performance:physical:collect \ - --platform ios \ - --device '${{ vars.ISSUE40_IOS_DEVICE }}' \ - --model '${{ vars.ISSUE40_IOS_MODEL }}' \ - --os-version '${{ vars.ISSUE40_IOS_VERSION }}' \ - --minimum-supported true \ - --out artifacts/issue-40/ios.json - - - name: Upload retained iOS evidence - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: issue-40-ios - if-no-files-found: error - retention-days: 30 - path: artifacts/issue-40/ - - android: - name: Minimum physical Android - runs-on: [self-hosted, Linux, physical-android-minimum] - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup - uses: ./.github/actions/setup - - - name: Build Release app - env: - JAVA_OPTS: '-XX:MaxHeapSize=6g' - run: | - yarn prepare - yarn workspace react-native-reorderable-example react-native build-android \ - --mode release \ - --extra-params "--no-daemon --console=plain -PreactNativeArchitectures=arm64-v8a" - - - name: Install exact Release app - run: | - yarn agent-device reinstall \ - reorderable.example \ - example/android/app/build/outputs/apk/release/app-release.apk \ - --platform android \ - --device '${{ vars.ISSUE40_ANDROID_DEVICE }}' - - - name: Collect paired physical measurements - run: | - yarn performance:physical:collect \ - --platform android \ - --device '${{ vars.ISSUE40_ANDROID_DEVICE }}' \ - --model '${{ vars.ISSUE40_ANDROID_MODEL }}' \ - --os-version '${{ vars.ISSUE40_ANDROID_VERSION }}' \ - --minimum-supported true \ - --out artifacts/issue-40/android.json - - - name: Upload retained Android evidence - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: issue-40-android - if-no-files-found: error - retention-days: 30 - path: artifacts/issue-40/ - - verify: - name: Enforce physical-device budgets - if: always() - needs: [ios, android] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Download physical reports - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - pattern: issue-40-* - path: artifacts/issue-40/downloads - - - name: Reject producer failures - if: needs.ios.result != 'success' || needs.android.result != 'success' - run: exit 1 - - - name: Enforce both physical reports - run: | - node scripts/verify-physical-performance.mjs \ - artifacts/issue-40/downloads/issue-40-ios/ios.json \ - artifacts/issue-40/downloads/issue-40-android/android.json diff --git a/.gitignore b/.gitignore index 2425f74..751fcac 100644 --- a/.gitignore +++ b/.gitignore @@ -100,7 +100,7 @@ artifacts/issue-39/detox/ artifacts/issue-39/feedback/ artifacts/issue-39/outcomes-by-case/ -# Issue 40 physical performance jobs (uploaded by CI, not committed) +# Local physical performance evidence (issue 40, pipeline removed in #75) artifacts/issue-40/ # Exact release candidate generated and uploaded by CI diff --git a/artifacts/issue-36/README.md b/artifacts/issue-36/README.md index 20bb67c..ad3a95d 100644 --- a/artifacts/issue-36/README.md +++ b/artifacts/issue-36/README.md @@ -6,8 +6,9 @@ Validated against fresh debug builds with `@shopify/flash-list@2.3.2` on: - iOS 26.5 (23F77), iPhone 17 Pro, Auto policy's runtime fallback; and - Android, Pixel 10 Pro emulator, Auto policy. -The reusable gate is `yarn verify:flash-device --platform ---device --engine --session `. Its four successful +The reusable gate at the time was `yarn verify:flash-device --platform + --device --engine --session ` +(removed with the manual device gates in #75). Its four successful runs and screenshots are under `matrix/`. It asserts provider windowing, selected multi-row reorder, edge auto-scroll into an initially off-window destination, section rendering, and in-flight cancellation by disabling three diff --git a/artifacts/issue-37/README.md b/artifacts/issue-37/README.md index 520b54e..6debbc3 100644 --- a/artifacts/issue-37/README.md +++ b/artifacts/issue-37/README.md @@ -6,8 +6,9 @@ Validated against `@legendapp/list@3.3.3` and the exact issue 37 source on: - iOS 26.5 (23F77), iPhone 17 Pro, Auto policy's runtime fallback; and - Android, Pixel 10 Pro emulator, Auto policy. -The reusable gate is `yarn verify:legend-device --platform ---device --engine --session `. Its four successful +The reusable gate at the time was `yarn verify:legend-device --platform + --device --engine --session ` +(removed with the manual device gates in #75). Its four successful runs and screenshots are under `matrix/`. It asserts provider windowing, selected multi-row reorder, edge auto-scroll into an initially off-window destination, provider-native section rendering, empty start/end sections, the diff --git a/example/README.md b/example/README.md index 1323cc1..3e7cf3d 100644 --- a/example/README.md +++ b/example/README.md @@ -46,20 +46,3 @@ reorderable://integrations/flash-section-list?preset=focused&engine=auto Examples and Integrations always normalize the engine to Auto. Scenario Lab remounts the selected scenario when its policy changes so each run begins from a deterministic state. - -## Device verification - -The shared verifier navigates through visible app controls and targets a named -scenario: - -```sh -node scripts/verify-named-scenario-device.mjs \ - --platform ios \ - --device "iPhone 17 Pro" \ - --scenario selection \ - --engine fallback \ - --artifacts artifacts/issue-38/selection-ios -``` - -The FlashList and Legend List package scripts delegate to this same named -scenario verifier rather than maintaining hidden automation surfaces. diff --git a/example/src/__tests__/scenario-catalog.test.ts b/example/src/__tests__/scenario-catalog.test.ts index 0208601..e48ba29 100644 --- a/example/src/__tests__/scenario-catalog.test.ts +++ b/example/src/__tests__/scenario-catalog.test.ts @@ -1,6 +1,4 @@ import { describe, expect, it } from '@jest/globals'; -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; import { AREAS, LAB_SCENARIOS, @@ -105,13 +103,4 @@ describe('example application scenario catalog', () => { ).toBeNull(); expect(parseExampleLink('not a link')).toBeNull(); }); - - it('keeps device automation on an app-owned visible activation target', () => { - const verifier = readFileSync( - resolve(process.cwd(), 'scripts/verify-named-scenario-device.mjs'), - 'utf8' - ); - expect(verifier).toContain('id="row-autoscroll-10"'); - expect(verifier).not.toMatch(/reorderable-(?:list|section-list)-wrapper-/); - }); }); diff --git a/package.json b/package.json index c6c17e3..533779e 100644 --- a/package.json +++ b/package.json @@ -52,12 +52,8 @@ "performance:reassure": "reassure --testMatch '**/src/__perf__/issue34.perf-test.tsx' '**/src/__perf__/issue35.perf-test.tsx' '**/src/__perf__/issue40.perf-test.tsx'", "performance:reassure:baseline": "reassure --baseline --testMatch '**/src/__perf__/issue34.perf-test.tsx' '**/src/__perf__/issue35.perf-test.tsx' '**/src/__perf__/issue40.perf-test.tsx'", "performance:reassure:verify": "node scripts/verify-reassure-output.mjs", - "performance:physical:collect": "node scripts/collect-physical-performance.mjs", - "performance:physical:verify": "node scripts/verify-physical-performance.mjs", - "verify:flash-device": "node scripts/verify-flash-list-device.mjs", "verify:flash-package": "node scripts/verify-flash-list-package.mjs", "verify:legend-package": "node scripts/verify-legend-list-package.mjs", - "verify:legend-device": "node scripts/verify-legend-list-device.mjs", "pack:candidate": "node scripts/pack-candidate.mjs", "verify:candidate": "node scripts/verify-package-artifact.mjs artifacts/candidate/react-native-reorderable.tgz", "docs:api": "node scripts/generate-api-docs.mjs", diff --git a/scripts/collect-physical-performance.mjs b/scripts/collect-physical-performance.mjs deleted file mode 100644 index 87d976f..0000000 --- a/scripts/collect-physical-performance.mjs +++ /dev/null @@ -1,452 +0,0 @@ -import { execFile, execFileSync } from 'node:child_process'; -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; -import { promisify } from 'node:util'; - -const options = new Map(); -for (let index = 2; index < process.argv.length; index += 2) - options.set(process.argv[index], process.argv[index + 1]); - -const platform = options.get('--platform'); -const device = options.get('--device'); -const model = options.get('--model'); -const osVersion = options.get('--os-version'); -const outputPath = options.get('--out'); -const declaredMinimumSupported = options.get('--minimum-supported') === 'true'; -if ( - !['ios', 'android'].includes(platform ?? '') || - device == null || - model == null || - osVersion == null || - outputPath == null -) { - throw new Error( - 'Usage: node scripts/collect-physical-performance.mjs --platform ios|android --device --model --os-version --minimum-supported true|false --out ' - ); -} - -const agentDevice = resolve('node_modules/.bin/agent-device'); -const budgets = JSON.parse( - readFileSync(resolve('performance/issue40-budgets.json'), 'utf8') -); -const minimumSupported = - declaredMinimumSupported && osVersion === budgets.minimumOsVersion[platform]; -if (declaredMinimumSupported && !minimumSupported) { - throw new Error( - `The ${platform} minimum-supported performance target must run OS ${budgets.minimumOsVersion[platform]}, received ${osVersion}` - ); -} -const artifactRoot = resolve(dirname(outputPath), platform); -const daemonState = resolve(artifactRoot, 'daemon-state'); -mkdirSync(artifactRoot, { recursive: true }); -const environment = { - ...process.env, - AGENT_DEVICE_STATE_DIR: daemonState, -}; -const session = `issue40-${platform}`; -const sampleCount = 3; -const execFileAsync = promisify(execFile); -let sessionOpen = false; - -function run(args, { json = false, timeout = 60_000 } = {}) { - const stdout = execFileSync(agentDevice, args, { - encoding: 'utf8', - env: environment, - timeout, - }); - return json ? JSON.parse(stdout) : stdout.trim(); -} - -function sessionRun(...args) { - return run([...args, '--session', session]); -} - -function sessionJson(...args) { - return run([...args, '--session', session], { json: true }); -} - -async function sessionJsonDuring(args, action) { - const capture = execFileAsync(agentDevice, [...args, '--session', session], { - encoding: 'utf8', - env: environment, - timeout: 60_000, - }); - await new Promise((resolveDelay) => setTimeout(resolveDelay, 2_000)); - let actionError; - try { - action(); - } catch (error) { - actionError = error; - } - const { stdout } = await capture; - if (actionError != null) throw actionError; - return JSON.parse(stdout); -} - -function writeJson(path, value) { - writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); - return path; -} - -function numericValue(value, keys) { - if (value == null || typeof value !== 'object') return null; - for (const key of keys) { - if (typeof value[key] === 'number' && Number.isFinite(value[key])) - return value[key]; - } - for (const child of Object.values(value)) { - const match = numericValue(child, keys); - if (match != null) return match; - } - return null; -} - -function cpuPercent(metrics) { - const value = numericValue(metrics?.data?.metrics?.cpu, [ - 'cpuPercent', - 'usagePercent', - 'processCpuPercent', - ]); - if (value == null) - throw new Error('Agent Device did not return a process CPU percentage'); - return value; -} - -function missedFramePercent(frames) { - const direct = numericValue(frames?.data?.metrics?.frames ?? frames, [ - 'missedDeadlinePercent', - 'droppedFramePercent', - ]); - if (direct != null) return direct; - const missed = numericValue(frames, [ - 'missedDeadline', - 'missedDeadlines', - 'droppedFrames', - ]); - const total = numericValue(frames, [ - 'totalFrameCount', - 'totalFrames', - 'frameCount', - ]); - if (missed == null || total == null || total <= 0) - throw new Error('Agent Device did not return frame-deadline metrics'); - return (missed / total) * 100; -} - -function frameCount(frames) { - const value = numericValue(frames, [ - 'totalFrameCount', - 'totalFrames', - 'frameCount', - ]); - if (value == null) - throw new Error('Agent Device did not return a frame sample count'); - return value; -} - -function residentMemoryKb(memory) { - const value = numericValue(memory?.data?.metrics?.memory ?? memory, [ - 'residentMemoryKb', - 'totalRssKb', - 'totalPssKb', - ]); - if (value == null) - throw new Error('Agent Device did not return process memory'); - return value; -} - -function resolvePhysicalDevice() { - const inventory = run(['devices', '--json'], { json: true }); - const matches = (inventory?.data?.devices ?? []).filter( - (candidate) => - candidate.platform === platform && - candidate.booted === true && - (candidate.name === device || candidate.id === device) - ); - if (matches.length !== 1) - throw new Error( - `Expected one booted Agent Device target for ${platform}/${device}, received ${matches.length}` - ); - if (matches[0].kind !== 'device') - throw new Error( - `Physical publication evidence cannot use Agent Device kind ${matches[0].kind}` - ); - return matches[0]; -} - -function parseVisibleMetrics(text) { - const start = text.indexOf('{'); - const end = text.lastIndexOf('}'); - if (start < 0 || end <= start) - throw new Error(`Visible performance metrics were not JSON: ${text}`); - return JSON.parse(text.slice(start, end + 1)); -} - -function readVisibleMetrics() { - for (let attempt = 0; attempt < 20; attempt += 1) { - const metrics = parseVisibleMetrics( - sessionRun('get', 'text', 'id="issue40-performance-metrics"') - ); - if (metrics.ready) return metrics; - sessionRun('wait', '250'); - } - throw new Error('Performance harness did not settle'); -} - -async function collectMode(mode, sampleIndex) { - const modeRoot = resolve(artifactRoot, mode, `sample-${sampleIndex + 1}`); - mkdirSync(modeRoot, { recursive: true }); - run([ - 'open', - 'reorderable.example', - '--platform', - platform, - '--device', - device, - '--session', - session, - '--relaunch', - ]); - sessionOpen = true; - sessionRun('wait', 'id="area-examples"', '15000'); - sessionRun('press', 'id="area-tab-lab"', '--settle'); - sessionRun('press', 'id="issue40-performance-launch"'); - sessionRun('wait', 'id="issue40-mode-fallback"', '15000'); - if (mode === 'fallback') sessionRun('press', 'id="issue40-mode-fallback"'); - const settled = readVisibleMetrics(); - if (platform === 'android') sessionJson('perf', 'frames', '--json'); - sessionRun('wait', '2500'); - const idleMetrics = sessionJson('perf', 'metrics', '--json'); - const idleFrames = idleMetrics; - const memory = sessionJson('perf', 'memory', 'sample', '--json'); - const baselineScreenshot = resolve(modeRoot, 'baseline.png'); - sessionRun('screenshot', baselineScreenshot); - - const drag = () => - sessionRun( - 'gesture', - 'drag', - 'label="Performance row 1"', - 'label="Performance row 6"', - '500', - '1800', - '500' - ); - let activeFrames; - if (platform === 'ios') { - activeFrames = await sessionJsonDuring(['perf', 'frames', '--json'], drag); - } else { - sessionJson('perf', 'frames', '--json'); - drag(); - activeFrames = sessionJson('perf', 'frames', '--json'); - } - sessionRun('wait', '750'); - const terminal = readVisibleMetrics(); - const terminalScreenshot = resolve(modeRoot, 'terminal.png'); - sessionRun('screenshot', terminalScreenshot); - - writeJson(resolve(modeRoot, 'visible-settled.json'), settled); - writeJson(resolve(modeRoot, 'visible-terminal.json'), terminal); - writeJson(resolve(modeRoot, 'idle-metrics.json'), idleMetrics); - writeJson(resolve(modeRoot, 'idle-frames.json'), idleFrames); - writeJson(resolve(modeRoot, 'active-frames.json'), activeFrames); - writeJson(resolve(modeRoot, 'memory.json'), memory); - sessionRun('close'); - sessionOpen = false; - return { - activeFrameCount: frameCount(activeFrames), - activeMissedDeadlinePercent: missedFramePercent(activeFrames), - baselineScreenshot, - idleCpuPercent: cpuPercent(idleMetrics), - idleFrameCount: frameCount(idleFrames), - idleMissedFramePercent: missedFramePercent(idleFrames), - memoryKb: residentMemoryKb(memory), - settled, - terminal, - terminalScreenshot, - }; -} - -function median(values) { - const sorted = [...values].sort((left, right) => left - right); - return sorted[Math.floor(sorted.length / 2)]; -} - -function medianObservedFramePercent(samples, countKey, percentKey) { - return median( - samples - .filter((sample) => sample[countKey] > 0) - .map((sample) => sample[percentKey]) - ); -} - -const controlSamples = []; -const fallbackSamples = []; -const agentDeviceVersion = run(['--version']); -if (agentDeviceVersion !== budgets.agentDeviceVersion) - throw new Error( - `Issue #40 requires Agent Device ${budgets.agentDeviceVersion}, received ${agentDeviceVersion}` - ); -const physicalDevice = resolvePhysicalDevice(); -try { - run(['daemon', 'stop']); - for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex += 1) { - controlSamples.push(await collectMode('control', sampleIndex)); - fallbackSamples.push(await collectMode('fallback', sampleIndex)); - } -} finally { - if (sessionOpen) { - try { - sessionRun('close'); - } catch { - // Preserve the measurement failure. - } - } - try { - run(['daemon', 'stop']); - } catch { - // Preserve the measurement failure. - } -} - -const geometry = JSON.parse( - execFileSync(process.execPath, ['benchmarks/issue34-geometry.mjs'], { - encoding: 'utf8', - }) -); -const report = { - schemaVersion: 1, - platform, - agentDeviceVersion, - buildMode: 'Release', - geometryOwner: 'production', - device: { - kind: 'physical', - minimumSupported, - model, - id: physicalDevice.id, - name: physicalDevice.name, - osVersion, - }, - dataset: { rowCount: 10_000 }, - samples: { - control: controlSamples, - fallback: fallbackSamples, - }, - control: { - mountedCells: median( - controlSamples.map((sample) => sample.settled.mountedCells) - ), - renderedCells: median( - controlSamples.map((sample) => sample.settled.renderedCells) - ), - settleMs: median(controlSamples.map((sample) => sample.settled.settleMs)), - idle: { - cpuPercent: median(controlSamples.map((sample) => sample.idleCpuPercent)), - missedFramePercent: medianObservedFramePercent( - controlSamples, - 'idleFrameCount', - 'idleMissedFramePercent' - ), - frameCount: controlSamples.reduce( - (total, sample) => total + sample.idleFrameCount, - 0 - ), - }, - active: { - missedDeadlinePercent: medianObservedFramePercent( - controlSamples, - 'activeFrameCount', - 'activeMissedDeadlinePercent' - ), - frameCount: controlSamples.reduce( - (total, sample) => total + sample.activeFrameCount, - 0 - ), - }, - }, - fallback: { - mountedCells: median( - fallbackSamples.map((sample) => sample.settled.mountedCells) - ), - renderedCells: median( - fallbackSamples.map((sample) => sample.settled.renderedCells) - ), - settleMs: median(fallbackSamples.map((sample) => sample.settled.settleMs)), - listSizeProportionalReactWork: false, - idle: { - cpuPercent: median( - fallbackSamples.map((sample) => sample.idleCpuPercent) - ), - missedFramePercent: medianObservedFramePercent( - fallbackSamples, - 'idleFrameCount', - 'idleMissedFramePercent' - ), - frameCount: fallbackSamples.reduce( - (total, sample) => total + sample.idleFrameCount, - 0 - ), - continuousTimer: false, - continuousFrameCallback: false, - }, - active: { - missedDeadlinePercent: medianObservedFramePercent( - fallbackSamples, - 'activeFrameCount', - 'activeMissedDeadlinePercent' - ), - frameCount: fallbackSamples.reduce( - (total, sample) => total + sample.activeFrameCount, - 0 - ), - p95LibraryUiWorkMs: Math.max( - ...fallbackSamples.map( - (sample) => sample.terminal.gesture?.p95LibraryUiWorkMs ?? Number.NaN - ) - ), - pointerJsCalls: Math.max( - ...fallbackSamples.map( - (sample) => sample.terminal.gesture?.pointerJsCalls ?? Number.NaN - ) - ), - }, - }, - memory: { - coldWorkletsActivationBytes: median( - fallbackSamples.map((sample, index) => - Math.max(0, (sample.memoryKb - controlSamples[index].memoryKb) * 1024) - ) - ), - warmGeometryBytes: geometry.bytes, - }, - geometry: { - maximumLookupSteps: Math.max( - geometry.maximumLookupSteps, - ...fallbackSamples.map( - (sample) => sample.terminal.gesture?.maximumLookupSteps ?? 0 - ) - ), - maximumCorrectionMs: geometry.maximumCorrectionMs, - maximumCorrectionsPerFrame: geometry.maximumCorrectionsPerFrame, - maximumAnchorDisplacementPx: geometry.anchorResidualPx, - mutationComplexity: 'O(log n)', - copiedListSizeArrays: false, - }, - finalization: { - jsResults: Math.max( - ...fallbackSamples.map( - (sample) => sample.terminal.gesture?.terminalJsResults ?? Number.NaN - ) - ), - reorderCommits: Math.max( - ...fallbackSamples.map((sample) => sample.terminal.reorderCommits) - ), - }, - evidence: [...controlSamples, ...fallbackSamples].flatMap((sample) => [ - sample.baselineScreenshot, - sample.terminalScreenshot, - ]), -}; -writeJson(resolve(outputPath), report); -process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); diff --git a/scripts/verify-flash-list-device.mjs b/scripts/verify-flash-list-device.mjs deleted file mode 100644 index 682d73a..0000000 --- a/scripts/verify-flash-list-device.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import { execFileSync } from 'node:child_process'; - -const forwarded = process.argv.slice(2); -for (const scenario of ['flash-list', 'flash-section-list']) { - execFileSync( - 'node', - [ - 'scripts/verify-named-scenario-device.mjs', - ...forwarded, - '--scenario', - scenario, - ], - { stdio: 'inherit' } - ); -} diff --git a/scripts/verify-legend-list-device.mjs b/scripts/verify-legend-list-device.mjs deleted file mode 100644 index 2f5cddd..0000000 --- a/scripts/verify-legend-list-device.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import { execFileSync } from 'node:child_process'; - -const forwarded = process.argv.slice(2); -for (const scenario of ['legend-list', 'legend-section-list']) { - execFileSync( - 'node', - [ - 'scripts/verify-named-scenario-device.mjs', - ...forwarded, - '--scenario', - scenario, - ], - { stdio: 'inherit' } - ); -} diff --git a/scripts/verify-named-scenario-device.mjs b/scripts/verify-named-scenario-device.mjs deleted file mode 100644 index 98f80c7..0000000 --- a/scripts/verify-named-scenario-device.mjs +++ /dev/null @@ -1,148 +0,0 @@ -import { execFileSync } from 'node:child_process'; -import { mkdirSync } from 'node:fs'; -import { resolve } from 'node:path'; - -const options = new Map(); -for (let index = 2; index < process.argv.length; index += 2) - options.set(process.argv[index], process.argv[index + 1]); - -const platform = options.get('--platform'); -const device = options.get('--device'); -const scenario = options.get('--scenario'); -const engine = options.get('--engine') ?? 'auto'; -const session = options.get('--session') ?? `issue-38-${platform}-${scenario}`; -const artifactDirectory = resolve( - options.get('--artifacts') ?? `artifacts/issue-38/matrix/${session}` -); - -const scenarioAreas = new Map([ - ...[ - 'free-form', - 'virtualized-list', - 'section-list', - 'multi-selection', - 'cross-panel-drop', - ].map((id) => [id, 'examples']), - ...[ - 'controlled-order', - 'exact-10000', - 'variable-1000', - 'sections-24x25', - 'autoscroll-200', - 'selection', - 'drop-zones', - 'cancellation', - 'accessibility', - ].map((id) => [id, 'lab']), - ...[ - 'flash-list', - 'flash-section-list', - 'legend-list', - 'legend-section-list', - ].map((id) => [id, 'integrations']), -]); - -if ( - !['ios', 'android'].includes(platform ?? '') || - device == null || - !scenarioAreas.has(scenario) -) - throw new Error( - 'Usage: verify-named-scenario-device.mjs --platform ios|android --device --scenario [--engine auto|fallback]' - ); -if (!['auto', 'fallback'].includes(engine)) - throw new Error('--engine must be auto or fallback'); - -mkdirSync(artifactDirectory, { recursive: true }); -const area = scenarioAreas.get(scenario); - -function run(...args) { - return execFileSync('agent-device', [...args, '--session', session], { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'inherit'], - }); -} - -function text(id) { - return run('get', 'text', `id="${id}"`).trim(); -} - -function expectIncludes(id, expected) { - const actual = text(id); - if (!actual.includes(expected)) - throw new Error(`${id}: expected ${JSON.stringify(expected)} in ${actual}`); -} - -try { - run( - 'open', - 'reorderable.example', - '--platform', - platform, - '--device', - device, - '--metro-host', - '127.0.0.1', - '--metro-port', - '8081', - '--relaunch' - ); - run('wait', 'id="area-examples"', '45000'); - run('press', `id="area-tab-${area}"`, '--settle'); - if (scenario === 'cancellation' || scenario === 'accessibility') - run('scroll', 'down', '3'); - run('press', `id="open-${scenario}"`, '--settle'); - run('wait', `id="scenario-${scenario}"`, '15000'); - - if (area === 'lab') run('press', `id="engine-${engine}"`, '--settle'); - - expectIncludes(`scenario-${scenario}-order`, 'Order:'); - expectIncludes(`scenario-${scenario}-selection`, 'Selection:'); - expectIncludes(`scenario-${scenario}-last-event`, 'Last committed event:'); - expectIncludes(`scenario-${scenario}-callback-count`, 'Callback count: 0'); - expectIncludes( - `scenario-${scenario}-deep-link`, - `reorderable://${area}/${scenario}?preset=` - ); - run('press', `id="scenario-${scenario}-reset"`, '--settle'); - expectIncludes(`scenario-${scenario}-callback-count`, 'Callback count: 0'); - run('screenshot', resolve(artifactDirectory, `${scenario}-${engine}.png`)); - - if (scenario === 'autoscroll-200' && platform === 'android') { - run('longpress', 'id="row-autoscroll-10"', '6500', '--settle'); - expectIncludes( - 'scenario-autoscroll-200-callback-count', - 'Callback count: 1' - ); - expectIncludes( - 'scenario-autoscroll-200-last-event', - '"sourceIds":["autoscroll-10"]' - ); - run('screenshot', resolve(artifactDirectory, 'autoscroll-committed.png')); - } - - if (scenario === 'cancellation') { - run('press', 'id="cancellation-arm-disable"', '--settle'); - run('longpress', 'id="cancellation-blue"', '6500', '--settle'); - expectIncludes('scenario-cancellation-callback-count', 'Callback count: 0'); - expectIncludes('cancellation-disable', 'Enable container'); - run('screenshot', resolve(artifactDirectory, 'cancelled-by-disable.png')); - - run('press', 'id="scenario-cancellation-reset"', '--settle'); - run('press', 'id="cancellation-arm-unmount"', '--settle'); - run('longpress', 'id="cancellation-blue"', '6500', '--settle'); - run('wait', 'id="cancellation-unmounted"', '5000'); - expectIncludes('scenario-cancellation-callback-count', 'Callback count: 0'); - run('screenshot', resolve(artifactDirectory, 'cancelled-by-unmount.png')); - } - - process.stdout.write( - `PASS area=${area} scenario=${scenario} platform=${platform} engine=${area === 'lab' ? engine : 'auto'} artifacts=${artifactDirectory}\n` - ); -} finally { - try { - run('close'); - } catch { - /* preserve the primary failure */ - } -} diff --git a/scripts/verify-physical-performance.mjs b/scripts/verify-physical-performance.mjs deleted file mode 100644 index 5c02302..0000000 --- a/scripts/verify-physical-performance.mjs +++ /dev/null @@ -1,363 +0,0 @@ -import { readFile } from 'node:fs/promises'; -import { pathToFileURL } from 'node:url'; - -const budgets = JSON.parse( - await readFile( - new URL('../performance/issue40-budgets.json', import.meta.url), - 'utf8' - ) -); - -function finite(report, path, failures) { - let value = report; - for (const part of path.split('.')) value = value?.[part]; - if (typeof value !== 'number' || !Number.isFinite(value)) { - failures.push(`${report.platform ?? 'unknown'}: ${path} must be finite`); - return Number.POSITIVE_INFINITY; - } - return value; -} - -function expect(report, condition, message, failures) { - if (!condition) failures.push(`${report.platform ?? 'unknown'}: ${message}`); -} - -export function verifyPhysicalPerformanceReport(report) { - const failures = []; - const platform = report.platform; - expect( - report, - report.schemaVersion === budgets.schemaVersion, - 'unsupported report schema', - failures - ); - expect( - report, - report.agentDeviceVersion === budgets.agentDeviceVersion, - `Agent Device ${budgets.agentDeviceVersion} is required`, - failures - ); - expect( - report, - platform === 'ios' || platform === 'android', - 'platform must be ios or android', - failures - ); - expect( - report, - report.device?.kind === 'physical', - 'device must be physical', - failures - ); - expect( - report, - report.device?.minimumSupported === true, - 'device must be a declared minimum-supported target', - failures - ); - expect( - report, - report.device?.osVersion === budgets.minimumOsVersion[platform], - `OS version must equal the minimum supported ${budgets.minimumOsVersion[platform] ?? 'target'}`, - failures - ); - expect( - report, - report.buildMode === 'Release', - 'measurements must use a Release build', - failures - ); - expect( - report, - report.geometryOwner === 'production', - 'measurements must use the production geometry owner', - failures - ); - expect( - report, - report.dataset?.rowCount === budgets.rowCount, - `dataset must contain ${budgets.rowCount} rows`, - failures - ); - expect( - report, - Array.isArray(report.evidence) && report.evidence.length > 0, - 'evidence artifacts are required', - failures - ); - - const controlMounted = finite(report, 'control.mountedCells', failures); - const fallbackMounted = finite(report, 'fallback.mountedCells', failures); - const controlRendered = finite(report, 'control.renderedCells', failures); - const fallbackRendered = finite(report, 'fallback.renderedCells', failures); - expect( - report, - Math.abs(fallbackMounted - controlMounted) <= - budgets.virtualizationCellDelta, - 'mounted cells exceed paired FlatList ±2', - failures - ); - expect( - report, - Math.abs(fallbackRendered - controlRendered) <= - budgets.virtualizationCellDelta, - 'rendered cells exceed paired FlatList ±2', - failures - ); - expect( - report, - report.fallback?.listSizeProportionalReactWork === false, - 'list-size-proportional React work is forbidden', - failures - ); - - const controlIdleCpu = finite(report, 'control.idle.cpuPercent', failures); - const fallbackIdleCpu = finite(report, 'fallback.idle.cpuPercent', failures); - const controlIdleMissed = finite( - report, - 'control.idle.missedFramePercent', - failures - ); - const fallbackIdleMissed = finite( - report, - 'fallback.idle.missedFramePercent', - failures - ); - const controlIdleFrames = finite(report, 'control.idle.frameCount', failures); - const fallbackIdleFrames = finite( - report, - 'fallback.idle.frameCount', - failures - ); - expect( - report, - report.fallback?.idle?.continuousTimer === false, - 'idle continuous timer is forbidden', - failures - ); - expect( - report, - report.fallback?.idle?.continuousFrameCallback === false, - 'idle continuous frame callback is forbidden', - failures - ); - expect( - report, - fallbackIdleCpu - controlIdleCpu <= budgets.idleCpuDeltaPercentagePoints, - 'idle CPU delta exceeds one percentage point', - failures - ); - expect( - report, - fallbackIdleMissed - controlIdleMissed <= - budgets.idleMissedFrameDeltaPercentagePoints, - 'idle missed-frame percentage exceeds FlatList', - failures - ); - expect( - report, - controlIdleFrames > 0 && fallbackIdleFrames > 0, - 'idle frame samples must contain attributable frames', - failures - ); - - const controlSettle = finite(report, 'control.settleMs', failures); - const fallbackSettle = finite(report, 'fallback.settleMs', failures); - const platformSettleDelta = budgets.settleDeltaMs[platform] ?? -1; - expect( - report, - fallbackSettle <= controlSettle * budgets.settleRatio, - 'settle exceeds twice FlatList', - failures - ); - expect( - report, - fallbackSettle - controlSettle <= platformSettleDelta, - `settle delta exceeds ${platformSettleDelta} ms`, - failures - ); - - const warmGeometryBytes = finite( - report, - 'memory.warmGeometryBytes', - failures - ); - const coldWorkletsBytes = finite( - report, - 'memory.coldWorkletsActivationBytes', - failures - ); - const warmBudget = - budgets.warmGeometryFixedBytes + - budgets.warmGeometryBytesPerItem * budgets.rowCount; - expect( - report, - warmGeometryBytes <= warmBudget, - `warm geometry exceeds ${warmBudget} bytes`, - failures - ); - expect( - report, - coldWorkletsBytes <= budgets.coldWorkletsActivationBytes, - 'cold Worklets activation exceeds 60 MiB', - failures - ); - - const controlActiveMissed = finite( - report, - 'control.active.missedDeadlinePercent', - failures - ); - const fallbackActiveMissed = finite( - report, - 'fallback.active.missedDeadlinePercent', - failures - ); - const controlActiveFrames = finite( - report, - 'control.active.frameCount', - failures - ); - const fallbackActiveFrames = finite( - report, - 'fallback.active.frameCount', - failures - ); - const p95UiWork = finite( - report, - 'fallback.active.p95LibraryUiWorkMs', - failures - ); - expect( - report, - fallbackActiveMissed < budgets.activeMissedDeadlinePercent, - 'active drag misses at least 5% of deadlines', - failures - ); - expect( - report, - fallbackActiveMissed - controlActiveMissed <= - budgets.activeMissedDeadlineDeltaPercentagePoints, - 'active missed-deadline delta exceeds two percentage points', - failures - ); - expect( - report, - controlActiveFrames > 0 && fallbackActiveFrames > 0, - 'active frame samples must contain attributable frames', - failures - ); - expect( - report, - p95UiWork <= budgets.p95LibraryUiWorkMs, - 'p95 library UI work exceeds 4 ms', - failures - ); - expect( - report, - report.fallback?.active?.pointerJsCalls === 0, - 'pointer updates crossed into JavaScript', - failures - ); - - const lookupSteps = finite(report, 'geometry.maximumLookupSteps', failures); - const correctionMs = finite(report, 'geometry.maximumCorrectionMs', failures); - const correctionsPerFrame = finite( - report, - 'geometry.maximumCorrectionsPerFrame', - failures - ); - const anchorDisplacement = finite( - report, - 'geometry.maximumAnchorDisplacementPx', - failures - ); - expect( - report, - report.geometry?.mutationComplexity === 'O(log n)', - 'geometry mutation must be O(log n)', - failures - ); - expect( - report, - report.geometry?.copiedListSizeArrays === false, - 'geometry must not copy list-size arrays', - failures - ); - expect( - report, - lookupSteps <= budgets.maximumLookupSteps, - 'lookup step budget exceeded', - failures - ); - expect( - report, - correctionMs <= budgets.maximumCorrectionMs, - 'correction latency budget exceeded', - failures - ); - expect( - report, - correctionsPerFrame <= budgets.maximumCorrectionsPerFrame, - 'corrections-per-frame budget exceeded', - failures - ); - expect( - report, - anchorDisplacement <= budgets.maximumAnchorDisplacementPx, - 'anchor displacement budget exceeded', - failures - ); - - const terminalResults = finite(report, 'finalization.jsResults', failures); - const commits = finite(report, 'finalization.reorderCommits', failures); - expect( - report, - terminalResults === budgets.terminalJsResults, - 'terminal JS result count must equal one', - failures - ); - expect( - report, - commits <= budgets.maximumReorderCommits, - 'more than one reorder commit was observed', - failures - ); - return failures; -} - -export function verifyPhysicalPerformanceMatrix(reports) { - const failures = reports.flatMap(verifyPhysicalPerformanceReport); - for (const platform of ['ios', 'android']) { - const count = reports.filter( - (report) => report.platform === platform - ).length; - if (count !== 1) - failures.push( - `matrix: expected exactly one ${platform} physical report, received ${count}` - ); - } - return failures; -} - -async function main() { - const paths = process.argv.slice(2); - if (paths.length === 0) - throw new Error( - 'Usage: node scripts/verify-physical-performance.mjs ' - ); - const reports = await Promise.all( - paths.map((path) => readFile(path, 'utf8').then(JSON.parse)) - ); - const failures = verifyPhysicalPerformanceMatrix(reports); - if (failures.length > 0) - throw new Error( - `Physical performance publication gate failed:\n- ${failures.join('\n- ')}` - ); - process.stdout.write( - 'Physical performance publication gate passed for iOS and Android.\n' - ); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) - await main(); diff --git a/src/__tests__/issue40-performance-gates.test.ts b/src/__tests__/issue40-performance-gates.test.ts index 6e8afbb..c82f9d4 100644 --- a/src/__tests__/issue40-performance-gates.test.ts +++ b/src/__tests__/issue40-performance-gates.test.ts @@ -2,8 +2,6 @@ import { execFileSync } from 'node:child_process'; import { resolve } from 'node:path'; import { describe, expect, it } from '@jest/globals'; -type PhysicalReport = Record; - function evaluateVerifier(file: string, expression: string): any { const verifierUrl = JSON.stringify(`file://${resolve(process.cwd(), file)}`); return JSON.parse( @@ -19,167 +17,7 @@ function evaluateVerifier(file: string, expression: string): any { ); } -function passingReport(platform: 'ios' | 'android'): PhysicalReport { - return { - schemaVersion: 1, - platform, - agentDeviceVersion: '0.20.10', - buildMode: 'Release', - geometryOwner: 'production', - device: { - kind: 'physical', - minimumSupported: true, - model: platform === 'ios' ? 'minimum-ios' : 'minimum-android', - osVersion: platform === 'ios' ? '15.1' : '24', - }, - dataset: { rowCount: 10_000 }, - control: { - mountedCells: 30, - renderedCells: 30, - settleMs: 30, - idle: { - cpuPercent: 0.5, - frameCount: 150, - missedFramePercent: 0, - }, - active: { frameCount: 150, missedDeadlinePercent: 2 }, - }, - fallback: { - mountedCells: 31, - renderedCells: 31, - settleMs: 45, - listSizeProportionalReactWork: false, - idle: { - cpuPercent: 1.4, - frameCount: 150, - missedFramePercent: 0, - continuousTimer: false, - continuousFrameCallback: false, - }, - active: { - frameCount: 150, - missedDeadlinePercent: 3.9, - p95LibraryUiWorkMs: 3.8, - pointerJsCalls: 0, - }, - }, - memory: { - warmGeometryBytes: 8_900_000, - coldWorkletsActivationBytes: 60_000_000, - }, - geometry: { - maximumLookupSteps: 14, - maximumCorrectionMs: 0.9, - maximumCorrectionsPerFrame: 4, - maximumAnchorDisplacementPx: 2, - mutationComplexity: 'O(log n)', - copiedListSizeArrays: false, - }, - finalization: { jsResults: 1, reorderCommits: 1 }, - evidence: ['screenshot.png', 'metrics.json', 'frames.json'], - }; -} - describe('issue #40 publication gates', () => { - it('accepts one minimum physical report for each platform', () => { - expect( - evaluateVerifier( - 'scripts/verify-physical-performance.mjs', - `verifier.verifyPhysicalPerformanceMatrix(${JSON.stringify([ - passingReport('ios'), - passingReport('android'), - ])})` - ) - ).toEqual([]); - }); - - it.each([ - [ - 'simulator evidence', - (report: PhysicalReport) => (report.device.kind = 'simulator'), - 'device must be physical', - ], - [ - 'unpinned Agent Device', - (report: PhysicalReport) => (report.agentDeviceVersion = '0.20.5'), - 'Agent Device 0.20.10 is required', - ], - [ - 'non-minimum hardware', - (report: PhysicalReport) => (report.device.minimumSupported = false), - 'minimum-supported', - ], - [ - 'newer operating system', - (report: PhysicalReport) => (report.device.osVersion = '27'), - 'OS version must equal the minimum supported', - ], - [ - 'virtualization growth', - (report: PhysicalReport) => (report.fallback.renderedCells = 33), - 'rendered cells', - ], - [ - 'idle scheduling', - (report: PhysicalReport) => - (report.fallback.idle.continuousFrameCallback = true), - 'continuous frame callback', - ], - [ - 'settle latency', - (report: PhysicalReport) => (report.fallback.settleMs = 61), - 'twice FlatList', - ], - [ - 'warm geometry memory', - (report: PhysicalReport) => - (report.memory.warmGeometryBytes = 10_000_000), - 'warm geometry', - ], - [ - 'active frame health', - (report: PhysicalReport) => - (report.fallback.active.missedDeadlinePercent = 5), - 'at least 5%', - ], - [ - 'empty active frame sample', - (report: PhysicalReport) => (report.control.active.frameCount = 0), - 'active frame samples must contain attributable frames', - ], - [ - 'UI-runtime work', - (report: PhysicalReport) => - (report.fallback.active.p95LibraryUiWorkMs = 4.1), - 'p95 library UI work', - ], - [ - 'lookup complexity', - (report: PhysicalReport) => (report.geometry.maximumLookupSteps = 16), - 'lookup step', - ], - [ - 'measurement correction', - (report: PhysicalReport) => (report.geometry.maximumCorrectionMs = 1.1), - 'correction latency', - ], - [ - 'terminal cardinality', - (report: PhysicalReport) => (report.finalization.jsResults = 2), - 'terminal JS result', - ], - ])('rejects %s outside budget', (_name, mutate, expected) => { - const report = passingReport('ios'); - mutate(report); - - expect( - evaluateVerifier( - 'scripts/verify-physical-performance.mjs', - `verifier.verifyPhysicalPerformanceReport(${JSON.stringify(report)})` - ).join('\n') - ).toContain(expected); - }); - it('fails a Reassure render-count increase and significant slowdown', () => { const measurement = { name: 'critical list path',