From ac561b1a9fced90bd4650a746a9e0640269c1b7d Mon Sep 17 00:00:00 2001 From: IC Date: Mon, 24 Aug 2026 18:17:25 +0200 Subject: [PATCH 1/3] fix[notask]: decouple mobile shard validation from desktop integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `generate-mobile-integration-tests.js` wrote integration.auto.cjs and then asserted that every runner was assigned to a Device Farm shard. The script is chained into `npm run test:integration`, so that assertion could abort desktop integration tests on all seven platforms — which is what happened when #4006 removed the pi05 group: `bare` aborted with exit 134 before a single test ran. Generation had already succeeded at that point. The failure was purely a mobile scheduling policy check running in the desktop path. - Remove validateGroups from the generator; it now only generates. - Move the rules to scripts/lib/validate-test-groups.js — pure, no fs, no process.exit — and call them from validate-mobile-tests.js, which already existed but was wired into nothing. - Add a top-level `deferred` key to test-groups.json recording runners that are intentionally not scheduled. pi05 mobile coverage is deferred pending a project-owned CDN mirror and is gated on-device by `_skipMobilePi05`, so this preserves #4006's outcome while keeping "not scheduled" distinguishable from "forgotten". It must stay top-level: the CI composites read only `.` and ignore sibling keys, as OCR's `perf_report_filter` already relies on. - Wire `test:mobile:validate` and a new unit suite into `test:unit`, which runs in the ungated ts-checks job — #4006 touched nothing native, so the gated integration job that would have caught it was skipped on its own PR. - Skip the mtime staleness heuristic when CI is set; a fresh clone stamps every file at checkout, so that comparison is meaningless there. Device Farm output is unchanged: same 2 specs, same greps, runPi05Test still absent from the runner list. integration.auto.cjs regenerates byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- packages/vla-ggml/package.json | 3 +- .../__tests__/mobile-test-groups.test.js | 86 +++++++++++++ .../generate-mobile-integration-tests.js | 42 +------ .../scripts/lib/validate-test-groups.js | 117 ++++++++++++++++++ .../vla-ggml/scripts/validate-mobile-tests.js | 54 ++++++-- packages/vla-ggml/test/mobile/README.md | 26 +++- .../vla-ggml/test/mobile/test-groups.json | 3 +- 7 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js create mode 100644 packages/vla-ggml/scripts/lib/validate-test-groups.js diff --git a/packages/vla-ggml/package.json b/packages/vla-ggml/package.json index 6fd91e1335..4cc8733b86 100644 --- a/packages/vla-ggml/package.json +++ b/packages/vla-ggml/package.json @@ -24,8 +24,9 @@ "test:integration": "npm run build:ts && npm run test:integration:generate && bare test/integration/all.js --exit", "test:integration:generate": "brittle -r test/integration/all.js test/integration/*.test.js && npm run test:mobile:generate", "test:unit:generate": "brittle -r test/unit/all.js test/unit/*.test.js", - "test:unit": "npm run build:ts && npm run test:unit:generate && bare test/unit/all.js --exit && npm run test:prestage", + "test:unit": "npm run build:ts && npm run test:unit:generate && bare test/unit/all.js --exit && npm run test:prestage && npm run test:mobile:groups && npm run test:mobile:validate", "test:prestage": "node --test scripts/__tests__/generate-prestage-block.test.js", + "test:mobile:groups": "node --test scripts/__tests__/mobile-test-groups.test.js", "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target addon-test", "test:cpp:run": "cd build/test/unit/ && ./addon-test --gtest_output=xml:cpp-test-results.xml", "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", diff --git a/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js new file mode 100644 index 0000000000..0338c212c0 --- /dev/null +++ b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js @@ -0,0 +1,86 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const path = require('node:path') + +const { validateTestGroups, platformNames } = require('../lib/validate-test-groups.js') +const groups = require('../../test/mobile/test-groups.json') + +const integrationAutoPath = path.resolve(__dirname, '../../test/mobile/integration.auto.cjs') + +function generatedRunners() { + const content = fs.readFileSync(integrationAutoPath, 'utf8') + return Array.from(content.matchAll(/^async function (run[A-Za-z0-9_]+)\s*\(/gm), (m) => m[1]) +} + +test('the committed test-groups.json covers every generated runner', () => { + assert.deepEqual(validateTestGroups(groups, generatedRunners()), []) +}) + +test('deferred runners are declared, not silently absent', () => { + // pi05 mobile coverage is deferred pending a project-owned CDN mirror, and + // pi05.test.js is gated on-device by `_skipMobilePi05`. Recording it here is + // what keeps "not scheduled" distinguishable from "forgotten". + assert.deepEqual(groups.deferred, ['runPi05Test']) + for (const platform of platformNames(groups)) { + const scheduled = Object.values(groups[platform]).flat() + assert.ok( + !scheduled.includes('runPi05Test'), + `runPi05Test must not be scheduled on ${platform}` + ) + } +}) + +test('"deferred" is a top-level key, never a platform', () => { + // The Device Farm composites read only `.`, so a `deferred` key + // nested inside ios/android would be scheduled as a real shard. + assert.ok(!platformNames(groups).includes('deferred')) + assert.deepEqual(platformNames(groups).sort(), ['android', 'ios']) +}) + +test('an unassigned runner is reported', () => { + const problems = validateTestGroups(groups, [...generatedRunners(), 'runBrandNewTest']) + assert.equal(problems.length, platformNames(groups).length) + assert.ok(problems.every((p) => p.includes('runBrandNewTest'))) +}) + +test('a typo in a group is reported', () => { + const typo = { + ios: { smolvla: ['runAddonTest', 'runTypoTest'] }, + deferred: [] + } + const problems = validateTestGroups(typo, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('runTypoTest') && p.includes('do not exist'))) +}) + +test('a stale deferred entry is reported', () => { + const stale = { + ios: { smolvla: ['runAddonTest'] }, + deferred: ['runRemovedTest'] + } + const problems = validateTestGroups(stale, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('runRemovedTest') && p.includes('do not exist'))) +}) + +test('a runner that is both scheduled and deferred is reported', () => { + const contradictory = { + ios: { smolvla: ['runAddonTest'] }, + deferred: ['runAddonTest'] + } + const problems = validateTestGroups(contradictory, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('both scheduled and listed'))) +}) + +test('metadata keys that are not platform maps are ignored', () => { + // OCR ships a top-level `perf_report_filter` string; the shape must tolerate + // sibling metadata without treating it as a platform. + const withMetadata = { + ios: { smolvla: ['runAddonTest'] }, + perf_report_filter: 'something|else', + deferred: [] + } + assert.deepEqual(platformNames(withMetadata), ['ios']) + assert.deepEqual(validateTestGroups(withMetadata, ['runAddonTest']), []) +}) diff --git a/packages/vla-ggml/scripts/generate-mobile-integration-tests.js b/packages/vla-ggml/scripts/generate-mobile-integration-tests.js index 325390c99b..9094876896 100644 --- a/packages/vla-ggml/scripts/generate-mobile-integration-tests.js +++ b/packages/vla-ggml/scripts/generate-mobile-integration-tests.js @@ -7,7 +7,6 @@ const repoRoot = path.resolve(__dirname, '..') const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileDir = path.join(repoRoot, 'test', 'mobile') const outputFile = path.join(mobileDir, 'integration.auto.cjs') -const groupsFile = path.join(mobileDir, 'test-groups.json') function getIntegrationFiles() { if (!fs.existsSync(integrationDir)) { @@ -64,50 +63,21 @@ function buildFileContents(files) { return `${lines.join('\n')}\n` } -function validateGroups(functionNames) { - if (!fs.existsSync(groupsFile)) { - console.warn('[warn] test-groups.json not found — skipping split validation') - return - } - const groups = JSON.parse(fs.readFileSync(groupsFile, 'utf-8')) - const nameSet = new Set(functionNames) - for (const [platform, splits] of Object.entries(groups)) { - const covered = new Set(Object.values(splits).flat()) - const missing = functionNames.filter((n) => !covered.has(n)) - const extra = [...covered].filter((n) => !nameSet.has(n)) - if (missing.length) { - throw new Error( - '[' + - platform + - '] Tests not assigned to any group in test-groups.json:\n ' + - missing.join('\n ') + - '\nAdd them to a group in test/mobile/test-groups.json.' - ) - } - if (extra.length) { - throw new Error( - '[' + - platform + - '] test-groups.json references non-existent tests:\n ' + - extra.join('\n ') + - '\nRemove them or check for typos.' - ) - } - } - console.log('Group coverage validated — all tests assigned for every platform.') -} - +// NOTE: this generator deliberately performs no test-groups.json validation. +// `npm run test:integration` chains it (so the committed integration.auto.cjs +// can never go stale), which means anything that throws here takes desktop +// integration tests down on every platform. Group coverage is a mobile +// scheduling concern, so it lives in `npm run test:mobile:validate` +// (scripts/validate-mobile-tests.js) and runs in the ungated ts-checks job. function main() { const files = getIntegrationFiles() if (files.length === 0) { throw new Error(`No integration test files found inside ${integrationDir}`) } - const functionNames = files.map(toFunctionName) const content = buildFileContents(files) fs.writeFileSync(outputFile, content, 'utf8') console.log(`Generated ${outputFile} with ${files.length} integration runners.`) - validateGroups(functionNames) } if (require.main === module) { diff --git a/packages/vla-ggml/scripts/lib/validate-test-groups.js b/packages/vla-ggml/scripts/lib/validate-test-groups.js new file mode 100644 index 0000000000..b0f1213ed2 --- /dev/null +++ b/packages/vla-ggml/scripts/lib/validate-test-groups.js @@ -0,0 +1,117 @@ +'use strict' + +// Group-coverage rules for test/mobile/test-groups.json. +// +// Deliberately dependency-free and side-effect-free (no fs, no process.exit) so +// the same rules run under `node` from validate-mobile-tests.js and are unit +// testable from scripts/__tests__/mobile-test-groups.test.js. +// +// This check lives OUTSIDE the generator on purpose. It answers a mobile +// scheduling question — "is every on-device runner assigned to a Device Farm +// shard?" — which has no bearing on whether integration.auto.cjs was written +// correctly. Bundling it into the generator once let a Device Farm scheduling +// edit abort `npm run test:integration`, taking desktop CI down on all seven +// platforms (PR #4006). + +// Runners deliberately not scheduled on Device Farm are listed under this +// top-level key. It sits beside the platform maps rather than inside one +// because the CI composites consume only `.` and ignore every other +// top-level key (see .github/actions/run-mobile-integration-tests/ +// upload-to-devicefarm/action.yml). Nesting it under `ios`/`android` would +// instead schedule it as a real shard. +const DEFERRED_KEY = 'deferred' + +// A platform entry is a `{ groupName: [runner, ...] }` map. Anything else at the +// top level is metadata for another consumer — `deferred` here, OCR's +// `perf_report_filter` — and is not a platform. +function isPlatformEntry(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function platformNames(groups) { + return Object.keys(groups).filter((key) => isPlatformEntry(groups[key])) +} + +function coveredRunners(platformEntry) { + return Object.values(platformEntry).filter(Array.isArray).flat() +} + +function deferredRunners(groups) { + const deferred = groups[DEFERRED_KEY] + return Array.isArray(deferred) ? deferred : [] +} + +// Returns a list of human-readable problem strings; empty means valid. +// `runners` is the authoritative runner-name list, derived from the generated +// integration.auto.cjs by the caller. +function validateTestGroups(groups, runners) { + const problems = [] + const known = new Set(runners) + const deferred = deferredRunners(groups) + + // A stale `deferred` entry is worse than a noisy one: it would silently + // excuse a runner that no longer exists, and mask a real gap if the name is + // ever reused. + const unknownDeferred = deferred.filter((name) => !known.has(name)) + if (unknownDeferred.length) { + problems.push( + `[${DEFERRED_KEY}] lists runners that do not exist:\n ` + + unknownDeferred.join('\n ') + + '\nRemove them or check for typos.' + ) + } + + const platforms = platformNames(groups) + if (platforms.length === 0) { + problems.push( + 'test-groups.json declares no platform maps.\n' + + 'Expected at least one top-level `{ "": { "": [runners] } }` entry.' + ) + return problems + } + + const deferredSet = new Set(deferred) + + for (const platform of platforms) { + const covered = new Set(coveredRunners(groups[platform])) + + const missing = runners.filter((name) => !covered.has(name) && !deferredSet.has(name)) + if (missing.length) { + problems.push( + `[${platform}] runners not assigned to any group:\n ` + + missing.join('\n ') + + `\nAdd them to a group in test/mobile/test-groups.json, or to the ` + + `top-level "${DEFERRED_KEY}" list if they are intentionally not run on device.` + ) + } + + const extra = [...covered].filter((name) => !known.has(name)) + if (extra.length) { + problems.push( + `[${platform}] groups reference runners that do not exist:\n ` + + extra.join('\n ') + + '\nRemove them or check for typos.' + ) + } + + // A runner in both a shard and `deferred` is contradictory: it would run on + // device while claiming to be deferred. + const contradictory = [...covered].filter((name) => deferredSet.has(name)) + if (contradictory.length) { + problems.push( + `[${platform}] runners are both scheduled and listed as "${DEFERRED_KEY}":\n ` + + contradictory.join('\n ') + + `\nRemove them from one or the other.` + ) + } + } + + return problems +} + +module.exports = { + DEFERRED_KEY, + validateTestGroups, + platformNames, + deferredRunners +} diff --git a/packages/vla-ggml/scripts/validate-mobile-tests.js b/packages/vla-ggml/scripts/validate-mobile-tests.js index 2a68be20c7..d7c8001f64 100644 --- a/packages/vla-ggml/scripts/validate-mobile-tests.js +++ b/packages/vla-ggml/scripts/validate-mobile-tests.js @@ -4,9 +4,12 @@ const fs = require('fs') const path = require('path') +const { validateTestGroups } = require('./lib/validate-test-groups.js') + const repoRoot = path.resolve(__dirname, '..') const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileAutoFile = path.join(repoRoot, 'test', 'mobile', 'integration.auto.cjs') +const groupsFile = path.join(repoRoot, 'test', 'mobile', 'test-groups.json') function getIntegrationTestFiles() { if (!fs.existsSync(integrationDir)) { @@ -32,6 +35,14 @@ function getGeneratedIntegrationRefs(content) { return references } +// integration.auto.cjs declares one `async function run` per on-device +// test. Once it is confirmed in sync with test/integration (above), it is the +// authoritative runner-name list — the same source .github/actions/ +// run-mobile-integration-tests/validate-devices uses. +function getGeneratedRunnerNames(content) { + return Array.from(content.matchAll(/^async function (run[A-Za-z0-9_]+)\s*\(/gm), (m) => m[1]) +} + function setDiff(left, right) { return [...left].filter((item) => !right.has(item)).sort() } @@ -74,19 +85,44 @@ try { } // Keep timestamp validation as a fast stale-content signal for edited tests. - const latestIntegrationTime = Math.max( - ...integrationFiles.map((f) => fs.statSync(path.join(integrationDir, f)).mtimeMs) - ) - const mobileAutoTime = fs.statSync(mobileAutoFile).mtimeMs + // Skipped in CI: a fresh clone stamps every working-tree file at checkout + // time, so the ordering this compares is meaningless there and would fail at + // random. The reference checks above are content-based and cover CI. + if (!process.env.CI) { + const latestIntegrationTime = Math.max( + ...integrationFiles.map((f) => fs.statSync(path.join(integrationDir, f)).mtimeMs) + ) + const mobileAutoTime = fs.statSync(mobileAutoFile).mtimeMs + + if (latestIntegrationTime > mobileAutoTime) { + console.error('❌ Mobile integration tests are out of date!') + console.error(' Integration tests modified after mobile tests were generated.') + console.error(' Run: npm run test:mobile:generate') + process.exit(1) + } + } - if (latestIntegrationTime > mobileAutoTime) { - console.error('❌ Mobile integration tests are out of date!') - console.error(' Integration tests modified after mobile tests were generated.') - console.error(' Run: npm run test:mobile:generate') + // Device Farm shard coverage. This lives here rather than in the generator so + // that a mobile scheduling mistake can never abort `npm run test:integration` + // and take desktop CI down with it. + if (!fs.existsSync(groupsFile)) { + console.log('✅ Mobile integration tests are up to date (no test-groups.json — single-spec)') + process.exit(0) + } + + const groups = JSON.parse(fs.readFileSync(groupsFile, 'utf8')) + const runners = getGeneratedRunnerNames(mobileAutoContent) + const problems = validateTestGroups(groups, runners) + + if (problems.length > 0) { + console.error('❌ test-groups.json does not cover every mobile runner\n') + problems.forEach((problem) => console.error(` ${problem}\n`)) process.exit(1) } - console.log('✅ Mobile integration tests are up to date') + console.log( + `✅ Mobile integration tests are up to date (${runners.length} runner(s), group coverage OK)` + ) process.exit(0) } catch (error) { console.error('Error validating mobile tests:', error.message) diff --git a/packages/vla-ggml/test/mobile/README.md b/packages/vla-ggml/test/mobile/README.md index a1dfa7848d..6bdd7ee61f 100644 --- a/packages/vla-ggml/test/mobile/README.md +++ b/packages/vla-ggml/test/mobile/README.md @@ -17,7 +17,31 @@ After adding a new file under `test/integration/`, regenerate the mobile entries npm run test:mobile:generate ``` -The generator walks `test/integration/`, derives a function name per test file, and rewrites `integration.auto.cjs`. If a `test-groups.json` is added later (per-platform iOS/Android shard split), the generator will validate that every runner is covered. +The generator walks `test/integration/`, derives a function name per test file, and rewrites `integration.auto.cjs`. It **only generates** — it performs no `test-groups.json` validation, because `npm run test:integration` chains it, so anything that throws there takes desktop integration tests down on every platform. + +## `test-groups.json` and deferred runners + +`test-groups.json` is the per-platform Device Farm shard split. Every runner in `integration.auto.cjs` must either appear in a group for each platform, or be listed under the top-level `deferred` key: + +```json +{ + "ios": { "smolvla": ["runAddonTest"], "groot": ["runGrootTest"] }, + "android": { "smolvla": ["runAddonTest"], "groot": ["runGrootTest"] }, + "deferred": ["runPi05Test"] +} +``` + +`deferred` records runners that are intentionally not scheduled on device — pi05 mobile coverage is deferred pending a project-owned CDN-fronted mirror, and `pi05.test.js` is gated on-device by `_skipMobilePi05`. Declaring it keeps "not scheduled" distinguishable from "forgotten"; deleting the runner from the file instead makes those two indistinguishable. + +`deferred` **must stay a top-level key**. The CI composites consume only `.` and ignore every other top-level key (as OCR's `perf_report_filter` already relies on), so nesting `deferred` inside `ios`/`android` would schedule it as a real Device Farm shard. + +Coverage is enforced by: + +```bash +npm run test:mobile:validate # also runs as part of `npm run test:unit` +``` + +which runs in the ungated `ts-checks` PR job, so a scheduling mistake is caught on every PR instead of only inside the expensive, label-gated integration suite. ## Running the Tests diff --git a/packages/vla-ggml/test/mobile/test-groups.json b/packages/vla-ggml/test/mobile/test-groups.json index b4cbad412c..ecab8ee608 100644 --- a/packages/vla-ggml/test/mobile/test-groups.json +++ b/packages/vla-ggml/test/mobile/test-groups.json @@ -6,5 +6,6 @@ "android": { "smolvla": ["runAddonTest", "runEsmNamedExportsTest"], "groot": ["runGrootTest"] - } + }, + "deferred": ["runPi05Test"] } From 7ef057275bbb652f438d15ee25628810b4e81c21 Mon Sep 17 00:00:00 2001 From: IC Date: Mon, 24 Aug 2026 19:16:15 +0200 Subject: [PATCH 2/3] fix[notask]: reserve "deferred" inside platform maps, drop the inert mtime check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups to the mobile shard validation split, both found in review. 1. A `deferred` key nested inside a platform validated completely clean. To coveredRunners it is just another array of runner names, so `missing` was satisfied, top-level deferredRunners stayed empty so the contradiction rule never fired, and the whole file returned zero problems. upload-to-devicefarm turns every `{ groupName: [runners] }` entry into a Device Farm spec, so the runners listed there would have been scheduled — and billed — under a shard literally named "deferred". That is exactly what the lib comment and test/mobile/README.md say must never happen, and nothing enforced it. The test named '"deferred" is a top-level key, never a platform' passed on that input too: it only asserted on platformNames() of the committed file, which is the top-level case that was never at risk. It is kept as-is and a second test now covers the hazard itself, verified to fail without the fix. 2. The mtime staleness check could not detect anything, and #4031 promoted it into a blocking gate by chaining test:mobile:validate into test:unit. buildFileContents derives integration.auto.cjs from the sorted filenames under test/integration/ and never opens a test file, so editing a test's body cannot make the generated file stale. The check could therefore only produce false positives — and as of the chaining, each one was a hard `npm run test:unit` failure telling the author to regenerate a byte-identical file. Removed rather than CI-gated; every staleness it could legitimately catch (a test added, renamed or removed) is already caught by the content-based reference diff above it, which is verified unchanged. Chaining test:mobile:validate into test:unit is kept — that is what gives the group-coverage check a home. Only the dead check riding along with it is gone. Verified: 17/17 script tests pass (9 mobile-test-groups, up from 8, plus the 8 pre-existing prestage tests); the new test fails against the unfixed lib with 0 problems where 2 are expected; `touch test/integration/addon.test.js` then validate-mobile-tests.js now exits 0; adding an unregenerated test file still exits 1 with "out of sync with test/integration". Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/mobile-test-groups.test.js | 20 ++++++++++++++ .../scripts/lib/validate-test-groups.js | 14 ++++++++++ .../vla-ggml/scripts/validate-mobile-tests.js | 26 +++++++------------ packages/vla-ggml/test/mobile/README.md | 2 ++ 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js index 0338c212c0..401535223d 100644 --- a/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js +++ b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js @@ -40,6 +40,26 @@ test('"deferred" is a top-level key, never a platform', () => { assert.deepEqual(platformNames(groups).sort(), ['android', 'ios']) }) +test('a "deferred" key nested inside a platform is reported', () => { + // The assertion above only covers the committed file's shape. This covers the + // hazard itself: nested under a platform, `deferred` is just another array of + // runner names, so every other rule is satisfied and the file would otherwise + // validate clean — while upload-to-devicefarm schedules it as a real shard. + const nested = { + ios: { smolvla: ['runAddonTest'], deferred: ['runPi05Test'] }, + android: { smolvla: ['runAddonTest'], deferred: ['runPi05Test'] } + } + const problems = validateTestGroups(nested, ['runAddonTest', 'runPi05Test']) + + assert.equal(problems.length, 2, 'exactly one problem per platform') + for (const platform of ['ios', 'android']) { + const reported = problems.some( + (p) => p.startsWith(`[${platform}]`) && p.includes('nested inside the platform map') + ) + assert.ok(reported, `${platform} must report the nested "deferred" key`) + } +}) + test('an unassigned runner is reported', () => { const problems = validateTestGroups(groups, [...generatedRunners(), 'runBrandNewTest']) assert.equal(problems.length, platformNames(groups).length) diff --git a/packages/vla-ggml/scripts/lib/validate-test-groups.js b/packages/vla-ggml/scripts/lib/validate-test-groups.js index b0f1213ed2..51c3d0a3d8 100644 --- a/packages/vla-ggml/scripts/lib/validate-test-groups.js +++ b/packages/vla-ggml/scripts/lib/validate-test-groups.js @@ -73,6 +73,20 @@ function validateTestGroups(groups, runners) { const deferredSet = new Set(deferred) for (const platform of platforms) { + // `deferred` nested inside a platform is indistinguishable from a shard: to + // `coveredRunners` below it is just another array of runner names, so the + // whole file would validate clean — and `upload-to-devicefarm` turns every + // `{ groupName: [runners] }` entry into a Device Farm spec, so those runners + // would be scheduled (and billed) under a shard literally named "deferred". + // Reserving the name here is what makes the top-level rule enforceable + // rather than merely documented. + if (Object.prototype.hasOwnProperty.call(groups[platform], DEFERRED_KEY)) { + problems.push( + `[${platform}] "${DEFERRED_KEY}" is nested inside the platform map.\n` + + `It must be a top-level key: nested here it is scheduled as a real Device Farm shard.` + ) + } + const covered = new Set(coveredRunners(groups[platform])) const missing = runners.filter((name) => !covered.has(name) && !deferredSet.has(name)) diff --git a/packages/vla-ggml/scripts/validate-mobile-tests.js b/packages/vla-ggml/scripts/validate-mobile-tests.js index d7c8001f64..81c2452d09 100644 --- a/packages/vla-ggml/scripts/validate-mobile-tests.js +++ b/packages/vla-ggml/scripts/validate-mobile-tests.js @@ -84,23 +84,15 @@ try { process.exit(0) } - // Keep timestamp validation as a fast stale-content signal for edited tests. - // Skipped in CI: a fresh clone stamps every working-tree file at checkout - // time, so the ordering this compares is meaningless there and would fail at - // random. The reference checks above are content-based and cover CI. - if (!process.env.CI) { - const latestIntegrationTime = Math.max( - ...integrationFiles.map((f) => fs.statSync(path.join(integrationDir, f)).mtimeMs) - ) - const mobileAutoTime = fs.statSync(mobileAutoFile).mtimeMs - - if (latestIntegrationTime > mobileAutoTime) { - console.error('❌ Mobile integration tests are out of date!') - console.error(' Integration tests modified after mobile tests were generated.') - console.error(' Run: npm run test:mobile:generate') - process.exit(1) - } - } + // There is deliberately no mtime comparison here. `buildFileContents` + // (generate-mobile-integration-tests.js) derives integration.auto.cjs from the + // sorted *filenames* under test/integration/ and never opens a test file, so + // editing a test's body cannot make the generated file stale. A timestamp + // check can therefore only produce false positives — and since this script now + // runs as part of `npm run test:unit`, each one would be a hard failure telling + // the author to regenerate a byte-identical file. Every staleness it could + // legitimately catch (a test added, renamed or removed) is already caught by + // the content-based reference diff above. // Device Farm shard coverage. This lives here rather than in the generator so // that a mobile scheduling mistake can never abort `npm run test:integration` diff --git a/packages/vla-ggml/test/mobile/README.md b/packages/vla-ggml/test/mobile/README.md index 6bdd7ee61f..96c7be2df7 100644 --- a/packages/vla-ggml/test/mobile/README.md +++ b/packages/vla-ggml/test/mobile/README.md @@ -19,6 +19,8 @@ npm run test:mobile:generate The generator walks `test/integration/`, derives a function name per test file, and rewrites `integration.auto.cjs`. It **only generates** — it performs no `test-groups.json` validation, because `npm run test:integration` chains it, so anything that throws there takes desktop integration tests down on every platform. +Run it **by hand** after adding, renaming or removing a `test/integration/*.test.js` file. `npm test` runs `test:unit` before `test:integration`, and `test:unit` now validates the committed `integration.auto.cjs`, so it will fail first and tell you to regenerate rather than regenerating for you. Editing the *body* of an existing test needs no regeneration: the generated file is derived from filenames alone. + ## `test-groups.json` and deferred runners `test-groups.json` is the per-platform Device Farm shard split. Every runner in `integration.auto.cjs` must either appear in a group for each platform, or be listed under the top-level `deferred` key: From 0afebd90653bac7a49921f938b6a4d17477c739f Mon Sep 17 00:00:00 2001 From: IC Date: Tue, 25 Aug 2026 10:15:41 +0200 Subject: [PATCH 3/3] fix[notask]: share the runner extractor, allow per-platform deferral Five review findings on the mobile shard validation split. 1. The unit suite asserted on its own copy of the runner-name regex. scripts/__tests__/mobile-test-groups.test.js carried a byte-copy of getGeneratedRunnerNames from validate-mobile-tests.js, so it proved nothing about the extractor that actually runs in CI. It is now generatedRunnerNames in lib/validate-test-groups.js, called by both. It takes the file contents rather than a path, so the lib stays fs-free. A new test pins it against the real committed integration.auto.cjs: a template change that renames the declarations can no longer yield zero runners, which would make every coverage rule below vacuously pass. 2. `deferred` was global, so per-platform deferral was unrepresentable. The contradiction rule runs inside the platform loop against one flat set, so "deferred on ios, scheduled on android" always reported a contradiction on android. Fine for pi05, but it blocks the llm-llamacpp and ocr-ggml ports, where the platform sets differ. `deferred` now accepts either a flat array (every platform, what VLA uses) or a `{ : [runner] }` map. That is also why platformNames now excludes DEFERRED_KEY by NAME rather than by shape: an object-form `deferred` is a non-array object, so shape-based inference would have read it as a platform demanding full coverage. A map keyed by a non-platform (`deferred: { io: [...] }`) deferred nothing while reading as a clean file, so that is reported too. 3. isPlatformEntry treated every top-level object as a platform. llm-llamacpp ships top-level iosWeekly/androidWeekly maps that are schedules, not platforms, so the same lib there would demand full coverage of them. validateTestGroups now takes options.platforms to pin the list. Inference stays the default, so VLA still picks up a newly added platform key automatically. A pinned platform absent from the file is now reported instead of silently passing. 4. The reference diff does not cover generator-template changes. getGeneratedIntegrationRefs compares only the runIntegrationModule paths, so editing the generator's own template (the __shouldRunTest guard, the header comments) leaves the committed file stale with both checks green. Wording softened to say so. Nothing regressed: the removed mtime check compared test-file timestamps, not the generator's, so it never caught this either. Not closed here because the generator requires bare-fs, so buildFileContents is not callable from the node validator. 5. No documented way to run a deferred runner on device. validate-devices builds its runner allowlist from `.` too, so a manual dispatch with `-f tests=runPi05Test` is rejected as an unknown runner. test/mobile/README.md now documents both halves of the gate: un-deferring the runner is not enough, because _skipMobilePi05 skips both pi05 cases on mobile regardless of scheduling, so that alone runs zero tests and passes green. Verified: 16/16 mobile-test-groups tests pass (up from 9) and 8/8 pre-existing prestage tests, no regression; validate-mobile-tests.js exits 0 with "4 runner(s), group coverage OK"; prettier --check with the real prettier-config-holepunch and node --check are clean on every changed file. The generator is untouched, so integration.auto.cjs is unaffected. Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/mobile-test-groups.test.js | 106 +++++++++++++++++- .../scripts/lib/validate-test-groups.js | 103 ++++++++++++++--- .../vla-ggml/scripts/validate-mobile-tests.js | 23 ++-- packages/vla-ggml/test/mobile/README.md | 16 +++ 4 files changed, 214 insertions(+), 34 deletions(-) diff --git a/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js index 401535223d..fe7d992b7b 100644 --- a/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js +++ b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js @@ -5,18 +5,33 @@ const assert = require('node:assert/strict') const fs = require('node:fs') const path = require('node:path') -const { validateTestGroups, platformNames } = require('../lib/validate-test-groups.js') +const { + validateTestGroups, + generatedRunnerNames, + platformNames +} = require('../lib/validate-test-groups.js') const groups = require('../../test/mobile/test-groups.json') const integrationAutoPath = path.resolve(__dirname, '../../test/mobile/integration.auto.cjs') -function generatedRunners() { - const content = fs.readFileSync(integrationAutoPath, 'utf8') - return Array.from(content.matchAll(/^async function (run[A-Za-z0-9_]+)\s*\(/gm), (m) => m[1]) +// The extractor under test, not a copy of it: validate-mobile-tests.js calls the +// same `generatedRunnerNames`, so a change to the pattern is caught here. +function committedRunners() { + return generatedRunnerNames(fs.readFileSync(integrationAutoPath, 'utf8')) } test('the committed test-groups.json covers every generated runner', () => { - assert.deepEqual(validateTestGroups(groups, generatedRunners()), []) + assert.deepEqual(validateTestGroups(groups, committedRunners()), []) +}) + +test('the runner extractor reads the committed integration.auto.cjs', () => { + // Pins the shared extractor against real generated output, so a template + // change that renames the declarations cannot silently yield zero runners — + // which would make every coverage rule below vacuously pass. + const runners = committedRunners() + assert.ok(runners.length > 0, 'integration.auto.cjs must declare at least one runner') + assert.ok(runners.includes('runPi05Test')) + assert.ok(runners.every((name) => name.startsWith('run'))) }) test('deferred runners are declared, not silently absent', () => { @@ -40,6 +55,17 @@ test('"deferred" is a top-level key, never a platform', () => { assert.deepEqual(platformNames(groups).sort(), ['android', 'ios']) }) +test('an object-form "deferred" is still not a platform', () => { + // Inference excludes `deferred` by name, not by shape, so the per-platform map + // form below cannot be mistaken for a platform needing full coverage. + const perPlatform = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'] }, + deferred: { ios: ['runPi05Test'] } + } + assert.deepEqual(platformNames(perPlatform).sort(), ['android', 'ios']) +}) + test('a "deferred" key nested inside a platform is reported', () => { // The assertion above only covers the committed file's shape. This covers the // hazard itself: nested under a platform, `deferred` is just another array of @@ -60,8 +86,67 @@ test('a "deferred" key nested inside a platform is reported', () => { } }) +test('deferral can be scoped per platform', () => { + // Deferring on one platform while scheduling on another is what llm-llamacpp + // and ocr-ggml need; the flat array form cannot express it, because the + // scheduled-and-deferred rule would fire on the platform that does run it. + const perPlatform = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'], heavy: ['runBigTest'] }, + deferred: { ios: ['runBigTest'] } + } + assert.deepEqual(validateTestGroups(perPlatform, ['runAddonTest', 'runBigTest']), []) + + const flat = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'], heavy: ['runBigTest'] }, + deferred: ['runBigTest'] + } + const problems = validateTestGroups(flat, ['runAddonTest', 'runBigTest']) + assert.ok(problems.some((p) => p.startsWith('[android]') && p.includes('both scheduled'))) +}) + +test('a per-platform "deferred" keyed by a non-platform is reported', () => { + // `io` defers nothing, so runBigTest is still unassigned on ios — without this + // rule the typo reads as a clean file. + const typo = { + ios: { smolvla: ['runAddonTest'] }, + deferred: { io: ['runBigTest'] } + } + const problems = validateTestGroups(typo, ['runAddonTest', 'runBigTest']) + assert.ok(problems.some((p) => p.includes('not platforms') && p.includes('io'))) + assert.ok(problems.some((p) => p.startsWith('[ios]') && p.includes('runBigTest'))) +}) + +test('an explicit platform list overrides shape inference', () => { + // llm-llamacpp ships top-level `iosWeekly`/`androidWeekly` maps that are + // schedules, not platforms; inferring from shape would demand full coverage of + // them too. Callers there pin the platform list instead. + const withSchedules = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'] }, + iosWeekly: { nightly: ['runAddonTest'] }, + androidWeekly: { nightly: ['runAddonTest'] } + } + const inferred = validateTestGroups(withSchedules, ['runAddonTest', 'runBigTest']) + assert.equal(inferred.length, 4, 'inference treats the weekly maps as platforms') + + const pinned = validateTestGroups(withSchedules, ['runAddonTest', 'runBigTest'], { + platforms: ['ios', 'android'] + }) + assert.equal(pinned.length, 2) + assert.ok(pinned.every((p) => p.startsWith('[ios]') || p.startsWith('[android]'))) +}) + +test('a pinned platform missing from the file is reported', () => { + const problems = validateTestGroups({ ios: { smolvla: ['runAddonTest'] } }, ['runAddonTest'], { + platforms: ['ios', 'android'] + }) + assert.ok(problems.some((p) => p.startsWith('[android]') && p.includes('no `{'))) +}) + test('an unassigned runner is reported', () => { - const problems = validateTestGroups(groups, [...generatedRunners(), 'runBrandNewTest']) + const problems = validateTestGroups(groups, [...committedRunners(), 'runBrandNewTest']) assert.equal(problems.length, platformNames(groups).length) assert.ok(problems.every((p) => p.includes('runBrandNewTest'))) }) @@ -84,6 +169,15 @@ test('a stale deferred entry is reported', () => { assert.ok(problems.some((p) => p.includes('runRemovedTest') && p.includes('do not exist'))) }) +test('a stale entry in a per-platform deferred is reported', () => { + const stale = { + ios: { smolvla: ['runAddonTest'] }, + deferred: { ios: ['runRemovedTest'] } + } + const problems = validateTestGroups(stale, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('runRemovedTest') && p.includes('do not exist'))) +}) + test('a runner that is both scheduled and deferred is reported', () => { const contradictory = { ios: { smolvla: ['runAddonTest'] }, diff --git a/packages/vla-ggml/scripts/lib/validate-test-groups.js b/packages/vla-ggml/scripts/lib/validate-test-groups.js index 51c3d0a3d8..ce180ffc8d 100644 --- a/packages/vla-ggml/scripts/lib/validate-test-groups.js +++ b/packages/vla-ggml/scripts/lib/validate-test-groups.js @@ -21,38 +21,85 @@ // instead schedule it as a real shard. const DEFERRED_KEY = 'deferred' +// integration.auto.cjs declares one `async function run` per on-device +// test; once it is confirmed in sync with test/integration/, those declarations +// are the authoritative runner-name list — the same source that +// .github/actions/run-mobile-integration-tests/validate-devices greps. +// +// The extractor lives here, beside the rules that consume its output, so +// validate-mobile-tests.js and the unit tests share one implementation instead +// of each keeping its own copy of the pattern (a test asserting on its own copy +// proves nothing about the extractor that actually runs). It takes the file +// contents rather than a path to keep this module fs-free. +function generatedRunnerNames(content) { + const declaration = /^async function (run[A-Za-z0-9_]+)\s*\(/gm + return Array.from(content.matchAll(declaration), (m) => m[1]) +} + // A platform entry is a `{ groupName: [runner, ...] }` map. Anything else at the -// top level is metadata for another consumer — `deferred` here, OCR's -// `perf_report_filter` — and is not a platform. +// top level is metadata for another consumer — OCR's `perf_report_filter`, or an +// object-form `deferred` — and is not a platform. function isPlatformEntry(value) { return typeof value === 'object' && value !== null && !Array.isArray(value) } -function platformNames(groups) { - return Object.keys(groups).filter((key) => isPlatformEntry(groups[key])) +// `platforms` overrides inference. Inference is a convenience for addons whose +// only top-level maps *are* platforms, which is VLA's case; it is not safe +// everywhere. llm-llamacpp ships top-level `iosWeekly`/`androidWeekly` maps that +// are schedules rather than platforms of their own, so a caller there must pass +// the platform list explicitly or full coverage would be demanded of them too. +function platformNames(groups, platforms) { + if (platforms) { + return [...platforms] + } + return Object.keys(groups).filter((key) => key !== DEFERRED_KEY && isPlatformEntry(groups[key])) } function coveredRunners(platformEntry) { return Object.values(platformEntry).filter(Array.isArray).flat() } -function deferredRunners(groups) { +// `deferred` is either a flat array — deferred on every platform, which is what +// VLA uses — or a `{ : [runner, ...] }` map, for a runner that is +// scheduled on one platform and deferred on another. The map form is why +// `platformNames` excludes DEFERRED_KEY by name rather than by shape. +function deferredRunners(groups, platform) { const deferred = groups[DEFERRED_KEY] - return Array.isArray(deferred) ? deferred : [] + if (Array.isArray(deferred)) { + return deferred + } + if (isPlatformEntry(deferred)) { + const forPlatform = deferred[platform] + return Array.isArray(forPlatform) ? forPlatform : [] + } + return [] +} + +function allDeferredRunners(groups) { + const deferred = groups[DEFERRED_KEY] + if (Array.isArray(deferred)) { + return deferred + } + if (isPlatformEntry(deferred)) { + return Object.values(deferred).filter(Array.isArray).flat() + } + return [] } // Returns a list of human-readable problem strings; empty means valid. // `runners` is the authoritative runner-name list, derived from the generated -// integration.auto.cjs by the caller. -function validateTestGroups(groups, runners) { +// integration.auto.cjs by the caller. `options.platforms` pins the platforms +// that must be covered instead of inferring them from the file's shape. +function validateTestGroups(groups, runners, options = {}) { const problems = [] const known = new Set(runners) - const deferred = deferredRunners(groups) // A stale `deferred` entry is worse than a noisy one: it would silently // excuse a runner that no longer exists, and mask a real gap if the name is // ever reused. - const unknownDeferred = deferred.filter((name) => !known.has(name)) + const unknownDeferred = [...new Set(allDeferredRunners(groups))].filter( + (name) => !known.has(name) + ) if (unknownDeferred.length) { problems.push( `[${DEFERRED_KEY}] lists runners that do not exist:\n ` + @@ -61,7 +108,7 @@ function validateTestGroups(groups, runners) { ) } - const platforms = platformNames(groups) + const platforms = platformNames(groups, options.platforms) if (platforms.length === 0) { problems.push( 'test-groups.json declares no platform maps.\n' + @@ -70,9 +117,31 @@ function validateTestGroups(groups, runners) { return problems } - const deferredSet = new Set(deferred) + // A per-platform `deferred` keyed by a name that is not a platform defers + // nothing, so a typo there reads as a clean file while the runner stays + // unassigned on the platform it was meant to excuse. + const deferredKeyed = groups[DEFERRED_KEY] + if (isPlatformEntry(deferredKeyed)) { + const unknownPlatforms = Object.keys(deferredKeyed).filter((key) => !platforms.includes(key)) + if (unknownPlatforms.length) { + problems.push( + `[${DEFERRED_KEY}] is keyed by names that are not platforms:\n ` + + unknownPlatforms.join('\n ') + + `\nExpected one of: ${platforms.join(', ')}.` + ) + } + } for (const platform of platforms) { + const entry = groups[platform] + if (!isPlatformEntry(entry)) { + problems.push( + `[${platform}] is required to be covered but test-groups.json has no ` + + `\`{ "": [runners] }\` map for it.` + ) + continue + } + // `deferred` nested inside a platform is indistinguishable from a shard: to // `coveredRunners` below it is just another array of runner names, so the // whole file would validate clean — and `upload-to-devicefarm` turns every @@ -80,14 +149,15 @@ function validateTestGroups(groups, runners) { // would be scheduled (and billed) under a shard literally named "deferred". // Reserving the name here is what makes the top-level rule enforceable // rather than merely documented. - if (Object.prototype.hasOwnProperty.call(groups[platform], DEFERRED_KEY)) { + if (Object.prototype.hasOwnProperty.call(entry, DEFERRED_KEY)) { problems.push( `[${platform}] "${DEFERRED_KEY}" is nested inside the platform map.\n` + `It must be a top-level key: nested here it is scheduled as a real Device Farm shard.` ) } - const covered = new Set(coveredRunners(groups[platform])) + const covered = new Set(coveredRunners(entry)) + const deferredSet = new Set(deferredRunners(groups, platform)) const missing = runners.filter((name) => !covered.has(name) && !deferredSet.has(name)) if (missing.length) { @@ -109,7 +179,9 @@ function validateTestGroups(groups, runners) { } // A runner in both a shard and `deferred` is contradictory: it would run on - // device while claiming to be deferred. + // device while claiming to be deferred. Scoped per platform, so the map form + // of `deferred` can legitimately defer a runner on ios while android + // schedules it. const contradictory = [...covered].filter((name) => deferredSet.has(name)) if (contradictory.length) { problems.push( @@ -126,6 +198,7 @@ function validateTestGroups(groups, runners) { module.exports = { DEFERRED_KEY, validateTestGroups, + generatedRunnerNames, platformNames, deferredRunners } diff --git a/packages/vla-ggml/scripts/validate-mobile-tests.js b/packages/vla-ggml/scripts/validate-mobile-tests.js index 81c2452d09..c925ca0c8a 100644 --- a/packages/vla-ggml/scripts/validate-mobile-tests.js +++ b/packages/vla-ggml/scripts/validate-mobile-tests.js @@ -4,7 +4,7 @@ const fs = require('fs') const path = require('path') -const { validateTestGroups } = require('./lib/validate-test-groups.js') +const { validateTestGroups, generatedRunnerNames } = require('./lib/validate-test-groups.js') const repoRoot = path.resolve(__dirname, '..') const integrationDir = path.join(repoRoot, 'test', 'integration') @@ -35,14 +35,6 @@ function getGeneratedIntegrationRefs(content) { return references } -// integration.auto.cjs declares one `async function run` per on-device -// test. Once it is confirmed in sync with test/integration (above), it is the -// authoritative runner-name list — the same source .github/actions/ -// run-mobile-integration-tests/validate-devices uses. -function getGeneratedRunnerNames(content) { - return Array.from(content.matchAll(/^async function (run[A-Za-z0-9_]+)\s*\(/gm), (m) => m[1]) -} - function setDiff(left, right) { return [...left].filter((item) => !right.has(item)).sort() } @@ -90,9 +82,14 @@ try { // editing a test's body cannot make the generated file stale. A timestamp // check can therefore only produce false positives — and since this script now // runs as part of `npm run test:unit`, each one would be a hard failure telling - // the author to regenerate a byte-identical file. Every staleness it could - // legitimately catch (a test added, renamed or removed) is already caught by - // the content-based reference diff above. + // the author to regenerate a byte-identical file. + // + // The reference diff above covers the staleness that matters day to day: a + // test file added, renamed or removed. It does not cover a change to the + // generator's own template (the `__shouldRunTest` guard, the header comments), + // since it compares only the `runIntegrationModule` paths — that still needs a + // manual `npm run test:mobile:generate`. The mtime check did not catch that + // either: it compared test-file timestamps, not the generator's. // Device Farm shard coverage. This lives here rather than in the generator so // that a mobile scheduling mistake can never abort `npm run test:integration` @@ -103,7 +100,7 @@ try { } const groups = JSON.parse(fs.readFileSync(groupsFile, 'utf8')) - const runners = getGeneratedRunnerNames(mobileAutoContent) + const runners = generatedRunnerNames(mobileAutoContent) const problems = validateTestGroups(groups, runners) if (problems.length > 0) { diff --git a/packages/vla-ggml/test/mobile/README.md b/packages/vla-ggml/test/mobile/README.md index 96c7be2df7..30db5d17e6 100644 --- a/packages/vla-ggml/test/mobile/README.md +++ b/packages/vla-ggml/test/mobile/README.md @@ -37,6 +37,22 @@ Run it **by hand** after adding, renaming or removing a `test/integration/*.test `deferred` **must stay a top-level key**. The CI composites consume only `.` and ignore every other top-level key (as OCR's `perf_report_filter` already relies on), so nesting `deferred` inside `ios`/`android` would schedule it as a real Device Farm shard. +A runner may also be deferred on one platform only, by keying `deferred` by platform instead of using a flat list: + +```json +{ "deferred": { "ios": ["runBigTest"] } } +``` + +### Running a deferred runner on device + +`validate-devices` builds its runner allowlist from `.` too, so a manual `workflow_dispatch` with `-f tests=runPi05Test` is rejected as an unknown runner — the deferral is enforced at both ends. To run pi05 on device from a branch: + +1. Move `runPi05Test` out of `deferred` into a group in `test-groups.json` (or into a platform-scoped `deferred` for the other platform). +2. Relax `_skipMobilePi05` in `test/integration/pi05.test.js` — it skips both pi05 cases on mobile regardless of scheduling, so step 1 alone runs zero tests and passes green. +3. Dispatch `integration-mobile-test-vla.yml` with `-f tests=runPi05Test` to collapse the run to a single spec. + +Step 1 alone is what CI validates; step 2 is why deferral is recorded here rather than enforced only by the skip. + Coverage is enforced by: ```bash