diff --git a/packages/vla-ggml/package.json b/packages/vla-ggml/package.json index 9728d61b44..2df0ea3206 100644 --- a/packages/vla-ggml/package.json +++ b/packages/vla-ggml/package.json @@ -24,8 +24,9 @@ "test:integration": "npm run build:ts && npm run test:integration:generate && bare test/integration/all.js --exit", "test:integration:generate": "brittle -r test/integration/all.js test/integration/*.test.js && npm run test:mobile:generate", "test:unit:generate": "brittle -r test/unit/all.js test/unit/*.test.js", - "test:unit": "npm run build:ts && npm run test:unit:generate && bare test/unit/all.js --exit && npm run test:prestage", + "test:unit": "npm run build:ts && npm run test:unit:generate && bare test/unit/all.js --exit && npm run test:prestage && npm run test:mobile:groups && npm run test:mobile:validate", "test:prestage": "node --test scripts/__tests__/generate-prestage-block.test.js", + "test:mobile:groups": "node --test scripts/__tests__/mobile-test-groups.test.js", "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target addon-test", "test:cpp:run": "cd build/test/unit/ && ./addon-test --gtest_output=xml:cpp-test-results.xml", "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", diff --git a/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js new file mode 100644 index 0000000000..fe7d992b7b --- /dev/null +++ b/packages/vla-ggml/scripts/__tests__/mobile-test-groups.test.js @@ -0,0 +1,200 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const path = require('node:path') + +const { + validateTestGroups, + generatedRunnerNames, + platformNames +} = require('../lib/validate-test-groups.js') +const groups = require('../../test/mobile/test-groups.json') + +const integrationAutoPath = path.resolve(__dirname, '../../test/mobile/integration.auto.cjs') + +// The extractor under test, not a copy of it: validate-mobile-tests.js calls the +// same `generatedRunnerNames`, so a change to the pattern is caught here. +function committedRunners() { + return generatedRunnerNames(fs.readFileSync(integrationAutoPath, 'utf8')) +} + +test('the committed test-groups.json covers every generated runner', () => { + assert.deepEqual(validateTestGroups(groups, committedRunners()), []) +}) + +test('the runner extractor reads the committed integration.auto.cjs', () => { + // Pins the shared extractor against real generated output, so a template + // change that renames the declarations cannot silently yield zero runners — + // which would make every coverage rule below vacuously pass. + const runners = committedRunners() + assert.ok(runners.length > 0, 'integration.auto.cjs must declare at least one runner') + assert.ok(runners.includes('runPi05Test')) + assert.ok(runners.every((name) => name.startsWith('run'))) +}) + +test('deferred runners are declared, not silently absent', () => { + // pi05 mobile coverage is deferred pending a project-owned CDN mirror, and + // pi05.test.js is gated on-device by `_skipMobilePi05`. Recording it here is + // what keeps "not scheduled" distinguishable from "forgotten". + assert.deepEqual(groups.deferred, ['runPi05Test']) + for (const platform of platformNames(groups)) { + const scheduled = Object.values(groups[platform]).flat() + assert.ok( + !scheduled.includes('runPi05Test'), + `runPi05Test must not be scheduled on ${platform}` + ) + } +}) + +test('"deferred" is a top-level key, never a platform', () => { + // The Device Farm composites read only `.`, so a `deferred` key + // nested inside ios/android would be scheduled as a real shard. + assert.ok(!platformNames(groups).includes('deferred')) + assert.deepEqual(platformNames(groups).sort(), ['android', 'ios']) +}) + +test('an object-form "deferred" is still not a platform', () => { + // Inference excludes `deferred` by name, not by shape, so the per-platform map + // form below cannot be mistaken for a platform needing full coverage. + const perPlatform = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'] }, + deferred: { ios: ['runPi05Test'] } + } + assert.deepEqual(platformNames(perPlatform).sort(), ['android', 'ios']) +}) + +test('a "deferred" key nested inside a platform is reported', () => { + // The assertion above only covers the committed file's shape. This covers the + // hazard itself: nested under a platform, `deferred` is just another array of + // runner names, so every other rule is satisfied and the file would otherwise + // validate clean — while upload-to-devicefarm schedules it as a real shard. + const nested = { + ios: { smolvla: ['runAddonTest'], deferred: ['runPi05Test'] }, + android: { smolvla: ['runAddonTest'], deferred: ['runPi05Test'] } + } + const problems = validateTestGroups(nested, ['runAddonTest', 'runPi05Test']) + + assert.equal(problems.length, 2, 'exactly one problem per platform') + for (const platform of ['ios', 'android']) { + const reported = problems.some( + (p) => p.startsWith(`[${platform}]`) && p.includes('nested inside the platform map') + ) + assert.ok(reported, `${platform} must report the nested "deferred" key`) + } +}) + +test('deferral can be scoped per platform', () => { + // Deferring on one platform while scheduling on another is what llm-llamacpp + // and ocr-ggml need; the flat array form cannot express it, because the + // scheduled-and-deferred rule would fire on the platform that does run it. + const perPlatform = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'], heavy: ['runBigTest'] }, + deferred: { ios: ['runBigTest'] } + } + assert.deepEqual(validateTestGroups(perPlatform, ['runAddonTest', 'runBigTest']), []) + + const flat = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'], heavy: ['runBigTest'] }, + deferred: ['runBigTest'] + } + const problems = validateTestGroups(flat, ['runAddonTest', 'runBigTest']) + assert.ok(problems.some((p) => p.startsWith('[android]') && p.includes('both scheduled'))) +}) + +test('a per-platform "deferred" keyed by a non-platform is reported', () => { + // `io` defers nothing, so runBigTest is still unassigned on ios — without this + // rule the typo reads as a clean file. + const typo = { + ios: { smolvla: ['runAddonTest'] }, + deferred: { io: ['runBigTest'] } + } + const problems = validateTestGroups(typo, ['runAddonTest', 'runBigTest']) + assert.ok(problems.some((p) => p.includes('not platforms') && p.includes('io'))) + assert.ok(problems.some((p) => p.startsWith('[ios]') && p.includes('runBigTest'))) +}) + +test('an explicit platform list overrides shape inference', () => { + // llm-llamacpp ships top-level `iosWeekly`/`androidWeekly` maps that are + // schedules, not platforms; inferring from shape would demand full coverage of + // them too. Callers there pin the platform list instead. + const withSchedules = { + ios: { smolvla: ['runAddonTest'] }, + android: { smolvla: ['runAddonTest'] }, + iosWeekly: { nightly: ['runAddonTest'] }, + androidWeekly: { nightly: ['runAddonTest'] } + } + const inferred = validateTestGroups(withSchedules, ['runAddonTest', 'runBigTest']) + assert.equal(inferred.length, 4, 'inference treats the weekly maps as platforms') + + const pinned = validateTestGroups(withSchedules, ['runAddonTest', 'runBigTest'], { + platforms: ['ios', 'android'] + }) + assert.equal(pinned.length, 2) + assert.ok(pinned.every((p) => p.startsWith('[ios]') || p.startsWith('[android]'))) +}) + +test('a pinned platform missing from the file is reported', () => { + const problems = validateTestGroups({ ios: { smolvla: ['runAddonTest'] } }, ['runAddonTest'], { + platforms: ['ios', 'android'] + }) + assert.ok(problems.some((p) => p.startsWith('[android]') && p.includes('no `{'))) +}) + +test('an unassigned runner is reported', () => { + const problems = validateTestGroups(groups, [...committedRunners(), 'runBrandNewTest']) + assert.equal(problems.length, platformNames(groups).length) + assert.ok(problems.every((p) => p.includes('runBrandNewTest'))) +}) + +test('a typo in a group is reported', () => { + const typo = { + ios: { smolvla: ['runAddonTest', 'runTypoTest'] }, + deferred: [] + } + const problems = validateTestGroups(typo, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('runTypoTest') && p.includes('do not exist'))) +}) + +test('a stale deferred entry is reported', () => { + const stale = { + ios: { smolvla: ['runAddonTest'] }, + deferred: ['runRemovedTest'] + } + const problems = validateTestGroups(stale, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('runRemovedTest') && p.includes('do not exist'))) +}) + +test('a stale entry in a per-platform deferred is reported', () => { + const stale = { + ios: { smolvla: ['runAddonTest'] }, + deferred: { ios: ['runRemovedTest'] } + } + const problems = validateTestGroups(stale, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('runRemovedTest') && p.includes('do not exist'))) +}) + +test('a runner that is both scheduled and deferred is reported', () => { + const contradictory = { + ios: { smolvla: ['runAddonTest'] }, + deferred: ['runAddonTest'] + } + const problems = validateTestGroups(contradictory, ['runAddonTest']) + assert.ok(problems.some((p) => p.includes('both scheduled and listed'))) +}) + +test('metadata keys that are not platform maps are ignored', () => { + // OCR ships a top-level `perf_report_filter` string; the shape must tolerate + // sibling metadata without treating it as a platform. + const withMetadata = { + ios: { smolvla: ['runAddonTest'] }, + perf_report_filter: 'something|else', + deferred: [] + } + assert.deepEqual(platformNames(withMetadata), ['ios']) + assert.deepEqual(validateTestGroups(withMetadata, ['runAddonTest']), []) +}) diff --git a/packages/vla-ggml/scripts/generate-mobile-integration-tests.js b/packages/vla-ggml/scripts/generate-mobile-integration-tests.js index 325390c99b..9094876896 100644 --- a/packages/vla-ggml/scripts/generate-mobile-integration-tests.js +++ b/packages/vla-ggml/scripts/generate-mobile-integration-tests.js @@ -7,7 +7,6 @@ const repoRoot = path.resolve(__dirname, '..') const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileDir = path.join(repoRoot, 'test', 'mobile') const outputFile = path.join(mobileDir, 'integration.auto.cjs') -const groupsFile = path.join(mobileDir, 'test-groups.json') function getIntegrationFiles() { if (!fs.existsSync(integrationDir)) { @@ -64,50 +63,21 @@ function buildFileContents(files) { return `${lines.join('\n')}\n` } -function validateGroups(functionNames) { - if (!fs.existsSync(groupsFile)) { - console.warn('[warn] test-groups.json not found — skipping split validation') - return - } - const groups = JSON.parse(fs.readFileSync(groupsFile, 'utf-8')) - const nameSet = new Set(functionNames) - for (const [platform, splits] of Object.entries(groups)) { - const covered = new Set(Object.values(splits).flat()) - const missing = functionNames.filter((n) => !covered.has(n)) - const extra = [...covered].filter((n) => !nameSet.has(n)) - if (missing.length) { - throw new Error( - '[' + - platform + - '] Tests not assigned to any group in test-groups.json:\n ' + - missing.join('\n ') + - '\nAdd them to a group in test/mobile/test-groups.json.' - ) - } - if (extra.length) { - throw new Error( - '[' + - platform + - '] test-groups.json references non-existent tests:\n ' + - extra.join('\n ') + - '\nRemove them or check for typos.' - ) - } - } - console.log('Group coverage validated — all tests assigned for every platform.') -} - +// NOTE: this generator deliberately performs no test-groups.json validation. +// `npm run test:integration` chains it (so the committed integration.auto.cjs +// can never go stale), which means anything that throws here takes desktop +// integration tests down on every platform. Group coverage is a mobile +// scheduling concern, so it lives in `npm run test:mobile:validate` +// (scripts/validate-mobile-tests.js) and runs in the ungated ts-checks job. function main() { const files = getIntegrationFiles() if (files.length === 0) { throw new Error(`No integration test files found inside ${integrationDir}`) } - const functionNames = files.map(toFunctionName) const content = buildFileContents(files) fs.writeFileSync(outputFile, content, 'utf8') console.log(`Generated ${outputFile} with ${files.length} integration runners.`) - validateGroups(functionNames) } if (require.main === module) { diff --git a/packages/vla-ggml/scripts/lib/validate-test-groups.js b/packages/vla-ggml/scripts/lib/validate-test-groups.js new file mode 100644 index 0000000000..ce180ffc8d --- /dev/null +++ b/packages/vla-ggml/scripts/lib/validate-test-groups.js @@ -0,0 +1,204 @@ +'use strict' + +// Group-coverage rules for test/mobile/test-groups.json. +// +// Deliberately dependency-free and side-effect-free (no fs, no process.exit) so +// the same rules run under `node` from validate-mobile-tests.js and are unit +// testable from scripts/__tests__/mobile-test-groups.test.js. +// +// This check lives OUTSIDE the generator on purpose. It answers a mobile +// scheduling question — "is every on-device runner assigned to a Device Farm +// shard?" — which has no bearing on whether integration.auto.cjs was written +// correctly. Bundling it into the generator once let a Device Farm scheduling +// edit abort `npm run test:integration`, taking desktop CI down on all seven +// platforms (PR #4006). + +// Runners deliberately not scheduled on Device Farm are listed under this +// top-level key. It sits beside the platform maps rather than inside one +// because the CI composites consume only `.` and ignore every other +// top-level key (see .github/actions/run-mobile-integration-tests/ +// upload-to-devicefarm/action.yml). Nesting it under `ios`/`android` would +// instead schedule it as a real shard. +const DEFERRED_KEY = 'deferred' + +// integration.auto.cjs declares one `async function run` per on-device +// test; once it is confirmed in sync with test/integration/, those declarations +// are the authoritative runner-name list — the same source that +// .github/actions/run-mobile-integration-tests/validate-devices greps. +// +// The extractor lives here, beside the rules that consume its output, so +// validate-mobile-tests.js and the unit tests share one implementation instead +// of each keeping its own copy of the pattern (a test asserting on its own copy +// proves nothing about the extractor that actually runs). It takes the file +// contents rather than a path to keep this module fs-free. +function generatedRunnerNames(content) { + const declaration = /^async function (run[A-Za-z0-9_]+)\s*\(/gm + return Array.from(content.matchAll(declaration), (m) => m[1]) +} + +// A platform entry is a `{ groupName: [runner, ...] }` map. Anything else at the +// top level is metadata for another consumer — OCR's `perf_report_filter`, or an +// object-form `deferred` — and is not a platform. +function isPlatformEntry(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +// `platforms` overrides inference. Inference is a convenience for addons whose +// only top-level maps *are* platforms, which is VLA's case; it is not safe +// everywhere. llm-llamacpp ships top-level `iosWeekly`/`androidWeekly` maps that +// are schedules rather than platforms of their own, so a caller there must pass +// the platform list explicitly or full coverage would be demanded of them too. +function platformNames(groups, platforms) { + if (platforms) { + return [...platforms] + } + return Object.keys(groups).filter((key) => key !== DEFERRED_KEY && isPlatformEntry(groups[key])) +} + +function coveredRunners(platformEntry) { + return Object.values(platformEntry).filter(Array.isArray).flat() +} + +// `deferred` is either a flat array — deferred on every platform, which is what +// VLA uses — or a `{ : [runner, ...] }` map, for a runner that is +// scheduled on one platform and deferred on another. The map form is why +// `platformNames` excludes DEFERRED_KEY by name rather than by shape. +function deferredRunners(groups, platform) { + const deferred = groups[DEFERRED_KEY] + if (Array.isArray(deferred)) { + return deferred + } + if (isPlatformEntry(deferred)) { + const forPlatform = deferred[platform] + return Array.isArray(forPlatform) ? forPlatform : [] + } + return [] +} + +function allDeferredRunners(groups) { + const deferred = groups[DEFERRED_KEY] + if (Array.isArray(deferred)) { + return deferred + } + if (isPlatformEntry(deferred)) { + return Object.values(deferred).filter(Array.isArray).flat() + } + return [] +} + +// Returns a list of human-readable problem strings; empty means valid. +// `runners` is the authoritative runner-name list, derived from the generated +// integration.auto.cjs by the caller. `options.platforms` pins the platforms +// that must be covered instead of inferring them from the file's shape. +function validateTestGroups(groups, runners, options = {}) { + const problems = [] + const known = new Set(runners) + + // A stale `deferred` entry is worse than a noisy one: it would silently + // excuse a runner that no longer exists, and mask a real gap if the name is + // ever reused. + const unknownDeferred = [...new Set(allDeferredRunners(groups))].filter( + (name) => !known.has(name) + ) + if (unknownDeferred.length) { + problems.push( + `[${DEFERRED_KEY}] lists runners that do not exist:\n ` + + unknownDeferred.join('\n ') + + '\nRemove them or check for typos.' + ) + } + + const platforms = platformNames(groups, options.platforms) + if (platforms.length === 0) { + problems.push( + 'test-groups.json declares no platform maps.\n' + + 'Expected at least one top-level `{ "": { "": [runners] } }` entry.' + ) + return problems + } + + // A per-platform `deferred` keyed by a name that is not a platform defers + // nothing, so a typo there reads as a clean file while the runner stays + // unassigned on the platform it was meant to excuse. + const deferredKeyed = groups[DEFERRED_KEY] + if (isPlatformEntry(deferredKeyed)) { + const unknownPlatforms = Object.keys(deferredKeyed).filter((key) => !platforms.includes(key)) + if (unknownPlatforms.length) { + problems.push( + `[${DEFERRED_KEY}] is keyed by names that are not platforms:\n ` + + unknownPlatforms.join('\n ') + + `\nExpected one of: ${platforms.join(', ')}.` + ) + } + } + + for (const platform of platforms) { + const entry = groups[platform] + if (!isPlatformEntry(entry)) { + problems.push( + `[${platform}] is required to be covered but test-groups.json has no ` + + `\`{ "": [runners] }\` map for it.` + ) + continue + } + + // `deferred` nested inside a platform is indistinguishable from a shard: to + // `coveredRunners` below it is just another array of runner names, so the + // whole file would validate clean — and `upload-to-devicefarm` turns every + // `{ groupName: [runners] }` entry into a Device Farm spec, so those runners + // would be scheduled (and billed) under a shard literally named "deferred". + // Reserving the name here is what makes the top-level rule enforceable + // rather than merely documented. + if (Object.prototype.hasOwnProperty.call(entry, DEFERRED_KEY)) { + problems.push( + `[${platform}] "${DEFERRED_KEY}" is nested inside the platform map.\n` + + `It must be a top-level key: nested here it is scheduled as a real Device Farm shard.` + ) + } + + const covered = new Set(coveredRunners(entry)) + const deferredSet = new Set(deferredRunners(groups, platform)) + + const missing = runners.filter((name) => !covered.has(name) && !deferredSet.has(name)) + if (missing.length) { + problems.push( + `[${platform}] runners not assigned to any group:\n ` + + missing.join('\n ') + + `\nAdd them to a group in test/mobile/test-groups.json, or to the ` + + `top-level "${DEFERRED_KEY}" list if they are intentionally not run on device.` + ) + } + + const extra = [...covered].filter((name) => !known.has(name)) + if (extra.length) { + problems.push( + `[${platform}] groups reference runners that do not exist:\n ` + + extra.join('\n ') + + '\nRemove them or check for typos.' + ) + } + + // A runner in both a shard and `deferred` is contradictory: it would run on + // device while claiming to be deferred. Scoped per platform, so the map form + // of `deferred` can legitimately defer a runner on ios while android + // schedules it. + const contradictory = [...covered].filter((name) => deferredSet.has(name)) + if (contradictory.length) { + problems.push( + `[${platform}] runners are both scheduled and listed as "${DEFERRED_KEY}":\n ` + + contradictory.join('\n ') + + `\nRemove them from one or the other.` + ) + } + } + + return problems +} + +module.exports = { + DEFERRED_KEY, + validateTestGroups, + generatedRunnerNames, + platformNames, + deferredRunners +} diff --git a/packages/vla-ggml/scripts/validate-mobile-tests.js b/packages/vla-ggml/scripts/validate-mobile-tests.js index 2a68be20c7..c925ca0c8a 100644 --- a/packages/vla-ggml/scripts/validate-mobile-tests.js +++ b/packages/vla-ggml/scripts/validate-mobile-tests.js @@ -4,9 +4,12 @@ const fs = require('fs') const path = require('path') +const { validateTestGroups, generatedRunnerNames } = require('./lib/validate-test-groups.js') + const repoRoot = path.resolve(__dirname, '..') const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileAutoFile = path.join(repoRoot, 'test', 'mobile', 'integration.auto.cjs') +const groupsFile = path.join(repoRoot, 'test', 'mobile', 'test-groups.json') function getIntegrationTestFiles() { if (!fs.existsSync(integrationDir)) { @@ -73,20 +76,42 @@ try { process.exit(0) } - // Keep timestamp validation as a fast stale-content signal for edited tests. - const latestIntegrationTime = Math.max( - ...integrationFiles.map((f) => fs.statSync(path.join(integrationDir, f)).mtimeMs) - ) - const mobileAutoTime = fs.statSync(mobileAutoFile).mtimeMs + // There is deliberately no mtime comparison here. `buildFileContents` + // (generate-mobile-integration-tests.js) derives integration.auto.cjs from the + // sorted *filenames* under test/integration/ and never opens a test file, so + // editing a test's body cannot make the generated file stale. A timestamp + // check can therefore only produce false positives — and since this script now + // runs as part of `npm run test:unit`, each one would be a hard failure telling + // the author to regenerate a byte-identical file. + // + // The reference diff above covers the staleness that matters day to day: a + // test file added, renamed or removed. It does not cover a change to the + // generator's own template (the `__shouldRunTest` guard, the header comments), + // since it compares only the `runIntegrationModule` paths — that still needs a + // manual `npm run test:mobile:generate`. The mtime check did not catch that + // either: it compared test-file timestamps, not the generator's. + + // Device Farm shard coverage. This lives here rather than in the generator so + // that a mobile scheduling mistake can never abort `npm run test:integration` + // and take desktop CI down with it. + if (!fs.existsSync(groupsFile)) { + console.log('✅ Mobile integration tests are up to date (no test-groups.json — single-spec)') + process.exit(0) + } - if (latestIntegrationTime > mobileAutoTime) { - console.error('❌ Mobile integration tests are out of date!') - console.error(' Integration tests modified after mobile tests were generated.') - console.error(' Run: npm run test:mobile:generate') + const groups = JSON.parse(fs.readFileSync(groupsFile, 'utf8')) + const runners = generatedRunnerNames(mobileAutoContent) + const problems = validateTestGroups(groups, runners) + + if (problems.length > 0) { + console.error('❌ test-groups.json does not cover every mobile runner\n') + problems.forEach((problem) => console.error(` ${problem}\n`)) process.exit(1) } - console.log('✅ Mobile integration tests are up to date') + console.log( + `✅ Mobile integration tests are up to date (${runners.length} runner(s), group coverage OK)` + ) process.exit(0) } catch (error) { console.error('Error validating mobile tests:', error.message) diff --git a/packages/vla-ggml/test/mobile/README.md b/packages/vla-ggml/test/mobile/README.md index a1dfa7848d..30db5d17e6 100644 --- a/packages/vla-ggml/test/mobile/README.md +++ b/packages/vla-ggml/test/mobile/README.md @@ -17,7 +17,49 @@ After adding a new file under `test/integration/`, regenerate the mobile entries npm run test:mobile:generate ``` -The generator walks `test/integration/`, derives a function name per test file, and rewrites `integration.auto.cjs`. If a `test-groups.json` is added later (per-platform iOS/Android shard split), the generator will validate that every runner is covered. +The generator walks `test/integration/`, derives a function name per test file, and rewrites `integration.auto.cjs`. It **only generates** — it performs no `test-groups.json` validation, because `npm run test:integration` chains it, so anything that throws there takes desktop integration tests down on every platform. + +Run it **by hand** after adding, renaming or removing a `test/integration/*.test.js` file. `npm test` runs `test:unit` before `test:integration`, and `test:unit` now validates the committed `integration.auto.cjs`, so it will fail first and tell you to regenerate rather than regenerating for you. Editing the *body* of an existing test needs no regeneration: the generated file is derived from filenames alone. + +## `test-groups.json` and deferred runners + +`test-groups.json` is the per-platform Device Farm shard split. Every runner in `integration.auto.cjs` must either appear in a group for each platform, or be listed under the top-level `deferred` key: + +```json +{ + "ios": { "smolvla": ["runAddonTest"], "groot": ["runGrootTest"] }, + "android": { "smolvla": ["runAddonTest"], "groot": ["runGrootTest"] }, + "deferred": ["runPi05Test"] +} +``` + +`deferred` records runners that are intentionally not scheduled on device — pi05 mobile coverage is deferred pending a project-owned CDN-fronted mirror, and `pi05.test.js` is gated on-device by `_skipMobilePi05`. Declaring it keeps "not scheduled" distinguishable from "forgotten"; deleting the runner from the file instead makes those two indistinguishable. + +`deferred` **must stay a top-level key**. The CI composites consume only `.` and ignore every other top-level key (as OCR's `perf_report_filter` already relies on), so nesting `deferred` inside `ios`/`android` would schedule it as a real Device Farm shard. + +A runner may also be deferred on one platform only, by keying `deferred` by platform instead of using a flat list: + +```json +{ "deferred": { "ios": ["runBigTest"] } } +``` + +### Running a deferred runner on device + +`validate-devices` builds its runner allowlist from `.` too, so a manual `workflow_dispatch` with `-f tests=runPi05Test` is rejected as an unknown runner — the deferral is enforced at both ends. To run pi05 on device from a branch: + +1. Move `runPi05Test` out of `deferred` into a group in `test-groups.json` (or into a platform-scoped `deferred` for the other platform). +2. Relax `_skipMobilePi05` in `test/integration/pi05.test.js` — it skips both pi05 cases on mobile regardless of scheduling, so step 1 alone runs zero tests and passes green. +3. Dispatch `integration-mobile-test-vla.yml` with `-f tests=runPi05Test` to collapse the run to a single spec. + +Step 1 alone is what CI validates; step 2 is why deferral is recorded here rather than enforced only by the skip. + +Coverage is enforced by: + +```bash +npm run test:mobile:validate # also runs as part of `npm run test:unit` +``` + +which runs in the ungated `ts-checks` PR job, so a scheduling mistake is caught on every PR instead of only inside the expensive, label-gated integration suite. ## Running the Tests diff --git a/packages/vla-ggml/test/mobile/test-groups.json b/packages/vla-ggml/test/mobile/test-groups.json index b4cbad412c..ecab8ee608 100644 --- a/packages/vla-ggml/test/mobile/test-groups.json +++ b/packages/vla-ggml/test/mobile/test-groups.json @@ -6,5 +6,6 @@ "android": { "smolvla": ["runAddonTest", "runEsmNamedExportsTest"], "groot": ["runGrootTest"] - } + }, + "deferred": ["runPi05Test"] }