diff --git a/.github/workflows/ci-janitor.yml b/.github/workflows/ci-janitor.yml index d29cfef938..4d5d207b53 100644 --- a/.github/workflows/ci-janitor.yml +++ b/.github/workflows/ci-janitor.yml @@ -44,7 +44,7 @@ jobs: run: | set -euo pipefail # Aggregates (ci-required, golden-path, run-summary) only echo another job's failure. - failed="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/jobs?filter=latest&per_page=100" --paginate \ + failed="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/jobs?filter=latest&per_page=10" --paginate \ --jq '.jobs[] | select(.conclusion == "failure") | select(.name | test("^(ci-required|golden-path|run-summary)$") | not) | .id')" [ -n "$failed" ] || { echo "No primary job failed; nothing to retry."; exit 0; } for id in $failed; do diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2d5274a27..f2097e235a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1664,7 +1664,7 @@ jobs: # with `--jq` or `--template`"), so the flatten of the paginated pages happens here. Paired # wrongly, the call failed, the guard below wrote `[]`, and every run's timing table was # header-only — an empty measurement read as a measured one. - if ! gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \ + if ! gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=10" \ | node -e 'let raw = ""; process.stdin.on("data", (chunk) => { raw += chunk; }).on("end", () => { process.stdout.write(JSON.stringify(JSON.parse(raw).flatMap((page) => page.jobs ?? []))); });' \ > "$RUNNER_TEMP/job-timings.json"; then echo '[]' > "$RUNNER_TEMP/job-timings.json" diff --git a/.github/workflows/native-platforms.yml b/.github/workflows/native-platforms.yml index 5f21d7508e..2e87cd9f74 100644 --- a/.github/workflows/native-platforms.yml +++ b/.github/workflows/native-platforms.yml @@ -1973,6 +1973,10 @@ jobs: # rcedit rewriting the PE no longer discards it and the container launches the real game. - name: Verify the relocated release container with the installed verifier shell: bash + env: + # The Windows runner's software adapter drew 300 frames at 0.6 to 1.3 s each on one tree + # (runs 41279577367, 41288321973), and the default 300 s bound cut it at frame 293. + TN_STARTER_TIMEOUT_MS: ${{ matrix.platform == 'Windows' && '600000' || '300000' }} run: | set -euo pipefail # Same POSIX normalisation as the relocate step, so the verifier sees one path style. @@ -2860,7 +2864,7 @@ jobs: ref: ${{ needs.scope.outputs.candidate_sha || inputs.candidate_sha || github.sha }} - uses: actions/download-artifact@v7 with: - pattern: ci-native-receipt-${{ github.run_id }}-${{ github.run_attempt }}-* + pattern: ci-native-receipt-${{ github.run_id }}-* path: artifacts/ci-native-receipts - id: collect env: diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index 96425eace0..6d00a90d94 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -220,7 +220,7 @@ jobs: fi printf '{"databaseId":%s,"attempt":1,"status":"%s","conclusion":"%s","event":"%s","headBranch":"%s","headSha":"%s"}\n' \ "$candidate_run" "$WORKFLOW_RUN_STATUS" "$WORKFLOW_RUN_CONCLUSION" "$WORKFLOW_RUN_EVENT" "$WORKFLOW_RUN_HEAD_BRANCH" "$candidate_sha" > "$evidence/ci-run.json" - gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$candidate_run/jobs?filter=latest&per_page=100" > "$evidence/ci-jobs.json" + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$candidate_run/jobs?filter=latest&per_page=10" > "$evidence/ci-jobs.json" CANDIDATE_RUN_ID="$candidate_run" CANDIDATE_SHA="$candidate_sha" node --input-type=module <<'NODE' import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -323,7 +323,7 @@ jobs: NODE )" gh run view "$candidate_run" --repo "$GITHUB_REPOSITORY" --json databaseId,attempt,status,conclusion,event,headBranch,headSha > "$evidence/ci-run.json" - gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$candidate_run/jobs?filter=latest&per_page=100" > "$evidence/ci-jobs.json" + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$candidate_run/jobs?filter=latest&per_page=10" > "$evidence/ci-jobs.json" CANDIDATE_RUN_ID="$candidate_run" node --input-type=module <<'NODE' import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; diff --git a/docs/verification/native-coverage-2026-08-28.md b/docs/verification/native-coverage-2026-08-28.md index 3361a6b73c..4421a2c470 100644 --- a/docs/verification/native-coverage-2026-08-28.md +++ b/docs/verification/native-coverage-2026-08-28.md @@ -8,13 +8,13 @@ targets could not be built and are named below. | Subsystem | Instrumented lines | Covered | Line coverage | | --- | ---: | ---: | ---: | | `src/async/` | 73 | 60 | 82.19% | -| `src/audio/` | 1471 | 1296 | 88.10% | +| `src/audio/` | 1471 | 1301 | 88.44% | | `src/canvas/` | 1334 | 1110 | 83.21% | | `src/cli/` | 1691 | 1243 | 73.51% | | `src/engine/` | 41259 | 34857 | 84.48% | | `src/fs/` | 235 | 189 | 80.43% | | `src/http/` | 410 | 377 | 91.95% | -| `src/js/` | 2835 | 2227 | 78.55% | +| `src/js/` | 2838 | 2230 | 78.58% | | `src/platform/` | 1507 | 1047 | 69.48% | | `src/raytracing/` | 643 | 543 | 84.45% | | `src/runtime.cpp` | 2410 | 1867 | 77.47% | @@ -22,12 +22,12 @@ targets could not be built and are named below. | `src/storage/` | 370 | 329 | 88.92% | | `src/utils/` | 0 | 0 | 0.00% | | `src/vfs/` | 239 | 195 | 81.59% | -| `src/webgpu/` | 10061 | 7719 | 76.72% | +| `src/webgpu/` | 10061 | 7716 | 76.69% | | `src/webtransport/` | 1391 | 1078 | 77.50% | -| `src/workers/` | 615 | 524 | 85.20% | -| **TOTAL** | **66571** | **54685** | **82.15%** | +| `src/workers/` | 615 | 527 | 85.69% | +| **TOTAL** | **66574** | **54693** | **82.15%** | -Source digest: `sha256:ff9b2800410629b80ec4e8838ebc325ddf0e5f49a2e10ef34760d239e1bd9bcc` +Source digest: `sha256:2bcd635f915fcac09cb8bfccf5fce40a6809afd04f4d5e6496907abe3839a664` The default `pnpm budgets` gate reads this committed measurement without configuring or compiling the native host. Any native source, native C++ test, CTest registration, or coverage aggregation diff --git a/packages/core/__tests__/auto-batch.spec.ts b/packages/core/__tests__/auto-batch.spec.ts index 12db65624e..903578918e 100644 --- a/packages/core/__tests__/auto-batch.spec.ts +++ b/packages/core/__tests__/auto-batch.spec.ts @@ -98,6 +98,28 @@ describe("AutoBatch", () => { expect(batch.census).toMatchObject({ meshes: 40, draws: 2, optedOut: 0, restored: 0 }); }); + it("walks the scene under the observer mark, so the bridge census never bills the pass to the game", () => { + // PR #595 run 41279573156: the walk landed on a census frame in collectathon-collect, and its + // __attributeNames re-reads pushed TN_BRIDGE_WARNING to 0.029 (limit 0.02) on a timing flake. + const host = globalThis as { __tnPlaytestObserving?: number }; + const scene = new Scene(); + const group = new Group(); + scene.add(group); + town(4, group); + const seen: (number | undefined)[] = []; + const children = group.children; + Object.defineProperty(group, "children", { + get: () => { + seen.push(host.__tnPlaytestObserving); + return children; + }, + }); + run(new AutoBatch(scene, { settleFrames: 3 }), 8); + // Only the snapshot passes hold the mark; the merge itself stays billed. + expect(seen.some((mark) => (mark ?? 0) > 0)).toBe(true); + expect(host.__tnPlaytestObserving ?? 0).toBe(0); + }); + it("keeps an opted-out subtree separate and reports it", () => { const scene = new Scene(); const group = new Group(); diff --git a/packages/core/src/auto-batch.ts b/packages/core/src/auto-batch.ts index de97a64977..80501f684e 100644 --- a/packages/core/src/auto-batch.ts +++ b/packages/core/src/auto-batch.ts @@ -4,6 +4,7 @@ import { isEngineRenderHook } from "./engine-render-hook.js"; import type { Registry } from "./entities.js"; import { mergeByMaterial } from "./merge-parts.js"; import { displacesVertices, hasMorphAttributes } from "./projection-plan.js"; +import { unbilled } from "./startup-offscreen-bytes.js"; /** * Static meshes that share a material become one draw, with no call from the game. @@ -489,17 +490,22 @@ export class AutoBatch { return this.#shapeChanges; } - /** Eligible meshes with the world matrix they were drawn at. */ + /** + * Eligible meshes with the world matrix they were drawn at. Runs only while the scene settles, so + * its reads are no game's steady cost and the bridge census leaves them out. + */ #snapshot(): Map { - this.#scene.updateMatrixWorld(true); - const found = new Map(); - this.#optedOut = 0; - this.#held = 0; - const held = heldObjects(this.#entities); - this.#walk(this.#scene, false, held, (mesh) => { - found.set(mesh, Float64Array.from(mesh.matrixWorld.elements)); + return unbilled(() => { + this.#scene.updateMatrixWorld(true); + const found = new Map(); + this.#optedOut = 0; + this.#held = 0; + const held = heldObjects(this.#entities); + this.#walk(this.#scene, false, held, (mesh) => { + found.set(mesh, Float64Array.from(mesh.matrixWorld.elements)); + }); + return found; }); - return found; } #walk( diff --git a/packages/core/src/startup-offscreen-bytes.ts b/packages/core/src/startup-offscreen-bytes.ts index 4d2ba86eca..9b9f7a43b5 100644 --- a/packages/core/src/startup-offscreen-bytes.ts +++ b/packages/core/src/startup-offscreen-bytes.ts @@ -87,7 +87,7 @@ function holders(root: Object3D, camera: Camera): Map(work: () => T): T { +export function unbilled(work: () => T): T { const host = globalThis as { __tnPlaytestObserving?: number }; host.__tnPlaytestObserving = (host.__tnPlaytestObserving ?? 0) + 1; try { diff --git a/packages/runtime-native/src/js/v8_engine.cpp b/packages/runtime-native/src/js/v8_engine.cpp index 67481672c7..f0705899e2 100644 --- a/packages/runtime-native/src/js/v8_engine.cpp +++ b/packages/runtime-native/src/js/v8_engine.cpp @@ -11,6 +11,7 @@ #include "mystral/js/engine.h" #include "mystral/cold_start.h" #include "mystral/js/module_system.h" +#include #include #include #include @@ -1414,15 +1415,22 @@ class V8Engine : public Engine { bool supportsBlockingTaskWait() const override { return true; } void waitForTask() override { + // processMicrotasks() between the caller's state check and this wait can consume a wake + // task. The flag keeps that wake: without it a terminate() in that gap joins a worker + // that sleeps forever. The waker changes its state before it sets the flag, and the + // caller rechecks that state after this returns, so clearing the flag loses nothing. + if (wakeRequested_.exchange(false)) return; V8EntryScope entry_scope(isolate_); v8::Local context = context_.Get(isolate_); entry_scope.enterContext(context); v8::platform::PumpMessageLoop( g_platform.get(), isolate_, v8::platform::MessageLoopBehavior::kWaitForWork); + wakeRequested_.store(false); isolate_->PerformMicrotaskCheckpoint(); } void wakeTaskWait() override { + wakeRequested_.store(true); g_platform->GetForegroundTaskRunner(isolate_)->PostTask( std::make_unique()); } @@ -2078,6 +2086,7 @@ class V8Engine : public Engine { std::deque> callbackArgsPool_; std::deque>> callbackLocalsPool_; bool inFrame_ = false; // True during animation frame execution + std::atomic wakeRequested_{false}; // A wake waitForTask() has not yet honoured bool frameTrackingSuspended_ = false; // When true, skip frame tracking for new allocations int nativeCallbackDepth_ = 0; bool exceptionFromNativeCallback_ = false; diff --git a/packages/runtime-native/tests/js_engine_contract_test.cpp b/packages/runtime-native/tests/js_engine_contract_test.cpp index cd6d51f375..f94feccb16 100644 --- a/packages/runtime-native/tests/js_engine_contract_test.cpp +++ b/packages/runtime-native/tests/js_engine_contract_test.cpp @@ -1,7 +1,10 @@ #include "mystral/js/engine.h" +#include +#include #include #include +#include namespace { @@ -163,6 +166,30 @@ bool checkNestedNativeCallbacks(mystral::js::Engine& engine) { } // namespace +// PR #595, macOS desktop core: a worker's loop pumps microtasks between its queue check and its +// blocking wait. A wake posted in that gap was consumed by the pump, the wait then had no task, +// and terminate() joined that worker forever on the main thread. The rescue thread keeps a +// regression a failure with a message, not a hung test. +bool checkConsumedWakeReleasesWait(mystral::js::Engine& engine) { + if (!engine.supportsBlockingTaskWait()) return true; + std::atomic returned{false}; + std::atomic rescued{false}; + std::thread rescue([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!returned.load() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (returned.load()) return; + rescued.store(true); + engine.wakeTaskWait(); + }); + engine.wakeTaskWait(); + engine.processMicrotasks(); + engine.waitForTask(); + returned.store(true); + rescue.join(); + return !rescued.load(); +} + int main() { struct EngineCase { mystral::js::EngineType type; @@ -198,6 +225,11 @@ int main() { << " nested callback semantics failed" << std::endl; return EXIT_FAILURE; } + if (!checkConsumedWakeReleasesWait(*engine)) { + std::cerr << "js-engine-contract: " << engineCase.name + << " a wake consumed before the wait stranded the wait" << std::endl; + return EXIT_FAILURE; + } std::cout << "js-engine-contract: engine=" << engineCase.name << " property=own-data global=assignment"; if (engineCase.checkNestedCallbacks) { diff --git a/packages/runtime-native/tests/native-platform-workflow.test.mjs b/packages/runtime-native/tests/native-platform-workflow.test.mjs index 3116d4fb4f..62a51a156e 100644 --- a/packages/runtime-native/tests/native-platform-workflow.test.mjs +++ b/packages/runtime-native/tests/native-platform-workflow.test.mjs @@ -560,7 +560,7 @@ function runReleaseGate({ runs = [candidateRun], detail = candidateRun, jobs = c case "$1 $2" in 'run list') test "$MOCK_GH_FAILURE" != list || exit 42; printf '%s' "$MOCK_GH_RUNS" ;; 'run view') test "$2" = view || exit 64; test "$MOCK_GH_FAILURE" != view || exit 42; printf '%s' "$MOCK_GH_DETAIL" ;; - 'api --paginate') test "$2" = --paginate || exit 64; test "$3" = --slurp || exit 64; test "$4" = 'repos/ThreeNativeHQ/threenative/actions/runs/123/jobs?filter=latest&per_page=100' || exit 64; test "$MOCK_GH_FAILURE" != jobs || exit 42; printf '%s' "$MOCK_GH_PAGES" ;; + 'api --paginate') test "$2" = --paginate || exit 64; test "$3" = --slurp || exit 64; test "$4" = 'repos/ThreeNativeHQ/threenative/actions/runs/123/jobs?filter=latest&per_page=10' || exit 64; test "$MOCK_GH_FAILURE" != jobs || exit 42; printf '%s' "$MOCK_GH_PAGES" ;; *) exit 64 ;; esac `); diff --git a/packages/three-native/__tests__/bridge-reads.spec.ts b/packages/three-native/__tests__/bridge-reads.spec.ts index 6eb0f13a63..4baec3ba33 100644 --- a/packages/three-native/__tests__/bridge-reads.spec.ts +++ b/packages/three-native/__tests__/bridge-reads.spec.ts @@ -411,6 +411,44 @@ describe("the bridge census", () => { expect(frames(500)).toEqual([]); }); + it("judges a census frame by its own length, so a scene rebuild is not steady waste", () => { + // PR run 41302359332: shooter `debug-restart` failed when the census landed on the restart + // frame, where Play.enter rebuilt the town: 77,840 redundant reads, 16 ms of a frame of seconds, + // judged as 16% of the 98 ms median. The next ordinary census still names a real waster. + host.__THREENATIVE_BRIDGE_CACHE__ = false; + const clock = { ms: 0 }; + vi.spyOn(performance, "now").mockImplementation(() => clock.ms); + const warnings: string[] = []; + vi.spyOn(console, "warn").mockImplementation((line: unknown) => { + warnings.push(String(line)); + }); + const { Light, collect } = scene(clock); + const key = new Light(0xffffff, 3); + const read = () => { + for (let at = 0; at < 500; at++) if (!(key.intensity >= 0)) throw new Error("bad"); + }; + const census = (safePoint: number) => + safePoint > WARMUP_FRAMES && safePoint % WINDOW_FRAMES === 0; + let safePoints = 0; + let rebuild = true; + let seen = 0; + const bridgeWarnings = () => warnings.filter((line) => line.startsWith(BRIDGE_WARNING_MARKER)); + while (seen < 2) { + collect(); + safePoints += 1; + clock.ms += 16; + if (!census(safePoints)) continue; + seen += 1; + read(); + if (rebuild) clock.ms += 5000; + rebuild = false; + collect(); + safePoints += 1; + if (seen === 1) expect(bridgeWarnings()).toEqual([]); + } + expect(bridgeWarnings()).toHaveLength(1); + }); + it("leaves the playtest observer's reads out, which no player run makes", () => { // Hosted CI's adventure `puzzle` run failed on 45,920 redundant reads, led by the playtest // sample's own scene walk (countMaterials, Box3.setFromObject) inside the census frame. diff --git a/packages/three-native/src/bridge-meter.ts b/packages/three-native/src/bridge-meter.ts index 4e6cf9c73f..bd5075b857 100644 --- a/packages/three-native/src/bridge-meter.ts +++ b/packages/three-native/src/bridge-meter.ts @@ -10,8 +10,9 @@ * also takes one stack for each name read redundantly, so the verdict names the caller. * * **The rule is derived, never a constant an author revisits.** It fires when the redundant reads, - * at this host's own measured cost per crossing, take more than `REDUNDANT_SHARE` of the frame's - * median interval: the frame's own length, so the guard holds on Xvfb and on a loaded host. The line + * at this host's own measured cost per crossing, take more than `REDUNDANT_SHARE` of the frame: its + * median interval or the census frame's own length, the longer, so the guard holds on Xvfb, on a + * loaded host and across a frame that rebuilds a scene. The line * is `TN_BRIDGE_WARNING`, which a playtest fails on (`packages/playtest` perf warnings). * * What *all* crossings cost is an observation, not a defect: a game that reads 995 distinct @@ -161,7 +162,9 @@ export function createBridgeMeter(host: IBridgeMeterHost): IBridgeMeter { ); } if (redundant === 0 || intervals.length === 0) return; - const frameMs = medianFrameMs(); + // The census frame's own length is the last interval. A scene rebuild reads a lot once in a frame + // of seconds; a steady waster's frame is as long as its median, so the next census names it. + const frameMs = Math.max(medianFrameMs(), intervals[intervals.length - 1] as number); const crossingUs = costMicros(); const redundantMs = (redundant * crossingUs) / 1000; if (!(redundantMs > frameMs * REDUNDANT_SHARE)) return; diff --git a/scripts/__tests__/ci-integration-receipts.spec.ts b/scripts/__tests__/ci-integration-receipts.spec.ts index 1408cf4078..c76f4753a4 100644 --- a/scripts/__tests__/ci-integration-receipts.spec.ts +++ b/scripts/__tests__/ci-integration-receipts.spec.ts @@ -6,6 +6,7 @@ import { makeTempDirSync } from "../../test-support/temp-dir.js"; const scope = await import(new URL("../ci-integration-scope.mjs", import.meta.url).href); const validate = scope.validateIntegrationReceipts; +const changeReceipts = await import(new URL("../ci-attempt-receipts.mjs", import.meta.url).href); const changeScope = await import(new URL("../ci-change-scope.mjs", import.meta.url).href); const fixture = await import( new URL("../../test-support/ci-integration-fixture.ts", import.meta.url).href @@ -144,6 +145,33 @@ describe("required current-attempt integration receipts", () => { ), ).toThrow("CI_INTEGRATION_JOB_IDENTITY"); }); + it("reads a carried leg's later-attempt copy as the same run, and a real rerun as a rerun", () => { + // PR #595 run 41262008820: job 124718810548 is listed in attempt 3 as 124750441136, with the + // same start, end and runner. Only the red leg really ran again. + const at = (row: object, started_at: string) => ({ + ...row, + started_at, + completed_at: `${started_at}:done`, + runner_name: "hosted", + }); + const first = [at(job(0, 1, 10), "t1"), at(job(1, 1, 6, "failure"), "t2")]; + const second = [at(job(0, 2, 100), "t1"), at(job(1, 2, 11), "t3")]; + const merged = changeReceipts.mergeAttemptListings([{ jobs: first }, { jobs: second }]); + const verdict = (listing: { jobs: unknown[] }) => + validate(expected, [receipt(0, "1"), receipt(1, "2")], { + totalCount: listing.jobs.length, + jobs: listing.jobs, + }); + expect(verdict(merged)).toBeUndefined(); + expect(() => verdict({ jobs: [...first, ...second] })).toThrow( + "ran again in a later attempt", + ); + const rerun = changeReceipts.mergeAttemptListings([ + { jobs: first }, + { jobs: [at(job(0, 2, 100), "t9"), second[1]] }, + ]); + expect(() => verdict(rerun)).toThrow("ran again in a later attempt"); + }); it("rejects a receipt from an attempt later than this run's", () => { expect(() => check([receipt(0, "3"), receipt(1, "2")], [job(0, 3, 10), job(1, 2, 11)]), diff --git a/scripts/__tests__/ci-native-chain.spec.ts b/scripts/__tests__/ci-native-chain.spec.ts index 1989ceda58..2c8a2f81d1 100644 --- a/scripts/__tests__/ci-native-chain.spec.ts +++ b/scripts/__tests__/ci-native-chain.spec.ts @@ -282,7 +282,7 @@ describe("actual native receipt writer, collector and protected verdict", () => if (!windows) throw new Error("missing fixture Windows job"); windows.run_attempt = 1; f.publish(); - expect(verdict().stderr).toContain("CI_NATIVE_JOB_IDENTITY"); + expect(verdict().stderr).toContain("CI_NATIVE_BUILD_IDENTITY"); expect( f.run("ci-native-receipts.mjs", [ "--collect", diff --git a/scripts/__tests__/ci-native-qualification.spec.ts b/scripts/__tests__/ci-native-qualification.spec.ts index 73a73d335a..0fe499b92f 100644 --- a/scripts/__tests__/ci-native-qualification.spec.ts +++ b/scripts/__tests__/ci-native-qualification.spec.ts @@ -141,6 +141,34 @@ describe("authoritative full native qualification through ci-required", () => { f.cleanup(); } }); + it("accepts a green leg carried from an earlier attempt beside one rerun leg", () => { + const f = fixture(); + try { + const carried = f.summary.receipts[0]; + const job = f.jobs.find((row) => String(row.id) === carried?.jobId); + if (carried === undefined || job === undefined) throw new Error("fixture has no leg"); + carried.runAttempt = "1"; + job.run_attempt = 1; + const result = f.run(); + expect(result.status, result.stderr).toBe(0); + } finally { + f.cleanup(); + } + }); + it("rejects a leg whose latest attempt is red, even with a green earlier attempt", () => { + const f = fixture(); + try { + const carried = f.summary.receipts[0]; + const job = f.jobs.find((row) => String(row.id) === carried?.jobId); + if (carried === undefined || job === undefined) throw new Error("fixture has no leg"); + carried.runAttempt = "1"; + f.jobs.push({ ...job, id: 998, run_attempt: 2, conclusion: "failure" }); + job.run_attempt = 1; + f.reject(); + } finally { + f.cleanup(); + } + }); it("never empties required Windows/macOS inventory when the entire desktop matrix skips", () => { const f = fixture(); try { diff --git a/scripts/__tests__/ci-structure.spec.ts b/scripts/__tests__/ci-structure.spec.ts index a936a0c95d..a9da3aaf02 100644 --- a/scripts/__tests__/ci-structure.spec.ts +++ b/scripts/__tests__/ci-structure.spec.ts @@ -4649,4 +4649,22 @@ describe("a CI-configuration-only pull request", () => { await removeFixture(fixture.root); } }); + + it("reads a run's job list in small pages, because a large page answers 502 after a --failed rerun", async () => { + // Run 41288322050 attempt 3: `jobs?filter=latest&per_page=100` answered 502 at 10.6 s on 9 of 9 + // reads, and the release gate of tag runtime-native-v0.3.7 could not read its own CI run. + const sources = [ + ".github/workflows/ci.yml", + ".github/workflows/ci-janitor.yml", + ".github/workflows/native-release.yml", + "scripts/ci-queue-watch.mjs", + "scripts/ci-merge-latency.mjs", + ]; + for (const source of sources) { + const large = (await readFile(source, "utf8")) + .split("\n") + .filter((line) => /\/jobs\?[^"'`\s]*per_page=(?:[2-9]\d|\d{3})/u.test(line)); + expect({ source, large }).toEqual({ source, large: [] }); + } + }); }); diff --git a/scripts/ci-attempt-receipts.mjs b/scripts/ci-attempt-receipts.mjs index 8e00794a48..a1eb03207f 100644 --- a/scripts/ci-attempt-receipts.mjs +++ b/scripts/ci-attempt-receipts.mjs @@ -87,6 +87,41 @@ export function readAttemptJobs( } } +/** + * Jobs of attempts 1..current, each as the attempt that really ran it. `gh run rerun --failed` + * lists every carried job in the later attempt as a copy: new id, later `run_attempt`, same start, + * end and runner (PR #595 run 41262008820: 124718810548 -> 124750441136). A copy is not a rerun, so + * the first record stays and the receipt that names it still matches. The current attempt is read + * last and carries the settled-name wait. + */ +export function readJobsAcrossAttempts(identity, prefix = "CI_INTEGRATION", settledNames = []) { + const current = Number(identity.runAttempt); + const listings = []; + for (let attempt = 1; attempt <= current; attempt++) + listings.push( + readAttemptJobs( + { ...identity, runAttempt: String(attempt) }, + prefix, + attempt === current ? settledNames : [], + ), + ); + return mergeAttemptListings(listings); +} + +export function mergeAttemptListings(listings) { + const ran = new Set(); + const jobs = new Map(); + for (const listing of listings) + for (const job of listing.jobs) { + const finished = job.started_at && job.completed_at; + const key = `${job.name}\0${job.started_at}\0${job.completed_at}\0${job.runner_name}`; + if (finished && ran.has(key)) continue; + if (finished) ran.add(key); + jobs.set(job.id, job); + } + return { totalCount: jobs.size, jobs: [...jobs.values()] }; +} + export function receiptFiles(directory, prefix = "CI_INTEGRATION") { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const file = path.join(directory, entry.name); diff --git a/scripts/ci-integration-receipts.mjs b/scripts/ci-integration-receipts.mjs index 81757da930..c781ccf083 100644 --- a/scripts/ci-integration-receipts.mjs +++ b/scripts/ci-integration-receipts.mjs @@ -2,28 +2,13 @@ import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { readAttemptJobs, receiptFiles } from "./ci-attempt-receipts.mjs"; +import { readAttemptJobs, readJobsAcrossAttempts, receiptFiles } from "./ci-attempt-receipts.mjs"; import { integrationCandidatePreflight, validateIntegrationReceipts, } from "./ci-integration-scope.mjs"; -// Jobs of attempts 1..current, so a leg that passed before `gh run rerun --failed` still reads as it -// did. The current attempt is read last and carries the settled-name wait. -export function readIntegrationJobs(identity, prefix = "CI_INTEGRATION", settledNames = []) { - const current = Number(identity.runAttempt); - const byId = new Map(); - for (let attempt = 1; attempt <= current; attempt++) { - const listing = readAttemptJobs( - { ...identity, runAttempt: String(attempt) }, - prefix, - attempt === current ? settledNames : [], - ); - for (const job of listing.jobs) byId.set(job.id, job); - } - const jobs = [...byId.values()]; - return { totalCount: jobs.length, jobs }; -} +export const readIntegrationJobs = readJobsAcrossAttempts; if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { diff --git a/scripts/ci-merge-latency.mjs b/scripts/ci-merge-latency.mjs index 53b2b935ca..f1ab13acf4 100644 --- a/scripts/ci-merge-latency.mjs +++ b/scripts/ci-merge-latency.mjs @@ -193,7 +193,7 @@ export function measure({ since, base = "develop", repo = process.env.GITHUB_REP "--paginate", "--jq", ".jobs[]", - `repos/${repository}/actions/runs/${run.id}/jobs?per_page=100`, + `repos/${repository}/actions/runs/${run.id}/jobs?per_page=10`, ]), ).map((job) => ({ name: job.name, diff --git a/scripts/ci-native-qualification.mjs b/scripts/ci-native-qualification.mjs index 15b9b41ece..4d6eaaf82a 100644 --- a/scripts/ci-native-qualification.mjs +++ b/scripts/ci-native-qualification.mjs @@ -205,14 +205,17 @@ export function nativeJobIdentity(expected, leg, listing, completed = true) { new Set(listing.jobs.map((job) => job.id)).size !== listing.jobs.length ) fail("API_INCOMPLETE", "incomplete or duplicate attempt listing"); - const matches = listing.jobs.filter((job) => job.name === leg.name); + // The attempt that last ran the leg decides: an earlier attempt's red never speaks for a rerun. + const named = listing.jobs.filter((job) => job.name === leg.name); + const latest = Math.max(0, ...named.map((job) => Number(job.run_attempt))); + const matches = named.filter((job) => Number(job.run_attempt) === latest); const job = matches[0]; if ( matches.length !== 1 || !Number.isSafeInteger(job.id) || job.id <= 0 || String(job.run_id) !== expected.runId || - String(job.run_attempt) !== expected.runAttempt || + latest > Number(expected.runAttempt) || job.head_sha !== expected.workflowHeadSha || (completed && (job.status !== "completed" || job.conclusion !== "success")) ) @@ -246,7 +249,7 @@ export function validateNativeReceipts(expected, summary, listing) { receipt.version !== 1 || receipt.candidateSha !== expected.candidateSha || receipt.runId !== expected.runId || - receipt.runAttempt !== expected.runAttempt || + receipt.runAttempt !== String(job.run_attempt) || receipt.planVersion !== expected.planVersion || receipt.jobId !== String(job.id) || receipt.conclusion !== "success" || diff --git a/scripts/ci-native-receipts.mjs b/scripts/ci-native-receipts.mjs index 4bf8e52194..d21ab912a4 100644 --- a/scripts/ci-native-receipts.mjs +++ b/scripts/ci-native-receipts.mjs @@ -4,7 +4,7 @@ import { createHash } from "node:crypto"; import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { readAttemptJobs, receiptFiles } from "./ci-attempt-receipts.mjs"; +import { readJobsAcrossAttempts, receiptFiles } from "./ci-attempt-receipts.mjs"; import { nativeBuildProfile, nativeCandidatePreflight, @@ -207,7 +207,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) workflowHeadSha: env.TN_CI_WORKFLOW_HEAD_SHA, reusable: env.TN_CI_NATIVE_REUSABLE !== "false", }); - const listing = readAttemptJobs( + const listing = readJobsAcrossAttempts( { repository: env.GITHUB_REPOSITORY, runId: expected.runId, runAttempt: expected.runAttempt }, "CI_NATIVE", ); @@ -247,7 +247,14 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) runId: expected.runId, runAttempt: expected.runAttempt, planVersion: plan.version, - receipts: receiptFiles(process.argv[3], "CI_NATIVE"), + // A carried leg's receipt was written by an earlier attempt; the newest per leg speaks. + receipts: [ + ...new Map( + receiptFiles(process.argv[3], "CI_NATIVE") + .sort((a, b) => Number(a.runAttempt) - Number(b.runAttempt)) + .map((receipt) => [receipt.job, receipt]), + ).values(), + ], }; validateNativeReceipts(expected, summary, listing); appendFileSync(env.GITHUB_OUTPUT, `coverage_receipt=${JSON.stringify(summary)}\n`); diff --git a/scripts/ci-queue-watch.mjs b/scripts/ci-queue-watch.mjs index 2b91a06826..748da4a793 100644 --- a/scripts/ci-queue-watch.mjs +++ b/scripts/ci-queue-watch.mjs @@ -83,7 +83,7 @@ function runJobs(repository, runId) { "api", "--paginate", "--slurp", - `repos/${repository}/actions/runs/${runId}/jobs?filter=latest&per_page=100`, + `repos/${repository}/actions/runs/${runId}/jobs?filter=latest&per_page=10`, ]), ); return pages.flatMap((page) => page.jobs ?? []); diff --git a/scripts/ci-required.mjs b/scripts/ci-required.mjs index f868bfe248..4eddc5e879 100644 --- a/scripts/ci-required.mjs +++ b/scripts/ci-required.mjs @@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process"; import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { readAttemptJobs } from "./ci-attempt-receipts.mjs"; +import { readAttemptJobs, readJobsAcrossAttempts } from "./ci-attempt-receipts.mjs"; import { assertMergeParentCandidate, classify, @@ -250,7 +250,7 @@ try { validateNativeReceipts( expected, summary, - readAttemptJobs({ repository, runId, runAttempt: summary.runAttempt }, "CI_NATIVE"), + readJobsAcrossAttempts({ repository, runId, runAttempt: summary.runAttempt }, "CI_NATIVE"), ); } catch (error) { failures.push(error instanceof Error ? error.message : String(error));