From aa4ce4ff43d3ba5ff25df460a751031fb91dc686 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 19 Aug 2026 02:41:28 +0000 Subject: [PATCH 1/2] A full artifact storage quota must not fail a green suite (#74) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An organisation hit the Actions artifact storage quota today and every consumer's CI went red at `Shard labels`: Failed to CreateArtifact: Artifact storage quota has been hit. before a single test executed. That job's real product is the `matrix` OUTPUT, and the log of the failing run reads `121 timing rows.` and `Set output 'matrix'` — the work succeeded and was discarded because an upload after it failed. Every shard `needs:` that matrix, so the whole suite was skipped, on every PR, in every repository using this workflow. The uploads that report on a run are now `continue-on-error`. The split is this file's own: coverage and timings are a REPORT, "every unit ran exactly once" is a CORRECTNESS CLAIM, and only the second should be able to fail a suite. They were sharing one exit status. - `-timings` (label job) — the load-bearing one; it gated the matrix. - `-coverage-`, `-out-` (shards) — both run AFTER the suite, so failing there converts a green shard into a red one over a report. - `-lcov`, `-records`, and the counter merge (collect) — same category. `-prebuild` stays fatal: the shards DOWNLOAD it, so a failure there is a real missing dependency rather than a lost report. `Every unit ran, exactly once` stays fatal and ungated. So does the refusal inside `restore_counters`: what changes is that a missing coverage report annotates the run with `::error::` and skips the Codecov upload instead of failing the gate. The silent-coverage-loss failure this file exists to prevent — 54.5 % reported for a 94.8 % suite — is still impossible, because a missing report is visible as an error and an absent upload, never as a quietly wrong number. --- .github/workflows/sharded-tests.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/sharded-tests.yml b/.github/workflows/sharded-tests.yml index 39d53b3..bad7fd3 100644 --- a/.github/workflows/sharded-tests.yml +++ b/.github/workflows/sharded-tests.yml @@ -312,7 +312,18 @@ jobs: # what has to run from this repository's own tree. - uses: QAtlasHub/TestShards.jl/actions/read-history@main + # NON-FATAL, and this is the load-bearing one (issue #74). This job's real product is the + # `matrix` OUTPUT, which is already set by the time this step runs — the log of a run that + # failed here still reads `121 timing rows.` and `Set output 'matrix'`. Letting an upload + # failure fail the JOB therefore discards a matrix that was computed successfully, and + # every shard `needs:` it, so the entire suite is skipped. + # + # Measured: an organisation hit the Actions artifact storage quota and every consumer's CI + # went red HERE — `Failed to CreateArtifact: Artifact storage quota has been hit` — before + # a single test executed. A storage problem should cost the timing HISTORY, which is an + # optimisation, not the TEST RESULT, which is the point of the run. - uses: actions/upload-artifact@v7 + continue-on-error: true with: name: ${{ inputs.artifact-prefix }}-timings path: timings.tsv @@ -580,8 +591,12 @@ jobs: echo "Captured $n coverage files." [ "$n" -gt 0 ] || echo "::warning::no .cov files — was the suite run with coverage on?" + # NON-FATAL for the same reason (issue #74): this step runs AFTER the suite, so failing the + # job here converts a green shard into a red one over a report. `collect` already treats + # missing counters as a loud reporting failure of its own, which is where that belongs. - name: Upload this shard's coverage counters if: ${{ inputs.coverage }} + continue-on-error: true uses: actions/upload-artifact@v7 with: name: ${{ inputs.artifact-prefix }}-coverage-${{ matrix.sid }} @@ -594,6 +609,7 @@ jobs: - name: Upload this shard's records and timings if: always() + continue-on-error: true uses: actions/upload-artifact@v7 with: name: ${{ inputs.artifact-prefix }}-out-${{ matrix.sid }} @@ -649,8 +665,20 @@ jobs: # shard-suffixed; putting them back where their sources are is what lets CoverageTools # find them, and it globs `.*.cov` so the suffix is free. # ONCE, here, rather than N times in the shards. + # NON-FATAL, but LOUD (issue #74). Coverage is a REPORT; "every unit ran exactly once", + # below, is a CORRECTNESS CLAIM, and only the second one should be able to fail a suite. + # Those are this file's own two categories and they were sharing one exit status: when the + # per-shard uploads were refused by a full artifact storage quota, this step found no + # counters, `collect` failed, and the `All shards passed` gate — which needs `collect` — + # reported a red suite for a run whose tests were all green. + # + # The refusal inside `restore_counters` stays exactly as loud as it was; what changes is + # that it annotates instead of gating. The silent-coverage-loss failure this guards + # (54.5 % reported for a 94.8 % suite) is still impossible: a missing report is an + # `::error::` in the run and an absent Codecov upload, not a quietly wrong number. - name: Convert every shard's counters into one report if: ${{ inputs.coverage }} + continue-on-error: true run: | set -euo pipefail julia --startup-file=no --project="$RUNNER_TEMP/tsenv" -e ' @@ -726,6 +754,7 @@ jobs: # lose nothing. - name: Publish the merged report if: ${{ inputs.coverage }} + continue-on-error: true uses: actions/upload-artifact@v7 with: name: ${{ inputs.artifact-prefix }}-lcov @@ -775,6 +804,7 @@ jobs: echo "Merged $(wc -l < records.jsonl) unit records." - name: Upload the merged records + continue-on-error: true uses: actions/upload-artifact@v7 with: name: ${{ inputs.artifact-prefix }}-records From 7d59dcf584cb14461d9ff4fb68344060a9328b2d Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 19 Aug 2026 02:42:49 +0000 Subject: [PATCH 2/2] =?UTF-8?q?Bump=20to=200.3.36=20=E2=80=94=20workflow-o?= =?UTF-8?q?nly=20change,=20but=20the=20repo=20gates=20every=20PR=20on=20a?= =?UTF-8?q?=20version=20step?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 97af3d9..3465407 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "TestShards" uuid = "acceef1d-f5e0-4fe4-a546-818dc56ce7b2" -version = "0.3.35" +version = "0.3.36" authors = ["sota shimozono "] [deps]