From 262ff7af94c3769f2731a232daa70c823dafbc7f Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 20 Aug 2026 03:54:47 +0000 Subject: [PATCH] feat: the pieces for carrying completeness evidence in the shards' job logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the transport and leaves it UNREFERENCED. The workflow wiring is a follow-up, because a `uses: QAtlasHub/TestShards.jl/actions/read-evidence@main` cannot resolve until the action is on `main` — measured, on the first attempt at this: Can't find 'action.yml' ... for action 'QAtlasHub/TestShards.jl/actions/read-evidence@main' `uses:` accepts no expression, so `@${{ github.sha }}` is not a way out, and a relative path is the trap `actions/read-history` documents (a `run:`/relative `uses:` in a REUSABLE workflow resolves in the CALLER's checkout). So a new action and its first reference cannot land together. Splitting also means nothing in this PR can change any consumer's behaviour: none of it is called yet. WHY THE TRANSPORT AT ALL. The completeness gate is the one job still failing across the fleet under a full artifact quota, and it fails CORRECTLY — `units observed = 0`, "no shard reported how many units it observed — cannot verify the run" — because the evidence never arrives. Artifact storage is metered in GigabyteHours against a per-period allowance; once crossed, `CreateArtifact` refuses BEFORE any bytes are sent, so a 0-byte artifact fails exactly as a large one does, and deleting artifacts does not un-accrue the period. `actions/cache` was measured as the replacement and does not work: save succeeds and the entry lists as active on the right ref, restore misses every key — including a default-branch entry written by this repo's own CI, and with a 45 s gap to rule out propagation. Four runs, three controls. What does work, verified end to end in a live run: the shard PRINTS its evidence and the collector reads that shard's job log back through the Actions API. No storage, no residue, and only `actions: read` — which a fork pull request also has. THE PARSING IS THE WHOLE RISK, SO IT IS IN THE PACKAGE. The runner echoes a step's script into the log before running it, so the marker appears TWICE: once as source with `$(...)` unexpanded, once as output. Taking the first match decodes the literal `$(base64 -w0 ev.tsv)`; that surfaced as `base64: invalid input` by luck rather than as a wrong answer. `evidence_from_log` discriminates by CONSTRUCTION — base64's alphabet cannot spell an unexpanded substitution — so no ordering convention is relied on. Three mutations confirm the tests can go red: widening the payload class, accepting a duplicate shard, and treating "no evidence anywhere" as success. Two more corrections the measurement forced, each of which would have shipped broken: * `gh api .../actions/jobs/{id}/logs` does not work. The endpoint answers 302 to a blob URL and gh exits 1 with no useful message; `curl -sSL` follows it. * The API names a reusable workflow's job with the CALLER's prefix — `ci / test / shard s5`, not `shard s5` — so a name filter would depend on what the caller called its job, and one that matches nothing reads zero logs and reports a complete-looking empty run. There is no name filter: the marker discriminates. `evidence_cli` rebuilds the same `all-shards.tsv` / `all-ran.tsv` the artifact path produced, against the real formats read off `src/json.jl`, and the test feeds the result to an UNCHANGED `completeness_cli`. The check itself is not touched by any of this. Co-Authored-By: Claude Opus 5 --- Project.toml | 2 +- actions/read-evidence/action.yml | 49 ++++ actions/read-evidence/read-evidence.sh | 63 +++++ actions/read-evidence/read-evidence.test.sh | 84 +++++++ src/TestShards.jl | 1 + src/evidence.jl | 248 ++++++++++++++++++++ test/core/test_evidence.jl | 154 ++++++++++++ 7 files changed, 600 insertions(+), 1 deletion(-) create mode 100644 actions/read-evidence/action.yml create mode 100755 actions/read-evidence/read-evidence.sh create mode 100755 actions/read-evidence/read-evidence.test.sh create mode 100644 src/evidence.jl create mode 100644 test/core/test_evidence.jl diff --git a/Project.toml b/Project.toml index 11716be..e74f6b2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "TestShards" uuid = "acceef1d-f5e0-4fe4-a546-818dc56ce7b2" -version = "0.3.37" +version = "0.3.38" authors = ["sota shimozono "] [deps] diff --git a/actions/read-evidence/action.yml b/actions/read-evidence/action.yml new file mode 100644 index 0000000..b21983e --- /dev/null +++ b/actions/read-evidence/action.yml @@ -0,0 +1,49 @@ +name: Read TestShards evidence from the shards' job logs +description: > + Download each shard job's log for one attempt of one run, so the collector can recover the + completeness evidence the shards printed. Uses no artifact storage and no cache; needs only + `actions: read`. + +# WHY THIS IS AN ACTION AND NOT A `run:` IN THE WORKFLOW — the same reason as +# `actions/read-history`. A `run:` step in a REUSABLE workflow executes in the CALLER's checkout, +# so it can only invoke files the caller has; a script referenced by path works in this +# repository's own CI and exits 127 on every consumer. An action's files are fetched with it. + +inputs: + repository: + description: owner/name of the repository whose run is being read. + required: false + default: ${{ github.repository }} + run-id: + required: false + default: ${{ github.run_id }} + attempt: + description: > + Run attempt. Reading `/runs/{id}/jobs` instead of this attempt's jobs merges every attempt, + so a re-run's shards double-count while the recovery looks clean. + required: false + default: ${{ github.run_attempt }} + job-prefix: + description: Only jobs whose name starts with this are read. + required: false + default: '' + path: + description: Directory to write the logs into. + required: false + default: shard-logs + token: + description: > + Needs `actions: read`. A fork pull request has it, and it needs no write scope at all. + required: false + default: ${{ github.token }} + +runs: + using: composite + steps: + - shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + run: > + bash "${{ github.action_path }}/read-evidence.sh" + "${{ inputs.repository }}" "${{ inputs.run-id }}" "${{ inputs.attempt }}" + "${{ inputs.path }}" "${{ inputs.job-prefix }}" diff --git a/actions/read-evidence/read-evidence.sh b/actions/read-evidence/read-evidence.sh new file mode 100755 index 0000000..594adee --- /dev/null +++ b/actions/read-evidence/read-evidence.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# Fetch every shard job's LOG for one attempt of one run, into a directory, so the collector can +# recover the evidence markers from them (TestShards.evidence_cli). +# +# Two things here are not obvious and both were measured against live runs: +# +# 1. `gh api .../actions/jobs/{id}/logs` DOES NOT WORK. The endpoint answers 302 with a +# short-lived blob URL; gh exits 1 with no useful message. curl -L follows it. +# 2. The jobs must come from `/runs/{id}/attempts/{n}/jobs`, NOT `/runs/{id}/jobs`. The latter +# returns every attempt's jobs, so a re-run merges two attempts' shards and the recovered +# evidence double-counts while looking complete. +# +# The network commands are indirected through TS_GH / TS_CURL so `read-evidence.test.sh` can run +# every branch below without a network or a token. +set -uo pipefail + +repo=${1:?repo} +run_id=${2:?run id} +attempt=${3:?attempt} +outdir=${4:?output directory} +filter=${5:-} # only jobs whose name starts with this; empty means all + +GH=${TS_GH:-gh} +CURL=${TS_CURL:-curl} + +mkdir -p "$outdir" + +ids=$("$GH" api "repos/$repo/actions/runs/$run_id/attempts/$attempt/jobs" \ + --paginate --jq ".jobs[]|select(.name|startswith(\"$filter\"))|\"\(.id)\"" 2>/dev/null) +rc=$? +if [ $rc -ne 0 ]; then + echo "::error::could not list the jobs of run $run_id attempt $attempt. Without the job list there is nothing to read; this is not a partial result." >&2 + exit 1 +fi +if [ -z "$ids" ]; then + # An empty list is NOT an empty-but-fine run: the collector runs after the shards, so at least + # one job must match. Reporting success here would hand the gate zero evidence and let a run + # with no shards look verified. + echo "::error::no job in run $run_id attempt $attempt matched '$filter'." >&2 + exit 1 +fi + +n=0; missed=0 +for id in $ids; do + if "$CURL" -sSL --fail-with-body \ + -H "Authorization: Bearer ${GH_TOKEN:-}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$repo/actions/jobs/$id/logs" -o "$outdir/job-$id.log" 2>/dev/null + then + n=$((n+1)) + else + # A job whose log cannot be read is reported and NOT fatal here: which shards are missing is + # the completeness gate's judgement to make, not this script's. Failing here would replace a + # precise "unit X ran nowhere" with an opaque transport error. + rm -f "$outdir/job-$id.log" + missed=$((missed+1)) + echo "::warning::could not read the log of job $id" >&2 + fi +done + +echo "read $n job log(s) into $outdir${missed:+, $missed unreadable}" +[ "$n" -gt 0 ] || { echo "::error::no job log could be read at all." >&2; exit 1; } diff --git a/actions/read-evidence/read-evidence.test.sh b/actions/read-evidence/read-evidence.test.sh new file mode 100755 index 0000000..3b2232c --- /dev/null +++ b/actions/read-evidence/read-evidence.test.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# The states this script must handle that a normal run never shows: an unreadable job, an empty +# job list, and an API call that fails outright. All three are reachable here without a network +# or a token, because the two network commands are indirected through TS_GH / TS_CURL. +# +# The failure this guards against is specific: `gh api .../jobs/{id}/logs` returns 302 and gh does +# not follow it, so a version of this that used gh for the log fetch reported "read 0 job logs" +# while every job had a perfectly good log. That is a transport that fails SILENTLY into an +# unverifiable run, which is the one outcome the completeness gate exists to prevent. +# +# Run it anywhere: `actions/read-evidence/read-evidence.test.sh` +set -uo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +script="$here/read-evidence.sh" +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +failures=0 + +check() { # name expected_rc actual_rc + if [ "$2" = "$3" ]; then echo " ok $1"; else echo " FAIL $1 (want rc=$2, got rc=$3)"; failures=$((failures+1)); fi +} + +# --- stubs ----------------------------------------------------------------------------------- +mkdir -p "$tmp/bin" +cat > "$tmp/bin/gh-three" <<'EOF' +#!/usr/bin/env bash +printf '11\n12\n13\n' +EOF +cat > "$tmp/bin/gh-none" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +cat > "$tmp/bin/gh-broken" <<'EOF' +#!/usr/bin/env bash +echo "api error" >&2; exit 1 +EOF +cat > "$tmp/bin/curl-ok" <<'EOF' +#!/usr/bin/env bash +out=""; prev="" +for a in "$@"; do [ "$prev" = "-o" ] && out=$a; prev=$a; done +echo "log body" > "$out" +EOF +# fails for job 12 only — a partial read, which must be reported and not fatal +cat > "$tmp/bin/curl-partial" <<'EOF' +#!/usr/bin/env bash +out=""; url=""; prev="" +for a in "$@"; do [ "$prev" = "-o" ] && out=$a; case $a in https://*) url=$a;; esac; prev=$a; done +case "$url" in *"/jobs/12/logs") exit 22;; esac +echo "log body" > "$out" +EOF +cat > "$tmp/bin/curl-allfail" <<'EOF' +#!/usr/bin/env bash +exit 22 +EOF +chmod +x "$tmp"/bin/* + +run() { TS_GH="$1" TS_CURL="$2" bash "$script" o/r 99 1 "$3" "" >"$tmp/out.log" 2>&1; echo $?; } + +echo "read-evidence.test.sh" + +rc=$(run "$tmp/bin/gh-three" "$tmp/bin/curl-ok" "$tmp/all") +check "three jobs, all readable" 0 "$rc" +n=$(find "$tmp/all" -name 'job-*.log' | wc -l) +[ "$n" = 3 ] && echo " ok wrote 3 logs" || { echo " FAIL wrote $n logs, want 3"; failures=$((failures+1)); } + +rc=$(run "$tmp/bin/gh-three" "$tmp/bin/curl-partial" "$tmp/part") +check "one job unreadable is NOT fatal (the gate decides what is missing)" 0 "$rc" +n=$(find "$tmp/part" -name 'job-*.log' | wc -l) +[ "$n" = 2 ] && echo " ok kept 2 logs, dropped the unreadable one" || { echo " FAIL kept $n, want 2"; failures=$((failures+1)); } +grep -q "could not read the log of job 12" "$tmp/out.log" \ + && echo " ok named the unreadable job" || { echo " FAIL did not name the unreadable job"; failures=$((failures+1)); } + +rc=$(run "$tmp/bin/gh-three" "$tmp/bin/curl-allfail" "$tmp/none1") +check "no log readable at all IS fatal" 1 "$rc" + +rc=$(run "$tmp/bin/gh-none" "$tmp/bin/curl-ok" "$tmp/none2") +check "an empty job list is fatal, not an empty-but-fine run" 1 "$rc" + +rc=$(run "$tmp/bin/gh-broken" "$tmp/bin/curl-ok" "$tmp/none3") +check "a failing job-list call is fatal" 1 "$rc" + +echo +[ "$failures" -eq 0 ] && { echo "all good"; exit 0; } || { echo "$failures failure(s)"; exit 1; } diff --git a/src/TestShards.jl b/src/TestShards.jl index e3c7a57..5eb4b3e 100644 --- a/src/TestShards.jl +++ b/src/TestShards.jl @@ -79,6 +79,7 @@ include("ownership.jl") # which units are this shard's, by assignment or by include("provider.jl") # the testset a unit runs in, and how a tool supplies its own include("run.jl") # running a unit, the evidence it leaves, and @shard / @unit include("json.jl") # minimal JSON writer — this package installs cold, with no deps +include("evidence.jl") # moving a shard's evidence to the collector without storage include("completeness.jl") # did the run cover the suite, or only look like it did include("coverage.jl") # one LCOV report out of N shards include("diagnose.jl") # read the history back: Observation, Diagnosis diff --git a/src/evidence.jl b/src/evidence.jl new file mode 100644 index 0000000..1eb7638 --- /dev/null +++ b/src/evidence.jl @@ -0,0 +1,248 @@ +# evidence.jl — moving a shard's evidence to the collector WITHOUT storage. +# +# The completeness gate compares what each shard was ASSIGNED against what it actually RAN, and that +# comparison is irreducibly cross-shard: `assign` is total only over the units the timing history has +# seen (everything else is placed by `_owns`'s round-robin over observation order), a shard checking +# itself compares `ctx.ran` against the very assignment that defined it, and under `Claimed` there is +# no assignment to compare against at all. So the evidence has to travel. +# +# It used to travel as an artifact. Actions artifact storage is metered in GigabyteHours against a +# per-period allowance, and once that allowance is crossed `CreateArtifact` refuses BEFORE any bytes +# are sent — a 0-byte artifact fails exactly as a large one does, and the accrual does not fall when +# artifacts are deleted. A suite that cannot verify itself for the rest of a billing period is not a +# suite. `actions/cache` was measured as a replacement and does not work: save succeeds and the entry +# is listed as active, restore misses every key. +# +# What is left needs no storage at all. A shard PRINTS its evidence; the collector reads that shard's +# job log back through the Actions API. Costs nothing, keeps nothing, needs only `actions: read` — +# which a fork pull request also has, unlike a write token. +# +# THE PARSING IS THE WHOLE RISK, WHICH IS WHY IT LIVES HERE AND NOT IN A STEP SCRIPT. The runner +# echoes a step's script into the log before running it, so the marker text appears TWICE: once as +# source, with `$(...)` unexpanded, and once as output. A collector that takes the first match +# decodes the literal string `$(base64 -w0 ev.tsv)` and reports success on garbage. That is a +# one-character bug, invisible in YAML, and pinned in `test/core/test_evidence.jl`. + +const EVIDENCE_MARK = "TESTSHARDS-EVIDENCE" + +# Base64's alphabet is exactly [A-Za-z0-9+/=], and an UNEXPANDED command substitution cannot be made +# of those characters — `$`, `(` and a space are all outside it. So the payload group discriminates +# the echoed source from the real output by CONSTRUCTION, and the collector needs no rule about which +# occurrence to prefer. An earlier version of this took the last match instead; that works until +# something else in the log happens to end with a marker, and "prefer the last" is a convention where +# this is a property. +const _EVIDENCE_RE = Regex("$(EVIDENCE_MARK):([A-Za-z0-9_.-]+):([A-Za-z0-9+/=]*):END") + +""" + evidence_line(sid, payload::AbstractString) -> String + +The one line a shard prints so its `payload` can be recovered from its job log. + +`payload` is base64-encoded, so a tab or a newline inside a unit name cannot break the framing — the +evidence is a TSV whose fields are exactly the strings the suite chose, and quoting them into a log +line any other way would make the collector's job depend on the units' names. +""" +function evidence_line(sid, payload::AbstractString) + s = String(sid) + occursin(r"^[A-Za-z0-9_.-]+$", s) || throw( + ArgumentError( + "evidence_line: shard id $(repr(s)) is not [A-Za-z0-9_.-]+. The id appears verbatim " * + "in the marker, so a character the pattern does not accept would make the line " * + "unreadable rather than merely unusual.", + ), + ) + return "$(EVIDENCE_MARK):$(s):$(_b64encode(payload)):END" +end + +""" + evidence_from_log(log::AbstractString) -> Union{Nothing,Pair{String,String}} + +Recover `sid => payload` from one job's log, or `nothing` if the log carries no evidence. + +Returns `nothing` rather than throwing: a shard that died before printing has no evidence and that is +a fact about the run for the caller to report, not an error in reading. A shard that printed a marker +whose payload will not decode DOES throw — that is corruption, and silently treating it as absent +would turn a broken transport into a missing shard. +""" +function evidence_from_log(log::AbstractString) + m = match(_EVIDENCE_RE, log) + m === nothing && return nothing + return String(m.captures[1]) => _b64decode(String(m.captures[2])) +end + +""" + evidence_scan(logs) -> Dict{String,String} + +`sid => payload` over an iterable of job logs, skipping the ones that carry no evidence. + +Duplicate shard ids are refused. Two logs claiming the same shard means the collector is reading the +wrong set of jobs — most often a run's jobs across two ATTEMPTS rather than one attempt's — and +merging them would produce exactly the double-counting the completeness gate exists to detect, while +looking like a clean recovery. +""" +function evidence_scan(logs) + out = Dict{String,String}() + for log in logs + e = evidence_from_log(log) + e === nothing && continue + haskey(out, e.first) && throw( + ArgumentError( + "evidence_scan: shard $(repr(e.first)) appears in two job logs. Read one " * + "ATTEMPT's jobs (`/runs/{id}/attempts/{n}/jobs`), not the run's, or the same " * + "shard from two attempts merges into one apparently complete run.", + ), + ) + out[e.first] = e.second + end + return out +end + +# --- base64, written out because this package installs cold and declares no dependencies ---------- +const _B64 = collect("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/") + +function _b64encode(s::AbstractString) + b = Vector{UInt8}(codeunits(s)) + io = IOBuffer() + i = 1 + while i + 2 <= length(b) + n = (Int(b[i]) << 16) | (Int(b[i + 1]) << 8) | Int(b[i + 2]) + for k in (18, 12, 6, 0) + print(io, _B64[((n >> k) & 0x3f) + 1]) + end + i += 3 + end + rem = length(b) - i + 1 + if rem == 1 + n = Int(b[i]) << 16 + print(io, _B64[((n >> 18) & 0x3f) + 1], _B64[((n >> 12) & 0x3f) + 1], "==") + elseif rem == 2 + n = (Int(b[i]) << 16) | (Int(b[i + 1]) << 8) + print( + io, + _B64[((n >> 18) & 0x3f) + 1], + _B64[((n >> 12) & 0x3f) + 1], + _B64[((n >> 6) & 0x3f) + 1], + "=", + ) + end + return String(take!(io)) +end + +function _b64decode(s::AbstractString) + rev = Dict(c => i - 1 for (i, c) in enumerate(_B64)) + chars = [c for c in s if c != '='] + all(c -> haskey(rev, c), chars) || throw( + ArgumentError( + "evidence: payload is not base64 — the marker is present but corrupt" + ), + ) + out = UInt8[] + acc = 0 + bits = 0 + for c in chars + acc = (acc << 6) | rev[c] + bits += 6 + if bits >= 8 + bits -= 8 + push!(out, UInt8((acc >> bits) & 0xff)) + end + end + return String(out) +end + +""" + evidence_payload(dir, tag) -> String + +The two files the completeness gate needs, as one string: every line of `shard-\$(tag).tsv` prefixed +`S\\t`, every line of `ran-\$(tag).tsv` prefixed `R\\t`. + +One payload per shard rather than two markers, because a shard that printed one marker and died would +otherwise be indistinguishable from a shard that ran nothing — and "ran nothing" is precisely the +state the gate exists to catch. + +The contents travel VERBATIM. `shard-*.tsv` and `ran-*.tsv` are the files +[`completeness_cli`](@ref) already reads, so changing how the evidence moves changes nothing about +what is checked. +""" +function evidence_payload(dir::AbstractString, tag::AbstractString) + io = IOBuffer() + for (prefix, name) in (("S", "shard-$(tag).tsv"), ("R", "ran-$(tag).tsv")) + path = joinpath(dir, name) + isfile(path) || continue + for line in eachline(path) + isempty(line) && continue + println(io, prefix, '\t', line) + end + end + return String(take!(io)) +end + +""" + emit_evidence(dir, tag) -> Nothing + +Print this shard's [`evidence_line`](@ref) to stdout, where the collector reads it back out of the +job log. Called from the shard's job AFTER the suite, so a failing suite still publishes what it ran. +""" +function emit_evidence(dir::AbstractString, tag::AbstractString) + println(evidence_line(tag, evidence_payload(dir, tag))) + return nothing +end + +""" + evidence_cli(args) -> Int + +`args` is a directory of job logs, then the two output paths. Each file in the directory is one job's +log; shard ids come from the markers, not from the file names, so the collector need not name files +after shards it has not read yet. + +Reconstructs `all-shards.tsv` and `all-ran.tsv` byte for byte as the artifact path produced them, so +[`completeness_cli`](@ref) is unchanged — only where its two inputs come from moves. + +Returns 1 when no log carried evidence. That is the same refusal the artifact path made when +`parts/` was empty (`units observed = 0`), and it is deliberate: a run that cannot be verified must +not report as verified. +""" +function evidence_cli(args) + length(args) == 3 || begin + println(stderr, "usage: evidence_cli(logdir, all-shards.tsv, all-ran.tsv)") + return 2 + end + dir, shards_out, ran_out = args + isdir(dir) || begin + println(stderr, "evidence_cli: $(dir) is not a directory") + return 2 + end + files = sort(filter(isfile, readdir(dir; join=true))) + found = evidence_scan(read(f, String) for f in files) + if isempty(found) + println( + stderr, + "evidence_cli: no job log carried a $(EVIDENCE_MARK) marker. Either no shard ran, or " * + "the collector read the wrong jobs. Not treating this as an empty-but-complete run.", + ) + return 1 + end + open(shards_out, "w") do sh + open(ran_out, "w") do rn + for sid in sort(collect(keys(found))) + for line in eachsplit(found[sid], '\n'; keepempty=false) + length(line) > 2 || continue + tag = line[1] + rest = SubString(line, 3) + if tag == 'S' + println(sh, rest) + elseif tag == 'R' + println(rn, rest) + else + nothing + end + end + end + end + end + println( + "Recovered evidence from $(length(found)) shard(s): ", + join(sort(collect(keys(found))), " "), + ) + return 0 +end diff --git a/test/core/test_evidence.jl b/test/core/test_evidence.jl new file mode 100644 index 0000000..6731230 --- /dev/null +++ b/test/core/test_evidence.jl @@ -0,0 +1,154 @@ +# test_evidence.jl — the log-based evidence transport (#82). +# +# Every case here is a mistake that was actually made while measuring this against live Actions runs, +# not a hypothetical. The transport itself was verified end to end in a real run; what a unit test +# can add is the PARSING, which is where all four of those mistakes were, and which a step script +# would have hidden. + +using TestShards +using Test + +const TS = TestShards + +@testset "round trip, including the characters that would break a delimiter" begin + for payload in ( + "A\tunit_one\nR\tunit_one", + "R\tweird unit name with spaces", + "R\tname\twith\ttabs\nR\ttrailing newline\n", + "", + "R\t" * repeat("x", 5000), # a long payload still fits one line + ) + line = TS.evidence_line("s1", payload) + @test !occursin('\n', line) # ONE line, or the collector cannot frame it + @test TS.evidence_from_log(line) == ("s1" => payload) + end +end + +@testset "the runner echoes the step script, so the marker appears TWICE" begin + # This is the log a real shard produces. The first occurrence is the runner echoing the step's + # source, with the command substitution UNEXPANDED; the second is the output. A collector that + # takes the first match decodes the literal `$(base64 -w0 ev.tsv)` — which is what happened, and + # it surfaced as `base64: invalid input` rather than as a wrong answer, purely by luck. + payload = "A\tunitA\nR\tunitA" + real = TS.evidence_line("s2", payload) + log = """ + 2026-08-20T03:35:56.2794941Z [36;1mecho "$(TS.EVIDENCE_MARK):s2:\$(base64 -w0 ev.tsv):END"[0m + 2026-08-20T03:35:56.2857806Z shell: /usr/bin/bash -e {0} + 2026-08-20T03:35:57.0947313Z $(real) + """ + @test TS.evidence_from_log(log) == ("s2" => payload) + + # …and the property that makes it work is that base64's alphabet cannot spell an unexpanded + # substitution, NOT an ordering convention. Asserted directly, so a future rewrite that switches + # to "take the last match" has to notice it is weakening the rule. + echoed_only = """ + 2026-08-20T03:35:56Z [36;1mecho "$(TS.EVIDENCE_MARK):s2:\$(base64 -w0 ev.tsv):END"[0m + """ + @test TS.evidence_from_log(echoed_only) === nothing +end + +@testset "a log with no evidence is a fact, not an error" begin + @test TS.evidence_from_log("") === nothing + @test TS.evidence_from_log("just some test output\nand a stack trace\n") === nothing + # a shard that died before printing: the collector must report a missing shard, and the + # completeness gate must be the thing that fails, not the reader + @test TS.evidence_from_log("ERROR: LoadError: something\n") === nothing +end + +@testset "a corrupt payload throws rather than reading as absent" begin + # `!` is outside base64's alphabet, so this does not match the marker at all and is absent… + @test TS.evidence_from_log("$(TS.EVIDENCE_MARK):s1:!!!!:END") === nothing + # …but a payload made of base64 characters that does not decode is corruption, and silently + # calling that "no evidence" would turn a broken transport into a missing shard. + @test_throws ArgumentError TS._b64decode("AB!CD") +end + +@testset "two logs claiming one shard is refused" begin + p = "R\tu" + a = TS.evidence_line("s1", p) + b = TS.evidence_line("s1", "R\tv") + @test TS.evidence_scan([a]) == Dict("s1" => p) + # the failure mode this guards: reading `/runs/{id}/jobs` (every attempt) instead of + # `/runs/{id}/attempts/{n}/jobs`, which merges a re-run's shard with the original's and looks + # like a clean recovery while double-counting + @test_throws ArgumentError TS.evidence_scan([a, b]) + @test TS.evidence_scan([a, TS.evidence_line("s2", p)]) == Dict("s1" => p, "s2" => p) + @test TS.evidence_scan(["nothing here", a]) == Dict("s1" => p) +end + +@testset "a shard id that cannot survive the marker is refused at emit time" begin + @test_throws ArgumentError TS.evidence_line("s:1", "R\tu") # `:` is the delimiter + @test_throws ArgumentError TS.evidence_line("", "R\tu") + @test_throws ArgumentError TS.evidence_line("s 1", "R\tu") + @test TS.evidence_line("s1", "R\tu") isa String + @test TS.evidence_line("shard-1.a_b", "R\tu") isa String +end + +@testset "evidence_cli reproduces the artifact path's two files, verbatim" begin + # The REAL formats, read off `src/json.jl`: `shard-.tsv` is one row + # `tag, started, finished, nrecords, seconds, seen`; `ran-.tsv` is rows of + # `index, tag, key`. The transport must reproduce the concatenation byte for byte, because + # `completeness_cli` is unchanged and parses exactly these. + mktempdir() do dir + out1 = joinpath(dir, "s1") + mkpath(out1) + write(joinpath(out1, "shard-s1.tsv"), "s1\t100.0\t160.5\t2\t42.0\t5\n") + write(joinpath(out1, "ran-s1.tsv"), "1\ts1\tapi/test_a.jl\n3\ts1\tapi/test_c.jl\n") + out2 = joinpath(dir, "s2") + mkpath(out2) + write(joinpath(out2, "shard-s2.tsv"), "s2\t101.0\t155.0\t1\t18.0\t5\n") + write(joinpath(out2, "ran-s2.tsv"), "2\ts2\tapi/test_b.jl\n") + + logs = joinpath(dir, "logs") + mkpath(logs) + write( + joinpath(logs, "job-1.txt"), + "some suite output\n" * + TS.evidence_line("s1", TS.evidence_payload(out1, "s1")) * + "\n", + ) + write( + joinpath(logs, "job-2.txt"), + TS.evidence_line("s2", TS.evidence_payload(out2, "s2")) * "\ntrailing\n", + ) + + sh = joinpath(dir, "all-shards.tsv") + rn = joinpath(dir, "all-ran.tsv") + @test TS.evidence_cli([logs, sh, rn]) == 0 + # byte for byte what `find parts -name 'shard-*.tsv' -exec cat {} + | sort -u` produced + @test sort(collect(eachline(sh))) == + ["s1\t100.0\t160.5\t2\t42.0\t5", "s2\t101.0\t155.0\t1\t18.0\t5"] + @test sort(collect(eachline(rn))) == + ["1\ts1\tapi/test_a.jl", "2\ts2\tapi/test_b.jl", "3\ts1\tapi/test_c.jl"] + + # and the recovered files are what the UNCHANGED completeness check consumes + @test TS.completeness_cli([sh, rn]) isa Integer + end +end + +@testset "a shard that wrote no ran-*.tsv still publishes its shard row" begin + # A shard that observed the suite and ran none of its units is a real state — every unit it + # owned may have been claimed by another shard — and it must still be counted, or the run looks + # smaller than it was. + mktempdir() do dir + write(joinpath(dir, "shard-s9.tsv"), "s9\t1.0\t2.0\t0\t0.0\t7\n") + p = TS.evidence_payload(dir, "s9") + @test occursin("S\ts9\t", p) + @test !occursin("R\t", p) + @test TS.evidence_from_log(TS.evidence_line("s9", p)) == ("s9" => p) + end +end + +@testset "no evidence at all is a FAILURE, not an empty-but-complete run" begin + # The exact shape of the outage: `collect` downloaded zero artifacts, every step before the gate + # reported success, and the gate correctly refused with `units observed = 0`. The replacement + # must refuse in the same place for the same reason. + mktempdir() do dir + logs = joinpath(dir, "logs") + mkpath(logs) + write(joinpath(logs, "job-1.txt"), "a shard that printed nothing useful\n") + @test TS.evidence_cli([logs, joinpath(dir, "s.tsv"), joinpath(dir, "r.tsv")]) == 1 + end + @test TS.evidence_cli(["definitely/not/a/dir", "a", "b"]) == 2 + @test TS.evidence_cli(["only-one-arg"]) == 2 +end