feat: the pieces for carrying completeness evidence in the shards' job logs

Project.toml

@@ -1,6 +1,6 @@
 name = "TestShards"
 uuid = "acceef1d-f5e0-4fe4-a546-818dc56ce7b2"
-version = "0.3.37"
+version = "0.3.38"
 authors = ["sota shimozono <shimozono-sota631@g.ecc.u-tokyo.ac.jp>"]
 
 [deps]

actions/read-evidence/action.yml

@@ -0,0 +1,49 @@
+name: Read TestShards evidence from the shards' job logs
+description: >
+  Download each shard job's log for one attempt of one run, so the collector can recover the
+  completeness evidence the shards printed. Uses no artifact storage and no cache; needs only
+  `actions: read`.
+
+# WHY THIS IS AN ACTION AND NOT A `run:` IN THE WORKFLOW — the same reason as
+# `actions/read-history`. A `run:` step in a REUSABLE workflow executes in the CALLER's checkout,
+# so it can only invoke files the caller has; a script referenced by path works in this
+# repository's own CI and exits 127 on every consumer. An action's files are fetched with it.
+
+inputs:
+  repository:
+    description: owner/name of the repository whose run is being read.
+    required: false
+    default: ${{ github.repository }}
+  run-id:
+    required: false
+    default: ${{ github.run_id }}
+  attempt:
+    description: >
+      Run attempt. Reading `/runs/{id}/jobs` instead of this attempt's jobs merges every attempt,
+      so a re-run's shards double-count while the recovery looks clean.
+    required: false
+    default: ${{ github.run_attempt }}
+  job-prefix:
+    description: Only jobs whose name starts with this are read.
+    required: false
+    default: ''
+  path:
+    description: Directory to write the logs into.
+    required: false
+    default: shard-logs
+  token:
+    description: >
+      Needs `actions: read`. A fork pull request has it, and it needs no write scope at all.
+    required: false
+    default: ${{ github.token }}
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+      run: >
+        bash "${{ github.action_path }}/read-evidence.sh"
+        "${{ inputs.repository }}" "${{ inputs.run-id }}" "${{ inputs.attempt }}"
+        "${{ inputs.path }}" "${{ inputs.job-prefix }}"

actions/read-evidence/read-evidence.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+#
+# Fetch every shard job's LOG for one attempt of one run, into a directory, so the collector can
+# recover the evidence markers from them (TestShards.evidence_cli).
+#
+# Two things here are not obvious and both were measured against live runs:
+#
+#  1. `gh api .../actions/jobs/{id}/logs` DOES NOT WORK. The endpoint answers 302 with a
+#     short-lived blob URL; gh exits 1 with no useful message. curl -L follows it.
+#  2. The jobs must come from `/runs/{id}/attempts/{n}/jobs`, NOT `/runs/{id}/jobs`. The latter
+#     returns every attempt's jobs, so a re-run merges two attempts' shards and the recovered
+#     evidence double-counts while looking complete.
+#
+# The network commands are indirected through TS_GH / TS_CURL so `read-evidence.test.sh` can run
+# every branch below without a network or a token.
+set -uo pipefail
+
+repo=${1:?repo}
+run_id=${2:?run id}
+attempt=${3:?attempt}
+outdir=${4:?output directory}
+filter=${5:-}          # only jobs whose name starts with this; empty means all
+
+GH=${TS_GH:-gh}
+CURL=${TS_CURL:-curl}
+
+mkdir -p "$outdir"
+
+ids=$("$GH" api "repos/$repo/actions/runs/$run_id/attempts/$attempt/jobs" \
+        --paginate --jq ".jobs[]|select(.name|startswith(\"$filter\"))|\"\(.id)\"" 2>/dev/null)
+rc=$?
+if [ $rc -ne 0 ]; then
+  echo "::error::could not list the jobs of run $run_id attempt $attempt. Without the job list there is nothing to read; this is not a partial result." >&2
+  exit 1
+fi
+if [ -z "$ids" ]; then
+  # An empty list is NOT an empty-but-fine run: the collector runs after the shards, so at least
+  # one job must match. Reporting success here would hand the gate zero evidence and let a run
+  # with no shards look verified.
+  echo "::error::no job in run $run_id attempt $attempt matched '$filter'." >&2
+  exit 1
+fi
+
+n=0; missed=0
+for id in $ids; do
+  if "$CURL" -sSL --fail-with-body \
+       -H "Authorization: Bearer ${GH_TOKEN:-}" \
+       -H "Accept: application/vnd.github+json" \
+       "https://api.github.com/repos/$repo/actions/jobs/$id/logs" -o "$outdir/job-$id.log" 2>/dev/null
+  then
+    n=$((n+1))
+  else
+    # A job whose log cannot be read is reported and NOT fatal here: which shards are missing is
+    # the completeness gate's judgement to make, not this script's. Failing here would replace a
+    # precise "unit X ran nowhere" with an opaque transport error.
+    rm -f "$outdir/job-$id.log"
+    missed=$((missed+1))
+    echo "::warning::could not read the log of job $id" >&2
+  fi
+done
+
+echo "read $n job log(s) into $outdir${missed:+, $missed unreadable}"
+[ "$n" -gt 0 ] || { echo "::error::no job log could be read at all." >&2; exit 1; }

actions/read-evidence/read-evidence.test.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# The states this script must handle that a normal run never shows: an unreadable job, an empty
+# job list, and an API call that fails outright. All three are reachable here without a network
+# or a token, because the two network commands are indirected through TS_GH / TS_CURL.
+#
+# The failure this guards against is specific: `gh api .../jobs/{id}/logs` returns 302 and gh does
+# not follow it, so a version of this that used gh for the log fetch reported "read 0 job logs"
+# while every job had a perfectly good log. That is a transport that fails SILENTLY into an
+# unverifiable run, which is the one outcome the completeness gate exists to prevent.
+#
+# Run it anywhere: `actions/read-evidence/read-evidence.test.sh`
+set -uo pipefail
+
+here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+script="$here/read-evidence.sh"
+tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
+failures=0
+
+check() { # name expected_rc actual_rc
+  if [ "$2" = "$3" ]; then echo "  ok   $1"; else echo "  FAIL $1 (want rc=$2, got rc=$3)"; failures=$((failures+1)); fi
+}
+
+# --- stubs -----------------------------------------------------------------------------------
+mkdir -p "$tmp/bin"
+cat > "$tmp/bin/gh-three" <<'EOF'
+#!/usr/bin/env bash
+printf '11\n12\n13\n'
+EOF
+cat > "$tmp/bin/gh-none" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+cat > "$tmp/bin/gh-broken" <<'EOF'
+#!/usr/bin/env bash
+echo "api error" >&2; exit 1
+EOF
+cat > "$tmp/bin/curl-ok" <<'EOF'
+#!/usr/bin/env bash
+out=""; prev=""
+for a in "$@"; do [ "$prev" = "-o" ] && out=$a; prev=$a; done
+echo "log body" > "$out"
+EOF
+# fails for job 12 only — a partial read, which must be reported and not fatal
+cat > "$tmp/bin/curl-partial" <<'EOF'
+#!/usr/bin/env bash
+out=""; url=""; prev=""
+for a in "$@"; do [ "$prev" = "-o" ] && out=$a; case $a in https://*) url=$a;; esac; prev=$a; done
+case "$url" in *"/jobs/12/logs") exit 22;; esac
+echo "log body" > "$out"
+EOF
+cat > "$tmp/bin/curl-allfail" <<'EOF'
+#!/usr/bin/env bash
+exit 22
+EOF
+chmod +x "$tmp"/bin/*
+
+run() { TS_GH="$1" TS_CURL="$2" bash "$script" o/r 99 1 "$3" "" >"$tmp/out.log" 2>&1; echo $?; }
+
+echo "read-evidence.test.sh"
+
+rc=$(run "$tmp/bin/gh-three" "$tmp/bin/curl-ok" "$tmp/all")
+check "three jobs, all readable" 0 "$rc"
+n=$(find "$tmp/all" -name 'job-*.log' | wc -l)
+[ "$n" = 3 ] && echo "  ok   wrote 3 logs" || { echo "  FAIL wrote $n logs, want 3"; failures=$((failures+1)); }
+
+rc=$(run "$tmp/bin/gh-three" "$tmp/bin/curl-partial" "$tmp/part")
+check "one job unreadable is NOT fatal (the gate decides what is missing)" 0 "$rc"
+n=$(find "$tmp/part" -name 'job-*.log' | wc -l)
+[ "$n" = 2 ] && echo "  ok   kept 2 logs, dropped the unreadable one" || { echo "  FAIL kept $n, want 2"; failures=$((failures+1)); }
+grep -q "could not read the log of job 12" "$tmp/out.log" \
+  && echo "  ok   named the unreadable job" || { echo "  FAIL did not name the unreadable job"; failures=$((failures+1)); }
+
+rc=$(run "$tmp/bin/gh-three" "$tmp/bin/curl-allfail" "$tmp/none1")
+check "no log readable at all IS fatal" 1 "$rc"
+
+rc=$(run "$tmp/bin/gh-none" "$tmp/bin/curl-ok" "$tmp/none2")
+check "an empty job list is fatal, not an empty-but-fine run" 1 "$rc"
+
+rc=$(run "$tmp/bin/gh-broken" "$tmp/bin/curl-ok" "$tmp/none3")
+check "a failing job-list call is fatal" 1 "$rc"
+
+echo
+[ "$failures" -eq 0 ] && { echo "all good"; exit 0; } || { echo "$failures failure(s)"; exit 1; }

src/TestShards.jl

@@ -79,6 +79,7 @@ include("ownership.jl")     # which units are this shard's, by assignment or by
 include("provider.jl")      # the testset a unit runs in, and how a tool supplies its own
 include("run.jl")           # running a unit, the evidence it leaves, and @shard / @unit
 include("json.jl")          # minimal JSON writer — this package installs cold, with no deps
+include("evidence.jl")     # moving a shard's evidence to the collector without storage
 include("completeness.jl")  # did the run cover the suite, or only look like it did
 include("coverage.jl")      # one LCOV report out of N shards
 include("diagnose.jl")      # read the history back: Observation, Diagnosis