test(runner): record portable suite observations

.github/workflows/ci.yml

@@ -115,7 +115,7 @@ jobs:
           ln -s /bin/bash "${RUNNER_TEMP}/bash32/bash"
           export PATH="${RUNNER_TEMP}/bash32:${PATH}"
           bash --version | head -n 1 | grep -F 'version 3.2'
-          bash scripts/run-tests.bash --jobs 2
+          bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-bash32.json"
 
       - name: PowerShell syntax
         if: runner.os == 'Windows'
@@ -140,7 +140,7 @@ jobs:
       # says where it can run, so a new suite needs no workflow change.
       - name: Run test suites
         run: |
-          bash scripts/run-tests.bash --jobs 2
+          bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-current.json"
 
       - name: Smoke gos commands
         run: |
@@ -157,6 +157,14 @@ jobs:
         run: |
           fish --no-config --no-execute completions/gos.fish
 
+      - name: Upload suite observations
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: suite-observations-${{ matrix.os }}
+          path: ${{ runner.temp }}/test-summary-*.json
+          if-no-files-found: warn
+
   action:
     name: Action (${{ matrix.os }})
     runs-on: ${{ matrix.os }}

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
 ### Changed
 
 - CI cancels superseded runs of the same pull request while retaining separate PRs, all OS/shell jobs, and existing canary/release serialization. Pushes to main are grouped per commit and never cancelled, so every merged commit keeps a complete CI result.
+- The portable test runner can write opt-in per-suite durations and machine-readable outcome/source/interpreter metadata; CI retains those observations without changing its wave scheduler or pass criteria.
 - Pinned GitHub Actions dependencies bumped by Dependabot: `actions/attest` 4.2.1 to 4.2.2, `softprops/action-gh-release` 3.0.2 to 3.0.3, and `github/codeql-action/upload-sarif`.
 
 ## [1.11.0] - 2026-09-06

CONTRIBUTING.md

@@ -110,6 +110,18 @@ constructs such as `mapfile`, `declare -A`, `${var^^}`, and unguarded
 `${array[@]:+"${array[@]}"}`). CI runs the suites under `/bin/bash` on macOS
 to enforce that floor.
 
+## Optional suite observations
+
+Use `scripts/run-tests.bash --jobs 2 --summary /tmp/gos-suite-summary.json` to
+record per-suite whole-second durations, pass/fail/OS-skip status, exit status,
+host/target OS, Bash version and source HEAD/dirty state. It adds no runtime
+dependency and does not change test criteria or the portable wave scheduler.
+The summary is written atomically even when a child fails; missing/corrupt
+status, duration or log evidence fails closed. Exported sources have a null HEAD.
+CI retains separate current-Bash and native macOS Bash 3.2 observations.
+Compare unchanged source/fixtures and matching runner/interpreter classes before
+proposing scheduling changes; these coarse measurements are not a claimed saving.
+
 ## Pull Request Guidelines
 
 - Keep changes focused and explain the user-visible behavior.

scripts/run-tests.bash

@@ -10,15 +10,16 @@ set -euo pipefail
 # (comma-separated lists of linux, macos, windows) and the runner reports the
 # skip instead of failing.
 #
-# Usage: scripts/run-tests.bash [--jobs N|auto] [--os linux|macos|windows] [--list] [suite ...]
+# Usage: scripts/run-tests.bash [--jobs N|auto] [--os linux|macos|windows] [--list] [--summary PATH] [suite ...]
 # A suite may be given as a path (tests/foo.bash) or a bare name (foo).
 
 usage() {
-  printf 'Usage: %s [--jobs N|auto] [--os linux|macos|windows] [--list] [suite ...]\n' "${0##*/}" >&2
+  printf 'Usage: %s [--jobs N|auto] [--os linux|macos|windows] [--list] [--summary PATH] [suite ...]\n' "${0##*/}" >&2
 }
 
 jobs="auto"
 list_only=0
+summary_path=""
 target_os=""
 requested=()
 while [ "$#" -gt 0 ]; do
@@ -39,6 +40,14 @@ while [ "$#" -gt 0 ]; do
       target_os="$2"
       shift 2
       ;;
+    --summary)
+      [ "$#" -ge 2 ] && [ -n "$2" ] || {
+        usage
+        exit 2
+      }
+      summary_path="$2"
+      shift 2
+      ;;
     --list)
       list_only=1
       shift
@@ -58,6 +67,13 @@ while [ "$#" -gt 0 ]; do
   esac
 done
 
+# A relative --summary path names a file under the caller's directory, not
+# under the repository root the runner switches to below.
+case "$summary_path" in
+  '' | /* | [A-Za-z]:[/\\]*) ;;
+  *) summary_path="${PWD%/}/${summary_path}" ;;
+esac
+
 repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "$repo_root"
 
@@ -104,8 +120,8 @@ fi
 discover_suites() {
   local path
   {
-    git -c core.quotePath=false ls-files 'tests/*.bash' 2>/dev/null || ls tests/*.bash
-  } | while IFS= read -r path; do
+    git ls-files -z 'tests/*.bash' 2>/dev/null || printf '%s\0' tests/*.bash
+  } | while IFS= read -r -d '' path; do
     case "${path##*/}" in
       lib*.bash) continue ;;
     esac
@@ -210,25 +226,136 @@ fi
 log_dir="$(mktemp -d)"
 trap 'rm -rf "$log_dir"' EXIT
 
+# Opt-in measurements use Bash's portable elapsed-seconds clock. Whole-second
+# resolution is explicit; this observes the existing waves, not a new scheduler.
+summary_count=0
+summary_paths=()
+summary_states=()
+summary_durations=()
+summary_statuses=()
+summary_reasons=()
+record_suite() {
+  [ -n "$summary_path" ] || return 0
+  summary_paths[summary_count]="$1"
+  summary_states[summary_count]="$2"
+  summary_durations[summary_count]="$3"
+  summary_statuses[summary_count]="$4"
+  summary_reasons[summary_count]="$5"
+  summary_count=$((summary_count + 1))
+}
+
 run_suite() {
-  # Writes the suite's combined output to its log and its status to a marker.
-  local path="$1" name status
+  # Writes the suite's combined output and status. No stdin: a suite must
+  # never consume the runner's input. Measurements never change pass criteria.
+  local path="$1" name status started elapsed
   name="$path"
   mkdir -p "${log_dir}/${name%/*}"
   status=0
-  # No stdin: a suite that reads it would otherwise eat the runner's own
-  # input (and CI has none anyway).
+  started=$SECONDS
   bash "$path" </dev/null >"${log_dir}/${name}.log" 2>&1 || status=$?
   printf '%s\n' "$status" >"${log_dir}/${name}.status"
+  if [ -n "$summary_path" ]; then
+    # SECONDS follows the wall clock; a backward clock step is not a failure.
+    elapsed=$((SECONDS - started))
+    [ "$elapsed" -ge 0 ] || elapsed=0
+    printf '%s\n' "$elapsed" >"${log_dir}/${name}.duration"
+  fi
 }
 
 report_suite() {
-  local path="$1" name status
+  # Unreadable or non-canonical markers fail closed, but the suite's header and
+  # log are still printed so the failure keeps its diagnostics.
+  local path="$1" name status duration=null reason=""
   name="$path"
-  status="$(cat "${log_dir}/${name}.status")" || return 1
-  printf '=== %s (%s) ===\n' "$path" "$([ "$status" -eq 0 ] && echo ok || echo "FAILED, status ${status}")"
-  cat "${log_dir}/${name}.log" || return 1
-  [ "$status" -eq 0 ]
+  if ! status="$(cat "${log_dir}/${name}.status")"; then
+    status=null reason=missing-status
+  else
+    case "$status" in
+      '' | 0?* | *[!0-9]*) status=null reason=invalid-status ;;
+    esac
+  fi
+  if [ -z "$reason" ] && [ -n "$summary_path" ]; then
+    if ! duration="$(cat "${log_dir}/${name}.duration")"; then
+      duration=null reason=missing-duration
+    else
+      case "$duration" in
+        '' | 0?* | *[!0-9]*) duration=null reason=invalid-duration ;;
+      esac
+    fi
+  fi
+  if [ -n "$reason" ]; then
+    printf '=== %s (FAILED, %s) ===\n' "$path" "$reason"
+  else
+    printf '=== %s (%s) ===\n' "$path" "$([ "$status" = 0 ] && echo ok || echo "FAILED, status ${status}")"
+  fi
+  cat "${log_dir}/${name}.log" || reason="${reason:-missing-log}"
+  [ -n "$reason" ] || [ "$status" = 0 ] || reason=child-exit
+  if [ -n "$reason" ]; then
+    record_suite "$path" failed "$duration" "$status" "$reason"
+    return 1
+  fi
+  record_suite "$path" passed "$duration" "$status" ""
+}
+
+# JSON quoting uses Bash builtins, including ASCII control characters, so the
+# summary introduces no jq/Python/runtime dependency and preserves literal names.
+json_string() {
+  local value="$1" i char code LC_ALL=C
+  printf '"'
+  for ((i = 0; i < ${#value}; i++)); do
+    char="${value:i:1}"
+    case "$char" in
+      '"') printf '\\"' ;;
+      \\) printf '%s%s' "$char" "$char" ;;
+      *)
+        printf -v code '%d' "'$char"
+        if [ "$code" -lt 32 ]; then
+          printf '\\u%04x' "$code"
+        else
+          printf '%s' "$char"
+        fi
+        ;;
+    esac
+  done
+  printf '"'
+}
+
+write_summary() {
+  [ -n "$summary_path" ] || return 0
+  local head host temp i comma="" dirty=false
+  head="$(git rev-parse HEAD 2>/dev/null)" || head=""
+  if [ -n "$head" ] && ! git diff --quiet HEAD --; then dirty=true; fi
+  host="$(uname -s)"
+  # mv would move the report into an existing directory and report success.
+  [ ! -d "$summary_path" ] || return 1
+  mkdir -p "$(dirname "$summary_path")" || return 1
+  temp="$(mktemp "${summary_path}.XXXXXX")" || return 1
+  if ! {
+    printf '{"schemaVersion":1,"os":'
+    json_string "$target_os"
+    printf ',"hostOs":'
+    json_string "$host"
+    printf ',"shell":'
+    json_string "$BASH_VERSION"
+    printf ',"head":'
+    if [ -n "$head" ]; then json_string "$head"; else printf null; fi
+    printf ',"dirty":%s,"jobs":%s,"durationUnit":"seconds","suites":[' "$dirty" "$jobs"
+    for ((i = 0; i < summary_count; i++)); do
+      printf '%s{"path":' "$comma"
+      json_string "${summary_paths[i]}"
+      printf ',"status":'
+      json_string "${summary_states[i]}"
+      printf ',"durationSeconds":%s,"exitStatus":%s,"reason":' "${summary_durations[i]}" "${summary_statuses[i]}"
+      json_string "${summary_reasons[i]}"
+      printf '}'
+      comma=,
+    done
+    printf ']}\n'
+  } >"$temp"; then
+    rm -f "$temp"
+    return 1
+  fi
+  mv "$temp" "$summary_path"
 }
 
 to_run=()
@@ -239,6 +366,7 @@ while IFS= read -r path; do
   if [ -n "$reason" ]; then
     printf '=== %s (skipped on %s: %s) ===\n' "$path" "$target_os" "$reason"
     skipped=$((skipped + 1))
+    record_suite "$path" skipped null null "$reason"
     continue
   fi
   to_run=(${to_run[@]:+"${to_run[@]}"} "$path")
@@ -282,8 +410,14 @@ else
   flush_wave
 fi
 
+summary_written=1
+if ! write_summary; then
+  printf 'not ok - could not write test summary: %s\n' "$summary_path" >&2
+  summary_written=0
+fi
 if [ -n "$failed" ]; then
   printf 'not ok - test suites failed: %s\n' "$failed" >&2
   exit 1
 fi
+[ "$summary_written" -eq 1 ] || exit 1
 printf 'ok - %s test suite(s) passed, %s skipped on %s\n' "$passed" "$skipped" "$target_os"

tests/runner.bash

@@ -144,6 +144,84 @@ run_runner --list invalid
 assert_status 2 "$status" 'tracked suite missing on disk' "$output"
 pass 'runner rejects malformed metadata and missing suites instead of reporting green'
 
+# Opt-in summaries preserve every selected result in deterministic order and
+# identify the exact interpreter/source rather than claiming scheduler savings.
+for mode in 1 2; do
+  summary="${test_root}/summary-${mode}.json"
+  run_runner --jobs "$mode" --summary "$summary" pass fail signal 'literal [x]' skipped
+  assert_status 1 "$status" "summary failure jobs=${mode}" "$output"
+  ruby -rjson -e '
+    report = JSON.parse(File.read(ARGV[0]))
+    abort "missing metadata" unless report.values_at("schemaVersion", "os", "hostOs", "durationUnit") == [1, "linux", ARGV[1], "seconds"]
+    abort "missing shell/head metadata" unless report["shell"].is_a?(String) && report.key?("head") && report.key?("dirty")
+    suites = report.fetch("suites")
+    states = suites.to_h { |suite| [suite.fetch("path"), suite.fetch("status")] }
+    abort "incorrect states" unless states == {"tests/pass.bash"=>"passed", "tests/fail.bash"=>"failed", "tests/signal.bash"=>"failed", "tests/literal [x].bash"=>"passed", "tests/skipped.bash"=>"skipped"}
+    abort "incorrect exit status" unless suites.find { |suite| suite["path"] == "tests/fail.bash" }["exitStatus"] == 7
+    abort "signal was lost" unless suites.find { |suite| suite["path"] == "tests/signal.bash" }["exitStatus"] > 128
+    suites.each do |suite|
+      duration = suite.fetch("durationSeconds")
+      abort "invalid duration" unless suite["status"] == "skipped" ? duration.nil? : duration.is_a?(Integer) && duration >= 0
+    end
+  ' "$summary" "$(uname -s)" || fail 'summary preserves metadata, statuses, signals and timing'
+done
+# JSON string quoting must be tested without creating filenames Windows forbids.
+# Exercise the exact runner helper with an argument string, then use a portable
+# filename for the duration observation.
+helper="${test_root}/json-string.bash"
+sed -n '/^json_string() {/,/^}/p' "${fixture}/scripts/run-tests.bash" >"$helper"
+quoted='quote" and \name'
+# shellcheck disable=SC2016 # The child shell expands its own arguments.
+quoted_json="$("$BASH" -c '. "$1"; json_string "$2"' _ "$helper" "$quoted")"
+ruby -rjson -e 'abort "JSON quoting lost" unless JSON.parse(ARGV[0]) == ARGV[1]' "$quoted_json" "$quoted" || fail 'JSON quotes and backslashes'
+printf '#!/usr/bin/env bash\nsleep 1\n' >"${fixture}/tests/duration.bash"
+git -C "$fixture" add tests/duration.bash
+run_runner --jobs 1 --summary "${test_root}/duration.json" duration
+assert_status 0 "$status" 'duration summary' "$output"
+ruby -rjson -e 'suite = JSON.parse(File.read(ARGV[0])).fetch("suites").fetch(0); abort "timing lost" unless suite["path"] == "tests/duration.bash" && suite["durationSeconds"] >= 1' "${test_root}/duration.json" || fail 'portable duration observation'
+for suffix in status log duration; do
+  GOS_TEST_RUNNER_REAL_CAT="$real_cat" GOS_TEST_RUNNER_FAIL_READ="$suffix" \
+    PATH="${test_root}/tools:${PATH}" run_runner --jobs 2 --summary "${test_root}/missing-${suffix}.json" pass 'literal [x]'
+  assert_status 1 "$status" "summary unreadable ${suffix}" "$output"
+  ruby -rjson -e 'report = JSON.parse(File.read(ARGV[0])); abort "missing failures" unless report["suites"].length == 2 && report["suites"].all? { |suite| suite["status"] == "failed" && suite["reason"].start_with?("missing-") }' "${test_root}/missing-${suffix}.json" || fail 'missing measurement/status/log fails closed in summary'
+done
+for corrupt in corrupt 007; do
+  cat >"${test_root}/tools/cat" <<TOOL
+#!/usr/bin/env bash
+case "\${1:-}" in
+  *."\$GOS_TEST_RUNNER_FAIL_READ") printf '%s\\n' '${corrupt}'; exit 0 ;;
+esac
+exec "\$GOS_TEST_RUNNER_REAL_CAT" "\$@"
+TOOL
+  for suffix in status duration; do
+    GOS_TEST_RUNNER_REAL_CAT="$real_cat" GOS_TEST_RUNNER_FAIL_READ="$suffix" \
+      PATH="${test_root}/tools:${PATH}" run_runner --jobs 1 --summary "${test_root}/corrupt-${suffix}.json" pass
+    assert_status 1 "$status" "summary corrupt ${suffix} ${corrupt}" "$output"
+    assert_contains "$output" 'PASS stdout' "corrupt ${suffix} keeps the suite log"
+    ruby -rjson -e 'suite = JSON.parse(File.read(ARGV[0])).fetch("suites").fetch(0); abort "corruption accepted" unless suite["status"] == "failed" && suite["reason"].start_with?("invalid-")' "${test_root}/corrupt-${suffix}.json" || fail 'corrupt markers fail closed'
+  done
+done
+run_runner --summary
+assert_status 2 "$status" 'missing summary argument' "$output"
+run_runner --summary "${test_root}/summary-1.json/child" pass fail
+assert_status 1 "$status" 'unwritable summary destination' "$output"
+assert_contains "$output" 'could not write test summary' 'summary write fails closed'
+assert_contains "$output" 'not ok - test suites failed: tests/fail.bash' 'summary write failure keeps the failed suite list'
+mkdir -p "${test_root}/summary-dir"
+run_runner --summary "${test_root}/summary-dir" pass
+assert_status 1 "$status" 'directory summary destination' "$output"
+[ -z "$(ls -A "${test_root}/summary-dir")" ] || fail 'a directory summary destination must not receive the report'
+# The runner changes into the repository root; a relative summary path must
+# still land beside the caller instead of dirtying the checkout.
+relative_dir="${test_root}/relative-cwd"
+mkdir -p "$relative_dir"
+status=0
+output="$(cd "$relative_dir" && "$BASH" "${fixture}/scripts/run-tests.bash" --os linux --jobs 1 --summary relative.json pass 2>&1)" || status=$?
+assert_status 0 "$status" 'relative summary path' "$output"
+[ -f "${relative_dir}/relative.json" ] || fail 'relative summary path resolves against the caller directory'
+[ ! -e "${fixture}/relative.json" ] || fail 'relative summary path must not write into the repository'
+pass 'optional summaries preserve serial/parallel outcomes, timings, metadata and failure evidence'
+
 # Exported sources have no git metadata; they still discover suites on disk.
 exported="${test_root}/exported"
 mkdir -p "${exported}/scripts" "${exported}/tests"

tests/workflows.bash

@@ -577,7 +577,7 @@ assert(psscriptanalyzer && psscriptanalyzer["if"] == "runner.os == 'Windows'" &&
 bash32 = step_named(smoke_steps, "Bash 3.2 compatibility")
 assert(bash32, "smoke job must exercise the bash 3.2 floor")
 assert(bash32["if"] == "runner.os == 'macOS'", "bash 3.2 compatibility step must run on macOS, the only runner shipping bash 3.2")
-assert(bash32["run"].to_s.include?("grep -F 'version 3.2'") && bash32["run"].to_s.include?("bash scripts/run-tests.bash") && bash32["run"].to_s.lines.any? { |line| line.strip == "bash scripts/run-tests.bash --jobs 2" }, "bash 3.2 compatibility step must verify the interpreter and run the feature suites under it")
+assert(bash32["run"].to_s.include?("grep -F 'version 3.2'") && bash32["run"].to_s.include?("bash scripts/run-tests.bash") && bash32["run"].to_s.lines.any? { |line| line.strip == 'bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-bash32.json"' }, "bash 3.2 compatibility step must verify the interpreter and run the feature suites under it")
 assert(command_surface_sync, "smoke job must check generated command surfaces")
 assert(command_surface_sync["run"].to_s.include?("bash scripts/sync-command-surfaces.bash --check"), "command surface sync must use the orchestrator")
 
@@ -601,6 +601,11 @@ assert(bash_syntax["run"].to_s.include?("git ls-files -z '*.sh' '*.bash' | xargs
 tracked_powershell_files.each do |path|
   assert(smoke_runs.include?(path), "smoke job PowerShell syntax must cover tracked PowerShell file #{path}")
 end
+summary_upload = step_named(smoke_steps, "Upload suite observations")
+assert(summary_upload && summary_upload["if"] == "always()", "suite summaries must survive failed suites")
+assert(summary_upload.dig("with", "name") == "suite-observations-${{ matrix.os }}", "summary artifact names must distinguish OS jobs")
+assert(summary_upload.dig("with", "path") == "${{ runner.temp }}/test-summary-*.json", "both current Bash and Bash 3.2 summaries must be retained")
+assert(smoke_runs.include?('bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-current.json"'), "ordinary suite runs must collect comparable observations")
 assert(!smoke_runs.match?(%r{bash tests/}), "smoke job must run suites through scripts/run-tests.bash, not hand-listed bash tests/ commands")
 assert(smoke_runs.include?("packaging/chocolatey/tools/chocolateyInstall.ps1"), "smoke job must parse the Chocolatey PowerShell installer")
 assert(smoke_runs.include?("packaging/chocolatey/tools/chocolateyUninstall.ps1"), "smoke job must parse the Chocolatey PowerShell uninstaller")