diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4cd65e..285bc6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: ln -s /bin/bash "${RUNNER_TEMP}/bash32/bash" export PATH="${RUNNER_TEMP}/bash32:${PATH}" bash --version | head -n 1 | grep -F 'version 3.2' - bash scripts/run-tests.bash --jobs 2 + bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-bash32.json" - name: PowerShell syntax if: runner.os == 'Windows' @@ -140,7 +140,7 @@ jobs: # says where it can run, so a new suite needs no workflow change. - name: Run test suites run: | - bash scripts/run-tests.bash --jobs 2 + bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-current.json" - name: Smoke gos commands run: | @@ -157,6 +157,14 @@ jobs: run: | fish --no-config --no-execute completions/gos.fish + - name: Upload suite observations + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: suite-observations-${{ matrix.os }} + path: ${{ runner.temp }}/test-summary-*.json + if-no-files-found: warn + action: name: Action (${{ matrix.os }}) runs-on: ${{ matrix.os }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3cca0..b48a8c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this ### Changed - CI cancels superseded runs of the same pull request while retaining separate PRs, all OS/shell jobs, and existing canary/release serialization. Pushes to main are grouped per commit and never cancelled, so every merged commit keeps a complete CI result. +- The portable test runner can write opt-in per-suite durations and machine-readable outcome/source/interpreter metadata; CI retains those observations without changing its wave scheduler or pass criteria. - Pinned GitHub Actions dependencies bumped by Dependabot: `actions/attest` 4.2.1 to 4.2.2, `softprops/action-gh-release` 3.0.2 to 3.0.3, and `github/codeql-action/upload-sarif`. ## [1.11.0] - 2026-09-06 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4215bfc..4196f57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,6 +110,18 @@ constructs such as `mapfile`, `declare -A`, `${var^^}`, and unguarded `${array[@]:+"${array[@]}"}`). CI runs the suites under `/bin/bash` on macOS to enforce that floor. +## Optional suite observations + +Use `scripts/run-tests.bash --jobs 2 --summary /tmp/gos-suite-summary.json` to +record per-suite whole-second durations, pass/fail/OS-skip status, exit status, +host/target OS, Bash version and source HEAD/dirty state. It adds no runtime +dependency and does not change test criteria or the portable wave scheduler. +The summary is written atomically even when a child fails; missing/corrupt +status, duration or log evidence fails closed. Exported sources have a null HEAD. +CI retains separate current-Bash and native macOS Bash 3.2 observations. +Compare unchanged source/fixtures and matching runner/interpreter classes before +proposing scheduling changes; these coarse measurements are not a claimed saving. + ## Pull Request Guidelines - Keep changes focused and explain the user-visible behavior. diff --git a/scripts/run-tests.bash b/scripts/run-tests.bash index 044b748..10e69b6 100755 --- a/scripts/run-tests.bash +++ b/scripts/run-tests.bash @@ -10,15 +10,16 @@ set -euo pipefail # (comma-separated lists of linux, macos, windows) and the runner reports the # skip instead of failing. # -# Usage: scripts/run-tests.bash [--jobs N|auto] [--os linux|macos|windows] [--list] [suite ...] +# Usage: scripts/run-tests.bash [--jobs N|auto] [--os linux|macos|windows] [--list] [--summary PATH] [suite ...] # A suite may be given as a path (tests/foo.bash) or a bare name (foo). usage() { - printf 'Usage: %s [--jobs N|auto] [--os linux|macos|windows] [--list] [suite ...]\n' "${0##*/}" >&2 + printf 'Usage: %s [--jobs N|auto] [--os linux|macos|windows] [--list] [--summary PATH] [suite ...]\n' "${0##*/}" >&2 } jobs="auto" list_only=0 +summary_path="" target_os="" requested=() while [ "$#" -gt 0 ]; do @@ -39,6 +40,14 @@ while [ "$#" -gt 0 ]; do target_os="$2" shift 2 ;; + --summary) + [ "$#" -ge 2 ] && [ -n "$2" ] || { + usage + exit 2 + } + summary_path="$2" + shift 2 + ;; --list) list_only=1 shift @@ -58,6 +67,13 @@ while [ "$#" -gt 0 ]; do esac done +# A relative --summary path names a file under the caller's directory, not +# under the repository root the runner switches to below. +case "$summary_path" in + '' | /* | [A-Za-z]:[/\\]*) ;; + *) summary_path="${PWD%/}/${summary_path}" ;; +esac + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" @@ -104,8 +120,8 @@ fi discover_suites() { local path { - git -c core.quotePath=false ls-files 'tests/*.bash' 2>/dev/null || ls tests/*.bash - } | while IFS= read -r path; do + git ls-files -z 'tests/*.bash' 2>/dev/null || printf '%s\0' tests/*.bash + } | while IFS= read -r -d '' path; do case "${path##*/}" in lib*.bash) continue ;; esac @@ -210,25 +226,136 @@ fi log_dir="$(mktemp -d)" trap 'rm -rf "$log_dir"' EXIT +# Opt-in measurements use Bash's portable elapsed-seconds clock. Whole-second +# resolution is explicit; this observes the existing waves, not a new scheduler. +summary_count=0 +summary_paths=() +summary_states=() +summary_durations=() +summary_statuses=() +summary_reasons=() +record_suite() { + [ -n "$summary_path" ] || return 0 + summary_paths[summary_count]="$1" + summary_states[summary_count]="$2" + summary_durations[summary_count]="$3" + summary_statuses[summary_count]="$4" + summary_reasons[summary_count]="$5" + summary_count=$((summary_count + 1)) +} + run_suite() { - # Writes the suite's combined output to its log and its status to a marker. - local path="$1" name status + # Writes the suite's combined output and status. No stdin: a suite must + # never consume the runner's input. Measurements never change pass criteria. + local path="$1" name status started elapsed name="$path" mkdir -p "${log_dir}/${name%/*}" status=0 - # No stdin: a suite that reads it would otherwise eat the runner's own - # input (and CI has none anyway). + started=$SECONDS bash "$path" "${log_dir}/${name}.log" 2>&1 || status=$? printf '%s\n' "$status" >"${log_dir}/${name}.status" + if [ -n "$summary_path" ]; then + # SECONDS follows the wall clock; a backward clock step is not a failure. + elapsed=$((SECONDS - started)) + [ "$elapsed" -ge 0 ] || elapsed=0 + printf '%s\n' "$elapsed" >"${log_dir}/${name}.duration" + fi } report_suite() { - local path="$1" name status + # Unreadable or non-canonical markers fail closed, but the suite's header and + # log are still printed so the failure keeps its diagnostics. + local path="$1" name status duration=null reason="" name="$path" - status="$(cat "${log_dir}/${name}.status")" || return 1 - printf '=== %s (%s) ===\n' "$path" "$([ "$status" -eq 0 ] && echo ok || echo "FAILED, status ${status}")" - cat "${log_dir}/${name}.log" || return 1 - [ "$status" -eq 0 ] + if ! status="$(cat "${log_dir}/${name}.status")"; then + status=null reason=missing-status + else + case "$status" in + '' | 0?* | *[!0-9]*) status=null reason=invalid-status ;; + esac + fi + if [ -z "$reason" ] && [ -n "$summary_path" ]; then + if ! duration="$(cat "${log_dir}/${name}.duration")"; then + duration=null reason=missing-duration + else + case "$duration" in + '' | 0?* | *[!0-9]*) duration=null reason=invalid-duration ;; + esac + fi + fi + if [ -n "$reason" ]; then + printf '=== %s (FAILED, %s) ===\n' "$path" "$reason" + else + printf '=== %s (%s) ===\n' "$path" "$([ "$status" = 0 ] && echo ok || echo "FAILED, status ${status}")" + fi + cat "${log_dir}/${name}.log" || reason="${reason:-missing-log}" + [ -n "$reason" ] || [ "$status" = 0 ] || reason=child-exit + if [ -n "$reason" ]; then + record_suite "$path" failed "$duration" "$status" "$reason" + return 1 + fi + record_suite "$path" passed "$duration" "$status" "" +} + +# JSON quoting uses Bash builtins, including ASCII control characters, so the +# summary introduces no jq/Python/runtime dependency and preserves literal names. +json_string() { + local value="$1" i char code LC_ALL=C + printf '"' + for ((i = 0; i < ${#value}; i++)); do + char="${value:i:1}" + case "$char" in + '"') printf '\\"' ;; + \\) printf '%s%s' "$char" "$char" ;; + *) + printf -v code '%d' "'$char" + if [ "$code" -lt 32 ]; then + printf '\\u%04x' "$code" + else + printf '%s' "$char" + fi + ;; + esac + done + printf '"' +} + +write_summary() { + [ -n "$summary_path" ] || return 0 + local head host temp i comma="" dirty=false + head="$(git rev-parse HEAD 2>/dev/null)" || head="" + if [ -n "$head" ] && ! git diff --quiet HEAD --; then dirty=true; fi + host="$(uname -s)" + # mv would move the report into an existing directory and report success. + [ ! -d "$summary_path" ] || return 1 + mkdir -p "$(dirname "$summary_path")" || return 1 + temp="$(mktemp "${summary_path}.XXXXXX")" || return 1 + if ! { + printf '{"schemaVersion":1,"os":' + json_string "$target_os" + printf ',"hostOs":' + json_string "$host" + printf ',"shell":' + json_string "$BASH_VERSION" + printf ',"head":' + if [ -n "$head" ]; then json_string "$head"; else printf null; fi + printf ',"dirty":%s,"jobs":%s,"durationUnit":"seconds","suites":[' "$dirty" "$jobs" + for ((i = 0; i < summary_count; i++)); do + printf '%s{"path":' "$comma" + json_string "${summary_paths[i]}" + printf ',"status":' + json_string "${summary_states[i]}" + printf ',"durationSeconds":%s,"exitStatus":%s,"reason":' "${summary_durations[i]}" "${summary_statuses[i]}" + json_string "${summary_reasons[i]}" + printf '}' + comma=, + done + printf ']}\n' + } >"$temp"; then + rm -f "$temp" + return 1 + fi + mv "$temp" "$summary_path" } to_run=() @@ -239,6 +366,7 @@ while IFS= read -r path; do if [ -n "$reason" ]; then printf '=== %s (skipped on %s: %s) ===\n' "$path" "$target_os" "$reason" skipped=$((skipped + 1)) + record_suite "$path" skipped null null "$reason" continue fi to_run=(${to_run[@]:+"${to_run[@]}"} "$path") @@ -282,8 +410,14 @@ else flush_wave fi +summary_written=1 +if ! write_summary; then + printf 'not ok - could not write test summary: %s\n' "$summary_path" >&2 + summary_written=0 +fi if [ -n "$failed" ]; then printf 'not ok - test suites failed: %s\n' "$failed" >&2 exit 1 fi +[ "$summary_written" -eq 1 ] || exit 1 printf 'ok - %s test suite(s) passed, %s skipped on %s\n' "$passed" "$skipped" "$target_os" diff --git a/tests/runner.bash b/tests/runner.bash index 8a84a52..89ad197 100644 --- a/tests/runner.bash +++ b/tests/runner.bash @@ -144,6 +144,84 @@ run_runner --list invalid assert_status 2 "$status" 'tracked suite missing on disk' "$output" pass 'runner rejects malformed metadata and missing suites instead of reporting green' +# Opt-in summaries preserve every selected result in deterministic order and +# identify the exact interpreter/source rather than claiming scheduler savings. +for mode in 1 2; do + summary="${test_root}/summary-${mode}.json" + run_runner --jobs "$mode" --summary "$summary" pass fail signal 'literal [x]' skipped + assert_status 1 "$status" "summary failure jobs=${mode}" "$output" + ruby -rjson -e ' + report = JSON.parse(File.read(ARGV[0])) + abort "missing metadata" unless report.values_at("schemaVersion", "os", "hostOs", "durationUnit") == [1, "linux", ARGV[1], "seconds"] + abort "missing shell/head metadata" unless report["shell"].is_a?(String) && report.key?("head") && report.key?("dirty") + suites = report.fetch("suites") + states = suites.to_h { |suite| [suite.fetch("path"), suite.fetch("status")] } + abort "incorrect states" unless states == {"tests/pass.bash"=>"passed", "tests/fail.bash"=>"failed", "tests/signal.bash"=>"failed", "tests/literal [x].bash"=>"passed", "tests/skipped.bash"=>"skipped"} + abort "incorrect exit status" unless suites.find { |suite| suite["path"] == "tests/fail.bash" }["exitStatus"] == 7 + abort "signal was lost" unless suites.find { |suite| suite["path"] == "tests/signal.bash" }["exitStatus"] > 128 + suites.each do |suite| + duration = suite.fetch("durationSeconds") + abort "invalid duration" unless suite["status"] == "skipped" ? duration.nil? : duration.is_a?(Integer) && duration >= 0 + end + ' "$summary" "$(uname -s)" || fail 'summary preserves metadata, statuses, signals and timing' +done +# JSON string quoting must be tested without creating filenames Windows forbids. +# Exercise the exact runner helper with an argument string, then use a portable +# filename for the duration observation. +helper="${test_root}/json-string.bash" +sed -n '/^json_string() {/,/^}/p' "${fixture}/scripts/run-tests.bash" >"$helper" +quoted='quote" and \name' +# shellcheck disable=SC2016 # The child shell expands its own arguments. +quoted_json="$("$BASH" -c '. "$1"; json_string "$2"' _ "$helper" "$quoted")" +ruby -rjson -e 'abort "JSON quoting lost" unless JSON.parse(ARGV[0]) == ARGV[1]' "$quoted_json" "$quoted" || fail 'JSON quotes and backslashes' +printf '#!/usr/bin/env bash\nsleep 1\n' >"${fixture}/tests/duration.bash" +git -C "$fixture" add tests/duration.bash +run_runner --jobs 1 --summary "${test_root}/duration.json" duration +assert_status 0 "$status" 'duration summary' "$output" +ruby -rjson -e 'suite = JSON.parse(File.read(ARGV[0])).fetch("suites").fetch(0); abort "timing lost" unless suite["path"] == "tests/duration.bash" && suite["durationSeconds"] >= 1' "${test_root}/duration.json" || fail 'portable duration observation' +for suffix in status log duration; do + GOS_TEST_RUNNER_REAL_CAT="$real_cat" GOS_TEST_RUNNER_FAIL_READ="$suffix" \ + PATH="${test_root}/tools:${PATH}" run_runner --jobs 2 --summary "${test_root}/missing-${suffix}.json" pass 'literal [x]' + assert_status 1 "$status" "summary unreadable ${suffix}" "$output" + ruby -rjson -e 'report = JSON.parse(File.read(ARGV[0])); abort "missing failures" unless report["suites"].length == 2 && report["suites"].all? { |suite| suite["status"] == "failed" && suite["reason"].start_with?("missing-") }' "${test_root}/missing-${suffix}.json" || fail 'missing measurement/status/log fails closed in summary' +done +for corrupt in corrupt 007; do + cat >"${test_root}/tools/cat" <&1)" || status=$? +assert_status 0 "$status" 'relative summary path' "$output" +[ -f "${relative_dir}/relative.json" ] || fail 'relative summary path resolves against the caller directory' +[ ! -e "${fixture}/relative.json" ] || fail 'relative summary path must not write into the repository' +pass 'optional summaries preserve serial/parallel outcomes, timings, metadata and failure evidence' + # Exported sources have no git metadata; they still discover suites on disk. exported="${test_root}/exported" mkdir -p "${exported}/scripts" "${exported}/tests" diff --git a/tests/workflows.bash b/tests/workflows.bash index b192891..5de1b4a 100644 --- a/tests/workflows.bash +++ b/tests/workflows.bash @@ -577,7 +577,7 @@ assert(psscriptanalyzer && psscriptanalyzer["if"] == "runner.os == 'Windows'" && bash32 = step_named(smoke_steps, "Bash 3.2 compatibility") assert(bash32, "smoke job must exercise the bash 3.2 floor") assert(bash32["if"] == "runner.os == 'macOS'", "bash 3.2 compatibility step must run on macOS, the only runner shipping bash 3.2") -assert(bash32["run"].to_s.include?("grep -F 'version 3.2'") && bash32["run"].to_s.include?("bash scripts/run-tests.bash") && bash32["run"].to_s.lines.any? { |line| line.strip == "bash scripts/run-tests.bash --jobs 2" }, "bash 3.2 compatibility step must verify the interpreter and run the feature suites under it") +assert(bash32["run"].to_s.include?("grep -F 'version 3.2'") && bash32["run"].to_s.include?("bash scripts/run-tests.bash") && bash32["run"].to_s.lines.any? { |line| line.strip == 'bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-bash32.json"' }, "bash 3.2 compatibility step must verify the interpreter and run the feature suites under it") assert(command_surface_sync, "smoke job must check generated command surfaces") assert(command_surface_sync["run"].to_s.include?("bash scripts/sync-command-surfaces.bash --check"), "command surface sync must use the orchestrator") @@ -601,6 +601,11 @@ assert(bash_syntax["run"].to_s.include?("git ls-files -z '*.sh' '*.bash' | xargs tracked_powershell_files.each do |path| assert(smoke_runs.include?(path), "smoke job PowerShell syntax must cover tracked PowerShell file #{path}") end +summary_upload = step_named(smoke_steps, "Upload suite observations") +assert(summary_upload && summary_upload["if"] == "always()", "suite summaries must survive failed suites") +assert(summary_upload.dig("with", "name") == "suite-observations-${{ matrix.os }}", "summary artifact names must distinguish OS jobs") +assert(summary_upload.dig("with", "path") == "${{ runner.temp }}/test-summary-*.json", "both current Bash and Bash 3.2 summaries must be retained") +assert(smoke_runs.include?('bash scripts/run-tests.bash --jobs 2 --summary "${RUNNER_TEMP}/test-summary-current.json"'), "ordinary suite runs must collect comparable observations") assert(!smoke_runs.match?(%r{bash tests/}), "smoke job must run suites through scripts/run-tests.bash, not hand-listed bash tests/ commands") assert(smoke_runs.include?("packaging/chocolatey/tools/chocolateyInstall.ps1"), "smoke job must parse the Chocolatey PowerShell installer") assert(smoke_runs.include?("packaging/chocolatey/tools/chocolateyUninstall.ps1"), "smoke job must parse the Chocolatey PowerShell uninstaller")