diff --git a/.github/scripts/summarize-mutation-report.sh b/.github/scripts/summarize-mutation-report.sh new file mode 100755 index 0000000000..2c13c47bef --- /dev/null +++ b/.github/scripts/summarize-mutation-report.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# summarize-mutation-report.sh โ€” turn one gremlins JSON report into a mutation-score summary. +# +# WHY (ksail#7849): agents write most of KSail's code and the tests that check it, so a green +# suite with good line coverage can still assert very little. The scheduled mutation pilot runs +# gremlins on packages where a silent regression is costly; this script publishes each package's +# score and its surviving mutants for triage. +# +# It fails closed: a missing or malformed report, or one in which gremlins examined no mutant at +# all, is an error, never a green run with an empty score. +# +# Usage: summarize-mutation-report.sh +# Prints a Markdown section (for $GITHUB_STEP_SUMMARY) on stdout. +# Exit codes: 0 summarized, 1 unusable or vacuous report, 2 usage error. + +set -euo pipefail + +if (($# != 2)); then + echo "usage: summarize-mutation-report.sh " >&2 + exit 2 +fi + +package="$1" +report="$2" + +if [[ ! -s "${report}" ]]; then + echo "mutation report for ${package} is missing or empty: ${report}" >&2 + exit 1 +fi + +counts="$(jq -er ' + [.mutants_killed, .mutants_lived, .mutants_not_covered, .mutants_not_viable, + (.mutants_timed_out // 0), .test_efficacy, .mutations_coverage, .elapsed_time] + | if all(type == "number") then . else error("non-numeric field") end + # Format here, not with printf, so the decimal separator never follows the runner locale. + | .[0:5] + [(.[5] * 100 | round / 100), (.[6] * 100 | round / 100), (.[7] | round)] + | map(tostring) | join(" ") +' "${report}" 2>/dev/null)" || { + echo "mutation report for ${package} is not a gremlins JSON report: ${report}" >&2 + exit 1 +} + +read -r killed lived not_covered not_viable timed_out efficacy coverage elapsed <<<"${counts}" + +examined=$((killed + lived + timed_out)) +if ((examined == 0)); then + echo "gremlins examined no mutants in ${package}; refusing a vacuous score" >&2 + exit 1 +fi + +# shellcheck disable=SC2016 # the backticks are Markdown, not a command substitution +printf '### Mutation score: `%s`\n\n' "${package}" +printf '| Killed | Lived | Timed out | Not covered | Not viable | Test efficacy | Mutant coverage | Elapsed |\n' +printf '|---:|---:|---:|---:|---:|---:|---:|---:|\n' +printf '| %s | %s | %s | %s | %s | %s%% | %s%% | %ss |\n\n' \ + "${killed}" "${lived}" "${timed_out}" "${not_covered}" "${not_viable}" \ + "${efficacy}" "${coverage}" "${elapsed}" + +if ((lived == 0)); then + printf 'No mutant survived.\n' + exit 0 +fi + +printf '
%s surviving mutants to triage\n\n' "${lived}" +printf '| File | Line | Mutation |\n|---|---:|---|\n' +jq -r ' + .files[]? | .file_name as $file | .mutations[]? | select(.status == "LIVED") + | "| `\($file)` | \(.line) | \(.type) |" +' "${report}" +printf '\n
\n' diff --git a/.github/scripts/summarize-mutation-report.test.sh b/.github/scripts/summarize-mutation-report.test.sh new file mode 100755 index 0000000000..f5b284d600 --- /dev/null +++ b/.github/scripts/summarize-mutation-report.test.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Tests for summarize-mutation-report.sh (ksail#7849): a real score is summarized with its +# survivors, and every unusable or vacuous report fails instead of printing a green score. + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +summarize="${script_dir}/summarize-mutation-report.sh" +work="$(mktemp -d)" +trap 'rm -rf -- "${work}"' EXIT + +failures=0 + +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +expect_status() { + local want="$1" name="$2" + shift 2 + + local status=0 + "$@" >"${work}/out" 2>"${work}/err" || status=$? + if ((status != want)); then + fail "${name}: exit ${status}, want ${want} ($(cat "${work}/err"))" + fi +} + +cat >"${work}/scored.json" <<'JSON' +{"go_module":"github.com/devantler-tech/ksail/v7","test_efficacy":75.0,"mutations_coverage":80.0, + "mutants_total":5,"mutants_killed":3,"mutants_lived":1,"mutants_not_viable":0,"mutants_not_covered":1, + "elapsed_time":12.3,"files":[{"file_name":"pkg/cli/lifecycle/simple.go","mutations":[ + {"line":10,"column":2,"type":"CONDITIONALS_NEGATION","status":"KILLED"}, + {"line":42,"column":7,"type":"CONDITIONALS_BOUNDARY","status":"LIVED"}]}]} +JSON + +expect_status 0 "scored report" "${summarize}" pkg/cli/lifecycle "${work}/scored.json" +grep -q '| 3 | 1 | 0 | 1 | 0 | 75% | 80% | 12s |' "${work}/out" || + fail "scored report: counts row missing: $(cat "${work}/out")" +# shellcheck disable=SC2016 # the backticks are Markdown +grep -q '| `pkg/cli/lifecycle/simple.go` | 42 | CONDITIONALS_BOUNDARY |' "${work}/out" || + fail "scored report: surviving mutant not listed" +if grep -q '| 10 |' "${work}/out"; then + fail "scored report: a killed mutant was listed as a survivor" +fi + +jq '.mutants_lived = 0 | .files[0].mutations |= map(select(.status != "LIVED"))' \ + "${work}/scored.json" >"${work}/clean.json" +expect_status 0 "no survivors" "${summarize}" pkg/cli/lifecycle "${work}/clean.json" +grep -q 'No mutant survived.' "${work}/out" || fail "no survivors: message missing" + +jq '.mutants_killed = 0 | .mutants_lived = 0 | .mutants_not_covered = 5' \ + "${work}/scored.json" >"${work}/vacuous.json" +expect_status 1 "vacuous report" "${summarize}" pkg/cli/lifecycle "${work}/vacuous.json" +grep -q 'examined no mutants' "${work}/err" || fail "vacuous report: wrong reason" + +: >"${work}/empty.json" +expect_status 1 "empty report" "${summarize}" pkg/cli/lifecycle "${work}/empty.json" +expect_status 1 "missing report" "${summarize}" pkg/cli/lifecycle "${work}/absent.json" + +echo '{"mutants_killed":"3"}' >"${work}/malformed.json" +expect_status 1 "malformed report" "${summarize}" pkg/cli/lifecycle "${work}/malformed.json" +grep -q 'not a gremlins JSON report' "${work}/err" || fail "malformed report: wrong reason" + +expect_status 2 "usage" "${summarize}" only-one-argument + +if ((failures > 0)); then + echo "${failures} check(s) failed" >&2 + exit 1 +fi + +echo "summarize-mutation-report: all checks passed" diff --git a/.github/workflows/mutation-testing.yaml b/.github/workflows/mutation-testing.yaml new file mode 100644 index 0000000000..ad7393744c --- /dev/null +++ b/.github/workflows/mutation-testing.yaml @@ -0,0 +1,92 @@ +name: Mutation Testing + +# A scheduled drift sensor for test quality, not a pull-request gate (ksail#7849). Agents write +# most of KSail's code and the tests that check it, so line coverage cannot tell a test that pins +# behaviour from one that only executes it. Gremlins mutates packages where a silent regression +# is costly (cluster create/delete orchestration and the EKS cleanup verifier) and reports how many +# mutants their tests kill. Surviving mutants that point at a missing assertion are triaged into +# issues. A PR-time gate is reconsidered only once the scores and run times are known. +on: + schedule: + - cron: "23 4 * * 1" + workflow_dispatch: + # Self-test: a change to the sensor runs it once on the pull request that makes it. + pull_request: + paths: + - .github/workflows/mutation-testing.yaml + - .github/scripts/summarize-mutation-report* + +permissions: {} + +concurrency: + group: mutation-testing-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + # Exact pin: go-gremlins/gremlins, Apache-2.0. Bump deliberately and compare the first scores. + GREMLINS_VERSION: v0.6.0 + +jobs: + mutate: + name: ๐Ÿงฌ Mutate ${{ matrix.package }} + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + package: + - pkg/cli/lifecycle + - internal/maintenance/eksinventory + steps: + - name: ๐Ÿ“„ Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: โš™๏ธ Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: โ™ป๏ธ Restore Go modules + uses: devantler-tech/ksail/.github/actions/restore-go-modules@46fdd72dff3ccc24a494508fde43426c7e65a0e3 + + - name: ๐Ÿงช Test the report summarizer + run: bash .github/scripts/summarize-mutation-report.test.sh + + - name: ๐Ÿ“ฅ Install gremlins + run: go install "github.com/go-gremlins/gremlins/cmd/gremlins@${GREMLINS_VERSION}" + + - name: ๐Ÿงฌ Run mutation testing + id: mutate + timeout-minutes: 75 + env: + PACKAGE: ${{ matrix.package }} + run: | + set -euo pipefail + started=$(date +%s) + # Two workers leave headroom on a standard runner; each mutant runs the package's tests. + gremlins unleash --workers 2 --output "$RUNNER_TEMP/mutation-report.json" "./$PACKAGE" + echo "elapsed-seconds=$(( $(date +%s) - started ))" >> "$GITHUB_OUTPUT" + + - name: ๐Ÿ“Š Publish the mutation score + env: + PACKAGE: ${{ matrix.package }} + ELAPSED_SECONDS: ${{ steps.mutate.outputs.elapsed-seconds }} + run: | + set -euo pipefail + bash .github/scripts/summarize-mutation-report.sh "$PACKAGE" \ + "$RUNNER_TEMP/mutation-report.json" >> "$GITHUB_STEP_SUMMARY" + printf '\nWall-clock time of the mutation step: %ss.\n' "$ELAPSED_SECONDS" >> "$GITHUB_STEP_SUMMARY" + + - name: ๐Ÿ“ค Keep the mutation report + if: always() && steps.mutate.outcome != 'skipped' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mutation-report-${{ strategy.job-index }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/mutation-report.json + if-no-files-found: warn + retention-days: 30