Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions .github/scripts/summarize-mutation-report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/usr/bin/env bash
# summarize-mutation-report.sh — turn one gremlins JSON report into a mutation-score summary.
#
# WHY (ksail#7849): agents write most of KSail's code and the tests that check it, so a green
# suite with good line coverage can still assert very little. The scheduled mutation pilot runs
# gremlins on packages where a silent regression is costly; this script publishes each package's
# score and its surviving mutants for triage.
#
# It fails closed: a missing or malformed report, or one in which gremlins examined no mutant at
# all, is an error, never a green run with an empty score.
#
# Usage: summarize-mutation-report.sh <package> <report.json>
# Prints a Markdown section (for $GITHUB_STEP_SUMMARY) on stdout.
# Exit codes: 0 summarized, 1 unusable or vacuous report, 2 usage error.

set -euo pipefail

if (($# != 2)); then
echo "usage: summarize-mutation-report.sh <package> <report.json>" >&2
exit 2
fi

package="$1"
report="$2"

if [[ ! -s "${report}" ]]; then
echo "mutation report for ${package} is missing or empty: ${report}" >&2
exit 1
fi

counts="$(jq -er '
[.mutants_killed, .mutants_lived, .mutants_not_covered, .mutants_not_viable,
(.mutants_timed_out // 0), .test_efficacy, .mutations_coverage, .elapsed_time]
| if all(type == "number") then . else error("non-numeric field") end
# Format here, not with printf, so the decimal separator never follows the runner locale.
| .[0:5] + [(.[5] * 100 | round / 100), (.[6] * 100 | round / 100), (.[7] | round)]
| map(tostring) | join(" ")
' "${report}" 2>/dev/null)" || {
echo "mutation report for ${package} is not a gremlins JSON report: ${report}" >&2
exit 1
}

read -r killed lived not_covered not_viable timed_out efficacy coverage elapsed <<<"${counts}"

examined=$((killed + lived + timed_out))
if ((examined == 0)); then
echo "gremlins examined no mutants in ${package}; refusing a vacuous score" >&2
exit 1
fi

# shellcheck disable=SC2016 # the backticks are Markdown, not a command substitution
printf '### Mutation score: `%s`\n\n' "${package}"
printf '| Killed | Lived | Timed out | Not covered | Not viable | Test efficacy | Mutant coverage | Elapsed |\n'
printf '|---:|---:|---:|---:|---:|---:|---:|---:|\n'
printf '| %s | %s | %s | %s | %s | %s%% | %s%% | %ss |\n\n' \
"${killed}" "${lived}" "${timed_out}" "${not_covered}" "${not_viable}" \
"${efficacy}" "${coverage}" "${elapsed}"

if ((lived == 0)); then
printf 'No mutant survived.\n'
exit 0
fi

printf '<details><summary>%s surviving mutants to triage</summary>\n\n' "${lived}"
printf '| File | Line | Mutation |\n|---|---:|---|\n'
jq -r '
.files[]? | .file_name as $file | .mutations[]? | select(.status == "LIVED")
| "| `\($file)` | \(.line) | \(.type) |"
' "${report}"
printf '\n</details>\n'
73 changes: 73 additions & 0 deletions .github/scripts/summarize-mutation-report.test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env bash
# Tests for summarize-mutation-report.sh (ksail#7849): a real score is summarized with its
# survivors, and every unusable or vacuous report fails instead of printing a green score.

set -euo pipefail

script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
summarize="${script_dir}/summarize-mutation-report.sh"
work="$(mktemp -d)"
trap 'rm -rf -- "${work}"' EXIT

failures=0

fail() {
echo "FAIL: $*" >&2
failures=$((failures + 1))
}

expect_status() {
local want="$1" name="$2"
shift 2

local status=0
"$@" >"${work}/out" 2>"${work}/err" || status=$?
if ((status != want)); then
fail "${name}: exit ${status}, want ${want} ($(cat "${work}/err"))"
fi
}

cat >"${work}/scored.json" <<'JSON'
{"go_module":"github.com/devantler-tech/ksail/v7","test_efficacy":75.0,"mutations_coverage":80.0,
"mutants_total":5,"mutants_killed":3,"mutants_lived":1,"mutants_not_viable":0,"mutants_not_covered":1,
"elapsed_time":12.3,"files":[{"file_name":"pkg/cli/lifecycle/simple.go","mutations":[
{"line":10,"column":2,"type":"CONDITIONALS_NEGATION","status":"KILLED"},
{"line":42,"column":7,"type":"CONDITIONALS_BOUNDARY","status":"LIVED"}]}]}
JSON

expect_status 0 "scored report" "${summarize}" pkg/cli/lifecycle "${work}/scored.json"
grep -q '| 3 | 1 | 0 | 1 | 0 | 75% | 80% | 12s |' "${work}/out" ||
fail "scored report: counts row missing: $(cat "${work}/out")"
# shellcheck disable=SC2016 # the backticks are Markdown
grep -q '| `pkg/cli/lifecycle/simple.go` | 42 | CONDITIONALS_BOUNDARY |' "${work}/out" ||
fail "scored report: surviving mutant not listed"
if grep -q '| 10 |' "${work}/out"; then
fail "scored report: a killed mutant was listed as a survivor"
fi

jq '.mutants_lived = 0 | .files[0].mutations |= map(select(.status != "LIVED"))' \
"${work}/scored.json" >"${work}/clean.json"
expect_status 0 "no survivors" "${summarize}" pkg/cli/lifecycle "${work}/clean.json"
grep -q 'No mutant survived.' "${work}/out" || fail "no survivors: message missing"

jq '.mutants_killed = 0 | .mutants_lived = 0 | .mutants_not_covered = 5' \
"${work}/scored.json" >"${work}/vacuous.json"
expect_status 1 "vacuous report" "${summarize}" pkg/cli/lifecycle "${work}/vacuous.json"
grep -q 'examined no mutants' "${work}/err" || fail "vacuous report: wrong reason"

: >"${work}/empty.json"
expect_status 1 "empty report" "${summarize}" pkg/cli/lifecycle "${work}/empty.json"
expect_status 1 "missing report" "${summarize}" pkg/cli/lifecycle "${work}/absent.json"

echo '{"mutants_killed":"3"}' >"${work}/malformed.json"
expect_status 1 "malformed report" "${summarize}" pkg/cli/lifecycle "${work}/malformed.json"
grep -q 'not a gremlins JSON report' "${work}/err" || fail "malformed report: wrong reason"

expect_status 2 "usage" "${summarize}" only-one-argument

if ((failures > 0)); then
echo "${failures} check(s) failed" >&2
exit 1
fi

echo "summarize-mutation-report: all checks passed"
92 changes: 92 additions & 0 deletions .github/workflows/mutation-testing.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
name: Mutation Testing

# A scheduled drift sensor for test quality, not a pull-request gate (ksail#7849). Agents write
# most of KSail's code and the tests that check it, so line coverage cannot tell a test that pins
# behaviour from one that only executes it. Gremlins mutates packages where a silent regression
# is costly (cluster create/delete orchestration and the EKS cleanup verifier) and reports how many
# mutants their tests kill. Surviving mutants that point at a missing assertion are triaged into
# issues. A PR-time gate is reconsidered only once the scores and run times are known.
on:
schedule:
- cron: "23 4 * * 1"
workflow_dispatch:
# Self-test: a change to the sensor runs it once on the pull request that makes it.
pull_request:
paths:
- .github/workflows/mutation-testing.yaml
- .github/scripts/summarize-mutation-report*

permissions: {}

concurrency:
group: mutation-testing-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

env:
# Exact pin: go-gremlins/gremlins, Apache-2.0. Bump deliberately and compare the first scores.
GREMLINS_VERSION: v0.6.0

jobs:
mutate:
name: 🧬 Mutate ${{ matrix.package }}
runs-on: ubuntu-latest
timeout-minutes: 90
permissions:
contents: read
strategy:
fail-fast: false
matrix:
package:
- pkg/cli/lifecycle
- internal/maintenance/eksinventory
steps:
- name: 📄 Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: ⚙️ Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
cache: false

- name: ♻️ Restore Go modules
uses: devantler-tech/ksail/.github/actions/restore-go-modules@46fdd72dff3ccc24a494508fde43426c7e65a0e3

- name: 🧪 Test the report summarizer
run: bash .github/scripts/summarize-mutation-report.test.sh

- name: 📥 Install gremlins
run: go install "github.com/go-gremlins/gremlins/cmd/gremlins@${GREMLINS_VERSION}"

- name: 🧬 Run mutation testing
id: mutate
timeout-minutes: 75
env:
PACKAGE: ${{ matrix.package }}
run: |
set -euo pipefail
started=$(date +%s)
# Two workers leave headroom on a standard runner; each mutant runs the package's tests.
gremlins unleash --workers 2 --output "$RUNNER_TEMP/mutation-report.json" "./$PACKAGE"
echo "elapsed-seconds=$(( $(date +%s) - started ))" >> "$GITHUB_OUTPUT"

- name: 📊 Publish the mutation score
env:
PACKAGE: ${{ matrix.package }}
ELAPSED_SECONDS: ${{ steps.mutate.outputs.elapsed-seconds }}
run: |
set -euo pipefail
bash .github/scripts/summarize-mutation-report.sh "$PACKAGE" \
"$RUNNER_TEMP/mutation-report.json" >> "$GITHUB_STEP_SUMMARY"
printf '\nWall-clock time of the mutation step: %ss.\n' "$ELAPSED_SECONDS" >> "$GITHUB_STEP_SUMMARY"
Comment thread
devantler marked this conversation as resolved.
Resolved

- name: 📤 Keep the mutation report
if: always() && steps.mutate.outcome != 'skipped'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: mutation-report-${{ strategy.job-index }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/mutation-report.json
if-no-files-found: warn
retention-days: 30
Loading