From 30ee806384be6062554c9827827ac1e3077163c5 Mon Sep 17 00:00:00 2001 From: Balogun Feranmi Date: Tue, 8 Sep 2026 10:49:27 +0100 Subject: [PATCH 1/2] test(e2e): cover fail2ban action family --- CONTRIBUTING.md | 4 ++-- docs/introduction.md | 2 +- scripts/check_evidence_claims.py | 2 +- tests/e2e/stories/story-134.sh | 19 +++++++++++++++++++ tests/e2e/stories/story-135.sh | 21 +++++++++++++++++++++ tests/e2e/stories/story-136.sh | 21 +++++++++++++++++++++ tests/e2e/stories/story-137.sh | 22 ++++++++++++++++++++++ tests/release/public-claims.test.sh | 2 +- 8 files changed, 88 insertions(+), 5 deletions(-) create mode 100755 tests/e2e/stories/story-134.sh create mode 100755 tests/e2e/stories/story-135.sh create mode 100755 tests/e2e/stories/story-136.sh create mode 100755 tests/e2e/stories/story-137.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65fac168..36810dcc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,10 +44,10 @@ table below. | Area | Why it matters | Difficulty | |---|---|---| -| **Ubuntu LTS support** | All three LTS releases are validated against the full story suite on a live VM, each with a committed replay twin that reproduces it: 22.04, 24.04 and 26.04 all at 79/79. `ubuntu-vm.sh` accepts `UBUNTU_RELEASE=jammy\|noble\|resolute`. Remaining: story coverage for the cross-family actions, and five Debian-only ones that still have none: the four fail2ban actions and `GrubSetKargs`. | medium | +| **Ubuntu LTS support** | All three LTS releases are validated against the full story suite on a live VM, each with a committed replay twin that reproduces it: 22.04, 24.04 and 26.04 all at 79/79. `ubuntu-vm.sh` accepts `UBUNTU_RELEASE=jammy\|noble\|resolute`. Remaining: story coverage for the cross-family actions, and one Debian-only action still has no story: `GrubSetKargs`. | medium | | **Distro detection coverage** | Robust `/etc/os-release` parsing for every release we claim to support. Pure-function tests against real fixture files, no integration mocks. The existing fixtures at the bottom of `crates/sysknife-core/src/distro.rs` show the shape. | easy | | **Action catalogue gaps** | Add a typed action (for example `EnableFirewallZone`). Small and isolated, and every PR carries the policy entry, the risk level and the tests. | easy | -| **E2E story coverage** | Real prompts, real LLM, real daemon. The suite is 133 stories: 54 atomic + 79 Ubuntu. What is left is the cross-family middle: of the action names available on both families, 59 are still untouched by any story, plus 10 Fedora-only and 5 Ubuntu-only ones. See #233 for the clustered map. | medium | +| **E2E story coverage** | Real prompts, real LLM, real daemon. The suite is 137 stories: 54 atomic + 83 Ubuntu. What is left is the cross-family middle: of the action names available on both families, 59 are still untouched by any story, plus 10 Fedora-only and 1 Ubuntu-only ones. See #233 for the clustered map. | medium | | **Fedora Atomic validation** | The action families exist and `DistroId::is_supported()` returns true for Atomic 41 and up. Nobody has run `tests/e2e/atomic-vm.sh` against a current release. Needs Fedora Atomic hardware or a VM host. | tedious | | **Demo recording on real hardware** | Replace the bundled demo GIF with a 30-second recording on real Ubuntu 26.04 with rollback visible. | easy | diff --git a/docs/introduction.md b/docs/introduction.md index f26a54d7..aec7668e 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -107,7 +107,7 @@ SysKnife, not an afterthought. > All three Ubuntu LTS releases have a committed live-VM run of the 79-story > Ubuntu suite, in `tests/evidence/story-runs/`: 22.04, 24.04 and 26.04 all at > 79/79. Each run has a replay twin that reproduces it, serving every call with -> zero misses. Five Debian-only actions still have no story. +> zero misses. One Debian-only action still has no story. > Fedora Atomic is supported by the rpm-ostree action family, but a current > Silverblue 44 VM run is a release gate. Plain Fedora remains experimental > until the `dnf` action family ships. diff --git a/scripts/check_evidence_claims.py b/scripts/check_evidence_claims.py index 2b2126e5..8bf99e9d 100755 --- a/scripts/check_evidence_claims.py +++ b/scripts/check_evidence_claims.py @@ -433,7 +433,7 @@ def _claim_count(value: str) -> int: DEBIAN_GAP_PROSE = ( r"\b(?P\d+|zero|one|two|three|four|five|six|seven|eight|" r"nine|ten)\s+Debian-only\s+(?:actions?|ones?)\s+" - r"(?:that\s+)?still\s+have\s+no(?:ne|\s+story)\b" + r"(?:that\s+)?still\s+(?:have|has)\s+no(?:ne|\s+story)\b" ) DEBIAN_EVERY_CLAIM = re.compile( r"\bEvery\s+Debian-only\s+action\s+(?:now\s+)?has\s+(?:one|a\s+story)\b", diff --git a/tests/e2e/stories/story-134.sh b/tests/e2e/stories/story-134.sh new file mode 100755 index 00000000..2e1c138d --- /dev/null +++ b/tests/e2e/stories/story-134.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Story 134 (ubuntu, read-only): Active fail2ban jail status +# Intent: "show me which fail2ban jails are active" +# Distro: ubuntu +set -euo pipefail +INTENT="show me which fail2ban jails are active" +echo "=== Story 134 (ubuntu): Fail2banStatus ===" +PLAN=$(sysknife --dry-run --json "$INTENT" 2>/tmp/sysknife-story-134-stderr.log) +echo "$PLAN" | jq . + +STEP_COUNT=$(echo "$PLAN" | jq '.plan.steps | length') +if [[ "$STEP_COUNT" != "1" ]]; then echo "FAIL: expected 1 step, got $STEP_COUNT"; exit 1; fi +STEP=$(echo "$PLAN" | jq '.plan.steps[0] | select(.action == "Fail2banStatus")') +if [[ -z "$STEP" || "$STEP" == "null" ]]; then echo "FAIL: expected Fail2banStatus"; exit 1; fi +RISK=$(echo "$STEP" | jq -r '.risk') +if [[ "$RISK" != "low" ]]; then echo "FAIL: expected risk low, got $RISK"; exit 1; fi +PARAMS=$(echo "$STEP" | jq -c '.params') +if [[ "$PARAMS" != "{}" ]]; then echo "FAIL: expected no jail parameter, got $PARAMS"; exit 1; fi +echo "PASS: Story 134" diff --git a/tests/e2e/stories/story-135.sh b/tests/e2e/stories/story-135.sh new file mode 100755 index 00000000..1e0219be --- /dev/null +++ b/tests/e2e/stories/story-135.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Story 135 (ubuntu, medium-risk): Unban a documented IP from sshd +# Intent: "unban 203.0.113.7 from the sshd fail2ban jail" +# Distro: ubuntu +set -euo pipefail +INTENT="unban 203.0.113.7 from the sshd fail2ban jail" +echo "=== Story 135 (ubuntu): Fail2banUnbanIp ===" +PLAN=$(sysknife --dry-run --json "$INTENT" 2>/tmp/sysknife-story-135-stderr.log) +echo "$PLAN" | jq . + +STEP_COUNT=$(echo "$PLAN" | jq '.plan.steps | length') +if [[ "$STEP_COUNT" != "1" ]]; then echo "FAIL: expected 1 step, got $STEP_COUNT"; exit 1; fi +STEP=$(echo "$PLAN" | jq '.plan.steps[0] | select(.action == "Fail2banUnbanIp")') +if [[ -z "$STEP" || "$STEP" == "null" ]]; then echo "FAIL: expected Fail2banUnbanIp"; exit 1; fi +RISK=$(echo "$STEP" | jq -r '.risk') +if [[ "$RISK" != "medium" ]]; then echo "FAIL: expected risk medium, got $RISK"; exit 1; fi +JAIL=$(echo "$STEP" | jq -r '.params.jail // ""') +if [[ "$JAIL" != "sshd" ]]; then echo "FAIL: expected jail=sshd, got $JAIL"; exit 1; fi +IP=$(echo "$STEP" | jq -r '.params.ip // ""') +if [[ "$IP" != "203.0.113.7" ]]; then echo "FAIL: expected ip=203.0.113.7, got $IP"; exit 1; fi +echo "PASS: Story 135" diff --git a/tests/e2e/stories/story-136.sh b/tests/e2e/stories/story-136.sh new file mode 100755 index 00000000..6870c903 --- /dev/null +++ b/tests/e2e/stories/story-136.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Story 136 (ubuntu, high-risk): Ban a documented IP in sshd +# Intent: "ban 203.0.113.7 in the sshd fail2ban jail" +# Distro: ubuntu +set -euo pipefail +INTENT="ban 203.0.113.7 in the sshd fail2ban jail" +echo "=== Story 136 (ubuntu): Fail2banBanIp ===" +PLAN=$(sysknife --dry-run --json "$INTENT" 2>/tmp/sysknife-story-136-stderr.log) +echo "$PLAN" | jq . + +STEP_COUNT=$(echo "$PLAN" | jq '.plan.steps | length') +if [[ "$STEP_COUNT" != "1" ]]; then echo "FAIL: expected 1 step, got $STEP_COUNT"; exit 1; fi +STEP=$(echo "$PLAN" | jq '.plan.steps[0] | select(.action == "Fail2banBanIp")') +if [[ -z "$STEP" || "$STEP" == "null" ]]; then echo "FAIL: expected Fail2banBanIp"; exit 1; fi +RISK=$(echo "$STEP" | jq -r '.risk') +if [[ "$RISK" != "high" ]]; then echo "FAIL: expected risk high, got $RISK"; exit 1; fi +JAIL=$(echo "$STEP" | jq -r '.params.jail // ""') +if [[ "$JAIL" != "sshd" ]]; then echo "FAIL: expected jail=sshd, got $JAIL"; exit 1; fi +IP=$(echo "$STEP" | jq -r '.params.ip // ""') +if [[ "$IP" != "203.0.113.7" ]]; then echo "FAIL: expected ip=203.0.113.7, got $IP"; exit 1; fi +echo "PASS: Story 136" diff --git a/tests/e2e/stories/story-137.sh b/tests/e2e/stories/story-137.sh new file mode 100755 index 00000000..d935a7f5 --- /dev/null +++ b/tests/e2e/stories/story-137.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Story 137 (ubuntu, high-risk): Enable the sshd fail2ban jail +# Intent: "enable the sshd jail in fail2ban" +# Distro: ubuntu +set -euo pipefail +INTENT="enable the sshd jail in fail2ban" +echo "=== Story 137 (ubuntu): ConfigureFail2banJail ===" +PLAN=$(sysknife --dry-run --json "$INTENT" 2>/tmp/sysknife-story-137-stderr.log) +echo "$PLAN" | jq . + +STEP_COUNT=$(echo "$PLAN" | jq '.plan.steps | length') +if [[ "$STEP_COUNT" != "1" ]]; then echo "FAIL: expected 1 step, got $STEP_COUNT"; exit 1; fi +STEP=$(echo "$PLAN" | jq '.plan.steps[0] | select(.action == "ConfigureFail2banJail")') +if [[ -z "$STEP" || "$STEP" == "null" ]]; then echo "FAIL: expected ConfigureFail2banJail"; exit 1; fi +RISK=$(echo "$STEP" | jq -r '.risk') +if [[ "$RISK" != "high" ]]; then echo "FAIL: expected risk high, got $RISK"; exit 1; fi +NAME=$(echo "$STEP" | jq -r '.params.name // ""') +if [[ "$NAME" != "sshd" ]]; then echo "FAIL: expected name=sshd, got $NAME"; exit 1; fi +if ! echo "$STEP" | jq -e '.params.enabled == true' >/dev/null; then + echo "FAIL: expected enabled=true, got $(echo "$STEP" | jq -c '.params.enabled // null')"; exit 1 +fi +echo "PASS: Story 137" diff --git a/tests/release/public-claims.test.sh b/tests/release/public-claims.test.sh index 479b6a22..f9eb5359 100755 --- a/tests/release/public-claims.test.sh +++ b/tests/release/public-claims.test.sh @@ -398,7 +398,7 @@ if [[ -z "${intro_published:-}" || -z "${intro_new:-}" ]]; then printf 'FAIL: introduction-count mutation produced no values\n' >&2 exit 1 fi -if ! grep -Eq "$intro_new Debian-only actions still have no story" \ +if ! grep -Eq "$intro_new Debian-only (actions still have|action still has) no story" \ "$fixture/docs/introduction.md"; then printf 'FAIL: introduction-count mutation did not apply\n' >&2 exit 1 From d580f2eaa9df6cf16fb1a30e0a80fe22569e3981 Mon Sep 17 00:00:00 2001 From: Balogun Feranmi Date: Wed, 9 Sep 2026 21:20:18 +0100 Subject: [PATCH 2/2] docs: clarify Ubuntu story evidence scope --- CONTRIBUTING.md | 2 +- docs/introduction.md | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 36810dcc..620497a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ table below. | Area | Why it matters | Difficulty | |---|---|---| -| **Ubuntu LTS support** | All three LTS releases are validated against the full story suite on a live VM, each with a committed replay twin that reproduces it: 22.04, 24.04 and 26.04 all at 79/79. `ubuntu-vm.sh` accepts `UBUNTU_RELEASE=jammy\|noble\|resolute`. Remaining: story coverage for the cross-family actions, and one Debian-only action still has no story: `GrubSetKargs`. | medium | +| **Ubuntu LTS support** | The current suite is 83 Ubuntu stories. Committed live-VM evidence covers 79 of those Ubuntu stories on each LTS release, with a committed replay twin that reproduces each run: 22.04, 24.04 and 26.04 all at 79/79. The four additional stories are not yet included in a committed live-VM run. `ubuntu-vm.sh` accepts `UBUNTU_RELEASE=jammy\|noble\|resolute`. Remaining: story coverage for the cross-family actions, and one Debian-only action still has no story: `GrubSetKargs`. | medium | | **Distro detection coverage** | Robust `/etc/os-release` parsing for every release we claim to support. Pure-function tests against real fixture files, no integration mocks. The existing fixtures at the bottom of `crates/sysknife-core/src/distro.rs` show the shape. | easy | | **Action catalogue gaps** | Add a typed action (for example `EnableFirewallZone`). Small and isolated, and every PR carries the policy entry, the risk level and the tests. | easy | | **E2E story coverage** | Real prompts, real LLM, real daemon. The suite is 137 stories: 54 atomic + 83 Ubuntu. What is left is the cross-family middle: of the action names available on both families, 59 are still untouched by any story, plus 10 Fedora-only and 1 Ubuntu-only ones. See #233 for the clustered map. | medium | diff --git a/docs/introduction.md b/docs/introduction.md index aec7668e..ef6b7950 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -104,10 +104,13 @@ SysKnife, not an afterthought. > **ℹ️ Distro support** > -> All three Ubuntu LTS releases have a committed live-VM run of the 79-story -> Ubuntu suite, in `tests/evidence/story-runs/`: 22.04, 24.04 and 26.04 all at -> 79/79. Each run has a replay twin that reproduces it, serving every call with -> zero misses. One Debian-only action still has no story. +> The current suite is 83 Ubuntu stories. All three Ubuntu LTS releases have a +> committed live-VM run covering 79 of those Ubuntu stories, in +> `tests/evidence/story-runs/`: +> 22.04, 24.04 and 26.04 all at 79/79. The four additional stories are not yet +> included in a committed live-VM run. Each run has a replay twin that +> reproduces it, serving every call with zero misses. +> One Debian-only action still has no story. > Fedora Atomic is supported by the rpm-ostree action family, but a current > Silverblue 44 VM run is a release gate. Plain Fedora remains experimental > until the `dnf` action family ships.