diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 28f188e..5835592 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "sage-instructor", - "version": "1.2.0", + "version": "1.2.1", "description": "Adaptive programming instructor — structured courses with discovery-first teaching, AskUserQuestion interactions, progress tracking, and pluggable curricula. Powered by the Three Axes Framework.", "author": { "name": "Lux Solari", diff --git a/.github/workflows/tier1-checks.yml b/.github/workflows/tier1-checks.yml new file mode 100644 index 0000000..e100b97 --- /dev/null +++ b/.github/workflows/tier1-checks.yml @@ -0,0 +1,24 @@ +name: Tier 1 regression checks + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + check-progress-schema: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Compile check + run: python3 -m py_compile tests/check_progress_schema.py tests/test_check_progress_schema.py + - name: Run checker unit tests against tests/fixtures/ + run: python3 tests/test_check_progress_schema.py -v + - name: Validate plugin structure + run: | + test -f .claude-plugin/plugin.json + python3 -c "import json; json.load(open('.claude-plugin/plugin.json'))" diff --git a/.gitignore b/.gitignore index fb8f331..2b849ad 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ CLAUDE.local.md # Never commit someone's actual progress/profile into the plugin repo. .sage-progress.json .sage-profile.md +# ...except the checked-in, synthetic fixtures the Tier 1 harness tests +# against (tests/fixtures/*/.sage-progress.json, .sage-profile.md) — those +# are committed on purpose, no real learner data. +!tests/fixtures/**/.sage-progress.json +!tests/fixtures/**/.sage-profile.md diff --git a/CHANGELOG.md b/CHANGELOG.md index afa0c2a..91fd348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,103 @@ # Changelog +## [1.2.1] — 2026-07-02 + +### Added +- **`tests/` regression harness.** Closes the gap where every fix in 1.2.0 + was validated by a one-off simulated session with no way to re-check it + after a future edit. Two tiers: + - `tests/check_progress_schema.py` — deterministic, no-LLM structural + checker for `.sage-progress.json`/`.sage-profile.md` against the Progress + Rules in `SKILL.md` (kebab-case topic keys, mutually-exclusive hint + streaks, `review_due` staying a subset of `topic_confidence`, exercise + slug format, profile file location). Fast enough for CI on every push. + - `tests/scenarios/*.md` — fixed, repeatable session scripts (not + open-ended exploration), one per bug class found and fixed in 1.2.0: + onboarding/profile location, topic-key derivation and reuse, toolchain + failure vs. learner bug, hint-streak scoping across a phase boundary + plus decline-resets-streak, and last-exercise track completion. + `tests/run_scenario_prompt.md` is the reusable agent prompt that plays + both Sage and the scripted learner, executes real commands, and grades + against each scenario's assertion checklist. See `tests/README.md`. +- **`tests/test_check_progress_schema.py` + `tests/fixtures/`.** The + checker itself is plain deterministic code, so it now has unit tests + against seven fixtures (a valid progress file, a valid track-completion + state, and five invalid variants each isolating one violation) instead of + relying on an expensive Tier 2 run to notice a bug in it — which is + exactly how the `next_up`/`warn()` bug below shipped in the first place. + Verified the new test would have caught that exact bug by reverting the + fix and confirming it fails. +- **`.github/workflows/tier1-checks.yml`.** Runs the Tier 1 checker's unit + tests and a plugin-manifest sanity check on every push/PR to `main` — Tier + 1 is now an actual automated gate, not a script someone has to remember to + run. +- **Pre-release checklist in `tests/README.md`/`CONTRIBUTING.md`.** Bumping + the plugin version now requires running all five Tier 2 scenarios (not + just the ones nearest the change) and recording the result in the + CHANGELOG entry, so "the harness was run" is checkable later. + +### Fixed +- **Found by actually running the harness against the live spec (scenarios + 02, 03, 05 — see `tests/scenarios/`):** + - Progress Rule 1 ("Checkpoints are explicit. Write ONLY on `/checkpoint` + or learner confirmation") read as an absolute gate on every field write, + contradicting Rules 5/6/8 and Step 6b, which already mandate immediate, + unconditional writes for `completed_exercises`, `topic_confidence`/ + `review_due`, and the hint streaks. Reworded to make explicit that Rule 1 + governs the narrated full-file save, not the field-level writes other + rules already require — an interrupted session shouldn't lose those. + - `tests/check_progress_schema.py`'s `next_up: null` advisory was + implemented as `warn(label, True, detail)` — since `warn()` only emits + `WARN` on a falsy condition, this line could never print anything but + `PASS`, silently suppressing the "verify track completion manually" + reminder in exactly the case it exists to flag. Added a dedicated + `note()` helper for unconditional advisories and switched this check to + use it. + - The Axis Re-Calibration section's two threshold bullets were worded + asymmetrically — only the `low_hint_streak` bullet stated "at the next + phase transition," leaving it ambiguous whether `high_hint_streak`'s + offer could also surface at a plain exercise-complete menu. Made + explicit that both signals are checked only at phase-transition points. + +### Verified +- **Full Tier 2 pre-release run (all five scenarios) against this + `SKILL.md`/`curricula/python-basics.md`, live via an agent playing both + Sage and each scripted learner:** + - `01-onboarding-and-profile-location`: 6/6 PASS. + - `02-topic-key-consistency`: 5/5 PASS. + - `03-toolchain-vs-learner-bug`: 6/6 PASS. + - `04-hint-streak-scoping-and-decline`: first run surfaced 4/5 PASS, 1 + FAIL — but the FAIL was a bug in the *scenario itself* (it asserted a + fresh recalibration offer after `P1-comprehension-refactor`, which isn't + Phase 1's last exercise, so per SKILL.md's phase-transition-gated + mechanism no offer was ever going to fire there). Fixed the scenario to + assert at `P1-json-roundtrip` (Phase 1's actual last exercise) instead; + also tightened the Axis Re-Calibration wording asymmetry noted above. + Re-ran end-to-end (fresh scratch project, all 7 steps replayed with real + exercises/verify runs): 6/6 PASS, including the two new assertions that + the offer correctly stays silent at both non-transition points and + re-fires as a genuinely fresh signal at the real Phase 1→2 transition. + - `05-track-completion-handling`: 5/6 PASS, 1 "FAIL" that was the + `check_progress_schema.py` bug documented above, not a spec issue. + - **Net result: all five Tier 2 scenarios pass cleanly against the current + `SKILL.md` and `curricula/python-basics.md`.** +- **Real-install smoke test.** Every scenario above (and the original 1.2.0 + bug hunt) validated the spec via an agent reading `SKILL.md` directly and + role-playing Sage — never through Claude Code's actual plugin-loading and + skill-triggering machinery. Ran `claude -p --plugin-dir` against this repo + from a fresh scratch project with no prior profile/progress files: the + skill triggered correctly from unscripted natural language ("teach me + Python, let's start," no slash command), ran Profile Setup, offered the + bundled "Python Foundations" track via Round 0, skipped straight to Phase + 0 on selection, and produced a real lesson (Steps 1-5, correctly bridging + to the stated Java/JS background including the `[]` truthiness + JS-vs-Python gotcha). The resulting `.sage-profile.md` and + `.sage-progress.json` — written by the real mechanism, not staged — pass + `check_progress_schema.py` with 0 failures. (`AskUserQuestion` isn't + available in headless `-p` mode; Sage correctly degraded to plain + numbered questions instead of erroring.) `claude plugin validate .` also + passes. + ## [1.2.0] — 2026-07-01 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98685cc..b90fbe2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,16 @@ Open an issue describing: what you expected, what happened, and your Claude Code 1. Clone the repo 2. `claude plugin validate .` to verify structure 3. Test locally: install from the local directory -4. Submit a PR with a clear description +4. If you're editing `SKILL.md` or a curriculum's spec-level behavior (progress + fields, verification, streaks, onboarding), run the relevant scenario(s) in + `tests/scenarios/` — see `tests/README.md`. These are fixed regression + scripts targeting bugs that were previously found and fixed; a spec edit + that silently breaks one of them is exactly what they're for catching. + `python3 tests/test_check_progress_schema.py` (also run in CI) covers the + deterministic Tier 1 checks and needs no agent. +5. Bumping the plugin version? Run `tests/README.md`'s full pre-release + checklist (all five Tier 2 scenarios) first. +6. Submit a PR with a clear description ## Code of Conduct Be kind. Be constructive. We're all here to learn. diff --git a/skills/sage-instructor/SKILL.md b/skills/sage-instructor/SKILL.md index 237a09a..02c32ae 100644 --- a/skills/sage-instructor/SKILL.md +++ b/skills/sage-instructor/SKILL.md @@ -177,7 +177,7 @@ Stored as `.sage-progress.json` in the project root. ``` ### Progress Rules -1. **Checkpoints are explicit.** Write ONLY on `/checkpoint` or learner confirmation. +1. **Checkpoints are explicit — but that governs the narrated full save, not every field.** Sage doesn't announce "checkpoint saved" or write the whole file speculatively except on `/checkpoint`, a phase-transition option that includes it, or explicit learner confirmation. That's distinct from the field-level writes other rules already mandate unconditionally — `completed_exercises` (Rule 5), `topic_confidence`/`review_due` (Rule 6), the hint streaks (Rule 8), and the Step 6b exercise-pointer promotion. Those persist immediately when their triggering event fires; they are not held back waiting for a narrated checkpoint, so an interrupted session doesn't silently lose them. 2. **Always read before writing.** Load, merge, write. 3. **hint_count resets** per exercise. 4. **observations** — pedagogical notes, under 200 chars. @@ -301,10 +301,10 @@ Before asking, check the recalibration signal below — if it fired, add a 5th o ### Axis Re-Calibration -Axis levels are declared once at track creation and go stale. `low_hint_streak` / `high_hint_streak` (see Progress File Format) are the signal for when they no longer match reality: +Axis levels are declared once at track creation and go stale. `low_hint_streak` / `high_hint_streak` (see Progress File Format) are the signal for when they no longer match reality. Both signals are only checked at phase-transition points (see the "Before asking, check the recalibration signal" step in Phase transitions above) — a streak crossing its threshold mid-phase doesn't surface an offer at that exercise's own "what next?" menu, only at the next phase transition, whether or not that transition happens to land in a different phase than where the streak started: - **`low_hint_streak >= 3`** (three exercises in a row, zero hints) → the declared Mastery is probably too low. Offer a bump at the next phase transition. -- **`high_hint_streak >= 2`** (two exercises in a row needing 3+ hints) → the declared Mastery or pace is probably too high. Offer to dial back — either Mastery down a level, or just slower pacing within the same level. +- **`high_hint_streak >= 2`** (two exercises in a row needing 3+ hints) → the declared Mastery or pace is probably too high. Offer to dial back — either Mastery down a level, or just slower pacing within the same level — at the next phase transition. - If the learner accepts, write the new level to `axis_overrides` in the progress file (never overwrite the curriculum file itself — the override layers on top, see Progress Rules). Confirm what changed in one sentence. - If the learner declines, reset the streak that triggered the offer to 0. Otherwise the same offer resurfaces at every subsequent phase transition until it's acted on — a fresh streak accumulating from here is a new signal and earns a fresh offer, but a stale one shouldn't nag. - This only ever surfaces as an offer, never a silent change. The learner decides. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..556ac72 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,106 @@ +# Regression harness + +Sage's "source code" is natural-language instructions (`SKILL.md`, `curricula/*.md`) +interpreted by an LLM at runtime, not a program you can unit-test in the usual +sense. This harness makes regression testing repeatable anyway, split into two +tiers: + +## Tier 1 — deterministic structural checks + +`check_progress_schema.py` validates a `.sage-progress.json` (and the presence +of `.sage-profile.md`) against the rules in `SKILL.md`'s Progress Rules +section — kebab-case topic keys (Rule 10), mutually-exclusive hint streaks +(Rule 8), `review_due` staying a subset of `topic_confidence`, observations +length, exercise slug format, profile file location. No LLM involved; runs in +milliseconds; wired into CI on every push and PR +(`.github/workflows/tier1-checks.yml`). + +``` +python3 tests/check_progress_schema.py +``` + +This catches schema drift (an artifact malformed relative to the spec) but +says nothing about whether Sage's *behavior* was pedagogically correct — that +needs Tier 2. + +### Testing the checker itself + +`check_progress_schema.py` is plain deterministic code, so it gets its own +unit tests rather than relying on an expensive Tier 2 run to notice a bug in +it (that's exactly how the `next_up`/`warn()` bug shipped originally — see +`CHANGELOG.md`). `test_check_progress_schema.py` runs the checker against the +fixtures in `fixtures/` (`valid/`, `valid-track-complete/`, and several +`invalid-*/` directories each isolating one violation) and pins the expected +exit code and failure message per fixture: + +``` +python3 tests/test_check_progress_schema.py -v +``` + +This is what `.github/workflows/tier1-checks.yml` runs on every push and PR +against `main` — Tier 1 is the actual automated gate; Tier 2 remains manual +(see below). + +## Tier 2 — scripted behavioral scenarios + +`scenarios/*.md` are fixed, repeatable session scripts — not open-ended +"go explore" prompts. Each one targets a specific bug class that was +previously found and fixed (see `CHANGELOG.md` 1.2.0), so re-running them +after a `SKILL.md` or curriculum edit tells you whether that fix still holds. + +Each scenario file has: +- **Regression target** — the specific bug this guards against, with a + `SKILL.md` line reference. +- **Setup** — the scratch project state to start from. +- **Script** — a fixed sequence of learner turns/commands. Not improvised. +- **Assertions** — a checklist split into `[mechanical]` (checkable by + `check_progress_schema.py` or by grepping the transcript for a literal) and + `[behavioral]` (requires reading the transcript and judging intent). + +### Running a scenario + +There's no separate test-runner binary — the "test runner" is an agent that +plays both Sage (per the current `SKILL.md`/`curricula/`) and the scripted +learner persona, executing real commands where the spec calls for real +execution (Step 6b `verify`), then grades itself against the scenario's +assertion checklist. `run_scenario_prompt.md` is the reusable prompt template +for that agent. + +Use it via Claude Code's `Agent` tool (or paste it into a fresh session): + +1. Give the agent `run_scenario_prompt.md`'s contents with `{SCENARIO}` + substituted for a path under `tests/scenarios/`. +2. It works in an isolated scratch directory (never inside this repo), reads + the relevant `SKILL.md`/curriculum sections directly (not from memory), + executes the script turn by turn, and produces a transcript plus real + `.sage-progress.json` / `.sage-profile.md` artifacts. +3. It runs `check_progress_schema.py` against those artifacts for the + mechanical assertions and self-grades the behavioral ones, citing the + specific transcript line and `SKILL.md` rule for each. +4. It reports PASS/FAIL per assertion — not a vibe summary. + +### Why Tier 2 isn't wired into CI + +Each scenario run is a multi-turn LLM conversation that costs real tokens and +minutes, and the "system under test" and the "test runner" are both LLMs +interpreting a spec, so results aren't bit-for-bit deterministic the way a +unit test is. Tier 2 is a **pre-release gate**, run manually, not a +per-commit CI check. + +### Pre-release checklist + +Before bumping the plugin version (`.claude-plugin/plugin.json` + +`CHANGELOG.md`), run **all five** scenarios in `scenarios/` — not just the +ones nearest whatever you changed. Record the result (pass/fail per +assertion) in the version's CHANGELOG entry the way 1.2.0 and the harness's +own rollout did, so "we ran the harness" is checkable later instead of +trusted on faith. If a scenario fails, that's either a spec bug (fix +`SKILL.md`/the curriculum) or a scenario gone stale (the curriculum changed +underneath it — update the scenario's setup/script to match). + +### Adding a scenario + +When a live-session bug hunt (like the ones behind 1.2.0) finds something new +and it gets fixed in `SKILL.md`, add a scenario file here targeting it, so the +next spec edit gets checked against it instead of relying on someone +remembering to re-discover the same bug. diff --git a/tests/check_progress_schema.py b/tests/check_progress_schema.py new file mode 100644 index 0000000..e12e514 --- /dev/null +++ b/tests/check_progress_schema.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Deterministic structural checker for Sage's runtime artifacts. + +Validates a .sage-progress.json (and optionally a sibling .sage-profile.md) +against the rules stated in skills/sage-instructor/SKILL.md's "Progress File +Format" / "Progress Rules" section. This catches spec-drift and schema +regressions mechanically -- it does NOT judge whether Sage's pedagogy was +good, only whether the artifacts it produced are well-formed. Pair with the +scenario transcripts in tests/scenarios/ for the behavioral checks. + +Usage: + python3 tests/check_progress_schema.py + +Exit code 0 if every check passes, 1 otherwise. Findings are printed as +PASS/FAIL/WARN lines so a grading agent (or a human) can read the report +without re-deriving the rules. +""" +import json +import os +import re +import sys + +KEBAB = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +SLUG = re.compile(r"^P\d+-[a-z0-9]+(-[a-z0-9]+)*$") +CONFIDENCE_LEVELS = {"solid", "shaky", "struggling"} +AXIS_KEYS = {"mastery", "consequence", "intent"} + +results = [] + + +def check(label, condition, detail=""): + status = "PASS" if condition else "FAIL" + results.append((status, label, detail)) + return condition + + +def warn(label, condition, detail=""): + status = "PASS" if condition else "WARN" + results.append((status, label, detail)) + return condition + + +def note(label, detail=""): + """Unconditional advisory -- always surfaces as NOTE with its detail + visible, for things the script can observe but not itself verify + (e.g. 'this requires a manual check'). Never fails the run.""" + results.append(("NOTE", label, detail)) + + +def check_track(name, t): + prefix = f"tracks.{name}" + + check(f"{prefix}.phase is a non-negative int", + isinstance(t.get("phase"), int) and t.get("phase") >= 0, + f"got {t.get('phase')!r}") + + exercises = t.get("completed_exercises", []) + check(f"{prefix}.completed_exercises entries match P{{N}}-{{slug}}", + all(isinstance(e, str) and SLUG.match(e) for e in exercises), + f"got {exercises!r}") + + topic_confidence = t.get("topic_confidence", {}) + check(f"{prefix}.topic_confidence keys are kebab-case (Rule 10)", + all(KEBAB.match(k) for k in topic_confidence), + f"got keys {list(topic_confidence.keys())!r}") + check(f"{prefix}.topic_confidence values are solid/shaky/struggling", + all(v in CONFIDENCE_LEVELS for v in topic_confidence.values()), + f"got {topic_confidence!r}") + + review_due = t.get("review_due", []) + check(f"{prefix}.review_due entries are a subset of topic_confidence keys", + all(k in topic_confidence for k in review_due), + f"review_due={review_due!r} topic_confidence keys={list(topic_confidence.keys())!r}") + warn(f"{prefix}.review_due has no 'solid' entries (Rule 6: cleared topics are removed)", + all(topic_confidence.get(k) != "solid" for k in review_due), + f"review_due={review_due!r}") + + observations = t.get("observations", "") + check(f"{prefix}.observations is under 200 chars (Rule 4)", + len(observations) < 200, + f"len={len(observations)}") + + axis_overrides = t.get("axis_overrides", {}) + check(f"{prefix}.axis_overrides only uses known axis keys (Rule 7)", + all(k in AXIS_KEYS for k in axis_overrides), + f"got {axis_overrides!r}") + + low = t.get("low_hint_streak", 0) + high = t.get("high_hint_streak", 0) + check(f"{prefix}.low_hint_streak/high_hint_streak are non-negative ints", + isinstance(low, int) and isinstance(high, int) and low >= 0 and high >= 0, + f"low={low!r} high={high!r}") + check(f"{prefix} streaks are mutually exclusive (Rule 8: incrementing one resets the other)", + low == 0 or high == 0, + f"low={low!r} high={high!r}") + + hint_count = t.get("hint_count", 0) + check(f"{prefix}.hint_count is a non-negative int", + isinstance(hint_count, int) and hint_count >= 0, + f"got {hint_count!r}") + + next_up = t.get("next_up") + current_exercise = t.get("current_exercise") + if next_up is None: + note(f"{prefix}.next_up is null -> should mean track completion (Step 6b point 8)", + "verify manually that this track's curriculum has no exercises left") + else: + check(f"{prefix}.next_up looks like an exercise slug when not null", + isinstance(next_up, str) and SLUG.match(next_up), + f"got {next_up!r}") + check(f"{prefix}.current_exercise looks like an exercise slug", + current_exercise is None or (isinstance(current_exercise, str) and SLUG.match(current_exercise)), + f"got {current_exercise!r}") + + +def main(): + if len(sys.argv) != 2: + print(__doc__) + sys.exit(2) + + project_dir = sys.argv[1] + progress_path = os.path.join(project_dir, ".sage-progress.json") + profile_path = os.path.join(project_dir, ".sage-profile.md") + + if not check("`.sage-progress.json` exists at project root", + os.path.isfile(progress_path), progress_path): + print_report() + sys.exit(1) + + with open(progress_path) as f: + raw = f.read() + try: + data = json.loads(raw) + check("`.sage-progress.json` is valid JSON", True) + except json.JSONDecodeError as e: + check("`.sage-progress.json` is valid JSON", False, str(e)) + print_report() + sys.exit(1) + + check("top-level has 'active_track'", "active_track" in data) + check("top-level has 'tracks'", isinstance(data.get("tracks"), dict)) + + for name, t in data.get("tracks", {}).items(): + check_track(name, t) + + check("`.sage-profile.md` exists at project root (not inside skills/sage-instructor/references/)", + os.path.isfile(profile_path), profile_path) + + print_report() + sys.exit(0 if all(s != "FAIL" for s, _, _ in results) else 1) + + +def print_report(): + for status, label, detail in results: + line = f"[{status}] {label}" + if detail and status != "PASS": + line += f"\n {detail}" + print(line) + fails = sum(1 for s, _, _ in results if s == "FAIL") + warns = sum(1 for s, _, _ in results if s == "WARN") + notes = sum(1 for s, _, _ in results if s == "NOTE") + print(f"\n{len(results)} checks, {fails} failed, {warns} warned, {notes} noted.") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/invalid-bad-json/.sage-progress.json b/tests/fixtures/invalid-bad-json/.sage-progress.json new file mode 100644 index 0000000..e4b6362 --- /dev/null +++ b/tests/fixtures/invalid-bad-json/.sage-progress.json @@ -0,0 +1,8 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": ["P0-fizzbuzz"], + } + } diff --git a/tests/fixtures/invalid-missing-profile/.sage-progress.json b/tests/fixtures/invalid-missing-profile/.sage-progress.json new file mode 100644 index 0000000..eea6728 --- /dev/null +++ b/tests/fixtures/invalid-missing-profile/.sage-progress.json @@ -0,0 +1,20 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": ["P0-fizzbuzz"], + "current_topic": "Control flow", + "current_exercise": "P0-temp-converter", + "next_up": "P0-word-counter", + "observations": "Fast grasp of loops and truthiness.", + "topic_confidence": {"variables": "solid", "control-flow": "shaky"}, + "review_due": ["control-flow"], + "axis_overrides": {}, + "low_hint_streak": 1, + "high_hint_streak": 0, + "last_session": "2026-07-02: Completed P0-fizzbuzz", + "hint_count": 0 + } + } +} diff --git a/tests/fixtures/invalid-review-due-dangling/.sage-progress.json b/tests/fixtures/invalid-review-due-dangling/.sage-progress.json new file mode 100644 index 0000000..6ce0a56 --- /dev/null +++ b/tests/fixtures/invalid-review-due-dangling/.sage-progress.json @@ -0,0 +1,20 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": [], + "current_topic": "Variables", + "current_exercise": "P0-fizzbuzz", + "next_up": "P0-temp-converter", + "observations": "", + "topic_confidence": {"variables": "solid"}, + "review_due": ["control-flow"], + "axis_overrides": {}, + "low_hint_streak": 0, + "high_hint_streak": 0, + "last_session": "", + "hint_count": 0 + } + } +} diff --git a/tests/fixtures/invalid-streak-overlap/.sage-progress.json b/tests/fixtures/invalid-streak-overlap/.sage-progress.json new file mode 100644 index 0000000..b942205 --- /dev/null +++ b/tests/fixtures/invalid-streak-overlap/.sage-progress.json @@ -0,0 +1,20 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": [], + "current_topic": "Variables", + "current_exercise": "P0-fizzbuzz", + "next_up": "P0-temp-converter", + "observations": "", + "topic_confidence": {}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 2, + "high_hint_streak": 1, + "last_session": "", + "hint_count": 0 + } + } +} diff --git a/tests/fixtures/invalid-topic-key/.sage-progress.json b/tests/fixtures/invalid-topic-key/.sage-progress.json new file mode 100644 index 0000000..513d3a7 --- /dev/null +++ b/tests/fixtures/invalid-topic-key/.sage-progress.json @@ -0,0 +1,20 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": ["P0-fizzbuzz"], + "current_topic": "Control flow", + "current_exercise": "P0-temp-converter", + "next_up": "P0-word-counter", + "observations": "", + "topic_confidence": {"Control_Flow": "shaky"}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 0, + "high_hint_streak": 0, + "last_session": "", + "hint_count": 0 + } + } +} diff --git a/tests/fixtures/valid-track-complete/.sage-profile.md b/tests/fixtures/valid-track-complete/.sage-profile.md new file mode 100644 index 0000000..a593e5b --- /dev/null +++ b/tests/fixtures/valid-track-complete/.sage-profile.md @@ -0,0 +1,8 @@ +# Learner Profile + +- Name: Test Learner +- Role: n/a +- Bridge languages: Python +- Experience: Mid-level (3-5yr) +- Learning style: Hands-on first +- Tone: Direct and concise diff --git a/tests/fixtures/valid-track-complete/.sage-progress.json b/tests/fixtures/valid-track-complete/.sage-progress.json new file mode 100644 index 0000000..aa67b1c --- /dev/null +++ b/tests/fixtures/valid-track-complete/.sage-progress.json @@ -0,0 +1,20 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 2, + "completed_exercises": ["P0-fizzbuzz", "P1-list-ops", "P2-persistent-tasklist"], + "current_topic": "Combining classes with json persistence", + "current_exercise": "P2-persistent-tasklist", + "next_up": null, + "observations": "Track complete.", + "topic_confidence": {"classes": "solid"}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 2, + "high_hint_streak": 0, + "last_session": "2026-07-02: Completed the track", + "hint_count": 0 + } + } +} diff --git a/tests/fixtures/valid/.sage-profile.md b/tests/fixtures/valid/.sage-profile.md new file mode 100644 index 0000000..a593e5b --- /dev/null +++ b/tests/fixtures/valid/.sage-profile.md @@ -0,0 +1,8 @@ +# Learner Profile + +- Name: Test Learner +- Role: n/a +- Bridge languages: Python +- Experience: Mid-level (3-5yr) +- Learning style: Hands-on first +- Tone: Direct and concise diff --git a/tests/fixtures/valid/.sage-progress.json b/tests/fixtures/valid/.sage-progress.json new file mode 100644 index 0000000..eea6728 --- /dev/null +++ b/tests/fixtures/valid/.sage-progress.json @@ -0,0 +1,20 @@ +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": ["P0-fizzbuzz"], + "current_topic": "Control flow", + "current_exercise": "P0-temp-converter", + "next_up": "P0-word-counter", + "observations": "Fast grasp of loops and truthiness.", + "topic_confidence": {"variables": "solid", "control-flow": "shaky"}, + "review_due": ["control-flow"], + "axis_overrides": {}, + "low_hint_streak": 1, + "high_hint_streak": 0, + "last_session": "2026-07-02: Completed P0-fizzbuzz", + "hint_count": 0 + } + } +} diff --git a/tests/run_scenario_prompt.md b/tests/run_scenario_prompt.md new file mode 100644 index 0000000..abc210b --- /dev/null +++ b/tests/run_scenario_prompt.md @@ -0,0 +1,56 @@ +Reusable prompt template for executing a Tier 2 regression scenario. Substitute +`{SCENARIO}` with a path to a file under `tests/scenarios/` and hand this to +an agent (e.g. via the `Agent` tool, `subagent_type: general-purpose`, with +Bash/Read/Write access). + +--- + +You are QA-testing the Sage Instructor Claude Code plugin. Your job is to run +ONE fixed regression scenario end-to-end and report PASS/FAIL per assertion — +not to freely explore, and not to be lenient. + +1. Read `skills/sage-instructor/SKILL.md`, `skills/sage-instructor/references/philosophy.md`, + and whichever `skills/sage-instructor/curricula/*.md` the scenario needs. + Follow those instructions literally — you are playing Sage exactly as + specified, not as you'd improvise a good tutor. If the scenario's script + and the spec conflict, the spec wins and that's worth flagging. + +2. Read the scenario file at `{SCENARIO}`. It defines: setup, a fixed script + of learner turns, and an assertion checklist. + +3. Work entirely inside a fresh scratch directory (use your scratchpad, never + this repo checkout) that stands in for "the learner's project root." Create + whatever `.sage-progress.json` / `.sage-profile.md` the setup section + specifies before starting. + +4. Play through the script turn by turn. You are simultaneously: + - **Sage**, generating exactly the output the real skill would (lessons, + AskUserQuestion-style prompts rendered as text, verify runs via real + Bash, progress file writes via real Write/Edit). + - **the learner**, whose responses are fixed by the script — don't + improvise better or worse answers than scripted, that defeats + repeatability. + + Where the scenario script says a command should be run (verify, etc.), run + it for real via Bash and use the actual output — don't narrate a + plausible-looking result. + +5. When the script ends, run: + ``` + python3 /tests/check_progress_schema.py + ``` + for the `[mechanical]` assertions. + +6. For each `[behavioral]` assertion, quote the specific transcript moment + that satisfies (or fails to satisfy) it, and cite the `SKILL.md` line/rule + it's checking. "Looks fine" is not a grade — point at the evidence. + +7. Report a checklist: every assertion from the scenario file, PASS or FAIL, + one line each, with a one-sentence reason. End with a summary count. If + anything failed, state plainly whether it's a spec bug (SKILL.md/curriculum + wording is ambiguous or wrong) or an execution slip (you didn't follow the + spec correctly) — those need different fixes. + +Do not soften a FAIL into a PASS because the overall session "felt" fine. +This scenario exists specifically because a past version of Sage got this +exact thing wrong. diff --git a/tests/scenarios/01-onboarding-and-profile-location.md b/tests/scenarios/01-onboarding-and-profile-location.md new file mode 100644 index 0000000..adb313d --- /dev/null +++ b/tests/scenarios/01-onboarding-and-profile-location.md @@ -0,0 +1,49 @@ +# Scenario 01 — Onboarding, existing-track offer, profile location + +## Regression target +- Generated learner profiles must land at `.sage-profile.md` in the project + root, never inside `skills/sage-instructor/references/` (fixed in 1.2.0 — + the plugin directory can be wiped on reinstall). See `SKILL.md` "Load + learner profile" and "Profile Setup." +- Track Setup Round 0 must offer the bundled `python-basics.md` curriculum by + its `title` and skip straight to Phase 0 when picked — not fall through to + the 4-round custom-track interview. See `SKILL.md` Round 0 and Track Setup. + +## Setup +Empty scratch directory. No `.sage-profile.md`, no `.sage-progress.json`. +`skills/sage-instructor/curricula/` contains `TEMPLATE.md` and +`python-basics.md` (title: "Python Foundations") exactly as shipped in this +repo — don't modify them for this scenario. + +## Script +1. Learner runs `/sage-start`. +2. Profile Setup, answer each round exactly: + - Identity: "Ana, backend engineer" + - Bridge languages: Python, Go + - Experience: "Mid-level (3-5yr)" + - Learning style: "Hands-on first" + - Tone: "Direct and concise" + Confirm the generated profile looks right. +3. Track Setup Round 0 should fire (no `active_track` yet, curricula/ has + more than just TEMPLATE.md). Learner picks "Python Foundations" (the + existing python-basics track), not "Build a custom track." +4. Learner runs `/sage-lesson`. + +## Assertions + +- `[mechanical]` `.sage-profile.md` exists at the scratch project root. +- `[mechanical]` No profile file was written under + `skills/sage-instructor/references/` (that directory only ever contains + `learner-profile-template.md`, unchanged). +- `[mechanical]` `python3 tests/check_progress_schema.py ` exits + 0 on the resulting `.sage-progress.json`. +- `[behavioral]` Round 0 presented "Python Foundations" as a pickable option + alongside "Build a custom track" — the learner was not forced through + Round 1-4 of the custom-track interview. +- `[behavioral]` Picking the existing track set `active_track` to + `python-basics` and jumped straight to Phase 0 content — no custom-track + questions ("What do you want to learn?", destination project, etc.) were + asked. +- `[behavioral]` The Step 1/2 lesson content actually referenced the + learner's stated bridge languages (Python and/or Go) per the Bridge step, + not a generic bridge language chosen without regard to the profile. diff --git a/tests/scenarios/02-topic-key-consistency.md b/tests/scenarios/02-topic-key-consistency.md new file mode 100644 index 0000000..eab14a9 --- /dev/null +++ b/tests/scenarios/02-topic-key-consistency.md @@ -0,0 +1,70 @@ +# Scenario 02 — Topic-key derivation and reuse (Rule 10) + +## Regression target +`topic_confidence`/`review_due` keys must be derived once per curriculum +Topics bullet (kebab-case, first-named concept for unlabeled/comma-separated +bullets) and reused verbatim everywhere after — not re-derived each time the +topic comes up. See `SKILL.md` Progress Rule 10. + +The target bullet is `python-basics.md` Phase 0's first Topics entry: +`Variables, dynamic typing, truthiness` — unlabeled and comma-separated, so +Rule 10 says the key must be `variables` (the first-named concept), not +`dynamic-typing`, `truthiness`, or a compound of all three. + +## Setup +Scratch directory with `.sage-progress.json`: +```json +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": [], + "current_topic": "Variables, dynamic typing, truthiness", + "current_exercise": "P0-fizzbuzz", + "next_up": "P0-temp-converter", + "observations": "", + "topic_confidence": {}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 0, + "high_hint_streak": 0, + "last_session": "", + "hint_count": 0 + } + } +} +``` +and a `.sage-profile.md` (any valid profile — bridge language Java, mid-level, +hands-on). + +## Script +1. Learner runs `/sage-lesson` (no topic given — picks up "Variables, dynamic + typing, truthiness," the current topic). +2. At Step 5 (Comprehension Check), learner answers the first question + **incorrectly** (deliberately pick a plausible-but-wrong option about + truthiness, e.g. treating `0` or `""` as truthy). Sage revisits per Step 5; + learner then answers **correctly**. +3. Learner completes `P0-fizzbuzz` with a correct solution; verify passes. +4. Learner runs `/sage-drill`. +5. Drill should pull from `review_due` first. Learner answers the + drill question on this same topic **correctly**. + +## Assertions + +- `[mechanical]` After step 2, `topic_confidence` has exactly one new key, + it is kebab-case, and its value is `"variables"` — not `"dynamic-typing"`, + `"truthiness"`, or any multi-concept compound. +- `[mechanical]` `python3 tests/check_progress_schema.py ` exits + 0 at every checkpoint (after step 2, after step 4). +- `[behavioral]` After step 2 (wrong-then-right), `topic_confidence["variables"] + == "shaky"` and `"variables"` was added to `review_due` (Step 5 outcome + rule: wrong-then-correct-after-revisit → `shaky`). +- `[behavioral]` The key used in step 5's drill round for this topic is + **byte-for-byte identical** to the key set in step 2 — quote both + occurrences from the transcript/file and confirm they match. This is the + actual regression: a spec that re-derives the key at drill time instead of + reusing the established one. +- `[behavioral]` After the correct drill answer in step 5, `"variables"` is + removed from `review_due` and `topic_confidence["variables"] == "solid"` + (drill-mode outcome rule). diff --git a/tests/scenarios/03-toolchain-vs-learner-bug.md b/tests/scenarios/03-toolchain-vs-learner-bug.md new file mode 100644 index 0000000..db0f3a4 --- /dev/null +++ b/tests/scenarios/03-toolchain-vs-learner-bug.md @@ -0,0 +1,75 @@ +# Scenario 03 — Toolchain failure must not count as a learner bug + +## Regression target +A `verify` run that fails for a reason that isn't a language-level +error/traceback from the learner's own code (missing interpreter, alias-stub +prompt, PATH issue, permissions error) must be recognized as a toolchain +problem, not counted against hint streaks, and not treated as a live Gotcha +requiring a code fix. See `SKILL.md` Step 6b point 3. This generalizes the +concrete `python3` Windows-alias-stub bug that shipped in `python-basics.md` +in 1.1.0 and was fixed in 1.2.0 — this scenario tests the general rule, not +just that one platform quirk. + +## Setup +Scratch directory with `.sage-progress.json`: +```json +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": ["P0-fizzbuzz"], + "current_topic": "Control flow", + "current_exercise": "P0-temp-converter", + "next_up": "P0-word-counter", + "observations": "", + "topic_confidence": {"variables": "solid"}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 1, + "high_hint_streak": 0, + "last_session": "", + "hint_count": 0 + } + } +} +``` +and a valid `.sage-profile.md`. + +## Script +1. Learner writes a genuinely correct `P0-temp-converter` solution + (Celsius↔Fahrenheit conversion function with self-check asserts, per the + curriculum's exercise description) and says they're done. +2. Before running the curriculum's real `verify` command, deliberately run + the verify step against a **fabricated, guaranteed-missing interpreter + name** instead — e.g. substitute the command with + `sage_test_nonexistent_interpreter_xyz {file}` — so it fails with a shell + "command not found" (exit 127), which is not a Python traceback and not a + language-level error from the learner's code. Run this for real via Bash; + don't narrate a plausible result. +3. Observe how Sage responds to that failure per Step 6b point 3. +4. Now run the *actual* curriculum `verify` command (`python {file}`) against + the same correct solution. It should pass. + +## Assertions + +- `[behavioral]` In response to step 2's failure, Sage explicitly identified + it as a toolchain/environment problem rather than a comprehension gap — + quote the specific reasoning (e.g. noting the error isn't a Python + traceback, or is a shell-level "not found"). +- `[behavioral]` Sage did NOT treat the step 2 failure as a live Gotcha + requiring the learner to fix their code, and did not ask a Socratic + "what do you think went wrong with your code" question about it. +- `[behavioral]` Sage suggested a plausible toolchain-level fix (check + interpreter name, check PATH) rather than silently retrying or blaming the + solution. +- `[mechanical]` `hint_count`, `low_hint_streak`, and `high_hint_streak` were + unchanged by step 2's failure — diff the progress file before/after step 2 + and confirm no streak fields moved. +- `[mechanical]` After step 4's real, passing verify run: `P0-temp-converter` + is in `completed_exercises`, `low_hint_streak == 2` (this exercise added 0 + hints on top of the setup's existing streak of 1), `high_hint_streak == 0`, + and `current_exercise`/`next_up` promoted per Step 6b point 8 + (`current_exercise` becomes `P0-word-counter`). +- `[mechanical]` `python3 tests/check_progress_schema.py ` exits + 0 on the final state. diff --git a/tests/scenarios/04-hint-streak-scoping-and-decline.md b/tests/scenarios/04-hint-streak-scoping-and-decline.md new file mode 100644 index 0000000..58d3a11 --- /dev/null +++ b/tests/scenarios/04-hint-streak-scoping-and-decline.md @@ -0,0 +1,91 @@ +# Scenario 04 — Hint-streak wording and decline-resets-streak + +## Regression target +- The recalibration offer must be worded as "the last few exercises," never + "this phase" — the streak is exercise-scoped and can span a phase boundary. + See `SKILL.md` Phase transitions section (the paragraph right after the + four standard options) and Axis Re-Calibration. +- Declining the offer must reset the streak that triggered it, so it doesn't + resurface at every subsequent phase transition — but a *fresh* streak that + accumulates afterward earns a genuinely new offer. See `SKILL.md` Axis + Re-Calibration, the "if the learner declines" bullet. + +## Setup +Scratch directory with `.sage-progress.json`: +```json +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 0, + "completed_exercises": ["P0-fizzbuzz"], + "current_topic": "Control flow", + "current_exercise": "P0-temp-converter", + "next_up": "P0-word-counter", + "observations": "", + "topic_confidence": {}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 0, + "high_hint_streak": 1, + "last_session": "", + "hint_count": 0 + } + } +} +``` +(`high_hint_streak: 1` simulates that `P0-fizzbuzz` already needed 3+ hints.) +Valid `.sage-profile.md`. + +## Script +1. Learner works `P0-temp-converter`, needs 3+ hints (drive `hint_count` to + 3 via real `/sage-hint` calls), then submits a correct solution. Verify + passes. `high_hint_streak` should become 2 — this exercise is **not** the + end of Phase 0 (one more, `P0-word-counter`, remains), so no phase + transition happens here, but the recalibration threshold (`>= 2`) is now + met. +2. Learner works `P0-word-counter` (Phase 0's last exercise), needs 3+ hints + again, submits a correct solution. Verify passes. This IS the end of + Phase 0 → the phase-transition AskUserQuestion fires. +3. At the phase-transition question, the recalibration option should be + present (per the streak state) — check its exact wording. +4. Learner picks a **non-recalibration** option ("Save checkpoint and stop + for now"), i.e. declines the offer. +5. Learner starts Phase 1, completes `P1-list-ops` needing 3+ hints again. +6. Learner completes `P1-comprehension-refactor` (Phase 1's *second* of + three exercises — `P1-list-ops`, `P1-comprehension-refactor`, + `P1-json-roundtrip`, per `curricula/python-basics.md` — not its last), + also needing 3+ hints. `high_hint_streak` reaches the `>= 2` threshold + here, but this is **not** a phase-transition point. +7. Learner completes `P1-json-roundtrip` (Phase 1's actual last exercise), + again needing 3+ hints so the streak isn't reset by a hint-free pass. + This IS the end of Phase 1 → the phase-transition AskUserQuestion fires. + +## Assertions + +- `[behavioral]` The recalibration option's wording at step 3 says "the last + few exercises" (or clearly equivalent phrasing that doesn't scope to a + single phase) — quote it directly. It must not say "this phase" given the + streak spans `P0-temp-converter` and `P0-word-counter`, both inside Phase + 0 here, but the copy itself must be the phase-agnostic wording regardless + of whether this particular run happens to stay within one phase. +- `[mechanical]` After step 4 (decline), `high_hint_streak` is reset to `0` + in the progress file — diff before/after the decline. +- `[behavioral]` After step 5 (first Phase 1 struggle exercise post-decline), + no recalibration offer fires yet — the streak was reset in step 4, so this + is only the first struggle exercise of a new streak (`high_hint_streak == + 1`), below threshold. +- `[behavioral]` After step 6, `high_hint_streak == 2` (threshold met) but + **no** recalibration offer is presented — `P1-comprehension-refactor` is + not Phase 1's last exercise, so no phase-transition question fires at all + here, and per SKILL.md (Axis Re-Calibration: "Both signals are only + checked at phase-transition points") the signal isn't surfaced outside + one. Sage should show only the standard 4-option "Exercise complete. What + next?" menu, with no 5th option. +- `[behavioral]` After step 7 (Phase 1's actual last exercise, phase + transition fires), `high_hint_streak == 3` and a **fresh** recalibration + offer is presented at the phase-transition question — confirm this isn't + suppressed as a repeat of the declined one (SKILL.md: "a fresh streak + accumulating from here is a new signal and earns a fresh offer"). +- `[mechanical]` `python3 tests/check_progress_schema.py ` exits + 0 at every checkpoint. diff --git a/tests/scenarios/05-track-completion-handling.md b/tests/scenarios/05-track-completion-handling.md new file mode 100644 index 0000000..4abff28 --- /dev/null +++ b/tests/scenarios/05-track-completion-handling.md @@ -0,0 +1,69 @@ +# Scenario 05 — Completing a track's actual last exercise + +## Regression target +When the exercise just completed is the curriculum's actual last exercise, +there's nothing left to promote to `current_exercise`/`next_up`. `SKILL.md` +Step 6b point 8 requires: set `next_up` to `null`, treat this as track +completion, congratulate the learner concretely on what the whole track +unlocked, and offer `/tracks` or `/new-track` instead of the standard +"Exercise complete. What next?" four-option prompt. + +## Setup +Scratch directory with `.sage-progress.json`: +```json +{ + "active_track": "python-basics", + "tracks": { + "python-basics": { + "phase": 2, + "completed_exercises": [ + "P0-fizzbuzz", "P0-temp-converter", "P0-word-counter", + "P1-list-ops", "P1-comprehension-refactor", "P1-json-roundtrip", + "P2-task-class", "P2-task-list" + ], + "current_topic": "Combining classes with json persistence", + "current_exercise": "P2-persistent-tasklist", + "next_up": null, + "observations": "Strong OOP fundamentals, fast through Phase 2.", + "topic_confidence": {"classes": "solid"}, + "review_due": [], + "axis_overrides": {}, + "low_hint_streak": 2, + "high_hint_streak": 0, + "last_session": "", + "hint_count": 0 + } + } +} +``` +(`P2-persistent-tasklist` is `python-basics.md`'s actual final exercise — +`next_up` is already `null` because nothing follows it.) +Valid `.sage-profile.md`. + +## Script +1. Learner works `P2-persistent-tasklist` — a `TaskList` extended to + save/load itself via JSON, per the curriculum's exercise description. + Submit a genuinely correct solution so `verify` (`python {file}`) passes + cleanly; the point of this scenario is the completion-handling branch, not + the failure path. +2. Observe what Sage does immediately after the passing verify run. + +## Assertions + +- `[mechanical]` `P2-persistent-tasklist` is added to `completed_exercises`. +- `[mechanical]` `next_up` remains `null` and `current_exercise` is NOT + advanced to some invented exercise name — there is nothing after this one + in the curriculum. +- `[behavioral]` Sage did NOT present the standard "Exercise complete. What + next?" AskUserQuestion with its four usual options (save/refactor/review/ + bonus challenge). +- `[behavioral]` Sage congratulated the learner concretely on what the whole + track unlocked — referencing the actual destination project (Taskwright) + and specifically what these exercises built toward it (e.g. `Task`/ + `TaskList` classes plus JSON persistence being Taskwright's domain model + and persistence layer), not a generic "great job, track complete." +- `[behavioral]` Sage then offered `/tracks` (start another existing track) + or `/new-track` (build a custom one) as the next step. +- `[mechanical]` `python3 tests/check_progress_schema.py ` exits + 0 on the final state (the `next_up: null` check should surface as a WARN + telling the grader to confirm this manually, not a FAIL). diff --git a/tests/test_check_progress_schema.py b/tests/test_check_progress_schema.py new file mode 100644 index 0000000..51b491c --- /dev/null +++ b/tests/test_check_progress_schema.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Regression tests for check_progress_schema.py itself. + +The checker was shipped with a bug (a `warn()` call hardcoded to always +report PASS -- see CHANGELOG [Unreleased]) that only surfaced because a +Tier 2 live scenario happened to exercise that exact branch. That's too +expensive a way to catch a bug in the *checker*, which is plain +deterministic code. These fixture-driven tests pin the checker's expected +exit code (and, for FAILs, which check should fire) against known-good and +known-bad `.sage-progress.json` files under tests/fixtures/, so a future +edit to the checker gets caught by `python3 -m unittest` in seconds -- +no agent, no tokens, CI-friendly. + +Run: python3 tests/test_check_progress_schema.py +""" +import os +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +CHECKER = os.path.join(HERE, "check_progress_schema.py") +FIXTURES = os.path.join(HERE, "fixtures") + + +def run_checker(fixture_name): + fixture_dir = os.path.join(FIXTURES, fixture_name) + result = subprocess.run( + [sys.executable, CHECKER, fixture_dir], + capture_output=True, text=True, + ) + return result.returncode, result.stdout + + +class TestValidFixtures(unittest.TestCase): + def test_valid_exits_zero(self): + code, out = run_checker("valid") + self.assertEqual(code, 0, out) + self.assertNotIn("[FAIL]", out) + + def test_valid_track_complete_exits_zero_with_note(self): + code, out = run_checker("valid-track-complete") + self.assertEqual(code, 0, out) + self.assertNotIn("[FAIL]", out) + # Regression guard: this NOTE must actually render, not be silently + # swallowed the way the pre-fix warn(label, True, ...) call was. + self.assertIn("[NOTE]", out) + self.assertIn("next_up is null", out) + + +class TestInvalidFixtures(unittest.TestCase): + def test_bad_topic_key_fails(self): + code, out = run_checker("invalid-topic-key") + self.assertEqual(code, 1, out) + self.assertIn("kebab-case", out) + self.assertIn("[FAIL]", out) + + def test_overlapping_streaks_fails(self): + code, out = run_checker("invalid-streak-overlap") + self.assertEqual(code, 1, out) + self.assertIn("mutually exclusive", out) + + def test_dangling_review_due_fails(self): + code, out = run_checker("invalid-review-due-dangling") + self.assertEqual(code, 1, out) + self.assertIn("review_due entries are a subset", out) + + def test_missing_profile_fails(self): + code, out = run_checker("invalid-missing-profile") + self.assertEqual(code, 1, out) + self.assertIn(".sage-profile.md", out) + + def test_malformed_json_fails(self): + code, out = run_checker("invalid-bad-json") + self.assertEqual(code, 1, out) + self.assertIn("valid JSON", out) + + +if __name__ == "__main__": + unittest.main()