diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..76af41c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Summary + +- + +## Testing + +- + +## Regression Protection + +- [ ] If this fixes a bug, I added or updated a regression test that would fail on the old behavior. +- [ ] The regression test is linked to the issue, PR, or incident it protects. +- [ ] If no regression test was added for a bug fix, I explained why in this PR. diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 0000000..3b2495c --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,59 @@ +name: PR checks + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + quality: + name: Tests and regression protection + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r backend/requirements.txt -r mcp_server/requirements.txt pytest httpx + + - name: Run backend tests + run: python -m pytest + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: client/package-lock.json + + - name: Install client dependencies + working-directory: client + run: npm ci + + - name: Run client chat tests + working-directory: client + run: npm run test:chat + + - name: Run client Electron tests + working-directory: client + run: npm run test:electron + + - name: Build client + working-directory: client + run: npm run build + + - name: Run explicit regression suite + run: python scripts/run_regression_checks.py diff --git a/README.md b/README.md index 87fad45..8b1d617 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,17 @@ cd client npm run build ``` +Run the explicit regression protection suite: + +```bash +python3 scripts/run_regression_checks.py +``` + +Backend regressions live in `tests/regressions/`. Client regressions live in +`client/tests/regressions/`. When a PR fixes a bug, add a regression test there +or update an existing one so the old failure mode cannot be reintroduced +silently. Name the test after the issue, PR, or incident when possible. + The tests use fake CLI agent fixtures so execution flow can be verified without hosted models or external agent tools. For the first public release gate, see diff --git a/client/package.json b/client/package.json index 358f13a..e08bf0d 100644 --- a/client/package.json +++ b/client/package.json @@ -16,6 +16,7 @@ "build:electron": "tsc -p tsconfig.electron.json", "test:chat": "node --test tests/app-state.test.mjs tests/chat-mode.test.mjs tests/embedded-scroll.test.mjs tests/workspace-attention.test.mjs tests/session-utils.test.mjs", "test:electron": "npm run build:electron && node --test tests/platform.test.mjs tests/input-sequencing.test.mjs tests/pty-write.test.mjs tests/agent-context.test.mjs tests/agent-routing.test.mjs tests/agent-skills.test.mjs tests/agent-mcp.test.mjs tests/athena-cli.test.mjs tests/agent-sessions.test.mjs tests/terminal-activity.test.mjs tests/terminal-env.test.mjs tests/external-links.test.mjs tests/terminal-restore.test.mjs tests/launch-state.test.mjs tests/terminal-launch.test.mjs tests/terminal-buffer.test.mjs tests/terminal-output-ack.test.mjs tests/file-prefix.test.mjs tests/ttl-cache.test.mjs tests/control-access.test.mjs tests/terminal-input.test.mjs tests/disk-guard.test.mjs tests/memory-guard.test.mjs tests/update-mode.test.mjs", + "test:regression": "node tests/run-regressions.mjs", "start": "electron .", "package": "npm run build && electron-builder --dir", "dist": "npm run build && electron-builder" diff --git a/client/tests/regressions/README.md b/client/tests/regressions/README.md new file mode 100644 index 0000000..bad1710 --- /dev/null +++ b/client/tests/regressions/README.md @@ -0,0 +1,12 @@ +# Client Regression Tests + +Put permanent Electron, browser-state, and UI logic regression tests here when +fixing client bugs. + +Each test should: + +- Reproduce the old failure mode as directly as possible. +- Assert the behavior that must never regress. +- Include the issue or PR number in the test name or a short comment. + +Use names like `issue-1234-terminal-restore.test.mjs`. diff --git a/client/tests/run-regressions.mjs b/client/tests/run-regressions.mjs new file mode 100644 index 0000000..a830021 --- /dev/null +++ b/client/tests/run-regressions.mjs @@ -0,0 +1,54 @@ +import { readdir } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import path from 'node:path'; +import process from 'node:process'; + +const root = path.resolve(import.meta.dirname, 'regressions'); + +async function collectTests(dir) { + let entries; + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch (error) { + if (error?.code === 'ENOENT') { + return []; + } + throw error; + } + + const tests = []; + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + tests.push(...await collectTests(fullPath)); + } else if (/\.test\.m?js$/.test(entry.name)) { + tests.push(fullPath); + } + } + return tests.sort(); +} + +const tests = await collectTests(root); + +if (tests.length === 0) { + console.log(`No client regression tests found under ${root}.`); + process.exit(0); +} + +const child = spawn(process.execPath, ['--test', ...tests], { + cwd: path.resolve(import.meta.dirname, '..'), + stdio: 'inherit', +}); + +child.on('exit', (code, signal) => { + if (signal) { + console.error(`Client regression tests stopped by signal ${signal}.`); + process.exit(1); + } + process.exit(code ?? 1); +}); + +child.on('error', (error) => { + console.error(`Failed to start client regression tests: ${error.message}`); + process.exit(1); +}); diff --git a/scripts/run_regression_checks.py b/scripts/run_regression_checks.py new file mode 100644 index 0000000..1745763 --- /dev/null +++ b/scripts/run_regression_checks.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +BACKEND_REGRESSION_DIR = ROOT / "tests" / "regressions" +CLIENT_DIR = ROOT / "client" + + +def has_python_regression_tests() -> bool: + if not BACKEND_REGRESSION_DIR.exists(): + return False + return any( + path.name.startswith("test_") and path.suffix == ".py" + for path in BACKEND_REGRESSION_DIR.rglob("*.py") + ) + + +def run_step(label: str, command: list[str], cwd: Path) -> int: + print(f"\n==> {label}", flush=True) + print(f"$ {' '.join(command)}", flush=True) + return subprocess.run(command, cwd=cwd).returncode + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run the permanent regression checks that protect fixed bugs." + ) + parser.add_argument( + "--python-only", + action="store_true", + help="Run only backend/Python regression tests.", + ) + parser.add_argument( + "--client-only", + action="store_true", + help="Run only client/Electron regression tests.", + ) + args = parser.parse_args() + + if args.python_only and args.client_only: + parser.error("--python-only and --client-only cannot be used together") + + failures = 0 + + if not args.client_only: + if has_python_regression_tests(): + failures += run_step( + "Backend regression tests", + [sys.executable, "-m", "pytest", str(BACKEND_REGRESSION_DIR)], + ROOT, + ) + else: + print("\n==> Backend regression tests") + print(f"No Python regression tests found under {BACKEND_REGRESSION_DIR}.") + + if not args.python_only: + failures += run_step( + "Client regression tests", + ["npm", "run", "test:regression"], + CLIENT_DIR, + ) + + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/regressions/README.md b/tests/regressions/README.md new file mode 100644 index 0000000..2166091 --- /dev/null +++ b/tests/regressions/README.md @@ -0,0 +1,11 @@ +# Backend Regression Tests + +Put permanent backend regression tests here when fixing production bugs. + +Each test should: + +- Reproduce the old failure mode as directly as possible. +- Assert the behavior that must never regress. +- Include the issue or PR number in the test name or a short comment. + +Use names like `test_issue_1234_recall_refresh_handles_missing_cache.py`.