From 4e0b04f28ccb052ace83c901499df53d525e01f1 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:55:25 +0200 Subject: [PATCH 1/6] Characterize the argh CLI before migrating to cw 31 argv-level cases recorded from the current argh-based console script: the whole help surface (top level + every subcommand), a happy path per subcommand, the short/long/`=` option spellings argh derives, seven usage errors that must keep exit code 2, and argh's own no-argument behaviour (usage on *stdout*, exit 0 -- which plain argparse does not do). Recorded first, on purpose: this is the baseline the migration is measured against, so it has to exist in history before any source changes. The corpus deliberately excludes `--format json` and every `install-skills` happy path. Both print the *resolved* project root, and a committed golden that embeds an absolute path cannot be replayed on another machine. Their grammar is still pinned by the tier-2 usage line of the corresponding `--help` case. Claude-Session: https://claude.ai/code/session_01K6LB3AwUmKDxaFNZ2NqPGr --- misc/cli_cases.txt | 60 +++++++ misc/cli_golden.json | 379 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 misc/cli_cases.txt create mode 100644 misc/cli_golden.json diff --git a/misc/cli_cases.txt b/misc/cli_cases.txt new file mode 100644 index 0000000..b50198b --- /dev/null +++ b/misc/cli_cases.txt @@ -0,0 +1,60 @@ +# opsward CLI characterization corpus -- see misc/README-cli-parity.md +# +# Recorded from the argh-based CLI *before* the cw migration and replayed after, so a +# dispatcher swap that changes anything a shell can see fails loudly. +# +# Rules for adding a case: +# * paths are relative to the repo root (characterize/replay run with cwd=repo root); +# * no case may print an absolute path -- the golden is committed and must be portable; +# * no case may write to the filesystem or reach the network; +# * no case may need an optional extra (`find`'s toolery), so the corpus replays on a +# bare install. `find --help` still pins find's grammar at tier 2. + +# -- tier 3: the whole help surface (top level + every subcommand) -- +--help +diagnose --help +generate --help +maintain --help +recommend --help +install-skills --help +find --help + +# -- tier 1: one happy path per subcommand (all dry-run / read-only) -- +diagnose tests/fixtures/bare_project +diagnose tests/fixtures/python_project +generate tests/fixtures/bare_project +maintain tests/fixtures/stale_project +recommend tests/fixtures/python_project + +# -- tier 1: option grammar. argh derives a short flag per parameter and dashes the long +# one; both spellings, the "=" form and repeated positionals are all pinned here. +diagnose tests/fixtures/bare_project --min-score 0 +diagnose tests/fixtures/bare_project --min-score=0 +diagnose tests/fixtures/bare_project -m 0 +diagnose --verbose tests/fixtures/python_project +diagnose -v tests/fixtures/python_project +# NOTE: `--format json` is deliberately absent. Its output embeds the *resolved* +# (absolute) project root, which would put a machine path in a committed golden and +# make it unreplayable anywhere else. tests/test_cli.py covers the JSON surface. +diagnose tests/fixtures/bare_project --format json-ish-typo --min-score 0 +diagnose tests/fixtures/bare_project tests/fixtures/python_project --min-score 0 +generate tests/fixtures/bare_project --agents-md --hooks +generate tests/fixtures/bare_project -a +# NOTE: `install-skills` has no happy-path case for the same reason as `--format json`: +# every one of its lines names the *resolved* target directory. `install-skills --help` +# pins its grammar at tier 2; tests/ covers its behaviour. +install-skills --target +install-skills -t + +# -- tier 1: usage errors. These must keep argparse's exit code 2; a dispatcher that +# starts exiting 0 on a bad command line breaks every CI step that checks it. +[] +nosuchcommand +diagnose --nosuchflag +diagnose --min-score +diagnose tests/fixtures/bare_project --min-score notanint +generate --write --nosuchflag tests/fixtures/bare_project +find + +# -- tier 1: a runtime error path (opsward's own sys.exit(2)) -- +diagnose ./no/such/directory/opsward-characterization diff --git a/misc/cli_golden.json b/misc/cli_golden.json new file mode 100644 index 0000000..14d113d --- /dev/null +++ b/misc/cli_golden.json @@ -0,0 +1,379 @@ +{ + "cases": [ + { + "argv": [ + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\n\npositional arguments:\n {diagnose,generate,maintain,recommend,install-skills,find}\n diagnose Diagnose the AI agent setup of one or more projects. :param project_roots:\n one or more paths to project directories :param format: output format —\n 'text' or 'json' :param verbose: show additional detail in text output\n :param min_score: minimum overall score to pass (exit 0); below this exits\n 1\n generate Generate missing AI setup artifacts for one or more projects. By default,\n shows what would be created (dry run). Use --write to actually write\n files. Existing files are never overwritten. :param project_roots: one or\n more paths to project directories :param write: actually write files\n (default: dry run) :param format: output format — 'text' or 'json' :param\n agents_md: also generate AGENTS.md (cross-platform agent instructions)\n :param hooks: also generate starter hook scripts\n maintain Check for stale references, out-of-sync docs, and other drift. :param\n project_roots: one or more paths to project directories :param format:\n output format — 'text' or 'json'\n recommend Recommend ecosystem skills based on the project's tech stack. Analyzes\n dependencies and suggests skills from the community catalog. :param\n project_roots: one or more paths to project directories :param format:\n output format — 'text' or 'json'\n install-skills Install opsward's Claude Code skills (and agents) into a project or\n globally. By default, shows what would be created (dry run). Use --write\n to actually write files. Existing files are never overwritten. :param\n target: project directory (ignored when --global is set) :param\n global_install: install into ~/.claude/ instead of the project :param\n agents: also install agent definitions (default: True) :param write:\n actually write files (default: dry run)\n find Find assets (skills, agents, docs) across one or more projects by QUERY.\n Cross-repo asset discovery via the toolery package (install with: pip\n install 'opsward[discovery]'). :param query: search query :param\n project_roots: one or more project directories (default: current dir)\n :param kinds: comma-separated asset kinds — skill, agent, doc :param\n semantic: use toolery's ir semantic backend (needs toolery[ir]) :param\n limit: max results to show\n\noptions:\n -h, --help show this help message and exit\n", + "tier": 3, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ..." + }, + { + "argv": [ + "diagnose", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]\n\nDiagnose the AI agent setup of one or more projects.\n\n:param project_roots: one or more paths to project directories\n:param format: output format — 'text' or 'json'\n:param verbose: show additional detail in text output\n:param min_score: minimum overall score to pass (exit 0); below this exits 1\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -f FORMAT, --format FORMAT\n 'text'\n -v, --verbose False\n -m MIN_SCORE, --min-score MIN_SCORE\n 80\n", + "tier": 3, + "usage": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]" + }, + { + "argv": [ + "generate", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward generate [-h] [-w] [-f FORMAT] [-a] [--hooks] [project-roots ...]\n\nGenerate missing AI setup artifacts for one or more projects.\n\nBy default, shows what would be created (dry run). Use --write to\nactually write files. Existing files are never overwritten.\n\n:param project_roots: one or more paths to project directories\n:param write: actually write files (default: dry run)\n:param format: output format — 'text' or 'json'\n:param agents_md: also generate AGENTS.md (cross-platform agent instructions)\n:param hooks: also generate starter hook scripts\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -w, --write False\n -f FORMAT, --format FORMAT\n 'text'\n -a, --agents-md False\n --hooks False\n", + "tier": 3, + "usage": "usage: opsward generate [-h] [-w] [-f FORMAT] [-a] [--hooks] [project-roots ...]" + }, + { + "argv": [ + "maintain", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward maintain [-h] [-f FORMAT] [project-roots ...]\n\nCheck for stale references, out-of-sync docs, and other drift.\n\n:param project_roots: one or more paths to project directories\n:param format: output format — 'text' or 'json'\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -f FORMAT, --format FORMAT\n 'text'\n", + "tier": 3, + "usage": "usage: opsward maintain [-h] [-f FORMAT] [project-roots ...]" + }, + { + "argv": [ + "recommend", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward recommend [-h] [-f FORMAT] [project-roots ...]\n\nRecommend ecosystem skills based on the project's tech stack.\n\nAnalyzes dependencies and suggests skills from the community catalog.\n\n:param project_roots: one or more paths to project directories\n:param format: output format — 'text' or 'json'\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -f FORMAT, --format FORMAT\n 'text'\n", + "tier": 3, + "usage": "usage: opsward recommend [-h] [-f FORMAT] [project-roots ...]" + }, + { + "argv": [ + "install-skills", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]\n\nInstall opsward's Claude Code skills (and agents) into a project or globally.\n\nBy default, shows what would be created (dry run). Use --write to\nactually write files. Existing files are never overwritten.\n\n:param target: project directory (ignored when --global is set)\n:param global_install: install into ~/.claude/ instead of the project\n:param agents: also install agent definitions (default: True)\n:param write: actually write files (default: dry run)\n\noptions:\n -h, --help show this help message and exit\n -t TARGET, --target TARGET\n '.'\n -g, --global-install False\n -a, --agents True\n -w, --write False\n", + "tier": 3, + "usage": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]" + }, + { + "argv": [ + "find", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...]\n\nFind assets (skills, agents, docs) across one or more projects by QUERY.\n\nCross-repo asset discovery via the toolery package\n(install with: pip install 'opsward[discovery]').\n\n:param query: search query\n:param project_roots: one or more project directories (default: current dir)\n:param kinds: comma-separated asset kinds — skill, agent, doc\n:param semantic: use toolery's ir semantic backend (needs toolery[ir])\n:param limit: max results to show\n\npositional arguments:\n query -\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -k KINDS, --kinds KINDS\n 'skill,agent'\n -s, --semantic False\n -l LIMIT, --limit LIMIT\n 10\n", + "tier": 3, + "usage": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...]" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found → run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json — run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/python_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found → add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found → add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity → name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) → flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md → link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json — no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found — run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "generate", + "tests/fixtures/bare_project" + ], + "returncode": 0, + "stderr": "", + "stdout": "bare_project: 12 artifact(s)\n\n Would create CLAUDE.md\n Would create misc/docs/docs_guide.md\n Would create misc/docs/architecture.md\n Would create misc/docs/known_issues.md\n Would create misc/docs/conventions.md\n Would create misc/docs/roadmap.md\n Would create misc/docs/decisions/0000-template.md\n Would create .claude/agents/setup-auditor.md\n Would create .claude/skills/opsward/SKILL.md\n Would create .claude/skills/opsward-diagnose/SKILL.md\n Would create .claude/skills/opsward-generate/SKILL.md\n Would create .claude/skills/opsward-maintain/SKILL.md\n\nDry run — pass --write to create files.\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "maintain", + "tests/fixtures/stale_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "stale_project: 5 issue(s)\n\n [stale_path] CLAUDE.md references `src/core.py` but it does not exist\n [stale_path] CLAUDE.md references `src/utils.py` but it does not exist\n [sync_issue] `unlisted_doc.md` exists in docs/ but is not listed in docs_guide.md\n + | [unlisted_doc.md](unlisted_doc.md) | |\n [sync_issue] docs_guide.md references `deleted_doc.md` but the file does not exist\n [empty_doc] `conventions.md` appears to be an empty stub (37 bytes)\n\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "recommend", + "tests/fixtures/python_project" + ], + "returncode": 0, + "stderr": "", + "stdout": "python_project: no skill recommendations\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--min-score", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found → run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json — run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--min-score=0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found → run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json — run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "-m", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found → run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json — run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "--verbose", + "tests/fixtures/python_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found → add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found → add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity → name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) → flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md → link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json — no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found — run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n\nDetailed inventory:\n Skills: 1\n - my-skill (SKILL.md: yes)\n Agents: 1\n - code-reviewer\n Rules: 1\n - no-print\n Docs: 2\n - architecture (54 bytes)\n - docs_guide (44 bytes)\n Hooks: yes\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "-v", + "tests/fixtures/python_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found → add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found → add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity → name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) → flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md → link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json — no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found — run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n\nDetailed inventory:\n Skills: 1\n - my-skill (SKILL.md: yes)\n Agents: 1\n - code-reviewer\n Rules: 1\n - no-print\n Docs: 2\n - architecture (54 bytes)\n - docs_guide (44 bytes)\n Hooks: yes\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--format", + "json-ish-typo", + "--min-score", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found → run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json — run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "tests/fixtures/python_project", + "--min-score", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found → run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json — run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n\n============================================================\n\nDiagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found → add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found → add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity → name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) → flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md → link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json — no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found — run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "generate", + "tests/fixtures/bare_project", + "--agents-md", + "--hooks" + ], + "returncode": 0, + "stderr": "", + "stdout": "bare_project: 14 artifact(s)\n\n Would create CLAUDE.md\n Would create misc/docs/docs_guide.md\n Would create misc/docs/architecture.md\n Would create misc/docs/known_issues.md\n Would create misc/docs/conventions.md\n Would create misc/docs/roadmap.md\n Would create misc/docs/decisions/0000-template.md\n Would create .claude/agents/setup-auditor.md\n Would create .claude/skills/opsward/SKILL.md\n Would create .claude/skills/opsward-diagnose/SKILL.md\n Would create .claude/skills/opsward-generate/SKILL.md\n Would create .claude/skills/opsward-maintain/SKILL.md\n Would create AGENTS.md\n Would create .claude/hooks.json\n\nDry run — pass --write to create files.\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "generate", + "tests/fixtures/bare_project", + "-a" + ], + "returncode": 0, + "stderr": "", + "stdout": "bare_project: 13 artifact(s)\n\n Would create CLAUDE.md\n Would create misc/docs/docs_guide.md\n Would create misc/docs/architecture.md\n Would create misc/docs/known_issues.md\n Would create misc/docs/conventions.md\n Would create misc/docs/roadmap.md\n Would create misc/docs/decisions/0000-template.md\n Would create .claude/agents/setup-auditor.md\n Would create .claude/skills/opsward/SKILL.md\n Would create .claude/skills/opsward-diagnose/SKILL.md\n Would create .claude/skills/opsward-generate/SKILL.md\n Would create .claude/skills/opsward-maintain/SKILL.md\n Would create AGENTS.md\n\nDry run — pass --write to create files.\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "install-skills", + "--target" + ], + "returncode": 2, + "stderr": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]\nopsward install-skills: error: argument -t/--target: expected one argument\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w] opsward install-skills: error: argument -t/--target: expected one argument" + }, + { + "argv": [ + "install-skills", + "-t" + ], + "returncode": 2, + "stderr": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]\nopsward install-skills: error: argument -t/--target: expected one argument\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w] opsward install-skills: error: argument -t/--target: expected one argument" + }, + { + "argv": [], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\n", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ..." + }, + { + "argv": [ + "nosuchcommand" + ], + "returncode": 2, + "stderr": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\nopsward: error: argument {diagnose,generate,maintain,recommend,install-skills,find}: invalid choice: 'nosuchcommand' (choose from diagnose, generate, maintain, recommend, install-skills, find)\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... opsward: error: argument {diagnose,generate,maintain,recommend,install-skills,find}: invalid choice: 'nosuchcommand' (choose from diagnose, generate, maintain, recommend, install-skills, find)" + }, + { + "argv": [ + "diagnose", + "--nosuchflag" + ], + "returncode": 2, + "stderr": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\nopsward: error: unrecognized arguments: --nosuchflag\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... opsward: error: unrecognized arguments: --nosuchflag" + }, + { + "argv": [ + "diagnose", + "--min-score" + ], + "returncode": 2, + "stderr": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]\nopsward diagnose: error: argument -m/--min-score: expected one argument\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...] opsward diagnose: error: argument -m/--min-score: expected one argument" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--min-score", + "notanint" + ], + "returncode": 2, + "stderr": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]\nopsward diagnose: error: argument -m/--min-score: invalid int value: 'notanint'\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...] opsward diagnose: error: argument -m/--min-score: invalid int value: 'notanint'" + }, + { + "argv": [ + "generate", + "--write", + "--nosuchflag", + "tests/fixtures/bare_project" + ], + "returncode": 2, + "stderr": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\nopsward: error: unrecognized arguments: --nosuchflag\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... opsward: error: unrecognized arguments: --nosuchflag" + }, + { + "argv": [ + "find" + ], + "returncode": 2, + "stderr": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...]\nopsward find: error: the following arguments are required: query\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...] opsward find: error: the following arguments are required: query" + }, + { + "argv": [ + "diagnose", + "./no/such/directory/opsward-characterization" + ], + "returncode": 2, + "stderr": "Error: ./no/such/directory/opsward-characterization is not a directory\n", + "stdout": "", + "tier": 1, + "usage": "" + } + ], + "cw_golden": 1, + "env": { + "COLUMNS": "100", + "NO_COLOR": "1", + "PYTHONHASHSEED": "0", + "PYTHONIOENCODING": "utf-8", + "PYTHONUTF8": "1", + "TERM": "dumb" + }, + "newlines": "lf", + "note": "argh 0.31.3, opsward 57b10ae, pre-cw-migration", + "prog": [ + "opsward" + ], + "recorded_with": { + "implementation": "CPython", + "platform": "Darwin", + "python": "3.12.12" + } +} From 4f415d4c957839f97032e64b9a1db7b01e87a74a Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:56:06 +0200 Subject: [PATCH 2/6] Wave 0: swap argh for cw's compat shim (one line) `from cw import compat as argh` -- nothing else changes. Replayed against the golden recorded in the previous commit: 31/31 identical (--strict-help, so the --help bodies are asserted too) no --help changes 135 passed Kept as its own commit because it is the evidence for the claim: the compat shim is a drop-in, and any behaviour difference in the next commit is attributable to the move to cw's own API rather than to leaving argh. Claude-Session: https://claude.ai/code/session_01K6LB3AwUmKDxaFNZ2NqPGr --- opsward/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opsward/__main__.py b/opsward/__main__.py index 4947011..e710eca 100644 --- a/opsward/__main__.py +++ b/opsward/__main__.py @@ -1,6 +1,6 @@ """CLI entry point: python -m opsward.""" -import argh +from cw import compat as argh from opsward.cli import _dispatch_funcs From 4de8a8b3918ef15b90562fb16f1e652f2c8f1450 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:59:04 +0200 Subject: [PATCH 3/6] Wave 2: dispatch through cw's own API, and pin the CLI surface in CI `__main__.py` now calls `cw.dispatch(_dispatch_funcs)` directly rather than going through the compat shim, and `argh` is replaced by `cw>=0.1,<0.2` in `dependencies`. `rg -n argh opsward/` returns nothing. Replay against the pre-migration golden, with the --help bodies asserted: 31/31 identical no --help changes 136 passed `prog=` is deliberately NOT passed. Letting argparse derive the program name from sys.argv[0] is what keeps `python -m opsward` reporting `__main__.py` and the console script reporting `opsward` -- pinning `prog='opsward'` would have changed the `-m` form's usage line, which is the invocation the module docstring documents and the whole existing test suite uses. New `tests/test_cli_parity.py` replays the golden as part of the ordinary test run, so this cannot rot: a refactor here, or a future cw release, that changes any exit code, any byte of stdout/stderr, or any `usage:` line fails the suite. Verified it can fail -- forcing a different naming convention produced a red test naming the exact flags that moved. The full --help *body* is asserted only when the running CPython matches the one that recorded the golden, because CPython rewrites its own help rendering between versions and the matrix here is 3.10 + 3.12. Docs updated to describe the pinned surface and how to re-record it. Claude-Session: https://claude.ai/code/session_01K6LB3AwUmKDxaFNZ2NqPGr --- AGENTS.md | 4 +-- CLAUDE.md | 34 +++++++++++++++------- misc/cli_cases.txt | 3 +- misc/docs/architecture.md | 2 +- opsward/__main__.py | 17 +++++++++-- pyproject.toml | 2 +- tests/test_cli_parity.py | 61 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 105 insertions(+), 18 deletions(-) create mode 100644 tests/test_cli_parity.py diff --git a/AGENTS.md b/AGENTS.md index 442c643..9a4045f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ Diagnose, generate, and maintain the AI agent setup of your projects — CLAUDE.md, skills, subagents, rules, and supporting docs. -**Tech stack:** Python 3.10+, `argh` (CLI), `string.Template` (templates), `dataclasses`. Lightweight by design — no heavy deps. +**Tech stack:** Python 3.10+, `cw` (CLI), `string.Template` (templates), `dataclasses`. Lightweight by design — no heavy deps. ## Build & Test Commands @@ -25,7 +25,7 @@ pytest --doctest-modules opsward/ # doctests embedded in the package - `generate.py` — template rendering + file generation (never overwrites; dry-run by default) - `maintain.py` — staleness / drift detection - `recommend.py` — tech-stack → curated ecosystem-skill recommendations - - `cli.py` / `__main__.py` — `argh` dispatch + - `cli.py` / `__main__.py` — `cw` dispatch (surface pinned by `tests/test_cli_parity.py`) - `base.py` — dataclasses (`ScanResult`, `DiagnosisReport`, `ComponentScore`, …) - `data/templates/` — bundled templates (`shared/`, `python/`, `jsts/`), accessed via `importlib.resources` - `tests/` — pytest tests + sample-project fixtures under `tests/fixtures/` diff --git a/CLAUDE.md b/CLAUDE.md index fae942a..2d8e0b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ There is also an `ow` PyPI package that is a thin re-export shim for `opsward`. ## Tech Stack - **Language:** Python 3.10+ -- **CLI:** `argh` for dispatching functions to CLI commands +- **CLI:** `cw` for dispatching functions to CLI commands (MIT; reproduces argh's grammar) - **Templates:** String-based (`string.Template` or simple f-string/jinja2-minimal) — keep deps light - **Data structures:** `dataclasses` for models, `Mapping`/`MutableMapping` where storage is involved - **File access:** `importlib.resources.files` for bundled templates in `opsward/data/` @@ -39,7 +39,7 @@ Key docs to read before starting work: - `generate.py` — template rendering and file generation (never overwrites without confirmation). Also provides `generate_skills()` for targeted skill/agent installation. - `maintain.py` — staleness detection, update suggestions, drift analysis - `recommend.py` — map detected tech-stack signals (deps, frameworks) to curated ecosystem skill recommendations - - `cli.py` — argh-based CLI entry point + - `cli.py` — CLI command functions (the `_dispatch_funcs` SSOT) - `util.py` — internal helpers (underscore-prefixed) - `data/` — bundled package resources (accessed via `importlib.resources.files`) - `templates/` — generation templates organized by target project type @@ -84,24 +84,38 @@ opsward install-skills --global-install --write # Install into ~/.claude/ ### CLI Pattern -Follow the argh SSOT dispatch pattern: +Follow the SSOT dispatch pattern: `cli.py` owns the command list, `__main__.py` +only runs it. ```python # In cli.py -_dispatch_funcs = [diagnose, generate, maintain, recommend, install_skills] - -if __name__ == "__main__": - import argh - argh.dispatch_commands(_dispatch_funcs) +_dispatch_funcs = [diagnose, generate, maintain, recommend, install_skills, find] ``` ```python # In __main__.py +import cw from opsward.cli import _dispatch_funcs -import argh -argh.dispatch_commands(_dispatch_funcs) + +def main(): + raise SystemExit(cw.dispatch(_dispatch_funcs)) +``` + +Do not pass `prog=` — leaving it to `argparse` is what keeps `python -m opsward` +reporting `__main__.py` and the console script reporting `opsward`. + +**The CLI surface is pinned.** `misc/cli_golden.json` records every argv in +`misc/cli_cases.txt` — exit code, stdout, stderr — and `tests/test_cli_parity.py` +replays it. Adding a command or a flag will fail that test; re-record with + +```bash +python -m cw.testing characterize opsward --cases misc/cli_cases.txt -o misc/cli_golden.json ``` +and put the resulting diff in the PR. Never re-record to make a red test green +without reading what changed. Cases must not print absolute paths — the golden is +committed and has to replay on someone else's machine. + ### Template Pattern Templates live in `opsward/data/templates/`. They are plain markdown files with `${variable}` placeholders (using `string.Template`). The generator reads them via `importlib.resources`, substitutes variables from the scan results, and writes to the target project. diff --git a/misc/cli_cases.txt b/misc/cli_cases.txt index b50198b..67384a7 100644 --- a/misc/cli_cases.txt +++ b/misc/cli_cases.txt @@ -1,4 +1,5 @@ -# opsward CLI characterization corpus -- see misc/README-cli-parity.md +# opsward CLI characterization corpus. Replayed by tests/test_cli_parity.py; +# re-recording procedure is in CLAUDE.md, "CLI Pattern". # # Recorded from the argh-based CLI *before* the cw migration and replayed after, so a # dispatcher swap that changes anything a shell can see fails loudly. diff --git a/misc/docs/architecture.md b/misc/docs/architecture.md index 02ac8e4..4f1535f 100644 --- a/misc/docs/architecture.md +++ b/misc/docs/architecture.md @@ -62,7 +62,7 @@ Generated artifacts include CLAUDE.md, docs, skills, agents, and rules. Future: | `generate.py` | Template loading + rendering | `ScanResult → list[GeneratedFile]` | | `maintain.py` | Drift/staleness detection | `ScanResult → list[MaintenanceSuggestion]` | | `base.py` | All dataclasses and type definitions | (no I/O) | -| `cli.py` | argh dispatch + pretty-printing | stdin/stdout | +| `cli.py` | command functions + pretty-printing | stdin/stdout | | `util.py` | Shared helpers | (internal) | ## Key Invariants diff --git a/opsward/__main__.py b/opsward/__main__.py index e710eca..de8fb11 100644 --- a/opsward/__main__.py +++ b/opsward/__main__.py @@ -1,12 +1,23 @@ -"""CLI entry point: python -m opsward.""" +"""CLI entry point: ``python -m opsward`` and the ``opsward`` console script. -from cw import compat as argh +The command list is :data:`opsward.cli._dispatch_funcs` -- one SSOT, no per-command +registration here. :func:`cw.dispatch` turns it into an ``argparse`` parser and runs it, +reproducing the grammar this CLI has always had (pinned by ``misc/cli_golden.json`` +and asserted by ``tests/test_cli_parity.py``). + +``prog`` is deliberately not passed: leaving it to ``argparse`` keeps the program name +derived from ``sys.argv[0]``, so ``python -m opsward`` still reports ``__main__.py`` and +the console script still reports ``opsward``, exactly as before. +""" + +import cw from opsward.cli import _dispatch_funcs def main(): - argh.dispatch_commands(_dispatch_funcs) + """Parse ``sys.argv`` and run the command it names; return its exit code.""" + raise SystemExit(cw.dispatch(_dispatch_funcs)) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 47a4a99..fbbcbb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Topic :: Software Development :: Documentation", ] dependencies = [ - "argh>=0.31", + "cw>=0.1,<0.2", ] [project.scripts] diff --git a/tests/test_cli_parity.py b/tests/test_cli_parity.py new file mode 100644 index 0000000..e5b2db7 --- /dev/null +++ b/tests/test_cli_parity.py @@ -0,0 +1,61 @@ +"""Assert opsward's command line has not moved since it was recorded. + +``misc/cli_golden.json`` was recorded from the argh-based CLI *before* the migration to +``cw``; ``misc/cli_cases.txt`` is the corpus it was recorded from. This test replays it +against whatever dispatcher is installed now, so neither a refactor here nor a new ``cw`` +release can change what a shell sees without a test going red. + +Two tiers are asserted (see :mod:`cw.testing`): + +* every case's exit code, stdout and stderr, verbatim; and +* the normalised ``usage:`` line of every ``--help`` case -- which names every option a + parser has, so a lost flag or a changed ``nargs`` shows up here. + +The full ``--help`` *body* is asserted only when the running CPython matches the one that +recorded the golden. CPython rewrites its own help rendering between versions (3.13 emits +``-f, --format FORMAT`` where 3.12 emits ``-f FORMAT, --format FORMAT``), and a golden +replayed across a version matrix would otherwise fail for something nobody caused. +""" + +import json +import shutil +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +GOLDEN = REPO_ROOT / "misc" / "cli_golden.json" + + +def _console_script() -> str: + """The installed ``opsward`` executable, however this environment lays it out.""" + found = shutil.which("opsward") + if found: + return found + # A venv whose bin/ is not on PATH -- common under `uv run` and tox. + for name in ("opsward", "opsward.exe"): + candidate = Path(sys.executable).parent / name + if candidate.exists(): + return str(candidate) + pytest.fail( + "the `opsward` console script is not installed; CLI parity cannot be checked. " + "Install the package (`pip install -e .`) before running this test." + ) + + +def _recorded_python() -> tuple: + """``(major, minor)`` of the CPython that recorded the golden.""" + recorded = json.loads(GOLDEN.read_text(encoding="utf-8"))["recorded_with"]["python"] + return tuple(int(part) for part in recorded.split(".")[:2]) + + +def test_cli_surface_is_unchanged(): + """Every recorded argv still produces the same exit code, stdout and stderr.""" + cw_testing = pytest.importorskip("cw.testing") + cw_testing.assert_replay( + GOLDEN, + prog=[_console_script()], + cwd=str(REPO_ROOT), + strict_help=_recorded_python() == sys.version_info[:2], + ) From e4c151fcdc05fcc0bb6231fea4ba34279b26da23 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:10:42 +0200 Subject: [PATCH 4/6] One CLI golden per CPython version in the CI matrix CI went red on Python 3.10, and it was not cw. argparse is stdlib and rewrites its own text between versions: * 3.12 stopped listing `nargs='*'` positionals among "the following arguments are required", so bare `opsward find` says `required: query` on 3.12 and `required: query, project-roots` on 3.10; * 3.12 also changed how `invalid choice` quotes the choices. Verified against plain argparse on 3.10/3.11/3.12/3.13 with no argh and no cw in the picture. Those two cases are the ONLY difference between the two recordings; argh produces them identically. A single golden asserted across a matrix fails for something nobody caused, which is the fastest way to teach a team to ignore a red parity test. So: `misc/cli_golden_py310.json` and `misc/cli_golden_py312.json`, and the test picks the one matching the running interpreter. A version with no recording fails loudly with instructions -- a parity test that quietly skips is worse than none. The 3.10 golden is recorded from *argh* too, not from the migrated code: a throwaway 3.10 venv over a worktree pinned at the pre-migration commit. It is a migration proof on both matrix versions, not a baseline taken after the fact. Replaying it against the cw code on 3.10 gives 31/31 identical, same as 3.12. `strict_help=True` unconditionally now. The gate on the recording interpreter is what makes that safe, and it upgrades the --help body from advisory to asserted. Claude-Session: https://claude.ai/code/session_01K6LB3AwUmKDxaFNZ2NqPGr --- CLAUDE.md | 28 +- misc/cli_golden_py310.json | 379 ++++++++++++++++++ ...{cli_golden.json => cli_golden_py312.json} | 0 opsward/__main__.py | 2 +- tests/test_cli_parity.py | 58 +-- 5 files changed, 436 insertions(+), 31 deletions(-) create mode 100644 misc/cli_golden_py310.json rename misc/{cli_golden.json => cli_golden_py312.json} (100%) diff --git a/CLAUDE.md b/CLAUDE.md index 2d8e0b7..3fb8b43 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,17 +104,31 @@ def main(): Do not pass `prog=` — leaving it to `argparse` is what keeps `python -m opsward` reporting `__main__.py` and the console script reporting `opsward`. -**The CLI surface is pinned.** `misc/cli_golden.json` records every argv in -`misc/cli_cases.txt` — exit code, stdout, stderr — and `tests/test_cli_parity.py` -replays it. Adding a command or a flag will fail that test; re-record with +**The CLI surface is pinned.** `misc/cli_golden_py.json` records every +argv in `misc/cli_cases.txt` — exit code, stdout, stderr, `usage:` line and the full +`--help` body — and `tests/test_cli_parity.py` replays the one matching the running +CPython. Adding a command or a flag will fail that test. Re-record **every** golden, +each on its own interpreter: ```bash -python -m cw.testing characterize opsward --cases misc/cli_cases.txt -o misc/cli_golden.json +python3.12 -m cw.testing characterize opsward --cases misc/cli_cases.txt \ + -o misc/cli_golden_py312.json ``` -and put the resulting diff in the PR. Never re-record to make a red test green -without reading what changed. Cases must not print absolute paths — the golden is -committed and has to replay on someone else's machine. +and put the resulting diff in the PR. Never re-record to make a red test green without +first reading what changed. + +Two rules for the corpus: + +- **No case may print an absolute path.** The goldens are committed and have to replay + on someone else's machine, so `--format json` and `install-skills` happy paths are + deliberately excluded (both name the *resolved* project root). +- **One golden per CPython version in the CI matrix.** `argparse` is stdlib and rewrites + its own text between versions — 3.12 stopped listing `nargs='*'` positionals among + "the following arguments are required", and changed how `invalid choice` quotes the + choices. Those are the only two cases that differ between the 3.10 and 3.12 + recordings. Adding a version to `[tool.wads.ci.testing]` means recording a golden for + it; the test fails loudly rather than skipping if one is missing. ### Template Pattern diff --git a/misc/cli_golden_py310.json b/misc/cli_golden_py310.json new file mode 100644 index 0000000..6ad045b --- /dev/null +++ b/misc/cli_golden_py310.json @@ -0,0 +1,379 @@ +{ + "cases": [ + { + "argv": [ + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\n\npositional arguments:\n {diagnose,generate,maintain,recommend,install-skills,find}\n diagnose Diagnose the AI agent setup of one or more projects. :param project_roots:\n one or more paths to project directories :param format: output format \u2014\n 'text' or 'json' :param verbose: show additional detail in text output\n :param min_score: minimum overall score to pass (exit 0); below this exits\n 1\n generate Generate missing AI setup artifacts for one or more projects. By default,\n shows what would be created (dry run). Use --write to actually write\n files. Existing files are never overwritten. :param project_roots: one or\n more paths to project directories :param write: actually write files\n (default: dry run) :param format: output format \u2014 'text' or 'json' :param\n agents_md: also generate AGENTS.md (cross-platform agent instructions)\n :param hooks: also generate starter hook scripts\n maintain Check for stale references, out-of-sync docs, and other drift. :param\n project_roots: one or more paths to project directories :param format:\n output format \u2014 'text' or 'json'\n recommend Recommend ecosystem skills based on the project's tech stack. Analyzes\n dependencies and suggests skills from the community catalog. :param\n project_roots: one or more paths to project directories :param format:\n output format \u2014 'text' or 'json'\n install-skills Install opsward's Claude Code skills (and agents) into a project or\n globally. By default, shows what would be created (dry run). Use --write\n to actually write files. Existing files are never overwritten. :param\n target: project directory (ignored when --global is set) :param\n global_install: install into ~/.claude/ instead of the project :param\n agents: also install agent definitions (default: True) :param write:\n actually write files (default: dry run)\n find Find assets (skills, agents, docs) across one or more projects by QUERY.\n Cross-repo asset discovery via the toolery package (install with: pip\n install 'opsward[discovery]'). :param query: search query :param\n project_roots: one or more project directories (default: current dir)\n :param kinds: comma-separated asset kinds \u2014 skill, agent, doc :param\n semantic: use toolery's ir semantic backend (needs toolery[ir]) :param\n limit: max results to show\n\noptions:\n -h, --help show this help message and exit\n", + "tier": 3, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ..." + }, + { + "argv": [ + "diagnose", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]\n\nDiagnose the AI agent setup of one or more projects.\n\n:param project_roots: one or more paths to project directories\n:param format: output format \u2014 'text' or 'json'\n:param verbose: show additional detail in text output\n:param min_score: minimum overall score to pass (exit 0); below this exits 1\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -f FORMAT, --format FORMAT\n 'text'\n -v, --verbose False\n -m MIN_SCORE, --min-score MIN_SCORE\n 80\n", + "tier": 3, + "usage": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]" + }, + { + "argv": [ + "generate", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward generate [-h] [-w] [-f FORMAT] [-a] [--hooks] [project-roots ...]\n\nGenerate missing AI setup artifacts for one or more projects.\n\nBy default, shows what would be created (dry run). Use --write to\nactually write files. Existing files are never overwritten.\n\n:param project_roots: one or more paths to project directories\n:param write: actually write files (default: dry run)\n:param format: output format \u2014 'text' or 'json'\n:param agents_md: also generate AGENTS.md (cross-platform agent instructions)\n:param hooks: also generate starter hook scripts\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -w, --write False\n -f FORMAT, --format FORMAT\n 'text'\n -a, --agents-md False\n --hooks False\n", + "tier": 3, + "usage": "usage: opsward generate [-h] [-w] [-f FORMAT] [-a] [--hooks] [project-roots ...]" + }, + { + "argv": [ + "maintain", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward maintain [-h] [-f FORMAT] [project-roots ...]\n\nCheck for stale references, out-of-sync docs, and other drift.\n\n:param project_roots: one or more paths to project directories\n:param format: output format \u2014 'text' or 'json'\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -f FORMAT, --format FORMAT\n 'text'\n", + "tier": 3, + "usage": "usage: opsward maintain [-h] [-f FORMAT] [project-roots ...]" + }, + { + "argv": [ + "recommend", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward recommend [-h] [-f FORMAT] [project-roots ...]\n\nRecommend ecosystem skills based on the project's tech stack.\n\nAnalyzes dependencies and suggests skills from the community catalog.\n\n:param project_roots: one or more paths to project directories\n:param format: output format \u2014 'text' or 'json'\n\npositional arguments:\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -f FORMAT, --format FORMAT\n 'text'\n", + "tier": 3, + "usage": "usage: opsward recommend [-h] [-f FORMAT] [project-roots ...]" + }, + { + "argv": [ + "install-skills", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]\n\nInstall opsward's Claude Code skills (and agents) into a project or globally.\n\nBy default, shows what would be created (dry run). Use --write to\nactually write files. Existing files are never overwritten.\n\n:param target: project directory (ignored when --global is set)\n:param global_install: install into ~/.claude/ instead of the project\n:param agents: also install agent definitions (default: True)\n:param write: actually write files (default: dry run)\n\noptions:\n -h, --help show this help message and exit\n -t TARGET, --target TARGET\n '.'\n -g, --global-install False\n -a, --agents True\n -w, --write False\n", + "tier": 3, + "usage": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]" + }, + { + "argv": [ + "find", + "--help" + ], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...]\n\nFind assets (skills, agents, docs) across one or more projects by QUERY.\n\nCross-repo asset discovery via the toolery package\n(install with: pip install 'opsward[discovery]').\n\n:param query: search query\n:param project_roots: one or more project directories (default: current dir)\n:param kinds: comma-separated asset kinds \u2014 skill, agent, doc\n:param semantic: use toolery's ir semantic backend (needs toolery[ir])\n:param limit: max results to show\n\npositional arguments:\n query -\n project-roots -\n\noptions:\n -h, --help show this help message and exit\n -k KINDS, --kinds KINDS\n 'skill,agent'\n -s, --semantic False\n -l LIMIT, --limit LIMIT\n 10\n", + "tier": 3, + "usage": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...]" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found \u2192 run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json \u2014 run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/python_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found \u2192 add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found \u2192 add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity \u2192 name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) \u2192 flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md \u2192 link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json \u2014 no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found \u2014 run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "generate", + "tests/fixtures/bare_project" + ], + "returncode": 0, + "stderr": "", + "stdout": "bare_project: 12 artifact(s)\n\n Would create CLAUDE.md\n Would create misc/docs/docs_guide.md\n Would create misc/docs/architecture.md\n Would create misc/docs/known_issues.md\n Would create misc/docs/conventions.md\n Would create misc/docs/roadmap.md\n Would create misc/docs/decisions/0000-template.md\n Would create .claude/agents/setup-auditor.md\n Would create .claude/skills/opsward/SKILL.md\n Would create .claude/skills/opsward-diagnose/SKILL.md\n Would create .claude/skills/opsward-generate/SKILL.md\n Would create .claude/skills/opsward-maintain/SKILL.md\n\nDry run \u2014 pass --write to create files.\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "maintain", + "tests/fixtures/stale_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "stale_project: 5 issue(s)\n\n [stale_path] CLAUDE.md references `src/core.py` but it does not exist\n [stale_path] CLAUDE.md references `src/utils.py` but it does not exist\n [sync_issue] `unlisted_doc.md` exists in docs/ but is not listed in docs_guide.md\n + | [unlisted_doc.md](unlisted_doc.md) | |\n [sync_issue] docs_guide.md references `deleted_doc.md` but the file does not exist\n [empty_doc] `conventions.md` appears to be an empty stub (37 bytes)\n\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "recommend", + "tests/fixtures/python_project" + ], + "returncode": 0, + "stderr": "", + "stdout": "python_project: no skill recommendations\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--min-score", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found \u2192 run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json \u2014 run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--min-score=0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found \u2192 run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json \u2014 run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "-m", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found \u2192 run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json \u2014 run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "--verbose", + "tests/fixtures/python_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found \u2192 add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found \u2192 add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity \u2192 name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) \u2192 flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md \u2192 link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json \u2014 no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found \u2014 run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n\nDetailed inventory:\n Skills: 1\n - my-skill (SKILL.md: yes)\n Agents: 1\n - code-reviewer\n Rules: 1\n - no-print\n Docs: 2\n - architecture (54 bytes)\n - docs_guide (44 bytes)\n Hooks: yes\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "-v", + "tests/fixtures/python_project" + ], + "returncode": 1, + "stderr": "", + "stdout": "Diagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found \u2192 add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found \u2192 add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity \u2192 name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) \u2192 flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md \u2192 link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json \u2014 no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found \u2014 run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n\nDetailed inventory:\n Skills: 1\n - my-skill (SKILL.md: yes)\n Agents: 1\n - code-reviewer\n Rules: 1\n - no-print\n Docs: 2\n - architecture (54 bytes)\n - docs_guide (44 bytes)\n Hooks: yes\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--format", + "json-ish-typo", + "--min-score", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found \u2192 run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json \u2014 run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "tests/fixtures/python_project", + "--min-score", + "0" + ], + "returncode": 0, + "stderr": "", + "stdout": "Diagnosis Report: bare_project\nProject type: unknown\nOverall score: 0/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [....................] 0/100\n - No CLAUDE.md found\n Documentation [....................] 0/100\n - No documentation files found \u2192 run `opsward generate` to scaffold architecture/conventions docs\n Skills [....................] 0/100\n - No skills defined\n Setup (rules/agents/hooks) [....................] 0/100\n - No rules defined\n - No agents defined\n - No hooks configured\n Cross-references [....................] 0/100\n - No CLAUDE.md\n\nMissing:\n [ ] CLAUDE.md\n [ ] docs_guide.md\n [ ] docs/architecture.md\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Create a CLAUDE.md at the project root\n 2. Create a docs_guide.md to index your documentation\n 3. Consider adding skills in .claude/skills/ for recurring tasks\n 4. Add hooks in .claude/hooks.json \u2014 run `opsward generate --hooks` for starter templates (auto-format on PostToolUse, session context on SessionStart)\n\n============================================================\n\nDiagnosis Report: python_project\nProject type: python\nOverall score: 54/100 (Grade: F)\n\nComponents:\n CLAUDE.md quality [##########..........] 49/100\n - Commands: no code blocks found \u2192 add a fenced code block with the build, test, and run commands\n - Architecture: no architecture/structure section found \u2192 add a '## Architecture' section mapping key modules to one-line descriptions\n - Conventions: low specificity \u2192 name the tools (e.g. ruff), cite file extensions, and show code snippets\n Documentation [###########.........] 55/100\n - 1 doc(s) appear to be empty stubs (docs_guide) \u2192 flesh them out with real content or remove them\n - CLAUDE.md does not reference docs_guide.md \u2192 link docs_guide.md from CLAUDE.md so agents discover the docs\n Skills [##############......] 70/100\n - Skill \"my-skill\": frontmatter missing required `name` field\n - Skill \"my-skill\": frontmatter missing required `description` field\n Setup (rules/agents/hooks) [########............] 40/100\n - Hooks config invalid: no top-level `hooks` key\n Cross-references [##########..........] 50/100\n - No file paths found in CLAUDE.md to validate\n\nMissing:\n [ ] docs/conventions.md\n [ ] docs/known_issues.md\n\nSuggestions:\n 1. Fix the hooks config in .claude/hooks.json \u2014 no top-level `hooks` key (an invalid hook silently never fires)\n 2. No AGENTS.md found \u2014 run `opsward generate --agents-md` to create cross-platform agent instructions (supported by 60,000+ projects)\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "generate", + "tests/fixtures/bare_project", + "--agents-md", + "--hooks" + ], + "returncode": 0, + "stderr": "", + "stdout": "bare_project: 14 artifact(s)\n\n Would create CLAUDE.md\n Would create misc/docs/docs_guide.md\n Would create misc/docs/architecture.md\n Would create misc/docs/known_issues.md\n Would create misc/docs/conventions.md\n Would create misc/docs/roadmap.md\n Would create misc/docs/decisions/0000-template.md\n Would create .claude/agents/setup-auditor.md\n Would create .claude/skills/opsward/SKILL.md\n Would create .claude/skills/opsward-diagnose/SKILL.md\n Would create .claude/skills/opsward-generate/SKILL.md\n Would create .claude/skills/opsward-maintain/SKILL.md\n Would create AGENTS.md\n Would create .claude/hooks.json\n\nDry run \u2014 pass --write to create files.\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "generate", + "tests/fixtures/bare_project", + "-a" + ], + "returncode": 0, + "stderr": "", + "stdout": "bare_project: 13 artifact(s)\n\n Would create CLAUDE.md\n Would create misc/docs/docs_guide.md\n Would create misc/docs/architecture.md\n Would create misc/docs/known_issues.md\n Would create misc/docs/conventions.md\n Would create misc/docs/roadmap.md\n Would create misc/docs/decisions/0000-template.md\n Would create .claude/agents/setup-auditor.md\n Would create .claude/skills/opsward/SKILL.md\n Would create .claude/skills/opsward-diagnose/SKILL.md\n Would create .claude/skills/opsward-generate/SKILL.md\n Would create .claude/skills/opsward-maintain/SKILL.md\n Would create AGENTS.md\n\nDry run \u2014 pass --write to create files.\n", + "tier": 1, + "usage": "" + }, + { + "argv": [ + "install-skills", + "--target" + ], + "returncode": 2, + "stderr": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]\nopsward install-skills: error: argument -t/--target: expected one argument\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w] opsward install-skills: error: argument -t/--target: expected one argument" + }, + { + "argv": [ + "install-skills", + "-t" + ], + "returncode": 2, + "stderr": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w]\nopsward install-skills: error: argument -t/--target: expected one argument\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward install-skills [-h] [-t TARGET] [-g] [-a] [-w] opsward install-skills: error: argument -t/--target: expected one argument" + }, + { + "argv": [], + "returncode": 0, + "stderr": "", + "stdout": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\n", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ..." + }, + { + "argv": [ + "nosuchcommand" + ], + "returncode": 2, + "stderr": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\nopsward: error: argument {diagnose,generate,maintain,recommend,install-skills,find}: invalid choice: 'nosuchcommand' (choose from 'diagnose', 'generate', 'maintain', 'recommend', 'install-skills', 'find')\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... opsward: error: argument {diagnose,generate,maintain,recommend,install-skills,find}: invalid choice: 'nosuchcommand' (choose from 'diagnose', 'generate', 'maintain', 'recommend', 'install-skills', 'find')" + }, + { + "argv": [ + "diagnose", + "--nosuchflag" + ], + "returncode": 2, + "stderr": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\nopsward: error: unrecognized arguments: --nosuchflag\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... opsward: error: unrecognized arguments: --nosuchflag" + }, + { + "argv": [ + "diagnose", + "--min-score" + ], + "returncode": 2, + "stderr": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]\nopsward diagnose: error: argument -m/--min-score: expected one argument\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...] opsward diagnose: error: argument -m/--min-score: expected one argument" + }, + { + "argv": [ + "diagnose", + "tests/fixtures/bare_project", + "--min-score", + "notanint" + ], + "returncode": 2, + "stderr": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...]\nopsward diagnose: error: argument -m/--min-score: invalid int value: 'notanint'\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward diagnose [-h] [-f FORMAT] [-v] [-m MIN_SCORE] [project-roots ...] opsward diagnose: error: argument -m/--min-score: invalid int value: 'notanint'" + }, + { + "argv": [ + "generate", + "--write", + "--nosuchflag", + "tests/fixtures/bare_project" + ], + "returncode": 2, + "stderr": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ...\nopsward: error: unrecognized arguments: --nosuchflag\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... opsward: error: unrecognized arguments: --nosuchflag" + }, + { + "argv": [ + "find" + ], + "returncode": 2, + "stderr": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...]\nopsward find: error: the following arguments are required: query, project-roots\n", + "stdout": "", + "tier": 1, + "usage": "usage: opsward find [-h] [-k KINDS] [-s] [-l LIMIT] query [project-roots ...] opsward find: error: the following arguments are required: query, project-roots" + }, + { + "argv": [ + "diagnose", + "./no/such/directory/opsward-characterization" + ], + "returncode": 2, + "stderr": "Error: ./no/such/directory/opsward-characterization is not a directory\n", + "stdout": "", + "tier": 1, + "usage": "" + } + ], + "cw_golden": 1, + "env": { + "COLUMNS": "100", + "NO_COLOR": "1", + "PYTHONHASHSEED": "0", + "PYTHONIOENCODING": "utf-8", + "PYTHONUTF8": "1", + "TERM": "dumb" + }, + "newlines": "lf", + "note": "argh 0.31.3, opsward 4e0b04f, pre-cw-migration, CPython 3.10", + "prog": [ + "opsward" + ], + "recorded_with": { + "implementation": "CPython", + "platform": "Darwin", + "python": "3.10.13" + } +} diff --git a/misc/cli_golden.json b/misc/cli_golden_py312.json similarity index 100% rename from misc/cli_golden.json rename to misc/cli_golden_py312.json diff --git a/opsward/__main__.py b/opsward/__main__.py index de8fb11..fde904d 100644 --- a/opsward/__main__.py +++ b/opsward/__main__.py @@ -2,7 +2,7 @@ The command list is :data:`opsward.cli._dispatch_funcs` -- one SSOT, no per-command registration here. :func:`cw.dispatch` turns it into an ``argparse`` parser and runs it, -reproducing the grammar this CLI has always had (pinned by ``misc/cli_golden.json`` +reproducing the grammar this CLI has always had (pinned by ``misc/cli_golden_py*.json`` and asserted by ``tests/test_cli_parity.py``). ``prog`` is deliberately not passed: leaving it to ``argparse`` keeps the program name diff --git a/tests/test_cli_parity.py b/tests/test_cli_parity.py index e5b2db7..91f55d9 100644 --- a/tests/test_cli_parity.py +++ b/tests/test_cli_parity.py @@ -1,23 +1,27 @@ """Assert opsward's command line has not moved since it was recorded. -``misc/cli_golden.json`` was recorded from the argh-based CLI *before* the migration to -``cw``; ``misc/cli_cases.txt`` is the corpus it was recorded from. This test replays it -against whatever dispatcher is installed now, so neither a refactor here nor a new ``cw`` -release can change what a shell sees without a test going red. +``misc/cli_golden_py3XX.json`` were recorded from the **argh-based** CLI, before the +migration to ``cw`` and before any source edit; ``misc/cli_cases.txt`` is the corpus they +were recorded from. This test replays them against whatever dispatcher is installed now, so +neither a refactor here nor a new ``cw`` release can change what a shell sees without a test +going red. -Two tiers are asserted (see :mod:`cw.testing`): +Every case's exit code, stdout, stderr and normalised ``usage:`` line is asserted, and so is +the full ``--help`` body (``strict_help``). Nothing is advisory. -* every case's exit code, stdout and stderr, verbatim; and -* the normalised ``usage:`` line of every ``--help`` case -- which names every option a - parser has, so a lost flag or a changed ``nargs`` shows up here. +**One golden per CPython minor version, and that is not a workaround.** ``argparse`` is part +of the standard library and rewrites its own text between versions -- 3.12 stopped listing +``nargs='*'`` positionals among "the following arguments are required", and it changed how +``invalid choice`` quotes the choices. Those two cases are the *only* difference between the +3.10 and the 3.12 recording, they predate this repo's migration, and argh produces them +identically. A single golden asserted across a matrix would therefore fail for something +nobody caused, which is the fastest way to teach a team to ignore a red parity test. -The full ``--help`` *body* is asserted only when the running CPython matches the one that -recorded the golden. CPython rewrites its own help rendering between versions (3.13 emits -``-f, --format FORMAT`` where 3.12 emits ``-f FORMAT, --format FORMAT``), and a golden -replayed across a version matrix would otherwise fail for something nobody caused. +To add a Python version to CI, record a golden for it (see CLAUDE.md, "CLI Pattern"). An +unrecorded version fails loudly rather than silently skipping: a parity test that quietly +does nothing is worse than no parity test. """ -import json import shutil import sys from pathlib import Path @@ -25,7 +29,21 @@ import pytest REPO_ROOT = Path(__file__).resolve().parent.parent -GOLDEN = REPO_ROOT / "misc" / "cli_golden.json" +GOLDEN_DIR = REPO_ROOT / "misc" + + +def _golden_for_this_python() -> Path: + """The golden recorded by the CPython running this test.""" + major, minor = sys.version_info[:2] + path = GOLDEN_DIR / f"cli_golden_py{major}{minor}.json" + if path.exists(): + return path + recorded = sorted(p.name for p in GOLDEN_DIR.glob("cli_golden_py*.json")) + pytest.fail( + f"no CLI golden recorded for Python {major}.{minor} (have: {', '.join(recorded)}). " + f"argparse's own wording differs between CPython versions, so each version in CI " + f"needs its own recording. See CLAUDE.md, 'CLI Pattern', for how to make one." + ) def _console_script() -> str: @@ -39,23 +57,17 @@ def _console_script() -> str: if candidate.exists(): return str(candidate) pytest.fail( - "the `opsward` console script is not installed; CLI parity cannot be checked. " + "the `opsward` console script is not installed, so the CLI cannot be checked. " "Install the package (`pip install -e .`) before running this test." ) -def _recorded_python() -> tuple: - """``(major, minor)`` of the CPython that recorded the golden.""" - recorded = json.loads(GOLDEN.read_text(encoding="utf-8"))["recorded_with"]["python"] - return tuple(int(part) for part in recorded.split(".")[:2]) - - def test_cli_surface_is_unchanged(): """Every recorded argv still produces the same exit code, stdout and stderr.""" cw_testing = pytest.importorskip("cw.testing") cw_testing.assert_replay( - GOLDEN, + _golden_for_this_python(), prog=[_console_script()], cwd=str(REPO_ROOT), - strict_help=_recorded_python() == sys.version_info[:2], + strict_help=True, ) From 1676b5fade33ba30702596d2e0d19a3e9d84d3c4 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:28:38 +0200 Subject: [PATCH 5/6] Require cw>=0.1.1: the Windows console-script fix argparse takes its `prog` from `basename(sys.argv[0])`, and on Windows the console script is installed as `opsward.EXE`, so 22 of the 31 recorded cases differed on the runner: - usage: opsward [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... + usage: opsward.EXE [-h] {diagnose,generate,maintain,recommend,install-skills,find} ... That is a fact about packaging, not about the command line. cw 0.1.1 scrubs it (i2mint/cw#34), a defect this migration found. The floor is raised rather than left at 0.1 because with `cw>=0.1` a resolver could pick 0.1.0 and this repo's own parity test would fail on Windows for a reason nobody could act on. Claude-Session: https://claude.ai/code/session_01K6LB3AwUmKDxaFNZ2NqPGr --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fbbcbb8..565605e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Topic :: Software Development :: Documentation", ] dependencies = [ - "cw>=0.1,<0.2", + "cw>=0.1.1,<0.2", ] [project.scripts] From d49cc6c6288c6128de670562213aefa8630dab52 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:31:18 +0200 Subject: [PATCH 6/6] Assert the CLI on Windows too, minus six known-bad content cases The Windows runner is down to six differing cases, and none is about the command line. opsward prints `misc\docs\...` where POSIX prints `misc/docs/...`, and it reports file sizes inflated by CRLF checkout (a 37-byte stub measures 40). Both are the pre-existing Windows bugs in #21 -- the same run fails test_generate.py::test_python_docs_path for the same reason -- and argh printed exactly the same thing. They are listed as `expect_diff` rather than skipping the test on Windows, so everything else there stays asserted: exit codes, stdout, stderr, usage lines and the whole --help body. That distinction is not academic. The `.exe` defect this migration found in cw (i2mint/cw#34) lived exactly in the part a platform-wide skip would have stopped checking, and it took a Windows run to see it. If one of the six starts matching, the test fails with `unexpected-match`. That is the correct outcome: it means #21 was fixed and the entry should be deleted. Claude-Session: https://claude.ai/code/session_01K6LB3AwUmKDxaFNZ2NqPGr --- tests/test_cli_parity.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_cli_parity.py b/tests/test_cli_parity.py index 91f55d9..612f7a0 100644 --- a/tests/test_cli_parity.py +++ b/tests/test_cli_parity.py @@ -20,6 +20,15 @@ To add a Python version to CI, record a golden for it (see CLAUDE.md, "CLI Pattern"). An unrecorded version fails loudly rather than silently skipping: a parity test that quietly does nothing is worse than no parity test. + +**Windows** is asserted too, minus the six cases in :data:`WINDOWS_CONTENT_DIFFS`. Those +six differ for reasons that have nothing to do with the command line and everything to do +with opsward's own output: it prints ``misc\\docs\\...`` where POSIX prints +``misc/docs/...``, and it reports file sizes inflated by CRLF checkout (a 37-byte stub +measures 40). Both are the pre-existing Windows bugs tracked in issue #21 -- argh printed +exactly the same thing -- and they are listed rather than skipped so that everything else +on Windows, the whole grammar included, stays asserted. That matters: the ``.exe`` defect +this migration found in cw lived precisely there. """ import shutil @@ -31,6 +40,20 @@ REPO_ROOT = Path(__file__).resolve().parent.parent GOLDEN_DIR = REPO_ROOT / "misc" +#: Cases whose *content* differs on Windows because of opsward's own output, not because of +#: anything about the command line: printed paths use the OS separator, and reported file +#: sizes count CRLF line endings. Tracked in issue #21; unchanged by the migration to cw. +#: If one of these starts matching, the test fails with `unexpected-match` -- which is the +#: correct outcome, and means #21 was fixed and the entry should be deleted. +WINDOWS_CONTENT_DIFFS = [ + ["generate", "tests/fixtures/bare_project"], + ["generate", "tests/fixtures/bare_project", "--agents-md", "--hooks"], + ["generate", "tests/fixtures/bare_project", "-a"], + ["maintain", "tests/fixtures/stale_project"], + ["diagnose", "--verbose", "tests/fixtures/python_project"], + ["diagnose", "-v", "tests/fixtures/python_project"], +] + def _golden_for_this_python() -> Path: """The golden recorded by the CPython running this test.""" @@ -70,4 +93,5 @@ def test_cli_surface_is_unchanged(): prog=[_console_script()], cwd=str(REPO_ROOT), strict_help=True, + expect_diff=WINDOWS_CONTENT_DIFFS if sys.platform == "win32" else (), )