From d5087d84202ed1a77913635d7546b3247d5e1ea9 Mon Sep 17 00:00:00 2001 From: Norair Arutshyan Date: Thu, 24 Sep 2026 11:04:24 +0100 Subject: [PATCH 1/2] feat(tooling): replace bash workflow scripts with flow.py CLI Consolidates new-issue.sh/start-group.sh into a single scripts/flow.py covering init/sync/start/pr/issue, with idempotent GitHub-as-source-of-truth reconciliation so a stale or branch-local tasks.md can never create a duplicate sub-issue. Updates the branch-naming hook, pre-push check, and CI workflow docs accordingly, and adds a tooling CI job for flow.py's test suite. Co-Authored-By: Claude Sonnet 5 --- .claude/hooks/block_manual_branch_creation.py | 9 +- .claude/skills/commit-push-pr/SKILL.md | 6 +- .github/workflows/branch-naming.yml | 2 +- .github/workflows/tooling.yml | 50 + CLAUDE.md | 54 +- docs/development/WORKFLOW.md | 170 +++- .../budget-feat-313-excel-export/tasks.md | 29 +- .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/budget-customer-cache/spec.md | 0 .../tasks.md | 15 +- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../branch-protection-enforcement/spec.md | 0 .../specs/ci-security-scanning/spec.md | 0 .../tasks.md | 39 +- openspec/config.yaml | 8 +- scripts/flow.py | 855 ++++++++++++++++++ scripts/git-hooks/pre-push | 11 +- scripts/new-issue.sh | 31 - scripts/start-group.sh | 127 --- scripts/test_flow.py | 242 +++++ 23 files changed, 1393 insertions(+), 255 deletions(-) create mode 100644 .github/workflows/tooling.yml rename openspec/changes/{budget-feat-customer-profile-cache => budget-feat-326-customer-profile-cache}/.openspec.yaml (100%) rename openspec/changes/{budget-feat-customer-profile-cache => budget-feat-326-customer-profile-cache}/proposal.md (100%) rename openspec/changes/{budget-feat-customer-profile-cache => budget-feat-326-customer-profile-cache}/specs/budget-customer-cache/spec.md (100%) rename openspec/changes/{budget-feat-customer-profile-cache => budget-feat-326-customer-profile-cache}/tasks.md (65%) rename openspec/changes/{ci-compliance-guardrails => platform-chore-328-compliance-guardrails}/.openspec.yaml (100%) rename openspec/changes/{ci-compliance-guardrails => platform-chore-328-compliance-guardrails}/design.md (100%) rename openspec/changes/{ci-compliance-guardrails => platform-chore-328-compliance-guardrails}/proposal.md (100%) rename openspec/changes/{ci-compliance-guardrails => platform-chore-328-compliance-guardrails}/specs/branch-protection-enforcement/spec.md (100%) rename openspec/changes/{ci-compliance-guardrails => platform-chore-328-compliance-guardrails}/specs/ci-security-scanning/spec.md (100%) rename openspec/changes/{ci-compliance-guardrails => platform-chore-328-compliance-guardrails}/tasks.md (66%) create mode 100755 scripts/flow.py delete mode 100755 scripts/new-issue.sh delete mode 100755 scripts/start-group.sh create mode 100644 scripts/test_flow.py diff --git a/.claude/hooks/block_manual_branch_creation.py b/.claude/hooks/block_manual_branch_creation.py index 484b563..23f8f57 100644 --- a/.claude/hooks/block_manual_branch_creation.py +++ b/.claude/hooks/block_manual_branch_creation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""PreToolUse hook (Bash): blocks manual branch creation that bypasses scripts/start-group.sh.""" +"""PreToolUse hook (Bash): blocks manual branch creation that bypasses scripts/flow.py.""" import json import re import sys @@ -36,9 +36,10 @@ def main(): "permissionDecisionReason": ( f"Blocked: manual branch creation with name '{name}', which doesn't " "match //Issue-/. Branches must be " - "created via `scripts/start-group.sh ` " - "(docs/development/WORKFLOW.md) so the sub-issue gets created and " - "linked. Do not skip it, even for small/chore work." + "created via `scripts/flow.py start ` " + "(docs/development/WORKFLOW.md) so the group's sub-issue is linked " + "and both it and the parent move to In Progress. Do not skip it, " + "even for small/chore work." ), } } diff --git a/.claude/skills/commit-push-pr/SKILL.md b/.claude/skills/commit-push-pr/SKILL.md index b8c975c..2422d9e 100644 --- a/.claude/skills/commit-push-pr/SKILL.md +++ b/.claude/skills/commit-push-pr/SKILL.md @@ -47,7 +47,7 @@ Commit the changes the user has already staged, push the current branch, and ope Run `gh pr view --json url,number,state 2>/dev/null` (or `gh pr list --head --json url,number`) to see if a PR already exists for this branch. - If one exists: report its URL, do not create a duplicate. - - If none exists: gather the branch's full commit history vs. the base branch (`git log ..HEAD`, `git diff ...HEAD`) and create one with `gh pr create`, using a HEREDOC body: + - If none exists: gather the branch's full commit history vs. the base branch (`git log ..HEAD`, `git diff ...HEAD`), run `scripts/flow.py pr` to get the issue-closing trailer for this branch, and create the PR with `gh pr create`, using a HEREDOC body: ``` ## Summary <1-3 bullets> @@ -56,8 +56,10 @@ Commit the changes the user has already staged, push the current branch, and ope πŸ€– Generated with [Claude Code](https://claude.com/claude-code) + + ``` - Keep the title under ~70 characters. + Keep the title under ~70 characters. Use the trailer exactly as printed β€” it emits `Closes #` alongside `Closes #` only when this is the last open group, and that is what closes the parent tracking issue and fires the board's Done automation. If `flow.py pr` errors (e.g. the branch isn't a task-group branch), say so and open the PR without a trailer rather than guessing issue numbers. 6. **Report** diff --git a/.github/workflows/branch-naming.yml b/.github/workflows/branch-naming.yml index 7ac21c7..f579195 100644 --- a/.github/workflows/branch-naming.yml +++ b/.github/workflows/branch-naming.yml @@ -18,6 +18,6 @@ jobs: # //Issue-/, see docs/development/WORKFLOW.md Β§2 if [[ ! "$BRANCH" =~ ^[A-Z][A-Za-z]*/(feat|fix|chore|refactor)/Issue-[0-9]+/.+$ ]]; then echo "::error::Branch '$BRANCH' doesn't match //Issue-/." - echo "::error::Create branches with scripts/start-group.sh β€” see docs/development/WORKFLOW.md" + echo "::error::Create branches with scripts/flow.py start β€” see docs/development/WORKFLOW.md" exit 1 fi diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml new file mode 100644 index 0000000..114806d --- /dev/null +++ b/.github/workflows/tooling.yml @@ -0,0 +1,50 @@ +name: Tooling CI + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.filter.outputs.relevant }} + steps: + - uses: actions/checkout@v4 + - id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + relevant: + - "scripts/**" + + tooling-test: + needs: changes + if: needs.changes.outputs.relevant == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest black flake8 + + - name: Lint + run: | + black --check scripts + flake8 --max-line-length=100 scripts + + - name: Run tests + run: python -m pytest scripts/test_flow.py -v diff --git a/CLAUDE.md b/CLAUDE.md index 48e9417..3f025ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,30 +1,58 @@ # Working in this repo -## Branch / issue workflow β€” mandatory +## OpenSpec β†’ issue β†’ branch β†’ PR β€” mandatory -Every OpenSpec task group gets its own branch and its own GitHub sub-issue, -created together by one command. Full details: `docs/development/WORKFLOW.md`. +One OpenSpec change = one GitHub parent issue. One task group = one sub-issue = +one branch = one PR. All of it runs through `scripts/flow.py`. Full details: +`docs/development/WORKFLOW.md`. -**Never create a branch with `git checkout -b` / `git switch -c` / `git branch` by -hand.** Always run: +There are three moments: + +**1. After proposing a change** β€” create the tracking issues: + +``` +scripts/flow.py init --title "" +``` + +Creates the parent issue, folds its number into the change directory name +(`---`), creates a sub-issue for **every** +task group up front, and writes the generated Tracking block into `tasks.md`. +Run it on `main` and commit the result before starting any group. + +If `tasks.md` gains, loses, or renames a group later, re-run +`scripts/flow.py sync ` on `main` and commit that too. + +**2. When starting a task group** β€” never create the branch by hand: + +``` +scripts/flow.py start +``` + +Moves the parent and that group to **In Progress**, then branches from a fresh +`main` as `//Issue-/-group`. Pass +`--base ` to branch from something other than `main`. + +**Never** use `git checkout -b` / `git switch -c` / `git branch` directly. + +**3. When opening the PR** β€” get the closing trailer from the branch: ``` -scripts/start-group.sh +scripts/flow.py pr ``` -This creates the sub-issue, links it under the parent issue, adds it to the -project board, and checks out a correctly-named branch: -`//Issue-/`. +Prints `Closes #`, plus `Closes #` when every sibling group +is already closed. Put those lines at the end of the PR body β€” that is what +fires the board's Done automation. This is enforced, not just documented: - `.claude/hooks/block_manual_branch_creation.py` (a `PreToolUse` hook on `Bash`) blocks manually-created branches whose name doesn't match the convention. - `scripts/git-hooks/pre-push` refuses to push a branch with a non-conforming name (requires `git config core.hooksPath scripts/git-hooks` β€” see - WORKFLOW.md Β§6, which also covers this hook's lint-mirroring behavior). + WORKFLOW.md Β§7, which also covers this hook's lint-mirroring behavior). - `.github/workflows/branch-naming.yml` re-checks the branch name on every push and PR as a backstop. -If a task genuinely doesn't fit an OpenSpec change/group (rare β€” e.g. a -one-off doc fix), ask the user before improvising a branch name outside the -convention. +For a ticket that isn't part of an OpenSpec change, use +`scripts/flow.py issue "" "<body>"`. If such a task still needs a branch, +ask the user before improvising a name outside the convention. diff --git a/docs/development/WORKFLOW.md b/docs/development/WORKFLOW.md index 7dccf43..43c8f4d 100644 --- a/docs/development/WORKFLOW.md +++ b/docs/development/WORKFLOW.md @@ -1,7 +1,22 @@ -# OpenSpec β†’ Issue β†’ Branch Workflow +# OpenSpec β†’ Issue β†’ Branch β†’ PR Workflow How a piece of work moves from an OpenSpec change to merged code, and how the -naming stays consistent end to end. +naming and tracking stay consistent end to end. + +Everything below is driven by one tool, `scripts/flow.py`: + +| Command | When | What it does | +| --- | --- | --- | +| `flow.py init <change> --title T` | right after proposing | parent issue + directory rename + a sub-issue per task group + Tracking block | +| `flow.py sync <change>` | whenever `tasks.md` groups change | idempotent reconcile of groups ↔ sub-issues | +| `flow.py start <change> <group>` | starting a group | parent + group β†’ In Progress, branch from `main` | +| `flow.py pr` | opening the PR | prints the `Closes` trailer | +| `flow.py issue "<t>" "<body>"` | one-off ticket | issue on the board, outside any change | + +Sub-issues are matched to task groups by the issue number recorded in `tasks.md`, +falling back to the `(group N)` title suffix when no number is recorded. +New issue numbers are saved before linking or updating the board, so `sync` +can resume those steps after a failure without creating another ticket. ## 1. OpenSpec change naming @@ -15,86 +30,143 @@ Change directories (`openspec/changes/<name>/`) are a flat identifier to the Example: `shared-feat-292-audit-mixin-auto-population` - **service** β€” lowercase, one of: `ai`, `backend`, `budget`, `chat`, - `feature`, `frontend`, `platform`, `shared`, `users` (same set used in - branch names, see below). + `feature`, `frontend`, `infra`, `platform`, `shared`, `users` (the `SERVICES` + map in `scripts/flow.py`; same set used in branch names, see Β§3). - **type** β€” `feat` | `fix` | `chore` | `refactor`, matching the commit prefixes already used in this repo (not GitHub's `bug` label). - `feat` β€” new capability - `fix` β€” bug fix - `chore` β€” infra/tooling/deps, no behavior change users would notice - `refactor` β€” structural change, no behavior change -- **issue** β€” the GitHub issue number of the **parent** issue for this - whole change. Its only job is to hold the per-group sub-issues (Β§3) β€” it - doesn't carry its own task list. +- **issue** β€” the GitHub issue number of the **parent** issue for this whole + change. Its only job is to hold the per-group sub-issues (Β§2) β€” it doesn't + carry its own task list. - **description** β€” short kebab-case summary. -**Ordering with the issue:** a change is often proposed before a ticket -exists. Flow is: +## 2. Creating the change's issues + +A change is proposed before any ticket exists, so: + +1. `/opsx:propose` with a name that has **no** issue number yet: + `<service>-<type>-<description>`. +2. Once `tasks.md` is real, on `main`: + + ``` + scripts/flow.py init <service>-<type>-<description> --title "<parent title>" + ``` + + which, in one step: + - creates the parent tracking issue and adds it to project board 8 as **Todo**; + - `git mv`s the change directory to fold in the new issue number; + - creates a sub-issue for **every** group in `tasks.md`, links each one under + the parent through GitHub's native sub-issues API (so the parent shows a + progress bar), and adds each to the board; + - writes the group's number back onto its header (`## N. Title β€” Issue #<n>`) + and generates the Tracking block at the top of `tasks.md`. +3. Commit the rename and the `tasks.md` edits **on `main`** before starting any + group. This is the whole point: the group↔issue mapping must exist on `main`, + not only on a feature branch. -1. `/opsx:propose` with a temporary kebab-case name (no issue number yet). -2. Once `tasks.md` exists and the scope is real, create the parent ticket: - `scripts/new-issue.sh "<title>" "<body>"`. -3. Rename the change directory to fold in the issue number: - `mv openspec/changes/{<temp-name>,<service>-<type>-<issue>-<description>}`. +Sub-issue titles are `<description>: <group title> (group N)`, with any +`β€” depends on N` planning suffix stripped. -## 2. Branch naming +If groups are added, removed or renamed afterwards, re-run +`scripts/flow.py sync <change-name>` on `main` and commit it. `sync` creates +what's missing, links a recorded issue if it is still unlinked, +refreshes the Tracking block and the parent issue's group list, and warns about +duplicates and orphaned sub-issues instead of silently papering over them. +If a recorded issue belongs to another parent, linking fails and `sync` stops +without moving it or creating a replacement. + +### The Tracking block + +`tasks.md` carries a generated block between `<!-- flow:tracking:start -->` and +`<!-- flow:tracking:end -->`, listing the parent issue, the board, and a +group β†’ sub-issue β†’ branch β†’ state table. It is rewritten wholesale by `init` +and `sync` β€” don't hand-edit it. + +## 3. Branch naming ``` -<Service>/<type>/Issue-<sub-issue>/<description>[-group<N>] +<Service>/<type>/Issue-<sub-issue>/<description>-group<N> ``` `Service` is title-cased (`Shared`, `Budget`, `Frontend`, `AI`, ...); `type` -stays lowercase, taken straight from the OpenSpec change name. `<sub-issue>` -is that **group's own** sub-issue number (Β§3), not the parent's β€” e.g. +stays lowercase, taken straight from the OpenSpec change name. `<sub-issue>` is +that **group's own** sub-issue number, not the parent's β€” e.g. `Shared/feat/Issue-301/audit-mixin-auto-population-group2`. -## 3. Task groups get their own sub-issue +## 4. Starting a task group + +``` +scripts/flow.py start <change-name> <group-number> +``` -Each `tasks.md` group is tracked as a real GitHub sub-issue of the parent -(GitHub's native sub-issue relationship, not just a text mention) β€” created -automatically the first time a group is started via `scripts/start-group.sh`: +- Moves the **parent** issue to In Progress (unless it's already Done) and the + **group's** sub-issue to In Progress. +- Fetches `origin/main`, fast-forwards, and branches from it. Use + `--base <ref>` to branch from something else. +- Refuses if the working tree is dirty, if the branch already exists, or if the + group has no sub-issue recorded in `tasks.md` on the current checkout β€” in + that last case it tells you to run `sync` on `main` and commit first, rather + than creating a duplicate issue. -- Title: `<description>: <group title> (group N)`. -- Body links back to the parent issue and the group's `tasks.md` section. -- Linked to the parent through GitHub's sub-issues API, so it shows up as a - checklist/progress bar on the parent issue. -- Added to project board 8. -- The sub-issue number is written back onto the group's header line in - `tasks.md` (`## N. Title β€” Issue #<sub-issue>`), so re-running the script - for the same group reuses it instead of creating a duplicate. +## 5. Opening the PR -## 4. Triggering a task group's branch +``` +scripts/flow.py pr +``` -Don't rely on remembering to create the sub-issue and branch correctly at -each group boundary β€” run: +Reads the sub-issue out of the current branch name, finds the change and group +that own it, and prints: ``` -scripts/start-group.sh <change-name> <group-number> +Closes #<sub-issue> ``` -First run for a group: creates its sub-issue (as in Β§3), links it under the -parent, and derives the branch from it. Later runs for the same group: reads -the sub-issue back out of `tasks.md` instead of recreating it. Either way it -creates and checks out the branch and flips the sub-issue's project item to -**In Progress**. Refuses to run if the change name doesn't parse or the -group doesn't exist in `tasks.md`. +plus `Closes #<parent>` when every other group's sub-issue is already closed β€” +so the parent closes itself with the last group's PR instead of relying on +someone remembering. Paste the output at the end of the PR body; it fires the +board's Done automation for each issue named. -## 5. PR / issue-closing convention +## 6. One-off tickets -Each group's PR closes its own sub-issue: `Closes #<sub-issue>` (fires the -board's Done automation for that item). Once every sub-issue under the -parent is closed, close the parent too β€” it's just a tracking issue at that -point. +For work that isn't an OpenSpec change (a doc fix, a bug found in passing): -## 6. Local lint enforcement before push +``` +scripts/flow.py issue "<title>" "<body>" +``` + +Creates the issue and puts it on the board as Todo. If it needs a branch, agree +a name with a human first β€” the convention above assumes a sub-issue exists. + +## 7. Local lint enforcement before push `scripts/git-hooks/pre-push` mirrors each service's CI lint step -(`black --check`, `mypy`, `flake8`) locally, scoped to whichever -service(s) the push actually touches (a `shared/` change checks all four). -One-time setup: +(`black --check`, `mypy`, `flake8`) locally, scoped to whichever service(s) the +push actually touches (a `shared/` change checks all four). A push touching +`scripts/*.py` runs `black --check`, `flake8 --max-line-length=100` and +`scripts/test_flow.py` over `scripts/` β€” the workflow tooling lints itself. The +hook also rejects a branch whose name doesn't match Β§3. One-time setup: ``` git config core.hooksPath scripts/git-hooks ``` Bypass with `git push --no-verify` when intentionally needed. + +## 8. Testing the tool itself + +`scripts/test_flow.py` covers the change-name/group parsing and the `tasks.md` +rewriting β€” the load-bearing, network-free parts: + +``` +python3 -m pytest scripts/test_flow.py -q +``` + +`.github/workflows/tooling.yml` runs the same tests plus `black`/`flake8` over +`scripts/` on any push or PR that touches `scripts/**`, and the pre-push hook +(Β§7) mirrors it locally. + +`scripts/flow.py --dry-run <subcommand> ...` prints every mutating `gh`/`git` +call instead of running it, which is the safe way to preview `init` or `sync`. diff --git a/openspec/changes/budget-feat-313-excel-export/tasks.md b/openspec/changes/budget-feat-313-excel-export/tasks.md index b292cb6..cab0d56 100644 --- a/openspec/changes/budget-feat-313-excel-export/tasks.md +++ b/openspec/changes/budget-feat-313-excel-export/tasks.md @@ -2,9 +2,26 @@ Workflow rule: one task group = one GitHub sub-issue (of this change's parent issue) = one PR, merged before the next group starts. +<!-- flow:tracking:start --> +## Tracking + +Parent issue: [#313](https://github.com/arutsh/OpenGrantFlow/issues/313) Β· [Project board](https://github.com/users/arutsh/projects/8) + +| Group | Sub-issue | Branch | State | +| --- | --- | --- | --- | +| 1. Export endpoint scaffolding + Sheet 1 (Original Budget) | [#316](https://github.com/arutsh/OpenGrantFlow/issues/316) | `Budget/feat/Issue-316/excel-export-group1` | closed | +| 2. Sheet 2 β€” Budget vs. Report Dashboard | [#318](https://github.com/arutsh/OpenGrantFlow/issues/318) | `Budget/feat/Issue-318/excel-export-group2` | open | +| 3. Sheet 3 β€” List of Expenses with per-allocation sublines | [#321](https://github.com/arutsh/OpenGrantFlow/issues/321) | `Budget/feat/Issue-321/excel-export-group3` | open | +| 4. Frontend export button | [#322](https://github.com/arutsh/OpenGrantFlow/issues/322) | `Budget/feat/Issue-322/excel-export-group4` | open | +| 5. Export templates: model, ownership, and candidate resolut… | [#323](https://github.com/arutsh/OpenGrantFlow/issues/323) | `Budget/feat/Issue-323/excel-export-group5` | open | +| 6. Apply template options to generation | [#324](https://github.com/arutsh/OpenGrantFlow/issues/324) | `Budget/feat/Issue-324/excel-export-group6` | open | +| 7. Template picker and management UI | [#325](https://github.com/arutsh/OpenGrantFlow/issues/325) | `Budget/feat/Issue-325/excel-export-group7` | open | + +_Generated by `scripts/flow.py sync budget-feat-313-excel-export` β€” do not edit by hand._ +<!-- flow:tracking:end --> + ## 1. Export endpoint scaffolding + Sheet 1 (Original Budget) β€” Issue #316 -- [x] 1.0 Run `scripts/start-group.sh budget-excel-export 1` to create/link this group's sub-issue and branch before starting any other work in this group. - [x] 1.1 Add `services/budget/app/services/excel_export_service.py` with a `generate_budget_export_workbook(budget, categories, lines, ...)` entry point that creates an `openpyxl.Workbook()` and returns bytes; verify with a unit test that it returns a valid `.xlsx` (round-trips through `openpyxl.load_workbook`) - [x] 1.2 Implement Sheet 1 generation: header block (org/donor name via `customer_client`, project name/period, estimated currency/rate), a Budget Summary section (its title row doubles as the column-header row; one row per category referencing its Detailed Budget subtotal cell, plus a `SUM`-formula TOTAL row), a Detailed Budget section (category header row, lines, then a `SUM`-formula "Subtotal" row β€” categories with zero lines still shown, at 0), and a footer (a "Total expenditures" row summing each category's subtotal cell, signature/contact-person lines); per-line local amount is the only literal value, every other figure is a formula, blank when `estimated_exchange_rate` is unset; column widths/number formats matched to a user-supplied reference file; verify with unit tests covering 2 categories, no `estimated_exchange_rate`, and an empty category, plus a LibreOffice headless round-trip confirming the formulas actually compute (not just parse) - [x] 1.3 Add `GET /budgets/{budget_id}/export.xlsx` to `services/budget/app/api/budget_routes.py`, authorized the same way as `GET /budgets/{budget_id}` (owner or funder), returning a `StreamingResponse` (mirroring `attachment_routes.py`'s pattern) with the correct `Content-Type`/`Content-Disposition`; verify with an integration test that owner and funder both get 200 and a non-owner/non-funder gets rejected @@ -20,20 +37,20 @@ Workflow rule: one task group = one GitHub sub-issue (of this change's parent is - [x] 2.5 Add `DashboardSheet`'s per-budget-line and category-subtotal rows (expenses local, expenses converted, deviation), reusing Sheet 1's category grouping/order and applying the estimated-portion cell style (italic + fill, per design.md Decision 4) where flagged; verify with a unit test asserting column values and that flagged cells carry the style β€” **superseded**: see Decision 13; deviation is donor-currency (Original βˆ’ Total Expenses), not local-currency - [ ] 2.6 Run backend lint/tests clean for `services/budget`; PR merged (`Closes` this group's sub-issue) -## 3. Sheet 3 β€” List of Expenses with per-allocation sublines β€” depends on 1 +## 3. Sheet 3 β€” List of Expenses with per-allocation sublines β€” depends on 1 β€” Issue #321 - [ ] 3.1 Extend `excel_export_crud.py` with a query returning every report line across all of the budget's reports, each with its ordered list of allocations (or none) - [ ] 3.2 Implement Sheet 3 as an `ExpenseListSheet(_SheetWriter)` subclass (see design.md Decision 12): one row per report line when zero or one allocation exists, one row per allocation (subline) when a report line has more than one, each subline carrying its own conversion date/rate/converted amount; verify with a unit test covering an unallocated expense, a single-lot expense, and a multi-lot expense (asserting the multi-lot rows sum to the expense's full amount) - [ ] 3.3 Run backend lint/tests clean for `services/budget`; PR merged (`Closes` this group's sub-issue) -## 4. Frontend export button β€” depends on 1, 2, 3 +## 4. Frontend export button β€” depends on 1, 2, 3 β€” Issue #322 - [ ] 4.1 Add an "Export to Excel" button to the single-budget detail view in `frontend-typescript`, visible whenever the viewer has read access to the budget (owner or funder), following existing button/permission conventions on that view β€” a plain button at this stage; group 7 turns it into a template picker - [ ] 4.2 Wire the button to `GET /budgets/{budget_id}/export.xlsx` and trigger a browser download of the response with a filename derived from the budget's name; verify manually that a downloaded file opens in Excel/LibreOffice with all 3 sheets populated - [ ] 4.3 Add inline error handling that shows a message without navigating away when the request fails; verify with a frontend test that simulates a failed request and asserts the user stays on the budget detail view with an error shown - [ ] 4.4 Run frontend lint/tests clean; manually verify end-to-end against a real budget with lines, receipts, multi-lot conversions, and report expenses (owner and funder logins); PR merged (`Closes` this group's sub-issue) -## 5. Export templates: model, ownership, and candidate resolution β€” depends on 1 +## 5. Export templates: model, ownership, and candidate resolution β€” depends on 1 β€” Issue #323 > The `budget-export-templates` capability's spec delta is not yet authored (see proposal.md β€” Capabilities). Write it before starting this group. @@ -45,14 +62,14 @@ Workflow rule: one task group = one GitHub sub-issue (of this change's parent is - [ ] 5.6 Add `GET /budgets/{budget_id}/export-templates` to `budget_routes.py`, authorized identically to `GET /budgets/{budget_id}`, returning each candidate tagged `system` / `own` / `donor`; verify with an integration test that owner and funder each get the candidate set the resolution rules predict and a non-viewer is rejected - [ ] 5.7 Run backend lint/tests clean for `services/budget`; PR merged (`Closes` this group's sub-issue) -## 6. Apply template options to generation β€” depends on 2, 3, 5 +## 6. Apply template options to generation β€” depends on 2, 3, 5 β€” Issue #324 - [ ] 6.1 Thread an optional `template_id` through `GET /budgets/{budget_id}/export.xlsx`: resolve against `list_candidate_templates`, use the single candidate when none is given, reject with 400 when none is given and more than one exists, reject with 403 when the given id is not a candidate, and reject with a distinct "no longer available" error when a previously valid id has been deleted or un-shared; verify with integration tests covering each of those four outcomes - [ ] 6.2 Make `generate_budget_export_workbook` take the resolved template's options and apply them β€” sheet subset, donor-currency estimate column visibility, column header label overrides, audit footer visibility β€” so the system default runs the same path as any other template; the sheet-subset option selects which of the fixed `OriginalBudgetSheet`/`DashboardSheet`/`ExpenseListSheet` classes (design.md Decision 12) the dispatcher instantiates, never a new layout; verify that group 1's and groups 2–3's existing tests pass unchanged when the system default is used, plus new tests asserting a Sheet-1-only template yields one sheet and a label override changes only the header text while every figure and formula is unchanged - [ ] 6.3 Extend `_audit_line()` to include the template's name and version alongside the exporting user and timestamp; verify with a unit test asserting the footer text for a named template and for the system default - [ ] 6.4 Run backend lint/tests clean for `services/budget`; PR merged (`Closes` this group's sub-issue) -## 7. Template picker and management UI β€” depends on 4, 5, 6 +## 7. Template picker and management UI β€” depends on 4, 5, 6 β€” Issue #325 - [ ] 7.1 Replace group 4's plain button with a template picker: fetch `GET /budgets/{budget_id}/export-templates` on the budget detail view, export directly when only one candidate exists, and present the options labelled by source (default / own / funder) when more than one does; verify with frontend tests covering the single-candidate path (no picker shown) and the multi-candidate path (picker shown, request sent only after a choice) - [ ] 7.2 Handle the "chosen template is no longer available" rejection by showing the reason inline and reopening the picker with freshly fetched options, rather than a generic failure; verify with a frontend test simulating that rejection diff --git a/openspec/changes/budget-feat-customer-profile-cache/.openspec.yaml b/openspec/changes/budget-feat-326-customer-profile-cache/.openspec.yaml similarity index 100% rename from openspec/changes/budget-feat-customer-profile-cache/.openspec.yaml rename to openspec/changes/budget-feat-326-customer-profile-cache/.openspec.yaml diff --git a/openspec/changes/budget-feat-customer-profile-cache/proposal.md b/openspec/changes/budget-feat-326-customer-profile-cache/proposal.md similarity index 100% rename from openspec/changes/budget-feat-customer-profile-cache/proposal.md rename to openspec/changes/budget-feat-326-customer-profile-cache/proposal.md diff --git a/openspec/changes/budget-feat-customer-profile-cache/specs/budget-customer-cache/spec.md b/openspec/changes/budget-feat-326-customer-profile-cache/specs/budget-customer-cache/spec.md similarity index 100% rename from openspec/changes/budget-feat-customer-profile-cache/specs/budget-customer-cache/spec.md rename to openspec/changes/budget-feat-326-customer-profile-cache/specs/budget-customer-cache/spec.md diff --git a/openspec/changes/budget-feat-customer-profile-cache/tasks.md b/openspec/changes/budget-feat-326-customer-profile-cache/tasks.md similarity index 65% rename from openspec/changes/budget-feat-customer-profile-cache/tasks.md rename to openspec/changes/budget-feat-326-customer-profile-cache/tasks.md index 46c9738..570fcbc 100644 --- a/openspec/changes/budget-feat-customer-profile-cache/tasks.md +++ b/openspec/changes/budget-feat-326-customer-profile-cache/tasks.md @@ -2,9 +2,20 @@ Workflow rule: one task group = one GitHub sub-issue (of this change's parent issue) = one PR, merged before the next group starts. -## 1. Local read-through customer cache +<!-- flow:tracking:start --> +## Tracking + +Parent issue: [#326](https://github.com/arutsh/OpenGrantFlow/issues/326) Β· [Project board](https://github.com/users/arutsh/projects/8) + +| Group | Sub-issue | Branch | State | +| --- | --- | --- | --- | +| 1. Local read-through customer cache | [#327](https://github.com/arutsh/OpenGrantFlow/issues/327) | `Budget/feat/Issue-327/customer-profile-cache-group1` | open | + +_Generated by `scripts/flow.py sync budget-feat-326-customer-profile-cache` β€” do not edit by hand._ +<!-- flow:tracking:end --> + +## 1. Local read-through customer cache β€” Issue #327 -- [ ] 1.0 Run `scripts/start-group.sh budget-feat-customer-profile-cache 1` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 1.1 Add `CustomerProfileModel` (`customer_profiles` table: `customer_id` PK, `name`, `is_donor`, `is_ngo`, `cached_at`) in `services/budget/app/models/`, mirroring `UserProfileModel`'s shape, and generate the Alembic migration. - [ ] 1.2 Change `get_customer_cached` in `services/budget/app/services/customer_client.py` to a DB-first, HTTP-fallback-and-backfill lookup mirroring `get_users_by_ids_cached` in `services/budget/app/services/user_cache.py`; drop the `lru_cache` decorator. - [ ] 1.3 Verify existing callers (`budget_services.py`'s `local_currency` lookup, `excel_export_service.py`'s organisation/donor name lookups) work unchanged against the new function signature. diff --git a/openspec/changes/ci-compliance-guardrails/.openspec.yaml b/openspec/changes/platform-chore-328-compliance-guardrails/.openspec.yaml similarity index 100% rename from openspec/changes/ci-compliance-guardrails/.openspec.yaml rename to openspec/changes/platform-chore-328-compliance-guardrails/.openspec.yaml diff --git a/openspec/changes/ci-compliance-guardrails/design.md b/openspec/changes/platform-chore-328-compliance-guardrails/design.md similarity index 100% rename from openspec/changes/ci-compliance-guardrails/design.md rename to openspec/changes/platform-chore-328-compliance-guardrails/design.md diff --git a/openspec/changes/ci-compliance-guardrails/proposal.md b/openspec/changes/platform-chore-328-compliance-guardrails/proposal.md similarity index 100% rename from openspec/changes/ci-compliance-guardrails/proposal.md rename to openspec/changes/platform-chore-328-compliance-guardrails/proposal.md diff --git a/openspec/changes/ci-compliance-guardrails/specs/branch-protection-enforcement/spec.md b/openspec/changes/platform-chore-328-compliance-guardrails/specs/branch-protection-enforcement/spec.md similarity index 100% rename from openspec/changes/ci-compliance-guardrails/specs/branch-protection-enforcement/spec.md rename to openspec/changes/platform-chore-328-compliance-guardrails/specs/branch-protection-enforcement/spec.md diff --git a/openspec/changes/ci-compliance-guardrails/specs/ci-security-scanning/spec.md b/openspec/changes/platform-chore-328-compliance-guardrails/specs/ci-security-scanning/spec.md similarity index 100% rename from openspec/changes/ci-compliance-guardrails/specs/ci-security-scanning/spec.md rename to openspec/changes/platform-chore-328-compliance-guardrails/specs/ci-security-scanning/spec.md diff --git a/openspec/changes/ci-compliance-guardrails/tasks.md b/openspec/changes/platform-chore-328-compliance-guardrails/tasks.md similarity index 66% rename from openspec/changes/ci-compliance-guardrails/tasks.md rename to openspec/changes/platform-chore-328-compliance-guardrails/tasks.md index cc844c8..4604980 100644 --- a/openspec/changes/ci-compliance-guardrails/tasks.md +++ b/openspec/changes/platform-chore-328-compliance-guardrails/tasks.md @@ -2,53 +2,64 @@ One task group = one GitHub sub-issue (of this change's parent issue) = one PR, merged before the next group starts. -## 1. Secret scanning pilot (gitleaks) on budget + repo-wide pre-commit hook +<!-- flow:tracking:start --> +## Tracking + +Parent issue: [#328](https://github.com/arutsh/OpenGrantFlow/issues/328) Β· [Project board](https://github.com/users/arutsh/projects/8) + +| Group | Sub-issue | Branch | State | +| --- | --- | --- | --- | +| 1. Secret scanning pilot (gitleaks) on budget + repo-wide pr… | [#329](https://github.com/arutsh/OpenGrantFlow/issues/329) | `Platform/chore/Issue-329/compliance-guardrails-group1` | open | +| 2. Bandit SAST pilot on budget | [#330](https://github.com/arutsh/OpenGrantFlow/issues/330) | `Platform/chore/Issue-330/compliance-guardrails-group2` | open | +| 3. Semgrep custom rules (PII-in-logs, raw-SQL) pilot on budg… | [#331](https://github.com/arutsh/OpenGrantFlow/issues/331) | `Platform/chore/Issue-331/compliance-guardrails-group3` | open | +| 4. Roll out gitleaks + Bandit + Semgrep (PII/SQL rules) to r… | [#332](https://github.com/arutsh/OpenGrantFlow/issues/332) | `Platform/chore/Issue-332/compliance-guardrails-group4` | open | +| 5. Tenant-scoping Semgrep rule (narrow scope, all services) | [#333](https://github.com/arutsh/OpenGrantFlow/issues/333) | `Platform/chore/Issue-333/compliance-guardrails-group5` | open | +| 6. Promotion tracking + documentation | [#334](https://github.com/arutsh/OpenGrantFlow/issues/334) | `Platform/chore/Issue-334/compliance-guardrails-group6` | open | +| 7. Promote checks to required + enable branch protection | [#335](https://github.com/arutsh/OpenGrantFlow/issues/335) | `Platform/chore/Issue-335/compliance-guardrails-group7` | open | + +_Generated by `scripts/flow.py sync platform-chore-328-compliance-guardrails` β€” do not edit by hand._ +<!-- flow:tracking:end --> + +## 1. Secret scanning pilot (gitleaks) on budget + repo-wide pre-commit hook β€” Issue #329 -- [ ] 1.0 Run `scripts/start-group.sh ci-compliance-guardrails 1` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 1.1 Add repo-root `.pre-commit-config.yaml` with a gitleaks hook; verify `pre-commit run --all-files` flags a locally-added test secret, then remove the test secret. - [ ] 1.2 Add a gitleaks CI job (diff-scoped, `continue-on-error: true`) to `.github/workflows/budget.yml`; verify it runs on a test PR and reports a planted secret finding without failing the build, then remove the planted secret. - [ ] 1.3 Run a one-time full-history gitleaks scan against the repo; record any findings in a tracked triage list and rotate/redact confirmed secrets before closing this task. - [ ] 1.4 Run budget service tests/lint clean; PR merged. -## 2. Bandit SAST pilot on budget β€” depends on 1 +## 2. Bandit SAST pilot on budget β€” depends on 1 β€” Issue #330 -- [ ] 2.0 Run `scripts/start-group.sh ci-compliance-guardrails 2` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 2.1 Add Bandit to budget's lint CI step (`continue-on-error: true`); verify it flags a planted insecure pattern (e.g. `eval` on untrusted input) in a throwaway commit, then remove the planted pattern. - [ ] 2.2 Run budget service tests/lint clean; PR merged. -## 3. Semgrep custom rules (PII-in-logs, raw-SQL) pilot on budget β€” depends on 1 +## 3. Semgrep custom rules (PII-in-logs, raw-SQL) pilot on budget β€” depends on 1 β€” Issue #331 -- [ ] 3.0 Run `scripts/start-group.sh ci-compliance-guardrails 3` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 3.1 Create `.semgrep/rules/pii-logging.yml`; verify it flags a planted logger call whose argument is named after a known PII category (email, ssn, password, token), then remove the planted call. - [ ] 3.2 Create `.semgrep/rules/raw-sql-interpolation.yml`; verify it flags a planted f-string/`.format()`-built SQL query, then remove the planted query. - [ ] 3.3 Add a Semgrep CI job (`continue-on-error: true`) to budget.yml running `.semgrep/rules/`. - [ ] 3.4 Run budget service tests/lint clean; PR merged. -## 4. Roll out gitleaks + Bandit + Semgrep (PII/SQL rules) to remaining services β€” depends on 1, 2, 3 +## 4. Roll out gitleaks + Bandit + Semgrep (PII/SQL rules) to remaining services β€” depends on 1, 2, 3 β€” Issue #332 -- [ ] 4.0 Run `scripts/start-group.sh ci-compliance-guardrails 4` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 4.1 Add the three advisory CI jobs to `chat.yml`, `ai.yml`, `users.yml`, `worker.yml`, `shared.yml`, `frontend.yml`, and `e2e.yml`, mirroring budget's configuration. - [ ] 4.2 Verify each workflow completes successfully (green, findings advisory-only) on a real PR touching that service. - [ ] 4.3 Run the full test suite across all affected services clean; PR merged. -## 5. Tenant-scoping Semgrep rule (narrow scope, all services) β€” depends on 4 +## 5. Tenant-scoping Semgrep rule (narrow scope, all services) β€” depends on 4 β€” Issue #333 -- [ ] 5.0 Run `scripts/start-group.sh ci-compliance-guardrails 5` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 5.1 Enumerate the known multi-tenant tables/models requiring `customer_id` scoping, cross-referencing the `model-audit-trail` spec and current AuditMixin inventory. - [ ] 5.2 Create `.semgrep/rules/tenant-scoping.yml` scoped to that table list; verify it flags a planted unscoped query against a known multi-tenant table and does not flag existing superuser/impersonation code paths (`customer-impersonation` capability), then remove the planted query. - [ ] 5.3 Add the rule to all 8 workflows in advisory mode (`continue-on-error: true`). - [ ] 5.4 Run the full test suite clean; PR merged. -## 6. Promotion tracking + documentation β€” depends on 4, 5 +## 6. Promotion tracking + documentation β€” depends on 4, 5 β€” Issue #334 -- [ ] 6.0 Run `scripts/start-group.sh ci-compliance-guardrails 6` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 6.1 Add a lightweight findings-tracking mechanism (e.g. a periodic export of CI annotation counts per check) so the promotion criteria has real data behind it. - [ ] 6.2 Write `docs/security/ci-compliance-guardrails.md` documenting: each check, its advisory start date, promotion criteria, the current required-check inventory (including checks pending from `gdpr-iso27001-priority-2` and `audit-mixin-coverage-guard`), and the waiver process. - [ ] 6.3 Run lint clean on the new docs; PR merged. -## 7. Promote checks to required + enable branch protection β€” depends on 6, and on each check's documented advisory period having elapsed +## 7. Promote checks to required + enable branch protection β€” depends on 6, and on each check's documented advisory period having elapsed β€” Issue #335 -- [ ] 7.0 Run `scripts/start-group.sh ci-compliance-guardrails 7` to create/link this group's sub-issue and branch before starting any other work in this group. - [ ] 7.1 Confirm each check meets its documented promotion criterion (false-positive rate, advisory duration) using the tracking data from group 6. - [ ] 7.2 Remove `continue-on-error` from each promoted check's CI step across all workflows. - [ ] 7.3 Enable GitHub branch protection on `main` requiring the promoted checks (repo admin action); verify a test PR with a deliberately failing check is blocked from merging, then revert the test change. diff --git a/openspec/config.yaml b/openspec/config.yaml index f569a5b..bcc1632 100644 --- a/openspec/config.yaml +++ b/openspec/config.yaml @@ -5,9 +5,9 @@ schema: spec-driven context: | Repo: GrantFlow (nonprofit grant budgeting/reporting platform). Services: services/budget, services/users, services/ai, services/chat (FastAPI); frontend-typescript (React + TypeScript, React Query); nginx gateway proxying /api/v1/<prefix>/ to each service. - OpenSpec change dir naming (see docs/development/WORKFLOW.md for the full convention): <service>-<type>-<issue>-<description>, flat kebab-case, e.g. shared-feat-292-audit-mixin-auto-population. service in ai, backend, budget, chat, feature, frontend, platform, shared, users; type in feat, fix, chore, refactor. Propose under a temp kebab-case name if the parent GitHub issue doesn't exist yet, then rename the dir once scripts/new-issue.sh creates it. Do NOT prefix the dir with a priority label (HIGH/MED/LOW) β€” priority lives only in .openspec.yaml's `priority` field. - Each tasks.md task group becomes its own GitHub sub-issue of that parent, created and linked via scripts/start-group.sh (not by hand) when the group's implementation starts. - Branch naming: <Service>/<type>/Issue-<sub-issue>/<description>[-group<N>] (Service title-cased: Platform, Frontend, Shared, Budget, Users, AI, Chat, Backend, Feature; type lowercase, same as the change's type; sub-issue is that group's own sub-issue number, not the parent's) β€” created by scripts/start-group.sh, not by hand. + OpenSpec change dir naming (see docs/development/WORKFLOW.md for the full convention): <service>-<type>-<issue>-<description>, flat kebab-case, e.g. shared-feat-292-audit-mixin-auto-population. service in ai, backend, budget, chat, feature, frontend, infra, platform, shared, users; type in feat, fix, chore, refactor. Propose under a temp <service>-<type>-<description> name (no issue number yet); `scripts/flow.py init` creates the parent issue and renames the dir to fold its number in. Do NOT prefix the dir with a priority label (HIGH/MED/LOW) β€” priority lives only in .openspec.yaml's `priority` field. + Each tasks.md task group becomes its own GitHub sub-issue of that parent. `scripts/flow.py init` creates and links one for every group up front, right after the change is proposed; `scripts/flow.py sync` reconciles them if groups are later added, removed or renamed. Never create or link them by hand. + Branch naming: <Service>/<type>/Issue-<sub-issue>/<description>[-group<N>] (Service title-cased: AI, Backend, Budget, Chat, Feature, Frontend, Infra, Platform, Shared, Users; type lowercase, same as the change's type; sub-issue is that group's own sub-issue number, not the parent's) β€” created by `scripts/flow.py start <change-name> <N>`, not by hand. Code comments: keep to short one- or two-liners explaining WHY, not WHAT; no multi-line rationale blocks in source files. Never `git commit`, `git push`, or open a PR without the user's explicit go-ahead for that specific action β€” a prior approval (e.g. "commit and push this one") authorizes only that instance, not standing permission for later commits/pushes in the same session or change. @@ -18,7 +18,7 @@ rules: - Keep proposals under 500 words tasks: - "Workflow rule: one task group = one GitHub sub-issue (of this change's parent issue) = one PR, merged before the next group starts. State this rule verbatim as the first line of tasks.md, above the first '## 1.' heading." - - "Every group's FIRST task (e.g. '1.0', '2.0') must be: 'Run `scripts/start-group.sh <change-name> <N>` to create/link this group's sub-issue and branch before starting any other work in this group.' Do not start on the rest of a group's tasks until this one is checked off." + - "Every group's FIRST task (e.g. '1.0', '2.0') must be: 'Run `scripts/flow.py start <change-name> <N>` to move this group to In Progress and create its branch before starting any other work in this group.' The sub-issue itself already exists β€” `scripts/flow.py init` creates one per group up front β€” so this task only starts the group, it does not create the ticket. Do not start on the rest of a group's tasks until this one is checked off." - Structure task groups as independently mergeable vertical slices (not horizontal implementation layers), ordered by dependency, so each group's PR is real, reviewable, shippable work on its own. - Every group's final task must be "Run <the project's tests/lint for the affected area> clean; PR merged" β€” do not bundle a separate cross-cutting "wrap-up" group at the end; fold final integration/manual-verification tasks into the last functional group instead. - Note in each group's heading which earlier group(s) it depends on (e.g. "## 3. <name> β€” ticket depends on 1, 2"), so the merge order is unambiguous. diff --git a/scripts/flow.py b/scripts/flow.py new file mode 100755 index 0000000..75c62d1 --- /dev/null +++ b/scripts/flow.py @@ -0,0 +1,855 @@ +#!/usr/bin/env python3 +"""OpenSpec change -> GitHub issues -> branch -> PR. + +Subcommands: + init <change> [--title T] create the parent issue, fold its number into the + change directory name, create every group's + sub-issue, write the tracking block + sync <change> reconcile groups against GitHub (idempotent) + start <change> <group> parent + group to In Progress, branch from main + pr [--branch B] print the Closes trailer for a group's PR + issue "<title>" ["<body>"] create a standalone issue on the board + +GitHub is the source of truth: a group is matched to the sub-issue number +tasks.md records, falling back to the "(group N)" title suffix, and every +sub-issue can be claimed only once β€” so re-running can never create a duplicate +even when tasks.md is stale, renumbered, or on the wrong branch. + +See docs/development/WORKFLOW.md +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +REPO = "arutsh/OpenGrantFlow" +PROJECT_NUMBER = "8" +PROJECT_OWNER = "arutsh" +PROJECT_ID = "PVT_kwHOAQDyGs4A-5Lx" +STATUS_FIELD_ID = "PVTSSF_lAHOAQDyGs4A-5LxzgyKO-Q" +STATUS_OPTIONS = { + "Backlog": "3dbf10e0", + "Todo": "f75ad846", + "In Progress": "47fc9ee4", + "Done": "98236657", +} + +CHANGES_DIR = Path("openspec/changes") +TYPES = ("feat", "fix", "chore", "refactor") +SERVICES = { + "ai": "AI", + "backend": "Backend", + "budget": "Budget", + "chat": "Chat", + "feature": "Feature", + "frontend": "Frontend", + "infra": "Infra", + "platform": "Platform", + "shared": "Shared", + "users": "Users", +} + +ISSUE_URL = f"https://github.com/{REPO}/issues" +BOARD_URL = f"https://github.com/users/{PROJECT_OWNER}/projects/{PROJECT_NUMBER}" + +TRACK_START = "<!-- flow:tracking:start -->" +TRACK_END = "<!-- flow:tracking:end -->" +GROUPS_START = "<!-- flow:groups:start -->" +GROUPS_END = "<!-- flow:groups:end -->" + +CHANGE_RE = re.compile( + r"^(?P<service>[a-z]+)-(?P<type>" + "|".join(TYPES) + r")-(?:(?P<issue>\d+)-)?(?P<desc>.+)$" +) +HEADER_RE = re.compile(r"^##\s+(\d+)\.\s+(.+?)\s*$") +ISSUE_SUFFIX_RE = re.compile(r"\s+β€”\s+Issue\s+#(\d+)$") +DEPENDS_RE = re.compile(r"\s+β€”\s+depends on .*$", re.IGNORECASE) +GROUP_TITLE_RE = re.compile(r"\(group (\d+)\)$") +BRANCH_RE = re.compile(r"^[A-Z][A-Za-z]*/(?:" + "|".join(TYPES) + r")/Issue-(\d+)/") + +DRY_RUN = False + + +# --------------------------------------------------------------------------- shell + + +def die(msg: str) -> "NoReturn": # noqa: F821 + print(f"error: {msg}", file=sys.stderr) + raise SystemExit(1) + + +def warn(msg: str) -> None: + print(f"warning: {msg}", file=sys.stderr) + + +def run(args: list[str], *, mutating: bool = False, check: bool = True) -> str: + """Run a command, returning stdout. Mutating commands are skipped on --dry-run.""" + if mutating and DRY_RUN: + print(f"[dry-run] {' '.join(args)}") + return "" + result = subprocess.run(args, capture_output=True, text=True) + if check and result.returncode != 0: + die(f"`{' '.join(args)}` failed:\n{(result.stderr or result.stdout).strip()}") + return (result.stdout or "").strip() + + +def gh_json(args: list[str]): + out = run(args) + return json.loads(out) if out else None + + +# --------------------------------------------------------------------------- model + + +@dataclass +class Change: + name: str + service: str + type: str + issue: int | None + desc: str + + @property + def path(self) -> Path: + return CHANGES_DIR / self.name + + @property + def tasks_file(self) -> Path: + return self.path / "tasks.md" + + @property + def service_title(self) -> str: + return SERVICES[self.service] + + +@dataclass +class Group: + number: int + line: int + title: str + issue: int | None + + @property + def github_title(self) -> str: + """The group title without the 'β€” depends on N' planning suffix.""" + return DEPENDS_RE.sub("", self.title).strip() + + +def parse_change(name: str, *, strict: bool = True) -> Change | None: + """Parse a change directory name. Returns None for a non-conforming name unless strict.""" + name = name.strip().strip("/").removeprefix("openspec/changes/") + match = CHANGE_RE.match(name) + if match and match["service"] not in SERVICES: + match = None + if not match: + if not strict: + return None + die( + f"change name '{name}' doesn't match <service>-<type>[-<issue>]-<description>\n" + f" service: {', '.join(SERVICES)}\n" + f" type: {', '.join(TYPES)}" + ) + return Change( + name=name, + service=match["service"], + type=match["type"], + issue=int(match["issue"]) if match["issue"] else None, + desc=match["desc"], + ) + + +def load_change(name: str, *, require_issue: bool = False) -> Change: + change = parse_change(name) + if not change.tasks_file.is_file(): + die(f"{change.tasks_file} not found (run from the repo root)") + if require_issue and change.issue is None: + die(f"change '{change.name}' has no parent issue yet β€” run `{sys.argv[0]} init` first") + return change + + +def read_tasks(change: Change) -> list[str]: + return change.tasks_file.read_text().splitlines() + + +def parse_groups(lines: list[str]) -> list[Group]: + groups: list[Group] = [] + for index, line in enumerate(lines): + match = HEADER_RE.match(line) + if not match: + continue + title = match.group(2) + issue = None + suffix = ISSUE_SUFFIX_RE.search(title) + if suffix: + issue = int(suffix.group(1)) + title = title[: suffix.start()] + groups.append(Group(int(match.group(1)), index, title.strip(), issue)) + if not groups: + die("no '## <N>. <title>' group headers found in tasks.md") + return groups + + +def branch_name(change: Change, group: Group) -> str: + return ( + f"{change.service_title}/{change.type}/Issue-{group.issue}/" + f"{change.desc}-group{group.number}" + ) + + +def subissue_title(change: Change, group: Group) -> str: + return f"{change.desc}: {group.github_title} (group {group.number})" + + +def change_marker(change: Change) -> str: + """Stable id for a change, so a re-run of `init` can find a parent it already made.""" + return f"<!-- flow:change:{change.service}-{change.type}-{change.desc} -->" + + +# --------------------------------------------------------------------------- github + + +def sub_issues(parent: int) -> list[dict]: + return gh_json(["gh", "api", f"repos/{REPO}/issues/{parent}/sub_issues", "--paginate"]) or [] + + +def find_parent_issue(marker: str) -> int | None: + """An existing parent carrying this marker β€” i.e. one an earlier `init` left behind.""" + found = ( + gh_json( + [ + "gh", + "issue", + "list", + "--repo", + REPO, + "--state", + "all", + "--search", + f'"{marker}" in:body', + "--json", + "number", + "--limit", + "5", + ] + ) + or [] + ) + return found[0]["number"] if found else None + + +def create_issue(title: str, body: str) -> int | None: + url = run( + ["gh", "issue", "create", "--repo", REPO, "--title", title, "--body", body], + mutating=True, + ) + if not url: + return None + return int(url.rsplit("/", 1)[-1]) + + +def link_sub_issue(parent: int, child: int) -> None: + child_id = run(["gh", "api", f"repos/{REPO}/issues/{child}", "--jq", ".id"]) + run( + [ + "gh", + "api", + f"repos/{REPO}/issues/{parent}/sub_issues", + "--method", + "POST", + "-F", + f"sub_issue_id={child_id}", + ], + mutating=True, + ) + + +PROJECT_ITEM_QUERY = """ +query($owner:String!, $repo:String!, $number:Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $number) { + projectItems(first: 20) { + nodes { + id + project { number } + fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { name } + } + } + } + } + } +} +""" + + +def board_item(issue: int) -> tuple[str, str | None] | None: + """This issue's (item id, status) on the project board, or None if it isn't on it. + + Queried per issue rather than via `gh project item-list`, which silently + truncates on a board this size and would report a present item as missing. + """ + owner, repo = REPO.split("/") + nodes = ( + gh_json( + [ + "gh", + "api", + "graphql", + "-f", + f"query={PROJECT_ITEM_QUERY}", + "-f", + f"owner={owner}", + "-f", + f"repo={repo}", + "-F", + f"number={issue}", + "--jq", + ".data.repository.issue.projectItems.nodes", + ] + ) + or [] + ) + for node in nodes: + if node.get("project", {}).get("number") == int(PROJECT_NUMBER): + return node["id"], (node.get("fieldValueByName") or {}).get("name") + return None + + +def board_add(issue: int) -> str | None: + """Add the issue to the project board, returning its item id. + + Not `check=True`: the board's own auto-add workflow races us, and once it + wins, `item-add` fails with "Content already exists in this project". Only + a re-query can tell that apart from a real failure, so ask before dying. + """ + out = run( + [ + "gh", + "project", + "item-add", + PROJECT_NUMBER, + "--owner", + PROJECT_OWNER, + "--url", + f"{ISSUE_URL}/{issue}", + "--format", + "json", + ], + mutating=True, + check=False, + ) + if out: + return json.loads(out)["id"] + if DRY_RUN: + return None + found = board_item(issue) + if found: + return found[0] + die(f"could not add #{issue} to project board {PROJECT_NUMBER}") + + +def board_status(issue: int, status: str, *, keep: tuple[str, ...] = ()) -> None: + """Put the issue on the board and set its status, leaving `keep` statuses alone.""" + found = board_item(issue) + if found is None: + item_id, current = board_add(issue), None + if item_id: + print(f" board: added #{issue}") + else: + item_id, current = found + if not item_id or current == status or current in keep: + return + run( + [ + "gh", + "project", + "item-edit", + "--id", + item_id, + "--project-id", + PROJECT_ID, + "--field-id", + STATUS_FIELD_ID, + "--single-select-option-id", + STATUS_OPTIONS[status], + ], + mutating=True, + ) + print(f" board: #{issue} -> {status}") + + +def board_ensure(issue: int) -> None: + """Add the issue to the board if missing, without disturbing an existing status.""" + board_status(issue, "Todo", keep=tuple(STATUS_OPTIONS)) + + +# --------------------------------------------------------------------------- tasks.md + + +def short(title: str, limit: int = 58) -> str: + title = DEPENDS_RE.sub("", title).strip() + return title if len(title) <= limit else title[: limit - 1].rstrip() + "…" + + +def render_tracking(change: Change, groups: list[Group], states: dict[int, str]) -> list[str]: + rows = [ + "| Group | Sub-issue | Branch | State |", + "| --- | --- | --- | --- |", + ] + for group in groups: + if group.issue: + issue_cell = f"[#{group.issue}]({ISSUE_URL}/{group.issue})" + branch_cell = f"`{branch_name(change, group)}`" + state_cell = states.get(group.issue, "unknown") + else: + issue_cell = branch_cell = "β€”" + state_cell = "not created" + rows.append( + f"| {group.number}. {short(group.title)} | {issue_cell}" + f" | {branch_cell} | {state_cell} |" + ) + return [ + TRACK_START, + "## Tracking", + "", + f"Parent issue: [#{change.issue}]({ISSUE_URL}/{change.issue})" + f" Β· [Project board]({BOARD_URL})", + "", + *rows, + "", + f"_Generated by `scripts/flow.py sync {change.name}` β€” do not edit by hand._", + TRACK_END, + ] + + +def write_tasks( + change: Change, lines: list[str], groups: list[Group], states: dict[int, str] +) -> None: + for group in groups: + if group.issue: + lines[group.line] = f"## {group.number}. {group.title} β€” Issue #{group.issue}" + + block = render_tracking(change, groups, states) + start = lines.index(TRACK_START) if TRACK_START in lines else -1 + end = lines.index(TRACK_END) + 1 if TRACK_END in lines else -1 + if start >= 0 and end > start: + lines[start:end] = block + elif start >= 0 or end >= 0: + # a lone marker means a bad merge ate part of the block; what survived is unknowable + die( + f"{change.tasks_file} has a damaged tracking block (one marker only) β€” " + f"delete both markers and the lines between them, then re-run sync" + ) + else: + header_at = next((i for i, line in enumerate(lines) if HEADER_RE.match(line)), len(lines)) + start = header_at + while start > 0 and not lines[start - 1].strip(): + start -= 1 + lines[start:header_at] = ["", *block, ""] + + text = "\n".join(lines).rstrip() + "\n" + if DRY_RUN: + print(f"[dry-run] would write {change.tasks_file}") + return + change.tasks_file.write_text(text) + print(f" wrote {change.tasks_file}") + + +# --------------------------------------------------------------------------- reconcile + + +def retitle_sub_issue(change: Change, group: Group, sub: dict) -> None: + """Move a sub-issue's '(group N)' suffix onto the group that now records it.""" + current = GROUP_TITLE_RE.search(sub["title"]) + if current and int(current.group(1)) == group.number: + return + title = subissue_title(change, group) + run( + ["gh", "issue", "edit", str(sub["number"]), "--repo", REPO, "--title", title], + mutating=True, + ) + print(f" group {group.number}: retitled #{sub['number']} β€” {title}") + + +def match_groups(change: Change, groups: list[Group]) -> dict[int, dict]: + """Group number -> its sub-issue, for the groups that already have one. + + Two passes, because the two signals disagree whenever groups are renumbered: + the number recorded in tasks.md wins, and the '(group N)' title suffix is + only the fallback for groups that record none. A sub-issue can be claimed + once, so no two groups can end up pointing at the same one. + """ + existing = sub_issues(change.issue) + by_number = {s["number"]: s for s in existing} + by_suffix: dict[int, list[dict]] = {} + for sub in existing: + suffix = GROUP_TITLE_RE.search(sub["title"]) + if suffix: + by_suffix.setdefault(int(suffix.group(1)), []).append(sub) + + matched: dict[int, dict] = {} + claimed: set[int] = set() + + for group in groups: + sub = by_number.get(group.issue) if group.issue else None + if group.issue and sub is None: + sub = gh_json(["gh", "api", f"repos/{REPO}/issues/{group.issue}"]) + if not sub or "pull_request" in sub: + die(f"#{group.issue} is not an issue that can be linked to #{change.issue}") + # Without reparenting permission, GitHub rejects issues owned by another parent. + link_sub_issue(change.issue, group.issue) + by_number[group.issue] = sub + if sub and sub["number"] not in claimed: + matched[group.number] = sub + claimed.add(sub["number"]) + + for group in groups: + if group.number in matched: + continue + candidates = [ + s + for s in sorted(by_suffix.get(group.number, []), key=lambda s: s["number"]) + if s["number"] not in claimed + ] + if not candidates: + continue + matched[group.number] = candidates[0] + claimed.add(candidates[0]["number"]) + if group.issue: + warn( + f"group {group.number} pointed at #{group.issue}, which is not a sub-issue " + f"of #{change.issue}; repointing at #{candidates[0]['number']}" + ) + + known = {g.number for g in groups} + for number, subs in sorted(by_suffix.items()): + loose = [f"#{s['number']}" for s in subs if s["number"] not in claimed] + if not loose: + continue + listed = ", ".join(loose) + if number in known: + warn(f"sub-issue(s) {listed} also claim group {number}, which matched another issue") + else: + warn( + f"sub-issue(s) {listed} reference group {number}, " + f"which no longer exists in tasks.md" + ) + return matched + + +def reconcile(change: Change, groups: list[Group]) -> dict[int, str]: + """Match every group to a sub-issue of the parent, creating what's missing. + + The sub-issue number recorded in tasks.md is the source of truth; the + '(group N)' title suffix is the fallback (see `match_groups`). Either way a + sub-issue is matched before anything is created, so a stale or branch-local + tasks.md can never cause a duplicate. + """ + matched = match_groups(change, groups) + + states: dict[int, str] = {} + for group in groups: + chosen = matched.get(group.number) + if chosen: + group.issue = chosen["number"] + retitle_sub_issue(change, group, chosen) + board_ensure(chosen["number"]) # it may predate us, or have been auto-added + states[chosen["number"]] = chosen["state"] + print(f" group {group.number}: #{chosen['number']} ({chosen['state']})") + continue + + title = subissue_title(change, group) + body = ( + f"Group {group.number} of the `{change.name}` OpenSpec change " + f"(parent: #{change.issue}).\n\n" + f"{group.title}\n\n" + f"See `openspec/changes/{change.name}/tasks.md`, group {group.number}.\n\n" + f"Branch: `{change.service_title}/{change.type}/Issue-<this issue>/" + f"{change.desc}-group{group.number}`" + ) + number = create_issue(title, body) + if number is None: + print(f" group {group.number}: [dry-run] would create sub-issue") + continue + group.issue = number + # Save the recovery handle before either subsequent GitHub mutation can fail. + lines = read_tasks(change) + lines[group.line] = f"## {group.number}. {group.title} β€” Issue #{number}" + if not DRY_RUN: + change.tasks_file.write_text("\n".join(lines) + "\n") + link_sub_issue(change.issue, number) + board_status(number, "Todo") + states[number] = "open" + print(f" group {group.number}: created #{number} β€” {title}") + return states + + +def refresh_parent_body(change: Change, groups: list[Group]) -> None: + body = gh_json(["gh", "issue", "view", str(change.issue), "--repo", REPO, "--json", "body"]) + text = (body or {}).get("body", "") + if GROUPS_START not in text or GROUPS_END not in text: + return + listing = "\n".join( + f"- Group {g.number}: {g.github_title}" + (f" β€” #{g.issue}" if g.issue else "") + for g in groups + ) + head, rest = text.split(GROUPS_START, 1) + _, tail = rest.split(GROUPS_END, 1) + updated = f"{head}{GROUPS_START}\n{listing}\n{GROUPS_END}{tail}" + if updated == text: + return + run( + ["gh", "issue", "edit", str(change.issue), "--repo", REPO, "--body", updated], mutating=True + ) + print(f" refreshed #{change.issue} body") + + +# --------------------------------------------------------------------------- commands + + +def cmd_init(args) -> None: + change = load_change(args.change) + if change.issue is not None: + die(f"'{change.name}' already carries parent issue #{change.issue} β€” use `sync` instead") + + lines = read_tasks(change) + groups = parse_groups(lines) + title = args.title or f"{change.desc.replace('-', ' ')}" + + listing = "\n".join(f"- Group {g.number}: {g.github_title}" for g in groups) + marker = change_marker(change) + body = ( + f"Tracking issue for the `{change.service}-{change.type}-<this issue>-{change.desc}` " + f"OpenSpec change.\n\n" + f"One task group = one sub-issue = one PR, merged before the next group starts. " + f"This issue holds no task list of its own β€” see the sub-issues below.\n\n" + f"{GROUPS_START}\n{listing}\n{GROUPS_END}\n\n" + f"Artifacts: `openspec/changes/{change.service}-{change.type}-<this issue>-{change.desc}/` " + f"(proposal.md, design.md, tasks.md).\n\n" + f"{marker}" + ) + + if DRY_RUN: + print(f"[dry-run] parent issue: {title}") + print( + f"[dry-run] rename: {change.path} -> " + f"{CHANGES_DIR}/{change.service}-{change.type}-<issue>-{change.desc}" + ) + for group in groups: + print(f"[dry-run] sub-issue: {subissue_title(change, group)}") + return + + parent = find_parent_issue(marker) + if parent: + print(f"Resuming '{change.name}' β€” parent #{parent} already exists") + else: + print(f"Creating parent issue for '{change.name}' ({len(groups)} groups)") + parent = create_issue(title, body) + print(f" parent: #{parent} β€” {title}") + + # the rename comes first: until it lands, only the marker search can find this parent + new_name = f"{change.service}-{change.type}-{parent}-{change.desc}" + new_path = CHANGES_DIR / new_name + try: + tracked = run(["git", "ls-files", str(change.path)], check=False) + if tracked: + run(["git", "mv", str(change.path), str(new_path)], mutating=True) + else: + change.path.rename(new_path) + print(f" renamed {change.path} -> {new_path}") + + change = parse_change(new_name) + board_status(parent, "Todo") + run( + [ + "gh", + "issue", + "edit", + str(parent), + "--repo", + REPO, + "--body", + body.replace("<this issue>", str(parent)), + ], + mutating=True, + ) + + lines = read_tasks(change) + groups = parse_groups(lines) + states = reconcile(change, groups) + write_tasks(change, lines, groups, states) + refresh_parent_body(change, groups) + except (Exception, SystemExit): + recovery = ( + f"{sys.argv[0]} sync {new_name}" + if change.name == new_name + else f"{sys.argv[0]} init {args.change}" + ) + print( + f"\nparent #{parent} exists. Resume with:\n {recovery}", + file=sys.stderr, + ) + raise + + print(f"\nParent #{parent}: {ISSUE_URL}/{parent}") + print(f"Commit the rename + tasks.md on main, then: {sys.argv[0]} start {new_name} 1") + + +def cmd_sync(args) -> None: + change = load_change(args.change, require_issue=True) + lines = read_tasks(change) + groups = parse_groups(lines) + print(f"Syncing '{change.name}' against parent #{change.issue}") + board_ensure(change.issue) + states = reconcile(change, groups) + write_tasks(change, lines, groups, states) + refresh_parent_body(change, groups) + + +def cmd_start(args) -> None: + change = load_change(args.change, require_issue=True) + groups = parse_groups(read_tasks(change)) + group = next((g for g in groups if g.number == args.group), None) + if group is None: + die(f"no '## {args.group}.' group in {change.tasks_file}") + + if group.issue is None: + die( + f"group {args.group} has no sub-issue recorded in tasks.md.\n" + f" Run `{sys.argv[0]} sync {change.name}` on main and commit it first, so the\n" + f" issue number is on main rather than only on a feature branch." + ) + if not any(s["number"] == group.issue for s in sub_issues(change.issue)): + die( + f"#{group.issue} is recorded for group {args.group} but is not a sub-issue of " + f"#{change.issue}.\n Run `{sys.argv[0]} sync {change.name}` on main to repair it." + ) + + branch = branch_name(change, group) + if run(["git", "rev-parse", "--verify", "--quiet", f"refs/heads/{branch}"], check=False): + die(f"branch {branch} already exists β€” `git switch {branch}` to resume it") + if run(["git", "status", "--porcelain"]): + die("working tree is not clean β€” commit or stash before starting a group") + + print(f"Change: {change.name}") + print(f"Group: {group.number}. {group.github_title}") + print(f"Parent: #{change.issue} Sub-issue: #{group.issue}") + print(f"Branch: {branch} (from {args.base})\n") + + run(["git", "fetch", "origin", args.base], mutating=True) + run(["git", "checkout", args.base], mutating=True) + run(["git", "pull", "--ff-only", "origin", args.base], mutating=True) + run(["git", "checkout", "-b", branch], mutating=True) + print(f" on {branch}") + + board_status(change.issue, "In Progress", keep=("Done",)) + board_status(group.issue, "In Progress", keep=("Done",)) + + print(f"\nWhen the group is done, open the PR with: {sys.argv[0]} pr") + + +def find_change_for_issue(sub: int) -> tuple[Change, Group, list[Group]] | None: + for tasks_file in sorted(CHANGES_DIR.glob("*/tasks.md")): + if "archive" in tasks_file.parts: + continue + change = parse_change(tasks_file.parent.name, strict=False) + if change is None or change.issue is None: + continue + groups = parse_groups(tasks_file.read_text().splitlines()) + group = next((g for g in groups if g.issue == sub), None) + if group: + return change, group, groups + return None + + +def cmd_pr(args) -> None: + branch = args.branch or run(["git", "branch", "--show-current"]) + match = BRANCH_RE.match(branch) + if not match: + die(f"branch '{branch}' doesn't match <Service>/<type>/Issue-<n>/<description>") + sub = int(match.group(1)) + + found = find_change_for_issue(sub) + closes = [sub] + if found is None: + warn(f"no change in {CHANGES_DIR} records sub-issue #{sub}; closing it alone") + else: + change, group, groups = found + # Include groups added since branching, even if their titles were edited. + siblings = sub_issues(change.issue) + unfinished = [ + f"#{s['number']}" for s in siblings if s["number"] != sub and s["state"] != "closed" + ] + linked = {s["number"] for s in siblings} + uncreated = [ + f"group {g.number} (no linked sub-issue)" for g in groups if g.issue not in linked + ] + unfinished += uncreated + total = len(siblings) + len(uncreated) + tail = f"; still open: {', '.join(unfinished)}" if unfinished else "; last open group" + print(f"# {change.name} β€” group {group.number} of {total}{tail}", file=sys.stderr) + if not unfinished: + closes.append(change.issue) + + print("\n".join(f"Closes #{n}" for n in closes)) + + +def cmd_issue(args) -> None: + number = create_issue(args.title, args.body or "") + if number is None: + return + board_status(number, "Todo") + print(f"{ISSUE_URL}/{number}") + + +# --------------------------------------------------------------------------- cli + + +def main() -> None: + global DRY_RUN + parser = argparse.ArgumentParser( + prog="scripts/flow.py", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--dry-run", action="store_true", help="print mutating commands only") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("init", help="create parent issue + every group sub-issue") + p.add_argument("change") + p.add_argument("--title", help="parent issue title (default: derived from the change name)") + p.set_defaults(func=cmd_init) + + p = sub.add_parser("sync", help="reconcile groups against GitHub (idempotent)") + p.add_argument("change") + p.set_defaults(func=cmd_sync) + + p = sub.add_parser("start", help="move to In Progress and branch") + p.add_argument("change") + p.add_argument("group", type=int) + p.add_argument("--base", default="main", help="branch off this ref instead of main") + p.set_defaults(func=cmd_start) + + p = sub.add_parser("pr", help="print the Closes trailer for this branch") + p.add_argument("--branch", help="inspect this branch instead of the current one") + p.set_defaults(func=cmd_pr) + + p = sub.add_parser("issue", help="create a standalone issue on the board") + p.add_argument("title") + p.add_argument("body", nargs="?") + p.set_defaults(func=cmd_issue) + + args = parser.parse_args() + DRY_RUN = args.dry_run + if not Path(".git").exists(): + die("run from the repository root") + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/git-hooks/pre-push b/scripts/git-hooks/pre-push index 7abde34..3723202 100755 --- a/scripts/git-hooks/pre-push +++ b/scripts/git-hooks/pre-push @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mirrors .github/workflows/{ai,budget,chat,users,shared,worker}.yml's lint +# Mirrors .github/workflows/{ai,budget,chat,users,shared,tooling,worker}.yml's lint # steps locally, scoped to whatever's actually being pushed. See # docs/development/WORKFLOW.md for setup (git config core.hooksPath). set -euo pipefail @@ -17,7 +17,7 @@ while read -r local_ref local_sha remote_ref remote_sha; do branch="${local_ref#refs/heads/}" if [[ "$branch" != "main" && ! "$branch" =~ $branch_name_re ]]; then echo "error: branch '${branch}' doesn't match <Service>/<type>/Issue-<n>/<description>" >&2 - echo " create branches with scripts/start-group.sh (docs/development/WORKFLOW.md)" >&2 + echo " create branches with scripts/flow.py start <change-name> <N> (docs/development/WORKFLOW.md)" >&2 exit 1 fi @@ -42,6 +42,7 @@ while read -r local_ref local_sha remote_ref remote_sha; do services/chat/*) areas["chat"]=1 ;; services/users/*) areas["users"]=1 ;; services/worker/*) areas["worker"]=1 ;; + scripts/*) areas["scripts"]=1 ;; shared/*) areas["ai"]=1; areas["budget"]=1; areas["chat"]=1; areas["users"]=1; areas["shared"]=1 ;; esac done <<< "$changed" @@ -80,6 +81,12 @@ for area in "${!areas[@]}"; do run black --check shared/ai_client shared/tests run flake8 --max-line-length=100 shared/ai_client shared/tests ;; + scripts) + echo "== scripts ==" + run black --check scripts/ + run flake8 --max-line-length=100 scripts/ + run python3 -m pytest scripts/test_flow.py -q + ;; esac done diff --git a/scripts/new-issue.sh b/scripts/new-issue.sh deleted file mode 100755 index bc47c6d..0000000 --- a/scripts/new-issue.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Usage: ./scripts/new-issue.sh "Issue title" "Issue body" -# ./scripts/new-issue.sh (interactive prompts) -set -euo pipefail - -REPO="arutsh/OpenGrantFlow" -PROJECT_NUMBER=8 -PROJECT_OWNER="arutsh" - -if [[ $# -ge 1 ]]; then - TITLE="$1" -else - read -rp "Title: " TITLE -fi - -if [[ $# -ge 2 ]]; then - BODY="$2" -else - echo "Body (end with a line containing only '.'):" - BODY="" - while IFS= read -r line; do - [[ "$line" == "." ]] && break - BODY+="$line"$'\n' - done -fi - -URL=$(gh issue create --repo "$REPO" --title "$TITLE" --body "$BODY") -echo "Created: $URL" - -gh project item-add "$PROJECT_NUMBER" --owner "$PROJECT_OWNER" --url "$URL" -echo "Added to project board." diff --git a/scripts/start-group.sh b/scripts/start-group.sh deleted file mode 100755 index b74cf4c..0000000 --- a/scripts/start-group.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash -# Usage: ./scripts/start-group.sh <change-name> <group-number> - see docs/development/WORKFLOW.md -set -euo pipefail - -REPO="arutsh/OpenGrantFlow" -PROJECT_NUMBER=8 -PROJECT_OWNER="arutsh" -PROJECT_ID="PVT_kwHOAQDyGs4A-5Lx" -STATUS_FIELD_ID="PVTSSF_lAHOAQDyGs4A-5LxzgyKO-Q" -STATUS_IN_PROGRESS="47fc9ee4" - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 <change-name> <group-number>" >&2 - exit 1 -fi - -CHANGE="$1" -GROUP="$2" - -TASKS_FILE="openspec/changes/${CHANGE}/tasks.md" - -if [[ ! -f "$TASKS_FILE" ]]; then - echo "error: $TASKS_FILE not found" >&2 - exit 1 -fi - -if [[ ! "$CHANGE" =~ ^([a-z]+)-(feat|fix|chore|refactor)-([0-9]+)-(.+)$ ]]; then - echo "error: change name '$CHANGE' doesn't match <service>-<type>-<issue>-<description>" >&2 - echo " type must be one of feat/fix/chore/refactor, issue must be numeric" >&2 - exit 1 -fi - -SERVICE_RAW="${BASH_REMATCH[1]}" -TYPE="${BASH_REMATCH[2]}" -PARENT_ISSUE="${BASH_REMATCH[3]}" -DESC="${BASH_REMATCH[4]}" - -GROUP_LINE_NUM=$(grep -nE "^## ${GROUP}\." "$TASKS_FILE" | head -1 | cut -d: -f1) -if [[ -z "$GROUP_LINE_NUM" ]]; then - echo "error: no '## ${GROUP}.' group header found in $TASKS_FILE" >&2 - exit 1 -fi -GROUP_LINE=$(sed -n "${GROUP_LINE_NUM}p" "$TASKS_FILE") - -case "$SERVICE_RAW" in - ai) SERVICE="AI" ;; - backend) SERVICE="Backend" ;; - budget) SERVICE="Budget" ;; - chat) SERVICE="Chat" ;; - feature) SERVICE="Feature" ;; - frontend) SERVICE="Frontend" ;; - platform) SERVICE="Platform" ;; - shared) SERVICE="Shared" ;; - users) SERVICE="Users" ;; - *) - echo "error: unknown service '$SERVICE_RAW' - add it to the case map in $0" >&2 - exit 1 - ;; -esac - -# Group header carries its own sub-issue once created: "## N. Title β€” Issue #123" -if [[ "$GROUP_LINE" =~ Issue\ \#([0-9]+) ]]; then - SUB_ISSUE="${BASH_REMATCH[1]}" - GROUP_TITLE=$(sed -E "s/^## ${GROUP}\. //; s/ β€” Issue #[0-9]+\$//" <<< "$GROUP_LINE") - echo "Reusing existing sub-issue #${SUB_ISSUE} for group ${GROUP}" -else - GROUP_TITLE=$(sed -E "s/^## ${GROUP}\. //" <<< "$GROUP_LINE") - SUB_TITLE="${DESC}: ${GROUP_TITLE} (group ${GROUP})" - SUB_BODY="Part of #${PARENT_ISSUE} (\`${CHANGE}\` OpenSpec change). See openspec/changes/${CHANGE}/tasks.md, group ${GROUP}." - - SUB_URL=$(gh issue create --repo "$REPO" --title "$SUB_TITLE" --body "$SUB_BODY") - SUB_ISSUE="${SUB_URL##*/}" - echo "Created sub-issue #${SUB_ISSUE}: ${SUB_TITLE}" - - gh project item-add "$PROJECT_NUMBER" --owner "$PROJECT_OWNER" --url "$SUB_URL" >/dev/null - echo "Added #${SUB_ISSUE} to project board" - - PARENT_DB_ID=$(gh api "repos/${REPO}/issues/${PARENT_ISSUE}" --jq .id) - SUB_DB_ID=$(gh api "repos/${REPO}/issues/${SUB_ISSUE}" --jq .id) - gh api "repos/${REPO}/issues/${PARENT_ISSUE}/sub_issues" --method POST -F "sub_issue_id=${SUB_DB_ID}" >/dev/null - echo "Linked #${SUB_ISSUE} as a sub-issue of #${PARENT_ISSUE}" - - sed -i "${GROUP_LINE_NUM}s/\$/ β€” Issue #${SUB_ISSUE}/" "$TASKS_FILE" - echo "Recorded Issue #${SUB_ISSUE} on group ${GROUP}'s header in $TASKS_FILE" -fi - -BRANCH="${SERVICE}/${TYPE}/Issue-${SUB_ISSUE}/${DESC}-group${GROUP}" - -echo -echo "Change: $CHANGE" -echo "Group: ${GROUP}. ${GROUP_TITLE}" -echo "Parent: #${PARENT_ISSUE}" -echo "Sub-issue: #${SUB_ISSUE}" -echo "Branch: ${BRANCH}" -echo - -if git show-ref --verify --quiet "refs/heads/${BRANCH}"; then - echo "error: branch ${BRANCH} already exists" >&2 - exit 1 -fi - -if [[ -n "$(git status --porcelain)" ]]; then - echo "error: working tree not clean - commit or stash before starting a new group" >&2 - exit 1 -fi - -git fetch origin main -git checkout main -git pull --ff-only origin main -git checkout -b "$BRANCH" - -echo "Created and checked out $BRANCH" - -ITEM_ID=$(gh project item-list "$PROJECT_NUMBER" --owner "$PROJECT_OWNER" --limit 200 --format json \ - | jq -r --argjson issue "$SUB_ISSUE" '.items[] | select(.content.number == $issue) | .id') - -if [[ -z "$ITEM_ID" ]]; then - echo "warning: no project item found for issue #${SUB_ISSUE}; skipping status update" >&2 -else - gh project item-edit --id "$ITEM_ID" --project-id "$PROJECT_ID" \ - --field-id "$STATUS_FIELD_ID" --single-select-option-id "$STATUS_IN_PROGRESS" - echo "Set project item for issue #${SUB_ISSUE} to In Progress" -fi - -echo -echo "Next: implement group ${GROUP}, then open a PR with 'Closes #${SUB_ISSUE}'." -echo "Once every sub-issue of #${PARENT_ISSUE} is closed, close #${PARENT_ISSUE} too." diff --git a/scripts/test_flow.py b/scripts/test_flow.py new file mode 100644 index 0000000..4876069 --- /dev/null +++ b/scripts/test_flow.py @@ -0,0 +1,242 @@ +"""Tests for scripts/flow.py's parsing and tasks.md rewriting (no network).""" + +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +spec = importlib.util.spec_from_file_location("flow", Path(__file__).with_name("flow.py")) +flow = importlib.util.module_from_spec(spec) +sys.modules["flow"] = flow # dataclasses resolves annotations through sys.modules +spec.loader.exec_module(flow) + + +TASKS = """# Tasks + +Workflow rule: one task group = one GitHub sub-issue = one PR. + +## 1. Export endpoint scaffolding β€” Issue #316 + +- [x] 1.1 Do the thing + +## 2. Dashboard sheet β€” depends on 1 β€” Issue #318 + +- [ ] 2.1 Do the other thing + +## 3. Frontend button β€” depends on 1, 2 +""" + + +@pytest.mark.parametrize( + "name,service,type_,issue,desc", + [ + ("budget-feat-313-excel-export", "budget", "feat", 313, "excel-export"), + ("shared-feat-292-audit-mixin", "shared", "feat", 292, "audit-mixin"), + ("budget-feat-customer-profile-cache", "budget", "feat", None, "customer-profile-cache"), + ("users-fix-2fa-enrolment", "users", "fix", None, "2fa-enrolment"), + ], +) +def test_parse_change(name, service, type_, issue, desc): + change = flow.parse_change(name) + assert (change.service, change.type, change.issue, change.desc) == (service, type_, issue, desc) + + +@pytest.mark.parametrize( + "name", ["ai-provider-model-catalog-api", "async-privileged-access-audit", "nope-feat-x"] +) +def test_parse_change_rejects_non_conforming(name): + assert flow.parse_change(name, strict=False) is None + with pytest.raises(SystemExit): + flow.parse_change(name) + + +def test_parse_groups_reads_numbers_titles_and_issues(): + groups = flow.parse_groups(TASKS.splitlines()) + assert [g.number for g in groups] == [1, 2, 3] + assert [g.issue for g in groups] == [316, 318, None] + assert groups[1].title == "Dashboard sheet β€” depends on 1" + assert groups[1].github_title == "Dashboard sheet" + + +def test_subissue_title_drops_the_depends_suffix(): + change = flow.parse_change("budget-feat-313-excel-export") + group = flow.parse_groups(TASKS.splitlines())[1] + assert flow.subissue_title(change, group) == "excel-export: Dashboard sheet (group 2)" + + +def test_branch_name(): + change = flow.parse_change("budget-feat-313-excel-export") + group = flow.parse_groups(TASKS.splitlines())[0] + assert flow.branch_name(change, group) == "Budget/feat/Issue-316/excel-export-group1" + + +def _write(tmp_path, monkeypatch, text=TASKS): + monkeypatch.chdir(tmp_path) + change = flow.parse_change("budget-feat-313-excel-export") + change.tasks_file.parent.mkdir(parents=True) + change.tasks_file.write_text(text) + return change + + +def test_write_tasks_inserts_tracking_block_before_the_first_group(tmp_path, monkeypatch): + change = _write(tmp_path, monkeypatch) + lines = flow.read_tasks(change) + groups = flow.parse_groups(lines) + flow.write_tasks(change, lines, groups, {316: "closed", 318: "open"}) + + out = change.tasks_file.read_text() + assert out.index(flow.TRACK_START) < out.index("## 1.") + assert "| 1. Export endpoint scaffolding | [#316]" in out + assert "`Budget/feat/Issue-316/excel-export-group1`" in out + assert "| 3. Frontend button | β€” | β€” | not created |" in out + assert "Workflow rule:" in out + assert "\n\n\n" not in out, "tracking block left a blank-line run behind" + + +def test_write_tasks_is_idempotent_and_replaces_a_stale_block(tmp_path, monkeypatch): + change = _write(tmp_path, monkeypatch) + for states in ({316: "closed", 318: "open"}, {316: "closed", 318: "closed"}): + lines = flow.read_tasks(change) + groups = flow.parse_groups(lines) + flow.write_tasks(change, lines, groups, states) + + out = change.tasks_file.read_text() + assert out.count(flow.TRACK_START) == 1 + assert out.count("## 1. Export endpoint scaffolding β€” Issue #316") == 1 + assert "| [#318](" in out and "| closed |" in out + + +def test_write_tasks_records_newly_created_issue_numbers(tmp_path, monkeypatch): + change = _write(tmp_path, monkeypatch) + lines = flow.read_tasks(change) + groups = flow.parse_groups(lines) + groups[2].issue = 400 + flow.write_tasks(change, lines, groups, {400: "open"}) + assert "## 3. Frontend button β€” depends on 1, 2 β€” Issue #400" in change.tasks_file.read_text() + + +def test_parse_groups_requires_at_least_one_group(): + with pytest.raises(SystemExit): + flow.parse_groups(["# Tasks", "", "no groups here"]) + + +def test_write_tasks_refuses_a_half_deleted_tracking_block(tmp_path, monkeypatch): + damaged = TASKS.replace("# Tasks", f"# Tasks\n\n{flow.TRACK_START}\n## Tracking\n") + change = _write(tmp_path, monkeypatch, damaged) + lines = flow.read_tasks(change) + groups = flow.parse_groups(lines) + with pytest.raises(SystemExit): + flow.write_tasks(change, lines, groups, {316: "closed"}) + assert change.tasks_file.read_text() == damaged, "damaged file was rewritten anyway" + + +def _sub(number, group, state="open", desc="excel-export", title="Thing"): + return {"number": number, "state": state, "title": f"{desc}: {title} (group {group})"} + + +def _match(monkeypatch, tasks, subs): + monkeypatch.setattr(flow, "sub_issues", lambda parent: subs) + change = flow.parse_change("budget-feat-313-excel-export") + groups = flow.parse_groups(tasks.splitlines()) + return {n: s["number"] for n, s in flow.match_groups(change, groups).items()} + + +def test_match_groups_uses_the_title_suffix_when_no_number_is_recorded(monkeypatch): + tasks = "## 1. A\n\n## 2. B\n" + assert _match(monkeypatch, tasks, [_sub(316, 1), _sub(318, 2)]) == {1: 316, 2: 318} + + +def test_match_groups_prefers_the_recorded_number_over_a_stale_suffix(monkeypatch): + # group 2 was renumbered to 3 in tasks.md; its sub-issue still says "(group 2)" + tasks = "## 1. A β€” Issue #316\n\n## 3. B β€” Issue #318\n" + assert _match(monkeypatch, tasks, [_sub(316, 1), _sub(318, 2)]) == {1: 316, 3: 318} + + +def test_match_groups_never_claims_one_sub_issue_twice(monkeypatch): + # a group left pointing at its neighbour's issue must not steal it + tasks = "## 1. A β€” Issue #316\n\n## 2. B β€” Issue #316\n" + assert _match(monkeypatch, tasks, [_sub(316, 1), _sub(318, 2)]) == {1: 316, 2: 318} + + +def test_match_groups_recovers_a_recorded_unlinked_issue(monkeypatch): + tasks = "## 1. A β€” Issue #999\n" + monkeypatch.setattr(flow, "gh_json", Mock(return_value=_sub(999, 1))) + link = Mock() + monkeypatch.setattr(flow, "link_sub_issue", link) + assert _match(monkeypatch, tasks, [_sub(316, 1)]) == {1: 999} + link.assert_called_once_with(313, 999) + + +def test_match_groups_leaves_an_unmatched_group_for_creation(monkeypatch): + tasks = "## 1. A β€” Issue #316\n\n## 2. B\n" + assert _match(monkeypatch, tasks, [_sub(316, 1)]) == {1: 316} + + +def test_change_marker_is_stable_across_the_init_rename(): + before = flow.change_marker(flow.parse_change("budget-feat-excel-export")) + after = flow.change_marker(flow.parse_change("budget-feat-313-excel-export")) + assert before == after == "<!-- flow:change:budget-feat-excel-export -->" + + +@pytest.mark.parametrize("failure", ["link", "board"]) +def test_reconcile_retries_without_creating_another_issue(tmp_path, monkeypatch, failure): + change = _write(tmp_path, monkeypatch, "## 1. A\n") + subs = [] + monkeypatch.setattr(flow, "sub_issues", lambda parent: list(subs)) + create = Mock(return_value=400) + monkeypatch.setattr(flow, "create_issue", create) + monkeypatch.setattr(flow, "gh_json", Mock(return_value=_sub(400, 1))) + monkeypatch.setattr(flow, "board_ensure", Mock()) + + def link(parent, child): + assert flow.parse_groups(flow.read_tasks(change))[0].issue == child + subs.append(_sub(child, 1)) + + monkeypatch.setattr(flow, "link_sub_issue", Mock(side_effect=SystemExit(1))) + if failure == "board": + monkeypatch.setattr(flow, "link_sub_issue", link) + monkeypatch.setattr(flow, "board_status", Mock(side_effect=SystemExit(1))) + with pytest.raises(SystemExit): + flow.reconcile(change, flow.parse_groups(flow.read_tasks(change))) + + monkeypatch.setattr(flow, "link_sub_issue", link) + states = flow.reconcile(change, flow.parse_groups(flow.read_tasks(change))) + assert states == {400: "open"} + assert len(subs) == 1 + create.assert_called_once() + + +def test_reconcile_stops_when_recorded_issue_cannot_be_linked(tmp_path, monkeypatch): + change = _write(tmp_path, monkeypatch, "## 1. A β€” Issue #999\n") + monkeypatch.setattr(flow, "sub_issues", Mock(return_value=[_sub(316, 1)])) + monkeypatch.setattr(flow, "gh_json", Mock(return_value=_sub(999, 1))) + monkeypatch.setattr(flow, "link_sub_issue", Mock(side_effect=SystemExit(1))) + create = Mock() + monkeypatch.setattr(flow, "create_issue", create) + with pytest.raises(SystemExit): + flow.reconcile(change, flow.parse_groups(flow.read_tasks(change))) + create.assert_not_called() + assert flow.parse_groups(flow.read_tasks(change))[0].issue == 999 + + +@pytest.mark.parametrize( + "siblings,close_parent", + [ + ([_sub(316, 1), {"number": 318, "title": "Renamed", "state": "open"}], False), + ([_sub(316, 1)], False), + ([_sub(318, 2, state="closed")], False), + ([_sub(316, 1), _sub(318, 2, state="closed")], True), + ([_sub(316, 1), _sub(318, 2, state="closed"), _sub(320, 3)], False), + ], +) +def test_pr_closes_parent_only_when_all_groups_are_linked_and_finished( + tmp_path, monkeypatch, capsys, siblings, close_parent +): + _write(tmp_path, monkeypatch, "## 1. A β€” Issue #316\n## 2. B β€” Issue #318\n") + monkeypatch.setattr(flow, "sub_issues", Mock(return_value=siblings)) + flow.cmd_pr(SimpleNamespace(branch="Budget/feat/Issue-316/excel-export-group1")) + lines = capsys.readouterr().out.splitlines() + assert lines == (["Closes #316", "Closes #313"] if close_parent else ["Closes #316"]) From e9cfcb911358367808fec286f77fda800f9a6590 Mon Sep 17 00:00:00 2001 From: Norair Arutshyan <n.arutshyan@gmail.com> Date: Thu, 24 Sep 2026 11:10:35 +0100 Subject: [PATCH 2/2] fix(ci): scope tooling.yml's GITHUB_TOKEN to contents:read Addresses two CodeQL findings (workflow does not contain permissions) by declaring the minimal permissions this checkout-and-test workflow needs. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --- .github/workflows/tooling.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index 114806d..ecffbec 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main, develop] +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true