From c43e749990f854e4647ef9052da5c7bb7c49ee58 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Wed, 23 Sep 2026 13:11:53 +1000 Subject: [PATCH 1/2] harness: reset lists open PRs with --limit 200 and stops if any survive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh pr list defaults to 30; a lane past that kept its oldest PR through the reset and it read as a missing verdict in the v0.29.2 tally. Post-reset check makes any survivor — page limit, failed listing, failed close — a hard stop before fixture PRs are created. Closes #321. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 ++ tool-test-action-on-github/README.md | 3 +- .../test-action-on-github.sh | 40 +++++++++++++++++-- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afd5304..ba26c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Harness reset no longer misses open PRs beyond the first page** (#321): the three `gh pr list` calls in `test-action-on-github.sh` had no `--limit`, so a repo with more than 30 open PRs kept its oldest through the reset — on 2026-09-21 the `.ml` lane had 31 after diagnosis work between two gates, and the survivor read as a missing verdict in the v0.29.2 tally. Now `--limit 200`, and a post-reset check that stops the run if any repo still has an open PR (a failed listing counts too: it no longer reads as "nothing to close"). + ## [0.29.2] - 2026-09-21 ### Fixed diff --git a/tool-test-action-on-github/README.md b/tool-test-action-on-github/README.md index 4217975..55f9346 100644 --- a/tool-test-action-on-github/README.md +++ b/tool-test-action-on-github/README.md @@ -76,7 +76,7 @@ cd /path/to/action-translation/tool-test-action-on-github ``` The script will: -1. Reset test repositories to clean state +1. Reset test repositories to clean state — every open PR on the source repo and each target repo is closed, and the script stops if any is still open afterwards (a survivor would be counted as one of this run's PRs) 2. Run 28 automated test scenarios 3. Create PRs in source repository with `test-translation` label 4. Label triggers action → creates translation PRs in **every** target repository @@ -239,6 +239,7 @@ Evaluation reports are saved to `reports/`: **Script fails to reset repositories:** - Check GitHub CLI authentication: `gh auth status` - Verify repository access permissions +- "Reset left open PRs behind": the post-reset check found a PR still open on the repo it names. Close it by hand (or work out why `gh` could not) and run the script again; do not create fixture PRs on top of it **PRs not created:** - Check source repository workflow configuration diff --git a/tool-test-action-on-github/test-action-on-github.sh b/tool-test-action-on-github/test-action-on-github.sh index c36292e..967854a 100755 --- a/tool-test-action-on-github/test-action-on-github.sh +++ b/tool-test-action-on-github/test-action-on-github.sh @@ -617,9 +617,15 @@ if [ "$DRY_RUN" = false ]; then cd "$WORK_DIR/$SOURCE_REPO" fi -# Close PRs on source repo +# Close PRs on source repo. +# Every `gh pr list` here carries --limit 200: the default is 30, and a lane that +# has had diagnosis work between two gates (a re-run, a local-bundle validation) +# can hold more open PRs than that. On 2026-09-21 the .ml lane had 31, the oldest +# was never listed, and it survived the reset to be read as a missing verdict +# (#321). The check after the close loops makes any survivor, whatever its +# cause, stop the run before a fixture PR is created. if [ "$DRY_RUN" = true ]; then - OPEN_PRS=$(gh pr list --repo "$OWNER/$SOURCE_REPO" --state open --json number --jq '.[].number' 2>/dev/null || echo "") + OPEN_PRS=$(gh pr list --repo "$OWNER/$SOURCE_REPO" --state open --limit 200 --json number --jq '.[].number' 2>/dev/null || echo "") if [ -z "$OPEN_PRS" ]; then echo -e "${CYAN}[DRY RUN] No open PRs to close on source repo${NC}" else @@ -630,7 +636,7 @@ if [ "$DRY_RUN" = true ]; then fi else # Get list of open PRs - OPEN_PRS=$(gh pr list --repo "$OWNER/$SOURCE_REPO" --state open --json number --jq '.[].number') + OPEN_PRS=$(gh pr list --repo "$OWNER/$SOURCE_REPO" --state open --limit 200 --json number --jq '.[].number') if [ -z "$OPEN_PRS" ]; then echo "No open PRs to close on source repo" @@ -652,7 +658,7 @@ for L in "${LANGUAGES[@]}"; do name="$(lang_name "$L")" repo="$OWNER/$SOURCE_REPO.$code" - TARGET_PRS=$(gh pr list --repo "$repo" --state open --json number --jq '.[].number' 2>/dev/null || echo "") + TARGET_PRS=$(gh pr list --repo "$repo" --state open --limit 200 --json number --jq '.[].number' 2>/dev/null || echo "") if [ -z "$TARGET_PRS" ]; then echo "No open PRs to close on $name target repo" continue @@ -667,6 +673,32 @@ for L in "${LANGUAGES[@]}"; do done done +# Verify the reset actually emptied every repo. A survivor here — a PR beyond the +# list page, a listing that failed and read as empty, a close that did not take — +# would be counted as one of this run's PRs and read as a missing verdict, so it +# is a hard stop, not a warning. +if [ "$DRY_RUN" != true ]; then + REPOS_TO_CHECK=("$OWNER/$SOURCE_REPO") + for L in "${LANGUAGES[@]}"; do + REPOS_TO_CHECK+=("$OWNER/$SOURCE_REPO.$(lang_code "$L")") + done + RESET_LEFTOVERS="" + for repo in "${REPOS_TO_CHECK[@]}"; do + # No `|| echo ""` here: if the listing itself fails, that is a failure too. + REMAINING=$(gh pr list --repo "$repo" --state open --limit 200 --json number --jq '.[].number') + if [ -n "$REMAINING" ]; then + RESET_LEFTOVERS="${RESET_LEFTOVERS} ${repo}: #$(echo "$REMAINING" | tr '\n' ' ' | sed 's/ $//; s/ / #/g')\n" + fi + done + if [ -n "$RESET_LEFTOVERS" ]; then + echo -e "${RED}✗ Reset left open PRs behind — stopping before any fixture PR is created:${NC}" + echo -e "$RESET_LEFTOVERS" + echo -e "${RED} Close them (or find out why gh could not) and run the script again.${NC}" + exit 1 + fi + echo -e "${GREEN}✓${NC} All ${#REPOS_TO_CHECK[@]} repos have no open PRs" +fi + echo "" # From 88f6070954b19f3affca0f35c1b56b5f4f768e96 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Wed, 23 Sep 2026 13:46:43 +1000 Subject: [PATCH 2/2] harness: a failed listing in the post-reset check names the repo and the error Co-Authored-By: Claude Fable 5.1 --- tool-test-action-on-github/test-action-on-github.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tool-test-action-on-github/test-action-on-github.sh b/tool-test-action-on-github/test-action-on-github.sh index 967854a..0f95841 100755 --- a/tool-test-action-on-github/test-action-on-github.sh +++ b/tool-test-action-on-github/test-action-on-github.sh @@ -684,8 +684,13 @@ if [ "$DRY_RUN" != true ]; then done RESET_LEFTOVERS="" for repo in "${REPOS_TO_CHECK[@]}"; do - # No `|| echo ""` here: if the listing itself fails, that is a failure too. - REMAINING=$(gh pr list --repo "$repo" --state open --limit 200 --json number --jq '.[].number') + # A listing that fails is a failure too — but say which repo and why, rather than + # letting `set -e` stop the run silently on the assignment. + if ! REMAINING=$(gh pr list --repo "$repo" --state open --limit 200 --json number --jq '.[].number' 2>&1); then + echo -e "${RED}✗ Could not list open PRs on ${repo}: ${REMAINING}${NC}" + echo -e "${RED} The reset cannot be verified — fix the listing and run the script again.${NC}" + exit 1 + fi if [ -n "$REMAINING" ]; then RESET_LEFTOVERS="${RESET_LEFTOVERS} ${repo}: #$(echo "$REMAINING" | tr '\n' ' ' | sed 's/ $//; s/ / #/g')\n" fi