From 612a989c94e9a155b500fcfd94978264d119a440 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 21 Nov 2026 18:28:53 -0500 Subject: [PATCH 1/2] fix(source-control): let a trusted spec amendment clear an earlier verify FAIL A counted pass cleared verify:earlier-fail only when the change differed from every FAILed head's change, so a FAIL against a done-when item that a trusted person later amended never cleared while the code stayed the same (pull request #18023 against issue #17908). _spec_amended clears a FAILed head of the same change when every FAIL there and every pass at the head record the same linked issues, and for each pass a trusted person amended one of them after the last FAIL's creation and up to the pass run's creation: a body edit whose last editor the pass records as trusted, or a spec comment (_spec_comments trust) created then. Requiring the same linked issues leaves relinks to _superseded, so an untrusted relink to a freshly commented issue clears nothing. No new GitHub read: the comments' createdAt comes from the closing-issues read already made, and the body editor from the pass's verdict record. Any failed read keeps the refusal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RAoAtUnz4WHB5auUnwQYq7 --- ...-18023-earlier-fail-spec-amend-4b9d2e71.md | 7 ++ .../scripts/babysit_verify_evidence.py | 81 ++++++++++++++++- .../tests/test_babysit_verify_evidence.py | 86 +++++++++++++++++++ 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 .changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md diff --git a/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md b/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md new file mode 100644 index 0000000000..daf4e11af6 --- /dev/null +++ b/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- The babysit merge gate clears `verify:earlier-fail` for a pass at a head of the same change when a trusted person amended a linked issue's spec (a body edit or a new spec comment) after the FAIL's run and before the pass's run was created, so a FAIL against a done-when item that was later amended no longer blocks the pull request forever. Untrusted edits or comments, a pass judged before the amendment, and any failed read keep the refusal ([#18023](https://github.com/melodic-software/claude-code-plugins/pull/18023)). diff --git a/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py b/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py index 535b12f7f9..9268dd8065 100755 --- a/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py +++ b/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py @@ -81,7 +81,12 @@ body change alone) and, for each counted pass at the head, a trusted person edited the body from the last such FAIL's creation to the pass run's creation and only trusted people edited the body or connected - or disconnected an issue up to its completion (`_body_fixed`). Any + or disconnected an issue up to its completion (`_body_fixed`), or every + counted FAIL there and every counted pass at the head record the same + linked issues and, for each pass, a trusted person amended one of them + from the last such FAIL's creation to the pass run's creation: a body + edit, last made by a trusted person as the pass records it, or a spec + comment (`_spec_comments`) created then (`_spec_amended`). Any other failed read of a run time or a diff refuses as the next item does; - a FAIL-named artifact tied to this pull request by number or SHA whose run cannot be read: `verify:unreadable`. An untied FAIL whose run cannot be read @@ -4392,6 +4397,7 @@ def capped(heads: set[str]) -> str: number, sha, fail_artifacts, passes, judged_at=head ) or self._body_fixed(number, sha, fail_artifacts, passes) + or self._spec_amended(number, sha, fail_artifacts, passes) ), ) ): @@ -4971,6 +4977,79 @@ def _body_fixed( return False return True + def _spec_amended( + self, + number: int, + sha: str, + fail_artifacts: dict[str, list[tuple[int, str, Any]]], + passes: list[tuple[int, str, Any]], + ) -> bool: + """Every counted FAIL at `sha` records this pull request, head and + verdict, each counted pass at the head records the same linked + issues as each FAIL, and for each pass a trusted person changed one + of those issues' spec after the last FAIL's creation, up to the pass + run's creation: its body, last edited by a trusted person + (`_trusted_actor`) as the pass records it, or a spec comment + (`_spec_comments`) created on it now. The pass judged an amended + spec the FAIL never read, which the same change cannot carry; a + relink stays `_superseded`'s. A record, time or read that fails is + False.""" + fails = fail_artifacts.get(sha, []) + if not passes or not fails: + return False + try: + judged: set[frozenset[int]] = set() + for run_id, name, artifact in fails: + record = self._record(run_id, name, artifact) + if ( + record.get("pr") != number + or record.get("head_sha") != sha + or record.get("verdict") != "fail" + ): + return False + judged.add(_linked_numbers(record)) + if len(judged) != 1: + return False + issues = next(iter(judged)) + created = [self._fresh_created(r) for r, _, _ in fails] + if None in created: + return False + last = max(c for c in created if c) + live = self._live_issues.get(number) + if live is None: + live = self._live_issues[number] = self._closing_issues(number) + commented = [] + for n in issues & set(live): + nodes = json_array(json_object(live[n].get("comments")).get("nodes")) + trusted = self._spec_comments(nodes) + commented += [ + postmerge._time(c.get("createdAt")) + for c in map(json_object, nodes) + if (c.get("databaseId"), c.get("updatedAt")) in trusted + ] + for run_id, name, artifact in passes: + record = self._record(run_id, name, artifact) + if _linked_numbers(record) != issues: + return False + edited = [ + postmerge._time(issue.get("last_edited_at")) + for issue in map( + json_object, json_array(record.get("linked_issues")) + ) + if issue.get("last_edited_at") is not None + and self._trusted_actor( + issue.get("editor_type"), issue.get("editor_id") + ) + ] + begun = self._fresh_created(run_id) + if begun is None or not any( + last < at <= begun for at in commented + edited + ): + return False + except (_Unreadable, postmerge.PostMergeReadError): + return False + return True + def _relinked_by_people( self, number: int, since: datetime, until: datetime, *, body_edit: bool = False ) -> bool: diff --git a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py index dae4563efc..aa56f124ef 100644 --- a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py +++ b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py @@ -1724,6 +1724,92 @@ def test_a_pass_created_before_the_fail_keeps_it(self) -> None: self.assertEqual(read(fake).refusal, "verify:earlier-fail") +class ATrustedSpecAmendmentClearsAnEarlierFailOnTheSameChange(unittest.TestCase): + """Pull request #18023's shape: a FAIL at an earlier head against issue + #17908's done-when item, the item amended by a trusted comment after the + FAIL's run, and a pass at a head of the same change whose run was + created after the comment. The pass judged the amended spec, so it + clears the FAIL (`_spec_amended`); an untrusted amendment or a pass + created before it keep `verify:earlier-fail`.""" + + STRANGER = {"__typename": "User", "login": "stranger", "databaseId": 999} + + @staticmethod + def amended( + author: dict[str, Any] = HUMAN, at: str = RELINK_AT, recorded: bool = True + ) -> FakeGitHub: + """A FAIL at EARLIER (run 2, created OLDER_START) noting a file, a + pass at HEAD (run 1, created RUN_STARTED) of the same change, no PR + body edit or relink, and comment 9 on issue 7 by `author` created at + `at`, which the pass records as spec when `recorded`.""" + fake = AFixedBodyClearsABodyOnlyFailOnTheSameChange.body_fail( + [NOTE_ON_A_FILE], relink_answer() + ) + fake.closing_comments[7] = [ + comment(9, HUMAN, at, createdAt=at) | {"author": author} + ] + spec = [{"id": 9, "updated_at": at}] if recorded else [] + fake.records[named(HEAD, "pass")] = record( + linked_issues=[{**TRUSTED_ISSUE, "spec_comments": spec}] + ) + return fake + + def test_pr_18023s_fail_clears_after_a_trusted_spec_comment(self) -> None: + evidence = read(self.amended()) + self.assertIsNone(evidence.refusal) + self.assertEqual(evidence.report()["earlierFails"], [EARLIER]) + + def test_a_trusted_issue_body_edit_after_the_fail_clears_it(self) -> None: + fake = AFixedBodyClearsABodyOnlyFailOnTheSameChange.body_fail( + [NOTE_ON_A_FILE], relink_answer() + ) + fake.closing_edited[7] = RELINK_AT + fake.records[named(HEAD, "pass")] = record( + linked_issues=[ + { + **TRUSTED_ISSUE, + "last_edited_at": RELINK_AT, + "editor_login": "maintainer", + "editor_id": 100, + "editor_type": "User", + } + ] + ) + self.assertIsNone(read(fake).refusal) + + def test_an_untrusted_comment_keeps_it(self) -> None: + for label, author in ( + ("a stranger", self.STRANGER), + ("the lanes App", LANES_ACTOR), + ): + with self.subTest(label): + fake = self.amended(author, recorded=False) + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + + def test_a_trusted_comment_an_outsider_edited_keeps_it(self) -> None: + fake = self.amended(recorded=False) + fake.closing_comments[7][0] |= { + "lastEditedAt": RELINK_AT, + "editor": self.STRANGER, + } + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + + def test_a_pass_created_before_the_amendment_keeps_it(self) -> None: + fake = self.amended(at=PASS_RUNNING) + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + + def test_an_amendment_before_the_fail_keeps_it(self) -> None: + fake = self.amended(at="2026-10-13T00:00:00Z") + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + + def test_a_pass_on_other_linked_issues_keeps_it(self) -> None: + fake = self.amended() + fake.records[named(EARLIER, "fail")]["linked_issues"] = [ + {**TRUSTED_ISSUE, "number": 8} + ] + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + + class AFailAnsweredInTheBodyGivesWayAtAMergeOnlyHead(unittest.TestCase): """A counted FAIL at an earlier head of the same change, answered by a trusted body edit, gives way at a head that only merged main since once From 473d3f5765f48e5084ce858fc2e6f121df24137e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 21 Nov 2026 19:30:15 -0500 Subject: [PATCH 2/2] fix(source-control): count only owner-decided spec amendments against an earlier FAIL The owner chose option B: a trusted spec change after the FAIL clears it only when it records an owner decision. _spec_amended now counts a trusted spec comment created in the window, or a linked issue body a trusted person last edited in the window, only when its text links an owner-decisions row and quotes the owner after `verbatim:`, read by the existing merge_policy.decision_gaps (the cost-decision parser). A plain trusted comment or body edit amends nothing. The issue body and its last editor now come from a live read with the existing ACTIONS_ISSUE_QUERY, made only when the issue's lastEditedAt falls after the FAIL, instead of from the pass's verdict record, which holds no body text. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RAoAtUnz4WHB5auUnwQYq7 --- ...-18023-earlier-fail-spec-amend-4b9d2e71.md | 2 +- .../scripts/babysit_verify_evidence.py | 55 ++++++----- .../tests/test_babysit_verify_evidence.py | 96 +++++++++++++++---- 3 files changed, 111 insertions(+), 42 deletions(-) diff --git a/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md b/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md index daf4e11af6..40c2342843 100644 --- a/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md +++ b/.changes/source-control/fix-18023-earlier-fail-spec-amend-4b9d2e71.md @@ -4,4 +4,4 @@ bump: patch ### Fixed -- The babysit merge gate clears `verify:earlier-fail` for a pass at a head of the same change when a trusted person amended a linked issue's spec (a body edit or a new spec comment) after the FAIL's run and before the pass's run was created, so a FAIL against a done-when item that was later amended no longer blocks the pull request forever. Untrusted edits or comments, a pass judged before the amendment, and any failed read keep the refusal ([#18023](https://github.com/melodic-software/claude-code-plugins/pull/18023)). +- The babysit merge gate clears `verify:earlier-fail` for a pass at a head of the same change when a trusted person recorded an owner decision on a linked issue after the FAIL's run and before the pass's run was created: a new spec comment, or an issue body edit, that links an owner-decisions row and quotes the owner after `verbatim:`. A FAIL against a done-when item the owner later amended no longer blocks the pull request forever. An amendment without that link, an untrusted edit or comment, a pass judged before the amendment, and any failed read keep the refusal ([#18023](https://github.com/melodic-software/claude-code-plugins/pull/18023)). diff --git a/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py b/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py index 9268dd8065..6de7b8999d 100755 --- a/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py +++ b/plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py @@ -83,10 +83,11 @@ pass run's creation and only trusted people edited the body or connected or disconnected an issue up to its completion (`_body_fixed`), or every counted FAIL there and every counted pass at the head record the same - linked issues and, for each pass, a trusted person amended one of them - from the last such FAIL's creation to the pass run's creation: a body - edit, last made by a trusted person as the pass records it, or a spec - comment (`_spec_comments`) created then (`_spec_amended`). Any + linked issues and, for each pass, a trusted person recorded an owner + decision on one of them from the last such FAIL's creation to the pass + run's creation: a spec comment (`_spec_comments`) created then, or a body + a trusted person last edited then, that links an owner-decisions row and + quotes the owner (`merge_policy.decision_gaps`; `_spec_amended`). Any other failed read of a run time or a diff refuses as the next item does; - a FAIL-named artifact tied to this pull request by number or SHA whose run cannot be read: `verify:unreadable`. An untied FAIL whose run cannot be read @@ -4986,17 +4987,23 @@ def _spec_amended( ) -> bool: """Every counted FAIL at `sha` records this pull request, head and verdict, each counted pass at the head records the same linked - issues as each FAIL, and for each pass a trusted person changed one - of those issues' spec after the last FAIL's creation, up to the pass - run's creation: its body, last edited by a trusted person - (`_trusted_actor`) as the pass records it, or a spec comment - (`_spec_comments`) created on it now. The pass judged an amended + issues as each FAIL, and for each pass a trusted person + (`_trusted_actor`) recorded an owner decision on one of those issues + after the last FAIL's creation, up to the pass run's creation: a spec + comment (`_spec_comments`) created then, or the issue body as last + edited then (ACTIONS_ISSUE_QUERY), that links an owner-decisions row + and quotes the owner (`merge_policy.decision_gaps`). A trusted + change without that link amends nothing. The pass judged an amended spec the FAIL never read, which the same change cannot carry; a relink stays `_superseded`'s. A record, time or read that fails is False.""" fails = fail_artifacts.get(sha, []) if not passes or not fails: return False + + def decided(text: Any) -> bool: + return not merge_policy.decision_gaps(text) + try: judged: set[frozenset[int]] = set() for run_id, name, artifact in fails: @@ -5018,33 +5025,33 @@ def _spec_amended( live = self._live_issues.get(number) if live is None: live = self._live_issues[number] = self._closing_issues(number) - commented = [] + amended = [] for n in issues & set(live): nodes = json_array(json_object(live[n].get("comments")).get("nodes")) trusted = self._spec_comments(nodes) - commented += [ + amended += [ postmerge._time(c.get("createdAt")) for c in map(json_object, nodes) if (c.get("databaseId"), c.get("updatedAt")) in trusted + and decided(c.get("body")) ] + edited = live[n].get("lastEditedAt") + if edited is None or postmerge._time(edited) <= last: + continue + node = json_object( + self._graphql(ACTIONS_ISSUE_QUERY, n, "spec issue").get("issue") + ) + editor = json_object(node.get("editor")) + if self._trusted_actor( + editor.get("__typename"), editor.get("databaseId") + ) and decided(node.get("body")): + amended.append(postmerge._time(node.get("lastEditedAt"))) for run_id, name, artifact in passes: record = self._record(run_id, name, artifact) if _linked_numbers(record) != issues: return False - edited = [ - postmerge._time(issue.get("last_edited_at")) - for issue in map( - json_object, json_array(record.get("linked_issues")) - ) - if issue.get("last_edited_at") is not None - and self._trusted_actor( - issue.get("editor_type"), issue.get("editor_id") - ) - ] begun = self._fresh_created(run_id) - if begun is None or not any( - last < at <= begun for at in commented + edited - ): + if begun is None or not any(last < at <= begun for at in amended): return False except (_Unreadable, postmerge.PostMergeReadError): return False diff --git a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py index aa56f124ef..3d528ebaaf 100644 --- a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py +++ b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_verify_evidence.py @@ -244,6 +244,7 @@ def __init__( self.closing_total: int | None = None # Each closing issue's body lastEditedAt now, by number. self.closing_edited: dict[int, str | None] = {} + self.issue_bodies: dict[int, dict[str, Any]] = {} # Each closing issue's comments now, and a totalCount that cuts them. self.closing_comments: dict[int, list[dict[str, Any]]] = {} self.comments_total: dict[int, int] = {} @@ -424,6 +425,9 @@ def missing() -> subprocess.CompletedProcess[str]: else {"__typename": "Issue"} | self.issue(n, kind) ) return ok({"data": {"repository": {"issueOrPullRequest": node}}}) + if any("issue(number:$n){title body" in a for a in args): + n = int(next(a for a in args if a.startswith("n="))[2:]) + return ok({"data": {"repository": {"issue": self.issue_bodies[n]}}}) nodes = [self.issue(n, repo) for n, repo in self.closing] refs = {"totalCount": self.closing_total or len(nodes), "nodes": nodes} pull = {"body": self.body, "closingIssuesReferences": refs} @@ -1728,25 +1732,50 @@ class ATrustedSpecAmendmentClearsAnEarlierFailOnTheSameChange(unittest.TestCase) """Pull request #18023's shape: a FAIL at an earlier head against issue #17908's done-when item, the item amended by a trusted comment after the FAIL's run, and a pass at a head of the same change whose run was - created after the comment. The pass judged the amended spec, so it - clears the FAIL (`_spec_amended`); an untrusted amendment or a pass - created before it keep `verify:earlier-fail`.""" + created after the comment. The amendment counts only when it links an + owner-decisions row and quotes the owner (owner decision, option B), so + the pass judged an owner-decided spec and clears the FAIL + (`_spec_amended`); an amendment without that link, an untrusted one or + a pass created before it keep `verify:earlier-fail`.""" STRANGER = {"__typename": "User", "login": "stranger", "databaseId": 999} + ROW = ( + "https://github.com/acme/records/blob/" + + "0" * 40 + + "/docs/owner-decisions.md#L42" + ) + DECIDED = f'Item 1 now reads as amended. Decision: {ROW} verbatim: "amend item 1"' - @staticmethod + def setUp(self) -> None: + patcher = mock.patch.object( + policy, + "DECISION_ROW", + policy.merge_settings.decision_row_pattern( + "acme/records/docs/owner-decisions.md" + ), + ) + patcher.start() + self.addCleanup(patcher.stop) + + @classmethod def amended( - author: dict[str, Any] = HUMAN, at: str = RELINK_AT, recorded: bool = True + cls, + author: dict[str, Any] = HUMAN, + at: str = RELINK_AT, + recorded: bool = True, + body: str | None = None, ) -> FakeGitHub: """A FAIL at EARLIER (run 2, created OLDER_START) noting a file, a pass at HEAD (run 1, created RUN_STARTED) of the same change, no PR body edit or relink, and comment 9 on issue 7 by `author` created at - `at`, which the pass records as spec when `recorded`.""" + `at` with `body` (DECIDED by default), which the pass records as spec + when `recorded`.""" fake = AFixedBodyClearsABodyOnlyFailOnTheSameChange.body_fail( [NOTE_ON_A_FILE], relink_answer() ) + text = cls.DECIDED if body is None else body fake.closing_comments[7] = [ - comment(9, HUMAN, at, createdAt=at) | {"author": author} + comment(9, HUMAN, at, createdAt=at, body=text) | {"author": author} ] spec = [{"id": 9, "updated_at": at}] if recorded else [] fake.records[named(HEAD, "pass")] = record( @@ -1754,28 +1783,61 @@ def amended( ) return fake - def test_pr_18023s_fail_clears_after_a_trusted_spec_comment(self) -> None: - evidence = read(self.amended()) - self.assertIsNone(evidence.refusal) - self.assertEqual(evidence.report()["earlierFails"], [EARLIER]) - - def test_a_trusted_issue_body_edit_after_the_fail_clears_it(self) -> None: + @classmethod + def body_edited( + cls, body: str | None = None, editor: dict[str, Any] = HUMAN + ) -> FakeGitHub: + """The same FAIL and pass, with issue 7's body last edited at + RELINK_AT by `editor` to `body` (DECIDED by default).""" fake = AFixedBodyClearsABodyOnlyFailOnTheSameChange.body_fail( [NOTE_ON_A_FILE], relink_answer() ) fake.closing_edited[7] = RELINK_AT + fake.issue_bodies[7] = { + "title": "t", + "body": cls.DECIDED if body is None else body, + "lastEditedAt": RELINK_AT, + "author": HUMAN, + "editor": editor, + } fake.records[named(HEAD, "pass")] = record( linked_issues=[ { **TRUSTED_ISSUE, "last_edited_at": RELINK_AT, - "editor_login": "maintainer", - "editor_id": 100, - "editor_type": "User", + "editor_login": editor.get("login"), + "editor_id": editor.get("databaseId"), + "editor_type": editor.get("__typename"), } ] ) - self.assertIsNone(read(fake).refusal) + return fake + + def test_pr_18023s_fail_clears_after_a_trusted_decision_comment(self) -> None: + evidence = read(self.amended()) + self.assertIsNone(evidence.refusal) + self.assertEqual(evidence.report()["earlierFails"], [EARLIER]) + + def test_a_trusted_issue_body_edit_linking_a_decision_clears_it(self) -> None: + self.assertIsNone(read(self.body_edited()).refusal) + + def test_a_trusted_amendment_without_a_decision_link_keeps_it(self) -> None: + for label, body in ( + ("plain text", "Item 1 now reads as amended."), + ("row link, no quote", f"Decision: {self.ROW}"), + ("quote, no row link", 'verbatim: "amend item 1"'), + ("link inside an HTML comment", f' verbatim: "x"'), + ): + with self.subTest(f"comment, {label}"): + fake = self.amended(body=body) + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + with self.subTest(f"issue body, {label}"): + fake = self.body_edited(body=body) + self.assertEqual(read(fake).refusal, "verify:earlier-fail") + + def test_an_issue_body_an_outsider_last_edited_keeps_it(self) -> None: + fake = self.body_edited(editor=self.STRANGER) + self.assertEqual(read(fake).refusal, "verify:earlier-fail") def test_an_untrusted_comment_keeps_it(self) -> None: for label, author in (