From 5aadc9f1c2143e6321c5b4661e3c9a8f7018b9ba Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 15:53:57 +0700 Subject: [PATCH 01/19] [WIP] Tri-state verification outcomes: PASS/FAIL/UNEVALUATED (Refs #3065) Replaces CheckResult.passed with CheckOutcome, extends the expectation grammar, classifies check tables by column contract, extracts the first backticked span as the command, converges validate_build onto one bound and one timeout disposition, and persists the graded aggregate. --- agent/verification_parser.py | 503 +++++++++++++++++++++++++++++++---- scripts/validate_build.py | 72 ++++- 2 files changed, 509 insertions(+), 66 deletions(-) diff --git a/agent/verification_parser.py b/agent/verification_parser.py index 22bbdc595..e9b680fac 100644 --- a/agent/verification_parser.py +++ b/agent/verification_parser.py @@ -43,9 +43,9 @@ consumes rows until a blank line or a line that cannot be part of the table). Every pipe-block in the section is classified on its own, independently: -- A block is a **check table** when it has at least three columns and one of - its first three column names is exactly ``Command`` (case-insensitive). - Every data row in it is parsed as a check, exactly as before. +- A block is a **check table** when its columns match the check contract: at + least three columns, the second named ``Command`` and the third ``Expected`` + (case-insensitive). Every data row in it is parsed as a check. - A block that is not a check table -- a red/green summary, a findings recap -- becomes a :class:`SkippedTable`: named, reported, and non-failing. A second markdown table in the section is legitimate plan authoring; treating @@ -57,6 +57,27 @@ :class:`SkippedTable` is produced in this branch -- a block is either skipped or malformed, never both. +Three-valued outcomes (#2791/#2901/#3022) +----------------------------------------- +A check result is :class:`CheckOutcome`.``PASS``, ``FAIL``, or +``UNEVALUATED`` -- never a boolean. ``UNEVALUATED`` means *the grader could +not answer the question*, and it is produced by a timeout, by any runner +exception, by an expectation form the grammar does not recognise (including +an empty cell), and by a command cell carrying no backticked span. Each +carries a ``reason``. + +``UNEVALUATED`` is **blocking** -- it does not pass -- but it is reported as +its own token and never as ``[FAIL]``. The distinction is the whole point: a +gate that says "your code is wrong" when it means "my grader is wrong" costs +a human the time to discover the difference, and the 2026-09 supervisor batch +hand-verified every such "failure" as actually passing. + +Table classification is by column **contract** -- columns 2 and 3 of the +header must be ``Command`` and ``Expected`` -- not by the word ``Command`` +appearing anywhere in the first three positions. A table shaped +``| Command | Observed stdout | Observed exit |`` used to be classified as a +check table and have its *second* column executed as a shell command (#3022). + The escape composes, which matters for basic-regex ``grep``: in a BRE, alternation is spelled ``\\|``, and to get that through the table you double the backslash. What lands in the shell is one level of unescaping:: @@ -70,14 +91,44 @@ from __future__ import annotations +import json +import logging import re import subprocess from dataclasses import dataclass +from datetime import UTC, datetime +from enum import StrEnum + +logger = logging.getLogger(__name__) # Split on a `|` that is not backslash-escaped. A row's cells are the pieces # between these; `\|` inside a cell survives the split and is unescaped after. _UNESCAPED_PIPE_RE = re.compile(r"(? list[str]: """Split one markdown table row into its cells, honoring ``\\|`` escapes. @@ -96,11 +147,22 @@ def split_row_cells(row: str) -> list[str]: @dataclass(frozen=True) class VerificationCheck: - """A single machine-readable verification check from a plan document.""" + """A single machine-readable verification check from a plan document. + + ``unevaluated_reason`` is non-empty when the row was read but cannot be + executed as written -- today, a command cell carrying no backticked span. + Such a check is never run; it grades ``UNEVALUATED`` with that reason. + + ``extraction_note`` records a non-obvious reading of the command cell (a + cell with two backticked spans, where the first is taken), so the report + says what ran rather than leaving the author to guess. + """ name: str command: str expected: str + unevaluated_reason: str = "" + extraction_note: str = "" @dataclass(frozen=True) @@ -141,13 +203,23 @@ class ParsedTable: @dataclass class CheckResult: - """Result of running a single verification check.""" + """Result of running a single verification check. + + ``outcome`` is three-valued (:class:`CheckOutcome`). There is deliberately + no ``passed`` boolean: keeping one alongside would let a caller keep asking + the ambiguous two-valued question, which is the defect this type exists to + remove. + + ``reason`` is populated exactly when ``outcome`` is ``UNEVALUATED`` and says + why the grader could not answer. + """ check: VerificationCheck - passed: bool + outcome: CheckOutcome exit_code: int output: str error: str = "" + reason: str = "" _SEPARATOR_ROW_RE = re.compile(r"^\|[\s\-:|]+\|$") @@ -176,11 +248,62 @@ def _iter_pipe_blocks(section: str) -> list[list[str]]: def _is_check_table_header(header_cells: list[str]) -> bool: - """A block is a check table when it has >=3 columns and one of its first - three column names is exactly "Command" (case-insensitive).""" + """A block is a check table when its columns match the check **contract**. + + The contract is positional: at least three columns, the second named + ``Command`` and the third named ``Expected`` (case-insensitive). The first + column is the check's name and may be called anything (``Check``, + ``Anti-criterion``, ...). + + The predicate this replaced asked whether *any* of the first three column + names was ``Command``, which is a question about vocabulary rather than + about shape. A table shaped ``| Command | Observed stdout | Observed exit |`` + -- a results recap, not a check list -- satisfied it, and its "Observed + stdout" column was then executed as a shell command with no diagnostic + emitted (#3022). A sweep of this repo's plans finds every genuine check + table is ``(, Command, Expected)``, and exactly one false positive + (``| # | Criterion | Check |``-shaped recaps) that the contract rejects. + """ if len(header_cells) < 3: return False - return any(cell.strip().lower() == "command" for cell in header_cells[:3]) + return ( + header_cells[1].strip().lower() == "command" + and header_cells[2].strip().lower() == "expected" + ) + + +# The first backticked span in a command cell. Anything outside it -- a +# trailing em-dash gloss, a parenthetical -- is prose about the command, not +# part of it. +_BACKTICKED_SPAN_RE = re.compile(r"`([^`]+)`") + + +def _extract_command(cell: str) -> tuple[str, str, str]: + """Read a command cell into ``(command, unevaluated_reason, note)``. + + The command is the cell's **first backticked span**. The prior reading was + ``cell.strip("`")``, which stripped the outer backticks and kept everything + between them -- so ``` `echo hi` -- this checks greeting ``` was executed + verbatim under ``shell=True`` as ``echo hi` -- this checks greeting``. + + A cell with no backticked span yields an ``unevaluated_reason``: there is + nothing unambiguous to run, and guessing is how the trailing-prose defect + happened. A cell with two or more spans takes the first and says so. + """ + spans = _BACKTICKED_SPAN_RE.findall(cell) + if not spans: + return ( + cell, + ( + "command cell carries no backticked span, so there is no " + "unambiguous command to run. Write the command as `cmd`." + ), + "", + ) + note = "" + if len(spans) > 1: + note = f"command cell carried {len(spans)} backticked spans; ran the first" + return spans[0], "", note def _block_data_rows(block: list[str]) -> list[str]: @@ -243,8 +366,8 @@ def parse_verification_table(markdown: str) -> ParsedTable: MalformedRow( line=block[0], reason=( - f"table has {len(block)} row(s) but none of its first three " - "column names is Command; the ## Verification section " + f"table has {len(block)} row(s) but its columns are not " + "(, Command, Expected); the ## Verification section " "yielded zero executable checks" ), ) @@ -258,7 +381,7 @@ def parse_verification_table(markdown: str) -> ParsedTable: SkippedTable( header=block[0], row_count=len(_block_data_rows(block)), - reason="not a check table: no column of the first three is named Command", + reason=("not a check table: its columns are not (, Command, Expected)"), ) for block in non_check_blocks ] @@ -284,10 +407,10 @@ def parse_verification_table(markdown: str) -> ParsedTable: continue name = cells[0] - command = cells[1].strip("`") + raw_command = cells[1] expected = cells[2] - if not name or not command or not expected: + if not name or not raw_command.strip() or not expected: malformed.append( MalformedRow( line=row, @@ -296,17 +419,56 @@ def parse_verification_table(markdown: str) -> ParsedTable: ) continue - checks.append(VerificationCheck(name=name, command=command, expected=expected)) + command, unevaluated_reason, note = _extract_command(raw_command) + checks.append( + VerificationCheck( + name=name, + command=command, + expected=expected, + unevaluated_reason=unevaluated_reason, + extraction_note=note, + ) + ) return ParsedTable(checks=checks, malformed=malformed, skipped=skipped) -def evaluate_expectation(expected: str, *, exit_code: int, output: str) -> bool: - """Evaluate whether a command result meets the expected outcome. +def timeout_reason(timeout: int) -> str: + """The one timeout disposition, shared by both runners of these tables.""" + return ( + f"command timed out after {timeout}s, so it never produced a result to grade " + "(this is not evidence that the check failed)" + ) + + +def unevaluated_reason(expected: str | None) -> str: + """Say why ``expected`` could not be graded, for an ``UNEVALUATED`` row.""" + if expected is None or not expected.strip(): + return "expectation cell is empty, so there is nothing to grade." + return ( + f"unrecognized expectation form: {expected.strip()!r}. " + "The grammar reads: exit code N, exit N, exit code != N, output contains X, " + "output does not contain X, match count == 0, output > N, > N, >= N, == N, " + "prints `N`, empty output." + ) + + +def evaluate_expectation(expected: str | None, *, exit_code: int, output: str) -> CheckOutcome: + """Grade a command result against its expected outcome, three-valued. + + Returns ``PASS``, ``FAIL``, or ``UNEVALUATED``. ``UNEVALUATED`` is returned + for an expectation the grammar does not recognise and for an empty, + whitespace-only, or ``None`` cell -- never ``FAIL``, because "I did not + understand the question" is not evidence about the code under test. Call + :func:`unevaluated_reason` for the accompanying reason text. Supported expectation formats (positive): - - ``exit code N`` -- passes when exit_code == N (positive exact-match) - - ``output > N`` -- passes when output (stripped) is numeric and > N + - ``exit code N`` / ``exit N`` -- passes when exit_code == N + - ``output > N`` / ``> N`` -- passes when output (stripped) is numeric and > N + - ``>= N`` -- passes when output (stripped) is numeric and >= N + - ``== N`` / ``output == N`` -- passes when output (stripped) is numeric and == N + - ``prints `N``` -- passes when stripped output equals N + - ``empty output`` -- passes when stdout is empty or whitespace-only - ``output contains X`` -- passes when substring X appears in stdout Supported expectation formats (inverse / anti-criteria): @@ -330,14 +492,32 @@ def evaluate_expectation(expected: str, *, exit_code: int, output: str) -> bool: branch, and ``output does not contain X`` is checked BEFORE ``output contains X``, so the inverse forms are always matched first and never captured by positive matchers. """ + if expected is None or not expected.strip(): + # An empty cell is not a failed check; it is an ungraded one. + return CheckOutcome.UNEVALUATED expected = expected.strip() + def verdict(ok: bool) -> CheckOutcome: + return CheckOutcome.PASS if ok else CheckOutcome.FAIL + + def numeric_verdict(op) -> CheckOutcome: + """Grade a numeric comparison against stripped stdout. + + Non-numeric stdout is a genuine FAIL, not UNEVALUATED: the expectation + was understood, and the command answered something that is not a + number. + """ + try: + return verdict(op(int(output.strip()))) + except (ValueError, TypeError): + return CheckOutcome.FAIL + # --- inverse forms (must be checked before positive forms) --- # exit code != N (inverse: passes when exit_code differs from N) m = re.match(r"exit code\s*!=\s*(\d+)", expected) if m: - return exit_code != int(m.group(1)) + return verdict(exit_code != int(m.group(1))) # output does not contain X (inverse: passes when X absent AND stdout non-empty) m = re.match(r"output does not contain (.+)", expected) @@ -345,58 +525,88 @@ def evaluate_expectation(expected: str, *, exit_code: int, output: str) -> bool: substring = m.group(1).strip() if not output.strip(): # empty-stdout gate: errored / stderr-only command must not false-pass - return False - return substring not in output + return CheckOutcome.FAIL + return verdict(substring not in output) # match count == 0 (inverse: passes when grep -c / -rc output shows zero matches) - if expected.strip() == "match count == 0": + if expected == "match count == 0": if not output.strip(): # empty-stdout gate: truly-empty stdout means the command errored or # wrote only to stderr; all(...) over an empty list would be vacuously # True without this guard. - return False + return CheckOutcome.FAIL lines = [ln.strip() for ln in output.strip().splitlines() if ln.strip()] - return all(ln == "0" or ln.endswith(":0") for ln in lines) + return verdict(all(ln == "0" or ln.endswith(":0") for ln in lines)) # --- positive forms --- - # exit code N (positive exact-match: passes when exit_code == N) - m = re.match(r"exit code (\d+)", expected) + # exit code N / exit N (positive exact-match: passes when exit_code == N) + m = re.match(r"exit(?: code)?\s+(\d+)\s*$", expected) if m: - return exit_code == int(m.group(1)) + return verdict(exit_code == int(m.group(1))) + + # empty output (passes when stdout is empty or whitespace-only) + if expected == "empty output": + return verdict(not output.strip()) - # output > N - m = re.match(r"output\s*>\s*(\d+)", expected) + # prints `N` (passes when stripped stdout equals N; backticks optional) + m = re.match(r"prints\s+`?([^`]+?)`?\s*$", expected) + if m: + return verdict(output.strip() == m.group(1).strip()) + + # output >= N / >= N + m = re.match(r"(?:output\s*)?>=\s*(\d+)\s*$", expected) if m: threshold = int(m.group(1)) - try: - value = int(output.strip()) - except (ValueError, TypeError): - return False - return value > threshold + return numeric_verdict(lambda value: value >= threshold) + + # output > N / > N + m = re.match(r"(?:output\s*)?>\s*(\d+)\s*$", expected) + if m: + threshold = int(m.group(1)) + return numeric_verdict(lambda value: value > threshold) + + # output == N / == N + m = re.match(r"(?:output\s*)?==\s*(\d+)\s*$", expected) + if m: + target = int(m.group(1)) + return numeric_verdict(lambda value: value == target) # output contains X m = re.match(r"output contains (.+)", expected) if m: substring = m.group(1).strip() - return substring in output + return verdict(substring in output) - return False + return CheckOutcome.UNEVALUATED def run_checks( checks: list[VerificationCheck], *, cwd: str | None = None, - timeout: int = 120, + timeout: int = DEFAULT_TIMEOUT_S, ) -> list[CheckResult]: """Run a list of verification checks and return results. - Each check is executed as a shell command. The result is evaluated against - the check's expected outcome. + Each check is executed as a shell command and graded three-valued. A check + the parser already marked unrunnable is not executed at all; a timeout and + any runner exception both grade ``UNEVALUATED`` with the reason attached, + because neither is evidence about the code under test. """ results: list[CheckResult] = [] for check in checks: + if check.unevaluated_reason: + results.append( + CheckResult( + check=check, + outcome=CheckOutcome.UNEVALUATED, + exit_code=-1, + output="", + reason=check.unevaluated_reason, + ) + ) + continue try: proc = subprocess.run( check.command, @@ -406,7 +616,7 @@ def run_checks( cwd=cwd, timeout=timeout, ) - passed = evaluate_expectation( + outcome = evaluate_expectation( check.expected, exit_code=proc.returncode, output=proc.stdout, @@ -414,30 +624,39 @@ def run_checks( results.append( CheckResult( check=check, - passed=passed, + outcome=outcome, exit_code=proc.returncode, output=proc.stdout.strip(), error=proc.stderr.strip(), + reason=( + unevaluated_reason(check.expected) + if outcome is CheckOutcome.UNEVALUATED + else "" + ), ) ) except subprocess.TimeoutExpired: + reason = timeout_reason(timeout) results.append( CheckResult( check=check, - passed=False, + outcome=CheckOutcome.UNEVALUATED, exit_code=-1, output="", - error=f"Command timed out after {timeout}s", + error=reason, + reason=reason, ) ) except Exception as e: + reason = f"runner error, the check never ran: {type(e).__name__}: {e}" results.append( CheckResult( check=check, - passed=False, + outcome=CheckOutcome.UNEVALUATED, exit_code=-1, output="", - error=f"Failed to execute: {e}", + error=reason, + reason=reason, ) ) @@ -466,11 +685,18 @@ def format_results( Skipped (non-check) tables are reported in their own section and do not participate in the pass/fail verdict (#2836): a summary table is legitimate plan authoring. + + ``UNEVALUATED`` rows render as their own token with their reason and are + never printed as ``[FAIL]``. They are blocking -- the run does not report + "All checks passed." -- but they say plainly that the grader, not the code, + is what could not answer. """ malformed = table.malformed skipped = table.skipped lines: list[str] = ["## Verification Results", ""] - all_passed = all(r.passed for r in results) and not malformed + unevaluated = [r for r in results if r.outcome is CheckOutcome.UNEVALUATED] + failed = [r for r in results if r.outcome is CheckOutcome.FAIL] + all_passed = not failed and not unevaluated and not malformed if malformed: lines.append(f"### Plan authoring errors ({len(malformed)})") @@ -496,9 +722,13 @@ def format_results( lines.append("") for r in results: - status = "PASS" if r.passed else "FAIL" - lines.append(f"- [{status}] {r.check.name}") - if not r.passed: + lines.append(f"- [{r.outcome.value}] {r.check.name}") + if r.outcome is CheckOutcome.UNEVALUATED: + lines.append(f" Command: `{r.check.command}`") + lines.append(f" Expected: {r.check.expected}") + lines.append(f" Reason: {r.reason}") + continue + if r.outcome is CheckOutcome.FAIL: lines.append(f" Command: `{r.check.command}`") lines.append(f" Expected: {r.check.expected}") lines.append(f" Got: exit code {r.exit_code}") @@ -506,12 +736,181 @@ def format_results( lines.append(f" Output: {r.output[:200]}") if r.error: lines.append(f" Error: {r.error[:200]}") + if r.check.extraction_note: + lines.append(f" Note: {r.check.extraction_note}") lines.append("") - if malformed and all(r.passed for r in results): + if all_passed: + summary = "All checks passed." + elif not failed and malformed and not unevaluated: summary = f"{len(malformed)} row(s) could not be parsed and were not run." + elif not failed: + summary = ( + f"{len(unevaluated)} check(s) could not be evaluated" + + (f" and {len(malformed)} row(s) could not be parsed" if malformed else "") + + "." + ) else: - summary = "All checks passed." if all_passed else "Some checks failed." + summary = "Some checks failed." lines.append(f"**{summary}**") return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Persisting the graded aggregate (#3065, Cluster B -> Cluster C) +# --------------------------------------------------------------------------- +# +# A verification run is graded at TEST/DOCS time and consumed by the merge +# predicate later, so the outcome has to outlive the process that produced it. +# It is stored as `_verification_outcomes` inside the issue-keyed +# PipelineLedger's `stage_states_json` blob -- an underscore-prefixed metadata +# key exactly like `_verdicts`, `_sdlc_dispatches`, and `_run_identities`, so +# it needs no schema field and no migration. +# +# The aggregate carries the PR head SHA it was graded against. Without that +# anchor a lane that passes verification and then takes a new commit merges on +# a cached PASS -- "a fact readable earlier, not now", which is the exact defect +# this mechanism exists to close. The SHA is resolved through +# `tools.pr_head_resolver.resolve_pr_head_sha` (git-first) and never a bare +# `gh` read: a stale `gh` head SHA is what flipped the verdict-staleness gate +# fail-open in #2895. + + +def aggregate_outcomes(results: list[CheckResult], table: ParsedTable | None = None) -> dict: + """Reduce a graded run to the record the merge predicate reads. + + Overall outcome is the worst thing present: any ``FAIL`` (or any malformed + row, which is an unrunnable check) makes the run ``FAIL``; otherwise any + ``UNEVALUATED`` makes it ``UNEVALUATED``; a run with no checks at all is + ``UNEVALUATED``, never a vacuous ``PASS``. + """ + malformed_count = len(table.malformed) if table else 0 + counts = { + CheckOutcome.PASS.value: 0, + CheckOutcome.FAIL.value: 0, + CheckOutcome.UNEVALUATED.value: 0, + } + for r in results: + counts[r.outcome.value] += 1 + + if counts[CheckOutcome.FAIL.value] or malformed_count: + overall = CheckOutcome.FAIL + elif counts[CheckOutcome.UNEVALUATED.value] or not results: + overall = CheckOutcome.UNEVALUATED + else: + overall = CheckOutcome.PASS + + return { + "outcome": overall.value, + "counts": counts, + "malformed": malformed_count, + "recorded_at": datetime.now(UTC).isoformat(), + "rows": [ + { + "name": r.check.name, + "outcome": r.outcome.value, + "reason": r.reason, + } + for r in results + ], + } + + +def record_verification_outcomes( + target_repo: str | None, + issue_number: int | None, + results: list[CheckResult], + *, + table: ParsedTable | None = None, + pr_number: int | None = None, + repo_root: str | None = None, +) -> bool: + """Persist this run's graded aggregate to the lane's ledger. + + Stamps ``head_sha`` with the PR head the run was graded against, resolved + through ``tools.pr_head_resolver.resolve_pr_head_sha``. A lane with no PR + at write time (or one whose head cannot be resolved) records the aggregate + with **no** ``head_sha`` key rather than a fabricated one, and does not + crash; the reader decides what an unanchored aggregate is worth. + + Fails OPEN: returns ``False`` on any failure and never raises. A grading + run that cannot write its record must still report its result to the human + in front of it. + """ + if not target_repo or not issue_number: + return False + + try: + aggregate = aggregate_outcomes(results, table) + + if pr_number: + try: + from tools.pr_head_resolver import resolve_pr_head_sha + + head_sha = resolve_pr_head_sha( + int(pr_number), repo=target_repo, repo_root=repo_root + ) + if head_sha: + aggregate["head_sha"] = head_sha + except Exception as exc: + logger.debug( + "record_verification_outcomes: head-SHA resolve failed for " + "%s#%s PR %s (%s: %s) -- recording without an anchor", + target_repo, + issue_number, + pr_number, + type(exc).__name__, + exc, + ) + + from agent.pipeline_ledger import PipelineLedger + from tools.stage_states_helpers import update_stage_states + + ledger = PipelineLedger.get_or_create(target_repo, issue_number) + + def write_outcomes(states: dict) -> dict: + states[VERIFICATION_OUTCOMES_KEY] = aggregate + return states + + return bool(update_stage_states(ledger, write_outcomes, field="stage_states_json")) + except Exception as exc: + logger.debug( + "record_verification_outcomes: write failed for %s#%s (%s: %s)", + target_repo, + issue_number, + type(exc).__name__, + exc, + ) + return False + + +def read_verification_outcomes(target_repo: str | None, issue_number: int | None) -> dict | None: + """Return the recorded aggregate for a lane, or ``None`` if there is none. + + Non-mutating (uses :meth:`PipelineLedger.get`, so a read never litters an + empty ledger) and fails OPEN to ``None`` on any error or malformed blob. + """ + if not target_repo or not issue_number: + return None + try: + from agent.pipeline_ledger import PipelineLedger + + ledger = PipelineLedger.get(target_repo, issue_number) + if ledger is None: + return None + raw = ledger.stage_states_json + blob = json.loads(raw) if isinstance(raw, str) else raw + if not isinstance(blob, dict): + return None + record = blob.get(VERIFICATION_OUTCOMES_KEY) + return record if isinstance(record, dict) else None + except Exception as exc: + logger.debug( + "read_verification_outcomes: read failed for %s#%s (%s: %s)", + target_repo, + issue_number, + type(exc).__name__, + exc, + ) + return None diff --git a/scripts/validate_build.py b/scripts/validate_build.py index 119aecdf7..0eafedc16 100644 --- a/scripts/validate_build.py +++ b/scripts/validate_build.py @@ -11,8 +11,8 @@ python scripts/validate_build.py --help Exit codes: - 0 - All checks pass or skip (no failures) - 1 - One or more checks failed + 0 - All checks pass (or are non-blocking skips) + 1 - One or more checks failed, or could not be evaluated """ import re @@ -23,9 +23,13 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from agent.verification_parser import ( # noqa: E402 + DEFAULT_TIMEOUT_S, + CheckOutcome, ParsedTable, evaluate_expectation, parse_verification_table, + timeout_reason, + unevaluated_reason, ) @@ -193,13 +197,18 @@ def check_file_assertions(assertions: list[dict[str, str]]) -> list[dict]: return results -def check_verification_table(table: ParsedTable) -> list[dict]: +def check_verification_table(table: ParsedTable, *, timeout: int = DEFAULT_TIMEOUT_S) -> list[dict]: """Run verification table commands and compare output. - Delegates table definition and expectation grammar to - ``agent.verification_parser`` (#2843) rather than carrying its own, - weaker evaluator. This runner keeps only what is genuinely its own: a - 30s-timeout, SKIP-on-timeout execution loop and its report shape. + Delegates table definition, expectation grammar, execution bound, and + timeout disposition to ``agent.verification_parser`` (#2843/#3065) rather + than carrying its own. This runner keeps only its report shape. + + The bound is ``DEFAULT_TIMEOUT_S`` and a timeout is ``UNEVALUATED``, both + shared with ``run_checks``. This module previously carried a private 30s + ceiling and called a timeout ``SKIP``, so the two runners graded the same + event two different ways -- and ``SKIP`` did not even block the exit code + (#2901). ``UNEVALUATED`` blocks: it is not a pass. """ results = [] @@ -234,18 +243,38 @@ def check_verification_table(table: ParsedTable) -> list[dict]: expected = check.expected name = check.name + if check.unevaluated_reason: + # Read but unrunnable as written (no backticked span): never + # executed on a guess, never reported as FAIL. + results.append( + { + "status": "UNEVALUATED", + "message": f"{name} -- {check.unevaluated_reason}", + } + ) + continue + try: - result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=30) + result = subprocess.run( + cmd, shell=True, capture_output=True, text=True, timeout=timeout + ) # `output` must be unstripped stdout -- run_checks passes proc.stdout # unmodified, and a stripped copy here would re-create divergence at # the exact seam this convergence closes. The stripped value is used # only in the FAIL message. actual_output = result.stdout actual_exit = result.returncode - passed = evaluate_expectation(expected, exit_code=actual_exit, output=actual_output) + outcome = evaluate_expectation(expected, exit_code=actual_exit, output=actual_output) - if passed: + if outcome is CheckOutcome.PASS: results.append({"status": "PASS", "message": name}) + elif outcome is CheckOutcome.UNEVALUATED: + results.append( + { + "status": "UNEVALUATED", + "message": f"{name} -- {unevaluated_reason(expected)}", + } + ) else: results.append( { @@ -258,9 +287,18 @@ def check_verification_table(table: ParsedTable) -> list[dict]: } ) except subprocess.TimeoutExpired: - results.append({"status": "SKIP", "message": f"{name} -- timed out after 30s"}) + results.append( + {"status": "UNEVALUATED", "message": f"{name} -- {timeout_reason(timeout)}"} + ) except Exception as e: - results.append({"status": "SKIP", "message": f"{name} -- error: {e}"}) + results.append( + { + "status": "UNEVALUATED", + "message": ( + f"{name} -- runner error, the check never ran: {type(e).__name__}: {e}" + ), + } + ) return results @@ -353,10 +391,16 @@ def main() -> int: pass_count = sum(1 for r in all_results if r["status"] == "PASS") fail_count = sum(1 for r in all_results if r["status"] == "FAIL") skip_count = sum(1 for r in all_results if r["status"] == "SKIP") + unevaluated_count = sum(1 for r in all_results if r["status"] == "UNEVALUATED") - print(f"\nResult: {pass_count} PASS, {fail_count} FAIL, {skip_count} SKIP") + print( + f"\nResult: {pass_count} PASS, {fail_count} FAIL, " + f"{unevaluated_count} UNEVALUATED, {skip_count} SKIP" + ) - return 1 if fail_count > 0 else 0 + # UNEVALUATED blocks. It is not a pass, and it is not a FAIL either: the + # exit code says "stop", the report says the grader could not answer. + return 1 if (fail_count or unevaluated_count) else 0 if __name__ == "__main__": From fdb91bf467a710ac470556b69daea7838b5816a1 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 15:56:20 +0700 Subject: [PATCH 02/19] [WIP] session-ensure: primary-key readback + candidate provenance gating (Refs #3065) --- .../test_sdlc_session_ensure_adoption.py | 23 ++- .../test_sdlc_session_ensure_core.py | 19 +- .../test_sdlc_session_ensure_issue_lock.py | 176 +++++++++++++++++- .../test_sdlc_session_ensure_run_identity.py | 3 + .../test_sdlc_session_ensure_short_circuit.py | 23 ++- tools/sdlc_session_ensure.py | 115 ++++++++++-- 6 files changed, 328 insertions(+), 31 deletions(-) diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_adoption.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_adoption.py index 9f5993ab3..2c525d825 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_adoption.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_adoption.py @@ -234,6 +234,7 @@ def test_divergent_owner_not_adopted(self, monkeypatch): mock_as = MagicMock() mock_as.query.filter.return_value = [issue_session] # post-save readback + mock_as.query.get.return_value = issue_session # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session", return_value=env_session), @@ -284,7 +285,8 @@ def test_ensure_session_records_the_lane_slug(self, monkeypatch): mock_new_session = MagicMock() mock_new_session.session_id = f"sdlc-local-{self._ISSUE}" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -318,7 +320,8 @@ def test_slug_resolution_failure_never_fails_the_ensure(self, monkeypatch): mock_new_session = MagicMock() mock_new_session.session_id = f"sdlc-local-{self._ISSUE}" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -384,6 +387,7 @@ def test_dry_run_lists_without_modifying(self): orphan = _make_orphan_session("sdlc-local-9991", ORPHAN_AGE_SECONDS + 60) mock_as = MagicMock() mock_as.query.filter.return_value = [orphan] + mock_as.query.get.return_value = orphan # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -398,6 +402,7 @@ def test_real_run_finalizes_orphans_via_finalize_session(self): orphan = _make_orphan_session("sdlc-local-9992", ORPHAN_AGE_SECONDS + 60) mock_as = MagicMock() mock_as.query.filter.return_value = [orphan] + mock_as.query.get.return_value = orphan # post-save readback (primary-key lookup) finalize_mock = MagicMock() with ( @@ -427,6 +432,7 @@ def test_finalize_session_failure_does_not_crash(self): orphan = _make_orphan_session("sdlc-local-9993", ORPHAN_AGE_SECONDS + 60) mock_as = MagicMock() mock_as.query.filter.return_value = [orphan] + mock_as.query.get.return_value = orphan # post-save readback (primary-key lookup) with ( patch("models.agent_session.AgentSession", mock_as), @@ -448,6 +454,7 @@ def test_newer_than_threshold_not_listed(self): fresh = _make_orphan_session("sdlc-local-9994", 60) # 1 minute old mock_as = MagicMock() mock_as.query.filter.return_value = [fresh] + mock_as.query.get.return_value = fresh # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -465,6 +472,7 @@ def test_session_with_heartbeat_never_listed(self): ) mock_as = MagicMock() mock_as.query.filter.return_value = [old_but_alive] + mock_as.query.get.return_value = old_but_alive # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -477,6 +485,7 @@ def test_boundary_at_threshold_is_listed(self): at_boundary = _make_orphan_session("sdlc-local-9996", ORPHAN_AGE_SECONDS) mock_as = MagicMock() mock_as.query.filter.return_value = [at_boundary] + mock_as.query.get.return_value = at_boundary # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -490,6 +499,7 @@ def test_boundary_one_second_under_not_listed(self): under = _make_orphan_session("sdlc-local-9997", ORPHAN_AGE_SECONDS - 1) mock_as = MagicMock() mock_as.query.filter.return_value = [under] + mock_as.query.get.return_value = under # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -502,6 +512,7 @@ def test_boundary_one_second_over_is_listed(self): over = _make_orphan_session("sdlc-local-9998", ORPHAN_AGE_SECONDS + 1) mock_as = MagicMock() mock_as.query.filter.return_value = [over] + mock_as.query.get.return_value = over # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -515,6 +526,7 @@ def test_non_sdlc_local_session_never_listed(self): bridge = _make_orphan_session("tg_valor_-1003449100931_691", ORPHAN_AGE_SECONDS + 3600) mock_as = MagicMock() mock_as.query.filter.return_value = [bridge] + mock_as.query.get.return_value = bridge # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -541,6 +553,7 @@ def test_live_local_pipeline_with_fresh_updated_at_not_listed(self): ) mock_as = MagicMock() mock_as.query.filter.return_value = [live] + mock_as.query.get.return_value = live # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -565,6 +578,7 @@ def test_stale_local_pipeline_no_heartbeat_still_listed(self): ) mock_as = MagicMock() mock_as.query.filter.return_value = [stale] + mock_as.query.get.return_value = stale # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -587,6 +601,7 @@ def test_fresh_updated_at_exempts_even_at_creation_boundary(self): ) mock_as = MagicMock() mock_as.query.filter.return_value = [s] + mock_as.query.get.return_value = s # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -608,6 +623,7 @@ def test_falls_back_to_started_at_when_updated_at_missing(self): s.started_at = datetime.now(UTC) - timedelta(seconds=30) mock_as = MagicMock() mock_as.query.filter.return_value = [s] + mock_as.query.get.return_value = s # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -629,6 +645,7 @@ def test_falls_back_to_created_at_when_no_activity_timestamps(self): s.started_at = None mock_as = MagicMock() mock_as.query.filter.return_value = [s] + mock_as.query.get.return_value = s # post-save readback (primary-key lookup) with patch("models.agent_session.AgentSession", mock_as): result = _kill_orphans(dry_run=True) @@ -654,6 +671,7 @@ def test_hollow_session_with_dead_locked_owner_is_reapable(self): ) mock_as = MagicMock() mock_as.query.filter.return_value = [hollow] + mock_as.query.get.return_value = hollow # post-save readback (primary-key lookup) dead_payload = { "run_id": "dead-run", @@ -696,6 +714,7 @@ def test_hollow_session_with_live_locked_owner_is_exempt(self): ) mock_as = MagicMock() mock_as.query.filter.return_value = [live] + mock_as.query.get.return_value = live # post-save readback (primary-key lookup) live_payload = { "run_id": "live-run", diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_core.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_core.py index 0f7c1ee0d..30cf89313 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_core.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_core.py @@ -23,6 +23,7 @@ def test_returns_existing_session_by_issue(self): mock_as = MagicMock() mock_as.query.filter.return_value = [mock_session] # post-save readback + mock_as.query.get.return_value = mock_session # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session_by_issue", return_value=mock_session), @@ -46,7 +47,8 @@ def test_creates_new_session(self): mock_as = MagicMock() # First filter call: idempotent existing-by-id check (none). Second: # the post-save run_id readback (the just-created session). - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -76,7 +78,8 @@ def test_creates_new_session_with_is_ledger_true_at_create_call(self): mock_new_session.session_id = "sdlc-local-947" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -102,6 +105,7 @@ def test_idempotent_by_session_id(self): mock_as = MagicMock() mock_as.query.filter.return_value = [mock_existing] + mock_as.query.get.return_value = mock_existing # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session_by_issue", return_value=None), @@ -138,7 +142,8 @@ def test_transition_status_failure_still_returns_session(self): mock_new_session.session_id = "sdlc-local-944" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -167,6 +172,7 @@ def test_project_key_resolution_error_returns_empty(self): mock_as = MagicMock() mock_as.query.filter.return_value = [] + mock_as.query.get.return_value = None # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session_by_issue", return_value=None), @@ -195,6 +201,7 @@ def test_projects_config_unavailable_error_returns_empty(self): mock_as = MagicMock() mock_as.query.filter.return_value = [] + mock_as.query.get.return_value = None # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session_by_issue", return_value=None), @@ -260,7 +267,8 @@ def test_create_local_receives_message_text(self): mock_new_session.session_id = "sdlc-local-1741" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -286,6 +294,7 @@ def test_message_text_is_issue_anchored(self): mock_as = MagicMock() mock_as.query.filter.return_value = [] + mock_as.query.get.return_value = None # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -309,6 +318,7 @@ def test_message_text_embeds_issue_url_when_provided(self): mock_as = MagicMock() mock_as.query.filter.return_value = [] + mock_as.query.get.return_value = None # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session issue_url = "https://github.com/tomcounsell/ai/issues/1743" @@ -335,6 +345,7 @@ def test_message_text_present_without_issue_url(self): mock_as = MagicMock() mock_as.query.filter.return_value = [] + mock_as.query.get.return_value = None # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_issue_lock.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_issue_lock.py index 74ca19362..1554c55a3 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_issue_lock.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_issue_lock.py @@ -30,6 +30,7 @@ def _readback_as(session): """Mock AgentSession whose readback query returns the bound session.""" mock_as = MagicMock() mock_as.query.filter.return_value = [session] + mock_as.query.get.return_value = session # post-save readback (primary-key lookup) return mock_as def test_mint_on_env_owns_issue_return(self, monkeypatch): @@ -134,6 +135,7 @@ def test_mint_on_idempotent_existing_by_id_return(self): mock_as = MagicMock() mock_as.query.filter.return_value = [existing] + mock_as.query.get.return_value = existing # post-save readback (primary-key lookup) lock_mock = MagicMock(return_value=self._lock_result(True, "sdlc-local-2004")) @@ -159,7 +161,8 @@ def test_mint_on_create_and_claim_return(self): mock_new_session.session_id = "sdlc-local-2005" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session lock_mock = MagicMock(return_value=self._lock_result(True, "sdlc-local-2005")) @@ -192,7 +195,8 @@ def test_acquire_run_lock_and_bind_pins_target_repo_from_resolver(self): mock_new_session.session_id = "sdlc-local-2006" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session lock_mock = MagicMock(return_value=self._lock_result(True, "sdlc-local-2006")) @@ -220,7 +224,8 @@ def test_acquire_run_lock_and_bind_passes_through_none_target_repo(self): mock_new_session.session_id = "sdlc-local-2007" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session lock_mock = MagicMock(return_value=self._lock_result(True, "sdlc-local-2007")) @@ -274,6 +279,7 @@ def test_blocked_shape_includes_owning_run_id(self): mock_as = MagicMock() mock_as.query.filter.return_value = [] + mock_as.query.get.return_value = None # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -401,6 +407,7 @@ def test_readback_mismatch_releases_lock(self): mock_as = MagicMock() mock_as.query.filter.return_value = [stale] # readback sees a stale value + mock_as.query.get.return_value = stale # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session_by_issue", return_value=session), @@ -411,6 +418,169 @@ def test_readback_mismatch_releases_lock(self): assert result.get("error") == "RUN_BIND_FAILED" assert rdb.POPOTO_REDIS_DB.get(f"session:issuelock:{issue_number}") is None + def test_readback_uses_primary_key_not_unordered_filter(self): + """Issue #3065 Cluster E: a lane whose session row was recreated after + a crash has TWO rows sharing one ``session_id``. The old readback + re-queried ``AgentSession.query.filter(session_id=...)`` and took + ``[0]`` from what Popoto resolves via an unordered Redis ``SMEMBERS`` + read -- a coin flip. The fix reads back by PRIMARY KEY + (``AgentSession.query.get(redis_key=...)``), so a ``query.filter`` + mock that puts the WRONG (stale-twin) row first must never affect the + result, across repeated invocations.""" + from tools.sdlc_session_ensure import ensure_session + + issue_number = 2061 + session = MagicMock() + session.session_id = f"sdlc-local-{issue_number}" + session.db_key.redis_key = f"AgentSession:sdlc-local-{issue_number}:real-row" + + stale_twin = MagicMock() + stale_twin.session_id = session.session_id + stale_twin.active_run_id = "stale-twin-run-id" + + mock_as = MagicMock() + # Simulate the unordered SMEMBERS coin flip: the stale duplicate + # sorts FIRST. The old `[0]`-taking readback would pick this row up + # and always report a mismatch; the new primary-key readback must + # never even consult this list. + mock_as.query.filter.return_value = [stale_twin, session] + + def _get(*, redis_key): + assert redis_key == session.db_key.redis_key + return session + + mock_as.query.get.side_effect = _get + + from models.session_lifecycle import release_issue_lock + + for _ in range(3): + with ( + patch("tools._sdlc_utils.find_session_by_issue", return_value=session), + patch("models.agent_session.AgentSession", mock_as), + ): + result = ensure_session(issue_number=issue_number) + + assert result.get("error") is None, result + assert result["run_id"] == session.active_run_id + # Release so the next iteration starts from a free lock, isolating + # each iteration's mint/bind/readback rather than testing renewal. + release_issue_lock(issue_number, result["run_id"]) + + def test_adopted_candidate_survives_readback_mismatch(self): + """An ADOPTED candidate (verified reuse against a live lock it + already owns) must NOT be released on a post-save readback mismatch + -- releasing it would compare-and-delete a lease this call never + minted, and the compare-and-delete would match by construction + because the adopted id already equals the live owner (#3065 Cluster + E). Real Redis lock.""" + import popoto.redis_db as rdb + + from models.session_lifecycle import touch_issue_lock + from tools.sdlc_session_ensure import ensure_session + + issue_number = 2062 + session_id = f"sdlc-local-{issue_number}" + adopted_run_id = "adopted-run-id-2062" + + # Pre-acquire the real lock under the id this call will adopt, so + # `_validated_reuse_candidate`'s live-lock-owner-match proof fires. + pre_acquire = touch_issue_lock(issue_number, adopted_run_id, session_id=session_id) + assert pre_acquire.acquired is True + + session = MagicMock() + session.session_id = session_id + + stale = MagicMock() + stale.session_id = session_id + stale.active_run_id = "some-other-run-entirely" # mismatch on readback + + mock_as = MagicMock() + mock_as.query.filter.return_value = [stale] + mock_as.query.get.return_value = stale # readback mismatch + + with ( + patch("tools._sdlc_utils.find_session_by_issue", return_value=session), + patch("models.agent_session.AgentSession", mock_as), + ): + result = ensure_session(issue_number=issue_number, reuse_run_id=adopted_run_id) + + assert result.get("error") == "RUN_BIND_FAILED" + # The lock this call adopted (but never minted) must survive. + assert rdb.POPOTO_REDIS_DB.get(f"session:issuelock:{issue_number}") is not None + peek = touch_issue_lock(issue_number, None, session_id=session_id, peek=True) + assert peek.acquired is False + assert peek.owner_run_id == adopted_run_id + + def test_adopted_candidate_survives_readback_exception(self): + """Same invariant as the mismatch case, but for the readback + `except` branch (:func:`_acquire_run_lock_and_bind`'s post-save + readback try/except): an adopted candidate must not be released when + the readback itself raises. Real Redis lock.""" + import popoto.redis_db as rdb + + from models.session_lifecycle import touch_issue_lock + from tools.sdlc_session_ensure import ensure_session + + issue_number = 2063 + session_id = f"sdlc-local-{issue_number}" + adopted_run_id = "adopted-run-id-2063" + + pre_acquire = touch_issue_lock(issue_number, adopted_run_id, session_id=session_id) + assert pre_acquire.acquired is True + + session = MagicMock() + session.session_id = session_id + + mock_as = MagicMock() + mock_as.query.filter.return_value = [session] + mock_as.query.get.side_effect = RuntimeError("redis hiccup during readback") + + with ( + patch("tools._sdlc_utils.find_session_by_issue", return_value=session), + patch("models.agent_session.AgentSession", mock_as), + ): + result = ensure_session(issue_number=issue_number, reuse_run_id=adopted_run_id) + + assert result.get("error") == "RUN_BIND_FAILED" + assert "readback failed" in result.get("reason", "") + # The lock this call adopted (but never minted) must survive. + assert rdb.POPOTO_REDIS_DB.get(f"session:issuelock:{issue_number}") is not None + peek = touch_issue_lock(issue_number, None, session_id=session_id, peek=True) + assert peek.acquired is False + assert peek.owner_run_id == adopted_run_id + + def test_minted_candidate_released_on_save_failure(self): + """A MINTED candidate (no reuse_run_id -- the ordinary create-and-claim + mint) IS released via compare-and-delete when the ``session.save()`` + bind itself raises, so the next caller does not wait out the TTL. + Real Redis lock.""" + import popoto.redis_db as rdb + + from tools.sdlc_session_ensure import ensure_session + + issue_number = 2064 + session_id = f"sdlc-local-{issue_number}" + + session = MagicMock() + session.session_id = session_id + session.save.side_effect = RuntimeError("redis write failure") + + mock_as = MagicMock() + mock_as.query.filter.return_value = [session] + mock_as.query.get.return_value = session + + with ( + patch("tools._sdlc_utils.find_session_by_issue", return_value=session), + patch("models.agent_session.AgentSession", mock_as), + ): + result = ensure_session(issue_number=issue_number) + + assert result.get("error") == "RUN_BIND_FAILED" + assert "active_run_id save failed" in result.get("reason", "") + # MINTED candidate: the lock this call itself acquired must be freed + # immediately rather than waiting out the 1800s TTL. + assert rdb.POPOTO_REDIS_DB.get(f"session:issuelock:{issue_number}") is None + def test_orphaned_lock_flagged_on_peek(self): """A lock whose recorded owner pid is dead is reported orphaned_lock=True by the peek path (issue #2305 defect 1: diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_run_identity.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_run_identity.py index 78c7cca11..334255391 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_run_identity.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_run_identity.py @@ -19,6 +19,7 @@ class TestVerifiedRunIdReuse: def _readback_as(session): mock_as = MagicMock() mock_as.query.filter.return_value = [session] + mock_as.query.get.return_value = session # post-save readback (primary-key lookup) return mock_as def test_consecutive_stage_reuse_survives_own_live_lock(self): @@ -140,6 +141,7 @@ class TestSupervisedRunSignal: def _readback_as(session): mock_as = MagicMock() mock_as.query.filter.return_value = [session] + mock_as.query.get.return_value = session # post-save readback (primary-key lookup) return mock_as def test_bare_ensure_under_live_signal_refuses_and_mints_nothing(self): @@ -383,6 +385,7 @@ class TestOwnedRunIdsSelfRecognition: def _readback_as(session): mock_as = MagicMock() mock_as.query.filter.return_value = [session] + mock_as.query.get.return_value = session # post-save readback (primary-key lookup) return mock_as def test_read_owned_run_ids_tolerant(self): diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py index 0b41ac270..d60dd4e31 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py @@ -35,6 +35,7 @@ def test_short_circuit_returns_env_session_when_live_eng(self, monkeypatch): mock_as = MagicMock() mock_as.query.filter.return_value = [bridge_session] # post-save readback + mock_as.query.get.return_value = bridge_session # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session", return_value=bridge_session), @@ -70,6 +71,7 @@ def test_non_owning_env_session_prefers_existing_issue_session(self, monkeypatch mock_as = MagicMock() # create_local must NOT be called (no duplicate). mock_as.query.filter.return_value = [issue_session] # post-save readback + mock_as.query.get.return_value = issue_session # post-save readback (primary-key lookup) with ( patch("tools._sdlc_utils.find_session", return_value=env_session), @@ -103,7 +105,8 @@ def test_non_owning_env_session_creates_when_no_issue_session(self, monkeypatch) mock_new_session.session_id = "sdlc-local-1172" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -132,7 +135,8 @@ def test_short_circuit_falls_through_when_env_session_missing(self, monkeypatch) mock_new_session.session_id = "sdlc-local-1141" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -160,7 +164,8 @@ def test_short_circuit_falls_through_when_only_agent_session_id_stale(self, monk mock_new_session.session_id = "sdlc-local-11411" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -186,7 +191,8 @@ def test_empty_env_var_does_not_short_circuit(self, monkeypatch): mock_new_session.session_id = "sdlc-local-1142" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session find_session_mock = MagicMock() @@ -225,7 +231,8 @@ def test_short_circuit_falls_through_for_non_owning_session(self, monkeypatch): mock_new_session.session_id = "sdlc-local-1143" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -273,7 +280,8 @@ def test_short_circuit_falls_through_for_terminal_status_eng_session(self, monke mock_new_session.session_id = local_id mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( @@ -303,7 +311,8 @@ def test_short_circuit_degrades_on_find_session_error(self, monkeypatch): mock_new_session.session_id = "sdlc-local-1145" mock_as = MagicMock() - mock_as.query.filter.side_effect = [[], [mock_new_session]] + mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) + mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) mock_as.create_local.return_value = mock_new_session with ( diff --git a/tools/sdlc_session_ensure.py b/tools/sdlc_session_ensure.py index 90998c0e3..24b224e23 100644 --- a/tools/sdlc_session_ensure.py +++ b/tools/sdlc_session_ensure.py @@ -29,7 +29,9 @@ signal to inherit (orphaned_lock=true means the owning run died before its next renewal; frees within the TTL) {"error": "RUN_BIND_FAILED", ...} -- lock acquired but the run_id could not be - persisted to the session record (lock released via compare-and-delete) + persisted to the session record. The lock is released via + compare-and-delete ONLY when this call minted the candidate; an adopted + candidate leaves the lease alone (see _acquire_run_lock_and_bind) {} on error {"orphans": [...], "count": N, "killed": false} -- --kill-orphans --dry-run {"results": [...], "count": N, "failures": M, "killed": true} -- --kill-orphans @@ -101,6 +103,23 @@ SDLC_RUN_IDENTITY_HISTORY_MAX = int(os.environ.get("SDLC_RUN_IDENTITY_HISTORY_MAX", "20")) +class _CandidateProvenance: + """How a lock candidate came to hold the value it has (issue #3065 Cluster E). + + ``release_issue_lock`` performs a correct compare-and-delete: it deletes + the lock iff the candidate it is handed still matches the live owner. + That correctness is exactly why passing it an ADOPTED candidate is unsafe + -- an adopted candidate matches the live owner *by construction*, so the + compare-and-delete "succeeds" by destroying a lease this call never + minted. Provenance is the caller-side fact ``release_issue_lock`` has no + way to see: only a MINTED candidate is safe to hand to it for release. + """ + + MINTED = "minted" + ADOPTED_SUPERVISED = "adopted_supervised" + ADOPTED_LIVE_LOCK = "adopted_live_lock" + + def _read_owned_run_ids(session) -> list[str]: """Return this session's recorded owned_run_ids as a list, tolerantly. @@ -422,10 +441,25 @@ def _acquire_run_lock_and_bind( to the fresh candidate, preserving no-adopt for foreign/stale callers. On acquisition, the candidate is saved to ``session.active_run_id`` and - read back from Redis (post-save readback, Race 3). On save failure or - readback mismatch, the lock is released via COMPARE-AND-DELETE - (``release_issue_lock`` -- never a raw DEL, cycle-2 CONCERN 2) so the - next caller acquires immediately instead of waiting out the 1800s TTL. + read back from Redis **by primary key** (``AgentSession.query.get(redis_key=...)``, + Race 3) -- never by re-querying the non-unique ``session_id`` index and + taking ``[0]`` from Popoto's unordered ``SMEMBERS`` result, which is a + coin flip on any lane whose row was duplicated (#3065 Cluster E). + + Release is gated on **provenance**, not on identity: ``candidate`` carries + an explicit ``_CandidateProvenance`` tag set where it is produced -- + ``MINTED`` (fresh ``uuid4().hex``), ``ADOPTED_SUPERVISED`` (inherited from + a live supervised-run signal), or ``ADOPTED_LIVE_LOCK`` (verified reuse via + :func:`_validated_reuse_candidate`). On save failure or readback mismatch, + the lock is released via COMPARE-AND-DELETE (``release_issue_lock`` -- + never a raw DEL, cycle-2 CONCERN 2) **only when the candidate is + ``MINTED``** -- releasing an adopted candidate would delete a lease this + call never created, using a compare-and-delete that matches by + construction because the adopted id equals the live owner. An adopted + candidate that fails to bind is returned as an error without touching the + lock; the next caller acquires immediately after a minted failure instead + of waiting out the 1800s TTL, and an adopted failure leaves the still-live + lease exactly as it was. Target-repo pinning (issue #2012): this is the ONE place ``target_repo`` is resolved for the issue-keyed ``PipelineLedger`` -- the process env @@ -458,6 +492,16 @@ def _acquire_run_lock_and_bind( session_id = getattr(session, "session_id", None) or "" + # Provenance of `reuse_run_id`, tracked separately from its value because + # the value alone is ambiguous (issue #3065 Cluster E): a bare ensure + # under a live supervised-run signal overwrites `reuse_run_id` below with + # the supervisor's id, and `_validated_reuse_candidate` further down + # validates it against the live-lock/self-remint/mirror/anchor proofs. + # Neither shape means "minted here" -- both are adoptions of an identity + # this call did not create, and only a MINTED candidate may ever be + # handed to `release_issue_lock`. + reuse_from_supervised_signal = False + # Supervised-run signal check (issue #2026, WS1). A BARE ensure (no # reuse_run_id) invoked while a LIVE supervised-run signal exists for this # issue must NOT contest the lock or mint — it returns the named @@ -515,6 +559,7 @@ def _acquire_run_lock_and_bind( # (verified reuse renews/re-acquires), returning a NORMAL success # payload carrying it -- never the refusal. reuse_run_id = supervised.run_id + reuse_from_supervised_signal = True elif supervised is not None and supervised.live: logger.debug( "sdlc_session_ensure: issue #%s has a LIVE supervised run (run_id=%s) -- " @@ -530,9 +575,20 @@ def _acquire_run_lock_and_bind( "owner_session_id": supervised.session_id, } - candidate = uuid.uuid4().hex + minted_candidate = uuid.uuid4().hex + candidate = minted_candidate + provenance = _CandidateProvenance.MINTED if reuse_run_id: - candidate = _validated_reuse_candidate(issue_number, session, reuse_run_id) or candidate + validated = _validated_reuse_candidate(issue_number, session, reuse_run_id) + if validated: + candidate = validated + provenance = ( + _CandidateProvenance.ADOPTED_SUPERVISED + if reuse_from_supervised_signal + else _CandidateProvenance.ADOPTED_LIVE_LOCK + ) + # else: validation failed -- fall back to the fresh mint above, + # candidate/provenance are already MINTED. target_repo = _resolve_target_repo() @@ -578,6 +634,26 @@ def _acquire_run_lock_and_bind( "orphaned_lock": orphaned, } + def _release_candidate_if_minted(reason: str) -> None: + """Compare-and-delete the lock, but ONLY for a MINTED candidate. + + An adopted candidate (ADOPTED_SUPERVISED / ADOPTED_LIVE_LOCK) equals + the live lock's owner by construction -- that is what "adopted" means + -- so handing it to `release_issue_lock`'s compare-and-delete would + "succeed" by deleting a lease this call never minted (#3065 Cluster + E). Only a call that minted the candidate itself may release it. + """ + if provenance == _CandidateProvenance.MINTED: + release_issue_lock(issue_number, candidate) + else: + logger.debug( + "sdlc_session_ensure: issue #%s candidate provenance=%s (not minted) -- " + "skipping release on %s; the lock's live owner is not this call's to delete", + issue_number, + provenance, + reason, + ) + # Acquired: bind the run_id to the session record (inspection mirror + # the identity source for the in-process renewal paths). try: @@ -596,13 +672,14 @@ def _acquire_run_lock_and_bind( # still refreshes updated_at via its own stage-state write (see #1676). session.save(update_fields=["active_run_id", "owned_run_ids"]) except Exception as e: - release_issue_lock(issue_number, candidate) + _release_candidate_if_minted("active_run_id save failure") logger.debug( "sdlc_session_ensure: active_run_id save failed for %s (%s: %s) -- " - "lock released via compare-and-delete", + "lock release gated on candidate provenance (%s)", session_id, type(e).__name__, e, + provenance, ) return None, { "error": "RUN_BIND_FAILED", @@ -611,20 +688,27 @@ def _acquire_run_lock_and_bind( } # Post-save readback: assert the record really carries the lock's run_id. + # Read back BY PRIMARY KEY (issue #3065 Cluster E) -- `session.db_key` is + # this row's actual identity (backed by the AutoKeyField `id`). Re-querying + # `AgentSession.query.filter(session_id=session_id)` and taking `[0]` reads + # an ordinary (non-unique) secondary index: any lane whose row was + # duplicated after a crash has two rows sharing one `session_id`, and + # Popoto resolves that filter via an unordered Redis `SMEMBERS`, making + # `[0]` a coin flip between the row this call just wrote and a stale twin. try: from models.agent_session import AgentSession - fresh_rows = list(AgentSession.query.filter(session_id=session_id)) - fresh = fresh_rows[0] if fresh_rows else None + fresh = AgentSession.query.get(redis_key=session.db_key.redis_key) readback_run_id = getattr(fresh, "active_run_id", None) if fresh is not None else None except Exception as e: - release_issue_lock(issue_number, candidate) + _release_candidate_if_minted("post-save readback failure") logger.debug( "sdlc_session_ensure: post-save readback failed for %s (%s: %s) -- " - "lock released via compare-and-delete", + "lock release gated on candidate provenance (%s)", session_id, type(e).__name__, e, + provenance, ) return None, { "error": "RUN_BIND_FAILED", @@ -633,13 +717,14 @@ def _acquire_run_lock_and_bind( } if readback_run_id != candidate: - release_issue_lock(issue_number, candidate) + _release_candidate_if_minted("post-save readback mismatch") logger.debug( "sdlc_session_ensure: post-save readback mismatch for %s " - "(expected %s, read %s) -- lock released via compare-and-delete", + "(expected %s, read %s) -- lock release gated on candidate provenance (%s)", session_id, candidate, readback_run_id, + provenance, ) return None, { "error": "RUN_BIND_FAILED", From 235da8ea493aaff6d70ffe602bf6895fc997b42b Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 15:58:03 +0700 Subject: [PATCH 03/19] [WIP] Migrate + extend verification parser tests to the tri-state (Refs #3065) Replaces test_unknown_expectation_returns_false (which pinned the bug), adds the extended grammar, command-cell extraction, run_checks timeout and exception paths, the #3022 header shape, and the persisted aggregate. --- tests/unit/test_verification_parser.py | 480 ++++++++++++++++++++++--- 1 file changed, 432 insertions(+), 48 deletions(-) diff --git a/tests/unit/test_verification_parser.py b/tests/unit/test_verification_parser.py index 769880e72..a51f6a971 100644 --- a/tests/unit/test_verification_parser.py +++ b/tests/unit/test_verification_parser.py @@ -1,21 +1,35 @@ """Unit tests for agent/verification_parser.py -- machine-readable verification checks.""" +import json from pathlib import Path +import pytest + +from agent.pipeline_ledger import PipelineLedger from agent.verification_parser import ( + CheckOutcome, CheckResult, MalformedRow, ParsedTable, SkippedTable, VerificationCheck, + aggregate_outcomes, evaluate_expectation, format_results, parse_verification_table, + read_verification_outcomes, + record_verification_outcomes, + run_checks, split_row_cells, + unevaluated_reason, ) FIXTURES_DIR = Path(__file__).parents[1] / "fixtures" / "verification" +PASS = CheckOutcome.PASS +FAIL = CheckOutcome.FAIL +UNEVALUATED = CheckOutcome.UNEVALUATED + # --------------------------------------------------------------------------- # parse_verification_table # --------------------------------------------------------------------------- @@ -111,7 +125,12 @@ def test_strips_backticks_from_command(self): checks = parse_verification_table(md).checks assert checks[0].command == "echo hello" - def test_command_without_backticks(self): + def test_command_without_backticks_is_unevaluated_not_guessed(self): + """A cell with no backticked span has no unambiguous command in it. + + Running the whole cell is how a trailing-prose gloss got executed under + `shell=True` (#3065); the row is reported UNEVALUATED instead. + """ md = """\ ## Verification @@ -120,7 +139,11 @@ def test_command_without_backticks(self): | Test | echo hello | exit code 0 | """ checks = parse_verification_table(md).checks - assert checks[0].command == "echo hello" + assert checks[0].unevaluated_reason + assert "backticked span" in checks[0].unevaluated_reason + results = run_checks(checks) + assert results[0].outcome is UNEVALUATED + assert results[0].reason == checks[0].unevaluated_reason def test_table_after_other_content(self): """Verification table can appear after other sections.""" @@ -171,44 +194,109 @@ class TestEvaluateExpectation: """Tests for checking if a command result meets the expectation.""" def test_exit_code_0_pass(self): - assert evaluate_expectation("exit code 0", exit_code=0, output="") is True + assert evaluate_expectation("exit code 0", exit_code=0, output="") is PASS def test_exit_code_0_fail(self): - assert evaluate_expectation("exit code 0", exit_code=1, output="") is False + assert evaluate_expectation("exit code 0", exit_code=1, output="") is FAIL def test_exit_code_nonzero(self): - assert evaluate_expectation("exit code 1", exit_code=1, output="") is True - assert evaluate_expectation("exit code 1", exit_code=0, output="") is False + assert evaluate_expectation("exit code 1", exit_code=1, output="") is PASS + assert evaluate_expectation("exit code 1", exit_code=0, output="") is FAIL def test_output_gt_pass(self): - assert evaluate_expectation("output > 0", exit_code=0, output="3") is True + assert evaluate_expectation("output > 0", exit_code=0, output="3") is PASS def test_output_gt_fail(self): - assert evaluate_expectation("output > 0", exit_code=0, output="0") is False + assert evaluate_expectation("output > 0", exit_code=0, output="0") is FAIL def test_output_gt_non_numeric(self): - assert evaluate_expectation("output > 0", exit_code=0, output="abc") is False + assert evaluate_expectation("output > 0", exit_code=0, output="abc") is FAIL def test_output_contains_pass(self): assert ( evaluate_expectation("output contains hello", exit_code=0, output="say hello world") - is True + is PASS ) def test_output_contains_fail(self): assert ( - evaluate_expectation("output contains hello", exit_code=0, output="say goodbye") - is False + evaluate_expectation("output contains hello", exit_code=0, output="say goodbye") is FAIL ) def test_output_contains_case_sensitive(self): assert ( - evaluate_expectation("output contains Hello", exit_code=0, output="hello world") - is False + evaluate_expectation("output contains Hello", exit_code=0, output="hello world") is FAIL ) - def test_unknown_expectation_returns_false(self): - assert evaluate_expectation("something weird", exit_code=0, output="ok") is False + def test_unknown_expectation_is_unevaluated_with_a_reason(self): + """An expectation the grammar cannot read is UNEVALUATED, not FAIL. + + This replaces `test_unknown_expectation_returns_false`, which asserted + the silent `False` fall-through as *intended* behavior and so pinned + the bug: a gate reporting red when it means "I did not understand the + question" asserts a fact it never read (#3065). + """ + assert evaluate_expectation("something weird", exit_code=0, output="ok") is UNEVALUATED + reason = unevaluated_reason("something weird") + assert "something weird" in reason + assert "unrecognized expectation form" in reason + + def test_empty_and_none_expectations_are_unevaluated(self): + """Empty, whitespace-only, and None cells: never FAIL, never PASS.""" + for cell in ("", " ", "\n\t ", None): + assert evaluate_expectation(cell, exit_code=0, output="ok") is UNEVALUATED + assert "empty" in unevaluated_reason("") + assert "empty" in unevaluated_reason(None) + + +class TestExtendedExpectationGrammar: + """The corpus #2836's spike-5 measured and deferred to #2791, which was + closed as consolidated without a fix. Re-derived by sweeping `Expected` + cells across the active plans in docs/plans/: every form below appears + there and every one of them returned a silent False before this change.""" + + def test_prints_backticked_value(self): + assert evaluate_expectation("prints `0`", exit_code=0, output="0\n") is PASS + assert evaluate_expectation("prints `0`", exit_code=0, output="1\n") is FAIL + assert evaluate_expectation("prints 0", exit_code=0, output="0") is PASS + + def test_equals_n(self): + assert evaluate_expectation("== 0", exit_code=0, output="0") is PASS + assert evaluate_expectation("== 2", exit_code=0, output="3") is FAIL + assert evaluate_expectation("output == 2", exit_code=0, output="2\n") is PASS + + def test_gte_n(self): + assert evaluate_expectation(">= 1", exit_code=0, output="5") is PASS + assert evaluate_expectation(">= 1", exit_code=0, output="0") is FAIL + assert evaluate_expectation("output >= 3", exit_code=0, output="3") is PASS + + def test_bare_gt_n(self): + assert evaluate_expectation("> 0", exit_code=0, output="2") is PASS + assert evaluate_expectation("> 0", exit_code=0, output="0") is FAIL + + def test_empty_output(self): + assert evaluate_expectation("empty output", exit_code=0, output="") is PASS + assert evaluate_expectation("empty output", exit_code=0, output=" \n") is PASS + assert evaluate_expectation("empty output", exit_code=0, output="x") is FAIL + + def test_exit_n_without_the_word_code(self): + assert evaluate_expectation("exit 0", exit_code=0, output="") is PASS + assert evaluate_expectation("exit 0", exit_code=1, output="") is FAIL + assert evaluate_expectation("exit 1", exit_code=1, output="") is PASS + + def test_non_numeric_output_for_a_numeric_form_is_a_real_fail(self): + """The expectation was understood; the command answered a non-number. + That is evidence about the code, so it is FAIL, not UNEVALUATED.""" + assert evaluate_expectation(">= 1", exit_code=0, output="abc") is FAIL + assert evaluate_expectation("== 1", exit_code=0, output="") is FAIL + + def test_trailing_prose_on_a_numeric_form_is_unevaluated(self): + """`output == 2 (the two read sites)` appears verbatim in a live plan. + Prefix-matching it would grade a sentence nobody wrote as a number.""" + assert ( + evaluate_expectation("output == 2 (the two read sites)", exit_code=0, output="2") + is UNEVALUATED + ) # --------------------------------------------------------------------------- @@ -223,17 +311,17 @@ class TestEvaluateExpectationInverse: def test_exit_code_ne_pass(self): """Passes when exit code differs from N.""" - assert evaluate_expectation("exit code != 0", exit_code=1, output="") is True - assert evaluate_expectation("exit code != 0", exit_code=2, output="") is True + assert evaluate_expectation("exit code != 0", exit_code=1, output="") is PASS + assert evaluate_expectation("exit code != 0", exit_code=2, output="") is PASS def test_exit_code_ne_fail(self): """Fails when exit code equals N (command should have failed but succeeded).""" - assert evaluate_expectation("exit code != 0", exit_code=0, output="") is False + assert evaluate_expectation("exit code != 0", exit_code=0, output="") is FAIL def test_exit_code_ne_nonzero_n(self): """Works for N != 0 too.""" - assert evaluate_expectation("exit code != 2", exit_code=0, output="") is True - assert evaluate_expectation("exit code != 2", exit_code=2, output="") is False + assert evaluate_expectation("exit code != 2", exit_code=0, output="") is PASS + assert evaluate_expectation("exit code != 2", exit_code=2, output="") is FAIL def test_exit_code_ne_grammar_collision_regression(self): """Regression: 'exit code != 0' must be evaluated by the inverse branch, @@ -245,9 +333,9 @@ def test_exit_code_ne_grammar_collision_regression(self): and evaluates correctly. """ # exit_code=0 should FAIL (code matches the forbidden value) - assert evaluate_expectation("exit code != 0", exit_code=0, output="") is False + assert evaluate_expectation("exit code != 0", exit_code=0, output="") is FAIL # exit_code=1 should PASS (code differs from forbidden value) - assert evaluate_expectation("exit code != 0", exit_code=1, output="") is True + assert evaluate_expectation("exit code != 0", exit_code=1, output="") is PASS # --- output does not contain X --- @@ -259,7 +347,7 @@ def test_output_does_not_contain_pass(self): exit_code=0, output="SELECT * FROM users", ) - is True + is PASS ) def test_output_does_not_contain_fail_present(self): @@ -270,7 +358,7 @@ def test_output_does_not_contain_fail_present(self): exit_code=0, output="ALTER TABLE; DROP TABLE users;", ) - is False + is FAIL ) def test_output_does_not_contain_empty_stdout_gate(self): @@ -285,7 +373,7 @@ def test_output_does_not_contain_empty_stdout_gate(self): exit_code=1, output="", ) - is False + is FAIL ) # Whitespace-only stdout also triggers the gate assert ( @@ -294,7 +382,7 @@ def test_output_does_not_contain_empty_stdout_gate(self): exit_code=0, output=" \n ", ) - is False + is FAIL ) def test_output_does_not_contain_ordering_regression(self): @@ -313,7 +401,7 @@ def test_output_does_not_contain_ordering_regression(self): exit_code=0, output="all clean, no matches", ) - is True + is PASS ) # FOO present → inverse branch → False (not positive branch which would be True) assert ( @@ -322,39 +410,39 @@ def test_output_does_not_contain_ordering_regression(self): exit_code=0, output="found FOO in file", ) - is False + is FAIL ) # --- match count == 0 --- def test_match_count_zero_bare_zero(self): """grep -c PATTERN file → emits literal '0', exit 1 → passes.""" - assert evaluate_expectation("match count == 0", exit_code=1, output="0") is True + assert evaluate_expectation("match count == 0", exit_code=1, output="0") is PASS def test_match_count_zero_whitespace_zero(self): """grep -r PATTERN dir | wc -l → emits ' 0' (leading whitespace) → passes.""" - assert evaluate_expectation("match count == 0", exit_code=0, output=" 0") is True + assert evaluate_expectation("match count == 0", exit_code=0, output=" 0") is PASS def test_match_count_zero_single_path_colon_zero(self): """grep -rc PATTERN file → emits 'path/to/file:0' → passes.""" assert ( - evaluate_expectation("match count == 0", exit_code=1, output="path/to/file:0") is True + evaluate_expectation("match count == 0", exit_code=1, output="path/to/file:0") is PASS ) def test_match_count_zero_multiline_path_colon_zero(self): """grep -rc PATTERN dir → emits multiple 'path:0' lines → passes.""" output = "a.txt:0\nb.txt:0\nc.py:0" - assert evaluate_expectation("match count == 0", exit_code=1, output=output) is True + assert evaluate_expectation("match count == 0", exit_code=1, output=output) is PASS def test_match_count_zero_nonzero_count_fails(self): """Any non-zero count fails.""" - assert evaluate_expectation("match count == 0", exit_code=0, output="3") is False - assert evaluate_expectation("match count == 0", exit_code=0, output="path:3") is False + assert evaluate_expectation("match count == 0", exit_code=0, output="3") is FAIL + assert evaluate_expectation("match count == 0", exit_code=0, output="path:3") is FAIL def test_match_count_zero_mixed_lines_fails(self): """Mixed zero and non-zero lines — one non-zero line must fail the whole check.""" output = "a.txt:0\nb.txt:2" - assert evaluate_expectation("match count == 0", exit_code=1, output=output) is False + assert evaluate_expectation("match count == 0", exit_code=1, output=output) is FAIL def test_match_count_zero_empty_stdout_gate(self): """Empty/whitespace-only stdout must NOT vacuously pass (empty-stdout gate). @@ -362,8 +450,8 @@ def test_match_count_zero_empty_stdout_gate(self): all(...) over an empty list is True in Python; without the gate, a command that errored or wrote only to stderr would produce empty stdout and pass. """ - assert evaluate_expectation("match count == 0", exit_code=1, output="") is False - assert evaluate_expectation("match count == 0", exit_code=0, output=" \n") is False + assert evaluate_expectation("match count == 0", exit_code=1, output="") is FAIL + assert evaluate_expectation("match count == 0", exit_code=0, output=" \n") is FAIL def test_match_count_zero_literal_zero_passes_not_gated(self): """Literal '0' (non-empty stdout) must NOT be blocked by the empty-stdout gate. @@ -371,22 +459,22 @@ def test_match_count_zero_literal_zero_passes_not_gated(self): This confirms the gate fires only on truly-empty output, not on a legitimately- clean grep -c result. """ - assert evaluate_expectation("match count == 0", exit_code=1, output="0") is True + assert evaluate_expectation("match count == 0", exit_code=1, output="0") is PASS # --- positive forms unchanged (regression) --- def test_positive_exit_code_still_works(self): - assert evaluate_expectation("exit code 0", exit_code=0, output="") is True - assert evaluate_expectation("exit code 0", exit_code=1, output="") is False - assert evaluate_expectation("exit code 1", exit_code=1, output="") is True + assert evaluate_expectation("exit code 0", exit_code=0, output="") is PASS + assert evaluate_expectation("exit code 0", exit_code=1, output="") is FAIL + assert evaluate_expectation("exit code 1", exit_code=1, output="") is PASS def test_positive_output_contains_still_works(self): - assert evaluate_expectation("output contains ok", exit_code=0, output="all ok") is True - assert evaluate_expectation("output contains ok", exit_code=0, output="bad") is False + assert evaluate_expectation("output contains ok", exit_code=0, output="all ok") is PASS + assert evaluate_expectation("output contains ok", exit_code=0, output="bad") is FAIL def test_positive_output_gt_still_works(self): - assert evaluate_expectation("output > 0", exit_code=0, output="3") is True - assert evaluate_expectation("output > 0", exit_code=0, output="0") is False + assert evaluate_expectation("output > 0", exit_code=0, output="3") is PASS + assert evaluate_expectation("output > 0", exit_code=0, output="0") is FAIL # --------------------------------------------------------------------------- @@ -500,7 +588,7 @@ def test_format_results_names_authoring_errors_separately(self): def test_a_malformed_row_fails_the_run(self): """A row nobody can execute is not a passing check.""" check = VerificationCheck(name="ok", command="true", expected="exit code 0") - results = [CheckResult(check=check, passed=True, exit_code=0, output="")] + results = [CheckResult(check=check, outcome=PASS, exit_code=0, output="")] clean_table = ParsedTable(checks=[check], malformed=[], skipped=[]) assert "All checks passed." in format_results(results, clean_table) malformed_table = ParsedTable( @@ -581,7 +669,7 @@ def test_skipped_table_does_not_fail_the_run(self): assert len(table.checks) == 2 assert len(table.skipped) == 1 results = [ - CheckResult(check=c, passed=True, exit_code=0, output="ok") for c in table.checks + CheckResult(check=c, outcome=PASS, exit_code=0, output="ok") for c in table.checks ] report = format_results(results, table) assert "All checks passed." in report @@ -602,3 +690,299 @@ def test_skipped_table_is_importable(self): def test_parsed_table_carries_skipped_field(self): assert "skipped" in ParsedTable.__dataclass_fields__ + + +# --------------------------------------------------------------------------- +# Command-cell extraction: first backticked span (#3065) +# --------------------------------------------------------------------------- + + +class TestCommandCellExtraction: + """The command is the cell's first backticked span, not the whole cell + with its outer backticks stripped. Spike-5 executed the old reading on + main: the cell ``` `echo hi` -- this checks greeting ``` produced the + shell string ``echo hi` -- this checks greeting`` and ran it.""" + + def _table(self, row: str) -> str: + return f"## Verification\n\n| Check | Command | Expected |\n|--|--|--|\n{row}\n" + + def test_em_dash_trailing_prose_is_not_part_of_the_command(self): + parsed = parse_verification_table( + self._table("| Greeting | `echo hi` -- this checks greeting | output contains hi |") + ) + assert parsed.malformed == [] + assert parsed.checks[0].command == "echo hi" + assert parsed.checks[0].unevaluated_reason == "" + assert run_checks(parsed.checks)[0].outcome is PASS + + def test_parenthetical_gloss_is_not_part_of_the_command(self): + parsed = parse_verification_table( + self._table("| Count | `echo 3` (three of them) | output > 0 |") + ) + assert parsed.checks[0].command == "echo 3" + + def test_two_spans_take_the_first_and_record_that_they_did(self): + parsed = parse_verification_table( + self._table("| Two | `echo first` then `echo second` | output contains first |") + ) + check = parsed.checks[0] + assert check.command == "echo first" + assert "2 backticked spans" in check.extraction_note + assert "ran the first" in check.extraction_note + results = run_checks(parsed.checks) + assert results[0].outcome is PASS + assert check.extraction_note in format_results(results, parsed) + + +# --------------------------------------------------------------------------- +# run_checks: timeout and runner-exception dispositions (#3065) +# --------------------------------------------------------------------------- + + +class TestRunChecksUnevaluatedPaths: + """A timeout and a runner exception are both UNEVALUATED with the reason + attached. Both used to be `passed=False`, rendered `[FAIL]` -- a gate + asserting the code is wrong when it never got an answer.""" + + def test_timeout_is_unevaluated_and_never_renders_as_fail(self): + check = VerificationCheck(name="slow", command="sleep 5", expected="exit code 0") + results = run_checks([check], timeout=1) + assert results[0].outcome is UNEVALUATED + assert "timed out after 1s" in results[0].reason + report = format_results(results, ParsedTable(checks=[check], malformed=[], skipped=[])) + assert "[FAIL]" not in report + assert "[UNEVALUATED] slow" in report + assert "All checks passed." not in report + + def test_runner_exception_is_unevaluated_with_an_observable_reason(self, monkeypatch): + def boom(*args, **kwargs): + raise OSError("no shell for you") + + monkeypatch.setattr("agent.verification_parser.subprocess.run", boom) + check = VerificationCheck(name="explodes", command="true", expected="exit code 0") + results = run_checks([check]) + assert results[0].outcome is UNEVALUATED + assert "OSError" in results[0].reason + assert "no shell for you" in results[0].reason + report = format_results(results, ParsedTable(checks=[check], malformed=[], skipped=[])) + assert "[FAIL]" not in report + assert "no shell for you" in report + + def test_unrecognized_expectation_row_is_unevaluated_end_to_end(self): + check = VerificationCheck(name="odd", command="echo ok", expected="ok") + results = run_checks([check]) + assert results[0].outcome is UNEVALUATED + assert "unrecognized expectation form" in results[0].reason + report = format_results(results, ParsedTable(checks=[check], malformed=[], skipped=[])) + assert "[FAIL]" not in report + + +# --------------------------------------------------------------------------- +# Check-table classification by column contract (#3022) +# --------------------------------------------------------------------------- + + +class TestCheckTableContract: + """A check table is `(, Command, Expected)`. The predicate this + replaced asked whether *any* of the first three headers was `Command`.""" + + def test_issue_3022_header_shape_is_not_executed(self): + """`| Command | Observed stdout | Observed exit |` is a results recap. + + On main it classified as a check table and its "Observed stdout" + column was executed as a shell command, with an empty `skipped` list + and no diagnostic at all. + """ + md = ( + "## Verification\n\n" + "| Check | Command | Expected |\n|--|--|--|\n" + "| Real check | `echo ok` | output contains ok |\n" + "\n" + "| Command | Observed stdout | Observed exit |\n|--|--|--|\n" + "| `grep -c x f` | 0 | 1 |\n" + ) + table = parse_verification_table(md) + assert [c.name for c in table.checks] == ["Real check"] + assert len(table.skipped) == 1 + assert table.skipped[0].header.startswith("| Command | Observed stdout") + assert "Command, Expected" in table.skipped[0].reason + assert table.malformed == [] + + def test_criterion_recap_table_is_skipped_not_executed(self): + """`| # | Criterion | Check |` -- the one false positive the `any` + predicate admits across this repo's plans. Its third column ("Check") + was read as the Expected cell and its second as a command.""" + md = ( + "## Verification\n\n" + "| Check | Command | Expected |\n|--|--|--|\n" + "| Real check | `echo ok` | output contains ok |\n" + "\n" + "| # | Criterion | Check |\n|--|--|--|\n" + "| 1 | something | manual |\n" + ) + table = parse_verification_table(md) + assert len(table.checks) == 1 + assert len(table.skipped) == 1 + + def test_a_non_command_second_column_is_not_a_check_table(self): + md = ( + "## Verification\n\n" + "| Check | Command | Expected |\n|--|--|--|\n" + "| Real check | `echo ok` | output contains ok |\n" + "\n" + "| Row | Pre-change | Meaning |\n|--|--|--|\n" + "| a | 1 | red |\n" + ) + assert len(parse_verification_table(md).skipped) == 1 + + def test_named_first_column_still_qualifies(self): + """`Anti-criterion | Command | Expected` is a real check table.""" + md = ( + "## Verification\n\n" + "| Anti-criterion | Command | Expected |\n|--|--|--|\n" + "| No leftovers | `echo 0` | == 0 |\n" + ) + table = parse_verification_table(md) + assert len(table.checks) == 1 + assert table.skipped == [] + + +# --------------------------------------------------------------------------- +# The graded aggregate (#3065, Cluster B -> Cluster C) +# --------------------------------------------------------------------------- + + +class TestAggregateOutcomes: + def _result(self, outcome, name="c"): + return CheckResult( + check=VerificationCheck(name=name, command="true", expected="exit code 0"), + outcome=outcome, + exit_code=0, + output="", + ) + + def test_all_pass(self): + agg = aggregate_outcomes([self._result(PASS), self._result(PASS)]) + assert agg["outcome"] == "PASS" + assert agg["counts"]["PASS"] == 2 + + def test_any_fail_dominates(self): + agg = aggregate_outcomes( + [self._result(PASS), self._result(FAIL), self._result(UNEVALUATED)] + ) + assert agg["outcome"] == "FAIL" + + def test_unevaluated_blocks_a_pass(self): + agg = aggregate_outcomes([self._result(PASS), self._result(UNEVALUATED)]) + assert agg["outcome"] == "UNEVALUATED" + + def test_no_checks_is_not_a_vacuous_pass(self): + assert aggregate_outcomes([])["outcome"] == "UNEVALUATED" + + def test_malformed_rows_make_the_run_fail(self): + table = ParsedTable( + checks=[], malformed=[MalformedRow(line="| x |", reason="r")], skipped=[] + ) + agg = aggregate_outcomes([self._result(PASS)], table) + assert agg["outcome"] == "FAIL" + assert agg["malformed"] == 1 + + def test_rows_carry_their_reasons(self): + r = self._result(UNEVALUATED, name="odd") + r.reason = "unrecognized expectation form: 'ok'" + agg = aggregate_outcomes([r]) + assert agg["rows"] == [ + { + "name": "odd", + "outcome": "UNEVALUATED", + "reason": "unrecognized expectation form: 'ok'", + } + ] + + +class TestRecordVerificationOutcomes: + """The aggregate is written to the issue-keyed ledger's `stage_states_json` + under `_verification_outcomes`, stamped with the PR head SHA it was graded + against. Real Redis, per this repo's testing philosophy; every test cleans + up the ledger it creates.""" + + REPO = "test-owner/test-repo" + ISSUE = 927380 + + @pytest.fixture(autouse=True) + def clean_ledger(self): + self._cleanup() + yield + self._cleanup() + + def _cleanup(self): + for record in PipelineLedger.query.filter(ledger_key=f"{self.REPO}:{self.ISSUE}"): + record.delete() + + def _results(self): + return [ + CheckResult( + check=VerificationCheck(name="ok", command="true", expected="exit code 0"), + outcome=PASS, + exit_code=0, + output="", + ) + ] + + def test_aggregate_carries_the_resolved_head_sha(self, monkeypatch): + sha = "a" * 40 + seen = {} + + def fake_resolver(pr, repo=None, repo_root=None, **kwargs): + seen["pr"] = pr + seen["repo"] = repo + return sha + + monkeypatch.setattr("tools.pr_head_resolver.resolve_pr_head_sha", fake_resolver) + assert record_verification_outcomes(self.REPO, self.ISSUE, self._results(), pr_number=4242) + record = read_verification_outcomes(self.REPO, self.ISSUE) + assert record["head_sha"] == sha + assert record["outcome"] == "PASS" + assert seen == {"pr": 4242, "repo": self.REPO} + + def test_lane_with_no_pr_records_no_head_sha_and_does_not_crash(self): + assert record_verification_outcomes(self.REPO, self.ISSUE, self._results()) + record = read_verification_outcomes(self.REPO, self.ISSUE) + assert "head_sha" not in record + assert record["outcome"] == "PASS" + + def test_unresolvable_head_records_without_an_anchor(self, monkeypatch): + def unresolvable(pr, repo=None, repo_root=None, **kwargs): + return None + + monkeypatch.setattr("tools.pr_head_resolver.resolve_pr_head_sha", unresolvable) + assert record_verification_outcomes(self.REPO, self.ISSUE, self._results(), pr_number=4242) + assert "head_sha" not in read_verification_outcomes(self.REPO, self.ISSUE) + + def test_resolver_failure_does_not_lose_the_aggregate(self, monkeypatch): + def boom(pr, repo=None, repo_root=None, **kwargs): + raise RuntimeError("ls-remote exploded") + + monkeypatch.setattr("tools.pr_head_resolver.resolve_pr_head_sha", boom) + assert record_verification_outcomes(self.REPO, self.ISSUE, self._results(), pr_number=4242) + record = read_verification_outcomes(self.REPO, self.ISSUE) + assert record["outcome"] == "PASS" + assert "head_sha" not in record + + def test_the_write_lands_under_the_pinned_key(self): + record_verification_outcomes(self.REPO, self.ISSUE, self._results()) + ledger = PipelineLedger.get(self.REPO, self.ISSUE) + blob = json.loads(ledger.stage_states_json) + assert "_verification_outcomes" in blob + + def test_a_later_run_replaces_the_earlier_aggregate(self): + record_verification_outcomes(self.REPO, self.ISSUE, self._results()) + failing = self._results() + failing[0].outcome = FAIL + record_verification_outcomes(self.REPO, self.ISSUE, failing) + assert read_verification_outcomes(self.REPO, self.ISSUE)["outcome"] == "FAIL" + + def test_reads_and_writes_without_a_lane_are_inert(self): + assert record_verification_outcomes(None, None, self._results()) is False + assert read_verification_outcomes(None, None) is None + assert read_verification_outcomes(self.REPO, 927381) is None From 35925b463bc6d181b59629e86b36bf3e884b4240 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 16:01:25 +0700 Subject: [PATCH 04/19] Converge both verification runners on one bound and one disposition (Refs #3065) The parity fixture gains the two shapes on which the runners actually disagreed and which it could not previously express: a timeout (FAIL at 120s in the canonical runner vs a non-blocking SKIP at 30s in validate_build) and an expectation neither grammar reads. validate_build now takes its bound and its timeout disposition from the canonical runner, and UNEVALUATED blocks the exit code where SKIP did not. --- tests/README.md | 4 +- .../fixtures/verification/runner_agreement.md | 20 ++++- tests/unit/test_validate_build.py | 85 ++++++++++++++++--- tests/unit/test_verification_parser.py | 2 +- 4 files changed, 94 insertions(+), 17 deletions(-) diff --git a/tests/README.md b/tests/README.md index 3d00e81f9..a0f067a16 100644 --- a/tests/README.md +++ b/tests/README.md @@ -299,8 +299,8 @@ tests/ | unit | `test_validate_documentation_section.py` | 41 | Documentation section validation, payload-targeted | | unit | `test_validate_verification_section.py` | 31 | Verification validation | | unit | `test_features_readme_sort.py` | 27 | README table sorting | -| unit | `test_verification_parser.py` | 58 | Verification section parsing (per-block table scoping, `SkippedTable`) | -| unit | `test_validate_build.py` | 43 | `scripts/validate_build.py` execution loop (30s-timeout SKIP, cross-runner agreement with `agent.verification_parser` on parse-only fixtures) | +| unit | `test_verification_parser.py` | 90 | Verification section parsing (per-block scoping, check-table column contract, first-backticked-span commands, PASS/FAIL/UNEVALUATED outcomes, persisted `_verification_outcomes` aggregate) | +| unit | `test_validate_build.py` | 47 | `scripts/validate_build.py` execution loop (shared timeout bound, UNEVALUATED-on-timeout, cross-runner agreement with `agent.verification_parser`) | | unit | `test_validate_commit_message.py` | 16 | Commit message format | | unit | `test_validate_sdlc_on_stop.py` | 12 | SDLC stop validation | | unit | `test_build_validation.py` | 6 | Build process validation | diff --git a/tests/fixtures/verification/runner_agreement.md b/tests/fixtures/verification/runner_agreement.md index d5bfaf9f0..3aa8a3ebc 100644 --- a/tests/fixtures/verification/runner_agreement.md +++ b/tests/fixtures/verification/runner_agreement.md @@ -1,9 +1,12 @@ ## Verification -Every command below is instantaneous, hermetic, and deterministic. Used by -`test_both_runners_agree_on_execution_fixture` to assert `validate_build`'s -per-check verdicts equal `run_checks`' verdicts row for row, across all six -`evaluate_expectation` branches. +Every command below is instantaneous, hermetic, and deterministic (the +`sleep` row is bounded by the short timeout the parity test passes both +runners). Used by `test_both_runners_agree_on_execution_fixture` to assert +`validate_build`'s per-check verdicts equal `run_checks`' verdicts row for +row, across every `evaluate_expectation` branch **and** across the two +dispositions the runners used to disagree on: a timeout (`FAIL` at 120s vs +`SKIP` at 30s) and an expectation neither grammar recognises. | Check | Command | Expected | |-------|---------|----------| @@ -13,3 +16,12 @@ per-check verdicts equal `run_checks`' verdicts row for row, across all six | output does not contain X | `echo hello` | output does not contain goodbye | | match count == 0 | `grep -c zzz /dev/null` | match count == 0 | | output > N | `echo 1` | output > 0 | +| exit N | `true` | exit 0 | +| prints N | `echo 0` | prints `0` | +| >= N | `echo 5` | >= 1 | +| == N | `echo 0` | == 0 | +| empty output | `true` | empty output | +| a real failure | `echo 1` | == 0 | +| timeout | `sleep 30` | exit code 0 | +| unparseable expectation | `true` | banana | +| no backticked span | echo hi | exit code 0 | diff --git a/tests/unit/test_validate_build.py b/tests/unit/test_validate_build.py index 9d4c372c2..65927a698 100644 --- a/tests/unit/test_validate_build.py +++ b/tests/unit/test_validate_build.py @@ -10,6 +10,8 @@ import pytest from agent.verification_parser import ( + DEFAULT_TIMEOUT_S, + CheckOutcome, MalformedRow, ParsedTable, SkippedTable, @@ -248,17 +250,70 @@ def test_output_check(self): assert len(results) == 1 assert results[0]["status"] == "PASS" - def test_timeout_skips(self): + def test_timeout_is_unevaluated_not_skip(self): + """This module called a timeout `SKIP` at a 30s bound while the + canonical runner called it `FAIL` at 120s -- two runners, two verdicts + on the same event, and `SKIP` did not even block the exit code. Both + now say UNEVALUATED at the shared bound (#2901/#3065).""" check = VerificationCheck(name="slow cmd", command="sleep 60", expected="exit code 0") table = ParsedTable(checks=[check], malformed=[], skipped=[]) with patch.object( subprocess, "run", - side_effect=subprocess.TimeoutExpired("sleep", 30), + side_effect=subprocess.TimeoutExpired("sleep", DEFAULT_TIMEOUT_S), ): results = validate_build.check_verification_table(table) assert len(results) == 1 - assert results[0]["status"] == "SKIP" + assert results[0]["status"] == "UNEVALUATED" + assert "timed out" in results[0]["message"] + + def test_the_execution_bound_is_the_shared_one(self): + """One bound, named once. A private ceiling here is how the two + runners drifted apart in the first place.""" + recorded = {} + + def capture(*args, **kwargs): + recorded["timeout"] = kwargs.get("timeout") + raise subprocess.TimeoutExpired("cmd", kwargs.get("timeout", 0)) + + check = VerificationCheck(name="any", command="true", expected="exit code 0") + table = ParsedTable(checks=[check], malformed=[], skipped=[]) + with patch.object(subprocess, "run", side_effect=capture): + validate_build.check_verification_table(table) + assert recorded["timeout"] == DEFAULT_TIMEOUT_S + + def test_unrecognized_expectation_is_unevaluated_not_fail(self): + check = VerificationCheck(name="odd", command="echo ok", expected="ok") + table = ParsedTable(checks=[check], malformed=[], skipped=[]) + results = validate_build.check_verification_table(table) + assert results[0]["status"] == "UNEVALUATED" + assert "unrecognized expectation form" in results[0]["message"] + + def test_command_cell_with_no_backticked_span_is_never_executed(self): + table = parse_verification_table( + "## Verification\n\n| Check | Command | Expected |\n|--|--|--|\n" + "| Bare | echo hi | exit code 0 |\n" + ) + with patch.object(subprocess, "run", side_effect=AssertionError("must not run")): + results = validate_build.check_verification_table(table) + assert results[0]["status"] == "UNEVALUATED" + + def test_unevaluated_blocks_the_exit_code(self, tmp_path, capsys): + """SKIP was non-blocking, which is how an ungraded row reached green. + UNEVALUATED blocks.""" + f = tmp_path / "unevaluated.md" + f.write_text( + textwrap.dedent("""\ + ## Verification + + | Check | Command | Expected | + |-------|---------|----------| + | Odd expectation | `echo ok` | banana | + """) + ) + with patch("sys.argv", ["validate_build.py", str(f)]): + assert validate_build.main() == 1 + assert "UNEVALUATED" in capsys.readouterr().out def test_malformed_row_fails(self): table = ParsedTable( @@ -426,7 +481,7 @@ def test_trailing_newline_parity_with_run_checks(self): vb_results = validate_build.check_verification_table(table) rc_results = run_checks([check]) assert vb_results[0]["status"] == "PASS" - assert rc_results[0].passed is True + assert rc_results[0].outcome is CheckOutcome.PASS # --------------------------------------------------------------------------- @@ -457,22 +512,32 @@ def test_trailing_newline_parity_with_run_checks(self): class TestCrossRunnerAgreement: def test_both_runners_agree_on_execution_fixture(self): + """Per-check parity, now including the two shapes on which the runners + genuinely disagreed and which the fixture could not previously express: + a timeout (FAIL@120s here, SKIP@30s there) and an expectation neither + grammar reads. Both runners are driven at a short bound so the timeout + row costs seconds, not minutes.""" p = FIXTURES_DIR / "runner_agreement.md" assert p.read_text().lstrip().startswith("## Verification") table = parse_verification_table(p.read_text()) assert table.malformed == [] - assert len(table.checks) == 6 + assert len(table.checks) == 15 - vb_results = validate_build.check_verification_table(table) - rc_results = run_checks(table.checks) + vb_results = validate_build.check_verification_table(table, timeout=2) + rc_results = run_checks(table.checks, timeout=2) assert len(vb_results) == len(rc_results) == len(table.checks) for vb, rc, check in zip(vb_results, rc_results, table.checks, strict=True): - vb_passed = vb["status"] == "PASS" - assert vb_passed == rc.passed, ( - f"{check.name}: validate_build={vb['status']!r} run_checks.passed={rc.passed!r}" + assert vb["status"] == rc.outcome.value, ( + f"{check.name}: validate_build={vb['status']!r} run_checks={rc.outcome.value!r}" ) + outcomes = [rc.outcome for rc in rc_results] + assert CheckOutcome.FAIL in outcomes, "fixture must exercise a real failure" + assert outcomes.count(CheckOutcome.UNEVALUATED) == 3, ( + "fixture must exercise timeout, unparseable expectation, and no-backticked-span" + ) + @pytest.mark.parametrize("fixture_name", PARSE_ONLY_FIXTURES) def test_parse_only_fixtures_parse_identically(self, fixture_name): """Parsed through `parse_verification_table` alone -- commands in diff --git a/tests/unit/test_verification_parser.py b/tests/unit/test_verification_parser.py index a51f6a971..60660fb40 100644 --- a/tests/unit/test_verification_parser.py +++ b/tests/unit/test_verification_parser.py @@ -741,7 +741,7 @@ def test_two_spans_take_the_first_and_record_that_they_did(self): class TestRunChecksUnevaluatedPaths: """A timeout and a runner exception are both UNEVALUATED with the reason - attached. Both used to be `passed=False`, rendered `[FAIL]` -- a gate + attached. Both used to grade false and render `[FAIL]` -- a gate asserting the code is wrong when it never got an answer.""" def test_timeout_is_unevaluated_and_never_renders_as_fail(self): From 13b69addac3c34765822cedff10e1014a1bc919b Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 16:04:35 +0700 Subject: [PATCH 05/19] Anchor only the newly added expectation forms (Refs #3065) A sweep of docs/plans/ found eleven live rows writing a trailing gloss on exit code N / output > N. Anchoring those would have turned working rows into blocking UNEVALUATED for a change nobody asked for, so the pre-existing forms keep prefix semantics and only the forms this lane adds are anchored. Verified against main: the only grading change across active plans is 'exit 0', which previously graded false. --- agent/verification_parser.py | 50 ++++++++++++++++++++++---- tests/unit/test_verification_parser.py | 21 +++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/agent/verification_parser.py b/agent/verification_parser.py index e9b680fac..de43eb3b2 100644 --- a/agent/verification_parser.py +++ b/agent/verification_parser.py @@ -117,6 +117,15 @@ # key in an already-flexible JSON blob: no schema field, no migration. VERIFICATION_OUTCOMES_KEY = "_verification_outcomes" +# Named refusal reason for a `_verification_outcomes` aggregate whose stamped +# `head_sha` does not match the PR's current head (task 8, #3065). Defined +# here -- where the aggregate is written and stamped -- rather than in +# `tools/merge_predicate.py`, which reads it: a Verification row greps for +# the constant *name*, matching how `reconcile_dispatch`, `decision_inputs`, +# `resolve_branch_truth`, and `G3_REDIRECT_REASON_DOCS_PENDING` are already +# pinned. Never read the prose; grep the symbol. +VERIFICATION_OUTCOMES_STALE_REASON = "verification outcome predates PR head commit" + class CheckOutcome(StrEnum): """The three things a verification check can say. @@ -491,6 +500,18 @@ def evaluate_expectation(expected: str | None, *, exit_code: int, output: str) - The inverse ``exit code != N`` branch is checked BEFORE the positive ``exit code N`` branch, and ``output does not contain X`` is checked BEFORE ``output contains X``, so the inverse forms are always matched first and never captured by positive matchers. + + **Anchoring rule.** The pre-existing forms (``exit code N``, ``output > N``, + ``output contains X``) stay prefix-matched, because a trailing gloss on + them -- ``exit code 0 (verified 2026-09-02 ...)`` -- is an established + authoring idiom in this repo's live plans and anchoring would turn eleven + working rows into blocking ``UNEVALUATED`` for a change nobody asked for. + The forms added here (``exit N``, ``prints `N```, ``>= N``, ``> N``, + ``== N``, ``empty output``) are **anchored**: a bare comparator followed by + prose is easy to write by accident, and grading a sentence as if it were a + number is exactly the guess this module exists to stop making. An anchored + form that does not match reports ``UNEVALUATED`` naming the cell, which + tells the author what to fix. """ if expected is None or not expected.strip(): # An empty cell is not a failed check; it is an ungraded one. @@ -540,8 +561,17 @@ def numeric_verdict(op) -> CheckOutcome: # --- positive forms --- - # exit code N / exit N (positive exact-match: passes when exit_code == N) - m = re.match(r"exit(?: code)?\s+(\d+)\s*$", expected) + # exit code N (positive exact-match: passes when exit_code == N). Left + # prefix-matched: `exit code 0 (verified 2026-09-02 to return exactly one + # EnvCall today)` is an established authoring idiom in live plans, and + # anchoring it here would turn eleven working rows across docs/plans/ into + # blocking UNEVALUATED for a change nobody asked for. + m = re.match(r"exit code (\d+)", expected) + if m: + return verdict(exit_code == int(m.group(1))) + + # exit N (anchored, see the note below) + m = re.match(r"exit\s+(\d+)\s*$", expected) if m: return verdict(exit_code == int(m.group(1))) @@ -554,19 +584,27 @@ def numeric_verdict(op) -> CheckOutcome: if m: return verdict(output.strip() == m.group(1).strip()) - # output >= N / >= N + # output >= N / >= N (anchored, see the note below) m = re.match(r"(?:output\s*)?>=\s*(\d+)\s*$", expected) if m: threshold = int(m.group(1)) return numeric_verdict(lambda value: value >= threshold) - # output > N / > N - m = re.match(r"(?:output\s*)?>\s*(\d+)\s*$", expected) + # output > N -- prefix-matched, preserving the long-standing reading of + # `output > 0 (a bare file-wide grep returns 3 today)`, which several live + # plans write. The bare `> N` form below is anchored instead. + m = re.match(r"output\s*>\s*(\d+)", expected) + if m: + threshold = int(m.group(1)) + return numeric_verdict(lambda value: value > threshold) + + # > N (anchored, see the note below) + m = re.match(r">\s*(\d+)\s*$", expected) if m: threshold = int(m.group(1)) return numeric_verdict(lambda value: value > threshold) - # output == N / == N + # output == N / == N (anchored, see the note below) m = re.match(r"(?:output\s*)?==\s*(\d+)\s*$", expected) if m: target = int(m.group(1)) diff --git a/tests/unit/test_verification_parser.py b/tests/unit/test_verification_parser.py index 60660fb40..36ef17552 100644 --- a/tests/unit/test_verification_parser.py +++ b/tests/unit/test_verification_parser.py @@ -986,3 +986,24 @@ def test_reads_and_writes_without_a_lane_are_inert(self): assert record_verification_outcomes(None, None, self._results()) is False assert read_verification_outcomes(None, None) is None assert read_verification_outcomes(self.REPO, 927381) is None + + +class TestExpectationAnchoringRule: + """Pre-existing forms keep prefix semantics; the forms added by #3065 are + anchored. A sweep of docs/plans/ found eleven live rows writing a trailing + gloss on `exit code N` / `output > N`; anchoring those would have turned + working rows into blocking UNEVALUATED for a change nobody asked for.""" + + def test_trailing_gloss_on_a_preexisting_form_still_grades(self): + assert ( + evaluate_expectation("exit code 0 (verified 2026-09-02)", exit_code=0, output="") + is PASS + ) + assert ( + evaluate_expectation("output > 0 (a bare grep returns 3)", exit_code=0, output="3") + is PASS + ) + + def test_trailing_gloss_on_a_new_bare_form_is_unevaluated(self): + for cell in ("== 2 (the two read sites)", ">= 1 nightly", "> 0 or so", "exit 0 maybe"): + assert evaluate_expectation(cell, exit_code=0, output="2") is UNEVALUATED From c0416aa31164853d29e6a06302af6680d5463edb Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 16:09:28 +0700 Subject: [PATCH 06/19] test: session-ensure readback identity and lock-release provenance (Refs #3065) Covers task 1 of docs/plans/sdlc-control-plane-asserted-facts.md (Cluster E): - duplicate-row readback stability: two real AgentSession rows sharing one session_id, six consecutive binds onto a specific row, all must succeed. Deterministically red on the pre-fix filter(session_id=...)[0] readback. - adopted candidates survive all three release sites (save failure, raising readback, readback mismatch) and both adopt shapes (live-lock reuse and supervised-signal self-recognition); the live lease stays owned. - minted candidates are still released at the save and readback sites, so provenance gating does not become never-release. - structural: the bind path calls query.get and never query.filter. Also stubs query.get on the multi-lineage integration readback mock, whose four concurrent-contention cases modelled the old session_id-index readback. --- tests/integration/test_sdlc_multi_lineage.py | 7 + ...sdlc_session_ensure_readback_provenance.py | 310 ++++++++++++++++++ .../test_sdlc_session_ensure_short_circuit.py | 3 +- 3 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py diff --git a/tests/integration/test_sdlc_multi_lineage.py b/tests/integration/test_sdlc_multi_lineage.py index 2015c5963..6f1c5a178 100644 --- a/tests/integration/test_sdlc_multi_lineage.py +++ b/tests/integration/test_sdlc_multi_lineage.py @@ -35,8 +35,15 @@ def _make_session(session_id: str) -> MagicMock: def _readback_as(session: MagicMock) -> MagicMock: + """Mock AgentSession that answers the bind path's post-save readback. + + The readback is a PRIMARY-KEY read (``query.get``), not a lookup on the + non-unique ``session_id`` index (#3065 Cluster E) -- ``query.filter`` is + still stubbed because other ensure_session paths use it. + """ mock_as = MagicMock() mock_as.query.filter.return_value = [session] + mock_as.query.get.return_value = session return mock_as diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py new file mode 100644 index 000000000..264c4ba7d --- /dev/null +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py @@ -0,0 +1,310 @@ +"""Post-save readback identity and lock-release provenance (#3065 Cluster E). + +Two defects, one test module, because they compose into the observed wedge: + +1. The post-save readback re-queried the non-unique ``session_id`` index and + took ``[0]``. ``AgentSession.session_id`` is a plain ``Field()``, not the + primary key, so a lane whose row was recreated after a crash has two rows + sharing one id and Popoto resolves that filter through an unordered Redis + ``SMEMBERS`` -- ``[0]`` is a coin flip. Spike-5 measured four + ``RUN_BIND_FAILED / post-save readback mismatch`` results in six + consecutive ensures on one duplicated lane. +2. The mismatch cleanup handed ``release_issue_lock`` a ``candidate`` that may + have been ADOPTED from the live lock rather than minted by this call. The + compare-and-delete is correct, which is precisely the problem: an adopted + candidate equals the live owner *by construction*, so the release always + "succeeds" -- destroying a lease this call never created. The missing + distinction is provenance, not identity. + +Every lock assertion here reads the raw ``session:issuelock:{n}`` key. That is +deliberate and permitted: the issue lock is a plain Redis key, not a +Popoto-managed model key, so the raw-Redis rule does not govern it (the same +reads already appear in ``test_sdlc_session_ensure_issue_lock.py``). +""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + + +def _lock_key(issue_number: int) -> str: + return f"session:issuelock:{issue_number}" + + +def _live_lock_owner(issue_number: int) -> str | None: + """The run_id currently holding the issue lock, or None if it is gone.""" + import popoto.redis_db as rdb + + raw = rdb.POPOTO_REDIS_DB.get(_lock_key(issue_number)) + if raw is None: + return None + return json.loads(raw).get("run_id") + + +@pytest.fixture +def _no_target_repo(): + """Keep the bind path off the network (``gh repo view``) and off signals.""" + with ( + patch("tools._sdlc_utils._resolve_target_repo", return_value=None), + patch("agent.supervised_run.write_supervised_run_signal"), + ): + yield + + +class TestReadbackByPrimaryKey: + """The readback resolves the row THIS call wrote, not an arbitrary row + sharing its ``session_id``.""" + + def test_duplicate_rows_do_not_destabilize_repeated_ensures(self, _no_target_repo): + """Demonstrated-red (spike-5): with two rows sharing one session_id, + repeated binds onto a specific row must succeed EVERY time. + + Under the old ``filter(session_id=...)[0]`` readback each iteration was + an independent coin flip between the row just written and its stale + twin, so this loop failed with probability ~1 - 2**-6. Real Redis, real + AgentSession rows -- a MagicMock would make the duplicate meaningless. + """ + from models.agent_session import AgentSession + from models.session_lifecycle import release_issue_lock + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306501 + session_id = f"sdlc-local-{issue_number}" + + target = AgentSession(session_id=session_id, project_key="test-3065", status="running") + target.save() + twin = AgentSession(session_id=session_id, project_key="test-3065", status="running") + twin.active_run_id = "stale-twin-run-id" + twin.save() + + # The duplicate the whole defect rests on: one session_id, two rows. + assert len(list(AgentSession.query.filter(session_id=session_id))) == 2 + assert target.agent_session_id != twin.agent_session_id + + run_ids = [] + for _ in range(6): + run_id, error = _acquire_run_lock_and_bind(issue_number, target) + assert error is None, f"bind failed on a duplicated-row lane: {error}" + assert run_id + run_ids.append(run_id) + release_issue_lock(issue_number, run_id) + + # Every iteration minted a distinct id and bound it to the SAME row. + assert len(set(run_ids)) == 6 + assert AgentSession.get_by_id(target.agent_session_id).active_run_id == run_ids[-1] + # The twin was never written through. + assert AgentSession.get_by_id(twin.agent_session_id).active_run_id == "stale-twin-run-id" + + def test_readback_never_queries_the_session_id_index(self, _no_target_repo): + """Structural: the bind path must not reach for ``filter(session_id=...)``. + + A behavioral-only assertion would pass on a lucky coin flip, so this + pins the mechanism: the readback is a primary-key read. + """ + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306502 + session = MagicMock() + session.session_id = f"sdlc-local-{issue_number}" + session.owned_run_ids = None + + mock_as = MagicMock() + + def _get(*args, **kwargs): + # Echo back whatever the call under test just bound. + fresh = MagicMock() + fresh.active_run_id = session.active_run_id + return fresh + + mock_as.query.get.side_effect = _get + + with patch("models.agent_session.AgentSession", mock_as): + run_id, error = _acquire_run_lock_and_bind(issue_number, session) + + assert error is None + assert run_id + mock_as.query.get.assert_called_once() + assert mock_as.query.filter.call_count == 0 + + +class TestReleaseGatedOnProvenance: + """``release_issue_lock`` is correct and unchanged; only a MINTED candidate + may ever be handed to it.""" + + @staticmethod + def _adopting_session(session_id: str, run_id: str) -> MagicMock: + """A session whose reuse claim is corroborated by its own history.""" + session = MagicMock() + session.session_id = session_id + session.owned_run_ids = json.dumps([run_id]) + return session + + def test_adopted_candidate_survives_readback_mismatch(self, _no_target_repo): + """The wedge itself: a reuse call that cannot confirm its bind must NOT + delete the live lease it merely adopted.""" + from models.session_lifecycle import touch_issue_lock + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306503 + session_id = f"sdlc-local-{issue_number}" + holder_run_id = "holder-run-id-306503" + + # A live lock this call did not create, owned by the id it will reuse. + assert touch_issue_lock(issue_number, holder_run_id, session_id=session_id).acquired + assert _live_lock_owner(issue_number) == holder_run_id + + session = self._adopting_session(session_id, holder_run_id) + + stale = MagicMock() + stale.active_run_id = "some-other-run-entirely" + mock_as = MagicMock() + mock_as.query.get.return_value = stale + + with patch("models.agent_session.AgentSession", mock_as): + run_id, error = _acquire_run_lock_and_bind( + issue_number, session, reuse_run_id=holder_run_id + ) + + assert run_id is None + assert error["error"] == "RUN_BIND_FAILED" + assert error["reason"] == "post-save readback mismatch" + # The lease is untouched: still live, still owned by the holder. + assert _live_lock_owner(issue_number) == holder_run_id + + def test_adopted_candidate_survives_a_raising_readback(self, _no_target_repo): + """Failure Path Test Strategy, ``:618``: the readback's own ``except`` + arm is a release site too, and an adopted candidate must survive it.""" + from models.session_lifecycle import touch_issue_lock + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306504 + session_id = f"sdlc-local-{issue_number}" + holder_run_id = "holder-run-id-306504" + + assert touch_issue_lock(issue_number, holder_run_id, session_id=session_id).acquired + + session = self._adopting_session(session_id, holder_run_id) + + mock_as = MagicMock() + mock_as.query.get.side_effect = RuntimeError("redis readback exploded") + + with patch("models.agent_session.AgentSession", mock_as): + run_id, error = _acquire_run_lock_and_bind( + issue_number, session, reuse_run_id=holder_run_id + ) + + assert run_id is None + assert error["error"] == "RUN_BIND_FAILED" + assert "post-save readback failed" in error["reason"] + assert _live_lock_owner(issue_number) == holder_run_id + + def test_adopted_candidate_survives_a_save_failure(self, _no_target_repo): + """Third release site (the ``session.save`` ``except`` arm).""" + from models.session_lifecycle import touch_issue_lock + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306505 + session_id = f"sdlc-local-{issue_number}" + holder_run_id = "holder-run-id-306505" + + assert touch_issue_lock(issue_number, holder_run_id, session_id=session_id).acquired + + session = self._adopting_session(session_id, holder_run_id) + session.save.side_effect = RuntimeError("redis save exploded") + + run_id, error = _acquire_run_lock_and_bind( + issue_number, session, reuse_run_id=holder_run_id + ) + + assert run_id is None + assert error["error"] == "RUN_BIND_FAILED" + assert _live_lock_owner(issue_number) == holder_run_id + + def test_supervised_adoption_also_survives_a_readback_mismatch(self, _no_target_repo): + """The second adopt shape (``ADOPTED_SUPERVISED``): a BARE ensure that + inherited the supervisor's run_id via self-recognition never minted it + either, so it may not release it. + + This is the shape ``reuse_run_id`` cannot distinguish -- it arrives + empty and is overwritten from the signal -- which is why provenance is + tracked separately from the value. + """ + from models.session_lifecycle import touch_issue_lock + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306506 + session_id = f"sdlc-local-{issue_number}" + supervisor_run_id = "supervisor-run-id-306506" + + assert touch_issue_lock(issue_number, supervisor_run_id, session_id=session_id).acquired + + session = self._adopting_session(session_id, supervisor_run_id) + + signal = MagicMock() + signal.live = True + signal.run_id = supervisor_run_id + signal.session_id = session_id + + stale = MagicMock() + stale.active_run_id = "some-other-run-entirely" + mock_as = MagicMock() + mock_as.query.get.return_value = stale + + with ( + patch("agent.supervised_run.supervised_run_status", return_value=signal), + patch("models.agent_session.AgentSession", mock_as), + ): + # BARE ensure -- no reuse_run_id. The supervised-self path supplies it. + run_id, error = _acquire_run_lock_and_bind(issue_number, session) + + assert run_id is None + assert error["error"] == "RUN_BIND_FAILED" + assert _live_lock_owner(issue_number) == supervisor_run_id + + def test_minted_candidate_is_still_released_on_save_failure(self, _no_target_repo): + """The other pole: provenance gating must not turn into "never release". + + A call that minted its own candidate and then failed to bind still + releases, so the next caller acquires immediately instead of waiting + out the 1800s TTL (cycle-2 CONCERN 2, unchanged by #3065). + """ + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306507 + session = MagicMock() + session.session_id = f"sdlc-local-{issue_number}" + session.owned_run_ids = None + session.save.side_effect = RuntimeError("redis save exploded") + + with patch("agent.supervised_run.supervised_run_status", return_value=None): + run_id, error = _acquire_run_lock_and_bind(issue_number, session) + + assert run_id is None + assert error["error"] == "RUN_BIND_FAILED" + assert _live_lock_owner(issue_number) is None + + def test_minted_candidate_is_still_released_on_readback_mismatch(self, _no_target_repo): + """Same pole at the readback site: an unverifiable bind of a + self-minted id frees the lock it just took.""" + from tools.sdlc_session_ensure import _acquire_run_lock_and_bind + + issue_number = 306508 + session = MagicMock() + session.session_id = f"sdlc-local-{issue_number}" + session.owned_run_ids = None + + stale = MagicMock() + stale.active_run_id = "some-other-run-entirely" + mock_as = MagicMock() + mock_as.query.get.return_value = stale + + with ( + patch("agent.supervised_run.supervised_run_status", return_value=None), + patch("models.agent_session.AgentSession", mock_as), + ): + run_id, error = _acquire_run_lock_and_bind(issue_number, session) + + assert run_id is None + assert error["reason"] == "post-save readback mismatch" + assert _live_lock_owner(issue_number) is None diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py index d60dd4e31..f087cf412 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_short_circuit.py @@ -281,7 +281,8 @@ def test_short_circuit_falls_through_for_terminal_status_eng_session(self, monke mock_as = MagicMock() mock_as.query.filter.side_effect = [[]] # existing_by_id lookup (none) - mock_as.query.get.return_value = mock_new_session # post-save readback (primary-key lookup) + # post-save readback (primary-key lookup) + mock_as.query.get.return_value = mock_new_session mock_as.create_local.return_value = mock_new_session with ( From 9c44266b3f0a771309e7a25508dcc6a67db852ce Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 16:29:29 +0700 Subject: [PATCH 07/19] [WIP] Merge predicate reads recorded verification outcomes (Refs #3065) Task 8: group (e) reads the _verification_outcomes aggregate, checks it fresh against the PR head fail-closed, and refuses on any FAIL/UNEVALUATED row by name. Reconciles a concurrent duplicate implementation of the same task in this worktree. --- agent/verification_parser.py | 9 - tests/unit/test_merge_predicate.py | 268 +++++++++++++++++++++++++++++ tools/merge_predicate.py | 227 +++++++++++++++++++++++- 3 files changed, 494 insertions(+), 10 deletions(-) diff --git a/agent/verification_parser.py b/agent/verification_parser.py index de43eb3b2..22091614f 100644 --- a/agent/verification_parser.py +++ b/agent/verification_parser.py @@ -117,15 +117,6 @@ # key in an already-flexible JSON blob: no schema field, no migration. VERIFICATION_OUTCOMES_KEY = "_verification_outcomes" -# Named refusal reason for a `_verification_outcomes` aggregate whose stamped -# `head_sha` does not match the PR's current head (task 8, #3065). Defined -# here -- where the aggregate is written and stamped -- rather than in -# `tools/merge_predicate.py`, which reads it: a Verification row greps for -# the constant *name*, matching how `reconcile_dispatch`, `decision_inputs`, -# `resolve_branch_truth`, and `G3_REDIRECT_REASON_DOCS_PENDING` are already -# pinned. Never read the prose; grep the symbol. -VERIFICATION_OUTCOMES_STALE_REASON = "verification outcome predates PR head commit" - class CheckOutcome(StrEnum): """The three things a verification check can say. diff --git a/tests/unit/test_merge_predicate.py b/tests/unit/test_merge_predicate.py index 24478c4d3..7ea090045 100644 --- a/tests/unit/test_merge_predicate.py +++ b/tests/unit/test_merge_predicate.py @@ -57,6 +57,9 @@ SIMPLE_PR = 990001 OTHER_REPO_ISSUE = 990999 NO_LEDGER_PR = 990999999 +# Group (e), the #3080 gate-row shape. +GATE_PR = 990077 +GATE_ISSUE = 990078 _SYNTHETIC_LEDGER_KEYS: list[tuple[str, int]] = [ (TARGET_REPO, UMBRELLA_ISSUE), @@ -66,6 +69,7 @@ (TARGET_REPO, SINGLE_ISSUE), (OTHER_REPO, UMBRELLA_ISSUE), (OTHER_REPO, OTHER_REPO_ISSUE), + (TARGET_REPO, GATE_ISSUE), ] @@ -679,3 +683,267 @@ def test_docs_stage_slash_bearing_head_ref_emits_honest_no_slug_refusal(monkeypa assert "no usable slug for the docs/features fallback" in failed[0] assert "docs/features/fix/router-blocked-on-conflict.md absent" not in failed[0] assert notes == [] + + +# --------------------------------------------------------------------------- +# Group (e): recorded plan-verification outcomes (#3065, task 8) +# +# The motivating incident is #3080 / commit ba092a06d. A plan carried a hard +# shipping gate in prose -- "FAIL and UNRESOLVED both hold the PR at REVIEW" -- +# and the PR merged straight past it, because nothing machine-readable stood +# between an APPROVED verdict and the merge. These tests reconstruct that exact +# state (APPROVED verdict, DOCS complete, CI green, one UNEVALUATED gate row) +# and require a refusal that NAMES the row. +# +# Every fixture below writes a REAL PipelineLedger aggregate through the +# production writer (``record_verification_outcomes``) and lets the predicate +# read it back through the production reader. Nothing here re-executes a +# plan-authored command: recorded state is the source of truth (PM ruling, +# 2026-09-03), and a predicate that shelled out to a test suite inside a merge +# gate would be a different -- and rejected -- design. +# --------------------------------------------------------------------------- + +HEAD_SHA = "a" * 40 +ADVANCED_SHA = "b" * 40 + + +def _plan_repo(tmp_path: Path, *, with_plan: bool = True) -> Path: + """A repo root whose plan doc is discoverable ONLY by `tracking:` frontmatter. + + The filename deliberately does not match the lane slug used in the PR head + ref: lane slug and plan filename are allowed to differ, so a resolver that + matched on filename would find nothing here and the enforcement tests would + silently pass for the wrong reason. + """ + plans = tmp_path / "docs" / "plans" + plans.mkdir(parents=True) + if with_plan: + (plans / "a-name-that-is-not-the-lane-slug.md").write_text( + f"---\ntracking: https://github.com/{TARGET_REPO}/issues/{GATE_ISSUE}\n---\n\n# Plan\n", + encoding="utf-8", + ) + return tmp_path + + +def _row(name: str, outcome, reason: str = ""): + """One graded CheckResult, built through the production dataclasses.""" + from agent.verification_parser import CheckResult, VerificationCheck + + return CheckResult( + check=VerificationCheck(name=name, command="grep -c foo bar.py", expected="output > 0"), + outcome=outcome, + exit_code=0, + output="", + reason=reason, + ) + + +@pytest.fixture +def gate_lane(monkeypatch, ledger_factory): + """Wire the #3080 state: APPROVED verdict, DOCS complete, CI green. + + Returns a callable that records a real ``_verification_outcomes`` aggregate + for the given graded rows and returns the ``PredicateResult``. + """ + + def _run(results, *, repo_root, recorded_sha=HEAD_SHA, pr_head=HEAD_SHA, strip_head=False): + from agent.verification_parser import record_verification_outcomes + + ledger_factory(TARGET_REPO, GATE_ISSUE, pr_number=GATE_PR) + + monkeypatch.setattr(mp, "_substrate_present", lambda root: True) + monkeypatch.setattr(mp, "_gh_repo_name_with_owner", lambda root: TARGET_REPO) + monkeypatch.setattr( + mp, + "_gh_pr_view", + lambda pr, root: { + "state": "OPEN", + "mergeable": "MERGEABLE", + "mergeStateStatus": "CLEAN", + "statusCheckRollup": [{"name": "ci", "conclusion": "SUCCESS"}], + "body": f"Closes #{GATE_ISSUE}", + "headRefName": "session/gate-lane", + }, + ) + monkeypatch.setattr( + mp, "_run_stage_query", lambda issue, root: {"stages": {"DOCS": "completed"}} + ) + monkeypatch.setattr( + mp, + "_run_verdict_get", + lambda issue, root: {"verdict": "APPROVED", "head_sha": pr_head}, + ) + monkeypatch.setattr(mp, "_gh_latest_commit", lambda pr, root: {"sha": pr_head, "date": ""}) + + # Both the writer's stamp and the predicate's current-head read go + # through the sanctioned git-first resolver, so the two poles of the + # freshness test are the same seam observed at two moments: the + # aggregate is stamped, and only THEN does the PR head advance. + import tools.pr_head_resolver as phr + + resolver_sha = {"sha": recorded_sha} + monkeypatch.setattr(phr, "resolve_pr_head_sha", lambda pr, **kw: resolver_sha["sha"]) + assert record_verification_outcomes(TARGET_REPO, GATE_ISSUE, results, pr_number=GATE_PR), ( + "the graded aggregate must actually persist for this test to mean anything" + ) + + if strip_head: + _corrupt_head_sha(strip_head) + + # The PR head as the predicate will now see it. + resolver_sha["sha"] = pr_head + return mp.evaluate_merge_predicate(GATE_PR, repo_root=repo_root) + + return _run + + +def _corrupt_head_sha(mode: str) -> None: + """Rewrite the persisted aggregate's ``head_sha`` to an absent/unparseable value.""" + import json as _json + + from agent.verification_parser import VERIFICATION_OUTCOMES_KEY + from tools.stage_states_helpers import update_stage_states + + ledger = PipelineLedger.get(TARGET_REPO, GATE_ISSUE) + + def _mutate(states: dict) -> dict: + aggregate = states[VERIFICATION_OUTCOMES_KEY] + if mode == "absent": + aggregate.pop("head_sha", None) + else: + aggregate["head_sha"] = "not-a-sha" + states[VERIFICATION_OUTCOMES_KEY] = aggregate + return states + + assert update_stage_states(ledger, _mutate, field="stage_states_json") + assert _json.loads(PipelineLedger.get(TARGET_REPO, GATE_ISSUE).stage_states_json) + + +def test_unevaluated_gate_row_refuses_and_names_it(gate_lane, tmp_path): + """The #3080 shape exactly: APPROVED, DOCS complete, CI green, one + UNEVALUATED gate row. On main today this state merges.""" + from agent.verification_parser import CheckOutcome + + result = gate_lane( + [ + _row("Tests pass", CheckOutcome.PASS), + _row( + "GATE: poll obligation recorded", + CheckOutcome.UNEVALUATED, + reason="expectation 'recorded' is not machine-readable", + ), + ], + repo_root=_plan_repo(tmp_path), + ) + + assert not result.allowed + named = [f for f in result.failed_checks if "GATE: poll obligation recorded" in f] + assert len(named) == 1, result.failed_checks + assert "UNEVALUATED" in named[0] + # The refusal carries the grader's own reason, not a bare false. + assert "not machine-readable" in named[0] + + +def test_failed_gate_row_refuses_and_names_it(gate_lane, tmp_path): + from agent.verification_parser import CheckOutcome + + result = gate_lane( + [_row("GATE: poll obligation recorded", CheckOutcome.FAIL)], + repo_root=_plan_repo(tmp_path), + ) + + assert not result.allowed + named = [f for f in result.failed_checks if "GATE: poll obligation recorded" in f] + assert len(named) == 1, result.failed_checks + assert "FAIL" in named[0] + + +def test_all_pass_fresh_aggregate_merges(gate_lane, tmp_path): + """A clean lane still merges -- group (e) is a gate, not a blanket refusal.""" + from agent.verification_parser import CheckOutcome + + result = gate_lane( + [_row("Tests pass", CheckOutcome.PASS), _row("Ruff clean", CheckOutcome.PASS)], + repo_root=_plan_repo(tmp_path), + ) + + assert result.allowed, result.failed_checks + assert any("all PASS" in n for n in result.notes) + + +def test_stale_aggregate_refuses_with_named_reason(gate_lane, tmp_path): + """Two-pole freshness: the SAME all-PASS aggregate, PR head advanced by one + commit. The cached PASS must NOT be read (#2404-shaped fail-open hole).""" + from agent.verification_parser import CheckOutcome + + result = gate_lane( + [_row("Tests pass", CheckOutcome.PASS)], + repo_root=_plan_repo(tmp_path), + recorded_sha=HEAD_SHA, + pr_head=ADVANCED_SHA, + ) + + assert not result.allowed + stale = [f for f in result.failed_checks if mp.VERIFICATION_OUTCOMES_STALE_REASON in f] + assert len(stale) == 1, result.failed_checks + assert "all PASS" not in " ".join(result.notes) + + +@pytest.mark.parametrize("mode", ["absent", "unparseable"]) +def test_unanchored_aggregate_refuses(gate_lane, tmp_path, mode): + """Missing or unparseable head_sha refuses. Deliberately stricter than the + REVIEW-verdict path's recorded_at fallback: that fallback exists for records + predating #2769, and there are no legacy aggregates to be compatible with.""" + from agent.verification_parser import CheckOutcome + + result = gate_lane( + [_row("Tests pass", CheckOutcome.PASS)], + repo_root=_plan_repo(tmp_path), + strip_head=mode, + ) + + assert not result.allowed + assert any("no usable head_sha" in f for f in result.failed_checks), result.failed_checks + + +def test_plan_less_lane_is_unaffected(gate_lane, tmp_path): + """No plan document tracks this issue -> REPORTED, never enforced. This must + stay a distinguishable branch from "present aggregate, not fresh".""" + from agent.verification_parser import CheckOutcome + + result = gate_lane( + [_row("GATE: poll obligation recorded", CheckOutcome.UNEVALUATED)], + repo_root=_plan_repo(tmp_path, with_plan=False), + ) + + assert result.allowed, result.failed_checks + assert any("no plan document" in n for n in result.notes) + + +def test_build_vs_ship_split_lives_on_the_consumer(gate_lane, tmp_path): + """The identical row and outcome that the merge predicate refuses does NOT + block the build-side write, and carries no severity/gate annotation of its + own. The split is a property of the consumer, not of the row -- a per-row + severity marker would be the first step back toward the gate DSL this plan + rejected.""" + from agent.verification_parser import ( + VERIFICATION_OUTCOMES_KEY, + CheckOutcome, + read_verification_outcomes, + ) + + result = gate_lane( + [_row("GATE: poll obligation recorded", CheckOutcome.UNEVALUATED)], + repo_root=_plan_repo(tmp_path), + ) + assert not result.allowed # ship side refuses + + # Build side: the grading run persisted, the lane's record is intact, and + # the row itself says only PASS/FAIL/UNEVALUATED -- no severity, no gate + # marker, nothing a consumer could read as "blocking for shipping only". + record = read_verification_outcomes(TARGET_REPO, GATE_ISSUE) + assert record is not None + assert VERIFICATION_OUTCOMES_KEY == "_verification_outcomes" + (row,) = record["rows"] + assert row["outcome"] == CheckOutcome.UNEVALUATED.value + assert set(row) == {"name", "outcome", "reason"} diff --git a/tools/merge_predicate.py b/tools/merge_predicate.py index 7505eaa38..59e9ff739 100644 --- a/tools/merge_predicate.py +++ b/tools/merge_predicate.py @@ -5,7 +5,7 @@ skill (via ``docs/sdlc/do-merge.md``). Consuming a single helper is what keeps the hook and the skill from drifting apart (#1944 class). -Four check groups: +Five check groups: - **Group (a) — PR state** (always enforced, fail-closed on any ``gh`` error): state OPEN, mergeable MERGEABLE, mergeStateStatus CLEAN (or UNSTABLE with a @@ -33,6 +33,19 @@ second layer keeps working; the ``/do-merge`` skill passes ``--run-id`` for the primary enforcement. Fails open on Redis errors (lease confirmed), closed on a substrate-present import failure. +- **Group (e) — verification outcomes** (substrate-present, plan-tracked + only): the #3080 / ``ba092a06d`` owner ruling -- "FAIL and UNEVALUATED both + hold the PR at REVIEW" -- lived only in plan prose, so PR #3080 merged past + it. This reads the graded aggregate the verification runner persists to the + lane's ``PipelineLedger`` (``_verification_outcomes``, #3065 Cluster B/C) + and refuses on a ``FAIL`` or ``UNEVALUATED`` row, naming it. Source of truth + is RECORDED state, never live re-execution (PM ruling, 2026-09-03): this + never shells out to a plan-authored command. A lane with no plan document, + or a plan document with no recorded aggregate, is reported and NOT + enforced -- there is no ruling to make machine-readable. A *present* + aggregate is checked for freshness against the PR's current head before + being trusted, fail-closed on a mismatch or an unresolvable anchor; see + ``_check_verification_outcomes``. Tracked-issue resolution for groups (b)/(c) (#2034, corrected mechanism): the two SDLC-substrate checks key on the **SDLC-tracked issue looked up from the @@ -78,6 +91,7 @@ import argparse import json import logging +import os import re import shutil import subprocess @@ -103,6 +117,13 @@ # Head refs that can never yield a usable slug for the docs/features fallback. _NO_SLUG_REFS = frozenset({"main", "master", "HEAD", ""}) +# Named refusal reason for a recorded `_verification_outcomes` aggregate whose +# stamped `head_sha` does not match the PR's current head (#3065, task 8). +# Defined HERE, where the predicate consumes it, rather than beside the writer: +# this is the ship-side rule, and a Verification row greps this file for the +# symbol. Never read the prose; grep the name. +VERIFICATION_OUTCOMES_STALE_REASON = "verification outcome predates PR head commit" + @dataclass class PredicateResult: @@ -681,6 +702,206 @@ def _check_lease_ownership( ) +def _find_plan_doc(issue_number: int, repo_root: Path) -> Path | None: + """Locate the plan document that *tracks* this issue, or ``None``. + + Delegates to ``tools.lane_identity.find_plan_path``, whose single + resolution rung is a ``tracking:`` frontmatter line naming the issue. A + filename match is deliberately NOT a rung: the lane slug and the plan + filename are allowed to differ, and a bare ``#N`` mention in a "Not + building" No-Gos line means the opposite of ownership (#2735). + + ``find_plan_path`` scopes its search by ``SDLC_TARGET_REPO``, else the + cwd's git toplevel. The merge predicate is handed an explicit + ``repo_root`` that need not be the cwd — the merge-guard hook and + ``/do-merge`` both pass one — so the override is set for the duration of + the call and restored afterwards. Duplicating the resolver's regex here + instead would be the replicated-value defect this lane exists to remove. + """ + from tools.lane_identity import find_plan_path + + prior = os.environ.get("SDLC_TARGET_REPO") + os.environ["SDLC_TARGET_REPO"] = str(repo_root) + try: + return find_plan_path(issue_number) + finally: + if prior is None: + os.environ.pop("SDLC_TARGET_REPO", None) + else: + os.environ["SDLC_TARGET_REPO"] = prior + + +def _check_verification_outcomes( + issue_number: int, + pr_number: int, + repo_root: Path, + failed: list[str], + notes: list[str], +) -> None: + """Group (e): the plan's graded verification outcomes (#3080, Cluster C). + + Makes the #3080 / ``ba092a06d`` owner ruling -- "FAIL and UNEVALUATED both + hold the PR at REVIEW" -- machine-readable. That ruling lived only in plan + prose, so PR #3080 merged past it; this reads the aggregate + ``agent.verification_parser.record_verification_outcomes`` persists to the + lane's ``PipelineLedger`` instead of re-deriving anything. + + Source of truth is RECORDED state, never live re-execution (PM ruling, + 2026-09-03, #3065): this never shells out to a plan-authored command -- + re-running a verification suite inside a merge gate is a non-starter, and + every other check in this module already reads recorded state via ``gh`` + / ``sdlc-tool`` rather than executing anything. + + Three outcomes are deliberately kept distinguishable, none collapsing into + another: + + - **No plan document** tracks this issue + (:func:`tools.lane_identity.find_plan_path`) -> reported, not enforced. + This plan has no evidence a plan-less lane should be blocked by a check + that exists only because a plan declared a ruling. + - **A plan document exists but no aggregate was ever recorded** -> also + reported, not enforced. There is no ruling to enforce when nothing was + graded; a new fail-closed behavior here has no incident backing it. + - **A recorded aggregate exists** -> it is graded, but ONLY after its + freshness against the PR's CURRENT head is established, fail-closed on + all three dispositions (task 8, round-2 concern): + * match -> the aggregate is fresh; grade its outcome. + * mismatch -> stale, treated as equivalent to ``UNEVALUATED``, refused + with ``VERIFICATION_OUTCOMES_STALE_REASON``. The cached PASS is + never read. + * missing/unparseable ``head_sha``, or a PR head that cannot be + resolved -> refuse. Deliberately stricter than group (c)'s + ``recorded_at`` fallback: there are no legacy + ``_verification_outcomes`` records to be compatible with, so a + weaker comparison would be a #2404-shaped fail-open hole added on + purpose. + + A ``FAIL`` or ``UNEVALUATED`` row holds the PR; the refusal names the row + rather than returning a bare failure. The build-vs-ship split lives + entirely on this consumer -- nothing marks a row with severity, and the + build gate (a different consumer) is free to treat ``UNEVALUATED`` as a + pause that still allows build progression. + """ + try: + plan_path = _find_plan_doc(issue_number, repo_root) + except Exception as exc: + notes.append(f"verification-outcomes check skipped: plan-doc resolution failed ({exc})") + return + + if plan_path is None: + notes.append( + "verification-outcomes check skipped: no plan document tracks issue" + f" #{issue_number} (reported, not enforced)" + ) + return + + try: + target_repo = _gh_repo_name_with_owner(repo_root) + except Exception as exc: + failed.append(f"verification outcomes: target repo unresolvable ({exc})") + return + + # Lazy import: keeps this module's stdlib-only load posture for the + # merge-guard hook (see module docstring). agent.verification_parser pulls + # in agent.pipeline_ledger, same posture as the _check_verdict_freshness + # trailer reader below. + from agent.verification_parser import read_verification_outcomes + + aggregate = read_verification_outcomes(target_repo, issue_number) + if aggregate is None: + notes.append( + "verification-outcomes check skipped: no recorded aggregate for" + f" issue #{issue_number} (reported, not enforced)" + ) + return + + from tools._sdlc_utils import head_sha_of_record + from tools.pr_head_resolver import resolve_pr_head_sha + + recorded_sha = head_sha_of_record(aggregate) if isinstance(aggregate, dict) else "" + + try: + current_head = resolve_pr_head_sha(pr_number, repo=target_repo, repo_root=str(repo_root)) + except Exception as exc: + failed.append(f"verification outcomes: PR head unresolvable ({exc})") + return + + if not current_head: + failed.append("verification outcomes: PR head unresolvable") + return + + if not recorded_sha: + # Kept distinct from the stale case on purpose: "graded against an + # older commit" and "we cannot tell what it was graded against" are + # different facts, and collapsing them would hide which one occurred. + # Both refuse; neither reads the cached outcome. + failed.append( + "verification outcomes: no usable head_sha on the recorded aggregate;" + " freshness is indeterminate and the aggregate cannot be trusted" + ) + return + + if recorded_sha.lower() != current_head.lower(): + failed.append( + f"verification outcomes: {VERIFICATION_OUTCOMES_STALE_REASON}" + f" (graded against {recorded_sha[:12]}, PR head is {current_head[:12]})" + ) + return + + # The tri-state tokens come from the writer's own enum, never re-spelled + # here: a literal "UNEVALUATED" in this module would be a replicated value + # that silently stops matching if the enum is ever renamed. + from agent.verification_parser import CheckOutcome + + rows = aggregate.get("rows") + if not isinstance(rows, list): + failed.append("verification outcomes: recorded aggregate has no readable rows") + return + + blocking = {CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value} + offending = 0 + for row in rows: + if not isinstance(row, dict): + offending += 1 + failed.append("verification outcomes: a recorded row is malformed") + continue + row_outcome = str(row.get("outcome") or "") + if row_outcome not in blocking: + continue + offending += 1 + reason = str(row.get("reason") or "").strip() + failed.append( + f"verification row {row.get('name') or ''!r} is {row_outcome}" + + (f": {reason}" if reason else "") + + " — FAIL and UNEVALUATED both hold the PR (owner ruling on #3080)" + ) + + malformed = aggregate.get("malformed") or 0 + if isinstance(malformed, int) and malformed > 0: + offending += 1 + failed.append( + f"verification outcomes: {malformed} malformed row(s) were never executed;" + " an unrunnable check is not a passing check" + ) + + outcome = str(aggregate.get("outcome") or "") + if offending: + return + if outcome != CheckOutcome.PASS.value: + # No blocking row and no malformed row, yet the aggregate is not PASS: + # a run with no checks at all, which grades UNEVALUATED rather than a + # vacuous PASS. Refuse rather than guess what it meant. + failed.append( + f"verification outcomes: recorded outcome is {outcome or ''!r}," + f" not {CheckOutcome.PASS.value} ({len(rows)} row(s) recorded)" + ) + return + notes.append( + f"verification outcomes fresh: {len(rows)} row(s), all {CheckOutcome.PASS.value}" + f" for issue #{issue_number}" + ) + + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @@ -760,6 +981,10 @@ def evaluate_merge_predicate( # Keyed on the same SDLC-tracked issue as groups (b)/(c) — the # lease is per-issue. _check_lease_ownership(effective_issue, run_id, failed, notes) + # Group (e): verification outcomes (#3080, #3065 Cluster C). + # Keyed on the same SDLC-tracked issue — the plan and its + # graded aggregate live on the tracking issue, not a sub-issue. + _check_verification_outcomes(effective_issue, pr_number, root, failed, notes) return PredicateResult( allowed=not failed, From 8cd3aba07bba96664df771c26317ced5dd8181b1 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 16:35:41 +0700 Subject: [PATCH 08/19] Merge predicate refuses on a FAIL/UNEVALUATED verification row (Refs #3065) Task 8. Group (e) reads the recorded `_verification_outcomes` aggregate -- never re-executing a plan-authored command (PM ruling, 2026-09-03) -- and: - refuses on any FAIL or UNEVALUATED row, naming the row and its reason. This is the #3080 / ba092a06d owner ruling made machine-readable; it lived only in plan prose and PR #3080 merged straight past it. - checks the aggregate's freshness against the PR's current head first, via head_sha_of_record vs resolve_pr_head_sha, fail-closed on all three dispositions: match grades, mismatch refuses with VERIFICATION_OUTCOMES_STALE_REASON without reading the cached PASS, and a missing/unparseable head_sha or unresolvable PR head refuses. Deliberately stricter than group (c)'s recorded_at fallback, which exists only for records predating #2769. - keeps three branches distinguishable: no plan document and no recorded aggregate are REPORTED; a present aggregate that cannot be shown fresh is ENFORCED. The plan document is resolved through `tracking:` frontmatter (lane_identity.find_plan_path, scoped to the predicate's explicit repo_root), never by filename -- the tests' plan file is deliberately named something other than the lane slug. VERIFICATION_OUTCOMES_STALE_REASON is defined in tools/merge_predicate.py, where the predicate consumes it; the unread copy in agent/verification_parser.py is removed so there is one definition, not two. --- tests/unit/test_merge_predicate.py | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/unit/test_merge_predicate.py b/tests/unit/test_merge_predicate.py index 7ea090045..f6742f9ed 100644 --- a/tests/unit/test_merge_predicate.py +++ b/tests/unit/test_merge_predicate.py @@ -947,3 +947,46 @@ def test_build_vs_ship_split_lives_on_the_consumer(gate_lane, tmp_path): (row,) = record["rows"] assert row["outcome"] == CheckOutcome.UNEVALUATED.value assert set(row) == {"name", "outcome", "reason"} + + +def test_plan_present_but_no_recorded_aggregate_is_reported_not_enforced( + monkeypatch, ledger_factory, tmp_path +): + """A plan exists, nothing was ever graded onto the ledger. Reported. + + This is the third branch, and it must stay distinguishable from the two + around it: "no plan" and "aggregate present but not provably fresh". Only + the last of those three is enforced -- blocking a lane because nothing was + graded would be a new fail-closed behavior with no incident behind it. + """ + ledger_factory(TARGET_REPO, GATE_ISSUE, pr_number=GATE_PR) + monkeypatch.setattr(mp, "_gh_repo_name_with_owner", lambda root: TARGET_REPO) + + failed: list[str] = [] + notes: list[str] = [] + mp._check_verification_outcomes(GATE_ISSUE, GATE_PR, _plan_repo(tmp_path), failed, notes) + + assert failed == [] + assert any("no recorded aggregate" in n for n in notes), notes + + +def test_unresolvable_pr_head_refuses(monkeypatch, ledger_factory, tmp_path): + """A present aggregate whose freshness cannot be established at all -- + the PR head does not resolve -- refuses. Fail-closed, like every other + indeterminate branch in this group.""" + import tools.pr_head_resolver as phr + from agent.verification_parser import CheckOutcome, record_verification_outcomes + + ledger_factory(TARGET_REPO, GATE_ISSUE, pr_number=GATE_PR) + monkeypatch.setattr(mp, "_gh_repo_name_with_owner", lambda root: TARGET_REPO) + monkeypatch.setattr(phr, "resolve_pr_head_sha", lambda pr, **kw: HEAD_SHA) + assert record_verification_outcomes( + TARGET_REPO, GATE_ISSUE, [_row("Tests pass", CheckOutcome.PASS)], pr_number=GATE_PR + ) + + monkeypatch.setattr(phr, "resolve_pr_head_sha", lambda pr, **kw: "") + failed: list[str] = [] + notes: list[str] = [] + mp._check_verification_outcomes(GATE_ISSUE, GATE_PR, _plan_repo(tmp_path), failed, notes) + + assert any("PR head unresolvable" in f for f in failed), failed From 1ed7c238daf01375305c39303eead6b53a288e01 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 16:49:15 +0700 Subject: [PATCH 09/19] =?UTF-8?q?fix(sdlc-router):=20route=20on=20read=20f?= =?UTF-8?q?acts=20=E2=80=94=20decision=20evidence,=20G3's=20docs=20arm,=20?= =?UTF-8?q?guard=20reconciliation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tasks 2, 3 and 4 of docs/plans/sdlc-control-plane-asserted-facts.md. Task 2 — decisions carry their evidence (Cluster D). Blocked(NO_RULE) now carries `decision_inputs`: the stage_states and meta it decided on, plus an `unrecorded_dispatch` signal, surfaced through sdlc-tool next-skill's JSON. The batch reported a NO_RULE on a state row 5 has owned since c1e991972 and the report could be neither confirmed nor refuted, because the payload carried nothing. detect_unrecorded_dispatch names all three "no confirming record" shapes — no record at all, a record naming another skill, and a router slot never confirmed by a stage entry — so a skipped `dispatch record` surfaces now instead of four turns later as a G4 oscillation block blamed on the wrong cause. next-skill still persists nothing; this is a read. Task 3 — G3's redirect ladder is complete. Adds the /do-docs arm (G3_REDIRECT_REASON_DOCS_PENDING) for REVIEW complete + APPROVED + DOCS pending, which previously fell to the else and was sent back to a /do-pr-review it had already passed. Arm 1 now requires a recorded APPROVED verdict rather than the REVIEW marker alone, matching the #1932 gap-(c) gate rows 9 and 10 already apply — a completed REVIEW marker with no verdict is an unearned marker, not evidence of approval, and must not fast-path to /do-merge. Task 4 — guards are reconciled against the selected dispatch (the keystone). Guards ran at :2246-2248, the table at :2250-2255, and nothing re-validated in between, so G3 constrained a suggestion but never a decision. reconcile_dispatch re-runs the guard list with the table's selection as the proposed skill. Bounded by construction: exactly one pass on the selection, at most one on the resulting redirect, then a Blocked(RECONCILE_DEADLOCK) carrying both the selected row and the vetoing guard. Never a third pass. A guard naming the skill already chosen is agreement, not a veto, and does not trigger a block. Row 2b's predicate is NOT edited — #1639 made it marker-agnostic deliberately; it is constrained from outside. On the #2771/#2334 shipped-lane shape the router now answers /do-docs where main answers /do-plan-critique. The guards are not pure and reconciliation runs them twice. This is a stated invariant, written into reconcile_dispatch's docstring: guard_g5_artifact_hash_cache mutates record["artifact_hash"] in place and logs a WARNING on legacy-hash migration, and double invocation is idempotent ONLY because stage_states is passed by reference. A defensive copy would re-run the migration. Tests assert the WARNING fires exactly once per decide call and that the guards receive the caller's own objects by identity. A raising guard during reconciliation is deliberately not caught, preserving the existing asymmetry (rule predicates are try/except-wrapped at :2260-2263, guards at 1083-1086 are not). Swallowing a bug into a NO_RULE would misreport it as a routing hole. Evidence fields are compare=False on both Dispatch and Blocked: evidence is not identity, and without it attaching evidence would silently redefine equality for every caller comparing against an expected Dispatch(...). That keeps tests/unit/test_sdlc_router.py purely additive — every pre-existing assertion, including the bfa4a6f7d / d9cf29dd6 / 3c689f211 regression floor, is unchanged. Demonstrated red against main (#2658), all four executed: - Blocked has no decision_inputs field at all - shipped lane routes /do-plan-critique instead of /do-docs - docs-pending falls to the else, producing a redundant /do-pr-review - REVIEW+DOCS completed with no verdict fast-paths to /do-merge Verified: 561 passed, 0 failed across tests/unit/test_sdlc_router.py, test_sdlc_verdict.py, test_sdlc_router_oscillation.py, test_sdlc_router_reconciliation.py, test_sdlc_next_skill.py, sdlc_router_decision/ and test_sdlc_dispatch.py. ruff check and ruff format clean on both touched source files. Refs #3065 --- agent/sdlc_router.py | 331 ++++++++++++- tests/unit/test_sdlc_next_skill.py | 104 ++++ tests/unit/test_sdlc_router.py | 384 +++++++++++++++ tests/unit/test_sdlc_router_oscillation.py | 34 +- tests/unit/test_sdlc_router_reconciliation.py | 460 ++++++++++++++++++ tools/sdlc_next_skill.py | 27 +- 6 files changed, 1327 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_sdlc_router_reconciliation.py diff --git a/agent/sdlc_router.py b/agent/sdlc_router.py index 9159c5499..e5daf2720 100644 --- a/agent/sdlc_router.py +++ b/agent/sdlc_router.py @@ -35,7 +35,7 @@ import os import re from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field, replace from datetime import UTC, datetime from typing import Any @@ -162,6 +162,14 @@ def _terminal_guard_enabled() -> bool: SKILL_DO_DOCS = "/do-docs" SKILL_DO_MERGE = "/do-merge" +# G3's docs-pending redirect reason (#3065 task 3), pinned as a named constant +# rather than inlined prose. A grep for ``SKILL_DO_DOCS`` would already pass +# today (row 9 uses it), making a Verification row incapable of failing; a +# grep for the literal sentence would break on an innocent copy-edit that +# leaves the logic unchanged. The constant name is what a Verification row +# greps for. +G3_REDIRECT_REASON_DOCS_PENDING = "review clean, docs pending" + # --------------------------------------------------------------------------- # Decision types @@ -175,6 +183,17 @@ class Dispatch: skill: str reason: str row_id: str | None = None + # #3065 Cluster D: the "your previous decision was never confirmed by a + # record" signal, or ``None`` when the last dispatch is accounted for. + # Purely diagnostic — it never changes which skill is dispatched. Shape is + # :func:`detect_unrecorded_dispatch`'s return value. + # + # ``compare=False`` because evidence is not identity: two dispatches of the + # same skill for the same reason ARE the same decision whether or not the + # previous one was recorded. Without it, attaching evidence would silently + # redefine equality for every existing caller that compares a decision + # against an expected ``Dispatch(...)``. + unrecorded_dispatch: dict | None = field(default=None, compare=False) @dataclass(frozen=True) @@ -211,10 +230,25 @@ class Blocked: the dispatch-table fallthrough — a genuine hole in the table rather than a guard verdict. Callers matching on specific guards must compare against the codes they care about, never merely test for presence. + + ``decision_inputs`` (#3065 Cluster D) is the evidence the decision was + made from — the ``stage_states`` and ``meta`` the router actually read, + plus whatever else the specific blocked path wants to surface (e.g. an + ``unrecorded_dispatch`` flag on the ``NO_RULE`` fallthrough, or the + vetoing guard's verdict on a reconciliation double-veto). It is optional + because most ``Blocked`` instances — the numbered guards — already state + their own reason in prose; this field exists for the paths where a human + reading the ``Blocked`` alone could not otherwise reconstruct what the + router saw. A control plane that cannot show its own inputs cannot be + checked against a field report (see the #3065 NO_RULE-on-row-5 incident). """ reason: str guard_id: str | None = None + # ``compare=False`` for the same reason as ``Dispatch.unrecorded_dispatch``: + # a refusal's identity is its reason and guard_id, not the state dump + # attached for a human to read. + decision_inputs: dict[str, Any] | None = field(default=None, compare=False) # Sentinel ``guard_id`` for the dispatch-table fallthrough (#2767b). Short-code @@ -222,6 +256,13 @@ class Blocked: # match it without parsing the reason string. NO_RULE_GUARD_ID = "NO_RULE" +# Sentinel ``guard_id`` for a reconciliation deadlock (#3065 Cluster A): the +# routing table's selection was vetoed by a guard, and the veto's own redirect +# target was vetoed in turn. Bounded by construction — the router stops here +# with both verdicts attached rather than iterating toward a G4 cap. See +# :func:`reconcile_dispatch`. +RECONCILE_DEADLOCK_GUARD_ID = "RECONCILE_DEADLOCK" + # Type alias for the predicate functions in DISPATCH_RULES. Each takes the # stage_states dict, the _meta dict, and an optional context dict, and returns @@ -239,6 +280,131 @@ class DispatchRule: reason: str +# --------------------------------------------------------------------------- +# Decision evidence (#3065 Cluster D) +# --------------------------------------------------------------------------- + + +def detect_unrecorded_dispatch(stage_states: dict, meta: dict) -> dict[str, Any] | None: + """Report a previous dispatch that has no confirming record, else ``None``. + + ``next-skill`` computes a decision and **persists nothing** (#2897); the + caller is expected to call ``sdlc-tool dispatch record`` before invoking + the skill, and ``PipelineStateMachine.start_stage`` then upgrades that + router slot to ``confirmed`` when the stage actually begins. When either + half of that protocol is skipped, the ledger keeps re-deriving the same row + from the same unchanged state until G4 caps the lane for "oscillating" — + an accurate observation attributed to the wrong cause. That misattribution + cost #2771 and #2334 a manual unwedge each. + + This is a **read**. It writes nothing and it never changes which skill is + dispatched; it only names the hole so a supervisor reading the decision can + see it now instead of inferring it from a G4 block four turns later. + + Three shapes count as "no confirming record", in the order checked: + + 1. ``last_dispatched_skill`` names a skill and ``_sdlc_dispatches`` is + empty — the dispatch was never recorded at all. + 2. The most recent history entry names a *different* skill — the record + belongs to an earlier turn, so the named dispatch went unrecorded. + 3. The most recent entry is still ``confirmed: False`` — a router slot + was recorded but the stage it reserved never started. + + Args: + stage_states: Stage-state dict; reads ``_sdlc_dispatches``. + meta: The ``_meta`` dict; reads ``last_dispatched_skill``. + + Returns: + A JSON-safe dict with ``last_dispatched_skill``, ``recorded_skill``, + ``confirmed`` and a human-readable ``reason``, or ``None`` when the + last dispatch is fully accounted for (including the case where nothing + has been dispatched yet). + """ + last = (meta or {}).get("last_dispatched_skill") or "" + if not last: + # Nothing has been dispatched yet — there is no obligation to report. + return None + + raw_history = (stage_states or {}).get("_sdlc_dispatches") + entries = ( + [e for e in raw_history if isinstance(e, dict)] if isinstance(raw_history, list) else [] + ) + + if not entries: + return { + "last_dispatched_skill": last, + "recorded_skill": None, + "confirmed": None, + "reason": ( + f"{last} was dispatched but no record exists in _sdlc_dispatches — " + f"the caller skipped `sdlc-tool dispatch record`" + ), + } + + tail = entries[-1] + recorded_skill = tail.get("skill") + if recorded_skill != last: + return { + "last_dispatched_skill": last, + "recorded_skill": recorded_skill, + "confirmed": tail.get("confirmed"), + "reason": ( + f"{last} was dispatched but the most recent dispatch record names " + f"{recorded_skill!r} — the caller skipped `sdlc-tool dispatch record`" + ), + } + + if tail.get("confirmed") is False: + return { + "last_dispatched_skill": last, + "recorded_skill": recorded_skill, + "confirmed": False, + "reason": ( + f"{last} was recorded as a router slot but never confirmed by a stage " + f"entry — the dispatched stage did not start" + ), + } + + return None + + +def build_decision_inputs(stage_states: dict, meta: dict, **extra: Any) -> dict[str, Any]: + """Package the facts a decision was computed from, for the decision to carry. + + Cluster D of #3065: a ``Blocked`` that names no inputs cannot be checked + against ground truth. The batch reported a ``NO_RULE`` on "CRITIQUE + APPROVED+completed, BUILD in_progress, no PR" — a state row 5 has owned + since ``c1e991972`` — and the report could neither be confirmed nor refuted, + because the payload carried no ``stage_states`` and no ``meta``. + + The key name ``decision_inputs`` is pinned: ``tools/sdlc_next_skill.py`` + surfaces it under exactly this name and a plan Verification row greps for + it. A grep for ``stage_states`` would pass today and prove nothing. + + Both dicts are shallow-copied so the returned evidence is a snapshot rather + than a live alias — the caller's dicts stay the same objects the guards + mutate by reference (see :func:`reconcile_dispatch`'s invariant), but this + payload will not silently change under a reader after the fact. + + Args: + stage_states: The stage-state dict the decision read. + meta: The ``_meta`` dict the decision read. + **extra: Additional named evidence for a specific blocked path, e.g. + ``selected_row`` and ``vetoing_guard`` on a reconciliation deadlock. + + Returns: + A JSON-serializable dict carrying ``stage_states``, ``meta``, + ``unrecorded_dispatch`` and any ``extra`` keys. + """ + payload: dict[str, Any] = { + "stage_states": dict(stage_states or {}), + "meta": dict(meta or {}), + "unrecorded_dispatch": detect_unrecorded_dispatch(stage_states or {}, meta or {}), + } + payload.update(extra) + return payload + + # --------------------------------------------------------------------------- # Stage-snapshot canonicalization (used by G4 counter + dispatch-history write) # --------------------------------------------------------------------------- @@ -472,8 +638,22 @@ def guard_g3_pr_lock(stage_states: dict, meta: dict, context: dict) -> Dispatch If an open PR exists for this issue AND the most recent dispatch was ``/do-plan`` or ``/do-plan-critique`` (or the LLM is asking the router about a plan-stage dispatch), redirect to the PR-stage skill appropriate - for the current state: ``/do-merge`` if review is APPROVED and docs are - done; ``/do-patch`` if review requested changes; otherwise ``/do-pr-review``. + for the current state: + + - ``/do-merge`` — review is APPROVED (recorded verdict) and docs done. + - ``/do-docs`` — review is APPROVED (recorded verdict) and docs are + NOT done (#3065 Cluster A: this arm was previously missing, so a + REVIEW-complete/APPROVED/DOCS-pending lane fell to the ``else`` and + was sent back to ``/do-pr-review`` it had already passed). + - ``/do-patch`` — review requested changes. + - ``/do-pr-review`` — otherwise (no verdict yet, or REVIEW not + complete, or a completed-but-unearned REVIEW marker). + + Arm 1 (merge) requires a recorded APPROVED verdict, not just + ``REVIEW == completed`` — REVIEW can be marked completed with no verdict + ever recorded (crash), which must not silently route to /do-merge. + Matches the same #1932 gap-(c) gate rows 9 and 10 already apply + (``_rule_review_approved_docs_not_done``, ``_rule_ready_to_merge``). """ pr_number = meta.get("pr_number") if not pr_number: @@ -497,10 +677,14 @@ def guard_g3_pr_lock(stage_states: dict, meta: dict, context: dict) -> Dispatch verdicts = stage_states.get("_verdicts") or {} review_verdict = _verdict_text(verdicts.get("REVIEW")) review_verdict_norm = normalize_verdict(review_verdict) + review_approved = REVIEW_APPROVED in review_verdict_norm - if review_status == STATUS_COMPLETED and docs_status == STATUS_COMPLETED: + if review_status == STATUS_COMPLETED and review_approved and docs_status == STATUS_COMPLETED: target = SKILL_DO_MERGE suffix = "review clean and docs complete" + elif review_status == STATUS_COMPLETED and review_approved and docs_status != STATUS_COMPLETED: + target = SKILL_DO_DOCS + suffix = G3_REDIRECT_REASON_DOCS_PENDING elif REVIEW_CHANGES_REQUESTED in review_verdict_norm or review_status == STATUS_FAILED: target = SKILL_DO_PATCH suffix = "review requested changes" @@ -2210,6 +2394,121 @@ def _stage_is_ready(stage_states: dict, stage: str) -> bool: return status in ("ready", "pending", "failed") +def reconcile_dispatch( + stage_states: dict, + meta: dict, + context: dict, + primary: Dispatch, +) -> Dispatch | Blocked | Terminal: + """Re-validate the dispatch table's selection against the guard list (#3065 + Cluster A/task 4 — "the reconciliation step"). + + ``guard_g3_pr_lock`` and friends (:1083-1086, the ``GUARDS`` list) only + ever constrained whatever the *caller* proposed via + ``context["proposed_skill"]`` — never the skill the dispatch table + actually selected. This function closes that gap: it re-runs the SAME + guard list with ``primary.skill`` substituted in as the proposed skill, + so a guard veto constrains the decision that is actually about to ship, + not just whatever the caller happened to ask about. + + Termination is the design constraint, not an afterthought: this function + runs the guard list AT MOST TWICE — once against ``primary``, and, only + if that pass redirects (returns a ``Dispatch``), once more against the + redirect. A second veto of any kind (redirect, block, or terminal) never + triggers a third pass; it is folded into a single ``Blocked`` that names + both the table's original selection and the vetoing guard's verdict. + This is deliberately fail-closed (Risk 2): a lane whose guards cannot + agree on a self-consistent answer stops immediately with evidence + instead of oscillating toward a G4 hard-cap, which is the failure mode + observed on #2771 and #2334. + + Row 2b (``_rule_critique_verdict_stale``) is not edited to make this + work, and must never be: #1639 made it marker-agnostic on purpose to + escape a CRITIQUE ``in_progress`` dead end (see its docstring). This + function is how row 2b's output on a shipped, open-PR lane gets + constrained — from outside, by G3 vetoing the redirect — while leaving + 2b free to fire unmodified on the lane shape it exists for. + + CRITICAL INVARIANT — ``stage_states`` and ``meta`` MUST be passed through + to ``evaluate_guards`` BY REFERENCE, never copied, here or in any caller. + The guards are not pure: ``guard_g5_artifact_hash_cache`` mutates + ``stage_states["_verdicts"]["CRITIQUE"]["artifact_hash"]`` in place on a + legacy-hash migration and logs a WARNING once. Because this function + calls ``evaluate_guards`` a second time on the SAME objects, that second + call sees the already-migrated record and silently steps aside — that is + the whole reason double invocation is idempotent today. A defensive copy + of ``stage_states``/``meta`` anywhere between the two passes would make + the second pass see the pre-migration hash again, re-running the + migration branch and doubling the log noise. Do not add one; this is + exactly the "safe" change a later contributor would make without + knowing this. + + A raising guard is NOT caught here, matching the existing guard/rule + asymmetry in this module: rule predicates in ``decide_next_dispatch`` are + try/except-wrapped, guards are not. Swallowing a raising guard into a + ``NO_RULE`` block here would misreport a bug as a routing hole. + """ + first_context = dict(context or {}) + first_context["proposed_skill"] = primary.skill + first_veto = evaluate_guards(stage_states, meta, first_context) + if first_veto is None: + return primary + if not isinstance(first_veto, Dispatch): + # A guard's own Blocked/Terminal is already a terminating decision + # with its own reason/guard_id — no need to wrap it further. + return first_veto + if first_veto.skill == primary.skill: + # The guard named the skill the table already chose. That is agreement, + # not a veto, so keep the table's own row_id and reason — reconciliation + # exists to WITHHOLD dispatches, never to relabel ones it endorses + # (Risk 1: it must not change a routing answer no guard objected to). + return primary + + second_context = dict(context or {}) + second_context["proposed_skill"] = first_veto.skill + second_veto = evaluate_guards(stage_states, meta, second_context) + if second_veto is None: + return first_veto + if isinstance(second_veto, Dispatch) and second_veto.skill == first_veto.skill: + # Same agreement case one level down: a guard re-proposing the redirect + # target is confirming it, not vetoing it. Blocking here would turn a + # self-consistent decision into a hard stop — the exact over-reach + # Risk 2 warns about. + return first_veto + + if isinstance(second_veto, Dispatch): + second_summary: dict[str, Any] = { + "skill": second_veto.skill, + "reason": second_veto.reason, + "row_id": second_veto.row_id, + } + elif isinstance(second_veto, Blocked): + second_summary = {"reason": second_veto.reason, "guard_id": second_veto.guard_id} + else: + second_summary = {"reason": second_veto.reason, "evidence": second_veto.evidence} + + return Blocked( + reason=( + f"reconciliation: guard veto did not converge — table selected row " + f"{primary.row_id!r} ({primary.skill!r}), redirect to " + f"{first_veto.skill!r} was itself vetoed" + ), + guard_id=RECONCILE_DEADLOCK_GUARD_ID, + decision_inputs=build_decision_inputs( + stage_states, + meta, + selected_row=primary.row_id, + selected_skill=primary.skill, + first_redirect={ + "skill": first_veto.skill, + "reason": first_veto.reason, + "row_id": first_veto.row_id, + }, + vetoing_guard=second_summary, + ), + ) + + def decide_next_dispatch( stage_states: dict, meta: dict | None = None, @@ -2221,8 +2520,13 @@ def decide_next_dispatch( 1. Evaluate the terminal guard, then G1–G9. If any trips, return its decision. A ``Terminal`` here means the lane is finished (#2894, #2817). 2. Otherwise, walk ``DISPATCH_RULES`` in row order. Take the first - rule whose ``state_predicate`` returns True as the primary dispatch. - 3. If no rule matches at all, return ``Blocked(reason="no matching rule")``. + rule whose ``state_predicate`` returns True as the primary dispatch, + then reconcile it against the guard list (``reconcile_dispatch``) — + the table's selection is re-validated, not just whatever the caller + proposed via ``context["proposed_skill"]``. + 3. If no rule matches at all, return ``Blocked(reason="no matching rule")`` + with ``decision_inputs`` carrying the ``stage_states``/``meta`` the + decision was made from (#3065 Cluster D). Args: stage_states: The stage-status dict from ``AgentSession.stage_states``. @@ -2281,9 +2585,22 @@ def decide_next_dispatch( return Blocked( reason="no matching dispatch rule", guard_id=NO_RULE_GUARD_ID, + decision_inputs=build_decision_inputs(stage_states, meta), ) - return primary + decision = reconcile_dispatch(stage_states, meta, context, primary) + + # #3065 Cluster D: a dispatch decision reports when the PREVIOUS decision + # was never recorded. Attached to the outgoing decision rather than + # computed by each caller, so the CLI path and the in-process + # ``agent/session_runner/runner.py`` path cannot disagree about it. Purely + # diagnostic: the skill is already chosen and this never changes it. + if isinstance(decision, Dispatch): + signal = detect_unrecorded_dispatch(stage_states, meta) + if signal is not None: + decision = replace(decision, unrecorded_dispatch=signal) + + return decision def stage_for_skill(skill: str | None) -> str | None: diff --git a/tests/unit/test_sdlc_next_skill.py b/tests/unit/test_sdlc_next_skill.py index 67f554dcc..63218fa61 100644 --- a/tests/unit/test_sdlc_next_skill.py +++ b/tests/unit/test_sdlc_next_skill.py @@ -1874,3 +1874,107 @@ def test_cli_exits_0_on_terminal_payload(self, monkeypatch, capsys): assert rc == 0 assert json.loads(capsys.readouterr().out)["decision"] == "terminal" + + +class TestDecisionEvidenceReachesTheCliJson: + """#3065 Cluster D: a decision's evidence is worthless if it dies inside + the router. These drive the actual CLI surface (``decide()``), following + ``test_decide_warm_cache_open_pr_defers_to_pr_review_not_plan``'s pattern + of injecting stage_states/meta rather than resolving live session state. + """ + + def _inject(self, monkeypatch, states, meta): + monkeypatch.setattr( + sdlc_next_skill, + "_resolve_enriched", + lambda issue_number, session_id: {"stages": states, "_meta": meta}, + ) + monkeypatch.setattr( + sdlc_next_skill, + "_build_context", + lambda proposed_skill, issue_number, stage_states=None, meta=None: {}, + ) + + def _no_rule_fixture(self, unconfirmed=False): + states = { + "ISSUE": STATUS_COMPLETED, + "PLAN": STATUS_COMPLETED, + "CRITIQUE": STATUS_COMPLETED, + "BUILD": STATUS_COMPLETED, + "TEST": STATUS_COMPLETED, + "REVIEW": STATUS_COMPLETED, + "DOCS": STATUS_COMPLETED, + "MERGE": "pending", + "_verdicts": { + "REVIEW": {"verdict": "LGTM", "recorded_at": "2026-09-01T00:00:00+00:00"} + }, + "_sdlc_dispatches": [ + { + "skill": "/do-test", + "at": "2026-09-03T00:00:00+00:00", + "stage_snapshot": {}, + "confirmed": not unconfirmed, + } + ], + } + meta = { + "pr_number": 4242, + "pr_merge_state": "CLEAN", + "pr_state": "OPEN", + "latest_review_verdict": "LGTM", + "last_dispatched_skill": "/do-test", + "ci_all_passing": True, + } + return states, meta + + def test_no_rule_block_round_trips_its_inputs_through_the_cli_json(self, monkeypatch): + states, meta = self._no_rule_fixture() + self._inject(monkeypatch, states, meta) + + result = sdlc_next_skill.decide(issue_number=28981) + + assert result["decision"] == "blocked" + assert result["guard_id"] == "NO_RULE" + # Serialized exactly as the CLI would print it -- the whole point is + # that a supervisor can read the inputs out of the JSON. + payload = json.loads(json.dumps(result)) + assert payload["decision_inputs"]["meta"]["pr_number"] == 4242 + assert payload["decision_inputs"]["stage_states"]["REVIEW"] == STATUS_COMPLETED + + def test_a_routed_decision_carries_no_decision_inputs_key(self, monkeypatch): + """Negative pole: the key is not sprayed onto every payload.""" + states, meta = self._no_rule_fixture() + states["_verdicts"]["REVIEW"]["verdict"] = "CHANGES REQUESTED" + meta["latest_review_verdict"] = "CHANGES REQUESTED" + self._inject(monkeypatch, states, meta) + + result = sdlc_next_skill.decide(issue_number=28981) + + assert result["decision"] == "dispatch" + assert "decision_inputs" not in result + + def test_dispatch_payload_reports_an_unrecorded_previous_dispatch(self, monkeypatch): + states, meta = self._no_rule_fixture(unconfirmed=True) + states["_verdicts"]["REVIEW"]["verdict"] = "CHANGES REQUESTED" + meta["latest_review_verdict"] = "CHANGES REQUESTED" + self._inject(monkeypatch, states, meta) + + result = sdlc_next_skill.decide(issue_number=28981) + + assert result["decision"] == "dispatch" + assert result["unrecorded_dispatch"]["confirmed"] is False + # Distinct from ``recorded``, which is about THIS decision and is + # always False because decide() never writes (#2897). + assert result["recorded"] is False + + def test_dispatch_payload_omits_the_signal_when_the_record_confirms(self, monkeypatch): + """Negative pole, one boolean apart from the test above.""" + states, meta = self._no_rule_fixture(unconfirmed=False) + states["_verdicts"]["REVIEW"]["verdict"] = "CHANGES REQUESTED" + meta["latest_review_verdict"] = "CHANGES REQUESTED" + self._inject(monkeypatch, states, meta) + + result = sdlc_next_skill.decide(issue_number=28981) + + assert result["decision"] == "dispatch" + assert "unrecorded_dispatch" not in result diff --git a/tests/unit/test_sdlc_router.py b/tests/unit/test_sdlc_router.py index f8d9d882f..c27f846b9 100644 --- a/tests/unit/test_sdlc_router.py +++ b/tests/unit/test_sdlc_router.py @@ -10,6 +10,7 @@ from __future__ import annotations from agent.sdlc_router import ( + G3_REDIRECT_REASON_DOCS_PENDING, GUARDS, MAX_PLAN_REVISING_DISPATCHES, MAX_SAME_STAGE_DISPATCHES, @@ -25,6 +26,7 @@ STATUS_FAILED, Blocked, Dispatch, + Terminal, _rule_pr_exists_no_review, _rule_review_approved_docs_not_done, build_stage_snapshot, @@ -32,8 +34,10 @@ decide_next_dispatch, evaluate_guards, guard_g2_critique_cycle_cap, + guard_g3_pr_lock, guard_g5_artifact_hash_cache, guard_g7_plan_revising, + reconcile_dispatch, record_dispatch, ) @@ -1565,3 +1569,383 @@ def test_g6_fast_path_fires_on_fresh_head(self): result = guard_g6_terminal_merge_ready(states, meta, {"pr_head_sha": _SHA_A}) assert isinstance(result, Dispatch) assert result.skill == SKILL_DO_MERGE + + +# --------------------------------------------------------------------------- +# #3065 task 3: G3's docs-pending redirect arm +# +# G3's ladder previously had exactly three arms and no ``/do-docs`` arm, so a +# lane with REVIEW complete, APPROVED, and DOCS pending fell to the ``else`` +# and was sent back to ``/do-pr-review`` it had already passed. +# --------------------------------------------------------------------------- + + +class TestG3DocsArm: + def _meta(self, **overrides): + return _base_meta( + pr_number=3065, + last_dispatched_skill=SKILL_DO_PLAN, + **overrides, + ) + + def test_docs_pending_after_approved_review_redirects_to_docs(self): + """The state that previously fell to the ``else`` and produced a + redundant /do-pr-review dispatch.""" + states = {"REVIEW": STATUS_COMPLETED, "DOCS": "pending"} + meta = self._meta(latest_review_verdict="APPROVED") + result = guard_g3_pr_lock(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_DOCS + assert G3_REDIRECT_REASON_DOCS_PENDING in result.reason + + def test_docs_complete_after_approved_review_redirects_to_merge(self): + states = {"REVIEW": STATUS_COMPLETED, "DOCS": STATUS_COMPLETED} + meta = self._meta(latest_review_verdict="APPROVED") + result = guard_g3_pr_lock(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_MERGE + + def test_review_completed_without_recorded_verdict_does_not_merge(self): + """Arm 1 now requires a recorded APPROVED verdict, matching rows 9/10 + (#1932 gap c) -- REVIEW==completed with no verdict is an unearned + marker, not evidence of approval.""" + states = {"REVIEW": STATUS_COMPLETED, "DOCS": STATUS_COMPLETED} + meta = self._meta(latest_review_verdict=None) + result = guard_g3_pr_lock(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PR_REVIEW + + def test_review_completed_without_recorded_verdict_does_not_route_to_docs_either(self): + """Same unearned-marker state, docs pending: still no dispatch to + /do-docs without a recorded APPROVED verdict.""" + states = {"REVIEW": STATUS_COMPLETED, "DOCS": "pending"} + meta = self._meta(latest_review_verdict=None) + result = guard_g3_pr_lock(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PR_REVIEW + + def test_changes_requested_still_routes_to_patch(self): + """Regression guard: the changes-requested arm is untouched by the + new docs arm.""" + states = {"REVIEW": STATUS_COMPLETED, "DOCS": "pending"} + meta = self._meta(latest_review_verdict="CHANGES REQUESTED") + result = guard_g3_pr_lock(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PATCH + + def test_review_not_yet_complete_still_routes_to_pr_review(self): + """Regression guard: the default arm is untouched.""" + states = {"REVIEW": "pending", "DOCS": "pending"} + meta = self._meta(latest_review_verdict=None) + result = guard_g3_pr_lock(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PR_REVIEW + + def test_through_decide_next_dispatch_docs_pending_state(self): + """End-to-end: the docs arm actually reaches the caller via + decide_next_dispatch, not just the guard in isolation.""" + states = dict(_ALL_COMPLETED, DOCS="pending", MERGE="pending") + meta = _base_meta( + pr_number=3065, + last_dispatched_skill=SKILL_DO_PLAN, + latest_review_verdict="APPROVED", + ) + result = decide_next_dispatch(states, meta, {}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_DOCS + + +# --------------------------------------------------------------------------- +# #3065 task 2: NO_RULE decision_inputs evidence +# +# A NO_RULE block previously carried no evidence of what the router actually +# read, so a field report claiming a NO_RULE on a state a rule demonstrably +# owns could not be checked. decision_inputs closes that gap. +# --------------------------------------------------------------------------- + + +def _unowned_state() -> tuple[dict, dict]: + """A genuine hole in the dispatch table (mirrors + sdlc_router_decision/test_sdlc_router_decision_post_patch.py's + TestNoRuleBlockIsDistinguishable._unowned_state): REVIEW pending while an + APPROVED verdict is already recorded -- row 8e needs REVIEW completed, + row 10 needs it settled, row 7 needs no verdict, so nothing owns it. + """ + states = { + "PLAN": "completed", + "CRITIQUE": "completed", + "BUILD": "completed", + "REVIEW": "pending", + "DOCS": "completed", + "MERGE": "pending", + "_verdicts": {"REVIEW": {"verdict": "APPROVED", "recorded_at": "2026-05-06T00:00:00Z"}}, + } + meta = _base_meta( + pr_number=4242, + last_dispatched_skill=SKILL_DO_MERGE, + pr_merge_state="BLOCKED", + ) + return states, meta + + +class TestNoRuleDecisionInputs: + def test_no_rule_block_carries_decision_inputs(self): + states, meta = _unowned_state() + result = decide_next_dispatch(states, meta) + assert isinstance(result, Blocked) + assert result.guard_id == "NO_RULE" + assert result.decision_inputs is not None + # build_decision_inputs shallow-copies (a snapshot, not a live alias + # of the caller's dicts) -- compare by value, not identity. + assert result.decision_inputs["stage_states"] == states + assert result.decision_inputs["meta"] == meta + + def test_decision_inputs_round_trips_through_cli_json(self, monkeypatch): + """Drives the fallthrough through the actual sdlc-tool next-skill + JSON surface (tools.sdlc_next_skill.decide), not just + decide_next_dispatch directly, so the CLI wiring a supervisor + actually reads is verified too.""" + import tools.sdlc_next_skill as sdlc_next_skill + + states, meta = _unowned_state() + monkeypatch.setattr( + sdlc_next_skill, + "_resolve_enriched", + lambda issue_number, session_id: {"stages": states, "_meta": meta}, + ) + monkeypatch.setattr( + sdlc_next_skill, + "_build_context", + lambda proposed_skill, issue_number, stage_states=None, meta=None: {}, + ) + + result = sdlc_next_skill.decide(issue_number=4242) + + assert result["decision"] == "blocked" + assert result["guard_id"] == "NO_RULE" + assert "decision_inputs" in result + assert result["decision_inputs"]["meta"]["pr_number"] == 4242 + assert result["decision_inputs"]["stage_states"]["REVIEW"] == "pending" + + def test_unrecorded_dispatch_present_when_last_slot_unconfirmed(self): + """The caller ran dispatch record but the stage never started -- + surfaced so this no longer has to be reverse-engineered from a later + G4 oscillation block attributed to the wrong cause.""" + states, meta = _unowned_state() + states["_sdlc_dispatches"] = [ + {"skill": SKILL_DO_MERGE, "stage": "MERGE", "confirmed": False} + ] + result = decide_next_dispatch(states, meta) + assert isinstance(result, Blocked) + signal = result.decision_inputs["unrecorded_dispatch"] + assert signal is not None + assert signal["confirmed"] is False + assert signal["last_dispatched_skill"] == SKILL_DO_MERGE + + def test_unrecorded_dispatch_absent_when_last_slot_confirmed(self): + states, meta = _unowned_state() + states["_sdlc_dispatches"] = [ + {"skill": SKILL_DO_MERGE, "stage": "MERGE", "confirmed": True} + ] + result = decide_next_dispatch(states, meta) + assert isinstance(result, Blocked) + assert result.decision_inputs["unrecorded_dispatch"] is None + + def test_unrecorded_dispatch_present_when_caller_skipped_dispatch_record(self): + """No _sdlc_dispatches entry at all for a meta that names a + last_dispatched_skill -- the caller skipped `sdlc-tool dispatch + record` entirely, not merely left a slot unconfirmed.""" + states, meta = _unowned_state() + result = decide_next_dispatch(states, meta) + assert isinstance(result, Blocked) + signal = result.decision_inputs["unrecorded_dispatch"] + assert signal is not None + assert signal["recorded_skill"] is None + + def test_unrecorded_dispatch_absent_when_nothing_dispatched_yet(self): + states, meta = _unowned_state() + meta["last_dispatched_skill"] = None + result = decide_next_dispatch(states, meta) + assert isinstance(result, Blocked) + assert result.decision_inputs["unrecorded_dispatch"] is None + + +# --------------------------------------------------------------------------- +# #3065 task 4: reconcile_dispatch — the keystone reconciliation step +# +# Guards previously validated only whatever the caller PROPOSED via +# context["proposed_skill"], never the skill the dispatch table actually +# SELECTED (:2246-2248 runs before the table at :2250-2255, and nothing +# re-validated in between). reconcile_dispatch re-runs the guard list +# against the table's selection, terminating after at most one redirect. +# --------------------------------------------------------------------------- + + +class TestReconcileDispatch: + def test_accepts_selection_untouched_when_no_guard_objects(self): + primary = Dispatch(skill=SKILL_DO_BUILD, reason="test", row_id="X") + result = reconcile_dispatch({}, _base_meta(), {}, primary) + assert result is primary + + def test_2771_2334_lane_shape_routes_to_docs_not_plan_critique(self): + """The #2771/#2334 lane shape: open PR, REVIEW APPROVED, DOCS + pending, and a CRITIQUE verdict stale enough that row 2b (which sits + fifteen positions before row 9 in table order) would otherwise + preempt DOCS unconditionally. Reconciliation lets G3 veto row 2b's + redirect target (/do-plan-critique) on this shipped lane, without + editing row 2b's predicate (#1639).""" + states = { + "PLAN": "completed", + "CRITIQUE": "completed", + "BUILD": "completed", + "TEST": "completed", + "REVIEW": STATUS_COMPLETED, + "DOCS": "pending", + "MERGE": "pending", + "_verdicts": { + "CRITIQUE": {"verdict": "READY TO BUILD", "recorded_at": "2026-01-01T00:00:00Z"}, + "REVIEW": {"verdict": "APPROVED", "recorded_at": "2026-05-01T00:00:00Z"}, + }, + "_sdlc_dispatches": [ + {"skill": SKILL_DO_PLAN, "at": "2026-06-01T00:00:00Z", "stage_snapshot": {}}, + ], + } + meta = _base_meta( + pr_number=2771, + last_dispatched_skill=SKILL_DO_MERGE, + latest_review_verdict="APPROVED", + ) + + result = decide_next_dispatch(states, meta, {}) + + assert isinstance(result, Dispatch), f"expected Dispatch, got {result!r}" + assert result.skill == SKILL_DO_DOCS + assert result.skill != SKILL_DO_PLAN_CRITIQUE + + def test_plan_critique_needed_with_no_pr_still_dispatched(self): + """A lane genuinely needing /do-plan-critique, with no PR open, must + still get it -- reconciliation must not withhold a dispatch G3 has + no jurisdiction over (G3 steps aside unconditionally when + meta['pr_number'] is falsy).""" + states = {"PLAN": "completed", "CRITIQUE": "pending"} + meta = _base_meta(pr_number=None, last_dispatched_skill=None) + + result = decide_next_dispatch(states, meta, {}) + + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PLAN_CRITIQUE + + def test_double_veto_produces_blocked_with_both_verdicts_and_terminates(self): + """A contrived guard pair that vetoes both the selection and its own + redirect must terminate as a single Blocked carrying both verdicts, + never a third pass.""" + primary = Dispatch(skill=SKILL_DO_BUILD, reason="table selected build", row_id="4a") + call_count = {"n": 0} + + def _always_redirect(stage_states, meta, context): + call_count["n"] += 1 + proposed = context.get("proposed_skill") + # Redirect every proposal to a DIFFERENT skill, so a naive loop + # would never converge -- proves this function bounds itself + # rather than relying on the guard to stop redirecting. + target = SKILL_DO_PATCH if proposed != SKILL_DO_PATCH else SKILL_DO_PR_REVIEW + return Dispatch(skill=target, reason=f"vetoing {proposed}", row_id="TESTGUARD") + + original_guards = list(GUARDS) + GUARDS.clear() + GUARDS.append(_always_redirect) + try: + result = reconcile_dispatch({}, _base_meta(), {}, primary) + finally: + GUARDS[:] = original_guards + + assert call_count["n"] == 2, "reconciliation must run the guard list at most twice" + assert isinstance(result, Blocked) + assert result.guard_id == "RECONCILE_DEADLOCK" + assert result.decision_inputs["selected_row"] == "4a" + assert result.decision_inputs["selected_skill"] == SKILL_DO_BUILD + assert result.decision_inputs["first_redirect"]["skill"] == SKILL_DO_PATCH + assert result.decision_inputs["vetoing_guard"]["skill"] == SKILL_DO_PR_REVIEW + + def test_raising_guard_during_reconciliation_is_not_swallowed(self): + """Preserves the existing asymmetry: rule predicates are + try/except-wrapped, guards are not. A raising guard must propagate + out of reconciliation rather than being folded into a NO_RULE.""" + + def _raises(stage_states, meta, context): + raise RuntimeError("boom") + + original_guards = list(GUARDS) + GUARDS.clear() + GUARDS.append(_raises) + try: + import pytest + + with pytest.raises(RuntimeError, match="boom"): + reconcile_dispatch({}, _base_meta(), {}, Dispatch(SKILL_DO_BUILD, "x", "4a")) + finally: + GUARDS[:] = original_guards + + def test_terminal_veto_on_reconciliation_returns_terminal_unmodified(self): + primary = Dispatch(skill=SKILL_DO_BUILD, reason="table selected build", row_id="4a") + terminal = Terminal(reason="pipeline complete", evidence="merge_marker", row_id="T") + + def _terminal_guard(stage_states, meta, context): + return terminal + + original_guards = list(GUARDS) + GUARDS.clear() + GUARDS.append(_terminal_guard) + try: + result = reconcile_dispatch({}, _base_meta(), {}, primary) + finally: + GUARDS[:] = original_guards + + assert result is terminal + + +class TestG5DoubleInvocationDuringReconciliation: + """Regression test (#3065 task 4): guard_g5_artifact_hash_cache's + legacy-hash migration branch must execute EXACTLY ONCE per + decide_next_dispatch call, even though reconciliation runs the guard + list a second time on the SAME stage_states/meta objects. Asserted via + the WARNING log-record count, not the return value -- the return value + is identical whether the migration ran once or twice, which is why this + needs its own test.""" + + def test_migration_warning_logged_exactly_once(self, caplog): + legacy_hash = "sha256:legacy-full-bytes" + current_hash = "sha256:body-only" + states = { + "PLAN": "completed", + "CRITIQUE": "completed", + "BUILD": STATUS_COMPLETED if False else "in_progress", + "_verdicts": { + "CRITIQUE": { + # Neither NEEDS REVISION/MAJOR REWORK nor READY TO BUILD -- + # G5 must migrate the hash and then still step aside (return + # None), so the table (not G5) selects the primary dispatch + # and reconciliation genuinely re-runs G5 a second time. + "verdict": "SOME OTHER VERDICT TEXT", + "artifact_hash": legacy_hash, + } + }, + } + meta = _base_meta(pr_number=None, last_dispatched_skill=None) + context = {"current_plan_hash": current_hash, "legacy_plan_hash": legacy_hash} + + with caplog.at_level("WARNING", logger="agent.sdlc_router"): + result = decide_next_dispatch(states, meta, context) + + assert isinstance(result, Dispatch), f"expected Dispatch, got {result!r}" + assert result.skill == SKILL_DO_BUILD + assert result.row_id == "5" + + migration_warnings = [r for r in caplog.records if "G5 migration" in r.message] + assert len(migration_warnings) == 1, ( + f"expected exactly one G5 migration WARNING, got {len(migration_warnings)}: " + f"{[r.message for r in migration_warnings]}" + ) + # The mutation itself is by-reference and idempotent: the second + # (reconciliation) pass must see the already-migrated hash. + assert states["_verdicts"]["CRITIQUE"]["artifact_hash"] == current_hash diff --git a/tests/unit/test_sdlc_router_oscillation.py b/tests/unit/test_sdlc_router_oscillation.py index a3f768f99..63ff6a83a 100644 --- a/tests/unit/test_sdlc_router_oscillation.py +++ b/tests/unit/test_sdlc_router_oscillation.py @@ -114,20 +114,48 @@ def test_g3_pr_lock_routes_to_patch_on_changes_requested(): assert result.skill == SKILL_DO_PATCH -def test_g3_pr_lock_routes_to_merge_when_review_and_docs_complete(): - """G3: PR + REVIEW completed + DOCS completed → /do-merge.""" +def test_g3_pr_lock_routes_to_merge_when_review_approved_and_docs_complete(): + """G3: PR + APPROVED verdict + REVIEW completed + DOCS completed → /do-merge. + + The APPROVED verdict is load-bearing, not decoration (#3065 task 3): G3's + merge arm now applies the same #1932 gap-(c) gate rows 9 and 10 already do. + See ``test_g3_pr_lock_does_not_merge_a_completed_review_with_no_verdict`` + for the other pole. + """ states = { "PLAN": "completed", "REVIEW": "completed", "DOCS": "completed", + "_verdicts": {"REVIEW": {"verdict": "APPROVED"}}, + } + meta = { + "pr_number": 42, + "last_dispatched_skill": SKILL_DO_PLAN, + "latest_review_verdict": "APPROVED", } - meta = {"pr_number": 42, "last_dispatched_skill": SKILL_DO_PLAN} result = decide_next_dispatch(states, meta) assert isinstance(result, Dispatch) assert result.row_id == "G3" assert result.skill == SKILL_DO_MERGE +def test_g3_pr_lock_does_not_merge_a_completed_review_with_no_verdict(): + """A REVIEW marker completed with no verdict ever recorded is the crash + state rows 8e/9/10 refuse to fast-path. G3 must refuse it too, and + re-review rather than merge.""" + states = { + "PLAN": "completed", + "REVIEW": "completed", + "DOCS": "completed", + } + meta = {"pr_number": 42, "last_dispatched_skill": SKILL_DO_PLAN} + result = decide_next_dispatch(states, meta) + assert isinstance(result, Dispatch) + assert result.row_id == "G3" + assert result.skill == SKILL_DO_PR_REVIEW + assert result.skill != SKILL_DO_MERGE + + def test_g4_oscillation_cap(): """G4: same_stage_dispatch_count >= MAX → Blocked.""" result = decide_next_dispatch( diff --git a/tests/unit/test_sdlc_router_reconciliation.py b/tests/unit/test_sdlc_router_reconciliation.py new file mode 100644 index 000000000..d0fdad85a --- /dev/null +++ b/tests/unit/test_sdlc_router_reconciliation.py @@ -0,0 +1,460 @@ +"""Router tasks 2-4 of #3065, second layer: the properties the row-level tests +in ``tests/unit/test_sdlc_router.py`` do not pin. + +That file already covers the headline behaviors — G3's docs arm, the NO_RULE +evidence payload, the #2771/#2334 shipped-lane redirect, the double-veto bound, +and the single G5 migration WARNING. This module deliberately does NOT repeat +them. It covers what is left, all of which is about the *mechanism* rather than +the routing answer: + + - ``detect_unrecorded_dispatch`` as a unit, across all three "no confirming + record" shapes and its silent cases. + - The evidence payload being a snapshot rather than a live alias, and being + JSON-serializable — it is worthless if it cannot reach the CLI output. + - Reconciliation's *agreement* cases, where a guard names the skill the table + already chose. Blocking those would turn self-consistent decisions into + hard stops, the over-reach Risk 2 warns about. + - The by-reference invariant, asserted on object identity rather than on the + log count. + - Every context shape ``decide_next_dispatch`` is called with, including the + ``context=None`` / ``context={}`` shape at + ``agent/session_runner/runner.py:1408``. + +Two-pole per #2658: each new gate has the state that must fire and the +neighbouring state that must not. +""" + +from __future__ import annotations + +import json +import logging + +import pytest + +from agent.sdlc_router import ( + NO_RULE_GUARD_ID, + SKILL_DO_DOCS, + SKILL_DO_MERGE, + SKILL_DO_PATCH, + SKILL_DO_PLAN, + SKILL_DO_PLAN_CRITIQUE, + SKILL_DO_PR_REVIEW, + STATUS_COMPLETED, + Blocked, + Dispatch, + _rule_critique_verdict_stale, + build_decision_inputs, + decide_next_dispatch, + detect_unrecorded_dispatch, + evaluate_guards, + guard_g3_pr_lock, + guard_g5_artifact_hash_cache, + reconcile_dispatch, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _shipped_lane_states(**overrides) -> dict: + """A lane that has shipped: PR open, REVIEW approved, DOCS still pending. + + The #2771 / #2334 shape. ``last_dispatched_skill`` is a PR-stage skill, not + a plan-family one — that is spike-2's actual hole. With a plan-family + ``last``, G3 trips in the pre-table pass and reconciliation is never the + thing under test. + """ + states = { + "ISSUE": STATUS_COMPLETED, + "PLAN": STATUS_COMPLETED, + "CRITIQUE": STATUS_COMPLETED, + "BUILD": STATUS_COMPLETED, + "TEST": STATUS_COMPLETED, + "REVIEW": STATUS_COMPLETED, + "DOCS": "pending", + "MERGE": "pending", + "_verdicts": { + # Recorded BEFORE the /do-plan dispatch below, which is what makes + # row 2b classify it stale. + "CRITIQUE": { + "verdict": "READY TO BUILD", + "recorded_at": "2026-09-01T00:00:00+00:00", + }, + "REVIEW": { + "verdict": "APPROVED", + "recorded_at": "2026-09-03T00:00:00+00:00", + }, + }, + "_sdlc_dispatches": [ + { + "skill": SKILL_DO_PLAN, + "at": "2026-09-02T00:00:00+00:00", + "stage_snapshot": {"PLAN": STATUS_COMPLETED}, + "confirmed": True, + }, + { + "skill": SKILL_DO_PR_REVIEW, + "at": "2026-09-03T00:00:00+00:00", + "stage_snapshot": {"REVIEW": STATUS_COMPLETED}, + "confirmed": True, + }, + ], + } + states.update(overrides) + return states + + +def _shipped_lane_meta(**overrides) -> dict: + meta = { + "pr_number": 2771, + "pr_merge_state": "CLEAN", + "pr_state": "OPEN", + "latest_critique_verdict": "READY TO BUILD", + "latest_review_verdict": "APPROVED", + "last_dispatched_skill": SKILL_DO_PR_REVIEW, + "ci_all_passing": True, + "same_stage_dispatch_count": 0, + } + meta.update(overrides) + return meta + + +def _no_rule_states(**overrides) -> dict: + """A state no dispatch-table row owns — an unrecognized REVIEW verdict. + + Not APPROVED (rows 8f/9/10 and G6 step aside), not CHANGES REQUESTED (rows + 8/8b step aside), and present (rows 8c/8d/8e require its absence). + """ + states = { + "ISSUE": STATUS_COMPLETED, + "PLAN": STATUS_COMPLETED, + "CRITIQUE": STATUS_COMPLETED, + "BUILD": STATUS_COMPLETED, + "TEST": STATUS_COMPLETED, + "REVIEW": STATUS_COMPLETED, + "DOCS": STATUS_COMPLETED, + "MERGE": "pending", + "_verdicts": {"REVIEW": {"verdict": "LGTM", "recorded_at": "2026-09-01T00:00:00+00:00"}}, + } + states.update(overrides) + return states + + +def _no_rule_meta(**overrides) -> dict: + meta = { + "pr_number": 77, + "pr_merge_state": "CLEAN", + "pr_state": "OPEN", + "latest_review_verdict": "LGTM", + "last_dispatched_skill": "/do-test", + "ci_all_passing": True, + } + meta.update(overrides) + return meta + + +# --------------------------------------------------------------------------- +# Task 2 — the evidence payload as a mechanism +# --------------------------------------------------------------------------- + + +class TestDecisionInputsPayload: + def test_fixture_really_is_a_no_rule_block(self): + """Guard the fixture: if a future row claims this state, the rest of + this class silently stops testing anything.""" + result = decide_next_dispatch(_no_rule_states(), _no_rule_meta()) + assert isinstance(result, Blocked) + assert result.guard_id == NO_RULE_GUARD_ID + + def test_payload_is_json_serializable(self): + """It has to survive to the CLI's JSON output to be worth anything.""" + result = decide_next_dispatch(_no_rule_states(), _no_rule_meta()) + round_tripped = json.loads(json.dumps(result.decision_inputs)) + assert round_tripped["meta"]["pr_number"] == 77 + assert round_tripped["stage_states"]["REVIEW"] == STATUS_COMPLETED + + def test_payload_is_a_snapshot_not_a_live_alias(self): + """A reader must see what the router saw, not what the dict became.""" + states = _no_rule_states() + result = decide_next_dispatch(states, _no_rule_meta()) + states["REVIEW"] = "mutated-after-the-decision" + assert result.decision_inputs["stage_states"]["REVIEW"] == STATUS_COMPLETED + + def test_a_routable_lane_produces_no_block_at_all(self): + """Negative pole: the evidence path must not fire on a routable lane.""" + assert isinstance( + decide_next_dispatch(_shipped_lane_states(), _shipped_lane_meta()), Dispatch + ) + + def test_extra_named_evidence_is_carried_through(self): + payload = build_decision_inputs({"PLAN": "completed"}, {"pr_number": 1}, selected_row="2b") + assert payload["selected_row"] == "2b" + assert payload["stage_states"] == {"PLAN": "completed"} + assert payload["meta"] == {"pr_number": 1} + + +class TestDetectUnrecordedDispatch: + """The signal fires exactly when the last dispatch has no confirming + record. Unit-level: the router-level wiring is covered in + ``test_sdlc_router.py::TestNoRuleDecisionInputs``. + """ + + def _history(self, skill, confirmed): + return [ + { + "skill": skill, + "at": "2026-09-03T00:00:00+00:00", + "stage_snapshot": {}, + "confirmed": confirmed, + } + ] + + def test_fires_when_the_recorded_slot_was_never_confirmed(self): + signal = detect_unrecorded_dispatch( + {"_sdlc_dispatches": self._history(SKILL_DO_PR_REVIEW, False)}, + {"last_dispatched_skill": SKILL_DO_PR_REVIEW}, + ) + assert signal is not None + assert signal["confirmed"] is False + assert "never confirmed" in signal["reason"] + + def test_silent_when_the_slot_was_confirmed(self): + """Negative pole for the same state, one field apart.""" + signal = detect_unrecorded_dispatch( + {"_sdlc_dispatches": self._history(SKILL_DO_PR_REVIEW, True)}, + {"last_dispatched_skill": SKILL_DO_PR_REVIEW}, + ) + assert signal is None + + def test_fires_when_no_record_exists_at_all(self): + signal = detect_unrecorded_dispatch( + {"_sdlc_dispatches": []}, + {"last_dispatched_skill": SKILL_DO_PR_REVIEW}, + ) + assert signal is not None + assert signal["recorded_skill"] is None + assert "dispatch record" in signal["reason"] + + def test_fires_when_the_newest_record_names_another_skill(self): + signal = detect_unrecorded_dispatch( + {"_sdlc_dispatches": self._history(SKILL_DO_PLAN, True)}, + {"last_dispatched_skill": SKILL_DO_PR_REVIEW}, + ) + assert signal is not None + assert signal["recorded_skill"] == SKILL_DO_PLAN + + def test_silent_when_nothing_has_been_dispatched_yet(self): + """An empty ledger is not an unrecorded dispatch.""" + assert detect_unrecorded_dispatch({}, {}) is None + assert detect_unrecorded_dispatch({}, {"last_dispatched_skill": None}) is None + + def test_silent_on_a_malformed_history(self): + assert ( + detect_unrecorded_dispatch( + {"_sdlc_dispatches": "not-a-list"}, + {"last_dispatched_skill": None}, + ) + is None + ) + + def test_the_signal_never_changes_which_skill_is_dispatched(self): + """Purely diagnostic — the two poles must agree on the skill.""" + confirmed = decide_next_dispatch(_shipped_lane_states(), _shipped_lane_meta()) + states = _shipped_lane_states() + states["_sdlc_dispatches"][-1]["confirmed"] = False + unconfirmed = decide_next_dispatch(states, _shipped_lane_meta()) + assert unconfirmed.unrecorded_dispatch is not None + assert confirmed.unrecorded_dispatch is None + assert confirmed.skill == unconfirmed.skill == SKILL_DO_DOCS + + +# --------------------------------------------------------------------------- +# Task 3 — the G3 arms the row-level tests do not pin +# --------------------------------------------------------------------------- + + +class TestG3LadderEdges: + def _states(self, review, docs, verdict): + verdicts = {} + if verdict is not None: + verdicts["REVIEW"] = {"verdict": verdict} + return {"PLAN": STATUS_COMPLETED, "REVIEW": review, "DOCS": docs, "_verdicts": verdicts} + + def _meta(self, verdict): + return { + "pr_number": 42, + "last_dispatched_skill": SKILL_DO_PLAN, + "latest_review_verdict": verdict, + } + + def test_changes_requested_beats_a_completed_docs_marker(self): + """Arm ordering: a completed DOCS marker must not launder a CHANGES + REQUESTED verdict into /do-merge.""" + result = guard_g3_pr_lock( + self._states(STATUS_COMPLETED, STATUS_COMPLETED, "CHANGES REQUESTED"), + self._meta("CHANGES REQUESTED"), + {}, + ) + assert result.skill == SKILL_DO_PATCH + assert result.skill != SKILL_DO_MERGE + + def test_guard_steps_aside_entirely_without_a_pr(self): + """Negative pole for the whole guard, not just an arm.""" + meta = self._meta("APPROVED") + meta["pr_number"] = None + assert ( + guard_g3_pr_lock(self._states(STATUS_COMPLETED, "pending", "APPROVED"), meta, {}) + is None + ) + + def test_guard_steps_aside_when_nothing_plan_family_is_in_play(self): + meta = self._meta("APPROVED") + meta["last_dispatched_skill"] = "/do-test" + assert ( + guard_g3_pr_lock(self._states(STATUS_COMPLETED, "pending", "APPROVED"), meta, {}) + is None + ) + + +# --------------------------------------------------------------------------- +# Task 4 — reconciliation mechanics +# --------------------------------------------------------------------------- + + +class TestReconciliationPremises: + """If either premise below stops holding, the shipped-lane tests in + ``test_sdlc_router.py`` silently stop exercising reconciliation.""" + + def test_the_table_really_does_select_row_2b_here(self): + states, meta = _shipped_lane_states(), _shipped_lane_meta() + assert _rule_critique_verdict_stale(states, meta, {}) is True + + def test_the_guards_do_not_trip_before_the_table_runs(self): + """The pre-table pass must be silent, or G3 would already have fired + on ``last_dispatched_skill`` and reconciliation would be untested.""" + states, meta = _shipped_lane_states(), _shipped_lane_meta() + assert evaluate_guards(states, meta, {}) is None + + @pytest.mark.parametrize("context", [None, {}, {"proposed_skill": SKILL_DO_PLAN_CRITIQUE}]) + def test_identical_answer_for_every_context_shape(self, context): + """``agent/session_runner/runner.py:1408`` passes no context at all. + Reconciliation must not silently no-op on that path.""" + result = decide_next_dispatch(_shipped_lane_states(), _shipped_lane_meta(), context) + assert result.skill == SKILL_DO_DOCS + + def test_a_lane_with_no_pr_still_gets_its_plan_critique(self): + """Negative pole. Reconciliation may only WITHHOLD a dispatch a guard + would already refuse; it must never take away a legitimate one.""" + meta = _shipped_lane_meta(pr_number=None, pr_merge_state=None, pr_state=None) + states = _shipped_lane_states(REVIEW="pending", DOCS="pending") + states["_verdicts"].pop("REVIEW") + result = decide_next_dispatch(states, meta) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PLAN_CRITIQUE + + +class TestReconciliationAgreementCases: + """A guard naming the skill the table already chose is agreeing with it, + not vetoing it. Treating that as a veto would either relabel a decision + nobody objected to (Risk 1) or hard-stop a self-consistent lane (Risk 2). + """ + + def test_a_guard_agreeing_with_the_table_keeps_the_table_row(self, monkeypatch): + def agrees(stage_states, meta, context): + proposed = context.get("proposed_skill") + return Dispatch(skill=proposed, reason="agree", row_id="GX") if proposed else None + + monkeypatch.setattr("agent.sdlc_router.GUARDS", [agrees]) + primary = Dispatch(skill=SKILL_DO_DOCS, reason="row 9", row_id="9") + result = reconcile_dispatch({}, {}, {}, primary) + assert result.row_id == "9" + assert result.reason == "row 9" + + def test_a_guard_agreeing_with_the_redirect_is_not_a_second_veto(self, monkeypatch): + def redirect_then_agree(stage_states, meta, context): + proposed = context.get("proposed_skill") + if proposed == SKILL_DO_PLAN_CRITIQUE: + return Dispatch(skill=SKILL_DO_DOCS, reason="veto", row_id="GX") + if proposed == SKILL_DO_DOCS: + return Dispatch(skill=SKILL_DO_DOCS, reason="agree", row_id="GX") + return None + + monkeypatch.setattr("agent.sdlc_router.GUARDS", [redirect_then_agree]) + result = reconcile_dispatch( + {}, {}, {}, Dispatch(skill=SKILL_DO_PLAN_CRITIQUE, reason="r", row_id="2b") + ) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_DOCS + + def test_a_single_veto_returns_the_redirect(self, monkeypatch): + def once(stage_states, meta, context): + if context.get("proposed_skill") == SKILL_DO_PLAN_CRITIQUE: + return Dispatch(skill=SKILL_DO_DOCS, reason="veto", row_id="GX") + return None + + monkeypatch.setattr("agent.sdlc_router.GUARDS", [once]) + result = reconcile_dispatch( + {}, {}, {}, Dispatch(skill=SKILL_DO_PLAN_CRITIQUE, reason="r", row_id="2b") + ) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_DOCS + + def test_a_guards_own_block_is_returned_unwrapped(self, monkeypatch): + def blocks(stage_states, meta, context): + if context.get("proposed_skill"): + return Blocked(reason="G4: capped", guard_id="G4") + return None + + monkeypatch.setattr("agent.sdlc_router.GUARDS", [blocks]) + result = reconcile_dispatch( + {}, {}, {}, Dispatch(skill=SKILL_DO_DOCS, reason="r", row_id="9") + ) + assert isinstance(result, Blocked) + assert result.guard_id == "G4" + + +class TestReconciliationPassesInputsByReference: + """STATED INVARIANT, not an accident: the guards are not pure and + reconciliation runs them twice. ``guard_g5_artifact_hash_cache`` mutates + ``record["artifact_hash"]`` in place and logs a WARNING on a legacy-hash + migration. Double invocation is idempotent ONLY because ``stage_states`` is + passed by reference and the second pass sees the already-migrated record. + A defensive copy anywhere between the passes re-runs the migration. + + ``test_sdlc_router.py::TestG5DoubleInvocationDuringReconciliation`` asserts + the resulting WARNING count. These two assert the mechanism underneath it. + """ + + def test_guards_receive_the_caller_s_own_objects(self, monkeypatch): + """Identity, not equality. A defensive copy would still compare equal + and would still break G5's in-place migration.""" + seen: list[tuple[int, int]] = [] + + def spy(stage_states, meta, context): + seen.append((id(stage_states), id(meta))) + return None + + monkeypatch.setattr("agent.sdlc_router.GUARDS", [spy]) + states, meta = _shipped_lane_states(), _shipped_lane_meta() + decide_next_dispatch(states, meta, {}) + assert len(seen) == 2, "one pre-table pass plus exactly one reconciliation pass" + assert seen[0] == seen[1] == (id(states), id(meta)) + + def test_a_second_g5_invocation_alone_emits_no_further_warning(self, caplog): + """Pins the idempotence the invariant relies on, without going through + the router at all.""" + states = _shipped_lane_states() + states["_verdicts"]["CRITIQUE"]["artifact_hash"] = "sha256:legacy" + meta = _shipped_lane_meta() + context = {"current_plan_hash": "sha256:new", "legacy_plan_hash": "sha256:legacy"} + + with caplog.at_level(logging.WARNING, logger="agent.sdlc_router"): + guard_g5_artifact_hash_cache(states, meta, context) + first = len([r for r in caplog.records if "G5 migration" in r.getMessage()]) + guard_g5_artifact_hash_cache(states, meta, context) + second = len([r for r in caplog.records if "G5 migration" in r.getMessage()]) + + assert first == 1, "the migration branch must actually have run" + assert second == 1, "the second invocation must see the already-migrated hash" + assert states["_verdicts"]["CRITIQUE"]["artifact_hash"] == "sha256:new" diff --git a/tools/sdlc_next_skill.py b/tools/sdlc_next_skill.py index 55ae0c20f..99b296c1b 100644 --- a/tools/sdlc_next_skill.py +++ b/tools/sdlc_next_skill.py @@ -601,7 +601,11 @@ def decide( is finished. No ``blocked`` key and no ``recorded`` claim: there is nothing to escalate and nothing to record. On ``Blocked``: ``{"blocked": True, "decision": "blocked", - "reason": "...", "guard_id": "..."}`` + "reason": "...", "guard_id": "..."}``, plus a ``"decision_inputs"`` + key (the ``stage_states``/``meta`` the router decided from, and on a + reconciliation double-veto the selected row and vetoing guard) when + the router populated one (#3065 Cluster D) -- notably on the + ``NO_RULE`` fallthrough and on a reconciliation double-veto. On issue-lock contention: ``{"blocked": True, "reason": "ISSUE_LOCKED", "guard_id": "ISSUE_LOCK", "owner_session_id": "...", "peek_identity": "caller" | "session_mirror" | "unresolved"}`` (and, only on the @@ -741,7 +745,7 @@ def decide( # this function has no write path, so a decision is never a # ledger advance. The caller records it via # ``sdlc-tool dispatch record`` before invoking the skill. - return { + dispatch_payload: dict = { "skill": result.skill, "reason": result.reason, "row_id": result.row_id, @@ -749,6 +753,14 @@ def decide( "recorded": False, "recorded_reason": NOT_RECORDED_REASON, } + # #3065 Cluster D: report a PREVIOUS dispatch that carries no + # confirming record. Distinct from ``recorded`` above, which is + # about THIS decision (always False -- ``decide`` never writes). + # Absent when the last dispatch is accounted for, so its mere + # presence is the signal. + if result.unrecorded_dispatch is not None: + dispatch_payload["unrecorded_dispatch"] = result.unrecorded_dispatch + return dispatch_payload elif isinstance(result, Terminal): # A finished lane is a SUCCESS, not an escalation (#2894, #2817). # Deliberately NOT folded into the blocked shape and carrying no @@ -763,12 +775,21 @@ def decide( "row_id": result.row_id, } elif isinstance(result, Blocked): - return { + payload = { "blocked": True, "decision": "blocked", "reason": result.reason, "guard_id": result.guard_id, } + # decision_inputs (#3065 Cluster D): the stage_states/meta (and, + # on a reconciliation double-veto, the selected row + vetoing + # guard) the router actually decided from. Optional — most + # Blocked instances (the numbered guards) already state their + # reason in prose and carry no decision_inputs — so this key is + # only added when the router populated it. + if result.decision_inputs is not None: + payload["decision_inputs"] = result.decision_inputs + return payload else: # Unexpected return type — treat as blocking error return { From 035119eb90d8cdd8d6f61d117dca57b6a9203f81 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 17:09:02 +0700 Subject: [PATCH 10/19] Resolve branch truth once, three-valued, for both router callers (Refs #3065) resolve_branch_truth answers found / absent / indeterminate from the PR head SHA (via tools/pr_head_resolver.resolve_pr_head_sha, never a bare gh read) matched against git ls-remote --heads origin. It replaces _check_branch_pushed, whose two-valued answer gave a wrong-but-present recorded slug, a genuinely unpushed branch, and an unreachable remote the same fail-closed /do-patch. G8 may fail closed on absent only; indeterminate is reported and deferred. An infra error in _verify_stage_artifacts now reports itself as indeterminate instead of being indistinguishable from a clean verification. The PATCH check's old 'no recorded PR number -> no-op' proxy is replaced by a MERGE-completed skip, since a lane with a PR whose head matches nothing is now indeterminate. agent/session_runner/runner.py stops deciding from an empty context: both decide_next_dispatch callers now assemble it through build_decision_context. --- agent/sdlc_router.py | 10 + agent/session_runner/runner.py | 10 +- tests/unit/test_lane_identity.py | 34 +- tests/unit/test_sdlc_next_skill.py | 507 +++++++++++++++++++++++++---- tools/sdlc_next_skill.py | 360 +++++++++++++++++--- 5 files changed, 797 insertions(+), 124 deletions(-) diff --git a/agent/sdlc_router.py b/agent/sdlc_router.py index e5daf2720..e82c7b326 100644 --- a/agent/sdlc_router.py +++ b/agent/sdlc_router.py @@ -762,6 +762,16 @@ def guard_g8_artifact_verification( explicit, verified mismatch). Absent/unset/``True`` is a no-op — this mirrors the context-assembly contract that a stage with no claimed artifact (or one that verified clean) never sets the flag to ``False``. + + **Branch truth is three-valued (#3065).** The PATCH arm's flag is set only + on ``resolve_branch_truth``'s *absent* verdict — no pushed branch holds + this lane's work and there is no PR to explain the gap. *Indeterminate* + (an unreadable remote, an unresolvable PR head, an ambiguous or mid-push + listing) sets ``context["branch_truth"]`` for reporting but never + ``stage_artifacts_verified``, so this guard steps aside. That asymmetry is + the whole point: dispatching a stage on an unreadable fact is what wedged + #2771 and #2334, and a wrong-but-present recorded slug used to be + indistinguishable here from a genuinely unpushed branch. """ if context.get("stage_artifacts_verified") is not False: return None diff --git a/agent/session_runner/runner.py b/agent/session_runner/runner.py index b3743b548..c6379cb20 100644 --- a/agent/session_runner/runner.py +++ b/agent/session_runner/runner.py @@ -1405,7 +1405,15 @@ def _load_ledger(self, issue_number: int) -> tuple[dict, dict, str | None, bool next_skill: str | None = None try: - decision = decide_next_dispatch(stage_states, meta) + # Same context builder the CLI path uses (#3065). Without it + # this call decided from a permanently empty context, so every + # context-fed guard (G3's proposed-skill arm, G5's plan-hash + # cache, G8's branch-truth artifact check) was inert here and + # the two paths could disagree on the same lane. + from tools.sdlc_next_skill import build_decision_context # noqa: PLC0415 + + context = build_decision_context(issue_number, stage_states, meta) + decision = decide_next_dispatch(stage_states, meta, context) next_skill = getattr(decision, "skill", None) except Exception as e: # noqa: BLE001 — nudge text only, never fatal logger.debug("[runner] next-skill for nudge failed: %s", e) diff --git a/tests/unit/test_lane_identity.py b/tests/unit/test_lane_identity.py index 76d7c837c..111110ea4 100644 --- a/tests/unit/test_lane_identity.py +++ b/tests/unit/test_lane_identity.py @@ -329,14 +329,12 @@ def test_g8_does_not_fire_when_lane_branch_diverges_from_plan_slug( ledger.slug = mint_lane_slug(_ISSUE_LANE) ledger.save(update_fields=["slug"]) - probed: list[str] = [] - - def fake_branch_pushed(name: str) -> bool: - # The world contains exactly one pushed branch for this lane. - probed.append(name) - return name.removeprefix("session/") == f"sdlc-{_ISSUE_LANE}" - - monkeypatch.setattr(sdlc_next_skill, "_check_branch_pushed", fake_branch_pushed) + # The world contains exactly one pushed branch for this lane. + monkeypatch.setattr( + sdlc_next_skill, + "_ls_remote_heads", + lambda: {f"refs/heads/session/sdlc-{_ISSUE_LANE}": "a" * 40}, + ) monkeypatch.setattr(sdlc_next_skill, "_check_plan_committed_on_main", lambda _: True) result = sdlc_next_skill._verify_stage_artifacts_live( @@ -345,10 +343,12 @@ def fake_branch_pushed(name: str) -> bool: _ISSUE_LANE, ) - assert result == {}, ( - f"G8 fired against a lane whose branch is pushed; probed {probed!r} " - f"instead of the recorded lane branch session/sdlc-{_ISSUE_LANE}" + assert result.get("stage_artifacts_verified") is not False, ( + "G8 fired against a lane whose branch is pushed: " + f"{result!r} (expected branch truth 'found' for session/sdlc-{_ISSUE_LANE})" ) + assert result["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_FOUND + assert result["branch_truth_branch"] == f"session/sdlc-{_ISSUE_LANE}" def test_g8_patch_check_noops_when_slug_unresolvable(self, tmp_path, monkeypatch): """With no recorded slug the PATCH check skips rather than guessing.""" @@ -361,13 +361,13 @@ def test_g8_patch_check_noops_when_slug_unresolvable(self, tmp_path, monkeypatch monkeypatch.setenv("SDLC_TARGET_REPO", str(repo_root)) monkeypatch.setenv("GH_REPO", _TEST_REPO) - probed: list[str] = [] + listed: list[bool] = [] - def fake_branch_pushed(name: str) -> bool: - probed.append(name) - return False + def fake_heads() -> dict[str, str]: + listed.append(True) + return {} - monkeypatch.setattr(sdlc_next_skill, "_check_branch_pushed", fake_branch_pushed) + monkeypatch.setattr(sdlc_next_skill, "_ls_remote_heads", fake_heads) monkeypatch.setattr(sdlc_next_skill, "_check_plan_committed_on_main", lambda _: True) result = sdlc_next_skill._verify_stage_artifacts_live( @@ -377,7 +377,7 @@ def fake_branch_pushed(name: str) -> bool: ) assert result == {} - assert probed == [], f"probed a guessed branch name: {probed!r}" + assert listed == [], "probed the remote for a lane with no recorded slug" # --------------------------------------------------------------------------- diff --git a/tests/unit/test_sdlc_next_skill.py b/tests/unit/test_sdlc_next_skill.py index 63218fa61..9081ea8ba 100644 --- a/tests/unit/test_sdlc_next_skill.py +++ b/tests/unit/test_sdlc_next_skill.py @@ -125,12 +125,15 @@ class TestBranchExistsCanonicalShape: """ @staticmethod - def _fake_git(stdout: str): + def _fake_git(*branches: str): + """Fake ``git ls-remote --heads origin`` listing the given branch names.""" + lines = "\n".join(f"{'a' * 40}\trefs/heads/{b}" for b in branches) + def _run(cmd, **kwargs): proc = MagicMock() - if cmd[:2] == ["git", "branch"]: + if cmd[:3] == ["git", "ls-remote", "--heads"]: proc.returncode = 0 - proc.stdout = stdout + proc.stdout = lines else: proc.returncode = 1 proc.stdout = "" @@ -144,7 +147,7 @@ def test_true_when_recorded_slug_branch_exists(self, monkeypatch): ) monkeypatch.setattr( "subprocess.run", - self._fake_git(" main\n session/my-feature-slug\n"), + self._fake_git("main", "session/my-feature-slug"), ) context = sdlc_next_skill._build_context(proposed_skill=None, issue_number=2003) @@ -161,7 +164,7 @@ def test_true_when_issue_derived_recorded_slug_branch_exists(self, monkeypatch): monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: "sdlc-2003") monkeypatch.setattr( "subprocess.run", - self._fake_git(" main\n session/sdlc-2003\n"), + self._fake_git("main", "session/sdlc-2003"), ) context = sdlc_next_skill._build_context(proposed_skill=None, issue_number=2003) @@ -169,7 +172,9 @@ def test_true_when_issue_derived_recorded_slug_branch_exists(self, monkeypatch): assert context["branch_exists"] is True def test_false_and_silent_when_no_slug_recorded(self, monkeypatch): - """No recorded slug -> cannot affirm existence -> False, and no probe.""" + """No recorded slug and no PR -> cannot affirm existence -> False, and + no live call (#3065: resolve_branch_truth short-circuits before any + subprocess when there is nothing to check).""" monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: None) run_mock = MagicMock() monkeypatch.setattr("subprocess.run", run_mock) @@ -178,14 +183,15 @@ def test_false_and_silent_when_no_slug_recorded(self, monkeypatch): assert context["branch_exists"] is False assert not any( - call.args and call.args[0][:2] == ["git", "branch"] for call in run_mock.call_args_list + call.args and call.args[0][:3] == ["git", "ls-remote", "--heads"] + for call in run_mock.call_args_list ) def test_false_when_branch_absent(self, monkeypatch): monkeypatch.setattr( "tools.lane_identity.resolve_lane_slug", lambda *a, **k: "my-feature-slug" ) - monkeypatch.setattr("subprocess.run", self._fake_git(" main\n")) + monkeypatch.setattr("subprocess.run", self._fake_git("main")) context = sdlc_next_skill._build_context(proposed_skill=None, issue_number=2003) @@ -284,15 +290,12 @@ def test_plan_committed_check_false_when_plan_absent_in_target(self, tmp_path, m assert sdlc_next_skill._check_plan_committed_on_main("docs/plans/no-such-slug.md") is False def test_branch_exists_probe_follows_target_repo(self, tmp_path, monkeypatch): - """_build_context's branch_exists probe reads the target's branches.""" + """_build_context's branch_exists probe (via resolve_branch_truth's + ``git ls-remote --heads origin``) runs with cwd pinned at the target + checkout, not the process cwd (#2078, #3065).""" target = tmp_path / "target" target.mkdir() self._init_fixture_repo(target, "sdlc-2078-fixture") - subprocess.run( - ["git", "-C", str(target), "branch", "session/sdlc-2078-fixture"], - check=True, - capture_output=True, - ) elsewhere = tmp_path / "elsewhere" elsewhere.mkdir() monkeypatch.chdir(elsewhere) @@ -301,9 +304,26 @@ def test_branch_exists_probe_follows_target_repo(self, tmp_path, monkeypatch): "tools.lane_identity.resolve_lane_slug", lambda *a, **k: "sdlc-2078-fixture" ) + calls = [] + + def _fake_run(cmd, **kwargs): + proc = MagicMock() + if cmd[:3] == ["git", "ls-remote", "--heads"]: + calls.append(kwargs.get("cwd")) + proc.returncode = 0 + sha = "a" * 40 + proc.stdout = f"{sha}\trefs/heads/session/sdlc-2078-fixture\n" + else: + proc.returncode = 1 + proc.stdout = "" + return proc + + monkeypatch.setattr("subprocess.run", _fake_run) + context = sdlc_next_skill._build_context(proposed_skill=None, issue_number=2078) assert context["branch_exists"] is True + assert calls == [str(target)], "ls-remote must run with cwd pinned at SDLC_TARGET_REPO" def test_decide_warm_cache_open_pr_defers_to_pr_review_not_plan(monkeypatch): @@ -957,24 +977,63 @@ def test_true_build_claim_leaves_context_unset_when_merged(self, monkeypatch): def test_patch_claim_skips_branch_check_when_pr_merged(self, monkeypatch, tmp_path): """#1267 g8 merged-pipeline misfire: PATCH claims completed, the PR - is MERGED, and the branch has already been deleted (delete-branch- - on-merge policy) -> still a no-op. The branch-pushed check must not - even run once the PR's live state proves MERGED.""" + is MERGED -> still a no-op, and ``resolve_branch_truth`` must not + even run (poisoned to explode if called) once the PR's live state + proves MERGED -- a deleted branch is the expected side effect of a + delete-branch-on-merge policy, not evidence of a fabricated claim. + + Calls ``_verify_stage_artifacts_live`` directly (not through + ``_build_context``) so this test is isolated from the UNRELATED + Row-5 ``branch_exists`` probe, which always resolves branch truth + once per tick regardless of PATCH's MERGED-skip (#3065).""" + monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: None) + monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: "my-slug") + monkeypatch.setattr("subprocess.run", self._fake_gh_pr_state("MERGED")) + + def _poison(*a, **k): + raise AssertionError("resolve_branch_truth must not run once PR is MERGED") + + monkeypatch.setattr(sdlc_next_skill, "resolve_branch_truth", _poison) + + stage_states = {"PATCH": "completed"} + meta = {"pr_number": 555} + + context = sdlc_next_skill._verify_stage_artifacts_live(stage_states, meta, 1267) + + assert "stage_artifacts_verified" not in context + assert "unverified_stage" not in context + + def test_patch_claim_found_via_pr_head_when_recorded_slug_is_wrong(self, monkeypatch, tmp_path): + """#3065 acceptance: a WRONG recorded lane slug with a live branch + under a DIFFERENT name must NOT dispatch /do-patch. The PR's head SHA + (git-first, never a bare `gh` read) uniquely matches a branch that + disagrees with the recorded slug -- resolve_branch_truth reports + ``found`` on the REAL branch, not a falsified PATCH claim. This is the + keystone regression: the old ``_check_branch_pushed`` probed only the + wrong-slug-derived name and reported it gone.""" plan_path = tmp_path / "my-slug.md" plan_path.write_text("---\nstatus: Ready\n---\n\n# Plan\n") monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: plan_path) + # The recorded slug names a branch that does not exist on origin -- + # the wrong-recorded-slug state this task fixes. + monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: "wrong-slug") - ls_remote_calls = [] + sha = "b" * 40 def _fake_run(cmd, **kwargs): proc = MagicMock() if cmd[:3] == ["gh", "pr", "view"]: proc.returncode = 0 - proc.stdout = json.dumps({"state": "MERGED"}) + proc.stdout = json.dumps({"state": "OPEN"}) + elif cmd[:3] == ["git", "ls-remote", "--heads"]: + proc.returncode = 0 + # The lane's REAL branch is named differently than the + # (wrong) recorded slug's derived name. + proc.stdout = f"{sha}\trefs/heads/session/dev-actual-branch\n" elif cmd[:2] == ["git", "ls-remote"]: - ls_remote_calls.append(cmd) + # refs/pull//head single-ref query inside resolve_pr_head_sha. proc.returncode = 0 - proc.stdout = "" # branch gone -- would fail if the check ran + proc.stdout = f"{sha}\trefs/pull/555/head\n" else: proc.returncode = 1 proc.stdout = "" @@ -994,12 +1053,15 @@ def _fake_run(cmd, **kwargs): assert "stage_artifacts_verified" not in context assert "unverified_stage" not in context - assert ls_remote_calls == [], "branch-pushed check must be skipped once PR is MERGED" - - def test_patch_claim_still_checks_branch_when_pr_open(self, monkeypatch, tmp_path): - """A PATCH claim against a still-OPEN PR (not yet merged) must still - run the real branch-pushed live check -- the MERGED skip is scoped - strictly to state == "MERGED", not to "PR exists".""" + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_FOUND + assert context["branch_truth_branch"] == "session/dev-actual-branch" + + def test_patch_claim_absent_when_pr_open_and_branch_genuinely_gone(self, monkeypatch, tmp_path): + """A PATCH claim against a still-OPEN PR whose head resolves to + nothing in the listing (ambiguous/mid-push territory) must defer + (indeterminate), not falsify -- see + test_patch_claim_with_no_pr_and_absent_branch_is_falsified below for + the one shape that DOES falsify (no PR at all).""" plan_path = tmp_path / "my-slug.md" plan_path.write_text("---\nstatus: Ready\n---\n\n# Plan\n") monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: plan_path) @@ -1012,7 +1074,7 @@ def _fake_run(cmd, **kwargs): proc.stdout = json.dumps({"state": "OPEN"}) elif cmd[:2] == ["git", "ls-remote"]: proc.returncode = 0 - proc.stdout = "" # branch gone -- should fail verification + proc.stdout = "" # nothing matches anywhere -- Race 1 else: proc.returncode = 1 proc.stdout = "" @@ -1030,8 +1092,13 @@ def _fake_run(cmd, **kwargs): meta=meta, ) - assert context["stage_artifacts_verified"] is False - assert context["unverified_stage"] == "PATCH" + # Race 1: a listing that does not contain the PR's head is a + # possibly-stale negative (mid-push), never an absence, while a PR + # is open. G8 must step aside rather than dispatch /do-patch on an + # unreadable fact. + assert "stage_artifacts_verified" not in context + assert "unverified_stage" not in context + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE def test_fails_open_on_infra_error(self, monkeypatch, caplog): """subprocess.TimeoutExpired/OSError from the gh/git call → advances @@ -1167,32 +1234,36 @@ def test_unverifiable_build_claim_makes_no_live_call(self, monkeypatch, meta, la assert "unverified_stage" not in context, label run_mock.assert_not_called() - def test_unverifiable_patch_claim_skips_the_branch_probe(self, monkeypatch, tmp_path): - """PATCH claims completed with a resolvable lane branch but no PR - number -> the `git ls-remote` branch probe never runs. - - The branch name alone is not enough to adjudicate this claim. With no - PR state to consult, `pr_state` stays None, the MERGED skip below it - cannot engage, and the entire verdict collapses onto "does the branch - still exist on origin" -- where "branch gone" is indistinguishable - from "deleted on merge". Probing anyway manufactures a PATCH mismatch - out of a successful merge. + def test_patch_claim_with_no_pr_and_absent_branch_is_falsified(self, monkeypatch, tmp_path): + """#3065: PATCH claims completed, no PR number is recorded, and the + recorded branch is genuinely absent from origin -> FALSIFIED (PATCH), + not "unverifiable". + + Before #3065, a missing PR number made this claim unverifiable + outright, because the old two-valued branch probe could not tell + "branch gone" from "deleted on merge" without a PR's state to + consult. ``resolve_branch_truth`` closes that ambiguity a different + way: a merge is impossible without a PR, so "no PR, branch absent" + can never be a deletion-on-merge false positive -- it can only mean + the branch was never pushed. This is the acceptance case for "a + genuinely unpushed branch STILL dispatches /do-patch". """ plan_path = tmp_path / "my-slug.md" plan_path.write_text("---\nstatus: Ready\n---\n\n# Plan\n") monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: plan_path) monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: "my-slug") - calls = [] - - def _record(cmd, **kwargs): - calls.append(list(cmd)) + def _fake_run(cmd, **kwargs): proc = MagicMock() - proc.returncode = 1 - proc.stdout = "" + if cmd[:3] == ["git", "ls-remote", "--heads"]: + proc.returncode = 0 + proc.stdout = "" # nothing on origin at all + else: + proc.returncode = 1 + proc.stdout = "" return proc - monkeypatch.setattr("subprocess.run", _record) + monkeypatch.setattr("subprocess.run", _fake_run) context = sdlc_next_skill._build_context( proposed_skill=None, @@ -1201,13 +1272,9 @@ def _record(cmd, **kwargs): meta={"pr_number": None}, ) - assert "stage_artifacts_verified" not in context - assert "unverified_stage" not in context - # `git branch -a` is _build_context's unrelated branch_exists signal and - # is expected. What must be absent is the verification gate's own live - # reads: the branch probe (`git ls-remote`) and any `gh` call. - assert [c for c in calls if c[:2] == ["git", "ls-remote"]] == [], calls - assert [c for c in calls if c[:1] == ["gh"]] == [], calls + assert context["stage_artifacts_verified"] is False + assert context["unverified_stage"] == "PATCH" + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_ABSENT def test_unverifiable_build_skip_logs_at_debug_not_warning(self, monkeypatch, caplog): """The skip is a normal, expected state and logs at DEBUG. @@ -1352,10 +1419,25 @@ def test_no_pr_number_skips_lookup_and_omits_key(self, monkeypatch): assert called == [] def test_no_recorded_review_verdict_skips_lookup(self, monkeypatch): - """No recorded verdict → no live call, key omitted (the router's - no-verdict recovery rows own that state; the signal stays inert).""" + """No recorded verdict → the WS3d head_sha verdict-staleness signal's + OWN lookup must not run, and the key stays omitted (the router's + no-verdict recovery rows own that state; the signal stays inert). + + Branch-truth resolution (#3065) independently calls + ``_fetch_pr_head_sha`` whenever a PR is recorded, regardless of + REVIEW verdicts -- ``resolve_branch_truth`` is stubbed out here so + that unrelated call site cannot pollute the ``called`` list this test + uses to prove the WS3d signal itself never fired. + """ called = [] monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: None) + monkeypatch.setattr( + sdlc_next_skill, + "resolve_branch_truth", + lambda *a, **k: sdlc_next_skill.BranchTruth( + status=sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE, reason="stubbed" + ), + ) monkeypatch.setattr( sdlc_next_skill, "_fetch_pr_head_sha", @@ -1978,3 +2060,316 @@ def test_dispatch_payload_omits_the_signal_when_the_record_confirms(self, monkey assert result["decision"] == "dispatch" assert "unrecorded_dispatch" not in result + + +# --------------------------------------------------------------------------- +# #3065 Task 5 -- branch truth is three-valued and shared by both router callers +# --------------------------------------------------------------------------- + +_SHA_A = "a" * 40 +_SHA_B = "b" * 40 + + +class TestResolveBranchTruth: + """``resolve_branch_truth`` answers found / absent / indeterminate. + + The two-valued ``_check_branch_pushed`` it replaces gave the same answer — + and the same fail-closed ``/do-patch`` dispatch — to a wrong-but-present + recorded slug, a genuinely unpushed branch, and an unreachable remote. + """ + + def _heads(self, monkeypatch, heads): + monkeypatch.setattr(sdlc_next_skill, "_ls_remote_heads", lambda: heads) + + def _head_sha(self, monkeypatch, sha): + monkeypatch.setattr(sdlc_next_skill, "_fetch_pr_head_sha", lambda pr, repo=None: sha) + + def test_pr_head_uniquely_matching_a_head_is_found(self, monkeypatch): + self._heads( + monkeypatch, + {"refs/heads/session/real-name": _SHA_A, "refs/heads/main": _SHA_B}, + ) + self._head_sha(monkeypatch, _SHA_A) + + truth = sdlc_next_skill.resolve_branch_truth("session/wrong-slug", pr_number=7) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_FOUND + # The branch is an OUTPUT of the SHA match, not the name we asked about. + assert truth.branch == "session/real-name" + + def test_two_or_more_matches_are_indeterminate(self, monkeypatch): + self._heads( + monkeypatch, + {"refs/heads/session/a": _SHA_A, "refs/heads/session/b": _SHA_A}, + ) + self._head_sha(monkeypatch, _SHA_A) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=7) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + assert "ambiguous" in truth.reason + + def test_unreachable_remote_is_indeterminate(self, monkeypatch): + self._heads(monkeypatch, None) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=7) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + assert "unreachable" in truth.reason + + def test_ls_remote_raising_is_indeterminate(self, monkeypatch): + def _boom(): + raise subprocess.TimeoutExpired(cmd="git ls-remote", timeout=10) + + monkeypatch.setattr(sdlc_next_skill, "_ls_remote_heads", _boom) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=7) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + + def test_listing_without_the_pr_head_is_indeterminate_not_absent(self, monkeypatch): + """Race 1: a mid-push (or merged-and-deleted) listing is a stale negative.""" + self._heads(monkeypatch, {"refs/heads/main": _SHA_B}) + self._head_sha(monkeypatch, _SHA_A) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=7) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + + def test_unresolvable_pr_head_is_indeterminate(self, monkeypatch): + self._heads(monkeypatch, {"refs/heads/session/a": _SHA_A}) + self._head_sha(monkeypatch, None) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=7) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + + def test_no_pr_and_branch_missing_is_absent(self, monkeypatch): + self._heads(monkeypatch, {"refs/heads/main": _SHA_B}) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=None) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_ABSENT + + def test_no_pr_and_branch_present_is_found(self, monkeypatch): + self._heads(monkeypatch, {"refs/heads/session/a": _SHA_A}) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=None) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_FOUND + assert truth.branch == "session/a" + + def test_no_branch_and_no_pr_is_indeterminate_never_absent(self, monkeypatch): + """Nothing to ask about is not evidence of absence, and guessing is #2718.""" + self._heads(monkeypatch, {"refs/heads/session/a": _SHA_A}) + + truth = sdlc_next_skill.resolve_branch_truth(None, pr_number=None) + + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + + def test_the_pr_head_is_resolved_through_the_authoritative_resolver(self, monkeypatch): + """Structural: the SHA comes from ``pr_head_resolver``, never a bare ``gh`` read. + + A stale ``gh`` head SHA is what flipped the verdict-staleness gate + fail-open in #2895, so this asserts the call actually routes through + ``resolve_pr_head_sha``. + """ + seen: list[int] = [] + + def _fake(pr_number, repo=None, repo_root=None, **kwargs): + seen.append(pr_number) + return _SHA_A + + monkeypatch.setattr("tools.pr_head_resolver.resolve_pr_head_sha", _fake) + self._heads(monkeypatch, {"refs/heads/session/a": _SHA_A}) + + truth = sdlc_next_skill.resolve_branch_truth("session/a", pr_number=99) + + assert seen == [99] + assert truth.status == sdlc_next_skill.BRANCH_TRUTH_FOUND + + +class TestG8ConsumesBranchTruth: + """G8 may fail closed on *absent* only; *indeterminate* makes it step aside.""" + + def _setup(self, monkeypatch, *, slug="sdlc-3065", heads, head_sha=None): + monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: None) + monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: slug) + monkeypatch.setattr(sdlc_next_skill, "_ls_remote_heads", lambda: heads) + monkeypatch.setattr(sdlc_next_skill, "_fetch_pr_head_sha", lambda pr, repo=None: head_sha) + monkeypatch.setattr(sdlc_next_skill, "_fetch_pr_state", lambda pr, repo=None: "OPEN") + + def _context(self, monkeypatch, stage_states, meta): + return sdlc_next_skill._build_context(None, 3065, stage_states, meta) + + def test_wrong_recorded_slug_with_a_live_branch_does_not_dispatch_patch(self, monkeypatch): + """The demonstrated red: pre-#3065 this force-dispatched ``/do-patch``. + + The recorded slug is stale (``sdlc-3065``) but the lane's work really + is pushed, on ``session/renamed-lane``, and the PR head proves it. + """ + from agent.sdlc_router import guard_g8_artifact_verification + + self._setup( + monkeypatch, + heads={"refs/heads/session/renamed-lane": _SHA_A}, + head_sha=_SHA_A, + ) + context = self._context( + monkeypatch, {"PATCH": "completed"}, {"pr_number": 41, "_resolved_target_repo": "o/r"} + ) + + assert context.get("stage_artifacts_verified") is not False + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_FOUND + assert guard_g8_artifact_verification({"PATCH": "completed"}, {}, context) is None + + def test_genuinely_unpushed_branch_still_dispatches_patch(self, monkeypatch): + from agent.sdlc_router import guard_g8_artifact_verification + + self._setup(monkeypatch, heads={"refs/heads/main": _SHA_B}) + context = self._context(monkeypatch, {"PATCH": "completed"}, {}) + + assert context["stage_artifacts_verified"] is False + assert context["unverified_stage"] == "PATCH" + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_ABSENT + + decision = guard_g8_artifact_verification({"PATCH": "completed"}, {}, context) + assert decision is not None + assert decision.skill == "/do-patch" + + def test_ambiguous_branch_truth_defers(self, monkeypatch): + from agent.sdlc_router import guard_g8_artifact_verification + + self._setup( + monkeypatch, + heads={"refs/heads/session/a": _SHA_A, "refs/heads/session/b": _SHA_A}, + head_sha=_SHA_A, + ) + context = self._context( + monkeypatch, {"PATCH": "completed"}, {"pr_number": 41, "_resolved_target_repo": "o/r"} + ) + + assert context.get("stage_artifacts_verified") is not False + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + assert context["branch_truth_reason"] + assert guard_g8_artifact_verification({"PATCH": "completed"}, {}, context) is None + + def test_unreachable_remote_defers(self, monkeypatch): + from agent.sdlc_router import guard_g8_artifact_verification + + self._setup(monkeypatch, heads=None) + context = self._context(monkeypatch, {"PATCH": "completed"}, {}) + + assert context.get("stage_artifacts_verified") is not False + assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_INDETERMINATE + assert guard_g8_artifact_verification({"PATCH": "completed"}, {}, context) is None + + def test_infra_error_is_reported_as_indeterminate_not_as_a_clean_pass(self, monkeypatch): + """The fail-open direction is right; the silence was not.""" + + def _boom(stage_states, meta, issue_number, branch_truth=None): + raise subprocess.TimeoutExpired(cmd="gh pr view", timeout=10) + + monkeypatch.setattr(sdlc_next_skill, "_verify_stage_artifacts_live", _boom) + + result = sdlc_next_skill._verify_stage_artifacts({"PATCH": "completed"}, {}, 3065) + + assert result["artifact_verification_indeterminate"] is True + assert "TimeoutExpired" in result["artifact_verification_reason"] + # Still fail-open for routing: G8 reads neither key. + assert "stage_artifacts_verified" not in result + + +class TestCliAndInProcessPathsAgree: + """Both ``decide_next_dispatch`` callers assemble context the same way. + + ``agent/session_runner/runner.py`` used to pass no context at all, so every + context-fed guard was permanently inert there and the in-process answer + could differ from the CLI's on the same lane (#3065 Cluster A). + """ + + def _lane(self): + stage_states = { + "ISSUE": STATUS_COMPLETED, + "PLAN": STATUS_COMPLETED, + "CRITIQUE": STATUS_COMPLETED, + "BUILD": STATUS_COMPLETED, + "TEST": STATUS_COMPLETED, + "PATCH": STATUS_COMPLETED, + } + return stage_states, {} + + def _patch_world(self, monkeypatch, stage_states, meta): + monkeypatch.setattr("tools.lane_identity.find_plan_path", lambda issue_number: None) + monkeypatch.setattr("tools.lane_identity.resolve_lane_slug", lambda *a, **k: "sdlc-30651") + # A genuinely unpushed lane branch: readable remote, no matching head, + # no PR. Branch truth is *absent*, so G8 must fail closed on BOTH paths. + monkeypatch.setattr( + sdlc_next_skill, "_ls_remote_heads", lambda: {"refs/heads/main": _SHA_B} + ) + monkeypatch.setattr( + "tools.sdlc_stage_query.query_enriched", + lambda **kwargs: {"stages": dict(stage_states), "_meta": dict(meta)}, + ) + + def test_both_paths_reach_the_same_dispatch(self, monkeypatch): + from agent.sdlc_router import decide_next_dispatch + + stage_states, meta = self._lane() + self._patch_world(monkeypatch, stage_states, meta) + + cli_context = sdlc_next_skill.build_decision_context(30651, dict(stage_states), dict(meta)) + cli_decision = decide_next_dispatch(dict(stage_states), dict(meta), cli_context) + + runner = self._make_runner() + _, _, in_process_skill, _, ok = runner._load_ledger(30651) + + assert ok is True + assert getattr(cli_decision, "skill", None) == "/do-patch" + assert in_process_skill == cli_decision.skill + + def test_in_process_path_without_the_shared_builder_would_disagree(self, monkeypatch): + """Pins WHY the two paths agree: the empty context reaches a different answer.""" + from agent.sdlc_router import decide_next_dispatch + + stage_states, meta = self._lane() + self._patch_world(monkeypatch, stage_states, meta) + + with_context = decide_next_dispatch( + dict(stage_states), + dict(meta), + sdlc_next_skill.build_decision_context(30651, dict(stage_states), dict(meta)), + ) + without_context = decide_next_dispatch(dict(stage_states), dict(meta)) + + assert getattr(with_context, "skill", None) == "/do-patch" + assert getattr(without_context, "skill", None) != "/do-patch" + + @staticmethod + def _make_runner(): + from agent.session_runner.adapter import SessionRunnerAdapter + from agent.session_runner.runner import SessionRunner + + class _Session: + session_id = "sess-3065-agree" + chat_id = 1 + telegram_message_id = 2 + session_events = None + issue_number = 30651 + session_type = "eng" + + def save(self, update_fields=None): + pass + + session = _Session() + adapter = SessionRunnerAdapter( + session, "test-proj", "telegram", resolve_callbacks=lambda pk, t: (None, None) + ) + return SessionRunner( + agent_session=session, + adapter=adapter, + working_dir="/tmp/wd", + session_type="eng", + driver=None, + steering_pop_fn=lambda: [], + ) diff --git a/tools/sdlc_next_skill.py b/tools/sdlc_next_skill.py index 99b296c1b..5bd45a14a 100644 --- a/tools/sdlc_next_skill.py +++ b/tools/sdlc_next_skill.py @@ -81,6 +81,7 @@ import os import subprocess import sys +from dataclasses import dataclass from pathlib import Path logger = logging.getLogger(__name__) @@ -178,24 +179,198 @@ def _fetch_pr_head_sha(pr_number: int, repo: str | None = None) -> str | None: return resolve_pr_head_sha(pr_number, repo=repo, repo_root=_target_repo_cwd()) -def _check_branch_pushed(branch_name: str) -> bool: - """Live-check (``git ls-remote``) that ``branch_name`` exists on origin. +def _ls_remote_heads() -> dict[str, str] | None: + """Return ``{refname: sha}`` for every head on ``origin``, ``None`` on failure. - Takes a FULL branch name, not a slug: the ``session/`` prefix is applied - by ``tools.lane_identity.lane_branch_name`` and nowhere else, so a caller - that has no lane slug gets ``None`` and no-ops instead of probing a name - it guessed. + ``None`` is deliberately distinct from ``{}``: an unreachable remote is not + an empty remote. Collapsing the two is what let a network blip read as + "this lane has no branch" (#3065 Cluster A). Callers turn ``None`` into + *indeterminate*. - Unlike the local ``git branch -a`` check elsewhere in this module (which - can be satisfied by a stale remote-tracking ref), this queries the - remote directly so a claimed "branch pushed" artifact is verified - against the live world, not local ref cache staleness. + Unlike a local ``git branch -a`` read (which a stale remote-tracking ref + can satisfy), this queries the remote directly, so branch truth is checked + against the live world rather than local ref-cache staleness. """ - cmd = ["git", "ls-remote", "--heads", "origin", branch_name] + cmd = ["git", "ls-remote", "--heads", "origin"] proc = subprocess.run(cmd, cwd=_target_repo_cwd(), capture_output=True, text=True, timeout=10) if proc.returncode != 0: - return False - return bool(proc.stdout.strip()) + logger.debug("branch-truth: git ls-remote --heads origin returned %s", proc.returncode) + return None + heads: dict[str, str] = {} + for line in (proc.stdout or "").splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[1].startswith("refs/heads/"): + heads[parts[1].strip()] = parts[0].strip() + return heads + + +# Branch-truth verdicts (#3065 Cluster A). Three values, because the two-valued +# answer ``_check_branch_pushed`` used to give could not tell "this lane has no +# pushed branch" from "the name I asked about is the wrong name" or "I could not +# read the remote" -- and all three got the same fail-closed consequence. +BRANCH_TRUTH_FOUND = "found" +BRANCH_TRUTH_ABSENT = "absent" +BRANCH_TRUTH_INDETERMINATE = "indeterminate" + + +@dataclass(frozen=True) +class BranchTruth: + """The answer to "which pushed branch holds this lane's work?". + + ``status`` is one of :data:`BRANCH_TRUTH_FOUND`, :data:`BRANCH_TRUTH_ABSENT`, + :data:`BRANCH_TRUTH_INDETERMINATE`. ``branch`` names the branch that holds + the work on *found* (which may differ from the branch the caller asked + about — that difference IS a wrong recorded slug, and is what + ``tools.lane_identity.repair_lane_slug`` acts on). ``reason`` is always + populated so an indeterminate answer is reportable rather than silent. + """ + + status: str + branch: str | None = None + reason: str = "" + matches: tuple[str, ...] = () + + @property + def is_found(self) -> bool: + return self.status == BRANCH_TRUTH_FOUND + + @property + def is_absent(self) -> bool: + return self.status == BRANCH_TRUTH_ABSENT + + @property + def is_indeterminate(self) -> bool: + return self.status == BRANCH_TRUTH_INDETERMINATE + + +_UNSET = object() + + +def resolve_branch_truth( + lane_branch: str | None, + pr_number: object = None, + repo: str | None = None, + heads: object = _UNSET, +) -> BranchTruth: + """Resolve which pushed branch holds this lane's work. Three-valued. + + Ground truth is the PR's head commit SHA — resolved through + ``tools.pr_head_resolver.resolve_pr_head_sha`` (git-first via ``git + ls-remote refs/pull/N/head``), **never a bare ``gh`` read**, per CLAUDE.md: + a stale ``gh`` head SHA is what flipped the verdict-staleness gate + fail-open in #2895 — matched against the ``git ls-remote --heads origin`` + listing. The branch name is an output of that match, not an input to it, + which is why a wrong recorded slug can no longer produce a wrong answer. + + Verdicts: + + - **found** — exactly one head carries the PR's head SHA (or, on a lane + with no PR, ``lane_branch`` is in the listing). ``branch`` names it. + - **absent** — the lane has **no PR** and its recorded branch is not in a + successfully-read listing. This is the only verdict a fail-closed + decision may act on. + - **indeterminate** — the remote could not be read, the PR head could not + be resolved, the head matches two or more heads, or the head matches + none of them. The last case is Race 1: a listing taken mid-push does not + yet contain the head, and a stale *negative* is the dangerous direction, + so it defers rather than claiming absence. ``lane_branch`` being + ``None`` is also indeterminate — there is no name to ask about, and + guessing one is #2718. + + This function makes no decision and writes nothing; it only reports what + the world says. + + Makes ZERO live calls when there is nothing to check at all -- no PR and + no recorded branch name. That case is answered (*indeterminate*) before + any subprocess runs, so a lane with no claimable artifact never pays for + a live probe (mirrors the #2757 "unverifiable costs no call" contract). + + ``heads`` lets a caller that already fetched the listing this tick (e.g. + ``_build_context``, which shares one resolution across ``branch_exists`` + and the G8 artifact check) inject it instead of paying for a second + ``git ls-remote --heads origin`` round trip. Omitted (the default), this + function fetches its own listing. + """ + lane_branch = (lane_branch or "").strip() or None + has_pr = isinstance(pr_number, int) and pr_number >= 1 + + if not has_pr and lane_branch is None: + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason="no lane branch recorded and no PR to resolve a head from", + ) + + if heads is _UNSET: + try: + heads = _ls_remote_heads() + except Exception as e: + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason=f"git ls-remote --heads origin failed ({type(e).__name__}: {e})", + ) + if heads is None: + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason="git ls-remote --heads origin was unreadable (remote unreachable)", + ) + + if has_pr: + try: + head_sha = _fetch_pr_head_sha(int(pr_number), repo=repo) + except Exception as e: + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason=f"PR #{pr_number} head SHA resolution failed ({type(e).__name__}: {e})", + ) + if not head_sha: + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason=f"PR #{pr_number} head SHA did not resolve", + ) + matches = tuple( + sorted(ref.removeprefix("refs/heads/") for ref, sha in heads.items() if sha == head_sha) + ) + if len(matches) == 1: + return BranchTruth( + status=BRANCH_TRUTH_FOUND, + branch=matches[0], + reason=f"PR #{pr_number} head {head_sha} uniquely matches {matches[0]}", + matches=matches, + ) + if matches: + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason=( + f"PR #{pr_number} head {head_sha} matches {len(matches)} heads " + f"({', '.join(matches)}) -- ambiguous" + ), + matches=matches, + ) + # Race 1: a listing that does not contain the head is a possibly-stale + # negative (mid-push, or merged-and-deleted). Never *absent* while a PR + # exists. + return BranchTruth( + status=BRANCH_TRUTH_INDETERMINATE, + reason=( + f"PR #{pr_number} head {head_sha} matches no head in the listing " + f"(mid-push or merged-and-deleted)" + ), + ) + + # has_pr is False here, and the (not has_pr and lane_branch is None) case + # already returned above without a live call -- lane_branch is guaranteed + # non-None below. + if f"refs/heads/{lane_branch}" in heads: + return BranchTruth( + status=BRANCH_TRUTH_FOUND, + branch=lane_branch, + reason=f"{lane_branch} is present on origin", + matches=(lane_branch,), + ) + return BranchTruth( + status=BRANCH_TRUTH_ABSENT, + reason=f"{lane_branch} is not on origin and this lane has no PR", + ) def _check_plan_committed_on_main(rel_plan_path: str) -> bool: @@ -215,7 +390,12 @@ def _check_plan_committed_on_main(rel_plan_path: str) -> bool: return proc.returncode == 0 -def _verify_stage_artifacts_live(stage_states: dict, meta: dict, issue_number: int) -> dict: +def _verify_stage_artifacts_live( + stage_states: dict, + meta: dict, + issue_number: int, + heads: object = _UNSET, +) -> dict: """Check the top-3 claimed stage artifacts against the live world. Only checks a stage whose marker actually claims completion -- a stage @@ -316,32 +496,40 @@ def _verify_stage_artifacts_live(stage_states: dict, meta: dict, issue_number: i # No recorded lane slug -> no branch to probe -> the PATCH check no-ops. # Probing a guessed name is what force-dispatches `/do-patch` against a # clean worktree until the G4 oscillation cap hard-blocks the lane (#2718). - # No recorded PR number is a second, independent reason to no-op (#2757): - # `pr_state` then stays None, the merged-skip below cannot engage, and the - # whole verdict falls to `git ls-remote` -- where "branch gone" is - # indistinguishable from "deleted on merge" with no PR state left to consult. - patch_claimed = patch_marked and bool(lane_branch) and pr_identifiable + # + # A lane whose MERGE is recorded completed is skipped outright (#2757): a + # missing remote branch is then the expected side effect of a + # delete-branch-on-merge policy, not evidence of a fabricated PATCH claim. + # This replaces the old "no recorded PR number -> no-op" proxy. That proxy + # existed only because the two-valued probe could not tell + # deletion-on-merge from a genuinely unpushed branch; + # :func:`resolve_branch_truth` now can -- a lane WITH a PR whose head + # matches nothing in the listing is *indeterminate*, never *absent* -- so + # the only lane that can still fail closed here is one with no PR and no + # merge, which is exactly the lane `/do-patch` exists for. + merge_recorded = stage_states.get("MERGE") == "completed" + patch_claimed = patch_marked and bool(lane_branch) and not merge_recorded if patch_marked and not lane_branch: logger.debug( "stage-artifact-verify: issue #%s PATCH claims completed but no lane slug is " "recorded; skipping the branch probe rather than guessing a branch name", issue_number, ) - elif patch_marked and not pr_identifiable: + elif patch_marked and merge_recorded: logger.debug( - "stage-artifact-verify: issue #%s PATCH claims completed but no PR number is " - "recorded; skipping the branch probe because a missing branch is " - "indistinguishable from deletion-on-merge with no PR state to consult", + "stage-artifact-verify: issue #%s PATCH claims completed on a lane whose MERGE " + "is recorded completed; skipping the branch probe because a deleted branch is " + "the expected side effect of merging", issue_number, ) # Resolve the live PR state at most once (used by both checks below) -- # only when a claim that needs it is actually present, so an unclaimed # BUILD/PATCH stage still makes zero live calls (test_no_claimed_artifact_is_a_noop). - # Both claims already require `pr_identifiable`, so a claim being present is - # itself proof that `pr_number` is truthy -- no separate check needed. + # `patch_claimed` no longer implies a recorded PR number, so the truthiness + # of `pr_number` is checked explicitly rather than inferred. pr_state: str | None = None - if build_claimed or patch_claimed: + if (build_claimed or patch_claimed) and pr_identifiable: pr_state = _fetch_pr_state(pr_number, repo=repo) if build_claimed: @@ -352,18 +540,45 @@ def _verify_stage_artifacts_live(stage_states: dict, meta: dict, issue_number: i ) return {"stage_artifacts_verified": False, "unverified_stage": "BUILD"} - if patch_claimed: - if pr_state != "MERGED" and not _check_branch_pushed(lane_branch): + if patch_claimed and pr_state != "MERGED": + truth = resolve_branch_truth(lane_branch, pr_number=pr_number, repo=repo, heads=heads) + if truth.is_absent: logger.warning( f"stage-artifact-verify: issue #{issue_number} PATCH claims completed " - f"but branch {lane_branch} is not pushed" + f"but no pushed branch holds its work ({truth.reason})" ) - return {"stage_artifacts_verified": False, "unverified_stage": "PATCH"} + return { + "stage_artifacts_verified": False, + "unverified_stage": "PATCH", + "branch_truth": truth.status, + "branch_truth_reason": truth.reason, + } + if truth.is_indeterminate: + # Report it AS indeterminate rather than as a silent clean pass: + # G8 must step aside here, but a supervisor reading the context has + # to be able to tell "verified" from "unreadable" (#3065). + logger.info( + f"stage-artifact-verify: issue #{issue_number} PATCH branch truth is " + f"indeterminate ({truth.reason}) — deferring rather than dispatching /do-patch" + ) + return { + "branch_truth": truth.status, + "branch_truth_reason": truth.reason, + } + return { + "branch_truth": truth.status, + "branch_truth_branch": truth.branch, + } return {} -def _verify_stage_artifacts(stage_states: dict, meta: dict, issue_number: int | None) -> dict: +def _verify_stage_artifacts( + stage_states: dict, + meta: dict, + issue_number: int | None, + heads: object = _UNSET, +) -> dict: """Verify claimed stage-completion artifacts against the live world (#1267). Sets ``stage_artifacts_verified`` / ``unverified_stage`` in the returned @@ -389,13 +604,20 @@ def _verify_stage_artifacts(stage_states: dict, meta: dict, issue_number: int | if not issue_number: return {} try: - return _verify_stage_artifacts_live(stage_states, meta, issue_number) + return _verify_stage_artifacts_live(stage_states, meta, issue_number, heads) except _INFRA_ERRORS as e: logger.warning( f"stage-artifact-verify: infra error verifying issue #{issue_number} " f"artifacts ({type(e).__name__}: {e}) — failing open (advancing)" ) - return {} + # The direction is right (advance), but silence is not: an infra + # failure used to be indistinguishable in the output from a genuine + # clean verification. Report it AS indeterminate (#3065) — the flags + # G8 reads are still unset, so nothing dispatches on it. + return { + "artifact_verification_indeterminate": True, + "artifact_verification_reason": f"{type(e).__name__}: {e}", + } except Exception: logger.error( f"stage-artifact-verify: unexpected (non-infra) error verifying issue " @@ -467,12 +689,17 @@ def _build_context( e, ) - # Check whether the lane's branch already exists (informs Row 5). - # The branch is named by the lane's RECORDED slug, which the lane minted - # once at lane start -- both the issue-derived shape and human-named - # shapes occur on this remote, so the name is read, never derived. Without - # a recorded slug we cannot affirm existence, so branch_exists stays False - # (#2003) and no git call is made at all. + # Branch truth, resolved ONCE per tick and shared by both consumers + # (#3065). ``branch_exists`` (Row 5) and the G8 PATCH artifact check used + # to ask two different questions of two different sources -- a local + # ``git branch -a`` read that a stale remote-tracking ref satisfies, and a + # live single-ref probe of a name derived from the recorded slug. One + # resolver, one live listing, one answer. + # + # ``branch_exists`` is True only on *found*: an unreadable remote must not + # assert existence. That matches the pre-#3065 behavior for the failure + # case while removing the stale-local-ref false positive. + heads: object = _UNSET if issue_number: context["branch_exists"] = False try: @@ -480,24 +707,24 @@ def _build_context( lane_branch = lane_branch_name(resolve_lane_slug(issue_number)) if lane_branch is not None: - proc2 = subprocess.run( - ["git", "branch", "-a"], - cwd=_target_repo_cwd(), - capture_output=True, - text=True, - timeout=5, + heads = _ls_remote_heads() + context["branch_exists"] = ( + heads is not None and f"refs/heads/{lane_branch}" in heads ) - branch_names = proc2.stdout if proc2.returncode == 0 else "" - context["branch_exists"] = lane_branch in branch_names - except Exception: + except Exception as e: + logger.debug("next-skill: branch-truth resolution failed (%s: %s)", type(e).__name__, e) context["branch_exists"] = False + heads = _UNSET # Stage-advance outcome verification gate (#1267): verify claimed # stage-completion artifacts against the live world. No-op when # stage_states/meta were not supplied (see the docstring above) or when - # no stage claims a checkable artifact this tick. + # no stage claims a checkable artifact this tick. The listing above is + # handed down so the whole tick costs one `git ls-remote --heads origin`; + # the PR-head resolve inside `resolve_branch_truth` stays lazy and runs + # only when a PATCH claim actually needs adjudicating. if issue_number and stage_states is not None and meta is not None: - context.update(_verify_stage_artifacts(stage_states, meta, issue_number)) + context.update(_verify_stage_artifacts(stage_states, meta, issue_number, heads)) # Head_sha verdict-staleness signal (WS3d, issue #2062): when a PR exists # AND a REVIEW verdict is recorded, fetch the live PR head so the router @@ -533,6 +760,39 @@ def _build_context( return context +def build_decision_context( + issue_number: int | None, + stage_states: dict | None = None, + meta: dict | None = None, + proposed_skill: str | None = None, +) -> dict: + """Public router-context builder, shared by BOTH ``decide_next_dispatch`` callers. + + The CLI path (:func:`decide`) and the in-process path + (``agent/session_runner/runner.py::_load_ledger``) used to disagree by + construction: the runner passed no context at all, so every context-fed + guard — G3's proposed-skill arm, G5's plan-hash cache, G8's artifact + verification — saw permanently empty inputs there and could reach a + different answer than the CLI on the same lane. This function is the one + place that assembles those facts, so "the two paths agree" is a property of + the code rather than something a test has to keep chasing. + + Never raises: every fact-gathering step inside is individually guarded, and + a total failure yields a partial (or empty) context, which is the same + fail-open shape both callers already had. + """ + try: + return _build_context(proposed_skill, issue_number, stage_states, meta) + except Exception as e: + logger.debug( + "build_decision_context failed for issue #%s (%s: %s)", + issue_number, + type(e).__name__, + e, + ) + return {} + + def _recover_stage_states_from_durable_signals(issue_number: int) -> dict: """Best-effort, read-only fallback: reconstruct stage_states from durable artifacts (committed plan, open/merged PR, review comments) when the From bdf72b7e4f833fe60b1e4f2cc718e81d28ef5336 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 17:15:55 +0700 Subject: [PATCH 11/19] Make a wrong recorded lane slug repairable, on unique evidence only (Refs #3065) repair_lane_slug corrects a recorded slug that branch truth contradicts. It fires only where a fail-closed decision is about to be taken on the recorded name, and only on a UNIQUE git ls-remote --heads origin match against the PR head SHA resolved through tools/pr_head_resolver.resolve_pr_head_sha. Zero and two-or-more matches leave the record alone, matching _adopt_from_pr's ambiguity discipline; one matcher (_match_pr_head_to_lane_branches) now serves both so that discipline cannot drift between them. _record_slug_if_empty is deliberately not reused: its no-overwrite behavior IS the defect. The repair re-reads the recorded value under the slug lock immediately before writing, so a concurrent repair converges to a no-op rather than a second write (Race 2), and a record that moved to a third value is left untouched. Every correction files its justification on the ledger. Rung 1 is unchanged: ordinary reads still return the recorded slug. The module docstring's 'a wrong adoption could never be corrected' claim is replaced. --- tests/unit/test_lane_identity.py | 197 +++++++++++++++++++++ tools/lane_identity.py | 286 ++++++++++++++++++++++++++++--- 2 files changed, 463 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_lane_identity.py b/tests/unit/test_lane_identity.py index 111110ea4..152b08026 100644 --- a/tests/unit/test_lane_identity.py +++ b/tests/unit/test_lane_identity.py @@ -38,6 +38,7 @@ _ISSUE_RESOLVER = 927353 # scratch issue for resolver-contract tests _ISSUE_ADOPT = 927354 # scratch issue for adopt_lane_slug tests _ISSUE_META = 927355 # scratch issue for the stage-query `_meta` slug read +_ISSUE_REPAIR = 927356 # scratch issue for the evidence-gated slug repair def _cleanup(*issue_numbers: int, target_repo: str = _TEST_REPO) -> None: @@ -56,6 +57,7 @@ def clean_ledgers(): _ISSUE_RESOLVER, _ISSUE_ADOPT, _ISSUE_META, + _ISSUE_REPAIR, ) _cleanup(*issues) yield @@ -729,3 +731,198 @@ def test_issue_number_from_message_replaces_it(self): assert _issue_number_from_message("Start the pipeline for issue 735") == 735 assert _issue_number_from_message("do something generic") is None assert _issue_number_from_message("") is None + + +# --------------------------------------------------------------------------- +# #3065 Task 6 -- a wrong recorded lane slug is repairable +# --------------------------------------------------------------------------- + + +class TestRepairLaneSlug: + """``repair_lane_slug`` corrects a contradicted slug, on unique evidence only. + + Demonstrated red (#2658): on main no code path can correct a wrong recorded + slug. Rung 1 returns it unconditionally, ``allow_heal`` only fills an + *empty* field, and both write paths are no-overwrite — the module docstring + conceded it. A lane mislabelled once stayed mislabelled forever, and G8 + force-dispatched ``/do-patch`` on the wrong branch name until the G4 + oscillation cap hard-blocked it. + """ + + _HEAD = "c" * 40 + _OTHER = "d" * 40 + + @pytest.fixture + def ledger(self, clean_ledgers): + record = PipelineLedger.get_or_create(_TEST_REPO, _ISSUE_REPAIR) + record.slug = "wrong-recorded-slug" + record.pr_number = 4242 + record.save(update_fields=["slug", "pr_number"]) + return record + + def _world(self, monkeypatch, heads, head_sha="c" * 40): + from tools import lane_identity + + monkeypatch.setattr(lane_identity, "_ls_remote_heads", lambda: heads) + monkeypatch.setattr("tools.pr_head_resolver.resolve_pr_head_sha", lambda *a, **k: head_sha) + + @staticmethod + def _recorded(): + return PipelineLedger.load(ledger_key=f"{_TEST_REPO}:{_ISSUE_REPAIR}").slug + + @staticmethod + def _evidence(): + import json + + from tools.lane_identity import _SLUG_REPAIR_KEY + + record = PipelineLedger.load(ledger_key=f"{_TEST_REPO}:{_ISSUE_REPAIR}") + raw = getattr(record, "stage_states_json", None) or "{}" + states = json.loads(raw) if isinstance(raw, str) else dict(raw) + return states.get(_SLUG_REPAIR_KEY, []) + + def test_unique_contradiction_repairs_and_records_evidence(self, ledger, monkeypatch): + from tools.lane_identity import repair_lane_slug + + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) == "real-lane-name" + assert self._recorded() == "real-lane-name" + + evidence = self._evidence() + assert len(evidence) == 1 + assert evidence[0]["from"] == "wrong-recorded-slug" + assert evidence[0]["to"] == "real-lane-name" + assert evidence[0]["head_sha"] == self._HEAD + assert evidence[0]["pr_number"] == 4242 + + def test_zero_matches_leave_the_record_alone(self, ledger, monkeypatch): + """Merged-and-deleted is the common zero case and is not a contradiction.""" + from tools.lane_identity import repair_lane_slug + + self._world(monkeypatch, {"refs/heads/main": self._OTHER}) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) is None + assert self._recorded() == "wrong-recorded-slug" + assert self._evidence() == [] + + def test_multiple_matches_leave_the_record_alone(self, ledger, monkeypatch): + """Ambiguity is not evidence — same discipline as ``_adopt_from_pr``.""" + from tools.lane_identity import repair_lane_slug + + self._world( + monkeypatch, + { + "refs/heads/session/candidate-a": self._HEAD, + "refs/heads/session/candidate-b": self._HEAD, + }, + ) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) is None + assert self._recorded() == "wrong-recorded-slug" + assert self._evidence() == [] + + def test_unresolvable_pr_head_leaves_the_record_alone(self, ledger, monkeypatch): + from tools.lane_identity import repair_lane_slug + + self._world( + monkeypatch, + {"refs/heads/session/real-lane-name": self._HEAD}, + head_sha=None, + ) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) is None + assert self._recorded() == "wrong-recorded-slug" + + def test_repeated_repair_is_idempotent(self, ledger, monkeypatch): + """Race 2: both callers compute the same value, so the second is a no-op.""" + from tools.lane_identity import repair_lane_slug + + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + first = repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) + second = repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) + + assert first == second == "real-lane-name" + assert self._recorded() == "real-lane-name" + # One correction happened, so exactly one justification is on file. + assert len(self._evidence()) == 1 + + def test_an_already_correct_slug_is_not_rewritten(self, clean_ledgers, monkeypatch): + from tools.lane_identity import repair_lane_slug + + record = PipelineLedger.get_or_create(_TEST_REPO, _ISSUE_REPAIR) + record.slug = "real-lane-name" + record.pr_number = 4242 + record.save(update_fields=["slug", "pr_number"]) + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) == "real-lane-name" + assert self._evidence() == [] + + def test_an_empty_slug_is_the_healing_arms_job_not_the_repairs( + self, clean_ledgers, monkeypatch + ): + from tools.lane_identity import repair_lane_slug + + record = PipelineLedger.get_or_create(_TEST_REPO, _ISSUE_REPAIR) + record.pr_number = 4242 + record.save(update_fields=["pr_number"]) + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) is None + assert not _nonempty_slug(_ISSUE_REPAIR) + + def test_repair_creates_no_ledger_for_a_non_lane_issue(self, clean_ledgers, monkeypatch): + from tools.lane_identity import repair_lane_slug + + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) is None + assert PipelineLedger.load(ledger_key=f"{_TEST_REPO}:{_ISSUE_REPAIR}") is None + + def test_ordinary_reads_still_return_the_recorded_slug(self, ledger, monkeypatch): + """Rung 1 is untouched: only the fail-closed decision path verifies.""" + from tools.lane_identity import resolve_lane_slug + + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + assert resolve_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) == "wrong-recorded-slug" + assert ( + resolve_lane_slug(_ISSUE_REPAIR, allow_heal=True, target_repo=_TEST_REPO) + == "wrong-recorded-slug" + ) + + def test_a_concurrent_repair_converges_rather_than_writing_twice(self, ledger, monkeypatch): + """The re-read immediately before the write is what makes this a no-op.""" + from tools import lane_identity + from tools.lane_identity import repair_lane_slug + + self._world(monkeypatch, {"refs/heads/session/real-lane-name": self._HEAD}) + + real_load = lane_identity.PipelineLedger.load + applied: list[str] = [] + + def _load_with_rival(*args, **kwargs): + record = real_load(*args, **kwargs) + # A rival repairer lands the same correction inside our window, + # exactly once, between our adjudication and our write. + if record is not None and not applied and record.slug == "wrong-recorded-slug": + applied.append("rival") + record.slug = "real-lane-name" + record.save(update_fields=["slug"]) + return real_load(*args, **kwargs) + return record + + monkeypatch.setattr(lane_identity.PipelineLedger, "load", _load_with_rival) + + assert repair_lane_slug(_ISSUE_REPAIR, target_repo=_TEST_REPO) == "real-lane-name" + assert self._recorded() == "real-lane-name" + # Converged, so our own repair filed no justification of its own. + assert self._evidence() == [] + + +def _nonempty_slug(issue_number: int) -> str | None: + record = PipelineLedger.load(ledger_key=f"{_TEST_REPO}:{issue_number}") + value = getattr(record, "slug", None) if record is not None else None + return value.strip() if isinstance(value, str) and value.strip() else None diff --git a/tools/lane_identity.py b/tools/lane_identity.py index 0e3468731..dafdbe346 100644 --- a/tools/lane_identity.py +++ b/tools/lane_identity.py @@ -35,8 +35,17 @@ branch, a PR's head ref. A plan document is not an identity; it is a document that mentions an issue, so a ``docs/plans/`` filename-stem rung is deliberately absent. Reading a plan filename to name a lane is derivation wearing adoption's -clothes, it is the precise defect this module closes, and because the write is -no-overwrite a wrong adoption could never be corrected. +clothes, and it is the precise defect this module closes. + +**A wrong recorded slug is repairable**, by :func:`repair_lane_slug` and by +nothing else. The adoption ladder above is conditional-on-empty by design, so +it prevents the bad state but cannot exit it; the repair is the separate, +evidence-gated path out. It fires only where a *fail-closed decision* is about +to be taken on the recorded name, and only on a **unique** ``git ls-remote +--heads origin`` match against the lane's PR head SHA -- zero and two-or-more +matches leave the record alone. Every correction files its justification on the +ledger, so a wrong repair is auditable rather than silent. Ordinary reads are +unaffected: rung 1 still returns the recorded value and never re-derives. A machine-local ``git worktree list`` rung is deliberately absent for a different reason: it would make two hosts reach different answers for the same lane, and a @@ -50,6 +59,7 @@ import re import subprocess import time +from dataclasses import dataclass from pathlib import Path from agent.pipeline_ledger import PipelineLedger @@ -295,17 +305,39 @@ def _slug_from_ref(ref: str) -> str | None: return _nonempty(ref[len(prefix) :]) -def _adopt_from_pr(pr_number: object, target_repo: str) -> str | None: - """Rung 2: recover the lane branch name via the PR's head SHA. +@dataclass(frozen=True) +class PrBranchTruth: + """Which lane branch a PR's head SHA points at, per ``git ls-remote``. - Shape-agnostic, which is why it precedes the fixed-shape probe: it is the - rung that recovers a lane whose branch a supervisor named something else - entirely. The match must be **unique** -- a re-created branch, a fork, or a - stale dev branch left at the same tip all produce duplicates, and a - listing-order-dependent answer would be a per-invocation identity. + ``sha`` is the resolved head (``None`` when it did not resolve) and + ``matches`` holds the lane slugs whose ``session/`` head sits at that SHA, + sorted. ``unique_slug`` is the ONLY answer any caller may act on: a + re-created branch, a fork, or a stale dev branch left at the same tip all + produce duplicates, and a listing-order-dependent answer would be a + per-invocation identity. Zero matches is the merged-and-deleted case. + """ + + sha: str | None = None + matches: tuple[str, ...] = () + + @property + def unique_slug(self) -> str | None: + return self.matches[0] if len(self.matches) == 1 else None + + +def _match_pr_head_to_lane_branches(pr_number: object, target_repo: str) -> PrBranchTruth: + """Resolve the PR's head SHA and find the lane branches sitting at it. + + The head SHA comes from ``tools.pr_head_resolver.resolve_pr_head_sha`` + (git-first via ``git ls-remote origin refs/pull/N/head``) and **never** a + bare ``gh`` read: a stale ``gh`` head SHA is what flipped the + verdict-staleness gate fail-open in #2895. + + One matcher serves both the adoption rung and the repair path, so the + ambiguity discipline cannot drift between them. """ if not isinstance(pr_number, int) or pr_number < 1: - return None + return PrBranchTruth() from tools.pr_head_resolver import resolve_pr_head_sha @@ -318,25 +350,37 @@ def _adopt_from_pr(pr_number: object, target_repo: str) -> str | None: ) except Exception as e: logger.debug(f"lane_identity: PR head resolution failed for PR {pr_number}: {e}") - return None + return PrBranchTruth() if not sha: - return None + return PrBranchTruth() - matches = [ + matches = sorted( slug for ref, ref_sha in _ls_remote_heads().items() if ref_sha == sha and (slug := _slug_from_ref(ref)) - ] - if len(matches) == 1: - return matches[0] - if matches: + ) + return PrBranchTruth(sha=sha, matches=tuple(matches)) + + +def _adopt_from_pr(pr_number: object, target_repo: str) -> str | None: + """Rung 2: recover the lane branch name via the PR's head SHA. + + Shape-agnostic, which is why it precedes the fixed-shape probe: it is the + rung that recovers a lane whose branch a supervisor named something else + entirely. The match must be **unique** (see :class:`PrBranchTruth`). + """ + truth = _match_pr_head_to_lane_branches(pr_number, target_repo) + unique = truth.unique_slug + if unique: + return unique + if truth.matches: logger.warning( "lane_identity: PR %s head %s matches %d lane branches (%s) -- " "ambiguous, falling through to the next rung", pr_number, - sha, - len(matches), - ", ".join(sorted(matches)), + truth.sha, + len(truth.matches), + ", ".join(truth.matches), ) # Zero matches is the merged-and-deleted case: a clean fall-through. return None @@ -433,6 +477,208 @@ def _record_slug_if_empty(ledger_key: str, candidate: str) -> str: _release_slug_lock(ledger_key) +# --------------------------------------------------------------------------- +# Evidence-gated repair of a wrong recorded slug +# --------------------------------------------------------------------------- + +# Where a repair's justification is filed on the ledger. A correction to a +# lane's identity must be auditable after the fact -- a wrong repair moves a +# lane's branch, worktree, and task list, so "it changed and nobody can say +# why" is not an acceptable end state (#3065 Risk 3). +_SLUG_REPAIR_KEY = "_slug_repairs" + + +def _record_repair_evidence( + ledger_key: str, + previous: str, + corrected: str, + pr_number: object, + head_sha: str | None, +) -> None: + """Append this repair's justification to the ledger. Best-effort. + + Written through ``update_stage_states``' optimistic retry rather than the + slug lock, because it touches ``stage_states_json`` — a blob with other + concurrent writers — while the slug write touches only ``slug``. A failure + here is logged and never raised: the correction itself already landed, and + losing the audit trail must not turn a good repair into an exception. + """ + try: + from tools.stage_states_helpers import update_stage_states + + ledger = PipelineLedger.load(ledger_key=ledger_key) + if ledger is None: + return + + def _append(states: dict) -> dict: + entries = states.setdefault(_SLUG_REPAIR_KEY, []) + if isinstance(entries, list): + entries.append( + { + "from": previous, + "to": corrected, + "pr_number": pr_number, + "head_sha": head_sha, + "at": int(time.time()), + } + ) + return states + + update_stage_states(ledger, _append, field="stage_states_json") + except Exception as e: + logger.warning( + "lane_identity: could not record slug-repair evidence for %r (%s -> %s): %s", + ledger_key, + previous, + corrected, + e, + ) + + +def _write_slug_repair( + ledger_key: str, + expected: str, + corrected: str, + pr_number: object, + head_sha: str | None, +) -> str | None: + """Overwrite a contradicted slug with ``corrected``. Returns what is recorded. + + Deliberately NOT :func:`_record_slug_if_empty`: that function's refusal to + overwrite is the whole defect this closes. What replaces it is not "write + unconditionally" but "write only against the value we adjudicated" — the + recorded slug is re-read under the slug lock immediately before the write + and compared to ``expected``: + + - already ``corrected`` — a concurrent repairer got there first. Both + callers computed the same value from the same ground truth, so this + converges to a **no-op** rather than a second write (Race 2). + - neither ``expected`` nor ``corrected`` — the record moved under us and + our evidence is about a value nobody has any more. Leave it alone. + + Returns the corrected slug on a write or a converged no-op, ``None`` when + the record was left untouched. + """ + lock_acquired = _acquire_slug_lock(ledger_key) + try: + if not lock_acquired: + # Another writer holds the lock. Wait for it and adopt its result + # if it is the same correction; never fight it. + for attempt in range(_SLUG_RACE_RETRY_ATTEMPTS): + fresh = PipelineLedger.load(ledger_key=ledger_key) + current = _nonempty(getattr(fresh, "slug", None)) if fresh is not None else None + if current == corrected: + return corrected + if attempt < _SLUG_RACE_RETRY_ATTEMPTS - 1: + time.sleep(_SLUG_RACE_RETRY_BACKOFF_S) + return None + + fresh = PipelineLedger.load(ledger_key=ledger_key) + if fresh is None: + logger.debug("lane_identity: ledger %r disappeared before the slug repair", ledger_key) + return None + current = _nonempty(getattr(fresh, "slug", None)) + if current == corrected: + return corrected + if current != expected: + logger.warning( + "lane_identity: recorded slug for %r changed from %r to %r under the repair " + "-- leaving it alone rather than writing evidence about a stale value", + ledger_key, + expected, + current, + ) + return None + fresh.slug = corrected + fresh.save(update_fields=["slug"]) + finally: + if lock_acquired: + _release_slug_lock(ledger_key) + + logger.warning( + "lane_identity: repaired the recorded lane slug for %r: %r -> %r " + "(PR %s head %s uniquely matches session/%s)", + ledger_key, + expected, + corrected, + pr_number, + head_sha, + corrected, + ) + _record_repair_evidence(ledger_key, expected, corrected, pr_number, head_sha) + return corrected + + +def repair_lane_slug( + issue_number: int, + *, + target_repo: str | None = None, +) -> str | None: + """Correct a recorded lane slug that branch truth contradicts. + + Ordinary reads keep going through :func:`resolve_lane_slug` rung 1, which + returns the recorded value and never re-derives over it. This function is + for the narrow case that made a wrong slug permanent: a **fail-closed + decision** about to be taken on the recorded name. Before failing a lane + closed on "the branch is not pushed", the decision must first establish + that it is asking about the right branch. + + The gate is deliberately narrow. A repair fires only when the lane's PR + head SHA resolves (through ``tools.pr_head_resolver.resolve_pr_head_sha``, + never a bare ``gh`` read) to **exactly one** branch in the ``git ls-remote + --heads origin`` listing and that branch's slug differs from the recorded + one. Zero matches and two-or-more matches both leave the record untouched, + matching :func:`_adopt_from_pr`'s existing ambiguity discipline — a wrong + repair is worse than a wrong original, because it moves a lane that was + merely mislabelled. + + Creates nothing: a lane with no ledger, no recorded slug, or no PR is not + a lane this function has anything to say about. + + Returns the slug branch truth confirms for this lane (the corrected value + after a repair, or the recorded value when it was already right), or + ``None`` when branch truth could not adjudicate. + """ + if not issue_number or issue_number < 1: + return None + + if target_repo is None: + target_repo = _sdlc_utils.resolve_target_repo_for_read(issue_number) + if not target_repo: + logger.debug( + "lane_identity: target repo unresolvable for issue #%s -- no repair", issue_number + ) + return None + + ledger = PipelineLedger.get(target_repo, issue_number) + if ledger is None: + return None + recorded = _nonempty(getattr(ledger, "slug", None)) + if not recorded: + # An empty slug is the healing arm's job (conditional-on-empty), not + # the repair's. There is no contradiction to adjudicate. + return None + + pr_number = getattr(ledger, "pr_number", None) + truth = _match_pr_head_to_lane_branches(pr_number, target_repo) + corrected = truth.unique_slug + if corrected is None: + if truth.matches: + logger.debug( + "lane_identity: issue #%s PR %s head matches %d lane branches -- " + "ambiguous, leaving the recorded slug %r alone", + issue_number, + pr_number, + len(truth.matches), + recorded, + ) + return None + if corrected == recorded: + return recorded + + return _write_slug_repair(ledger.ledger_key, recorded, corrected, pr_number, truth.sha) + + def adopt_lane_slug( issue_number: int, slug: str | None, From 43f48a67ddb84fee438844a627d104615ddb3118 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 17:17:27 +0700 Subject: [PATCH 12/19] Repair a proven-wrong lane slug at the fail-closed decision point (Refs #3065) Branch truth resolving to a branch other than the one the recorded slug names IS the proof repair_lane_slug is gated on, and the G8 PATCH check is where that proof is produced. Wire the two together so a slug proven wrong is corrected then, rather than left to confuse every future tick and every other consumer of the recorded identity. Best-effort and non-fatal; repair_lane_slug re-adjudicates uniqueness itself, so a stale read here cannot force a bad write. --- tools/sdlc_next_skill.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tools/sdlc_next_skill.py b/tools/sdlc_next_skill.py index 5bd45a14a..de20d8047 100644 --- a/tools/sdlc_next_skill.py +++ b/tools/sdlc_next_skill.py @@ -565,6 +565,29 @@ def _verify_stage_artifacts_live( "branch_truth": truth.status, "branch_truth_reason": truth.reason, } + if truth.branch and truth.branch != lane_branch: + # #3065 Task 6: branch truth just proved the recorded slug wrong + # -- the PR's work lives on `truth.branch`, not the name derived + # from the recorded slug. This IS the fail-closed decision point + # `repair_lane_slug` exists for: correct the ledger now, rather + # than leaving a proven-wrong slug to keep confusing every future + # tick and every other consumer of the recorded identity + # (worktree, branch, task list). Best-effort and non-fatal: a + # failed repair must never turn a successful verification into an + # error, and the gate inside repair_lane_slug re-adjudicates + # uniqueness independently, so a stale `truth` here cannot force + # a bad write. + try: + from tools.lane_identity import repair_lane_slug + + repair_lane_slug(issue_number, target_repo=repo) + except Exception as e: + logger.debug( + "stage-artifact-verify: issue #%s slug repair attempt failed (%s: %s)", + issue_number, + type(e).__name__, + e, + ) return { "branch_truth": truth.status, "branch_truth_branch": truth.branch, From 15d9b57b094dffc9f7df25c1b5afca05b043e9b0 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 17:28:26 +0700 Subject: [PATCH 13/19] Test the slug-repair trigger at the G8 decision point (Refs #3065) Both poles: a FOUND branch that disagrees with the recorded slug calls repair_lane_slug with the issue number and resolved repo; a FOUND branch that agrees must not call it at all. --- tests/unit/test_sdlc_next_skill.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/test_sdlc_next_skill.py b/tests/unit/test_sdlc_next_skill.py index 9081ea8ba..b5a58593e 100644 --- a/tests/unit/test_sdlc_next_skill.py +++ b/tests/unit/test_sdlc_next_skill.py @@ -2223,6 +2223,45 @@ def test_wrong_recorded_slug_with_a_live_branch_does_not_dispatch_patch(self, mo assert context["branch_truth"] == sdlc_next_skill.BRANCH_TRUTH_FOUND assert guard_g8_artifact_verification({"PATCH": "completed"}, {}, context) is None + def test_wrong_recorded_slug_triggers_a_repair_attempt(self, monkeypatch): + """#3065 Task 6: a FOUND branch that disagrees with the recorded slug + is exactly the fail-closed decision point ``repair_lane_slug`` exists + for -- this asserts the call actually happens, with the issue number + and resolved repo, not just that G8 tolerates the mismatch.""" + calls = [] + monkeypatch.setattr( + "tools.lane_identity.repair_lane_slug", + lambda issue_number, target_repo=None: calls.append((issue_number, target_repo)), + ) + self._setup( + monkeypatch, + heads={"refs/heads/session/renamed-lane": _SHA_A}, + head_sha=_SHA_A, + ) + self._context( + monkeypatch, {"PATCH": "completed"}, {"pr_number": 41, "_resolved_target_repo": "o/r"} + ) + + assert calls == [(3065, "o/r")] + + def test_matching_slug_triggers_no_repair_attempt(self, monkeypatch): + """FOUND and the branch agrees with the recorded slug -- nothing to + repair, so ``repair_lane_slug`` must not even be called.""" + + def _poison(issue_number, target_repo=None): + raise AssertionError("repair_lane_slug must not run when the slug is already right") + + monkeypatch.setattr("tools.lane_identity.repair_lane_slug", _poison) + self._setup( + monkeypatch, + slug="sdlc-3065", + heads={"refs/heads/session/sdlc-3065": _SHA_A}, + head_sha=_SHA_A, + ) + self._context( + monkeypatch, {"PATCH": "completed"}, {"pr_number": 41, "_resolved_target_repo": "o/r"} + ) + def test_genuinely_unpushed_branch_still_dispatches_patch(self, monkeypatch): from agent.sdlc_router import guard_g8_artifact_verification From 27f5a18e9164e549529b4a09f42cf99d7821aff7 Mon Sep 17 00:00:00 2001 From: valorengels Date: Thu, 3 Sep 2026 17:32:41 +0700 Subject: [PATCH 14/19] Docs: a wrong recorded lane slug is repairable on unique evidence (Refs #3065) --- docs/features/sdlc-lane-identity.md | 52 ++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/docs/features/sdlc-lane-identity.md b/docs/features/sdlc-lane-identity.md index f34da6432..d6d90d703 100644 --- a/docs/features/sdlc-lane-identity.md +++ b/docs/features/sdlc-lane-identity.md @@ -61,9 +61,12 @@ Two rungs are deliberately **absent**: filename stem. It was removed because a plan document is not an identity — it is a document that *mentions* an issue. Reading its filename to name a lane is derivation wearing adoption's clothes, which is the precise defect - this feature closes. Recording the result would have made it *worse* than - the prior guessing, not better, because the write is no-overwrite and a - wrong adoption could never be corrected. + this feature closes. Recording the result would have been worse than the + prior guessing, because the ladder's write is no-overwrite: a wrong + adoption from a plan filename would have been indistinguishable from a + correct one, and — before `repair_lane_slug` (see "A wrong recorded slug + is repairable" below) — recorded, permanent guesses had no path back to + the truth. - **No machine-local `git worktree list` rung.** A rung that reads local filesystem state would make two hosts reach different answers for the same lane. A per-host identity is not an identity. @@ -88,13 +91,16 @@ With `allow_heal=False` (the default), the function stops after rung 1: no git subprocess, no write, and — critically — **no ledger creation**. A read path can never bring a `PipelineLedger` into existence for a non-lane issue. -Exactly three callers write lane identity, and only two of them heal: +Exactly four callers write lane identity: two heal an empty slug, one adopts a +known one, and one repairs a wrong one (see "A wrong recorded slug is +repairable" below). | Caller | Mechanism | Why | |---|---|---| | `tools/sdlc_session_ensure.py::ensure_session` | `resolve_lane_slug(N, allow_heal=True)` | The minter. Runs at lane start with no identity in hand. | | `reflections/sdlc_upvote_lanes.py` lane pickup | `resolve_lane_slug(N, allow_heal=True, target_repo=repo)` | Lane start on the reflection path, past every gate, about to create a real branch. It also has no identity in hand — it scanned an issue, not a branch. | | `reflections/sdlc_progress.py` stalled-lane respawn | `adopt_lane_slug(N, slug, target_repo=target_repo)` | See "Adopt vs. resolve" below — this caller already knows the identity, so it does not heal. | +| `tools/sdlc_next_skill.py::_verify_stage_artifacts_live` (PATCH's G8 check) | `repair_lane_slug(N, target_repo=repo)` | The fail-closed decision point: branch truth just proved the recorded slug wrong (`resolve_branch_truth` returned *found* on a branch that disagrees with it), so the ledger is corrected here rather than left wrong forever. | ## Adopt vs. resolve: the three-way rule @@ -136,6 +142,44 @@ described above is tested by calling `_attempt_action` directly. arm of `resolve_lane_slug`. It walks no ladder because the caller is not asking a question. +## A wrong recorded slug is repairable + +`resolve_lane_slug` rung 1 and `adopt_lane_slug` are both no-overwrite by +design — neither can fix a slug that was already recorded wrong. That used to +mean a wrong recorded slug was permanent (#3065): `_check_branch_pushed` +probed a branch name derived from the recorded slug, found nothing, and +reported a healthy, pushed lane as unverified forever, because nothing in this +module could ever correct the record it was reading from. + +`repair_lane_slug(issue_number, *, target_repo=None)` closes that gap with a +**fourth write path**, deliberately separate from the conditional-on-empty +ones above. It fires only at the moment a fail-closed decision is about to act +on the recorded name — today that is the G8 artifact-verification check in +`tools/sdlc_next_skill.py::_verify_stage_artifacts_live`, via +`resolve_branch_truth` — and only under a gate as narrow as the adoption +ladder's rung 2: + +- The lane's PR head SHA (via `tools.pr_head_resolver.resolve_pr_head_sha`, + git-first, never a bare `gh` read) must resolve to **exactly one** branch in + a `git ls-remote --heads origin` listing. +- That branch's slug must differ from the recorded one — no contradiction, no + write. +- Zero matches or two-or-more matches both leave the record untouched, + matching rung 2's existing ambiguity discipline. A wrong repair is worse + than a wrong original, because it moves a lane that was merely mislabelled + (the worktree, the branch, and the task list all key off the slug). + +Every correction is written under the same slug lock as the other write paths, +re-reading the recorded value immediately before writing so a repair another +caller already applied converges to a no-op instead of a second write, and its +justification (`from`, `to`, `pr_number`, `head_sha`, `at`) is appended to the +ledger's `stage_states_json` under `_slug_repairs` — a wrong repair is +auditable, not silent. + +Ordinary reads are unaffected: rung 1 of `resolve_lane_slug` still returns the +recorded value directly and never triggers a repair or any live call. Only the +fail-closed decision path pays for the check. + ## Discovery reads identity The write direction above answers "what is this lane called." A companion From 4f005170faac2fbecbace559bf713a76e885bb5e Mon Sep 17 00:00:00 2001 From: valorengels Date: Fri, 4 Sep 2026 08:56:05 +0700 Subject: [PATCH 15/19] Documentation cascade for the read-facts router and tri-state grading (Refs #3065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New feature doc for reconciliation: why a guard must see the selected dispatch and not only a caller's proposal, the at-most-two-pass bound, the by-reference invariant that keeps the impure G5 guard idempotent, and how to read a Blocked carrying two verdicts. machine-readable-dod gains the expanded expectation grammar, the anchored-vs- prefix-matched split, and the UNEVALUATED outcome with its build-gate/merge-gate consumer split. The pipe-escape rule is now reachable from the expectation list instead of only from the anti-criteria section. PLAN_TEMPLATE carries the grammar, three-valued grading, and the escape rule inline, with a sample anti-criterion that actually demonstrates BRE alternation (doubled backslash) — proven two-pole through the parser rather than asserted. CLAUDE.md is deliberately unedited: reconciliation changes no caller-visible router contract, and that file is regex-parsed into worker prompts. The features README sort fix on the two Telegram rows is pre-existing drift on main, not from this lane; the sort validator blocks any edit to the file until it is corrected. --- .../skills-global/do-plan/PLAN_TEMPLATE.md | 36 +++- docs/features/README.md | 5 +- docs/features/machine-readable-dod.md | 65 ++++++- .../sdlc-router-decision-reconciliation.md | 173 ++++++++++++++++++ docs/sdlc/do-build.md | 17 +- docs/sdlc/do-pr-review.md | 6 +- 6 files changed, 291 insertions(+), 11 deletions(-) create mode 100644 docs/features/sdlc-router-decision-reconciliation.md diff --git a/.claude/skills-global/do-plan/PLAN_TEMPLATE.md b/.claude/skills-global/do-plan/PLAN_TEMPLATE.md index ce4ee5036..067d82873 100644 --- a/.claude/skills-global/do-plan/PLAN_TEMPLATE.md +++ b/.claude/skills-global/do-plan/PLAN_TEMPLATE.md @@ -445,7 +445,12 @@ Each row is a named check with an executable command and expected result. **Positive expectations** (the command must succeed or produce the expected output): - `exit code N` — passes when exit_code == N (positive exact-match; e.g. `exit code 0` for success, `exit code 1` for "grep found no matches") -- `output > N` — passes when stdout (as integer) is greater than N +- `exit N` — the same assertion, shorter spelling +- `output > N` / `> N` — passes when stdout (stripped, as an integer) is greater than N +- `output >= N` / `>= N` — passes when stdout is numeric and >= N +- `output == N` / `== N` — passes when stdout is numeric and exactly N +- `` prints `N` `` — passes when stripped stdout equals N exactly (backticks optional) +- `empty output` — passes when stdout is empty or whitespace-only - `output contains X` — passes when substring X appears in stdout **Inverse expectations / anti-criteria** (the command must NOT produce a forbidden result): @@ -458,6 +463,33 @@ Each row is a named check with an executable command and expected result. disjoint and unambiguous. The existing `exit code 1` sample row ("No stale xfails") is a positive exact-match — it stays as-is. +**Every row grades three-valued: `PASS`, `FAIL`, or `UNEVALUATED`.** There is no +pass/fail boolean. `UNEVALUATED` means the grader could not answer the question — +an expectation form not in the list above, an empty `Expected` cell, a `Command` +cell with no backticked span, a timeout, or a runner exception. It blocks exactly +like `FAIL` but is reported as its own token, because it is a finding about the +*row you wrote*, not about the code. If a row comes back `UNEVALUATED`, fix the +row. Note that the three older forms (`exit code N`, `output > N`, +`output contains X`) tolerate a trailing gloss, while every newer form is anchored: +`>= 1 (one call site today)` grades `UNEVALUATED`, not `>= 1`. + +**Pipes must be escaped, and the escape composes.** A `|` is the table's own column +separator, so a command containing one is written `\|`; a bare `|` is rejected as a +plan-authoring error rather than executed truncated. The parser unescapes once, after +splitting, which means a cell reaching `grep -E` as alternation and a cell reaching +basic-regex `grep -c` as alternation are spelled differently: + +| In the table cell | Reaches the shell as | Under `grep -E` | Under `grep -c` (BRE) | +|---|---|---|---| +| `a\|b` | `a\|b` | alternation | literal `a\|b` | +| `a\\\|b` | `a\\|b` | literal `a\\|b` | alternation | + +Anti-criteria are the rows that most often need alternation and most often use +`grep -c`, so they usually want the **doubled** backslash. The sample row below is +written that way on purpose. Prove any anti-criterion two-pole (red against a +deliberately-violating input, green against clean) before trusting it — a row that +cannot fail is not a gate. + **Anti-criteria** are inverse rows in this table that assert a forbidden code-level outcome from a No-Go cannot be detected in the PR. They are opt-in: only add an inverse row when you can write a command that mechanically detects the violation. @@ -480,7 +512,7 @@ zero checks -- give at least one table a `Command` column, or drop the rows.] | Lint clean | `python -m ruff check .` | exit code 0 | | Format clean | `python -m ruff format --check .` | exit code 0 | | No stale xfails | `grep -rn 'xfail' tests/ \| grep -v '# open bug'` | exit code 1 | -| [Anti-criterion example] | `grep -c "forbidden_pattern" changed/file.py` | match count == 0 | +| [Anti-criterion example] | `grep -c "r\.delete\\\|r\.srem" changed/file.py` | match count == 0 | | [Feature-specific check] | `[command]` | [expected] | ## Critique Results diff --git a/docs/features/README.md b/docs/features/README.md index a1e3ac11d..d96e186fb 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -111,7 +111,7 @@ Feature documentation for the Valor AI system. Each document describes an implem | [Local Ollama Model Policy](local-model-policy.md) | Classification → `granite4.1:3b`; generation → `gemma4:31b-cloud` by default; embeddings → `nomic-embed-text`. | Shipped | | [Log Rotation](log-rotation.md) | User-space log rotation via LaunchAgent (`com.valor.log-rotate`): 30-minute schedule, 10 MB/3 backups, self-exclusion. | Shipped | | [Long-Task Checkpointing](long-task-checkpointing.md) | PROGRESS.md scratchpad and frequent-commit guidance for dev sessions to survive context compaction. | Shipped | -| [Machine-Readable Definition of Done](machine-readable-dod.md) | Structured `## Verification` table in plan documents with six executable expectation types, executed automatically by /do-build and /do-pr-review. | Shipped | +| [Machine-Readable Definition of Done](machine-readable-dod.md) | Structured `## Verification` table in plan documents, graded three-valued (`PASS` / `FAIL` / `UNEVALUATED`) and executed automatically by /do-build and /do-pr-review; the recorded aggregate is a merge gate. | Shipped | | [Markitdown Ingestion](markitdown-ingestion.md) | Multi-format document ingestion (PDF, DOCX, PPTX, XLSX, HTML, images) for the knowledge pipeline via `.md` sidecars; `valor-ingest` CLI. | Shipped | | [Media Enrichment](media-enrichment.md) | Bridge-side Telethon download + worker-side AI (vision/Whisper/extract) for photos, voice, audio, and documents. | Shipped | | [Memory Hook Performance](memory-hook-performance.md) | Import-chain optimization for the PostToolUse memory recall hook. | Shipped | @@ -193,6 +193,7 @@ Feature documentation for the Valor AI system. Each document describes an implem | [SDLC Pipeline Portability](sdlc-pipeline-portability.md) | Generic robustness so `/sdlc` runs unattended in any repo: git-root plan resolution, finished-PR routing guard, `pr_number` recovery, and cross-repo plan resolution. | Shipped | | [SDLC Pipeline State](sdlc-pipeline-state.md) | Local Claude Code session state tracking via `--issue-number` and `sdlc_session_ensure`; `AgentSession.pr_number` single-writer field. | Shipped | | [SDLC Repo Addenda](sdlc-repo-addenda.md) | Per-stage `docs/sdlc/` notes injected into global SDLC skills at runtime; a reflection proposes updates every 3 days. | Shipped | +| [SDLC Router Decision Reconciliation](sdlc-router-decision-reconciliation.md) | The guard list re-runs against the skill the dispatch table selected, bounded to one pass on the selection and one on its redirect; a non-converging veto returns `Blocked(RECONCILE_DEADLOCK)` carrying `decision_inputs`. | Shipped | | [SDLC Router Oscillation Guard](sdlc-router-oscillation-guard.md) | The Legal Dispatch Guards G1-G9, the plan-revising lock, the stage-advance artifact verification gate, the blocked-on-conflict escalation gate, and a single-writer verdict recorder. | Shipped | | [SDLC Run Identity Self-Heal](sdlc-run-identity-self-heal.md) | State-mutating `sdlc-tool` writes self-heal run identity on a resumed turn instead of silently no-op'ing. | Shipped | | [SDLC Run Self-Recognition](sdlc-run-self-recognition.md) | `AgentSession.owned_run_ids` makes self-recognition structural: a lease-lapse re-mint returns the live owner id, and a run-health disposition makes sustained marker-write failures loud. | Shipped | @@ -244,10 +245,10 @@ Feature documentation for the Valor AI system. Each document describes an implem | [Teammate Session Permissions](teammate-session-permissions.md) | Code-level enforcement of the one teammate hard rule (source-code writes require a Dev session); two-pass allowlist, `[teammate-audit]` Bash log, and a capable prompt. | Shipped | | [Telegram History & Links](telegram-history.md) | Searchable message history and link compilation from Telegram. | Shipped | | [Telegram Inbound Attachments](telegram-inbound-attachments.md) | Files arriving in a chat with a live session are enriched with their extracted content (document text, image description, voice transcription) before the steering push, and fire-and-forget copied into `~/work-vault/telegram-attachments/` for the `KnowledgeWatcher` to index. | Shipped | -| [Telegram Poll Questions](telegram-poll-questions.md) | `/ask-me` renders a blocked eng session's question as a native Telegram poll in a group chat, so one tap unblocks it; poll registry, vote→steering translation, and a nudge-loop pause so the asker actually waits. | Shipped | | [Telegram Message Edit Handling](telegram-message-edit-handling.md) | Handles Telegram MessageEdited events — steers running sessions with edited text or spawns a fresh session. | Shipped | | [Telegram Messaging](telegram-messaging.md) | Unified interface for reading and sending Telegram messages via the `valor-telegram` CLI. | Shipped | | [Telegram PM Guide](telegram-pm-guide.md) | PM-facing guide for Telegram interaction patterns, session resumption, and pipeline signals. | Shipped | +| [Telegram Poll Questions](telegram-poll-questions.md) | `/ask-me` renders a blocked eng session's question as a native Telegram poll in a group chat, so one tap unblocks it; poll registry, vote→steering translation, and a nudge-loop pause so the asker actually waits. | Shipped | | [Test Baseline Verification](test-baseline-verification.md) | Verified classification of test failures as regressions vs pre-existing by running failing tests against main. | Shipped | | [Test Concurrency Coordination](test-concurrency-coordination.md) | Defense-in-depth sentinel-ID namespacing to prevent cross-run Redis contention. | Shipped | | [Test Coverage Standards](test-coverage-standards.md) | Standards and tooling for preventing silent failure classes: exception swallowing, empty output loops, coupled tests, missing error rendering, silent builds. | Shipped | diff --git a/docs/features/machine-readable-dod.md b/docs/features/machine-readable-dod.md index 4c220db7e..f8d7c9390 100644 --- a/docs/features/machine-readable-dod.md +++ b/docs/features/machine-readable-dod.md @@ -42,13 +42,22 @@ Each row defines: ### Supported Expectations +`evaluate_expectation` (`agent/verification_parser.py`) is the sole grader. +It is three-valued: every row grades `PASS`, `FAIL`, or `UNEVALUATED` (see +"Three-Valued Grading" below), never a plain boolean. + **Positive expectations** (the command must succeed or produce the expected output): | Format | Meaning | Example | |--------|---------|---------| -| `exit code N` | Command must exit with code N (positive exact-match) | `exit code 0` | -| `output > N` | Command output (as integer) must be greater than N | `output > 0` | +| `exit code N` | Command must exit with code N (positive exact-match, prefix-matched) | `exit code 0` | +| `exit N` | Same as `exit code N`, shorter spelling (anchored) | `exit 0` | | `output contains X` | Command stdout must contain substring X | `output contains ok` | +| `output > N` / `> N` | Stdout (stripped, parsed as an integer) must be greater than N | `output > 0`, `> 0` | +| `>= N` / `output >= N` | Stdout must be numeric and `>= N` | `>= 1` | +| `== N` / `output == N` | Stdout must be numeric and exactly N | `== 3` | +| `prints \`N\`` | Stripped stdout must equal N exactly (backticks optional) | ``prints `ok` `` | +| `empty output` | Stdout must be empty or whitespace-only | `empty output` | **Inverse expectations / anti-criteria** (the command must NOT produce a forbidden result): @@ -60,8 +69,60 @@ Each row defines: **Important distinction:** `exit code N` is a positive exact-match — it passes when `exit_code == N`. `exit code != N` is the inverse — it passes when `exit_code != N`. The two are syntactically disjoint and unambiguous. The existing `exit code 1` check ("No stale xfails") is a positive exact-match: grep exits 1 when it finds no matches, so `exit code 1` asserts "no stale xfails found". It is NOT an inverse. +The inverse forms are matched before their positive counterparts (`exit code != N` before `exit code N`, `output does not contain X` before `output contains X`), so an inverse row can never be misread as a positive one. + +A command containing a `|` needs the escape rule in "Authoring Rule: Pipes Must Be Escaped" below — and the escape composes differently for `grep -E` than for basic-regex `grep -c`, which is the single most common way an otherwise-correct row silently stops asserting what it says. `.claude/skills-global/do-plan/PLAN_TEMPLATE.md` carries the same rule and a worked alternation sample inline, so a plan author meets it while writing the row rather than by following a link. + **Empty-stdout gate:** Both `output does not contain X` and `match count == 0` reject truly-empty stdout. An errored command or one that writes only to stderr produces empty stdout; without the gate, a trivially-absent substring or `all(...)` over an empty list would silently pass. A legitimately-clean `grep -c` returns a literal `0` (one byte of non-empty stdout), so the gate fires only when the command produced no output at all. +**Prefix-matched vs. anchored.** The three pre-existing forms — `exit code N`, `output > N`, `output contains X` — stay prefix-matched: a trailing gloss (`exit code 0 (verified 2026-09-02 to return exactly one EnvCall today)`) is an established authoring idiom in this repo's live plans, and anchoring them would have turned already-working rows into blocking `UNEVALUATED` for a change nobody asked for. Every form added afterward (`exit N`, `` prints `N` ``, `>= N`, `> N`, `== N`, `empty output`) is **anchored**: the cell must match the pattern to its end. A bare comparator followed by prose is easy to write by accident, and grading a sentence as if it were a number is exactly the guess this module exists to stop making — an anchored form that fails to match reports `UNEVALUATED` naming the cell, telling the author what to fix rather than silently misgrading it. + +Non-numeric stdout under a numeric comparator (`> N`, `>= N`, `== N`) is a genuine `FAIL`, not `UNEVALUATED`: the expectation was understood, the command just answered with something that is not a number. + +### Three-Valued Grading: `UNEVALUATED` + +A check never grades a plain pass/fail boolean. `evaluate_expectation` and +`run_checks` return one of `CheckOutcome.PASS`, `CheckOutcome.FAIL`, or +`CheckOutcome.UNEVALUATED`. `UNEVALUATED` means *the grader could not answer +the question* — it is produced by: + +- an expectation cell that is empty, whitespace-only, or `None` +- an expectation form the grammar above does not recognise +- a command cell carrying no backticked span +- a command timeout (`DEFAULT_TIMEOUT_S`, currently 120s — provisional and + tunable; the one bound shared by both runners of these tables, #2901) +- any runner exception + +`UNEVALUATED` is **blocking** — it never counts as a pass — but it is +reported as its own token, never folded into `[FAIL]`. The distinction +matters for whoever reads the report: a `FAIL` says the code under test is +wrong; an `UNEVALUATED` says the grader itself could not answer, which is a +claim about the check's authoring (a malformed expectation, a missing +backtick span, a suite that took too long), not about the code. The 2026-09 +supervisor batch hand-verified every `UNEVALUATED` "failure" it encountered +as code that was actually correct — the row's expectation was unreadable, +not the code under test. + +The distinction changes what a human does next, but not whether the gate +stops: + +- **Build gate** (`/do-build` Step 5.1, `scripts/validate_build.py`): + `UNEVALUATED` blocks exactly like `FAIL` — the script's exit code is `1` if + either count is non-zero — but the printed report line reads + `UNEVALUATED: ` rather than `FAIL: ...`, so the triaging agent knows + to fix the *row* (rewrite the expectation, add the missing backtick span) + rather than debug the code. +- **Merge/review gate** (`/do-pr-review` Step 4.5): the reviewer runs the + same table and reports `UNEVALUATED` rows as their own category in the + review, never as a code-quality finding — a blocker that reads "grader + could not answer" is a different fix than a blocker that reads "code is + wrong," and conflating them sends the wrong person down the wrong path. + +A run with zero checks at all (an empty `## Verification` table, or a section +that produced nothing executable) grades `UNEVALUATED` for the run, never a +vacuous `PASS` — there being nothing to check is not evidence that +everything checked out. + ### Table Scoping: One `## Verification` Section, Many Pipe-Blocks (#2836) A `## Verification` section can carry more than one markdown table -- a check diff --git a/docs/features/sdlc-router-decision-reconciliation.md b/docs/features/sdlc-router-decision-reconciliation.md new file mode 100644 index 000000000..16de1b480 --- /dev/null +++ b/docs/features/sdlc-router-decision-reconciliation.md @@ -0,0 +1,173 @@ +# SDLC Router Decision Reconciliation + +The SDLC router decides in two movements: the guard list (G1-G9) runs first, and +if nothing trips, the dispatch table (`DISPATCH_RULES`) picks a row. Those two +movements ask the guards about **different skills**. Reconciliation is the step +that makes them ask about the same one. + +`agent/sdlc_router.py::reconcile_dispatch` re-runs the guard list against the +skill the dispatch table actually selected, so a guard veto constrains the +decision that is about to ship. + +## Why a guard must see the selected dispatch + +Guards read `context["proposed_skill"]` — a value supplied by whoever called +`decide_next_dispatch`. `guard_g3_pr_lock` is the clearest case: it fires only +when the proposed or previously dispatched skill is in the plan-stage family +(`/do-plan`, `/do-plan-critique`), and it exists to stop a lane with an open PR +from being sent back to planning. + +The dispatch table's selection was never run past that. A caller invoking +`next-skill` with no `--proposed-skill` supplies nothing, the guard list sees an +empty proposal and steps aside, and the table is then free to select +`/do-plan-critique` on a lane with an open PR and an APPROVED review — precisely +the dispatch G3 is written to forbid. The guard's coverage depended on the +caller having independently guessed the answer first. + +Reconciliation closes that by substituting `primary.skill` into the context and +running the same guard list again. Nothing about the guards changes; what +changes is that they are now asked about the real decision. + +The row that most needs this constraint is row 2b +(`_rule_critique_verdict_stale`). That row is marker-agnostic on purpose +(#1639), so it can rescue a lane out of a CRITIQUE `in_progress` dead end, and +it must stay that way. Reconciliation constrains its output from the outside — +G3 vetoes the redirect on a shipped, open-PR lane — while leaving 2b free to +fire unmodified on the lane shape it exists for. Editing 2b to special-case the +open-PR lane would reintroduce the dead end it was written to escape. + +G3's redirect target on a clean-review lane with DOCS outstanding is `/do-docs`, +carrying the reason constant `G3_REDIRECT_REASON_DOCS_PENDING`. + +## The single-pass bound + +Termination is the design constraint, not an afterthought. `reconcile_dispatch` +runs the guard list **at most twice**: + +1. Once with `primary.skill` proposed. + - No veto (`None`) → the table's selection ships unchanged. + - A guard's own `Blocked` or `Terminal` → returned as-is. It is already a + terminating decision with its own reason and `guard_id`; wrapping it would + add nothing. + - A guard `Dispatch` naming the same skill the table chose → **agreement, + not a veto**. The table's own `row_id` and `reason` are kept. + Reconciliation exists to withhold dispatches, never to relabel ones no + guard objected to. + - A guard `Dispatch` naming a different skill → a redirect, checked once + more. +2. Once with the redirect proposed. + - No veto → the redirect ships. + - A `Dispatch` re-proposing the redirect target → agreement again; the + redirect ships. + - Anything else → a single `Blocked`, described below. + +A second veto never triggers a third pass. The alternative — iterating until the +guards agree — converges on nothing and reaches G4's oscillation cap several +turns later, reporting "stage oscillation" for a condition that is actually +"the guards contradict each other". That misattribution cost #2771 and #2334 a +manual unwedge each. Stopping immediately with evidence is the fail-closed +direction. + +## The by-reference invariant + +`stage_states` and `meta` **must** be passed through to `evaluate_guards` by +reference, never copied, here or in any caller. + +The guards are not pure. `guard_g5_artifact_hash_cache` rewrites +`stage_states["_verdicts"]["CRITIQUE"]["artifact_hash"]` in place when it +detects a legacy full-bytes hash, and logs a WARNING on that rewrite. Because +reconciliation calls `evaluate_guards` a second time on the *same objects*, the +second call sees the already-migrated record and silently steps aside. That is +the whole reason double invocation is idempotent. + +A defensive copy of `stage_states` or `meta` anywhere between the two passes +would hand the second pass the pre-migration hash again, re-running the +migration branch and doubling the log noise for every reconciled decision. This +is exactly the "safe" cleanup a later contributor would make without knowing +the guards mutate. Do not add one. + +`build_decision_inputs` shallow-copies both dicts for the evidence payload +only, so that a `Blocked` a supervisor reads later is a snapshot rather than a +live alias. That copy is downstream of the guard passes and does not break the +invariant. + +A raising guard is not caught here. Rule predicates in `decide_next_dispatch` +are try/except-wrapped; guards are not, and reconciliation preserves that +asymmetry. Swallowing a raising guard into a `NO_RULE` block would misreport a +bug as a routing hole. + +## Reading a `Blocked` that carries two verdicts + +When the redirect is itself vetoed, the router returns: + +``` +Blocked( + reason="reconciliation: guard veto did not converge — table selected row + '2b' ('/do-plan-critique'), redirect to '/do-docs' was itself vetoed", + guard_id=RECONCILE_DEADLOCK_GUARD_ID, # "RECONCILE_DEADLOCK" + decision_inputs={...}, +) +``` + +`RECONCILE_DEADLOCK` is a sentinel `guard_id` in the same short-code namespace +as `G2`/`G4`/`G7` and `NO_RULE`, so a consumer matches on it without parsing +prose. It does not mean a numbered guard fired. It means the routing table and +the guard list produced answers that do not compose, and no third opinion is +going to be requested. + +`decision_inputs` carries what the decision was made from and what disagreed: + +| Key | What it holds | +|---|---| +| `stage_states` / `meta` | The facts the router read, snapshotted | +| `unrecorded_dispatch` | The previous-dispatch-was-never-recorded signal, or `None` | +| `selected_row` / `selected_skill` | The dispatch table's own answer | +| `first_redirect` | The first guard's counter-proposal (`skill`, `reason`, `row_id`) | +| `vetoing_guard` | The second pass's verdict, summarized by its type | + +For a supervisor, this is the difference between "the router refused" and "the +router refused *because*". A control plane that cannot show its own inputs +cannot be checked against a field report: the #3065 batch reported a `NO_RULE` +on a state (CRITIQUE APPROVED and completed, BUILD in progress, no PR) that a +dispatch row has owned since `c1e991972`, and the report could be neither +confirmed nor refuted, because the payload carried no `stage_states` and no +`meta`. The `NO_RULE` fallthrough now carries the same evidence for the same +reason. + +Two verdicts in one `Blocked` is an instruction about where to look. The +disagreement is between the named row and the named guard, and one of them is +wrong about this lane. Resolving it means changing a rule or a guard, not +re-running the router: the state it read is attached, and re-running on the +same state produces the same deadlock. + +## Diagnostic-only evidence never changes the decision + +`Dispatch.unrecorded_dispatch` and `Blocked.decision_inputs` are both declared +`compare=False`. Evidence is not identity: two dispatches of the same skill for +the same reason are the same decision whether or not the previous one was +recorded, and a refusal's identity is its reason and `guard_id`, not the state +dump attached for a human. Without `compare=False`, attaching evidence would +silently redefine equality for every caller that compares a decision against an +expected `Dispatch(...)`. + +`detect_unrecorded_dispatch` is a pure read. It writes nothing and never +changes which skill is dispatched; it names the hole — a skill dispatched with +no record in `_sdlc_dispatches`, a record naming a different skill, or a router +slot still `confirmed: False` — so a supervisor sees it on the decision itself +rather than inferring it from a G4 block four turns later. + +## Related + +- [SDLC Router Oscillation Guard](sdlc-router-oscillation-guard.md) — the guard + list G1-G9 this step re-runs, including G4's oscillation cap and G8's + artifact verification. +- [SDLC Lane Identity](sdlc-lane-identity.md) — the recorded slug G8's branch + probe resolves through, and the evidence-gated repair at that decision point. +- [Machine-Readable Definition of Done](machine-readable-dod.md) — the graded + verification outcomes the merge gate reads. +- Source: `agent/sdlc_router.py` (`reconcile_dispatch`, `build_decision_inputs`, + `detect_unrecorded_dispatch`), `tools/sdlc_next_skill.py` + (`build_decision_context`, the one context builder both `decide_next_dispatch` + callers share, and `decide`, which surfaces `decision_inputs` in the CLI's + JSON payload). +- GitHub issue: #3065 diff --git a/docs/sdlc/do-build.md b/docs/sdlc/do-build.md index 9b077f6de..a7c4db726 100644 --- a/docs/sdlc/do-build.md +++ b/docs/sdlc/do-build.md @@ -184,12 +184,21 @@ prevent. (cd $TARGET_REPO/.worktrees/{slug} && python scripts/validate_build.py $PLAN_PATH) # exit 1 → /do-patch, ≤3 iters (cd $TARGET_REPO/.worktrees/{slug} && python scripts/evaluate_build.py $PLAN_PATH) # exit 2 → bundle FAILs to /do-patch, ≤2 iters; 3 = no criteria; 1 = non-blocking # Verification table runner: -python -c "import sys; from agent.verification_parser import parse_verification_table, run_checks, format_results; t = parse_verification_table(open(PLAN_PATH).read()); r = run_checks(t.checks); print(format_results(r, t)); sys.exit(1 if t.malformed or not all(x.passed for x in r) else 0)" +python -c "import sys; from agent.verification_parser import parse_verification_table, run_checks, format_results; t = parse_verification_table(open(PLAN_PATH).read()); r = run_checks(t.checks); print(format_results(r, t)); sys.exit(1 if t.malformed or not all(x.outcome == 'PASS' for x in r) else 0)" +# Each result carries a three-valued `outcome` (PASS / FAIL / UNEVALUATED), never a +# boolean. Only PASS clears the gate: UNEVALUATED means the GRADER could not answer +# (a timeout, a runner exception, an expectation form the grammar does not recognise, +# an empty Expected cell, or a Command cell with no backticked span), and it blocks +# exactly like FAIL while reporting itself as its own token so nobody debugs the code +# for a grader problem. `CheckOutcome` is a StrEnum, so `x.outcome == 'PASS'` is the +# comparison; there is deliberately no `.passed` attribute to fall back on. # A row in `t.malformed` is a PLAN-AUTHORING error (an unescaped `|` split it, or a # pipe-block with rows but no Command column), not a finding about the code. Write -# pipes in the table as `\|`. A row in `t.skipped` is a non-check -# table (a summary, a findings recap) -- named in the report but never counted toward -# the exit code. +# pipes in the table as `\|`. Malformed rows are never executed and always fail the +# gate. A `SkippedTable` in `t.skipped` is a non-check pipe-block (a summary, a +# findings recap, anything whose columns are not `(, Command, Expected)`) -- +# named in the report as a diagnostic, never executed, and never counted toward the +# exit code. ``` **Documentation gate scripts (Step 6):** diff --git a/docs/sdlc/do-pr-review.md b/docs/sdlc/do-pr-review.md index 747a0f440..353709132 100644 --- a/docs/sdlc/do-pr-review.md +++ b/docs/sdlc/do-pr-review.md @@ -69,7 +69,11 @@ inheritance, not a block: use the returned `run_id` and continue; only a foreign **Verification-table runner (§ 4.5):** ```bash -python -c "import sys; from agent.verification_parser import parse_verification_table, run_checks, format_results; t = parse_verification_table(open(PLAN_PATH).read()); r = run_checks(t.checks); print(format_results(r, t)); sys.exit(1 if t.malformed or not all(x.passed for x in r) else 0)" +python -c "import sys; from agent.verification_parser import parse_verification_table, run_checks, format_results; t = parse_verification_table(open(PLAN_PATH).read()); r = run_checks(t.checks); print(format_results(r, t)); sys.exit(1 if t.malformed or not all(x.outcome == 'PASS' for x in r) else 0)" +# Each result carries a three-valued `outcome` (PASS / FAIL / UNEVALUATED), never a +# boolean; `CheckOutcome` is a StrEnum, so `x.outcome == 'PASS'` is the comparison. +# UNEVALUATED blocks like FAIL but says the GRADER could not answer -- report it as +# UNEVALUATED in the review, never as a finding about the code. # A row in `t.malformed` is a PLAN-AUTHORING error (an unescaped `|` split it, or a # pipe-block with rows but no Command column), not a finding about the code. Write # pipes in the table as `\|`. See #2570, #2836. A row in `t.skipped` is a non-check From 2f09345778bb2038b13078b69cdaecb614f7c3e6 Mon Sep 17 00:00:00 2001 From: valorengels Date: Fri, 4 Sep 2026 09:10:31 +0700 Subject: [PATCH 16/19] Record the merge gate's new fail-closed read in the SDLC principle (Refs #3065) Principle 9 already carried the head-SHA rule for one gate. The merge predicate now reads a second recorded fact under the same rule: a plan's Verification rows grade three-valued, a FAIL or UNEVALUATED row refuses the merge, and the recorded aggregate is trusted only while its stamped head SHA matches the PR head. The build-vs-ship asymmetry is stated where callers will look for it: UNEVALUATED may let a build proceed and may never let a lane ship. That ruling previously lived only in one plan's prose, which is how PR #3080 merged past it. Confined to principle 9; `## Work Completion Criteria` is untouched because it is regex-parsed into worker system prompts and asserted byte-for-byte. --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5704f1850..9dc491e70 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ When creating AgentSessions manually to test worker or queue behavior, use a rec 6. **MINIMAL TOOLS** — loading all tools pollutes context and degrades performance. Start minimal, expand only if needed. 7. **DEFINITION OF DONE** — the authoritative list lives in [`.claude/skills-global/do-build/SKILL.md`](.claude/skills-global/do-build/SKILL.md) and is enforced by `/do-build` and the builder agent. 8. **PARALLEL EXECUTION** — spawn parallel sub-agents for genuinely independent tasks; never for sequential or dependent work. Aggregate results before reporting. -9. **SDLC PIPELINE** — an Eng-role AgentSession handles both orchestration and execution. `/sdlc` is a **single-stage router**: assess state, invoke ONE sub-skill, return. Never write code, run tests, or create plans directly; always delegate through sub-skills. Agent gating reads of a PR's head SHA must resolve through `tools/pr_head_resolver.py::resolve_pr_head_sha` (git-first via `git ls-remote refs/pull/N/head`), never a bare `gh` read: a stale `gh` head SHA matches the recorded verdict's trailer and flips the verdict-staleness gate from fail-closed to fail-open (see [`docs/features/gh-stale-state-verdict-gate.md`](docs/features/gh-stale-state-verdict-gate.md)). Ground truth on stages: [`.claude/skills-global/do-sdlc/SKILL.md`](.claude/skills-global/do-sdlc/SKILL.md). +9. **SDLC PIPELINE** — an Eng-role AgentSession handles both orchestration and execution. `/sdlc` is a **single-stage router**: assess state, invoke ONE sub-skill, return. Never write code, run tests, or create plans directly; always delegate through sub-skills. Agent gating reads of a PR's head SHA must resolve through `tools/pr_head_resolver.py::resolve_pr_head_sha` (git-first via `git ls-remote refs/pull/N/head`), never a bare `gh` read: a stale `gh` head SHA matches the recorded verdict's trailer and flips the verdict-staleness gate from fail-closed to fail-open (see [`docs/features/gh-stale-state-verdict-gate.md`](docs/features/gh-stale-state-verdict-gate.md)). The same rule now governs a second gate: a plan's `## Verification` rows grade `PASS` / `FAIL` / `UNEVALUATED`, and `tools/merge_predicate.py` refuses to merge on a `FAIL` or `UNEVALUATED` row, reading the aggregate the runner recorded rather than re-executing anything. That aggregate is trusted only while its stamped head SHA matches the PR's current head; a mismatched, absent, or unresolvable SHA refuses. `UNEVALUATED` may let a build proceed and may never let a lane ship — the split lives on the consumer, not on a row marker (see [`docs/features/machine-readable-dod.md`](docs/features/machine-readable-dod.md)). Ground truth on stages: [`.claude/skills-global/do-sdlc/SKILL.md`](.claude/skills-global/do-sdlc/SKILL.md). 10. **RESTART RUNNING SERVICES** — see the restart note under Commands. ## Development Workflow From 2ed0f61789b74c36bb3bade83378efb7984d4400 Mon Sep 17 00:00:00 2001 From: valorengels Date: Fri, 4 Sep 2026 09:55:16 +0700 Subject: [PATCH 17/19] Give the merge gate a writer, and stop rejecting indexed check tables (Refs #3065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both blockers from PR #3123's review, and both were instances of the defect class this lane exists to remove: a gate structurally incapable of firing. record_verification_outcomes had zero production callers. Neither runner called it, so the merge predicate's verification check always took its "no aggregate" branch and was reported-not-enforced for every lane. scripts/validate_build.py now carries the write behind --record-outcomes/--repo/--issue/--pr. REVIEW is the recording stage and BUILD deliberately is not. The record is only worth reading when stamped with the PR head SHA it was graded against, and BUILD grades before the lane has a PR; recording there would have written an unanchored aggregate that the predicate refuses, blocking every lane with a reason no lane could clear. docs/sdlc/do-pr-review.md invokes the script with the flag, docs/sdlc/do-build.md without it. To record what a run graded without running every command twice, this runner now delegates execution to run_checks and derives its report shape from the results, rather than carrying a parallel execution loop. That is the convergence the plan asked for: the two runners can no longer drift on what a check did, only on how it is printed. The check-table column contract searched only indices 1 and 2, which rejected the leading-index shape (| # | Check | Command | Expected |) that live plans already use — docs/plans/overclaim-guard-greps-whole-worktree.md went from 30 executable checks to 0 checks and 2 malformed rows. The contract now locates an Expected column immediately after a Command column with at least one column ahead of it, which still rejects both false positives the docstring names, and takes the check's name from the column before Command so an index column does not become the name. CLAUDE.md's new sentence claimed UNEVALUATED "may let a build proceed", which contradicted the three artifacts in this branch that say it blocks. Restated as what is actually true: each consumer owns its disposition, and the build gate blocks today. --- CLAUDE.md | 2 +- agent/verification_parser.py | 63 ++++++---- docs/features/machine-readable-dod.md | 21 ++++ docs/sdlc/do-build.md | 3 + docs/sdlc/do-pr-review.md | 8 +- scripts/validate_build.py | 149 ++++++++++++---------- tests/unit/test_validate_build.py | 171 ++++++++++++++++++++++++++ 7 files changed, 330 insertions(+), 87 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9dc491e70..d0d5b4964 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ When creating AgentSessions manually to test worker or queue behavior, use a rec 6. **MINIMAL TOOLS** — loading all tools pollutes context and degrades performance. Start minimal, expand only if needed. 7. **DEFINITION OF DONE** — the authoritative list lives in [`.claude/skills-global/do-build/SKILL.md`](.claude/skills-global/do-build/SKILL.md) and is enforced by `/do-build` and the builder agent. 8. **PARALLEL EXECUTION** — spawn parallel sub-agents for genuinely independent tasks; never for sequential or dependent work. Aggregate results before reporting. -9. **SDLC PIPELINE** — an Eng-role AgentSession handles both orchestration and execution. `/sdlc` is a **single-stage router**: assess state, invoke ONE sub-skill, return. Never write code, run tests, or create plans directly; always delegate through sub-skills. Agent gating reads of a PR's head SHA must resolve through `tools/pr_head_resolver.py::resolve_pr_head_sha` (git-first via `git ls-remote refs/pull/N/head`), never a bare `gh` read: a stale `gh` head SHA matches the recorded verdict's trailer and flips the verdict-staleness gate from fail-closed to fail-open (see [`docs/features/gh-stale-state-verdict-gate.md`](docs/features/gh-stale-state-verdict-gate.md)). The same rule now governs a second gate: a plan's `## Verification` rows grade `PASS` / `FAIL` / `UNEVALUATED`, and `tools/merge_predicate.py` refuses to merge on a `FAIL` or `UNEVALUATED` row, reading the aggregate the runner recorded rather than re-executing anything. That aggregate is trusted only while its stamped head SHA matches the PR's current head; a mismatched, absent, or unresolvable SHA refuses. `UNEVALUATED` may let a build proceed and may never let a lane ship — the split lives on the consumer, not on a row marker (see [`docs/features/machine-readable-dod.md`](docs/features/machine-readable-dod.md)). Ground truth on stages: [`.claude/skills-global/do-sdlc/SKILL.md`](.claude/skills-global/do-sdlc/SKILL.md). +9. **SDLC PIPELINE** — an Eng-role AgentSession handles both orchestration and execution. `/sdlc` is a **single-stage router**: assess state, invoke ONE sub-skill, return. Never write code, run tests, or create plans directly; always delegate through sub-skills. Agent gating reads of a PR's head SHA must resolve through `tools/pr_head_resolver.py::resolve_pr_head_sha` (git-first via `git ls-remote refs/pull/N/head`), never a bare `gh` read: a stale `gh` head SHA matches the recorded verdict's trailer and flips the verdict-staleness gate from fail-closed to fail-open (see [`docs/features/gh-stale-state-verdict-gate.md`](docs/features/gh-stale-state-verdict-gate.md)). The same rule now governs a second gate: a plan's `## Verification` rows grade `PASS` / `FAIL` / `UNEVALUATED`, and `tools/merge_predicate.py` refuses to merge on a `FAIL` or `UNEVALUATED` row, reading the aggregate the runner recorded rather than re-executing anything. That aggregate is trusted only while its stamped head SHA matches the PR's current head; a mismatched, absent, or unresolvable SHA refuses. Each consumer owns its own disposition for `UNEVALUATED` rather than reading a per-row marker: the merge predicate may never let one ship, and the build gate blocks on one today (see [`docs/features/machine-readable-dod.md`](docs/features/machine-readable-dod.md)). The aggregate is recorded at REVIEW, the first stage with a PR head to stamp it against; BUILD grades the same table but records nothing. Ground truth on stages: [`.claude/skills-global/do-sdlc/SKILL.md`](.claude/skills-global/do-sdlc/SKILL.md). 10. **RESTART RUNNING SERVICES** — see the restart note under Commands. ## Development Workflow diff --git a/agent/verification_parser.py b/agent/verification_parser.py index 22091614f..d35b661b9 100644 --- a/agent/verification_parser.py +++ b/agent/verification_parser.py @@ -247,29 +247,42 @@ def _iter_pipe_blocks(section: str) -> list[list[str]]: return blocks -def _is_check_table_header(header_cells: list[str]) -> bool: - """A block is a check table when its columns match the check **contract**. +def check_column_indices(header_cells: list[str]) -> tuple[int, int] | None: + """Locate a check table's ``(command, expected)`` column indices. - The contract is positional: at least three columns, the second named - ``Command`` and the third named ``Expected`` (case-insensitive). The first - column is the check's name and may be called anything (``Check``, - ``Anti-criterion``, ...). + The contract is a **shape**, not a fixed pair of offsets: an ``Expected`` + column immediately following a ``Command`` column (case-insensitive), with + at least one column ahead of ``Command`` to name the check. Returns the two + indices, or ``None`` when the block is not a check table. The predicate this replaced asked whether *any* of the first three column names was ``Command``, which is a question about vocabulary rather than about shape. A table shaped ``| Command | Observed stdout | Observed exit |`` -- a results recap, not a check list -- satisfied it, and its "Observed stdout" column was then executed as a shell command with no diagnostic - emitted (#3022). A sweep of this repo's plans finds every genuine check - table is ``(, Command, Expected)``, and exactly one false positive - (``| # | Criterion | Check |``-shaped recaps) that the contract rejects. + emitted (#3022). + + Pinning the pair to indices 1 and 2 fixed that but over-corrected: a + leading index column (``| # | Check | Command | Expected |``) is an + established shape in this repo's live plans, and pinning silently turned + one such plan's 30 executable checks into 0 checks and 2 malformed rows. + Searching for the adjacent pair keeps both false positives rejected -- + ``| Command | Observed stdout | Observed exit |`` has no column ahead of + ``Command`` and no following ``Expected``, and ``| # | Criterion | Check |`` + has no ``Command`` at all -- while accepting every genuine shape. """ - if len(header_cells) < 3: - return False - return ( - header_cells[1].strip().lower() == "command" - and header_cells[2].strip().lower() == "expected" - ) + for i in range(1, len(header_cells) - 1): + if ( + header_cells[i].strip().lower() == "command" + and header_cells[i + 1].strip().lower() == "expected" + ): + return i, i + 1 + return None + + +def _is_check_table_header(header_cells: list[str]) -> bool: + """Whether a block's header matches the check contract.""" + return check_column_indices(header_cells) is not None # The first backticked span in a command cell. Anything outside it -- a @@ -321,12 +334,14 @@ def parse_verification_table(markdown: str) -> ParsedTable: found or the section has no pipe-blocks at all. The section is split into pipe-blocks (see :func:`_iter_pipe_blocks`) and - each is classified independently. A **check table** (header carries a - ``Command`` column among its first three) contributes its data rows as + each is classified independently. A **check table** (an ``Expected`` column + immediately after a ``Command`` column, with at least one column ahead of + them -- see :func:`check_column_indices`) contributes its data rows as checks; the expected column count comes from its own header, so a table that carries an extra annotation column is read correctly instead of - having every row rejected. Only the first three columns of a check table - are used: Check, Command, Expected. + having every row rejected. Three columns of a check table are read: the one + ahead of ``Command`` for the name, then ``Command`` and ``Expected`` + wherever the header puts them. A non-check table becomes a non-failing :class:`SkippedTable`. When the section has pipe-blocks but none of them is a check table, that is a loud @@ -388,6 +403,10 @@ def parse_verification_table(markdown: str) -> ParsedTable: for block, header_cells in check_blocks: expected_columns = max(len(header_cells), 3) + command_idx, expected_idx = check_column_indices(header_cells) + # The check's name is the column immediately ahead of Command, so a + # leading index column yields the descriptive name rather than "1". + name_idx = command_idx - 1 for row in _block_data_rows(block): cells = split_row_cells(row) @@ -406,9 +425,9 @@ def parse_verification_table(markdown: str) -> ParsedTable: ) continue - name = cells[0] - raw_command = cells[1] - expected = cells[2] + name = cells[name_idx] + raw_command = cells[command_idx] + expected = cells[expected_idx] if not name or not raw_command.strip() or not expected: malformed.append( diff --git a/docs/features/machine-readable-dod.md b/docs/features/machine-readable-dod.md index f8d7c9390..4f195549e 100644 --- a/docs/features/machine-readable-dod.md +++ b/docs/features/machine-readable-dod.md @@ -123,6 +123,27 @@ that produced nothing executable) grades `UNEVALUATED` for the run, never a vacuous `PASS` — there being nothing to check is not evidence that everything checked out. +### Where the graded aggregate is recorded + +The merge gate reads a **recorded** aggregate; it never re-executes a plan's +commands. `record_verification_outcomes` writes it to `_verification_outcomes` +in the lane ledger's existing `stage_states` JSON, stamped with the PR head SHA +the run was graded against (resolved through +`tools.pr_head_resolver.resolve_pr_head_sha`, never a bare `gh` read). + +**REVIEW is the recording stage, and BUILD deliberately is not.** The stamp is +what makes the record trustworthy later, and BUILD grades the table before the +lane has a PR to stamp against. An unanchored record is refused at merge, so a +BUILD-time write would block every lane with a reason no lane could clear. +`scripts/validate_build.py --record-outcomes --repo --issue --pr

` is +the one production writer; `/do-pr-review` § 4.5 invokes it, `/do-build` +Step 5.1 runs the same script without the flag. Re-run it at DOCS if the head +moved after review. + +The write happens after the report is printed and never changes the exit code: +a run that cannot reach the ledger must still tell the human what its checks +found. + ### Table Scoping: One `## Verification` Section, Many Pipe-Blocks (#2836) A `## Verification` section can carry more than one markdown table -- a check diff --git a/docs/sdlc/do-build.md b/docs/sdlc/do-build.md index a7c4db726..41644a588 100644 --- a/docs/sdlc/do-build.md +++ b/docs/sdlc/do-build.md @@ -185,6 +185,9 @@ prevent. (cd $TARGET_REPO/.worktrees/{slug} && python scripts/evaluate_build.py $PLAN_PATH) # exit 2 → bundle FAILs to /do-patch, ≤2 iters; 3 = no criteria; 1 = non-blocking # Verification table runner: python -c "import sys; from agent.verification_parser import parse_verification_table, run_checks, format_results; t = parse_verification_table(open(PLAN_PATH).read()); r = run_checks(t.checks); print(format_results(r, t)); sys.exit(1 if t.malformed or not all(x.outcome == 'PASS' for x in r) else 0)" +# BUILD grades the table but records nothing: `--record-outcomes` is REVIEW's job, +# because the record is only trustworthy when stamped with a PR head SHA and there +# is no PR yet at BUILD time. See docs/features/machine-readable-dod.md. # Each result carries a three-valued `outcome` (PASS / FAIL / UNEVALUATED), never a # boolean. Only PASS clears the gate: UNEVALUATED means the GRADER could not answer # (a timeout, a runner exception, an expectation form the grammar does not recognise, diff --git a/docs/sdlc/do-pr-review.md b/docs/sdlc/do-pr-review.md index 353709132..12c3eb5b6 100644 --- a/docs/sdlc/do-pr-review.md +++ b/docs/sdlc/do-pr-review.md @@ -69,7 +69,13 @@ inheritance, not a block: use the returned `run_id` and continue; only a foreign **Verification-table runner (§ 4.5):** ```bash -python -c "import sys; from agent.verification_parser import parse_verification_table, run_checks, format_results; t = parse_verification_table(open(PLAN_PATH).read()); r = run_checks(t.checks); print(format_results(r, t)); sys.exit(1 if t.malformed or not all(x.outcome == 'PASS' for x in r) else 0)" +python scripts/validate_build.py $PLAN_PATH --record-outcomes --repo $TARGET_REPO --issue $ISSUE_NUMBER --pr $PR_NUMBER +# REVIEW is where the graded aggregate gets RECORDED, and it is the only stage +# that can: `--record-outcomes` stamps the record with the PR head SHA it was +# graded against, and the merge predicate refuses an aggregate it cannot show +# is fresh. BUILD runs the same table but has no PR yet, so it must NOT record +# -- an unanchored record is refused at merge with a reason no lane can clear. +# Re-run this at DOCS too if the head moved after review. # Each result carries a three-valued `outcome` (PASS / FAIL / UNEVALUATED), never a # boolean; `CheckOutcome` is a StrEnum, so `x.outcome == 'PASS'` is the comparison. # UNEVALUATED blocks like FAIL but says the GRADER could not answer -- report it as diff --git a/scripts/validate_build.py b/scripts/validate_build.py index 0eafedc16..03d1e7b83 100644 --- a/scripts/validate_build.py +++ b/scripts/validate_build.py @@ -25,11 +25,11 @@ from agent.verification_parser import ( # noqa: E402 DEFAULT_TIMEOUT_S, CheckOutcome, + CheckResult, ParsedTable, - evaluate_expectation, parse_verification_table, - timeout_reason, - unevaluated_reason, + record_verification_outcomes, + run_checks, ) @@ -197,13 +197,23 @@ def check_file_assertions(assertions: list[dict[str, str]]) -> list[dict]: return results -def check_verification_table(table: ParsedTable, *, timeout: int = DEFAULT_TIMEOUT_S) -> list[dict]: +def check_verification_table( + table: ParsedTable, + *, + timeout: int = DEFAULT_TIMEOUT_S, + check_results: list[CheckResult] | None = None, +) -> list[dict]: """Run verification table commands and compare output. - Delegates table definition, expectation grammar, execution bound, and + Delegates table definition, expectation grammar, **execution**, bound, and timeout disposition to ``agent.verification_parser`` (#2843/#3065) rather than carrying its own. This runner keeps only its report shape. + Execution goes through ``run_checks``, so the two runners cannot drift on + what a check *did*, only on how it is printed. ``check_results``, when + given, is extended with the graded :class:`CheckResult` objects so a caller + can persist the aggregate without running every command a second time. + The bound is ``DEFAULT_TIMEOUT_S`` and a timeout is ``UNEVALUATED``, both shared with ``run_checks``. This module previously carried a private 30s ceiling and called a timeout ``SKIP``, so the two runners graded the same @@ -238,64 +248,24 @@ def check_verification_table(table: ParsedTable, *, timeout: int = DEFAULT_TIMEO } ) - for check in table.checks: - cmd = check.command - expected = check.expected - name = check.name - - if check.unevaluated_reason: - # Read but unrunnable as written (no backticked span): never - # executed on a guess, never reported as FAIL. + graded = run_checks(table.checks, timeout=timeout) + if check_results is not None: + check_results.extend(graded) + + for r in graded: + name = r.check.name + if r.outcome is CheckOutcome.PASS: + results.append({"status": "PASS", "message": name}) + elif r.outcome is CheckOutcome.UNEVALUATED: + results.append({"status": "UNEVALUATED", "message": f"{name} -- {r.reason}"}) + else: results.append( { - "status": "UNEVALUATED", - "message": f"{name} -- {check.unevaluated_reason}", - } - ) - continue - - try: - result = subprocess.run( - cmd, shell=True, capture_output=True, text=True, timeout=timeout - ) - # `output` must be unstripped stdout -- run_checks passes proc.stdout - # unmodified, and a stripped copy here would re-create divergence at - # the exact seam this convergence closes. The stripped value is used - # only in the FAIL message. - actual_output = result.stdout - actual_exit = result.returncode - outcome = evaluate_expectation(expected, exit_code=actual_exit, output=actual_output) - - if outcome is CheckOutcome.PASS: - results.append({"status": "PASS", "message": name}) - elif outcome is CheckOutcome.UNEVALUATED: - results.append( - { - "status": "UNEVALUATED", - "message": f"{name} -- {unevaluated_reason(expected)}", - } - ) - else: - results.append( - { - "status": "FAIL", - "message": ( - f"{name} -- expected: {expected}," - f" got exit={actual_exit}" - f" output={actual_output.strip()[:100]}" - ), - } - ) - except subprocess.TimeoutExpired: - results.append( - {"status": "UNEVALUATED", "message": f"{name} -- {timeout_reason(timeout)}"} - ) - except Exception as e: - results.append( - { - "status": "UNEVALUATED", + "status": "FAIL", "message": ( - f"{name} -- runner error, the check never ran: {type(e).__name__}: {e}" + f"{name} -- expected: {r.check.expected}," + f" got exit={r.exit_code}" + f" output={r.output[:100]}" ), } ) @@ -339,18 +309,43 @@ def check_success_criteria(criteria: list[dict[str, str]]) -> list[dict]: def main() -> int: if len(sys.argv) < 2 or sys.argv[1] in ("--help", "-h"): - print("Usage: python scripts/validate_build.py ") + print("Usage: python scripts/validate_build.py [options]") print() print("Validates a build against the plan specification.") print("Checks file path assertions, verification table commands,") print("and success criteria commands.") print() + print("Options:") + print(" --record-outcomes Persist the graded aggregate to the lane's ledger,") + print(" where the merge predicate reads it. Requires") + print(" --repo and --issue; pass --pr so the record is") + print(" stamped with the head SHA it was graded against.") + print(" An unstamped record is refused at merge, so record") + print(" at REVIEW/DOCS time, once the lane has a PR.") + print(" --repo OWNER/NAME Target repo for the ledger key.") + print(" --issue N Issue number for the ledger key.") + print(" --pr N PR whose head SHA anchors the record.") + print() print("Exit codes:") print(" 0 - All checks pass or skip") print(" 1 - One or more checks failed") return 0 - plan_path = Path(sys.argv[1]) + argv = sys.argv[1:] + + def _opt(flag: str) -> str | None: + if flag in argv: + i = argv.index(flag) + if i + 1 < len(argv): + return argv[i + 1] + return None + + record_outcomes = "--record-outcomes" in argv + opt_repo = _opt("--repo") + opt_issue = _opt("--issue") + opt_pr = _opt("--pr") + + plan_path = Path(argv[0]) if not plan_path.exists(): print(f"Plan file not found: {plan_path}") print("Nothing to validate.") @@ -371,8 +366,9 @@ def main() -> int: # 2. Verification table verification_table = parse_verification_table(plan_text) + graded: list[CheckResult] = [] if verification_table.checks or verification_table.malformed or verification_table.skipped: - all_results.extend(check_verification_table(verification_table)) + all_results.extend(check_verification_table(verification_table, check_results=graded)) # 3. Success criteria commands success_criteria = parse_success_criteria_commands(plan_text) @@ -398,6 +394,33 @@ def main() -> int: f"{unevaluated_count} UNEVALUATED, {skip_count} SKIP" ) + # Persist last, and never let a ledger failure change what the human is + # told: the write reports its own success or failure on its own line and + # does not touch the exit code, which belongs to the checks. + if record_outcomes: + if not opt_repo or not opt_issue: + print("RECORD: skipped -- --record-outcomes requires --repo and --issue") + else: + wrote = record_verification_outcomes( + opt_repo, + int(opt_issue), + graded, + table=verification_table, + pr_number=int(opt_pr) if opt_pr else None, + ) + if wrote: + anchor = f"anchored to PR #{opt_pr} head" if opt_pr else "UNANCHORED" + print( + f"RECORD: verification outcomes written for {opt_repo}#{opt_issue} ({anchor})" + ) + if not opt_pr: + print( + "RECORD: no --pr given, so no head SHA was stamped. " + "The merge predicate refuses an unanchored record." + ) + else: + print(f"RECORD: FAILED to write verification outcomes for {opt_repo}#{opt_issue}") + # UNEVALUATED blocks. It is not a pass, and it is not a FAIL either: the # exit code says "stop", the report says the grader could not answer. return 1 if (fail_count or unevaluated_count) else 0 diff --git a/tests/unit/test_validate_build.py b/tests/unit/test_validate_build.py index 65927a698..45ed5750a 100644 --- a/tests/unit/test_validate_build.py +++ b/tests/unit/test_validate_build.py @@ -552,3 +552,174 @@ def test_parse_only_fixtures_parse_identically(self, fixture_name): assert len(table.checks) == expected["checks"] assert len(table.malformed) == expected["malformed"] assert len(table.skipped) == expected["skipped"] + + +class TestLeadingIndexColumnIsACheckTable: + """A check table may carry a leading index column (review of PR #3123). + + Pinning the `(Command, Expected)` pair to indices 1 and 2 rejected the + `| # | Check | Command | Expected |` shape that live plans in this repo + already use, silently turning one plan's 30 executable checks into 0 + checks and 2 malformed rows. That is a gate incapable of firing -- the + defect class #3065 exists to remove -- so the shape is pinned here. + """ + + INDEXED = textwrap.dedent("""\ + ## Verification + | # | Check | Command | Expected | + |---|-------|---------|----------| + | 1 | Echo works | `echo hi` | output contains hi | + | 2 | True exits 0 | `true` | exit code 0 | + """) + + def test_indexed_header_yields_executable_checks(self): + table = parse_verification_table(self.INDEXED) + assert len(table.checks) == 2 + assert not table.malformed + assert not table.skipped + + def test_name_comes_from_the_column_before_command(self): + """Not from column 0, which is the index.""" + table = parse_verification_table(self.INDEXED) + assert [c.name for c in table.checks] == ["Echo works", "True exits 0"] + assert [c.command for c in table.checks] == ["echo hi", "true"] + + def test_indexed_table_actually_grades(self): + table = parse_verification_table(self.INDEXED) + results = run_checks(table.checks, timeout=10) + assert [r.outcome for r in results] == [CheckOutcome.PASS, CheckOutcome.PASS] + + def test_results_recap_shape_is_still_rejected(self): + """The #3022 false positive must stay rejected: no column ahead of + `Command`, and no `Expected` following it.""" + recap = textwrap.dedent("""\ + ## Verification + | Command | Observed stdout | Observed exit | + |---------|-----------------|---------------| + | `echo hi` | hi | 0 | + """) + table = parse_verification_table(recap) + assert not table.checks + + def test_criterion_recap_shape_is_still_rejected(self): + recap = textwrap.dedent("""\ + ## Verification + | # | Criterion | Check | + |---|-----------|-------| + | 1 | Something | Manual | + """) + table = parse_verification_table(recap) + assert not table.checks + + +class TestRecordOutcomesHasAProductionCaller: + """`record_verification_outcomes` must be reachable from a real runner. + + Review of PR #3123 found the writer had zero production callers, so the + merge predicate's verification group always took its `aggregate is None` + branch -- a gate that could never fire. The recording flag on this runner + is that caller; these tests fail if it is removed or silently no-ops. + """ + + PLAN = textwrap.dedent("""\ + ## Verification + | Check | Command | Expected | + |-------|---------|----------| + | Echo works | `echo hi` | output contains hi | + """) + + def _plan_file(self, tmp_path): + f = tmp_path / "plan.md" + f.write_text(self.PLAN) + return f + + def test_recording_flag_calls_the_writer_with_graded_results(self, tmp_path): + f = self._plan_file(tmp_path) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "owner/name", + "--issue", + "4242", + "--pr", + "77", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + writer.return_value = True + assert validate_build.main() == 0 + + writer.assert_called_once() + args, kwargs = writer.call_args + assert args[0] == "owner/name" + assert args[1] == 4242 + assert kwargs["pr_number"] == 77 + graded = args[2] + assert [r.outcome for r in graded] == [CheckOutcome.PASS], ( + "the writer must receive the results this run actually graded" + ) + + def test_no_flag_records_nothing(self, tmp_path): + f = self._plan_file(tmp_path) + with ( + patch("sys.argv", ["validate_build.py", str(f)]), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + assert validate_build.main() == 0 + writer.assert_not_called() + + def test_recording_without_repo_or_issue_is_refused_not_guessed(self, tmp_path): + f = self._plan_file(tmp_path) + with ( + patch("sys.argv", ["validate_build.py", str(f), "--record-outcomes"]), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + assert validate_build.main() == 0 + writer.assert_not_called() + + def test_a_failed_write_does_not_change_the_exit_code(self, tmp_path): + """The exit code belongs to the checks, not to the ledger.""" + f = self._plan_file(tmp_path) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "owner/name", + "--issue", + "4242", + "--pr", + "77", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + writer.return_value = False + assert validate_build.main() == 0 + + def test_commands_are_executed_once_not_twice(self, tmp_path): + """Recording reuses the graded results; it must not re-run the table.""" + f = self._plan_file(tmp_path) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "owner/name", + "--issue", + "4242", + "--pr", + "77", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes", return_value=True), + patch.object(validate_build, "run_checks", wraps=validate_build.run_checks) as rc, + ): + validate_build.main() + assert rc.call_count == 1 From 76e8d30557830e3814e0e18331d80a11cbccaf1e Mon Sep 17 00:00:00 2001 From: valorengels Date: Fri, 4 Sep 2026 13:20:36 +0700 Subject: [PATCH 18/19] Arm the merge gate safely: strict argv, fail-closed read, router-side twin (Refs #3065) Round-2 review returned three blockers, all in the arming of the gate rather than its design. The writer's argv parser accepted a flag as a value. The documented invocation interpolates shell variables, so an empty one collapsed the argument list and the naive reading took the next flag: an empty issue number raised ValueError after the summary had printed and rewrote the exit code the checks own, an empty repo wrote a real ledger row under a repo named "--issue" and reported success, and a flag ahead of the plan path exited 0 having run no checks at all. _parse_argv now rejects a value that is itself a flag, takes the plan path from the first non-flag positional, and reports an unparseable --issue/--pr instead of raising through the report. All three modes have tests. read_verification_outcomes failed open. It was the one read in the merge path that fetched the blocking evidence, and the predicate could not tell an error from an absence, so a store blip converted a recorded FAIL into an unenforced pass -- inverted against every neighbouring group. Absence still returns None; anything that prevents an answer now raises VerificationOutcomesUnavailableError and the predicate refuses on it. Group (e) was a merge refusal no dispatch rule could see. Row 8g is its routing-side twin: a blocking or unfresh aggregate routes to /do-pr-review, which re-records it, instead of dead-ending on a merge the predicate is certain to refuse until G4 blocks the lane. Its dispositions mirror the predicate's so the two cannot drift, and a test asserts that agreement across the whole CheckOutcome enum. Also from the review, in blast radius: the per-check bound gets a lever (VERIFICATION_TIMEOUT_S, --timeout) now that a timeout is a durable merge refusal rather than a non-blocking skip; merge-troubleshooting.md documents both new refusals and their recovery; the REVIEW invocation is pinned by test rather than by prose alone; the trailing-gloss rule is symmetric across >, >= and == , which unblocks three live rows in other plans; G4's block carries the evidence Cluster D exists to attach; and four tests stop leaking real issue locks into the shared Redis. Plan verification table: 29 PASS, 0 FAIL, 0 UNEVALUATED. --- agent/sdlc_router.py | 118 ++++++++++- agent/verification_parser.py | 135 ++++++++++--- .../sdlc-control-plane-asserted-facts.md | 16 ++ docs/sdlc/do-pr-review.md | 6 +- docs/sdlc/merge-troubleshooting.md | 48 +++++ scripts/validate_build.py | 114 +++++++++-- ...sdlc_session_ensure_readback_provenance.py | 46 ++++- tests/unit/test_merge_predicate.py | 35 ++++ tests/unit/test_sdlc_router.py | 182 +++++++++++++++++ tests/unit/test_validate_build.py | 115 +++++++++++ tests/unit/test_verification_parser.py | 185 +++++++++++++++++- tools/merge_predicate.py | 20 +- 12 files changed, 960 insertions(+), 60 deletions(-) diff --git a/agent/sdlc_router.py b/agent/sdlc_router.py index e82c7b326..54bbbd553 100644 --- a/agent/sdlc_router.py +++ b/agent/sdlc_router.py @@ -45,6 +45,7 @@ STAGE_TO_SKILL, ) from agent.pipeline_state import SETTLED_STATUSES +from agent.verification_parser import VERIFICATION_OUTCOMES_KEY, CheckOutcome logger = logging.getLogger(__name__) @@ -615,6 +616,13 @@ def guard_g2_critique_cycle_cap( f"Escalating to human." ), guard_id="G2", + decision_inputs=build_decision_inputs( + stage_states, + meta, + critique_cycle_count=cycles, + max_critique_cycles=MAX_CRITIQUE_CYCLES, + critique_status=critique_status, + ), ) @@ -682,7 +690,7 @@ def guard_g3_pr_lock(stage_states: dict, meta: dict, context: dict) -> Dispatch if review_status == STATUS_COMPLETED and review_approved and docs_status == STATUS_COMPLETED: target = SKILL_DO_MERGE suffix = "review clean and docs complete" - elif review_status == STATUS_COMPLETED and review_approved and docs_status != STATUS_COMPLETED: + elif review_status == STATUS_COMPLETED and review_approved: target = SKILL_DO_DOCS suffix = G3_REDIRECT_REASON_DOCS_PENDING elif REVIEW_CHANGES_REQUESTED in review_verdict_norm or review_status == STATUS_FAILED: @@ -728,6 +736,13 @@ def guard_g4_oscillation( "`sdlc-tool dispatch reset --issue-number N`." ), guard_id="G4", + decision_inputs=build_decision_inputs( + stage_states, + meta, + last_dispatched_skill=skill, + same_stage_dispatch_count=count, + max_same_stage_dispatches=MAX_SAME_STAGE_DISPATCHES, + ), ) @@ -1029,6 +1044,12 @@ def guard_g7_plan_revising( f"revision is already complete." ), guard_id="G7", + decision_inputs=build_decision_inputs( + stage_states, + meta, + recent_skills=recent_skills, + max_plan_revising_dispatches=MAX_PLAN_REVISING_DISPATCHES, + ), ) # A plan dispatch is already in the recent history — let dispatch table route. @@ -1246,6 +1267,12 @@ def guard_g9_blocked_on_conflict( f"resolves conflicts — this needs a human." ), guard_id="G9", + decision_inputs=build_decision_inputs( + stage_states, + meta, + pr_number=pr_number, + pr_merge_state=merge_state, + ), ) @@ -2153,6 +2180,64 @@ def _rule_review_approved_docs_not_done(stage_states: dict, meta: dict, context: return docs_status not in (STATUS_COMPLETED,) +def _rule_verification_outcomes_hold_pr(stage_states: dict, meta: dict, context: dict) -> bool: + """Recorded verification aggregate holds the PR — re-review, do not merge. + + The dispatch-table twin of ``tools/merge_predicate``'s group (e), and the + same construction row 8f uses for group (c). Without it, group (e) is a + merge-refusal condition no dispatch rule can see: row 10 fires, + ``/do-merge`` is dispatched, the predicate refuses on a ``FAIL`` or + ``UNEVALUATED`` row, and the router re-dispatches ``/do-merge`` unchanged + until ``guard_g4_oscillation`` blocks the lane for a human. That is exactly + the router-predicate oscillation loop WS3d/#2062 existed to end, + reintroduced on the verification axis. + + The dispositions mirror the predicate's, so the two cannot disagree: + + - no recorded aggregate → **False**. Absence is reported and not enforced + on the ship side either; a lane that never recorded one is not blocked. + - ``FAIL`` or ``UNEVALUATED`` → **True**. The #3080 / ``ba092a06d`` ruling: + both hold the PR. + - ``PASS`` but not provably fresh — no ``head_sha`` on the record, a live + head the CLI could not resolve (the empty-string sentinel), or a head + that differs from the record's → **True**. A cached PASS from before the + current head is the defect this whole mechanism exists to close. + - ``PASS`` anchored to the current head → **False**. Row 10 may merge. + + Reads ``stage_states`` only; the aggregate already travels in that blob, so + this rule makes no network call. Scoped to APPROVED verdicts because a lane + that is not approved is owned by the review/patch rows. + + Termination: ``/do-pr-review`` re-runs the table and re-records an anchored + aggregate, so a stale or unanchored record converges in one pass. A record + that is genuinely ``FAIL`` converges the other way -- the re-review records + findings and flips the verdict, handing the lane to the patch rows -- and + is loop-bound by G4 in the interim, exactly as row 8f is. + """ + if not meta.get("pr_number"): + return False + if REVIEW_APPROVED not in normalize_verdict(_latest_review_verdict(stage_states, meta)): + return False + + aggregate = stage_states.get(VERIFICATION_OUTCOMES_KEY) + if not isinstance(aggregate, dict): + return False + + outcome = str(aggregate.get("outcome") or "").strip().upper() + if outcome in (CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value): + return True + + if "pr_head_sha" not in context: + return False + head_sha = context.get("pr_head_sha") or "" + if not head_sha: + return True + recorded_head = str(aggregate.get("head_sha") or "") + if not recorded_head: + return True + return recorded_head.lower() != head_sha.lower() + + def _rule_ready_to_merge(stage_states: dict, meta: dict, context: dict) -> bool: """Review APPROVED, zero findings, docs done, ready to merge.""" if not meta.get("pr_number"): @@ -2213,6 +2298,11 @@ def _rule_ready_to_merge(stage_states: dict, meta: dict, context: dict) -> bool: _rule_review_approved_docs_not_done.__doc__ = ( "Review APPROVED with zero findings, docs NOT done (see Step 3)" ) +_rule_verification_outcomes_hold_pr.__doc__ = ( + "Recorded verification outcomes carry a FAIL/UNEVALUATED row, or a PASS not anchored " + "to the current PR head — re-review re-records them instead of dispatching a merge " + "the predicate will refuse" +) _rule_ready_to_merge.__doc__ = ( "Review APPROVED (recorded verdict, head_sha-fresh) with zero findings, docs done, " "AND all display stages show completed in stage_states " @@ -2378,6 +2468,19 @@ def _rule_ready_to_merge(stage_states: dict, meta: dict, context: dict) -> bool: skill=SKILL_DO_DOCS, reason="Docs are required before merge", ), + # Row 8g mirrors merge_predicate group (e) on the routing side. Ordered + # immediately before row 10 so it preempts only the merge dispatch: a lane + # with docs outstanding still goes to row 9 first, and only a lane that + # would otherwise be sent to a gate certain to refuse is re-reviewed. + DispatchRule( + row_id="8g", + state_predicate=_rule_verification_outcomes_hold_pr, + skill=SKILL_DO_PR_REVIEW, + reason=( + "recorded verification outcomes hold the PR (blocking row, or a PASS that " + "is not anchored to the current head) — re-review re-records them" + ), + ), DispatchRule( row_id="10", state_predicate=_rule_ready_to_merge, @@ -2497,6 +2600,12 @@ def reconcile_dispatch( else: second_summary = {"reason": second_veto.reason, "evidence": second_veto.evidence} + # RECONCILE_DEADLOCK is the loop-bound for this bounded reconciliation pass: + # it returns both vetoes' verdicts here, on the second guard pass, instead of + # letting the caller iterate into G4's oscillation cap several turns later. + # Currently reachable only from this guard-reconciliation path, so it has no + # production coverage yet — it exists as the fail-closed stop for the two-veto + # case Risk 2 (above) describes. return Blocked( reason=( f"reconciliation: guard veto did not converge — table selected row " @@ -2591,6 +2700,13 @@ def decide_next_dispatch( f"(target repo: {resolved_repo}; check GH_REPO / SDLC_TARGET_REPO env)" ), guard_id=None, + decision_inputs=build_decision_inputs( + stage_states, + meta, + pr_number=pr_num, + pr_merge_state=pr_state, + resolved_target_repo=resolved_repo, + ), ) return Blocked( reason="no matching dispatch rule", diff --git a/agent/verification_parser.py b/agent/verification_parser.py index d35b661b9..457adc7ae 100644 --- a/agent/verification_parser.py +++ b/agent/verification_parser.py @@ -43,9 +43,11 @@ consumes rows until a blank line or a line that cannot be part of the table). Every pipe-block in the section is classified on its own, independently: -- A block is a **check table** when its columns match the check contract: at - least three columns, the second named ``Command`` and the third ``Expected`` - (case-insensitive). Every data row in it is parsed as a check. +- A block is a **check table** when its columns match the check contract: an + ``Expected`` column immediately after a ``Command`` column (case-insensitive), + with at least one column ahead of them naming the check. The pair is located, + not pinned to fixed offsets, so a leading index column is fine. Every data row + in it is parsed as a check. - A block that is not a check table -- a red/green summary, a findings recap -- becomes a :class:`SkippedTable`: named, reported, and non-failing. A second markdown table in the section is legitimate plan authoring; treating @@ -72,11 +74,13 @@ a human the time to discover the difference, and the 2026-09 supervisor batch hand-verified every such "failure" as actually passing. -Table classification is by column **contract** -- columns 2 and 3 of the -header must be ``Command`` and ``Expected`` -- not by the word ``Command`` -appearing anywhere in the first three positions. A table shaped -``| Command | Observed stdout | Observed exit |`` used to be classified as a -check table and have its *second* column executed as a shell command (#3022). +Table classification is by column **contract** -- an ``Expected`` column +directly after a ``Command`` column, with something ahead of them to name the +check -- not by the word ``Command`` appearing anywhere in the first three +positions. A table shaped ``| Command | Observed stdout | Observed exit |`` used +to be classified as a check table and have its *second* column executed as a +shell command (#3022); it has no name column and no following ``Expected``, so +the contract rejects it. The escape composes, which matters for basic-regex ``grep``: in a BRE, alternation is spelled ``\\|``, and to get that through the table you double @@ -93,6 +97,7 @@ import json import logging +import os import re import subprocess from dataclasses import dataclass @@ -108,9 +113,19 @@ # One bound, shared by every runner of this repo's verification tables. The # second runner (`scripts/validate_build.py`) carried its own 30s ceiling and # its own SKIP-on-timeout disposition, so the two graded the same event two -# different ways (#2901). Provisional and tunable: raise it if a legitimate -# suite starts brushing the ceiling rather than letting rows go UNEVALUATED. -DEFAULT_TIMEOUT_S = 120 +# different ways (#2901). Provisional and tunable. +# +# The bound needs a lever because a timeout is now a durable merge refusal, not +# the non-blocking SKIP it used to be: the slowest row in this repo's plans sits +# around a quarter of the bound on a quiet machine, so heavy contention alone +# can push a legitimate suite over and hold a PR. Raise it via +# VERIFICATION_TIMEOUT_S (or `--timeout`) when a real suite brushes the ceiling +# -- but contention is a load problem, not a bound problem, so prefer rerunning +# on a quiet machine over permanently inflating this. +try: + DEFAULT_TIMEOUT_S = int(os.environ.get("VERIFICATION_TIMEOUT_S", "") or 120) +except ValueError: + DEFAULT_TIMEOUT_S = 120 # Underscore-prefixed metadata key inside the ledger's `stage_states_json` # blob, mirroring `_verdicts` / `_sdlc_dispatches` / `_run_identities`. A new @@ -403,7 +418,20 @@ def parse_verification_table(markdown: str) -> ParsedTable: for block, header_cells in check_blocks: expected_columns = max(len(header_cells), 3) - command_idx, expected_idx = check_column_indices(header_cells) + indices = check_column_indices(header_cells) + if indices is None: + # Unreachable in practice: check_blocks was filtered through + # _is_check_table_header, which is this same call returning + # non-None. Guarded explicitly anyway so the invariant is + # enforced in code rather than assumed at this type seam. + malformed.append( + MalformedRow( + line=block[0], + reason="table header no longer matches the check contract", + ) + ) + continue + command_idx, expected_idx = indices # The check's name is the column immediately ahead of Command, so a # leading index column yields the descriptive name rather than "1". name_idx = command_idx - 1 @@ -594,28 +622,47 @@ def numeric_verdict(op) -> CheckOutcome: if m: return verdict(output.strip() == m.group(1).strip()) - # output >= N / >= N (anchored, see the note below) - m = re.match(r"(?:output\s*)?>=\s*(\d+)\s*$", expected) + # Trailing-gloss rule (applies uniformly to >, >=, ==): the `output`-prefixed + # spellings (`output > N`, `output >= N`, `output == N`) are the established + # authoring idiom in live plans -- e.g. `output > 0 (a bare file-wide grep + # returns 3 today)` or `output == 2 (the two read sites)` -- so they are + # prefix-matched and tolerate a trailing gloss. The bare spellings (`> N`, + # `>= N`, `== N`) have no such idiom behind them and stay anchored, so a + # trailing gloss on a bare form is UNEVALUATED. Each `output`-prefixed + # branch is tried before its bare counterpart so the prefix match wins. + + # output >= N -- prefix-matched (see the trailing-gloss rule above). + m = re.match(r"output\s*>=\s*(\d+)", expected) if m: threshold = int(m.group(1)) return numeric_verdict(lambda value: value >= threshold) - # output > N -- prefix-matched, preserving the long-standing reading of - # `output > 0 (a bare file-wide grep returns 3 today)`, which several live - # plans write. The bare `> N` form below is anchored instead. + # >= N (anchored, see the trailing-gloss rule above) + m = re.match(r">=\s*(\d+)\s*$", expected) + if m: + threshold = int(m.group(1)) + return numeric_verdict(lambda value: value >= threshold) + + # output > N -- prefix-matched (see the trailing-gloss rule above). m = re.match(r"output\s*>\s*(\d+)", expected) if m: threshold = int(m.group(1)) return numeric_verdict(lambda value: value > threshold) - # > N (anchored, see the note below) + # > N (anchored, see the trailing-gloss rule above) m = re.match(r">\s*(\d+)\s*$", expected) if m: threshold = int(m.group(1)) return numeric_verdict(lambda value: value > threshold) - # output == N / == N (anchored, see the note below) - m = re.match(r"(?:output\s*)?==\s*(\d+)\s*$", expected) + # output == N -- prefix-matched (see the trailing-gloss rule above). + m = re.match(r"output\s*==\s*(\d+)", expected) + if m: + target = int(m.group(1)) + return numeric_verdict(lambda value: value == target) + + # == N (anchored, see the trailing-gloss rule above) + m = re.match(r"==\s*(\d+)\s*$", expected) if m: target = int(m.group(1)) return numeric_verdict(lambda value: value == target) @@ -933,11 +980,30 @@ def write_outcomes(states: dict) -> dict: return False +class VerificationOutcomesUnavailableError(Exception): + """The recorded aggregate could not be read, as distinct from absent. + + A merge gate must tell these two apart. "No aggregate was ever recorded" + is a lane the gate deliberately does not block; "the aggregate exists but + the read failed" is a lane about which nothing is known, and treating the + second as the first converts a recorded ``FAIL`` into an unenforced pass on + a Redis blip. Every neighbouring group in ``tools/merge_predicate.py`` + fails closed on its own read error; this makes that possible here. + """ + + def read_verification_outcomes(target_repo: str | None, issue_number: int | None) -> dict | None: """Return the recorded aggregate for a lane, or ``None`` if there is none. - Non-mutating (uses :meth:`PipelineLedger.get`, so a read never litters an - empty ledger) and fails OPEN to ``None`` on any error or malformed blob. + Non-mutating: uses :meth:`PipelineLedger.get`, so a read never litters an + empty ledger. + + Fails **closed**. ``None`` means genuine absence -- no ledger, no + ``stage_states`` blob, or no ``_verification_outcomes`` key in it. Anything + that prevents an answer (an unreachable store, an unparseable blob, a + record of the wrong shape) raises :class:`VerificationOutcomesUnavailableError` + rather than reporting absence, so the caller can refuse instead of + silently passing a lane it could not check. """ if not target_repo or not issue_number: return None @@ -948,11 +1014,11 @@ def read_verification_outcomes(target_repo: str | None, issue_number: int | None if ledger is None: return None raw = ledger.stage_states_json - blob = json.loads(raw) if isinstance(raw, str) else raw - if not isinstance(blob, dict): + if raw is None or raw == "": return None - record = blob.get(VERIFICATION_OUTCOMES_KEY) - return record if isinstance(record, dict) else None + blob = json.loads(raw) if isinstance(raw, str) else raw + except VerificationOutcomesUnavailableError: + raise except Exception as exc: logger.debug( "read_verification_outcomes: read failed for %s#%s (%s: %s)", @@ -961,4 +1027,21 @@ def read_verification_outcomes(target_repo: str | None, issue_number: int | None type(exc).__name__, exc, ) + raise VerificationOutcomesUnavailableError( + f"could not read verification outcomes for {target_repo}#{issue_number}: " + f"{type(exc).__name__}: {exc}" + ) from exc + + if not isinstance(blob, dict): + raise VerificationOutcomesUnavailableError( + f"stage_states for {target_repo}#{issue_number} is {type(blob).__name__}, not an object" + ) + if VERIFICATION_OUTCOMES_KEY not in blob: return None + record = blob[VERIFICATION_OUTCOMES_KEY] + if not isinstance(record, dict): + raise VerificationOutcomesUnavailableError( + f"recorded verification outcomes for {target_repo}#{issue_number} are " + f"{type(record).__name__}, not an object" + ) + return record diff --git a/docs/plans/sdlc-control-plane-asserted-facts.md b/docs/plans/sdlc-control-plane-asserted-facts.md index 040342e7f..d80b02953 100644 --- a/docs/plans/sdlc-control-plane-asserted-facts.md +++ b/docs/plans/sdlc-control-plane-asserted-facts.md @@ -728,6 +728,22 @@ table, and the anti-criteria rows are written to fail if the tri-state is absent truth depends on the modified evaluator is a row that proves nothing; the Verification section keeps those to direct `grep`/`pytest` assertions. +### Risk 8: The #3080 gate only fires on a RECORDED aggregate; nothing enforces that § 4.5 ran + +**Impact:** `tools/merge_predicate.py::_check_verification_outcomes` grades a recorded aggregate +against the PR's current head, but a lane where the § 4.5 verification-table runner is simply never +invoked has no aggregate to grade at all — that case is "reported, not enforced" by design (see the +module's own comment at `tools/merge_predicate.py:762-764`). The invocation itself is not a +mechanical trigger anywhere in this codebase; it is prose in `docs/sdlc/do-pr-review.md` telling the +REVIEW stage to run it. A lane that skips or forgets that step merges unimpeded by this gate, which +means the #3080 ruling is machine-readable once graded but not machine-*guaranteed* to be graded. +**Mitigation:** this is a known, accepted open edge, not a defect — fail-closed-on-absence was +deliberately rejected because it has no incident backing it (no observed case of a lane skipping +§ 4.5 to dodge the ruling) and because it would block every lane whose plan predates this mechanism, +none of which ever recorded an aggregate. Closing this edge, if it is ever worth closing, means making +the § 4.5 invocation itself mechanical (a hook or a stage-transition check) rather than tightening +this gate's absence handling. + ## Race Conditions ### Race 1: Branch truth is a live remote read taken mid-push diff --git a/docs/sdlc/do-pr-review.md b/docs/sdlc/do-pr-review.md index 12c3eb5b6..21a9091d1 100644 --- a/docs/sdlc/do-pr-review.md +++ b/docs/sdlc/do-pr-review.md @@ -69,7 +69,11 @@ inheritance, not a block: use the returned `run_id` and continue; only a foreign **Verification-table runner (§ 4.5):** ```bash -python scripts/validate_build.py $PLAN_PATH --record-outcomes --repo $TARGET_REPO --issue $ISSUE_NUMBER --pr $PR_NUMBER +python scripts/validate_build.py "$PLAN_PATH" --record-outcomes --repo "$TARGET_REPO" --issue "$ISSUE_NUMBER" --pr "$PR_NUMBER" +# Quote every variable. An empty unquoted one collapses the argument list, and +# the parser then rejects the flag rather than eating the next one -- but an +# empty *quoted* value is at least visible as an empty value instead of +# silently shifting every later argument. # REVIEW is where the graded aggregate gets RECORDED, and it is the only stage # that can: `--record-outcomes` stamps the record with the PR head SHA it was # graded against, and the merge predicate refuses an aggregate it cannot show diff --git a/docs/sdlc/merge-troubleshooting.md b/docs/sdlc/merge-troubleshooting.md index 8d05de62e..4278f8309 100644 --- a/docs/sdlc/merge-troubleshooting.md +++ b/docs/sdlc/merge-troubleshooting.md @@ -99,6 +99,54 @@ should have `created_at > $LATEST`. Re-dispatch `/do-merge {pr}`. --- +## Verification Outcomes Hold the PR + +**Symptom.** `/do-merge` refuses with one of: + +- `verification row '' is UNEVALUATED` (or `is FAIL`) +- `verification outcomes: verification outcome predates PR head commit` +- `verification outcomes: no usable head_sha on the recorded aggregate` +- `verification outcomes: recorded aggregate unreadable (...)` + +**Cause.** The merge predicate reads the aggregate the verification runner +recorded for this lane and refuses on a blocking row or on one it cannot show +is fresh. `FAIL` and `UNEVALUATED` both hold the PR (owner ruling, `ba092a06d`): +`FAIL` says the code is wrong, `UNEVALUATED` says the grader could not answer, +and the second is usually a plan-authoring problem, not a code problem. + +**Diagnose.** Run the table and read the report: + +```bash +python scripts/validate_build.py "$PLAN_PATH" +``` + +**Fix.** Depends on which line you got: + +- **A `FAIL` row** — real finding. Route to `/do-patch`, not around the gate. +- **An `UNEVALUATED` row** — fix the *row*. It names its own reason: an + unrecognised expectation form, an empty `Expected` cell, a `Command` cell + with no backticked span, or a timeout. For a timeout on a genuinely slow but + legitimate suite, re-run on a quiet machine first; raise the bound only if it + is really too low (`--timeout N`, or `VERIFICATION_TIMEOUT_S`). Contention is + a load problem, not a bound problem. +- **A stale, unanchored, or unreadable aggregate** — nothing is wrong with the + code; the record just cannot be trusted at this head. Re-record it: + +```bash +python scripts/validate_build.py "$PLAN_PATH" --record-outcomes \ + --repo "$TARGET_REPO" --issue "$ISSUE_NUMBER" --pr "$PR_NUMBER" +``` + +Quote every variable: an empty unquoted one collapses the argument list and the +run refuses to record rather than writing under a garbage key. + +**Verify.** Re-dispatch `/do-merge {pr}`. Note that the router reaches this +state on its own too — dispatch row 8g sends a lane with a blocking or unfresh +aggregate back to `/do-pr-review`, which re-runs § 4.5 and re-records, rather +than looping on a merge the predicate will refuse. + +--- + ## Lockfile Drift **Symptom.** The Lockfile Sync Check reports diff --git a/scripts/validate_build.py b/scripts/validate_build.py index 03d1e7b83..57845ac22 100644 --- a/scripts/validate_build.py +++ b/scripts/validate_build.py @@ -307,6 +307,54 @@ def check_success_criteria(criteria: list[dict[str, str]]) -> list[dict]: return results +_VALUE_FLAGS = ("--repo", "--issue", "--pr", "--timeout") +_BARE_FLAGS = ("--record-outcomes",) + + +def _parse_argv(argv: list[str]) -> tuple[dict[str, str], list[str], list[str]]: + """Split argv into ``(options, positionals, rejected_flags)``. + + A value flag whose next token is missing or is itself a flag is + **rejected**, not silently satisfied. The documented production invocation + interpolates unquoted shell variables (``--issue $ISSUE_NUMBER``), so an + empty variable collapses the argument list and the naive reading takes the + following flag as the value. That produced three real failures: a + ``ValueError`` that escaped after the report and changed the exit code, a + ledger row written under a repo literally named ``--issue``, and a plan + path silently read from a flag so the run exited 0 having checked nothing. + + Positionals are tokens that are neither a flag nor a flag's value, so + ``--record-outcomes plan.md`` finds the plan rather than mistaking the flag + for it. + """ + opts: dict[str, str] = {} + positionals: list[str] = [] + rejected: list[str] = [] + + i = 0 + while i < len(argv): + token = argv[i] + if token in _BARE_FLAGS: + opts[token] = "" + i += 1 + elif token in _VALUE_FLAGS: + value = argv[i + 1] if i + 1 < len(argv) else None + if value is None or value.startswith("--"): + rejected.append(token) + i += 1 + else: + opts[token] = value + i += 2 + elif token.startswith("--"): + rejected.append(token) + i += 1 + else: + positionals.append(token) + i += 1 + + return opts, positionals, rejected + + def main() -> int: if len(sys.argv) < 2 or sys.argv[1] in ("--help", "-h"): print("Usage: python scripts/validate_build.py [options]") @@ -325,27 +373,40 @@ def main() -> int: print(" --repo OWNER/NAME Target repo for the ledger key.") print(" --issue N Issue number for the ledger key.") print(" --pr N PR whose head SHA anchors the record.") + print(" --timeout N Per-check bound in seconds (env: VERIFICATION_TIMEOUT_S).") + print(" A timeout is UNEVALUATED, which blocks; raise this when a") + print(" legitimate suite brushes the ceiling.") print() print("Exit codes:") print(" 0 - All checks pass or skip") print(" 1 - One or more checks failed") return 0 - argv = sys.argv[1:] - - def _opt(flag: str) -> str | None: - if flag in argv: - i = argv.index(flag) - if i + 1 < len(argv): - return argv[i + 1] - return None - - record_outcomes = "--record-outcomes" in argv - opt_repo = _opt("--repo") - opt_issue = _opt("--issue") - opt_pr = _opt("--pr") + opts, positionals, bad_flags = _parse_argv(sys.argv[1:]) + record_outcomes = "--record-outcomes" in opts + opt_repo = opts.get("--repo") + opt_issue = opts.get("--issue") + opt_pr = opts.get("--pr") + opt_timeout = opts.get("--timeout") + + for flag in bad_flags: + print(f"ARGS: ignoring {flag} -- unknown flag, or its value was missing or another flag") + + if not positionals: + # Never return 0 having run nothing: a green exit with zero checks is + # indistinguishable from a clean plan, which is the whole failure this + # module exists to make impossible. + print("No plan path given. Usage: python scripts/validate_build.py [options]") + return 1 + + timeout = DEFAULT_TIMEOUT_S + if opt_timeout: + try: + timeout = int(opt_timeout) + except ValueError: + print(f"ARGS: ignoring --timeout {opt_timeout!r} -- not an integer") - plan_path = Path(argv[0]) + plan_path = Path(positionals[0]) if not plan_path.exists(): print(f"Plan file not found: {plan_path}") print("Nothing to validate.") @@ -368,7 +429,9 @@ def _opt(flag: str) -> str | None: verification_table = parse_verification_table(plan_text) graded: list[CheckResult] = [] if verification_table.checks or verification_table.malformed or verification_table.skipped: - all_results.extend(check_verification_table(verification_table, check_results=graded)) + all_results.extend( + check_verification_table(verification_table, timeout=timeout, check_results=graded) + ) # 3. Success criteria commands success_criteria = parse_success_criteria_commands(plan_text) @@ -398,15 +461,28 @@ def _opt(flag: str) -> str | None: # told: the write reports its own success or failure on its own line and # does not touch the exit code, which belongs to the checks. if record_outcomes: - if not opt_repo or not opt_issue: - print("RECORD: skipped -- --record-outcomes requires --repo and --issue") + try: + issue_int = int(opt_issue) if opt_issue else None + pr_int = int(opt_pr) if opt_pr else None + except ValueError: + # Reported, never raised: an unparseable argument must not escape + # after the summary has printed and rewrite the exit code. + issue_int = pr_int = None + print( + f"RECORD: skipped -- --issue/--pr must be integers (got {opt_issue!r}/{opt_pr!r})" + ) else: + if not opt_repo or issue_int is None: + print("RECORD: skipped -- --record-outcomes requires --repo and --issue") + issue_int = None + + if issue_int is not None and opt_repo: wrote = record_verification_outcomes( opt_repo, - int(opt_issue), + issue_int, graded, table=verification_table, - pr_number=int(opt_pr) if opt_pr else None, + pr_number=pr_int, ) if wrote: anchor = f"anchored to PR #{opt_pr} head" if opt_pr else "UNANCHORED" diff --git a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py index 264c4ba7d..e92972d2e 100644 --- a/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py +++ b/tests/unit/sdlc_session_ensure/test_sdlc_session_ensure_readback_provenance.py @@ -52,6 +52,32 @@ def _no_target_repo(): yield +@pytest.fixture +def _release_issue_lock_after(): + """Deterministic teardown for tests that acquire a REAL issue lock via + ``touch_issue_lock`` against the shared Redis and assert it survives. + + Several agents test on this machine at once, so a lock a test forgets to + release wedges another lane. The test body registers ``(issue_number, + run_id)`` pairs via the returned callable; teardown releases every + registered pair through ``release_issue_lock`` (the sanctioned + compare-and-delete helper -- never a raw Redis ``DELETE`` on this + Popoto-adjacent key), even if the test body raises. + """ + from models.session_lifecycle import release_issue_lock + + registered: list[tuple[int, str]] = [] + + def _register(issue_number: int, run_id: str) -> None: + registered.append((issue_number, run_id)) + + try: + yield _register + finally: + for issue_number, run_id in registered: + release_issue_lock(issue_number, run_id) + + class TestReadbackByPrimaryKey: """The readback resolves the row THIS call wrote, not an arbitrary row sharing its ``session_id``.""" @@ -140,7 +166,9 @@ def _adopting_session(session_id: str, run_id: str) -> MagicMock: session.owned_run_ids = json.dumps([run_id]) return session - def test_adopted_candidate_survives_readback_mismatch(self, _no_target_repo): + def test_adopted_candidate_survives_readback_mismatch( + self, _no_target_repo, _release_issue_lock_after + ): """The wedge itself: a reuse call that cannot confirm its bind must NOT delete the live lease it merely adopted.""" from models.session_lifecycle import touch_issue_lock @@ -152,6 +180,7 @@ def test_adopted_candidate_survives_readback_mismatch(self, _no_target_repo): # A live lock this call did not create, owned by the id it will reuse. assert touch_issue_lock(issue_number, holder_run_id, session_id=session_id).acquired + _release_issue_lock_after(issue_number, holder_run_id) assert _live_lock_owner(issue_number) == holder_run_id session = self._adopting_session(session_id, holder_run_id) @@ -172,7 +201,9 @@ def test_adopted_candidate_survives_readback_mismatch(self, _no_target_repo): # The lease is untouched: still live, still owned by the holder. assert _live_lock_owner(issue_number) == holder_run_id - def test_adopted_candidate_survives_a_raising_readback(self, _no_target_repo): + def test_adopted_candidate_survives_a_raising_readback( + self, _no_target_repo, _release_issue_lock_after + ): """Failure Path Test Strategy, ``:618``: the readback's own ``except`` arm is a release site too, and an adopted candidate must survive it.""" from models.session_lifecycle import touch_issue_lock @@ -183,6 +214,7 @@ def test_adopted_candidate_survives_a_raising_readback(self, _no_target_repo): holder_run_id = "holder-run-id-306504" assert touch_issue_lock(issue_number, holder_run_id, session_id=session_id).acquired + _release_issue_lock_after(issue_number, holder_run_id) session = self._adopting_session(session_id, holder_run_id) @@ -199,7 +231,9 @@ def test_adopted_candidate_survives_a_raising_readback(self, _no_target_repo): assert "post-save readback failed" in error["reason"] assert _live_lock_owner(issue_number) == holder_run_id - def test_adopted_candidate_survives_a_save_failure(self, _no_target_repo): + def test_adopted_candidate_survives_a_save_failure( + self, _no_target_repo, _release_issue_lock_after + ): """Third release site (the ``session.save`` ``except`` arm).""" from models.session_lifecycle import touch_issue_lock from tools.sdlc_session_ensure import _acquire_run_lock_and_bind @@ -209,6 +243,7 @@ def test_adopted_candidate_survives_a_save_failure(self, _no_target_repo): holder_run_id = "holder-run-id-306505" assert touch_issue_lock(issue_number, holder_run_id, session_id=session_id).acquired + _release_issue_lock_after(issue_number, holder_run_id) session = self._adopting_session(session_id, holder_run_id) session.save.side_effect = RuntimeError("redis save exploded") @@ -221,7 +256,9 @@ def test_adopted_candidate_survives_a_save_failure(self, _no_target_repo): assert error["error"] == "RUN_BIND_FAILED" assert _live_lock_owner(issue_number) == holder_run_id - def test_supervised_adoption_also_survives_a_readback_mismatch(self, _no_target_repo): + def test_supervised_adoption_also_survives_a_readback_mismatch( + self, _no_target_repo, _release_issue_lock_after + ): """The second adopt shape (``ADOPTED_SUPERVISED``): a BARE ensure that inherited the supervisor's run_id via self-recognition never minted it either, so it may not release it. @@ -238,6 +275,7 @@ def test_supervised_adoption_also_survives_a_readback_mismatch(self, _no_target_ supervisor_run_id = "supervisor-run-id-306506" assert touch_issue_lock(issue_number, supervisor_run_id, session_id=session_id).acquired + _release_issue_lock_after(issue_number, supervisor_run_id) session = self._adopting_session(session_id, supervisor_run_id) diff --git a/tests/unit/test_merge_predicate.py b/tests/unit/test_merge_predicate.py index f6742f9ed..f9122c5bc 100644 --- a/tests/unit/test_merge_predicate.py +++ b/tests/unit/test_merge_predicate.py @@ -990,3 +990,38 @@ def test_unresolvable_pr_head_refuses(monkeypatch, ledger_factory, tmp_path): mp._check_verification_outcomes(GATE_ISSUE, GATE_PR, _plan_repo(tmp_path), failed, notes) assert any("PR head unresolvable" in f for f in failed), failed + + +class TestUnreadableAggregateFailsClosed: + """An unreadable aggregate must refuse, not pass as absent. + + Review of PR #3123: group (e)'s reader was the one read in this module + that failed open, so a store error on a lane carrying a recorded FAIL + merged unimpeded. Every neighbouring group already fails closed on its own + read error; this pins group (e) to the same posture. + """ + + def test_read_error_refuses_and_names_the_cause(self, monkeypatch, tmp_path): + import agent.verification_parser as vp + + def boom(*a, **kw): + raise vp.VerificationOutcomesUnavailableError("redis is down") + + monkeypatch.setattr(vp, "read_verification_outcomes", boom) + + failed: list[str] = [] + notes: list[str] = [] + monkeypatch.setattr(mp, "_gh_repo_name_with_owner", lambda root: TARGET_REPO) + # A plan doc must resolve, or the check reports "no plan document" and + # returns before ever reaching the read this test is about. + monkeypatch.setattr(mp, "_find_plan_doc", lambda issue, root: tmp_path / "plan.md") + mp._check_verification_outcomes( + pr_number=GATE_PR, + issue_number=GATE_ISSUE, + repo_root=tmp_path, + failed=failed, + notes=notes, + ) + assert failed, "an unreadable aggregate must fail the gate" + assert any("unreadable" in f for f in failed) + assert not any("reported, not enforced" in n for n in notes) diff --git a/tests/unit/test_sdlc_router.py b/tests/unit/test_sdlc_router.py index c27f846b9..63edbead9 100644 --- a/tests/unit/test_sdlc_router.py +++ b/tests/unit/test_sdlc_router.py @@ -1949,3 +1949,185 @@ def test_migration_warning_logged_exactly_once(self, caplog): # The mutation itself is by-reference and idempotent: the second # (reconciliation) pass must see the already-migrated hash. assert states["_verdicts"]["CRITIQUE"]["artifact_hash"] == current_hash + + +class TestRow8gVerificationOutcomesHoldPr: + """Row 8g is the routing-side twin of merge_predicate group (e). + + Review of PR #3123: group (e) was a merge-refusal condition no dispatch + rule could see. Row 10 fired, `/do-merge` was dispatched, the predicate + refused on a FAIL/UNEVALUATED row, and the router re-dispatched `/do-merge` + unchanged until G4 blocked the lane -- the router-predicate oscillation + loop WS3d/#2062 ended, reintroduced on the verification axis. + """ + + HEAD = "a" * 40 + OTHER = "b" * 40 + + def _states(self, aggregate=None): + from agent.verification_parser import VERIFICATION_OUTCOMES_KEY + + states = dict(_ALL_COMPLETED, PATCH="completed") + if aggregate is not None: + states[VERIFICATION_OUTCOMES_KEY] = aggregate + return states + + def _meta(self): + # The REVIEW verdict must itself be head-fresh, or row 8f (group (c)'s + # twin) preempts row 8g and every case below reads the same. + return _base_meta( + pr_number=3123, + last_dispatched_skill=SKILL_DO_DOCS, + latest_review_verdict="APPROVED", + latest_review_head_sha=self.HEAD, + ) + + def _decide(self, aggregate, *, head=None): + context = {} if head is None else {"pr_head_sha": head} + return decide_next_dispatch(self._states(aggregate), self._meta(), context) + + def test_fail_row_re_reviews_instead_of_merging(self): + result = self._decide({"outcome": "FAIL", "head_sha": self.HEAD}, head=self.HEAD) + assert isinstance(result, Dispatch) + assert result.row_id == "8g" + assert result.skill == SKILL_DO_PR_REVIEW + assert result.skill != SKILL_DO_MERGE + + def test_unevaluated_row_re_reviews_instead_of_merging(self): + """The #3080 shape: UNEVALUATED holds the PR exactly as FAIL does.""" + result = self._decide({"outcome": "UNEVALUATED", "head_sha": self.HEAD}, head=self.HEAD) + assert isinstance(result, Dispatch) + assert result.row_id == "8g" + assert result.skill == SKILL_DO_PR_REVIEW + + def test_fresh_pass_still_merges(self): + result = self._decide({"outcome": "PASS", "head_sha": self.HEAD}, head=self.HEAD) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_MERGE + assert result.row_id == "10" + + def test_pass_graded_against_an_older_head_re_reviews(self): + """Two-pole against the fresh case above: same aggregate, head moved.""" + result = self._decide({"outcome": "PASS", "head_sha": self.OTHER}, head=self.HEAD) + assert isinstance(result, Dispatch) + assert result.row_id == "8g" + assert result.skill == SKILL_DO_PR_REVIEW + + def test_pass_with_no_anchor_re_reviews(self): + result = self._decide({"outcome": "PASS"}, head=self.HEAD) + assert isinstance(result, Dispatch) + assert result.row_id == "8g" + + def test_unresolvable_live_head_re_reviews(self): + """The empty-string sentinel is a lookup failure, not a match. + + Row 8f reaches this state first -- an unresolvable head makes the + REVIEW verdict unattributable too -- and both rows send the lane to + re-review, so the dispatch is what matters here, not which row owns + it. 8g's own disposition is asserted directly below. + """ + result = self._decide({"outcome": "PASS", "head_sha": self.HEAD}, head="") + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_PR_REVIEW + assert result.skill != SKILL_DO_MERGE + + def test_rule_holds_pr_on_an_unresolvable_live_head(self): + """8g fails closed on the sentinel independently of row 8f.""" + from agent.sdlc_router import _rule_verification_outcomes_hold_pr + + held = _rule_verification_outcomes_hold_pr( + self._states({"outcome": "PASS", "head_sha": self.HEAD}), + self._meta(), + {"pr_head_sha": ""}, + ) + assert held is True + + def test_absent_aggregate_is_not_enforced(self): + """Mirrors the predicate: absence is reported there, never enforced. + A lane that never recorded one must still be able to merge.""" + result = self._decide(None, head=self.HEAD) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_MERGE + + def test_blocking_aggregate_does_not_preempt_docs(self): + """Ordered before row 10 but after row 9: docs still come first.""" + from agent.verification_parser import VERIFICATION_OUTCOMES_KEY + + states = dict(_ALL_COMPLETED, PATCH="completed", DOCS="pending") + states[VERIFICATION_OUTCOMES_KEY] = {"outcome": "FAIL", "head_sha": self.HEAD} + result = decide_next_dispatch(states, self._meta(), {"pr_head_sha": self.HEAD}) + assert isinstance(result, Dispatch) + assert result.skill == SKILL_DO_DOCS + + def test_inert_without_an_approved_verdict(self): + """A non-approved lane belongs to the review/patch rows, not to 8g.""" + from agent.sdlc_router import _rule_verification_outcomes_hold_pr + + states = self._states({"outcome": "FAIL"}) + meta = _base_meta(pr_number=3123, latest_review_verdict="CHANGES REQUESTED") + assert _rule_verification_outcomes_hold_pr(states, meta, {}) is False + + def test_router_and_predicate_agree_on_the_blocking_set(self): + """The two sides must not drift on which outcomes hold a PR.""" + from agent.sdlc_router import _rule_verification_outcomes_hold_pr + from agent.verification_parser import CheckOutcome + + blocking = {CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value} + for outcome in (o.value for o in CheckOutcome): + held = _rule_verification_outcomes_hold_pr( + self._states({"outcome": outcome, "head_sha": self.HEAD}), + self._meta(), + {"pr_head_sha": self.HEAD}, + ) + assert held is (outcome in blocking), outcome + + +class TestG4BlockCarriesItsEvidence: + """G4's Blocked is the verdict Cluster D names as its own motivation. + + Review of PR #3123: `decision_inputs` was attached to dispatches and to + the RECONCILE_DEADLOCK block, but the one blocked verdict the cluster + cites -- a lane stopped for human intervention, where the evidence is the + whole point -- returned without any. A human clearing an oscillation + streak needs to see what the router saw. + """ + + def _blocked(self): + from agent.sdlc_router import guard_g4_oscillation + + meta = _base_meta( + same_stage_dispatch_count=MAX_SAME_STAGE_DISPATCHES, + last_dispatched_skill=SKILL_DO_MERGE, + ) + return guard_g4_oscillation(_base_states(), meta, {}) + + def test_g4_blocks_with_decision_inputs_attached(self): + blocked = self._blocked() + assert isinstance(blocked, Blocked) + assert blocked.guard_id == "G4" + assert blocked.decision_inputs is not None + + def test_evidence_names_the_streak_it_tripped(self): + """The three facts that make a G4 block diagnosable without a re-run.""" + inputs = self._blocked().decision_inputs + assert inputs["last_dispatched_skill"] == SKILL_DO_MERGE + assert inputs["same_stage_dispatch_count"] == MAX_SAME_STAGE_DISPATCHES + assert inputs["max_same_stage_dispatches"] == MAX_SAME_STAGE_DISPATCHES + + def test_evidence_is_not_identity(self): + """`decision_inputs` is compare=False, so it never affects equality. + + Two G4 blocks reached from different stage_states must still compare + equal on their verdict; evidence travels alongside the decision, it + does not define it. + """ + from agent.sdlc_router import guard_g4_oscillation + + meta = _base_meta( + same_stage_dispatch_count=MAX_SAME_STAGE_DISPATCHES, + last_dispatched_skill=SKILL_DO_MERGE, + ) + a = guard_g4_oscillation(_base_states(), meta, {}) + b = guard_g4_oscillation(_base_states(REVIEW=STATUS_FAILED), meta, {}) + assert a.decision_inputs != b.decision_inputs + assert a == b diff --git a/tests/unit/test_validate_build.py b/tests/unit/test_validate_build.py index 45ed5750a..e94f981f4 100644 --- a/tests/unit/test_validate_build.py +++ b/tests/unit/test_validate_build.py @@ -723,3 +723,118 @@ def test_commands_are_executed_once_not_twice(self, tmp_path): ): validate_build.main() assert rc.call_count == 1 + + +class TestRecordingIsWiredIntoTheReviewStage: + """The flag's only production invocation is one line of markdown. + + Review of PR #3123 (tech debt 1): every other test in this file patches + the writer and asserts on `main()`, which proves the flag path works, not + that anything invokes it. Delete the flag from the REVIEW addendum and + those stay green while the merge predicate returns to its permanently + `aggregate is None` branch. These assertions pin the wiring itself. + """ + + REPO_ROOT = Path(__file__).parents[2] + REVIEW_ADDENDUM = REPO_ROOT / "docs" / "sdlc" / "do-pr-review.md" + BUILD_ADDENDUM = REPO_ROOT / "docs" / "sdlc" / "do-build.md" + + def test_review_addendum_invokes_the_recording_flag(self): + text = self.REVIEW_ADDENDUM.read_text() + assert "--record-outcomes" in text, ( + "the REVIEW stage is the only production caller of the verification-outcomes " + "writer; without it the merge predicate's group (e) can never fire" + ) + assert "validate_build.py" in text + + def test_review_invocation_passes_every_argument_the_writer_needs(self): + """A record with no --pr is unanchored, and the predicate refuses it.""" + line = next( + line + for line in self.REVIEW_ADDENDUM.read_text().splitlines() + if "--record-outcomes" in line + ) + for flag in ("--repo", "--issue", "--pr"): + assert flag in line, f"{flag} missing from the recording invocation" + + def test_build_addendum_does_not_record(self): + """BUILD has no PR to anchor against, so recording there would write an + unanchored aggregate the merge gate refuses -- blocking every lane. + + Scoped to invocation lines: the addendum is free to *explain* why it + does not record, and does. + """ + invocations = [ + line + for line in self.BUILD_ADDENDUM.read_text().splitlines() + if "validate_build.py" in line and not line.lstrip().startswith("#") + ] + assert invocations, "the BUILD addendum must still run the validator" + assert not any("--record-outcomes" in line for line in invocations) + + def test_the_flag_the_doc_passes_is_the_flag_the_script_accepts(self): + """Pins the doc and the parser together, so renaming one breaks here.""" + opts, positionals, rejected = validate_build._parse_argv( + ["plan.md", "--record-outcomes", "--repo", "o/n", "--issue", "1", "--pr", "2"] + ) + assert not rejected + assert positionals == ["plan.md"] + assert "--record-outcomes" in opts + assert opts["--repo"] == "o/n" + + +class TestArgvParsingRejectsFlagShapedValues: + """The three failure modes reproduced in review of PR #3123. + + The documented invocation interpolates shell variables, so an empty one + collapses the argument list. The naive reading took the next flag as the + value, which produced a ValueError that escaped after the report and + changed the exit code, a ledger row written under a repo named `--issue`, + and a plan path read from a flag so the run exited 0 having checked nothing. + """ + + def test_missing_value_rejects_the_flag(self): + opts, _, rejected = validate_build._parse_argv(["p.md", "--issue", "--pr", "77"]) + assert "--issue" in rejected + assert "--issue" not in opts + assert opts["--pr"] == "77" + + def test_trailing_flag_with_no_value_is_rejected(self): + opts, _, rejected = validate_build._parse_argv(["p.md", "--repo"]) + assert rejected == ["--repo"] + assert "--repo" not in opts + + def test_positional_is_found_after_a_bare_flag(self): + _, positionals, _ = validate_build._parse_argv(["--record-outcomes", "p.md"]) + assert positionals == ["p.md"] + + def test_unknown_flag_is_rejected_not_treated_as_a_positional(self): + _, positionals, rejected = validate_build._parse_argv(["p.md", "--bogus"]) + assert positionals == ["p.md"] + assert rejected == ["--bogus"] + + def test_no_positional_exits_nonzero_rather_than_green_on_zero_checks(self): + with patch("sys.argv", ["validate_build.py", "--record-outcomes", "--repo", "o/n"]): + assert validate_build.main() == 1 + + def test_non_integer_issue_does_not_change_the_exit_code(self, tmp_path): + f = tmp_path / "plan.md" + f.write_text( + "## Verification\n| Check | Command | Expected |\n" + "|---|---|---|\n| Echo | `echo hi` | output contains hi |\n" + ) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "o/n", + "--issue", + "not-a-number", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + assert validate_build.main() == 0 + writer.assert_not_called() diff --git a/tests/unit/test_verification_parser.py b/tests/unit/test_verification_parser.py index 36ef17552..e19e33e6e 100644 --- a/tests/unit/test_verification_parser.py +++ b/tests/unit/test_verification_parser.py @@ -290,12 +290,14 @@ def test_non_numeric_output_for_a_numeric_form_is_a_real_fail(self): assert evaluate_expectation(">= 1", exit_code=0, output="abc") is FAIL assert evaluate_expectation("== 1", exit_code=0, output="") is FAIL - def test_trailing_prose_on_a_numeric_form_is_unevaluated(self): + def test_trailing_gloss_on_the_output_prefixed_form_still_grades(self): """`output == 2 (the two read sites)` appears verbatim in a live plan. - Prefix-matching it would grade a sentence nobody wrote as a number.""" + The `output`-prefixed spellings are prefix-matched (see + TestNumericComparatorTrailingGlossSymmetry), so this grades rather + than going UNEVALUATED.""" assert ( evaluate_expectation("output == 2 (the two read sites)", exit_code=0, output="2") - is UNEVALUATED + is PASS ) @@ -559,8 +561,13 @@ def test_empty_cell_is_rejected_not_silently_dropped(self): assert len(parsed.malformed) == 1 def test_column_count_comes_from_the_header(self): - """One plan in docs/plans/ carries a 4-column Verification table. - Hardcoding 3 would reject every one of its rows.""" + """A trailing extra column (Check, Command, Expected, Notes) is + tolerated: the row width comes from the header, not a hardcoded 3. + No plan under docs/plans/ actually carries this trailing-column + shape today -- the leading-index-column shape + (# | Check | Command | Expected) that plans do carry is pinned + separately by TestLeadingIndexColumnIsACheckTable in + tests/unit/test_validate_build.py.""" table = ( "## Verification\n\n" "| Check | Command | Expected | Notes |\n|--|--|--|--|\n" @@ -1007,3 +1014,171 @@ def test_trailing_gloss_on_a_preexisting_form_still_grades(self): def test_trailing_gloss_on_a_new_bare_form_is_unevaluated(self): for cell in ("== 2 (the two read sites)", ">= 1 nightly", "> 0 or so", "exit 0 maybe"): assert evaluate_expectation(cell, exit_code=0, output="2") is UNEVALUATED + + +class TestNumericComparatorTrailingGlossSymmetry: + """The trailing-gloss rule is now symmetric across >, >=, and ==: the + `output`-prefixed spellings are prefix-matched (gloss tolerated) and the + bare spellings stay anchored (gloss makes it UNEVALUATED), for all three + comparators alike. Previously only `output > N` had this idiom; `output + >= N` and `output == N` were anchored, which silently turned three live + plan rows UNEVALUATED (durability-m1-fence-canary.md:1012, + watch-skill-video-scoping-controls.md:606-607).""" + + def test_gt_output_prefixed_with_gloss_evaluates(self): + assert ( + evaluate_expectation( + "output > 0 (a bare grep returns 3 today)", exit_code=0, output="3" + ) + is PASS + ) + assert ( + evaluate_expectation( + "output > 5 (a bare grep returns 3 today)", exit_code=0, output="3" + ) + is FAIL + ) + + def test_gt_bare_with_gloss_is_unevaluated(self): + assert evaluate_expectation( + "> 0 (a bare grep returns 3 today)", exit_code=0, output="3" + ) is (UNEVALUATED) + + def test_gte_output_prefixed_with_gloss_evaluates(self): + assert ( + evaluate_expectation( + "output >= 3 (holds since the last sweep)", exit_code=0, output="3" + ) + is PASS + ) + assert ( + evaluate_expectation( + "output >= 9 (holds since the last sweep)", exit_code=0, output="3" + ) + is FAIL + ) + + def test_gte_bare_with_gloss_is_unevaluated(self): + assert ( + evaluate_expectation(">= 3 (holds since the last sweep)", exit_code=0, output="3") + is UNEVALUATED + ) + + def test_eq_output_prefixed_with_gloss_evaluates(self): + assert ( + evaluate_expectation("output == 2 (the two read sites)", exit_code=0, output="2") + is PASS + ) + assert ( + evaluate_expectation("output == 2 (the two read sites)", exit_code=0, output="3") + is FAIL + ) + + def test_eq_bare_with_gloss_is_unevaluated(self): + assert ( + evaluate_expectation("== 2 (the two read sites)", exit_code=0, output="2") + is UNEVALUATED + ) + + def test_live_plan_rows_verbatim(self): + """Pinned verbatim from real plans so a regression here is caught at + the exact string that would otherwise hold a PR's merge.""" + assert ( + evaluate_expectation( + "output == 2 (the `:3198` reason-string read and the `:3216` " + "`logger.warning` read)", + exit_code=0, + output="2", + ) + is PASS + ) + assert ( + evaluate_expectation("output == 1 (exactly `import os`)", exit_code=0, output="1") + is PASS + ) + assert ( + evaluate_expectation( + "output == 1 (exactly `from __future__ import annotations`)", + exit_code=0, + output="1", + ) + is PASS + ) + + +class TestReadVerificationOutcomesFailsClosed: + """Absent and unreadable must be distinguishable (review of PR #3123). + + The reader previously returned None on any error, and the merge predicate + reads None as "no aggregate -- reported, not enforced". A Redis blip or a + corrupt blob therefore converted a recorded FAIL into an unenforced pass, + inverted against every neighbouring group in merge_predicate, which all + fail closed on their own read errors. + """ + + REPO = "owner/name" + ISSUE = 991234 + + def test_absent_ledger_reads_as_absence(self): + from agent.verification_parser import read_verification_outcomes + + assert read_verification_outcomes(self.REPO, self.ISSUE) is None + + def test_missing_identifiers_read_as_absence(self): + from agent.verification_parser import read_verification_outcomes + + assert read_verification_outcomes(None, None) is None + + def test_store_error_raises_rather_than_reporting_absence(self, monkeypatch): + import agent.pipeline_ledger as pl + from agent.verification_parser import ( + VerificationOutcomesUnavailableError, + read_verification_outcomes, + ) + + def boom(*a, **kw): + raise ConnectionError("redis is down") + + monkeypatch.setattr(pl.PipelineLedger, "get", staticmethod(boom)) + with pytest.raises(VerificationOutcomesUnavailableError): + read_verification_outcomes(self.REPO, self.ISSUE) + + def test_unparseable_blob_raises(self, monkeypatch): + import agent.pipeline_ledger as pl + from agent.verification_parser import ( + VerificationOutcomesUnavailableError, + read_verification_outcomes, + ) + + class _Row: + stage_states_json = "{not json" + + monkeypatch.setattr(pl.PipelineLedger, "get", staticmethod(lambda *a, **kw: _Row())) + with pytest.raises(VerificationOutcomesUnavailableError): + read_verification_outcomes(self.REPO, self.ISSUE) + + def test_record_of_the_wrong_shape_raises(self, monkeypatch): + import agent.pipeline_ledger as pl + from agent.verification_parser import ( + VERIFICATION_OUTCOMES_KEY, + VerificationOutcomesUnavailableError, + read_verification_outcomes, + ) + + class _Row: + stage_states_json = json.dumps({VERIFICATION_OUTCOMES_KEY: "PASS"}) + + monkeypatch.setattr(pl.PipelineLedger, "get", staticmethod(lambda *a, **kw: _Row())) + with pytest.raises(VerificationOutcomesUnavailableError): + read_verification_outcomes(self.REPO, self.ISSUE) + + def test_blob_present_but_key_missing_is_absence_not_an_error(self, monkeypatch): + """A lane that simply never recorded one is not blocked.""" + import agent.pipeline_ledger as pl + from agent.verification_parser import read_verification_outcomes + + class _Row: + stage_states_json = json.dumps({"BUILD": "completed"}) + + monkeypatch.setattr(pl.PipelineLedger, "get", staticmethod(lambda *a, **kw: _Row())) + assert read_verification_outcomes(self.REPO, self.ISSUE) is None diff --git a/tools/merge_predicate.py b/tools/merge_predicate.py index 59e9ff739..07ea2d964 100644 --- a/tools/merge_predicate.py +++ b/tools/merge_predicate.py @@ -805,9 +805,21 @@ def _check_verification_outcomes( # merge-guard hook (see module docstring). agent.verification_parser pulls # in agent.pipeline_ledger, same posture as the _check_verdict_freshness # trailer reader below. - from agent.verification_parser import read_verification_outcomes + from agent.verification_parser import ( + VerificationOutcomesUnavailableError, + read_verification_outcomes, + ) + + try: + aggregate = read_verification_outcomes(target_repo, issue_number) + except VerificationOutcomesUnavailableError as exc: + # An unreadable record is not an absent one. Absence is a lane this + # gate deliberately does not block; a failed read is a lane about + # which nothing is known, and passing it would turn a recorded FAIL + # into an unenforced pass on a transient store error. + failed.append(f"verification outcomes: recorded aggregate unreadable ({exc})") + return - aggregate = read_verification_outcomes(target_repo, issue_number) if aggregate is None: notes.append( "verification-outcomes check skipped: no recorded aggregate for" @@ -871,7 +883,7 @@ def _check_verification_outcomes( offending += 1 reason = str(row.get("reason") or "").strip() failed.append( - f"verification row {row.get('name') or ''!r} is {row_outcome}" + f"verification row {(row.get('name') or '')!r} is {row_outcome}" + (f": {reason}" if reason else "") + " — FAIL and UNEVALUATED both hold the PR (owner ruling on #3080)" ) @@ -892,7 +904,7 @@ def _check_verification_outcomes( # a run with no checks at all, which grades UNEVALUATED rather than a # vacuous PASS. Refuse rather than guess what it meant. failed.append( - f"verification outcomes: recorded outcome is {outcome or ''!r}," + f"verification outcomes: recorded outcome is {(outcome or '')!r}," f" not {CheckOutcome.PASS.value} ({len(rows)} row(s) recorded)" ) return From a9d411bebc9e186ae42a7239d838240273cd8178 Mon Sep 17 00:00:00 2001 From: valorengels Date: Fri, 4 Sep 2026 14:13:06 +0700 Subject: [PATCH 19/19] Make the gate reachable, and stop it refusing lanes that declared no gate (Refs #3065) Round 3 found the gate armed but aimed wrong in three places. Row 8g never fired. Both router callers take stage_states from query_enriched()["stages"], which filters to ALL_STAGES and threads exactly two underscore keys back in. _verification_outcomes was not one of them, so the rule returned False for every real lane while 626 tests passed over it -- every 8g test hand-built its payload with the key already injected. The key is now threaded, and a seam test drives row 8g from a real query_enriched payload, so deleting it from the tuple fails a test rather than silently disarming the rule. Verified by removing it and watching the test fail. A plan with no verification table recorded a 0-row UNEVALUATED, which the predicate then refused forever. Four of the twenty live plans are in that shape. Absence of a contract is not a failed contract: the runner now declines to record when the plan declares no check table, and says so, which is what Risk 8 claimed the design already did. Two in-flight PRs were about to be refused on expectation forms the grammar did not cover -- output `N`, output is N, and a gloss set off from exit N by a comma. All three are legitimate authoring. The gloss rule is now stated once as a rule: a gloss delimited by a comma or a paren/backtick span is ignored, a bare trailing word stays ambiguous and UNEVALUATED. `no output` is accepted as an exact synonym of `empty output`. Regraded every Expected cell in docs/plans against origin/main's grammar across eight (exit, output) samples: 58 differences, and zero on any cell the old grammar actually recognized. Old code returned a bare False on fallthrough, so its "FAIL" on an unrecognized form was a lie rather than a verdict; every difference is that lie being corrected. Also: the blocking set is defined once and shared, so the router and the predicate cannot drift on which outcomes hold a PR; a non-positive timeout is rejected at both entry points; do-merge.md points at the recovery section; and the troubleshooting remedy names the step where plan edits land on main while the lane grades its own worktree. Carried tech debt and nits are tracked in #3125. Plan verification table: 29 PASS, 0 FAIL, 0 UNEVALUATED, exit 0. --- agent/sdlc_router.py | 24 ++- agent/verification_parser.py | 104 +++++++++++-- docs/sdlc/do-merge.md | 15 ++ docs/sdlc/merge-troubleshooting.md | 7 + scripts/validate_build.py | 130 ++++++++++++----- tests/unit/test_sdlc_router.py | 23 ++- tests/unit/test_sdlc_stage_query.py | 85 +++++++++++ tests/unit/test_validate_build.py | 195 ++++++++++++++++++++++++- tests/unit/test_verification_parser.py | 64 ++++++++ tools/merge_predicate.py | 8 +- tools/sdlc_stage_query.py | 17 ++- 11 files changed, 610 insertions(+), 62 deletions(-) diff --git a/agent/sdlc_router.py b/agent/sdlc_router.py index 54bbbd553..c60edd053 100644 --- a/agent/sdlc_router.py +++ b/agent/sdlc_router.py @@ -45,7 +45,7 @@ STAGE_TO_SKILL, ) from agent.pipeline_state import SETTLED_STATUSES -from agent.verification_parser import VERIFICATION_OUTCOMES_KEY, CheckOutcome +from agent.verification_parser import BLOCKING_OUTCOMES, VERIFICATION_OUTCOMES_KEY logger = logging.getLogger(__name__) @@ -2192,7 +2192,9 @@ def _rule_verification_outcomes_hold_pr(stage_states: dict, meta: dict, context: the router-predicate oscillation loop WS3d/#2062 existed to end, reintroduced on the verification axis. - The dispositions mirror the predicate's, so the two cannot disagree: + The dispositions mirror the predicate's on the overall outcome, and the + blocking set itself is shared rather than re-spelled (``BLOCKING_OUTCOMES``), + so the two cannot drift on *which* outcomes hold a PR: - no recorded aggregate → **False**. Absence is reported and not enforced on the ship side either; a lane that never recorded one is not blocked. @@ -2204,9 +2206,21 @@ def _rule_verification_outcomes_hold_pr(stage_states: dict, meta: dict, context: current head is the defect this whole mechanism exists to close. - ``PASS`` anchored to the current head → **False**. Row 10 may merge. + The mirroring is on the overall outcome, not row-by-row: the predicate also + refuses a malformed aggregate (``rows`` not a list, a non-dict row, a + non-zero ``malformed`` count), and this rule inspects none of those. Those + shapes are unreachable from the sanctioned writer, and a lane in one of them + should reach the predicate's refusal rather than be re-routed by a rule that + cannot explain why. If a future writer can produce them, they belong here + too. + Reads ``stage_states`` only; the aggregate already travels in that blob, so - this rule makes no network call. Scoped to APPROVED verdicts because a lane - that is not approved is owned by the review/patch rows. + this rule makes no network call -- but it does require that + ``tools/sdlc_stage_query.query_enriched`` thread the key through to the + router, which is why that threading is pinned by a seam test rather than + left to the hand-built payloads the rest of these tests use. Scoped to + APPROVED verdicts because a lane that is not approved is owned by the + review/patch rows. Termination: ``/do-pr-review`` re-runs the table and re-records an anchored aggregate, so a stale or unanchored record converges in one pass. A record @@ -2224,7 +2238,7 @@ def _rule_verification_outcomes_hold_pr(stage_states: dict, meta: dict, context: return False outcome = str(aggregate.get("outcome") or "").strip().upper() - if outcome in (CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value): + if outcome in BLOCKING_OUTCOMES: return True if "pr_head_sha" not in context: diff --git a/agent/verification_parser.py b/agent/verification_parser.py index 457adc7ae..36a4dbe12 100644 --- a/agent/verification_parser.py +++ b/agent/verification_parser.py @@ -122,10 +122,23 @@ # VERIFICATION_TIMEOUT_S (or `--timeout`) when a real suite brushes the ceiling # -- but contention is a load problem, not a bound problem, so prefer rerunning # on a quiet machine over permanently inflating this. +# A non-positive bound is rejected rather than honored: it would time out every +# row, and since the recorder writes regardless of exit code, a stray +# VERIFICATION_TIMEOUT_S=0 in a launchd environment would persist an +# all-UNEVALUATED aggregate that holds the lane with no self-evident cause. +_FALLBACK_TIMEOUT_S = 120 try: - DEFAULT_TIMEOUT_S = int(os.environ.get("VERIFICATION_TIMEOUT_S", "") or 120) + _configured_timeout = int(os.environ.get("VERIFICATION_TIMEOUT_S", "") or _FALLBACK_TIMEOUT_S) except ValueError: - DEFAULT_TIMEOUT_S = 120 + _configured_timeout = _FALLBACK_TIMEOUT_S +if _configured_timeout <= 0: + logger.warning( + "VERIFICATION_TIMEOUT_S=%r is not positive; using %ss", + os.environ.get("VERIFICATION_TIMEOUT_S"), + _FALLBACK_TIMEOUT_S, + ) + _configured_timeout = _FALLBACK_TIMEOUT_S +DEFAULT_TIMEOUT_S = _configured_timeout # Underscore-prefixed metadata key inside the ledger's `stage_states_json` # blob, mirroring `_verdicts` / `_sdlc_dispatches` / `_run_identities`. A new @@ -145,6 +158,20 @@ class CheckOutcome(StrEnum): UNEVALUATED = "UNEVALUATED" +# The outcomes that hold a PR (owner ruling on #3080, `ba092a06d`): FAIL says +# the code is wrong, UNEVALUATED says the grader could not answer, and neither +# is evidence that the lane is shippable. +# +# Defined here rather than on either consumer because it has exactly two, and +# they sit on opposite sides of an architectural boundary: `tools/merge_predicate` +# refuses the merge, and `agent/sdlc_router`'s row 8g re-routes rather than +# dispatching one the predicate would refuse. The router may not import `tools/` +# (see `guard_g8_artifact_verification`), so a set spelled out on the predicate +# could not be shared with it -- and a set spelled out twice is one edit away +# from the two sides silently disagreeing about which outcomes block. +BLOCKING_OUTCOMES = frozenset({CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value}) + + def split_row_cells(row: str) -> list[str]: """Split one markdown table row into its cells, honoring ``\\|`` escapes. @@ -496,10 +523,32 @@ def unevaluated_reason(expected: str | None) -> str: f"unrecognized expectation form: {expected.strip()!r}. " "The grammar reads: exit code N, exit N, exit code != N, output contains X, " "output does not contain X, match count == 0, output > N, > N, >= N, == N, " - "prints `N`, empty output." + "output is N, prints `N`, output `N`, empty output, no output." ) +_TRAILING_GLOSS_DELIMITERS = (",", "(", "`") + + +def _match_bare_with_delimited_gloss(pattern: str, expected: str) -> re.Match | None: + """Match a bare anchored form, tolerating a trailing gloss set off by a delimiter. + + **Delimited-gloss rule.** A trailing gloss is only recognised as a gloss -- + and therefore ignored -- when it is set off by a delimiter: a comma, or a + parenthesis/backtick span (``exit 0, JSON `"compatible": true``` reads as + ``exit 0`` plus a comma-delimited gloss). Bare trailing words with no + delimiter are ambiguous (``exit 0 maybe`` could be a typo for a different + number) and the form stays unmatched, i.e. UNEVALUATED. + """ + m = re.match(pattern, expected) + if not m: + return None + rest = expected[m.end() :].lstrip() + if not rest or rest[0] in _TRAILING_GLOSS_DELIMITERS: + return m + return None + + def evaluate_expectation(expected: str | None, *, exit_code: int, output: str) -> CheckOutcome: """Grade a command result against its expected outcome, three-valued. @@ -510,12 +559,15 @@ def evaluate_expectation(expected: str | None, *, exit_code: int, output: str) - :func:`unevaluated_reason` for the accompanying reason text. Supported expectation formats (positive): - - ``exit code N`` / ``exit N`` -- passes when exit_code == N + - ``exit code N`` / ``exit N`` -- passes when exit_code == N. ``exit N`` also + tolerates a delimited trailing gloss (see the delimited-gloss rule below), + e.g. ``exit 0, JSON `"compatible": true```. - ``output > N`` / ``> N`` -- passes when output (stripped) is numeric and > N - ``>= N`` -- passes when output (stripped) is numeric and >= N - ``== N`` / ``output == N`` -- passes when output (stripped) is numeric and == N - - ``prints `N``` -- passes when stripped output equals N - - ``empty output`` -- passes when stdout is empty or whitespace-only + - ``output is N`` -- equality, same semantics as ``output == N`` + - ``prints `N``` / ```output `N``` -- passes when stripped output equals N + - ``empty output`` / ``no output`` -- passes when stdout is empty or whitespace-only - ``output contains X`` -- passes when substring X appears in stdout Supported expectation formats (inverse / anti-criteria): @@ -608,13 +660,18 @@ def numeric_verdict(op) -> CheckOutcome: if m: return verdict(exit_code == int(m.group(1))) - # exit N (anchored, see the note below) - m = re.match(r"exit\s+(\d+)\s*$", expected) + # exit N (anchored, but tolerates a delimited trailing gloss -- see the + # delimited-gloss rule on `_match_bare_with_delimited_gloss` -- e.g. + # `exit 0, JSON `"compatible": true``. A bare trailing word with no + # delimiter, e.g. `exit 0 maybe`, stays unmatched and UNEVALUATED.) + m = _match_bare_with_delimited_gloss(r"exit\s+(\d+)", expected) if m: return verdict(exit_code == int(m.group(1))) - # empty output (passes when stdout is empty or whitespace-only) - if expected == "empty output": + # empty output / no output (passes when stdout is empty or whitespace-only). + # `no output` is an exact synonym a live plan already writes; accepting it is + # an alias, not a widening -- there is no reading of it that differs. + if expected in ("empty output", "no output"): return verdict(not output.strip()) # prints `N` (passes when stripped stdout equals N; backticks optional) @@ -622,6 +679,14 @@ def numeric_verdict(op) -> CheckOutcome: if m: return verdict(output.strip() == m.group(1).strip()) + # output `N` (backtick-wrapped integer, equality against stripped stdout; + # same semantics as `output == N` below, and prefix-matched for the same + # reason -- it is the `output`-prefixed idiom's backtick spelling). + m = re.match(r"output\s*`(\d+)`", expected) + if m: + target = int(m.group(1)) + return numeric_verdict(lambda value: value == target) + # Trailing-gloss rule (applies uniformly to >, >=, ==): the `output`-prefixed # spellings (`output > N`, `output >= N`, `output == N`) are the established # authoring idiom in live plans -- e.g. `output > 0 (a bare file-wide grep @@ -661,6 +726,13 @@ def numeric_verdict(op) -> CheckOutcome: target = int(m.group(1)) return numeric_verdict(lambda value: value == target) + # output is N -- `is` used as the equality verb; same semantics and + # prefix-matching as `output == N` above. + m = re.match(r"output\s+is\s+(\d+)", expected) + if m: + target = int(m.group(1)) + return numeric_verdict(lambda value: value == target) + # == N (anchored, see the trailing-gloss rule above) m = re.match(r"==\s*(\d+)\s*$", expected) if m: @@ -948,6 +1020,11 @@ def record_verification_outcomes( ) if head_sha: aggregate["head_sha"] = head_sha + else: + # Resolver returned no exception but no usable SHA either -- + # stamp this distinctly from "never anchored" so a merge + # refusal can say *why* there is no head_sha. + aggregate["head_sha_anchor_failed"] = True except Exception as exc: logger.debug( "record_verification_outcomes: head-SHA resolve failed for " @@ -958,6 +1035,11 @@ def record_verification_outcomes( type(exc).__name__, exc, ) + # A transient resolver failure at record time must not read + # the same as a lane that was never anchored: stamp the + # failed-anchor case distinctly so the refusal message and + # troubleshooting can tell the two apart. + aggregate["head_sha_anchor_failed"] = True from agent.pipeline_ledger import PipelineLedger from tools.stage_states_helpers import update_stage_states @@ -1017,8 +1099,6 @@ def read_verification_outcomes(target_repo: str | None, issue_number: int | None if raw is None or raw == "": return None blob = json.loads(raw) if isinstance(raw, str) else raw - except VerificationOutcomesUnavailableError: - raise except Exception as exc: logger.debug( "read_verification_outcomes: read failed for %s#%s (%s: %s)", diff --git a/docs/sdlc/do-merge.md b/docs/sdlc/do-merge.md index 7bd6e3395..4cacbbefb 100644 --- a/docs/sdlc/do-merge.md +++ b/docs/sdlc/do-merge.md @@ -304,6 +304,21 @@ blocks at the choke point. A stale-but-safe diff (docs-only re-push after approval) needs a fresh review or a matching-trailer re-record — the predicate does not re-admit prior approvals by diff shape. +### Verification Outcomes + +Group (e) of the same predicate refuses a merge when the lane's recorded +verification aggregate carries a blocking row or cannot be shown fresh against +the PR head. A `FAIL` row says the code is wrong; an `UNEVALUATED` row says the +grader could not answer, and both hold the PR (owner ruling, `ba092a06d`). A +lane whose plan declares no verification table has declared no gate, and the +predicate stands down rather than refusing. + +Recovery differs by which line you got, and none of it is guesswork — +**`docs/sdlc/merge-troubleshooting.md`, "Verification Outcomes Hold the PR"** +names each refusal string and its remedy, including the `--record-outcomes` +re-run that re-anchors a stale aggregate. Read that section rather than +re-deriving the fix here. + ### Lockfile Sync Check ```bash diff --git a/docs/sdlc/merge-troubleshooting.md b/docs/sdlc/merge-troubleshooting.md index 4278f8309..d263cfc12 100644 --- a/docs/sdlc/merge-troubleshooting.md +++ b/docs/sdlc/merge-troubleshooting.md @@ -129,6 +129,13 @@ python scripts/validate_build.py "$PLAN_PATH" legitimate suite, re-run on a quiet machine first; raise the bound only if it is really too low (`--timeout N`, or `VERIFICATION_TIMEOUT_S`). Contention is a load problem, not a bound problem. + + Mind where the edit lands. Plan files live on `main` and never travel in a + feature-branch PR (`docs/sdlc/do-docs.md`), but `/do-pr-review` grades + `"$PLAN_PATH"` **in the lane worktree**. So an `Expected`-cell fix committed + to `main` keeps grading the old cell until you bring `main` into the branch. + Merge or rebase first, then re-run the table — otherwise the row you just + fixed refuses the merge again and reads as though the fix did not work. - **A stale, unanchored, or unreadable aggregate** — nothing is wrong with the code; the record just cannot be trusted at this head. Re-record it: diff --git a/scripts/validate_build.py b/scripts/validate_build.py index 57845ac22..c30ead8b0 100644 --- a/scripts/validate_build.py +++ b/scripts/validate_build.py @@ -311,8 +311,8 @@ def check_success_criteria(criteria: list[dict[str, str]]) -> list[dict]: _BARE_FLAGS = ("--record-outcomes",) -def _parse_argv(argv: list[str]) -> tuple[dict[str, str], list[str], list[str]]: - """Split argv into ``(options, positionals, rejected_flags)``. +def _parse_argv(argv: list[str]) -> tuple[dict[str, str], list[str], list[str], list[str]]: + """Split argv into ``(options, positionals, rejected_flags, warnings)``. A value flag whose next token is missing or is itself a flag is **rejected**, not silently satisfied. The documented production invocation @@ -322,27 +322,38 @@ def _parse_argv(argv: list[str]) -> tuple[dict[str, str], list[str], list[str]]: ``ValueError`` that escaped after the report and changed the exit code, a ledger row written under a repo literally named ``--issue``, and a plan path silently read from a flag so the run exited 0 having checked nothing. + A value that merely *looks* like a flag -- any token starting with ``-``, + not only ``--`` -- is rejected the same way, so ``--issue -5`` is refused + rather than silently accepted as a negative issue number. Positionals are tokens that are neither a flag nor a flag's value, so ``--record-outcomes plan.md`` finds the plan rather than mistaking the flag - for it. + for it. Only the first positional is used (the plan path); ``warnings`` + carries a ready-to-print ``ARGS: ...`` line for any positional beyond the + first, and for a repeated flag (last occurrence wins), so both conditions + are reported instead of being swallowed silently. """ opts: dict[str, str] = {} positionals: list[str] = [] rejected: list[str] = [] + warnings: list[str] = [] i = 0 while i < len(argv): token = argv[i] if token in _BARE_FLAGS: + if token in opts: + warnings.append(f"ARGS: {token} repeated -- using the last occurrence") opts[token] = "" i += 1 elif token in _VALUE_FLAGS: value = argv[i + 1] if i + 1 < len(argv) else None - if value is None or value.startswith("--"): + if value is None or value.startswith("-"): rejected.append(token) i += 1 else: + if token in opts: + warnings.append(f"ARGS: {token} repeated -- using the last occurrence") opts[token] = value i += 2 elif token.startswith("--"): @@ -352,37 +363,52 @@ def _parse_argv(argv: list[str]) -> tuple[dict[str, str], list[str], list[str]]: positionals.append(token) i += 1 - return opts, positionals, rejected + if len(positionals) > 1: + extras = ", ".join(positionals[1:]) + warnings.append(f"ARGS: ignoring extra positional argument(s): {extras}") + + return opts, positionals, rejected, warnings + + +def _print_usage() -> None: + print("Usage: python scripts/validate_build.py [options]") + print() + print("Validates a build against the plan specification.") + print("Checks file path assertions, verification table commands,") + print("and success criteria commands.") + print() + print("Options:") + print(" --record-outcomes Persist the graded aggregate to the lane's ledger,") + print(" where the merge predicate reads it. Requires") + print(" --repo and --issue; pass --pr so the record is") + print(" stamped with the head SHA it was graded against.") + print(" An unstamped record is refused at merge, so record") + print(" at REVIEW/DOCS time, once the lane has a PR.") + print(" --repo OWNER/NAME Target repo for the ledger key.") + print(" --issue N Issue number for the ledger key.") + print(" --pr N PR whose head SHA anchors the record.") + print(" --timeout N Per-check bound in seconds (env: VERIFICATION_TIMEOUT_S).") + print(" A timeout is UNEVALUATED, which blocks; raise this when a") + print(" legitimate suite brushes the ceiling.") + print() + print("Exit codes:") + print(" 0 - All checks pass or skip") + print(" 1 - One or more checks failed") def main() -> int: - if len(sys.argv) < 2 or sys.argv[1] in ("--help", "-h"): - print("Usage: python scripts/validate_build.py [options]") - print() - print("Validates a build against the plan specification.") - print("Checks file path assertions, verification table commands,") - print("and success criteria commands.") - print() - print("Options:") - print(" --record-outcomes Persist the graded aggregate to the lane's ledger,") - print(" where the merge predicate reads it. Requires") - print(" --repo and --issue; pass --pr so the record is") - print(" stamped with the head SHA it was graded against.") - print(" An unstamped record is refused at merge, so record") - print(" at REVIEW/DOCS time, once the lane has a PR.") - print(" --repo OWNER/NAME Target repo for the ledger key.") - print(" --issue N Issue number for the ledger key.") - print(" --pr N PR whose head SHA anchors the record.") - print(" --timeout N Per-check bound in seconds (env: VERIFICATION_TIMEOUT_S).") - print(" A timeout is UNEVALUATED, which blocks; raise this when a") - print(" legitimate suite brushes the ceiling.") - print() - print("Exit codes:") - print(" 0 - All checks pass or skip") - print(" 1 - One or more checks failed") + if len(sys.argv) < 2: + _print_usage() return 0 - opts, positionals, bad_flags = _parse_argv(sys.argv[1:]) + # --help/-h is honored anywhere in argv, not only as the first token: a + # documented invocation like `--record-outcomes --help` must still show + # help rather than being parsed as a (rejected) unknown-flag run. + if "--help" in sys.argv[1:] or "-h" in sys.argv[1:]: + _print_usage() + return 0 + + opts, positionals, bad_flags, arg_warnings = _parse_argv(sys.argv[1:]) record_outcomes = "--record-outcomes" in opts opt_repo = opts.get("--repo") opt_issue = opts.get("--issue") @@ -391,6 +417,8 @@ def main() -> int: for flag in bad_flags: print(f"ARGS: ignoring {flag} -- unknown flag, or its value was missing or another flag") + for warning in arg_warnings: + print(warning) if not positionals: # Never return 0 having run nothing: a green exit with zero checks is @@ -402,12 +430,25 @@ def main() -> int: timeout = DEFAULT_TIMEOUT_S if opt_timeout: try: - timeout = int(opt_timeout) + parsed_timeout = int(opt_timeout) except ValueError: print(f"ARGS: ignoring --timeout {opt_timeout!r} -- not an integer") + else: + if parsed_timeout <= 0: + # A non-positive bound times out every row immediately, and a + # timeout grades UNEVALUATED -- which now blocks merge + # permanently once recorded. Fall back rather than let a + # misconfigured `--timeout 0` (or a negative value) silently + # turn every check into a durable refusal. + print( + f"ARGS: ignoring --timeout {opt_timeout!r} -- must be positive, " + f"using default {DEFAULT_TIMEOUT_S}" + ) + else: + timeout = parsed_timeout plan_path = Path(positionals[0]) - if not plan_path.exists(): + if not plan_path.is_file(): print(f"Plan file not found: {plan_path}") print("Nothing to validate.") return 0 @@ -428,7 +469,17 @@ def main() -> int: # 2. Verification table verification_table = parse_verification_table(plan_text) graded: list[CheckResult] = [] - if verification_table.checks or verification_table.malformed or verification_table.skipped: + # Whether the plan declares a check table at all -- a table with rows, a + # malformed row, or a non-check table it named and stood down on. Shared + # by the run site below and the record site further down so the two + # can never drift apart: a plan with none of these has declared no gate, + # and recording a 0-row UNEVALUATED aggregate for it would block the lane + # forever on a contract it never made (Risk 8, + # docs/plans/sdlc-control-plane-asserted-facts.md). + table_declared = bool( + verification_table.checks or verification_table.malformed or verification_table.skipped + ) + if table_declared: all_results.extend( check_verification_table(verification_table, timeout=timeout, check_results=graded) ) @@ -460,7 +511,18 @@ def main() -> int: # Persist last, and never let a ledger failure change what the human is # told: the write reports its own success or failure on its own line and # does not touch the exit code, which belongs to the checks. - if record_outcomes: + if record_outcomes and not table_declared: + # Absence of a contract is not a failed contract: a plan that names no + # `## Verification` table has declared no gate, so there is nothing + # for the merge predicate to enforce. Recording anyway would write a + # 0-row aggregate that `aggregate_outcomes([], table)` grades + # UNEVALUATED, and UNEVALUATED blocks merge permanently with no + # self-heal -- re-reviewing re-records the identical aggregate. This + # is the one case the writer must never be called for. + print( + "RECORD: skipped -- plan declares no verification table, so there is no gate to record" + ) + elif record_outcomes: try: issue_int = int(opt_issue) if opt_issue else None pr_int = int(opt_pr) if opt_pr else None diff --git a/tests/unit/test_sdlc_router.py b/tests/unit/test_sdlc_router.py index 63edbead9..88b9ceb2b 100644 --- a/tests/unit/test_sdlc_router.py +++ b/tests/unit/test_sdlc_router.py @@ -9,6 +9,8 @@ from __future__ import annotations +import inspect + from agent.sdlc_router import ( G3_REDIRECT_REASON_DOCS_PENDING, GUARDS, @@ -2067,12 +2069,29 @@ def test_inert_without_an_approved_verdict(self): meta = _base_meta(pr_number=3123, latest_review_verdict="CHANGES REQUESTED") assert _rule_verification_outcomes_hold_pr(states, meta, {}) is False + def test_predicate_reads_the_shared_blocking_set(self): + """Pins the two sides to ONE definition, not to matching spellings. + + Review of PR #3123: the earlier version of this test re-spelled + `{FAIL, UNEVALUATED}` locally and compared the router to itself, so a + drift in the predicate would not have failed it. Both consumers now + read `BLOCKING_OUTCOMES`, and this asserts the predicate really does. + """ + import agent.verification_parser as vp + import tools.merge_predicate as mp + + source = inspect.getsource(mp._check_verification_outcomes) + assert "BLOCKING_OUTCOMES" in source, ( + "the merge predicate must read the shared blocking set, not re-spell it" + ) + assert vp.BLOCKING_OUTCOMES == {"FAIL", "UNEVALUATED"} + def test_router_and_predicate_agree_on_the_blocking_set(self): """The two sides must not drift on which outcomes hold a PR.""" from agent.sdlc_router import _rule_verification_outcomes_hold_pr - from agent.verification_parser import CheckOutcome + from agent.verification_parser import BLOCKING_OUTCOMES, CheckOutcome - blocking = {CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value} + blocking = BLOCKING_OUTCOMES for outcome in (o.value for o in CheckOutcome): held = _rule_verification_outcomes_hold_pr( self._states({"outcome": outcome, "head_sha": self.HEAD}), diff --git a/tests/unit/test_sdlc_stage_query.py b/tests/unit/test_sdlc_stage_query.py index d91958794..e08af57da 100644 --- a/tests/unit/test_sdlc_stage_query.py +++ b/tests/unit/test_sdlc_stage_query.py @@ -546,6 +546,91 @@ def test_stages_exposes_router_underscore_keys(self): assert _critique_verdict_is_stale(result["stages"]) is True + def test_verification_outcomes_survive_through_the_real_seam_and_route_router(self): + """Round-3 review blocker 1 on PR #3123: row 8g + (``_rule_verification_outcomes_hold_pr``) reads ``_verification_outcomes`` + directly off ``stage_states``, but both router callers (the CLI at + ``tools/sdlc_next_skill.py`` and the in-process runner) source + ``stage_states`` from ``query_enriched()["stages"]`` -- never from the + raw blob. Every existing row-8g router test hand-builds a stage_states + dict with the key already injected, so 626 tests passed over an + aggregate that never actually reached the router in production. + + This test goes THROUGH the real ``query_enriched`` seam instead of + around it: a mocked session carries the aggregate in its raw + ``stage_states`` JSON, exactly as the ledger writer + (``verification_parser``) leaves it, and the test asserts the + aggregate survives into the enriched payload's ``stages`` AND that + feeding that real payload into ``decide_next_dispatch`` routes to + ``/do-pr-review`` (row 8g) rather than ``/do-merge`` (row 10). + """ + from agent.sdlc_router import ( + SKILL_DO_MERGE, + SKILL_DO_PR_REVIEW, + Dispatch, + decide_next_dispatch, + ) + from agent.verification_parser import VERIFICATION_OUTCOMES_KEY + from tools.sdlc_stage_query import query_enriched + + head_sha = "a" * 40 + aggregate = {"outcome": "FAIL", "head_sha": head_sha} + verdicts = { + "REVIEW": { + "verdict": "APPROVED", + "recorded_at": "2026-09-01T00:00:00+00:00", + "artifact_hash": "sha256:def", + "head_sha": head_sha, + } + } + mock_session = MagicMock() + mock_session.stage_states = json.dumps( + { + "ISSUE": "completed", + "PLAN": "completed", + "CRITIQUE": "completed", + "BUILD": "completed", + "TEST": "completed", + "REVIEW": "completed", + "DOCS": "completed", + "MERGE": "pending", + "_verdicts": verdicts, + "_sdlc_dispatches": [{"skill": "/do-pr-review", "at": "2026-09-01T00:00:00+00:00"}], + VERIFICATION_OUTCOMES_KEY: aggregate, + } + ) + mock_session.pr_number = 3123 + + with patch("tools.sdlc_stage_query._find_session_by_id", return_value=mock_session): + with patch("tools.sdlc_stage_query._lookup_pr", return_value=None): + with patch("tools.sdlc_stage_query._find_plan_path", return_value=None): + result = query_enriched(session_id="sid") + + # The seam: the aggregate (and its siblings) must survive filtering. + assert result["stages"][VERIFICATION_OUTCOMES_KEY] == aggregate + assert result["stages"]["_verdicts"] == verdicts + assert result["stages"]["_sdlc_dispatches"] == [ + {"skill": "/do-pr-review", "at": "2026-09-01T00:00:00+00:00"} + ] + + meta = dict(result["_meta"]) + meta["pr_number"] = 3123 + meta["latest_review_verdict"] = "APPROVED" + meta["latest_review_head_sha"] = head_sha + # Neutralize G6's fast-path merge guard (mergeable + CI green + docs done + # + APPROVED short-circuits straight to /do-merge before DISPATCH_RULES + # ever runs) so this test isolates row 8g's own seam-threading behavior, + # matching agent.sdlc_router's own row-8g test fixtures (_base_meta). + meta["pr_merge_state"] = None + meta["ci_all_passing"] = None + + decision = decide_next_dispatch(result["stages"], meta, {"pr_head_sha": head_sha}) + + assert isinstance(decision, Dispatch) + assert decision.skill == SKILL_DO_PR_REVIEW + assert decision.skill != SKILL_DO_MERGE + assert decision.row_id == "8g" + def test_pr_number_resolved_from_session_field(self): """#2003 T1.7: the AgentSession.pr_number FIELD is the first rung — when set, the read-only recovery rungs (gh lookup) are never needed.""" diff --git a/tests/unit/test_validate_build.py b/tests/unit/test_validate_build.py index e94f981f4..f4c10188e 100644 --- a/tests/unit/test_validate_build.py +++ b/tests/unit/test_validate_build.py @@ -358,6 +358,20 @@ def test_missing_plan_file(self, tmp_path): with patch("sys.argv", ["validate_build.py", nonexistent]): assert validate_build.main() == 0 + def test_empty_plan_path_reports_cleanly_instead_of_traceback(self): + """TD 2 (review of PR #3123): `Path("")` is `Path(".")`, and + `Path(".").exists()` is True since it's the cwd -- so the old + `.exists()` guard let `read_text()` raise `IsADirectoryError` past + `main()`. An empty `$PLAN_PATH` interpolated into the documented + shell invocation must report cleanly, not crash.""" + with patch("sys.argv", ["validate_build.py", ""]): + assert validate_build.main() == 0 + + def test_directory_plan_path_reports_cleanly(self, tmp_path): + """A non-blank path that names a directory hits the same guard.""" + with patch("sys.argv", ["validate_build.py", str(tmp_path)]): + assert validate_build.main() == 0 + def test_empty_plan_file(self, tmp_path): f = tmp_path / "empty.md" f.write_text("") @@ -725,6 +739,104 @@ def test_commands_are_executed_once_not_twice(self, tmp_path): assert rc.call_count == 1 +class TestRecordingStandsDownWithNoVerificationTable: + """Blocker 2 (review of PR #3123): a plan with no ``## Verification`` + table has declared no gate. Recording anyway writes a 0-row aggregate + that grades UNEVALUATED, and UNEVALUATED blocks merge permanently with + no self-heal. The writer must never be called for this shape, and the + stand-down must be visible in the output, not silent. + """ + + def test_no_table_skips_the_writer_and_says_so(self, tmp_path): + f = tmp_path / "no_table.md" + f.write_text( + textwrap.dedent("""\ + ## Success Criteria + - [ ] `test -e /dev/null` passes + """) + ) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "owner/name", + "--issue", + "4242", + "--pr", + "77", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + exit_code = validate_build.main() + + writer.assert_not_called() + assert exit_code == 0, "the write being skipped must not change the exit code" + + def test_no_table_skip_message_is_distinct(self, tmp_path, capsys): + f = tmp_path / "no_table.md" + f.write_text( + textwrap.dedent("""\ + ## Success Criteria + - [ ] `test -e /dev/null` passes + """) + ) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "owner/name", + "--issue", + "4242", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes"), + ): + validate_build.main() + out = capsys.readouterr().out + assert ( + "RECORD: skipped -- plan declares no verification table, " + "so there is no gate to record" in out + ) + + def test_a_table_still_records_normally(self, tmp_path): + """The stand-down is scoped to the no-table case only; a plan that + does declare a table must still record exactly as before.""" + f = tmp_path / "with_table.md" + f.write_text( + textwrap.dedent("""\ + ## Verification + | Check | Command | Expected | + |-------|---------|----------| + | Echo works | `echo hi` | output contains hi | + """) + ) + argv = [ + "validate_build.py", + str(f), + "--record-outcomes", + "--repo", + "owner/name", + "--issue", + "4242", + "--pr", + "77", + ] + with ( + patch("sys.argv", argv), + patch.object(validate_build, "record_verification_outcomes") as writer, + ): + writer.return_value = True + exit_code = validate_build.main() + + writer.assert_called_once() + assert exit_code == 0, "recording must not change the exit code" + + class TestRecordingIsWiredIntoTheReviewStage: """The flag's only production invocation is one line of markdown. @@ -774,7 +886,7 @@ def test_build_addendum_does_not_record(self): def test_the_flag_the_doc_passes_is_the_flag_the_script_accepts(self): """Pins the doc and the parser together, so renaming one breaks here.""" - opts, positionals, rejected = validate_build._parse_argv( + opts, positionals, rejected, _ = validate_build._parse_argv( ["plan.md", "--record-outcomes", "--repo", "o/n", "--issue", "1", "--pr", "2"] ) assert not rejected @@ -794,29 +906,102 @@ class TestArgvParsingRejectsFlagShapedValues: """ def test_missing_value_rejects_the_flag(self): - opts, _, rejected = validate_build._parse_argv(["p.md", "--issue", "--pr", "77"]) + opts, _, rejected, _ = validate_build._parse_argv(["p.md", "--issue", "--pr", "77"]) assert "--issue" in rejected assert "--issue" not in opts assert opts["--pr"] == "77" def test_trailing_flag_with_no_value_is_rejected(self): - opts, _, rejected = validate_build._parse_argv(["p.md", "--repo"]) + opts, _, rejected, _ = validate_build._parse_argv(["p.md", "--repo"]) assert rejected == ["--repo"] assert "--repo" not in opts def test_positional_is_found_after_a_bare_flag(self): - _, positionals, _ = validate_build._parse_argv(["--record-outcomes", "p.md"]) + _, positionals, _, _ = validate_build._parse_argv(["--record-outcomes", "p.md"]) assert positionals == ["p.md"] def test_unknown_flag_is_rejected_not_treated_as_a_positional(self): - _, positionals, rejected = validate_build._parse_argv(["p.md", "--bogus"]) + _, positionals, rejected, _ = validate_build._parse_argv(["p.md", "--bogus"]) assert positionals == ["p.md"] assert rejected == ["--bogus"] + def test_single_dash_value_is_rejected_like_a_flag(self): + """The docstring promises a value that 'is itself a flag' is + rejected; a single-dash token (`-x`, or a negative number like `-5`) + must be rejected the same way as a double-dash one, not silently + accepted as the value.""" + opts, _, rejected, _ = validate_build._parse_argv(["p.md", "--issue", "-5"]) + assert "--issue" in rejected + assert "--issue" not in opts + + def test_repo_value_that_looks_like_a_flag_is_rejected(self): + opts, _, rejected, _ = validate_build._parse_argv(["p.md", "--repo", "-x"]) + assert "--repo" in rejected + assert "--repo" not in opts + + def test_repeated_flag_last_wins_and_is_reported(self): + opts, _, _, warnings = validate_build._parse_argv( + ["p.md", "--repo", "first/one", "--repo", "second/one"] + ) + assert opts["--repo"] == "second/one" + assert any("--repo" in w and "repeated" in w for w in warnings) + + def test_extra_positionals_are_dropped_and_reported(self): + _, positionals, _, warnings = validate_build._parse_argv(["a.md", "b.md", "c.md"]) + assert positionals == ["a.md", "b.md", "c.md"] + assert any("b.md" in w and "c.md" in w for w in warnings) + + def test_no_repeats_or_extras_yields_no_warnings(self): + _, _, _, warnings = validate_build._parse_argv(["p.md", "--repo", "o/n"]) + assert warnings == [] + + def test_help_anywhere_in_argv_is_honored(self): + with patch("sys.argv", ["validate_build.py", "--record-outcomes", "--help"]): + assert validate_build.main() == 0 + + def test_h_anywhere_in_argv_is_honored(self): + with patch("sys.argv", ["validate_build.py", "p.md", "-h"]): + assert validate_build.main() == 0 + def test_no_positional_exits_nonzero_rather_than_green_on_zero_checks(self): with patch("sys.argv", ["validate_build.py", "--record-outcomes", "--repo", "o/n"]): assert validate_build.main() == 1 + def test_zero_timeout_falls_back_to_default_not_all_unevaluated(self, tmp_path, capsys): + """TD 3 (review of PR #3123): `--timeout 0` (or negative) converted + every row to UNEVALUATED, which now blocks merge permanently once + recorded. A non-positive bound must be rejected with the established + `ARGS: ignoring ...` line and fall back to the default, not silently + grade the whole table UNEVALUATED.""" + f = tmp_path / "plan.md" + f.write_text( + "## Verification\n| Check | Command | Expected |\n" + "|---|---|---|\n| Echo | `echo hi` | output contains hi |\n" + ) + with patch("sys.argv", ["validate_build.py", str(f), "--timeout", "0"]): + assert validate_build.main() == 0 + out = capsys.readouterr().out + assert "ARGS: ignoring --timeout '0'" in out + assert "UNEVALUATED:" not in out + assert "Result: 1 PASS, 0 FAIL, 0 UNEVALUATED" in out + + def test_negative_timeout_value_is_rejected_at_the_argv_level(self, tmp_path, capsys): + """A value starting with `-` is rejected as flag-shaped before it + ever reaches the int-and-sign check, so `--timeout -5` never grades + the table UNEVALUATED either -- it falls back to the default via the + same `bad_flags` path as any other malformed value flag.""" + f = tmp_path / "plan.md" + f.write_text( + "## Verification\n| Check | Command | Expected |\n" + "|---|---|---|\n| Echo | `echo hi` | output contains hi |\n" + ) + with patch("sys.argv", ["validate_build.py", str(f), "--timeout", "-5"]): + assert validate_build.main() == 0 + out = capsys.readouterr().out + assert "ARGS: ignoring --timeout" in out + assert "UNEVALUATED:" not in out + assert "Result: 1 PASS, 0 FAIL, 0 UNEVALUATED" in out + def test_non_integer_issue_does_not_change_the_exit_code(self, tmp_path): f = tmp_path / "plan.md" f.write_text( diff --git a/tests/unit/test_verification_parser.py b/tests/unit/test_verification_parser.py index e19e33e6e..9c62d8de9 100644 --- a/tests/unit/test_verification_parser.py +++ b/tests/unit/test_verification_parser.py @@ -1106,6 +1106,70 @@ def test_live_plan_rows_verbatim(self): ) +class TestNewlyRecognizedAuthoringForms: + """Round 3 of the #3065 review found two in-flight PRs (#3093, #3089) + would be refused the moment this gate lands: their plans use three + legitimate authoring shapes the grammar did not recognise -- + backtick-wrapped-integer equality, `is` as the equality verb, and a + comma-delimited gloss after a bare `exit N`. Each gets a PASS/FAIL pair + plus the verbatim live-plan cell that must grade.""" + + # --- output `N` (backtick-wrapped integer equality) --- + + def test_output_backtick_int_passes(self): + assert evaluate_expectation("output `0`", exit_code=0, output="0") is PASS + + def test_output_backtick_int_fails(self): + assert evaluate_expectation("output `0`", exit_code=0, output="1") is FAIL + + def test_live_plan_row_output_backtick_zero(self): + """Verbatim from docs/plans/promise-gate-recorded-obligations.md:433-434.""" + assert evaluate_expectation("output `0`", exit_code=1, output="0") is PASS + + # --- output is N --- + + def test_output_is_n_passes(self): + assert evaluate_expectation("output is 0", exit_code=0, output="0") is PASS + + def test_output_is_n_fails(self): + assert evaluate_expectation("output is 0", exit_code=0, output="1") is FAIL + + def test_live_plan_row_output_is_zero(self): + """Verbatim from + docs/plans/unblock-dependency-bumps-coupled-set-gate.md:2047.""" + assert evaluate_expectation("output is 0", exit_code=0, output="0") is PASS + + # --- exit N, --- + + def test_exit_n_comma_gloss_passes(self): + assert evaluate_expectation("exit 0, some gloss text", exit_code=0, output="") is PASS + + def test_exit_n_comma_gloss_fails(self): + assert evaluate_expectation("exit 0, some gloss text", exit_code=1, output="") is FAIL + + def test_live_plan_row_exit_0_json_gloss(self): + """Verbatim from + docs/plans/unblock-dependency-bumps-coupled-set-gate.md:2008.""" + assert ( + evaluate_expectation('exit 0, JSON `"compatible": true`', exit_code=0, output="") + is PASS + ) + + def test_exit_n_bare_trailing_word_still_unevaluated(self): + """The anchoring counter-example (`exit 0 maybe`, no delimiter) must + keep grading UNEVALUATED -- pinned again here alongside the new forms + so a regression in the delimiter rule is caught in the same place.""" + assert evaluate_expectation("exit 0 maybe", exit_code=0, output="") is UNEVALUATED + + def test_bare_comparator_gloss_counter_examples_still_unevaluated(self): + """The delimited-gloss rule is scoped to `exit N` only; the + pre-existing bare `>`, `>=`, `==` forms stay fully anchored and a + parenthesized gloss on them is still UNEVALUATED (see + TestExpectationAnchoringRule and TestNumericComparatorTrailingGlossSymmetry).""" + for cell in ("== 2 (the two read sites)", ">= 1 nightly", "> 0 or so"): + assert evaluate_expectation(cell, exit_code=0, output="2") is UNEVALUATED + + class TestReadVerificationOutcomesFailsClosed: """Absent and unreadable must be distinguishable (review of PR #3123). diff --git a/tools/merge_predicate.py b/tools/merge_predicate.py index 07ea2d964..554592217 100644 --- a/tools/merge_predicate.py +++ b/tools/merge_predicate.py @@ -862,15 +862,17 @@ def _check_verification_outcomes( # The tri-state tokens come from the writer's own enum, never re-spelled # here: a literal "UNEVALUATED" in this module would be a replicated value - # that silently stops matching if the enum is ever renamed. - from agent.verification_parser import CheckOutcome + # that silently stops matching if the enum is ever renamed. The blocking set + # itself is shared with the router's row 8g for the same reason -- see + # BLOCKING_OUTCOMES. + from agent.verification_parser import BLOCKING_OUTCOMES, CheckOutcome rows = aggregate.get("rows") if not isinstance(rows, list): failed.append("verification outcomes: recorded aggregate has no readable rows") return - blocking = {CheckOutcome.FAIL.value, CheckOutcome.UNEVALUATED.value} + blocking = BLOCKING_OUTCOMES offending = 0 for row in rows: if not isinstance(row, dict): diff --git a/tools/sdlc_stage_query.py b/tools/sdlc_stage_query.py index 1560b149c..fc8cfa446 100644 --- a/tools/sdlc_stage_query.py +++ b/tools/sdlc_stage_query.py @@ -1002,13 +1002,28 @@ def query_enriched( except Exception: stages = {k: v for k, v in raw_states.items() if not k.startswith("_")} + try: + from agent.verification_parser import VERIFICATION_OUTCOMES_KEY + except Exception: # pragma: no cover - mirrors the ALL_STAGES import above + VERIFICATION_OUTCOMES_KEY = "_verification_outcomes" # noqa: N806 + # Thread the router-helper underscore keys into the stages dict. The router's # staleness rules (``_critique_verdict_is_stale`` / ``_latest_dispatch_at`` → # row 2b/8b) read ``_verdicts`` and ``_sdlc_dispatches`` directly off the # ``stage_states`` arg. Without them here, those rules are structurally inert # in the CLI path: a revised plan with a stale NEEDS REVISION verdict can never # route to re-critique and dead-ends on ``/do-plan`` until G4 oscillation fires. - for _router_key in ("_verdicts", "_sdlc_dispatches"): + # + # ``_verification_outcomes`` is here for exactly the same reason and was + # missed once already. Row 8g (``_rule_verification_outcomes_hold_pr``) reads + # the recorded aggregate off ``stage_states`` to re-route a lane the merge + # predicate would refuse. The predicate reads the ledger directly, so it sees + # the aggregate either way — meaning that without this key the two disagree: + # the router keeps dispatching ``/do-merge`` and the predicate keeps refusing + # it, until G4 blocks the lane for a human. That is the oscillation loop row + # 8g exists to prevent, so the whole tuple is pinned by a seam test rather + # than by the hand-built payloads the router's own tests use. + for _router_key in ("_verdicts", "_sdlc_dispatches", VERIFICATION_OUTCOMES_KEY): if _router_key in raw_states: stages[_router_key] = raw_states[_router_key]