diff --git a/.github/scripts/test-required-ci-gate.py b/.github/scripts/test-required-ci-gate.py new file mode 100644 index 00000000..2ee8c17b --- /dev/null +++ b/.github/scripts/test-required-ci-gate.py @@ -0,0 +1,1035 @@ +#!/usr/bin/env python3 +"""Regression tests for the required CI gate in .github/workflows/ci.yml (#796). + +`Required CI gate` is the only required check on main. It passes when +`implementation-gate` ("SpecSync implementation ready") passes, and that job +reads the results of the jobs it needs. GitHub reports `skipped` for a job +classify deselected and for a job whose own dependency failed, so the gate has +to tell the two apart. Before #796 it could not: the lifecycle gate failed, +test, audit, coverage and spec-check were skipped, and both gates went green. + +These tests hold the workflow to three rules: + +1. Every job that can finish before implementation-gate is in its `needs`, + `preflight` and `lifecycle-gate` included. +2. The gate's row for each job evaluates that job's own `if:` again, so + `skipped` passes only where classify deselected the job. +3. Simulated over every classify path, the required gate is green when every + selected job succeeds, and red when any one of them fails or is cancelled. + +The workflow is parsed with Ruby's standard-library Psych, like the other +workflow validators here, and the gate's own shell steps are executed the way +the runner invokes them. `--truth-table` prints the per-path table. +""" + +from __future__ import annotations + +import copy +from concurrent.futures import ThreadPoolExecutor +import functools +import itertools +import json +import math +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import tempfile +import unittest +from dataclasses import dataclass, field +from typing import Any, Callable + + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/ci.yml" +GATE_JOB = "implementation-gate" +REQUIRED_JOB = "ci-gate" +ALWAYS_REQUIRED = ("classify", "preflight", "lifecycle-gate") +CLASSIFY_FLAGS = ( + "full", + "site", + "vscode", + "archive_only", + "legacy_archive_only", + "review_only", + "review_required", +) +STATUS_FUNCTIONS = frozenset({"always", "success", "failure", "cancelled"}) + + +# MARK: - Workflow loading + + +@functools.lru_cache(maxsize=None) +def load_workflow(path: Path = WORKFLOW) -> dict[str, Any]: + """Parse a workflow through Ruby's standard-library Psych parser (once; never mutate).""" + ruby = r""" +require "json" +require "psych" + +document = Psych.safe_load(File.read(ARGV.fetch(0), encoding: "UTF-8"), permitted_classes: [], aliases: false) +raise "workflow is not a mapping" unless document.is_a?(Hash) +raise "workflow has no jobs mapping" unless document["jobs"].is_a?(Hash) +puts JSON.generate({ "env" => document["env"] || {}, "jobs" => document["jobs"] }) +""" + try: + result = subprocess.run( + ["ruby", "-e", ruby, str(path)], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + except FileNotFoundError as error: + raise RuntimeError("Ruby with standard-library Psych is required") from error + if result.returncode != 0: + raise RuntimeError(f"{path}: workflow parsing failed: {result.stderr.strip()[-2000:]}") + document = json.loads(result.stdout) + jobs = {name: normalize_job(name, job) for name, job in document["jobs"].items()} + return {"env": document["env"], "jobs": jobs} + + +def normalize_job(name: str, job: Any) -> dict[str, Any]: + """Return one job with list `needs`, optional string `if`, and a step list.""" + if not isinstance(job, dict): + raise RuntimeError(f"job {name} is not a mapping") + needs = job.get("needs", []) + if isinstance(needs, str): + needs = [needs] + condition = job.get("if") + if isinstance(condition, bool): + condition = "true" if condition else "false" + return { + "needs": list(needs), + "if": condition, + "outputs": job.get("outputs") or {}, + "env": job.get("env") or {}, + "steps": job.get("steps") or [], + } + + +def ancestors(jobs: dict[str, Any], name: str) -> set[str]: + """Every job `name` needs, directly or transitively.""" + found: set[str] = set() + pending = list(jobs[name]["needs"]) + while pending: + current = pending.pop() + if current in found: + continue + if current not in jobs: + raise RuntimeError(f"{name} needs unknown job {current}") + found.add(current) + pending.extend(jobs[current]["needs"]) + return found + + +def topological_order(jobs: dict[str, Any]) -> list[str]: + """Jobs in an order where each follows everything it needs.""" + order: list[str] = [] + placed: set[str] = set() + remaining = dict(jobs) + while remaining: + ready = sorted(name for name, job in remaining.items() if set(job["needs"]) <= placed) + if not ready: + raise RuntimeError(f"workflow needs contain a cycle among {sorted(remaining)}") + for name in ready: + order.append(name) + placed.add(name) + del remaining[name] + return order + + +# MARK: - GitHub expressions + + +TOKEN = re.compile( + r"""\s*(?: + (?P'(?:[^']|'')*') + | (?P\d+(?:\.\d+)?) + | (?P==|!=|&&|\|\||!|\(|\)|,|\.) + | (?P\*) + | (?P[A-Za-z_][A-Za-z0-9_-]*) + )""", + re.VERBOSE, +) + + +def tokenize(source: str) -> list[tuple[str, str]]: + """Split one expression into (kind, text) tokens, failing on anything unsupported.""" + tokens: list[tuple[str, str]] = [] + position = 0 + source = source.rstrip() + while position < len(source): + match = TOKEN.match(source, position) + if match is None or match.end() == position: + raise ValueError(f"unsupported expression syntax at {source[position:]!r} in {source!r}") + kind = match.lastgroup or "" + tokens.append((kind, match.group(kind))) + position = match.end() + return tokens + + +def called_functions(source: str) -> set[str]: + """Names called as functions anywhere in an expression.""" + tokens = tokenize(source) + return { + text.lower() + for (kind, text), (_, following) in zip(tokens, tokens[1:] + [("", "")]) + if kind == "name" and following == "(" + } + + +def to_number(value: Any) -> float: + if value is None: + return 0.0 + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, float): + return value + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return 0.0 + try: + return float(stripped) + except ValueError: + return math.nan + return math.nan + + +def truthy(value: Any) -> bool: + if value is None or value is False: + return False + if isinstance(value, float): + return value != 0 and not math.isnan(value) + if isinstance(value, str): + return value != "" + return True + + +def loose_equal(left: Any, right: Any) -> bool: + """GitHub's `==`: strings compare case-insensitively, mixed types as numbers.""" + if isinstance(left, str) and isinstance(right, str): + return left.casefold() == right.casefold() + if type(left) is type(right) and not isinstance(left, (list, dict)): + return left == right + return to_number(left) == to_number(right) + + +def to_text(value: Any) -> str: + """How GitHub renders an expression value inside a string.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, float): + return str(int(value)) if value.is_integer() else str(value) + if isinstance(value, str): + return value + return json.dumps(value) + + +def lookup(container: Any, key: str) -> Any: + if isinstance(container, list): + return [lookup(item, key) for item in container] + if not isinstance(container, dict): + return None + if key in container: + return container[key] + folded = {str(name).casefold(): value for name, value in container.items()} + return folded.get(key.casefold()) + + +@dataclass +class Evaluator: + """Recursive-descent evaluator for the expression subset ci.yml uses.""" + + context: dict[str, Any] + functions: dict[str, Callable[..., Any]] + tokens: list[tuple[str, str]] = field(default_factory=list) + index: int = 0 + + def evaluate(self, source: str) -> Any: + self.tokens = tokenize(source) + self.index = 0 + value = self.either() + if self.index != len(self.tokens): + raise ValueError(f"unexpected {self.tokens[self.index][1]!r} in {source!r}") + return value + + def peek(self) -> str: + return self.tokens[self.index][1] if self.index < len(self.tokens) else "" + + def take(self, expected: str | None = None) -> tuple[str, str]: + if self.index >= len(self.tokens): + raise ValueError("expression ended early") + token = self.tokens[self.index] + if expected is not None and token[1] != expected: + raise ValueError(f"expected {expected!r}, found {token[1]!r}") + self.index += 1 + return token + + def either(self) -> Any: + value = self.both() + while self.peek() == "||": + self.take() + right = self.both() + value = value if truthy(value) else right + return value + + def both(self) -> Any: + value = self.comparison() + while self.peek() == "&&": + self.take() + right = self.comparison() + value = right if truthy(value) else value + return value + + def comparison(self) -> Any: + value = self.unary() + while self.peek() in ("==", "!="): + operator = self.take()[1] + right = self.unary() + equal = loose_equal(value, right) + value = equal if operator == "==" else not equal + return value + + def unary(self) -> Any: + if self.peek() == "!": + self.take() + return not truthy(self.unary()) + return self.primary() + + def primary(self) -> Any: + kind, text = self.take() + if text == "(" and kind == "op": + value = self.either() + self.take(")") + return value + if kind == "string": + return text[1:-1].replace("''", "'") + if kind == "number": + return float(text) + if kind != "name": + raise ValueError(f"unexpected {text!r}") + lowered = text.lower() + if lowered in ("true", "false"): + return lowered == "true" + if lowered == "null": + return None + if self.peek() == "(": + self.take("(") + arguments: list[Any] = [] + while self.peek() != ")": + arguments.append(self.either()) + if self.peek() == ",": + self.take(",") + self.take(")") + function = self.functions.get(lowered) + if function is None: + raise ValueError(f"unsupported function {text}()") + return function(*arguments) + value = lookup(self.context, text) + while self.peek() == ".": + self.take(".") + kind, key = self.take() + if kind == "star": + value = list(value.values()) if isinstance(value, dict) else None + elif kind == "name": + value = lookup(value, key) + else: + raise ValueError(f"unsupported property {key!r}") + return value + + +def join(values: Any, separator: Any = ",") -> str: + if isinstance(values, list): + return str(separator).join(to_text(value) for value in values) + return to_text(values) + + +def strip_wrapper(source: str) -> str: + """Drop one `${{ ... }}` wrapping a whole condition.""" + text = source.strip() + match = re.fullmatch(r"\$\{\{(.*)\}\}", text, re.DOTALL) + return match.group(1).strip() if match else text + + +def interpolate(template: Any, evaluate: Callable[[str], Any]) -> str: + """Replace every `${{ expression }}` in a string the way the runner does.""" + return re.sub(r"\$\{\{(.*?)\}\}", lambda match: to_text(evaluate(match.group(1))), str(template)) + + +# MARK: - Job selection + + +def selection_expression(job: dict[str, Any]) -> str: + """The job's own `if:` with any leading `always() &&` removed; `true` when unconditional.""" + condition = job["if"] + if condition is None: + return "true" + text = " ".join(strip_wrapper(condition).split()) + text = re.sub(r"^always\(\)\s*(&&\s*|$)", "", text) + return text or "true" + + +def gate_rows(jobs: dict[str, Any]) -> tuple[dict[str, tuple[str, str]], list[str]]: + """Parse the gate's GATES rows into {job: (selection, result)} plus parse errors.""" + errors: list[str] = [] + rows: dict[str, tuple[str, str]] = {} + tables = [ + step["env"]["GATES"] + for step in jobs[GATE_JOB]["steps"] + if isinstance(step.get("env"), dict) and "GATES" in step["env"] + ] + if len(tables) != 1: + return rows, [f"{GATE_JOB} must have exactly one step with a GATES table, found {len(tables)}"] + row_pattern = re.compile( + r"(?P[A-Za-z_][A-Za-z0-9_-]*) " + r"(?Ptrue|\$\{\{ .+? \}\}) " + r"(?P\$\{\{ needs\.[A-Za-z0-9_-]+\.result \}\})" + ) + for line in str(tables[0]).splitlines(): + if not line.strip(): + continue + match = row_pattern.fullmatch(line.strip()) + if match is None: + errors.append(f"{GATE_JOB} GATES row is malformed: {line!r}") + continue + job = match.group("job") + if job in rows: + errors.append(f"{GATE_JOB} GATES lists {job} twice") + rows[job] = (match.group("selected"), match.group("result")) + return rows, errors + + +def gate_contract_errors(jobs: dict[str, Any]) -> list[str]: + """Structural rules for implementation-gate; empty when the workflow holds to them.""" + errors: list[str] = [] + for name in (GATE_JOB, REQUIRED_JOB): + if name not in jobs: + return [f"workflow has no {name} job"] + gate = jobs[GATE_JOB] + needs = set(gate["needs"]) + if GATE_JOB not in ancestors(jobs, REQUIRED_JOB): + errors.append(f"{REQUIRED_JOB} must need {GATE_JOB}") + if selection_expression(gate) != "true" or "always" not in called_functions(strip_wrapper(gate["if"] or "")): + errors.append(f"{GATE_JOB} must run with if: always() so a failure upstream cannot skip it") + + for name in ALWAYS_REQUIRED: + if name not in needs: + errors.append(f"{GATE_JOB}.needs must include {name}") + downstream = {name for name in jobs if GATE_JOB in ancestors(jobs, name)} | {GATE_JOB} + for name in sorted(set(jobs) - downstream): + if name in needs or name in ALWAYS_REQUIRED: + continue + reason = ( + "gates on lifecycle-gate" + if "lifecycle-gate" in ancestors(jobs, name) + else "can finish before the gate" + ) + errors.append(f"{name} {reason} but is missing from {GATE_JOB}.needs") + for name in sorted(needs & downstream): + errors.append(f"{GATE_JOB}.needs contains {name}, which runs after the gate") + + rows, row_errors = gate_rows(jobs) + errors.extend(row_errors) + if row_errors and not rows: + return errors + for name in sorted(needs - set(rows)): + errors.append(f"{name} is in {GATE_JOB}.needs but has no GATES row, so the gate ignores it") + for name in sorted(set(rows) - needs): + errors.append(f"{GATE_JOB} GATES has a row for {name}, which is not in its needs") + for name in sorted(needs & set(rows)): + selected, result = rows[name] + if result != f"${{{{ needs.{name}.result }}}}": + errors.append(f"{name} GATES row reads {result}, not its own result") + expected = selection_expression(jobs[name]) + leftover = called_functions(expected) & STATUS_FUNCTIONS + if leftover: + errors.append( + f"{name} if: calls {sorted(leftover)}; the gate cannot tell when it is selected" + ) + continue + actual = "true" if selected == "true" else " ".join(strip_wrapper(selected).split()) + if actual != expected: + errors.append( + f"{name} GATES row selects on {actual!r} but the job runs on {expected!r}" + ) + return errors + + +# MARK: - Simulation + + +@dataclass(frozen=True) +class Scenario: + """One classify outcome for one event.""" + + label: str + event: str + flags: tuple[str, ...] = () + ref: str = "refs/pull/1/merge" + + def outputs(self, jobs: dict[str, Any]) -> dict[str, str]: + keys = jobs["classify"]["outputs"].keys() + values = {key: ("false" if key in CLASSIFY_FLAGS else "") for key in keys} + for flag in self.flags: + if flag not in values: + raise KeyError(f"classify has no output {flag}") + values[flag] = "true" + return values + + +# Every lane classify-ci-paths.sh and select-ci-lane.sh can choose today. +SCENARIOS = ( + Scenario("Full PR (src/, tests/, workflows, docs/, *.md)", "pull_request", ("full",)), + Scenario("Full PR, change awaiting scoped review", "pull_request", ("full", "review_required")), + Scenario("Site-only PR (site/**)", "pull_request", ("site",)), + Scenario("VS Code-only PR (vscode-extension/**)", "pull_request", ("vscode",)), + Scenario("Site and VS Code PR", "pull_request", ("site", "vscode")), + Scenario("Specs/lifecycle-only PR (specs/**, .specsync/changes/**)", "pull_request"), + Scenario("Specs/lifecycle-only PR, awaiting scoped review", "pull_request", ("review_required",)), + Scenario("Archive-only PR (workflow-v2 archive move)", "pull_request", ("archive_only",)), + Scenario( + "Legacy archive-only PR (workflow-v1)", + "pull_request", + ("legacy_archive_only", "full"), + ), + Scenario("Review-only PR (review.json + review-attempts.json)", "pull_request", ("review_only",)), + Scenario("Push to main (full)", "push", ("full",), "refs/heads/main"), + Scenario("Push to main, verifying change present", "push", ("full", "review_required"), "refs/heads/main"), + Scenario("Push to main (site-only)", "push", ("site",), "refs/heads/main"), + Scenario("Push to main (specs/lifecycle-only)", "push", (), "refs/heads/main"), + Scenario("workflow_dispatch (forced full)", "workflow_dispatch", ("full",), "refs/heads/main"), +) + + +def expression_context( + jobs: dict[str, Any], + name: str, + results: dict[str, str], + scenario: Scenario, + outputs: dict[str, str], +) -> dict[str, Any]: + needs = {} + for dependency in jobs[name]["needs"]: + needs[dependency] = { + "result": results[dependency], + "outputs": outputs if dependency == "classify" and results[dependency] == "success" else {}, + } + return { + "needs": needs, + "github": {"event_name": scenario.event, "ref": scenario.ref}, + } + + +def job_functions(jobs: dict[str, Any], name: str, results: dict[str, str]) -> dict[str, Callable[..., Any]]: + # Job-level status functions look at every transitive dependency, so a job + # whose grandparent was skipped is skipped too. That is observable here: + # `attest` is skipped on every push to main, where `corvid-pet`, upstream of + # `ci-gate`, is skipped. + upstream = [results[job] for job in ancestors(jobs, name)] + return { + "always": lambda: True, + "success": lambda: all(result == "success" for result in upstream), + "failure": lambda: any(result == "failure" for result in upstream), + "cancelled": lambda: False, + "join": join, + } + + +def job_runs( + jobs: dict[str, Any], + name: str, + results: dict[str, str], + scenario: Scenario, + outputs: dict[str, str], +) -> bool: + functions = job_functions(jobs, name, results) + evaluator = Evaluator(expression_context(jobs, name, results, scenario, outputs), functions) + condition = jobs[name]["if"] + source = "true" if condition is None else strip_wrapper(condition) + decision = truthy(evaluator.evaluate(source)) + if not called_functions(source) & STATUS_FUNCTIONS: + decision = decision and functions["success"]() + return decision + + +def run_gate_steps( + workflow: dict[str, Any], + name: str, + results: dict[str, str], + scenario: Scenario, + outputs: dict[str, str], +) -> tuple[str, str]: + """Execute a gate job's `run` steps; return (result, combined output).""" + jobs = workflow["jobs"] + evaluator = Evaluator( + expression_context(jobs, name, results, scenario, outputs), + job_functions(jobs, name, results), + ) + transcript: list[str] = [] + for step in jobs[name]["steps"]: + if "run" not in step: + continue + if "if" in step: + raise NotImplementedError(f"{name} step conditions are not simulated") + environment = {"PATH": os.environ.get("PATH", "/usr/bin:/bin")} + for scope in (workflow["env"], jobs[name]["env"], step.get("env") or {}): + for key, value in scope.items(): + environment[key] = interpolate(value, evaluator.evaluate) + script = interpolate(step["run"], evaluator.evaluate) + returncode, output = run_step( + step.get("shell"), script, tuple(sorted(environment.items())) + ) + transcript.append(output) + if returncode != 0: + return "failure", "".join(transcript) + return "success", "".join(transcript) + + +@functools.lru_cache(maxsize=None) +def run_step( + shell: str | None, script: str, environment: tuple[tuple[str, str], ...] +) -> tuple[int, str]: + """Run one step script under the runner's invocation for its shell; same input, same answer.""" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "step.sh" + path.write_text(script, encoding="utf-8") + if shell is None: + # A step that names no shell runs as `bash -e {0}` on Linux runners + # (the job log prints `shell: /usr/bin/bash -e {0}`), without pipefail. + command = ["bash", "-e", str(path)] + elif shell == "bash": + command = ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(path)] + elif shell == "sh": + command = ["sh", "-e", str(path)] + elif "{0}" in shell: + command = shlex.split(shell.replace("{0}", str(path))) + else: + raise NotImplementedError(f"unsupported shell {shell!r}") + completed = subprocess.run( + command, + cwd=ROOT, + env=dict(environment), + capture_output=True, + text=True, + timeout=30, + check=False, + ) + return completed.returncode, completed.stdout + completed.stderr + + +def simulate( + workflow: dict[str, Any], + scenario: Scenario, + forced: dict[str, str] | None = None, +) -> tuple[dict[str, str], str]: + """Every job's result for one scenario; `forced` sets a job's result if it runs.""" + jobs = workflow["jobs"] + forced = forced or {} + outputs = scenario.outputs(jobs) + results: dict[str, str] = {} + transcript = "" + for name in topological_order(jobs): + if not job_runs(jobs, name, results, scenario, outputs): + results[name] = "skipped" + elif name in forced: + results[name] = forced[name] + elif name in (GATE_JOB, REQUIRED_JOB): + results[name], output = run_gate_steps(workflow, name, results, scenario, outputs) + transcript += output + else: + results[name] = "success" + return results, transcript + + +def simulate_many( + workflow: dict[str, Any], + cases: list[tuple[Scenario, dict[str, str]]], +) -> list[tuple[dict[str, str], str]]: + """`simulate` for many independent cases, in parallel (each spawns real shells).""" + with ThreadPoolExecutor(max_workers=max(4, os.cpu_count() or 4)) as pool: + return list(pool.map(lambda case: simulate(workflow, case[0], case[1]), cases)) + + +def product_jobs(workflow: dict[str, Any]) -> list[str]: + """Jobs that finish before the gate, in workflow order.""" + jobs = workflow["jobs"] + downstream = {name for name in jobs if GATE_JOB in ancestors(jobs, name)} | {GATE_JOB} + return [name for name in jobs if name not in downstream] + + +# The gate as it stood when #796 was found, kept to prove the harness sees the bug. +PRE_796_GATE_NEEDS = [ + "classify", "test", "fmt", "hi-check", "validate-action", "action-consumer", + "spec-check", "audit", "coverage", "site", "vscode-extension", "corvid-pet", +] +PRE_796_GATE_STEP = { + "name": "Require every selected gate", + "env": {"RESULTS": "${{ join(needs.*.result, ' ') }}"}, + "run": ( + 'for result in $RESULTS; do\n' + ' case "$result" in\n' + ' success|skipped) ;;\n' + ' *) echo "Selected CI gate ended with: $result" >&2; exit 1 ;;\n' + ' esac\n' + 'done\n' + ), +} + + +def pre_796_workflow(workflow: dict[str, Any]) -> dict[str, Any]: + before = copy.deepcopy(workflow) + before["jobs"][GATE_JOB]["needs"] = list(PRE_796_GATE_NEEDS) + before["jobs"][GATE_JOB]["steps"] = [copy.deepcopy(PRE_796_GATE_STEP)] + return before + + +# MARK: - Tests + + +class RequiredGateContractTests(unittest.TestCase): + """The workflow on disk holds to the gate's structural rules.""" + + workflow: dict[str, Any] + + @classmethod + def setUpClass(cls) -> None: + cls.workflow = load_workflow() + + def test_workflow_holds_to_the_gate_contract(self) -> None: + self.assertEqual(gate_contract_errors(self.workflow["jobs"]), []) + + def test_gate_needs_preflight_and_the_lifecycle_gate(self) -> None: + needs = self.workflow["jobs"][GATE_JOB]["needs"] + for name in ALWAYS_REQUIRED: + with self.subTest(job=name): + self.assertIn(name, needs) + + def test_every_job_gated_on_the_lifecycle_gate_is_required(self) -> None: + jobs = self.workflow["jobs"] + needs = set(jobs[GATE_JOB]["needs"]) + gated = [ + name + for name in product_jobs(self.workflow) + if "lifecycle-gate" in ancestors(jobs, name) + ] + self.assertTrue({"test", "audit", "coverage", "spec-check"} <= set(gated)) + for name in gated: + with self.subTest(job=name): + self.assertIn(name, needs) + + def test_required_gate_depends_on_the_implementation_gate(self) -> None: + self.assertIn(GATE_JOB, self.workflow["jobs"][REQUIRED_JOB]["needs"]) + + def test_these_tests_run_through_fledge_and_ci(self) -> None: + script = ".github/scripts/test-required-ci-gate.py" + fledge = (ROOT / "fledge.toml").read_text(encoding="utf-8") + task = re.search( + r'(?ms)^\[tasks\.([A-Za-z0-9_-]+)\]\ncmd\s*=\s*"[^"]*' + re.escape(script) + r'[^"]*"\s*$', + fledge, + ) + self.assertIsNotNone(task, "Fledge must define a task that runs the required-gate tests") + if task is None: + return + verify = re.search(r'(?ms)^\[lanes\.verify\]\n(.*?)^\]', fledge) + self.assertIsNotNone(verify) + if verify is not None: + self.assertIn(f'"{task.group(1)}"', verify.group(1)) + commands = [ + str(step.get("run", "")) + for job in self.workflow["jobs"].values() + for step in job["steps"] + ] + self.assertTrue( + any(script in command for command in commands), + "CI must run the required-gate tests", + ) + + +class GuardMutationTests(unittest.TestCase): + """The structural guard fails on each way the gate can drift.""" + + workflow: dict[str, Any] + + @classmethod + def setUpClass(cls) -> None: + cls.workflow = load_workflow() + + def mutated(self) -> dict[str, Any]: + return copy.deepcopy(self.workflow["jobs"]) + + def assert_guard_reports(self, jobs: dict[str, Any], fragment: str) -> None: + errors = gate_contract_errors(jobs) + self.assertTrue( + any(fragment in error for error in errors), + f"expected an error containing {fragment!r}, got {errors}", + ) + + def test_dropping_the_lifecycle_gate_from_needs_fails(self) -> None: + jobs = self.mutated() + jobs[GATE_JOB]["needs"].remove("lifecycle-gate") + self.assert_guard_reports(jobs, f"{GATE_JOB}.needs must include lifecycle-gate") + + def test_dropping_preflight_from_needs_fails(self) -> None: + jobs = self.mutated() + jobs[GATE_JOB]["needs"].remove("preflight") + self.assert_guard_reports(jobs, f"{GATE_JOB}.needs must include preflight") + + def test_a_new_job_gated_on_the_lifecycle_gate_must_be_required(self) -> None: + jobs = self.mutated() + jobs["new-product-check"] = normalize_job( + "new-product-check", + { + "needs": ["classify", "lifecycle-gate"], + "if": "needs.classify.outputs.full == 'true'", + "steps": [{"run": "true"}], + }, + ) + self.assert_guard_reports(jobs, "new-product-check gates on lifecycle-gate but is missing") + + def test_a_new_selected_job_must_be_required(self) -> None: + jobs = self.mutated() + jobs["new-site-check"] = normalize_job( + "new-site-check", + {"needs": "classify", "if": "needs.classify.outputs.site == 'true'", "steps": []}, + ) + self.assert_guard_reports(jobs, "new-site-check can finish before the gate but is missing") + + def test_a_needed_job_without_a_row_fails(self) -> None: + jobs = self.mutated() + jobs["new-site-check"] = normalize_job("new-site-check", {"needs": "classify", "steps": []}) + jobs[GATE_JOB]["needs"].append("new-site-check") + self.assert_guard_reports(jobs, "new-site-check is in implementation-gate.needs but has no GATES row") + + def test_a_row_that_no_longer_matches_its_job_condition_fails(self) -> None: + jobs = self.mutated() + jobs["test"]["if"] = "needs.classify.outputs.site == 'true'" + self.assert_guard_reports(jobs, "test GATES row selects on") + + def test_a_condition_the_gate_cannot_mirror_fails(self) -> None: + jobs = self.mutated() + jobs["test"]["if"] = "failure() && needs.classify.outputs.full == 'true'" + self.assert_guard_reports(jobs, "test if: calls ['failure']") + + +class RequiredGateSimulationTests(unittest.TestCase): + """Simulated over every classify path, the required gate matches reality.""" + + workflow: dict[str, Any] + + @classmethod + def setUpClass(cls) -> None: + cls.workflow = load_workflow() + + def test_issue_796_lifecycle_gate_failure_turns_the_required_gate_red(self) -> None: + scenario = SCENARIOS[0] + results, transcript = simulate(self.workflow, scenario, {"lifecycle-gate": "failure"}) + for name in ("test", "audit", "coverage", "spec-check"): + self.assertEqual(results[name], "skipped", name) + self.assertEqual(results[GATE_JOB], "failure") + self.assertEqual(results[REQUIRED_JOB], "failure") + self.assertIn("::error::lifecycle-gate was selected and ended with: failure", transcript) + self.assertIn("::error::test was selected but skipped", transcript) + + def test_the_pre_796_gate_reproduces_the_bug(self) -> None: + results, _ = simulate( + pre_796_workflow(self.workflow), SCENARIOS[0], {"lifecycle-gate": "failure"} + ) + self.assertEqual(results["lifecycle-gate"], "failure") + self.assertEqual(results[GATE_JOB], "success") + self.assertEqual(results[REQUIRED_JOB], "success") + + def test_every_path_is_green_when_every_selected_job_succeeds(self) -> None: + for scenario in SCENARIOS: + with self.subTest(path=scenario.label): + results, transcript = simulate(self.workflow, scenario) + self.assertEqual(results[GATE_JOB], "success", transcript) + self.assertEqual(results[REQUIRED_JOB], "success", transcript) + + def test_every_classify_output_combination_is_green_when_everything_succeeds(self) -> None: + referenced = ("full", "site", "vscode", "archive_only", "review_only", "review_required") + scenarios = [ + Scenario(f"{event} {flags}", event, flags, ref) + for event, ref in ( + ("pull_request", "refs/pull/1/merge"), + ("push", "refs/heads/main"), + ("workflow_dispatch", "refs/heads/main"), + ) + for bits in itertools.product((False, True), repeat=len(referenced)) + for flags in [tuple(flag for flag, bit in zip(referenced, bits) if bit)] + ] + outcomes = simulate_many(self.workflow, [(scenario, {}) for scenario in scenarios]) + for scenario, (results, transcript) in zip(scenarios, outcomes): + with self.subTest(path=scenario.label): + self.assertEqual(results[REQUIRED_JOB], "success", transcript) + + def test_any_selected_job_failing_or_cancelled_turns_the_required_gate_red(self) -> None: + cases: list[tuple[Scenario, dict[str, str]]] = [] + for scenario, (green, _) in zip( + SCENARIOS, simulate_many(self.workflow, [(scenario, {}) for scenario in SCENARIOS]) + ): + ran = [name for name in product_jobs(self.workflow) if green[name] != "skipped"] + self.assertTrue(set(ALWAYS_REQUIRED) <= set(ran), scenario.label) + cases.extend( + (scenario, {name: outcome}) + for name, outcome in itertools.product(ran, ("failure", "cancelled")) + ) + for (scenario, forced), (results, transcript) in zip(cases, simulate_many(self.workflow, cases)): + with self.subTest(path=scenario.label, forced=forced): + self.assertEqual(results[GATE_JOB], "failure", transcript) + self.assertEqual(results[REQUIRED_JOB], "failure", transcript) + + def test_deselected_jobs_are_skipped_and_do_not_block(self) -> None: + expectations = { + "Archive-only PR (workflow-v2 archive move)": { + "test", "fmt", "hi-check", "validate-action", "action-consumer", + "spec-check", "audit", "coverage", "site", "vscode-extension", "corvid-pet", + }, + "Review-only PR (review.json + review-attempts.json)": { + "test", "fmt", "hi-check", "validate-action", "action-consumer", + "spec-check", "audit", "coverage", "site", "vscode-extension", "corvid-pet", + }, + "Site-only PR (site/**)": { + "test", "fmt", "hi-check", "action-consumer", "audit", "coverage", + "vscode-extension", "corvid-pet", + }, + } + by_label = {scenario.label: scenario for scenario in SCENARIOS} + for label, skipped in expectations.items(): + with self.subTest(path=label): + results, _ = simulate(self.workflow, by_label[label]) + self.assertEqual( + {name for name in product_jobs(self.workflow) if results[name] == "skipped"}, + skipped, + ) + for name in ALWAYS_REQUIRED: + self.assertEqual(results[name], "success") + self.assertEqual(results[REQUIRED_JOB], "success") + + +class GateScriptTests(unittest.TestCase): + """The gate's own step fails closed on inputs a consistent run never produces.""" + + def run_gate(self, gates: str, needs_results: str) -> tuple[int, str]: + steps = [step for step in load_workflow()["jobs"][GATE_JOB]["steps"] if "run" in step] + self.assertEqual(len(steps), 1) + environment = ( + ("GATES", gates), + ("NEEDS_RESULTS", needs_results), + ("PATH", os.environ.get("PATH", "/usr/bin:/bin")), + ) + return run_step(steps[0].get("shell"), steps[0]["run"], environment) + + def test_consistent_rows_pass(self) -> None: + code, output = self.run_gate("lifecycle-gate true success\nsite false skipped\n", "success skipped") + self.assertEqual(code, 0, output) + + def test_a_selected_job_that_was_skipped_fails(self) -> None: + code, output = self.run_gate("lifecycle-gate true failure\ntest true skipped\n", "failure skipped") + self.assertNotEqual(code, 0) + self.assertIn("::error::test was selected but skipped", output) + + def test_a_deselected_job_that_ran_fails(self) -> None: + code, output = self.run_gate("site false success\n", "success") + self.assertNotEqual(code, 0) + self.assertIn("its row no longer matches its if:", output) + + def test_a_malformed_selection_fails(self) -> None: + code, output = self.run_gate("site skipped\n", "skipped") + self.assertNotEqual(code, 0) + self.assertIn("has no usable selection", output) + + def test_rows_that_do_not_cover_needs_fail(self) -> None: + code, output = self.run_gate("site false skipped\n", "skipped skipped") + self.assertNotEqual(code, 0) + self.assertIn("1 rows for 2 jobs in needs", output) + + def test_a_failure_in_needs_fails_whatever_the_rows_say(self) -> None: + code, output = self.run_gate("site false skipped\n", "failure") + self.assertNotEqual(code, 0) + self.assertIn("a job this gate needs ended with: failure", output) + + +class ExpressionTests(unittest.TestCase): + """The evaluator agrees with GitHub on the operators ci.yml uses.""" + + def evaluate(self, source: str, **context: Any) -> Any: + return Evaluator(context, {"always": lambda: True, "join": join}).evaluate(source) + + def test_string_equality_is_case_insensitive(self) -> None: + self.assertIs(self.evaluate("'TRUE' == 'true'"), True) + + def test_logical_operators_return_operands(self) -> None: + self.assertEqual(self.evaluate("'' || 'fallback'"), "fallback") + self.assertEqual(self.evaluate("'a' && 'b'"), "b") + + def test_missing_outputs_compare_unequal_to_true(self) -> None: + self.assertIs(self.evaluate("needs.classify.outputs.full == 'true'", needs={}), False) + self.assertIs(self.evaluate("needs.classify.outputs.full != 'true'", needs={}), True) + + def test_join_over_a_wildcard(self) -> None: + needs = {"a": {"result": "success"}, "b": {"result": "skipped"}} + self.assertEqual(self.evaluate("join(needs.*.result, ' ')", needs=needs), "success skipped") + + def test_unsupported_syntax_fails_loudly(self) -> None: + with self.assertRaises(ValueError): + self.evaluate("contains(github.ref, 'main')") + with self.assertRaises(ValueError): + self.evaluate("1 < 2") + + +# MARK: - Truth table + + +def truth_table() -> str: + """Markdown table of the required gate's verdict per classify path.""" + workflow = load_workflow() + before = pre_796_workflow(workflow) + lines = [ + "| Classify path | Event | Selected, must succeed | Deselected, `skipped` passes " + "| All selected green | `lifecycle-gate` fails | `preflight` fails " + "| Any one selected job fails or is cancelled | `lifecycle-gate` fails, before this fix |", + "|---|---|---|---|---|---|---|---|---|", + ] + + def verdict(results: dict[str, str]) -> str: + return "green" if results[REQUIRED_JOB] == "success" else "**red**" + + for scenario in SCENARIOS: + green, _ = simulate(workflow, scenario) + ran = [name for name in product_jobs(workflow) if green[name] != "skipped"] + skipped = [name for name in product_jobs(workflow) if green[name] == "skipped"] + injected = [ + simulate(workflow, scenario, {name: outcome})[0][REQUIRED_JOB] + for name, outcome in itertools.product(ran, ("failure", "cancelled")) + ] + red = sum(result != "success" for result in injected) + lines.append( + "| {label} | `{event}` | {ran} | {skipped} | {green} | {lifecycle} | {preflight} " + "| **red** in {red}/{total} | {before} |".format( + label=scenario.label, + event=scenario.event, + ran=", ".join(f"`{name}`" for name in ran), + skipped=", ".join(f"`{name}`" for name in skipped) or "none", + green=verdict(green), + lifecycle=verdict(simulate(workflow, scenario, {"lifecycle-gate": "failure"})[0]), + preflight=verdict(simulate(workflow, scenario, {"preflight": "failure"})[0]), + red=red, + total=len(injected), + before=verdict(simulate(before, scenario, {"lifecycle-gate": "failure"})[0]), + ) + ) + return "\n".join(lines) + + +if __name__ == "__main__": + if sys.argv[1:] == ["--truth-table"]: + print(truth_table()) + raise SystemExit(0) + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 397b0321..891475d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -430,6 +430,8 @@ jobs: run: python3 .github/scripts/validate-workflow-runtime-pins.py - name: Validate release version consistency run: python3 .github/scripts/validate-release-version.py + - name: Test the required CI gate + run: python3 .github/scripts/test-required-ci-gate.py action-consumer: name: Packaged GitHub Action consumer @@ -591,19 +593,70 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 if: ${{ always() }} - needs: [classify, test, fmt, hi-check, validate-action, action-consumer, spec-check, audit, coverage, site, vscode-extension, corvid-pet] + # Every job that can finish before this one belongs in `needs`, preflight and + # the lifecycle gate included. A job missing from here cannot fail the gate + # (#796). .github/scripts/test-required-ci-gate.py fails when one is missing. + needs: [classify, preflight, lifecycle-gate, test, fmt, hi-check, validate-action, action-consumer, spec-check, audit, coverage, site, vscode-extension, corvid-pet] steps: - name: Require every selected gate + # One row per job in `needs`: ` `. is + # that job's own `if:`, less a leading `always() &&`, evaluated again over + # the same classify outputs (a job with no `if:` is always selected). So + # `skipped` passes only where classify deselected the job. A selected job + # that was skipped had a dependency that did not succeed, and fails the + # gate: before #796 a failed lifecycle gate left test, audit, coverage and + # spec-check skipped and this gate green. test-required-ci-gate.py holds + # each row to its job's `if:` and runs this script on every classify path. env: - RESULTS: >- - ${{ join(needs.*.result, ' ') }} + NEEDS_RESULTS: ${{ join(needs.*.result, ' ') }} + GATES: | + classify true ${{ needs.classify.result }} + preflight true ${{ needs.preflight.result }} + lifecycle-gate true ${{ needs.lifecycle-gate.result }} + test ${{ needs.classify.outputs.full == 'true' }} ${{ needs.test.result }} + fmt ${{ needs.classify.outputs.full == 'true' }} ${{ needs.fmt.result }} + hi-check ${{ needs.classify.outputs.full == 'true' }} ${{ needs.hi-check.result }} + validate-action ${{ needs.classify.outputs.archive_only != 'true' && needs.classify.outputs.review_only != 'true' }} ${{ needs.validate-action.result }} + action-consumer ${{ needs.classify.outputs.full == 'true' }} ${{ needs.action-consumer.result }} + spec-check ${{ needs.classify.outputs.archive_only != 'true' && needs.classify.outputs.review_only != 'true' }} ${{ needs.spec-check.result }} + audit ${{ needs.classify.outputs.full == 'true' }} ${{ needs.audit.result }} + coverage ${{ needs.classify.outputs.full == 'true' }} ${{ needs.coverage.result }} + site ${{ needs.classify.outputs.full == 'true' || needs.classify.outputs.site == 'true' }} ${{ needs.site.result }} + vscode-extension ${{ needs.classify.outputs.full == 'true' || needs.classify.outputs.vscode == 'true' }} ${{ needs.vscode-extension.result }} + corvid-pet ${{ github.event_name == 'pull_request' && needs.classify.outputs.review_required == 'true' }} ${{ needs.corvid-pet.result }} run: | - for result in $RESULTS; do + status=0 + rows=0 + while read -r job selected result; do + [[ -n "$job" ]] || continue + rows=$((rows + 1)) + case "$selected:$result" in + true:success|false:skipped) continue ;; + true:skipped) why="was selected but skipped, so a job it needs did not succeed" ;; + true:*) why="was selected and ended with: ${result:-no result}" ;; + false:*) why="was not selected yet ended with: ${result:-no result}; its row no longer matches its if:" ;; + *) why="has no usable selection '${selected}'; its row is malformed" ;; + esac + echo "::error::${job} ${why}" + status=1 + done <<<"$GATES" + # Belt and braces: the rows must cover `needs` exactly, and nothing in + # `needs` may have failed or been cancelled whatever its row says. + read -r -a results <<<"$NEEDS_RESULTS" + if [[ "$rows" -ne "${#results[@]}" ]]; then + echo "::error::this gate has ${rows} rows for ${#results[@]} jobs in needs" + status=1 + fi + for result in "${results[@]}"; do case "$result" in success|skipped) ;; - *) echo "Selected CI gate ended with: $result" >&2; exit 1 ;; + *) echo "::error::a job this gate needs ended with: $result"; status=1 ;; esac done + if [[ "$status" -ne 0 ]]; then + exit 1 + fi + echo "Every selected gate succeeded; every skipped job was deselected by classify." ci-gate: name: Required CI gate @@ -613,8 +666,9 @@ jobs: needs: [classify, implementation-gate] steps: # One aggregate context, suitable for a required status check. Lifecycle - # coherence is proven by `specsync change audit --strict` in spec-check, - # not by inspecting commit topology. + # coherence is proven by `specsync change audit --strict` in the lifecycle + # gate and in spec-check, not by inspecting commit topology, and the + # implementation gate requires both of them. - name: Require the implementation gate env: IMPLEMENTATION_RESULT: ${{ needs.implementation-gate.result }} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/accepted-state.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/accepted-state.json new file mode 100644 index 00000000..e33275c8 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/accepted-state.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "workflow_version": 2, + "workflow_origin_version": 2, + "id": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "slug": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "title": "Required CI gate fails when the lifecycle gate fails", + "description": "Required CI gate fails when the lifecycle gate fails", + "kind": "bug_fix", + "state": "accepted", + "canonical_applied": true, + "base_commit": "cddc39e478dcc1f111940a3cfb02134bba9804cc", + "created_at": 1790434419, + "updated_at": 1790441718, + "affected_specs": [ + "github" + ], + "affected_paths": [ + ".github/workflows/ci.yml", + ".github/scripts/test-required-ci-gate.py", + "fledge.toml", + "docs/HLD.md" + ], + "no_spec_change": false, + "no_spec_change_rationale": null, + "acceptance_criteria": [ + "Required CI gate fails whenever Lifecycle preflight or Lifecycle gate fails or is cancelled. implementation-gate (SpecSync implementation ready) needs classify, preflight, lifecycle-gate and every other job that can finish before it, and accepts skipped for a job only when that job's own if: condition, evaluated again over the classify outputs, deselected it; a selected job that was skipped because something it needs did not succeed fails the gate. Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and review-only pull requests, pushes to main and workflow_dispatch runs stay green when every selected job succeeds. .github/scripts/test-required-ci-gate.py runs in the validate-action CI job and in the Fledge verify lane; it fails if preflight or lifecycle-gate is missing from implementation-gate.needs, if a job that gates on lifecycle-gate or can otherwise finish before the gate is missing, or if a gate row no longer matches its job's if:, and it simulates every classify path to show the required gate red when any one selected job fails or is cancelled and green otherwise, and reproduces #796 against the pre-fix gate." + ], + "selected_artifacts": [ + "context", + "testing", + "tasks", + "research", + "design", + "plan" + ], + "dependencies": [], + "answers": { + "architecture_risk": "yes", + "public_contract": "no" + } +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/approvals.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/approvals.json new file mode 100644 index 00000000..bd4d9b1b --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/approvals.json @@ -0,0 +1,49 @@ +{ + "approvals": [ + { + "gate": "definition", + "actor": "user:0xLeif", + "timestamp": 1790441012, + "digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "note": "Approved by Leif in the orc session, 2026-09-26: spec-sync #797", + "approved_scope": { + "schema_version": 1, + "change_id": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "title": "Required CI gate fails when the lifecycle gate fails", + "description": "Required CI gate fails when the lifecycle gate fails", + "kind": "bug_fix", + "affected_specs": [ + "github" + ], + "affected_paths": [ + ".github/scripts/test-required-ci-gate.py", + ".github/workflows/ci.yml", + "docs/HLD.md", + "fledge.toml" + ], + "no_spec_change": false, + "no_spec_change_rationale": null, + "acceptance_criteria": [ + "Required CI gate fails whenever Lifecycle preflight or Lifecycle gate fails or is cancelled. implementation-gate (SpecSync implementation ready) needs classify, preflight, lifecycle-gate and every other job that can finish before it, and accepts skipped for a job only when that job's own if: condition, evaluated again over the classify outputs, deselected it; a selected job that was skipped because something it needs did not succeed fails the gate. Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and review-only pull requests, pushes to main and workflow_dispatch runs stay green when every selected job succeeds. .github/scripts/test-required-ci-gate.py runs in the validate-action CI job and in the Fledge verify lane; it fails if preflight or lifecycle-gate is missing from implementation-gate.needs, if a job that gates on lifecycle-gate or can otherwise finish before the gate is missing, or if a gate row no longer matches its job's if:, and it simulates every classify path to show the required gate red when any one selected job fails or is cancelled and green otherwise, and reproduces #796 against the pre-fix gate." + ], + "dependencies": [], + "supersedes": [], + "answers": { + "architecture_risk": "yes", + "public_contract": "no" + } + }, + "approved_delta_digests": { + "github": "e1e69409edadb41bd7af8b1992f79007fa2138651348cb2d82b1c652b1490ec7" + } + }, + { + "gate": "finalization", + "actor": "specsync:finalization", + "timestamp": 1790441717, + "digest": "846c770fee65169aee2f680122bf3db12bb73e23401ef8c3614dfeb633df9e7e", + "note": "Same-PR finalization closing digest" + } + ], + "reopenings": [] +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/change.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/change.md new file mode 100644 index 00000000..866de91f --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/change.md @@ -0,0 +1,24 @@ +--- +id: required-ci-gate-fails-when-the-lifecycle-gate-fails +state: archived +type: bug_fix +base_commit: cddc39e478dcc1f111940a3cfb02134bba9804cc +--- + +# Required CI gate fails when the lifecycle gate fails + +## Intent + +Required CI gate fails when the lifecycle gate fails + +## Affected Canonical Specs + +- `github` + +## Acceptance Criteria + +- Required CI gate fails whenever Lifecycle preflight or Lifecycle gate fails or is cancelled. implementation-gate (SpecSync implementation ready) needs classify, preflight, lifecycle-gate and every other job that can finish before it, and accepts skipped for a job only when that job's own if: condition, evaluated again over the classify outputs, deselected it; a selected job that was skipped because something it needs did not succeed fails the gate. Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and review-only pull requests, pushes to main and workflow_dispatch runs stay green when every selected job succeeds. .github/scripts/test-required-ci-gate.py runs in the validate-action CI job and in the Fledge verify lane; it fails if preflight or lifecycle-gate is missing from implementation-gate.needs, if a job that gates on lifecycle-gate or can otherwise finish before the gate is missing, or if a gate row no longer matches its job's if:, and it simulates every classify path to show the required gate red when any one selected job fails or is cancelled and green otherwise, and reproduces #796 against the pre-fix gate. + +## No-spec Rationale + +Not applicable diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/context.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/context.md new file mode 100644 index 00000000..ab655d1a --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/context.md @@ -0,0 +1,44 @@ +--- +change: required-ci-gate-fails-when-the-lifecycle-gate-fails +artifact: context +--- + +# Context + +`Required CI gate` is the only required check on `main`, and it passes when +`implementation-gate` ("SpecSync implementation ready") passes. On #795 at `bb1d80f2`, +`Lifecycle gate` and `trust` failed, `test`, `audit`, `coverage` and `spec-check` never ran, +and both gates were green (#796). orc found it while shipping #795. + +The cause is in `.github/workflows/ci.yml`: + +- `test`, `audit`, `coverage` and `spec-check` need `lifecycle-gate`. When it fails, GitHub + reports them as `skipped`, not failed. +- `implementation-gate` needed neither `preflight` nor `lifecycle-gate`, and it accepted + `success` or `skipped` from every job it did need. +- `ci-gate` only required `implementation-gate == success`. + +So a failed lifecycle gate became four skipped jobs, and skipped read as green. + +What a session picking this up needs to know: + +- `skipped` alone cannot say why a job did not run. GitHub reports the same result for a job + classify deselected and for a job whose dependency failed. The gate can only tell them apart by + asking the question the job asked, so each gate row carries the job's own `if:` condition, + evaluated again over the same classify outputs. +- Classify outputs are fixed once `classify` finishes, and the gate and the job use the same + expression evaluator. The only way the two answers can differ is if the row and the job's `if:` + are different text. The test compares them as text. +- `preflight` and `lifecycle-gate` have no `if:`, so they are selected on every path, archive-only + and review-only included. Recent archive-only and review-only pull requests (#785, #790, #791) + and the #795 product and archive tips all show `Lifecycle gate: success`, so requiring success + blocks nothing that merges today. +- Job-level `success()` looks at every transitive dependency, not only the direct ones. That is + why `attest` has been skipped on every push to `main` (it needs `ci-gate`, and `corvid-pet`, + two levels up, is skipped on pushes). The simulation models this. Fixing `attest` is out of + scope here and is reported separately. +- `act` is not installed and CI cannot be run locally, so the proof is a simulation of the job + graph that executes the gate's own bash under GitHub's default invocation, plus the live run on + this pull request. +- Ruled out: requiring `success` from every job in `needs`. Archive-only and review-only pull + requests legitimately skip most jobs, and that would block them. diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/deltas/github.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/deltas/github.md new file mode 100644 index 00000000..f463ec0b --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/deltas/github.md @@ -0,0 +1,56 @@ +## MODIFIED + +### SPEC SECTION Invariants + +Every path that can be merged can reach the required CI gate; a path the CI +workflow cannot trigger can never report the gate and blocks its pull request. + +The required CI gate fails whenever a job it depends on did not succeed, unless classify +deselected that job. A skipped job passes only when its own `if:` condition, evaluated over the +classify outputs, left it unselected. A selected job that was skipped was skipped because a job it +needs did not succeed, and it fails the gate: `skipped` is never read as green on its own. The +lifecycle preflight and the lifecycle gate are selected on every path, so either one failing turns +the required gate red. + +Release qualification verifies exactly the tag protections this repository actually has, and names +every protection it does not verify on every run, green runs included. A gate that demands an +unprovisioned policy fails on every candidate and therefore verifies nothing — it is not a safe +default, because the protections that DO exist are never reached. Dropping a check from the gate is +permitted; dropping it silently is not. The tag protections that remain admit no bypass actor and +no broadening — where that can be observed. GitHub returns `bypass_actors` only to a caller with +admin access to repository settings, and the workflow token is not one, so the field is ABSENT +from every payload CI fetches. Absence means UNOBSERVED, never "no bypass actors": it is checked +when visible, refused when it grants anyone, and named in the unenforced disclosure when it cannot +be read. Requiring it made the gate impossible to satisfy from CI, which is how a lane stayed red +on every candidate while appearing to enforce something. + +Release authority is stated wherever it is exercised. The final tag is created by the release +workflow's own token under a permission scoped to the single job that writes it, so the authority +to run the release lane is the authority to create a release tag; that equivalence is announced by +every run and recorded at the job itself, never left to be inferred from a green result. A named +deployment environment that does not exist is not a gate — GitHub materializes it unprotected on +first use — so the workflow names no environment rather than publish a gate that gates nothing. + +## ADDED + +### REQUIREMENT REQ-github-021 + +The required CI gate SHALL fail whenever the lifecycle preflight, the lifecycle gate, or any job +classify selected for the run fails, is cancelled, or is skipped because a job it needs did not +succeed. + +Acceptance Criteria + +- `implementation-gate` (SpecSync implementation ready) needs `classify`, `preflight`, + `lifecycle-gate` and every other job that can finish before it, and `ci-gate` (Required CI gate) + passes only when `implementation-gate` succeeds. +- A skipped job passes only when that job's own `if:` condition, evaluated again over the classify + outputs, deselected it. +- Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and + review-only pull requests, pushes to `main` and `workflow_dispatch` runs stay green when every + selected job succeeds. +- `.github/scripts/test-required-ci-gate.py` fails when `preflight` or `lifecycle-gate` is missing + from the gate's `needs`, when a job that gates on `lifecycle-gate` or can otherwise finish before + the gate is missing, or when a gate row no longer matches its job's `if:`. +- The same test runs the gate's own script over every classify path and requires the required gate + to be red when any one selected job fails or is cancelled. diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/design.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/design.md new file mode 100644 index 00000000..ca00b786 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/design.md @@ -0,0 +1,63 @@ +--- +change: required-ci-gate-fails-when-the-lifecycle-gate-fails +artifact: design +--- + +# Design + +## The gate asks each job's own question + +`implementation-gate` needs every job that can finish before it: `classify`, `preflight`, +`lifecycle-gate`, the product jobs and `corvid-pet`. Its single step reads a `GATES` table with +one row per job: + +```text + +test ${{ needs.classify.outputs.full == 'true' }} ${{ needs.test.result }} +``` + +`` is the job's own `if:` with any leading `always() &&` removed, or `true` for a job +with no `if:`. GitHub renders it as `true` or `false`. The step then applies: + +| selected | result | verdict | +|---|---|---| +| true | success | pass | +| false | skipped | pass (classify deselected it) | +| true | skipped | **fail**: a dependency did not succeed | +| true | failure, cancelled | **fail** | +| false | anything but skipped | **fail**: the row no longer matches the job's `if:` | +| anything else | any | **fail**: malformed row | + +It also keeps the previous check, so any `failure` or `cancelled` in `join(needs.*.result)` fails, +and it fails when the number of rows differs from the number of jobs in `needs`. `ci-gate` is +unchanged: it passes only when `implementation-gate` succeeds. + +## Why the row repeats the condition rather than the rule living in a script + +The gate evaluates the same expression, over the same fixed classify outputs, with the same +evaluator as the job's `if:`. It cannot disagree with the job unless the text differs, and the +test compares the text. A script outside the workflow would need a checkout in the gate and a +second copy of the lane rules. + +## The guard + +`.github/scripts/test-required-ci-gate.py` parses `ci.yml` with Psych, like the other workflow +validators, and checks: + +1. Every job that does not itself depend on `implementation-gate` is in its `needs`. Any new job + that gates on `lifecycle-gate`, or is otherwise selected before the gate, fails the test until + it is added. `classify`, `preflight` and `lifecycle-gate` are named explicitly. +2. Every job in `needs` has exactly one row, each row reads its own job's result, and its + selection text equals the job's `if:`. A job whose `if:` uses a status function other than a + leading `always()` fails, because the gate cannot mirror it. +3. A simulation of the job graph, with GitHub's implicit and transitive `success()`, runs the + gate's own step and `ci-gate`'s step under `bash -e`, as the runner does for a step that names + no shell. It covers every classify lane, every combination of the classify flags the + conditions read, and every event. The required gate must be green when every selected job + succeeds, and red when any one of them fails or is cancelled. +4. The pre-#796 gate definition, kept as a fixture, reproduces the bug in the same simulation, so + the harness can see what it guards against. + +It runs in the `validate-action` CI job, which runs whenever `ci.yml` changes because a workflow +change selects the full lane, and as the Fledge task `ci-gate-test` in the `verify`, `ci` and +`repo` lanes. diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/finalization.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/finalization.json new file mode 100644 index 00000000..c330b27d --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/finalization.json @@ -0,0 +1,12 @@ +{ + "schema_version": 2, + "change_id": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "implementation_commit": "3ffef4758ab6654507183cab5942b412dc6064fa", + "implementation_tree": "15dc0b08430b89b2cbb6d866eb9f17962f41b2a9", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "closing_digest": "846c770fee65169aee2f680122bf3db12bb73e23401ef8c3614dfeb633df9e7e", + "review_digest": "90e1655610732ad67f229d54288e3edd2b73e904539f720b46d80d373fd2bc0d", + "finalization_digest": "9b7de0951756847938dbc2ce0a087110054ea049212094654db63f4768aae1a1", + "timestamp": 1790441718 +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/lesson-bundle.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/lesson-bundle.md new file mode 100644 index 00000000..cffb87e7 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/lesson-bundle.md @@ -0,0 +1,203 @@ +# Lesson bundle — required-ci-gate-fails-when-the-lifecycle-gate-fails + +Material for folding this change's lessons into the affected specs' `context.md`. +Synthesise from what actually happened below; do not restate the change description. + +## What this change was + +- **Title**: Required CI gate fails when the lifecycle gate fails +- **Kind**: BugFix +- **Specs**: github +- **Paths**: .github/workflows/ci.yml, .github/scripts/test-required-ci-gate.py, fledge.toml, docs/HLD.md +- **Acceptance**: Required CI gate fails whenever Lifecycle preflight or Lifecycle gate fails or is cancelled. implementation-gate (SpecSync implementation ready) needs classify, preflight, lifecycle-gate and every other job that can finish before it, and accepts skipped for a job only when that job's own if: condition, evaluated again over the classify outputs, deselected it; a selected job that was skipped because something it needs did not succeed fails the gate. Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and review-only pull requests, pushes to main and workflow_dispatch runs stay green when every selected job succeeds. .github/scripts/test-required-ci-gate.py runs in the validate-action CI job and in the Fledge verify lane; it fails if preflight or lifecycle-gate is missing from implementation-gate.needs, if a job that gates on lifecycle-gate or can otherwise finish before the gate is missing, or if a gate row no longer matches its job's if:, and it simulates every classify path to show the required gate red when any one selected job fails or is cancelled and green otherwise, and reproduces #796 against the pre-fix gate. + +## Evidence + +- Verification commit: `3ffef4758ab6654507183cab5942b412dc6064fa` +- Base commit: `cddc39e478dcc1f111940a3cfb02134bba9804cc` +- Verified by: `specsync check --spec github` + +## From the change's context.md + +# Context + +`Required CI gate` is the only required check on `main`, and it passes when +`implementation-gate` ("SpecSync implementation ready") passes. On #795 at `bb1d80f2`, +`Lifecycle gate` and `trust` failed, `test`, `audit`, `coverage` and `spec-check` never ran, +and both gates were green (#796). orc found it while shipping #795. + +The cause is in `.github/workflows/ci.yml`: + +- `test`, `audit`, `coverage` and `spec-check` need `lifecycle-gate`. When it fails, GitHub + reports them as `skipped`, not failed. +- `implementation-gate` needed neither `preflight` nor `lifecycle-gate`, and it accepted + `success` or `skipped` from every job it did need. +- `ci-gate` only required `implementation-gate == success`. + +So a failed lifecycle gate became four skipped jobs, and skipped read as green. + +What a session picking this up needs to know: + +- `skipped` alone cannot say why a job did not run. GitHub reports the same result for a job + classify deselected and for a job whose dependency failed. The gate can only tell them apart by + asking the question the job asked, so each gate row carries the job's own `if:` condition, + evaluated again over the same classify outputs. +- Classify outputs are fixed once `classify` finishes, and the gate and the job use the same + expression evaluator. The only way the two answers can differ is if the row and the job's `if:` + are different text. The test compares them as text. +- `preflight` and `lifecycle-gate` have no `if:`, so they are selected on every path, archive-only + and review-only included. Recent archive-only and review-only pull requests (#785, #790, #791) + and the #795 product and archive tips all show `Lifecycle gate: success`, so requiring success + blocks nothing that merges today. +- Job-level `success()` looks at every transitive dependency, not only the direct ones. That is + why `attest` has been skipped on every push to `main` (it needs `ci-gate`, and `corvid-pet`, + two levels up, is skipped on pushes). The simulation models this. Fixing `attest` is out of + scope here and is reported separately. +- `act` is not installed and CI cannot be run locally, so the proof is a simulation of the job + graph that executes the gate's own bash under GitHub's default invocation, plus the live run on + this pull request. +- Ruled out: requiring `success` from every job in `needs`. Archive-only and review-only pull + requests legitimately skip most jobs, and that would block them. + +## From the change's design.md + +# Design + +## The gate asks each job's own question + +`implementation-gate` needs every job that can finish before it: `classify`, `preflight`, +`lifecycle-gate`, the product jobs and `corvid-pet`. Its single step reads a `GATES` table with +one row per job: + +```text + +test ${{ needs.classify.outputs.full == 'true' }} ${{ needs.test.result }} +``` + +`` is the job's own `if:` with any leading `always() &&` removed, or `true` for a job +with no `if:`. GitHub renders it as `true` or `false`. The step then applies: + +| selected | result | verdict | +|---|---|---| +| true | success | pass | +| false | skipped | pass (classify deselected it) | +| true | skipped | **fail**: a dependency did not succeed | +| true | failure, cancelled | **fail** | +| false | anything but skipped | **fail**: the row no longer matches the job's `if:` | +| anything else | any | **fail**: malformed row | + +It also keeps the previous check, so any `failure` or `cancelled` in `join(needs.*.result)` fails, +and it fails when the number of rows differs from the number of jobs in `needs`. `ci-gate` is +unchanged: it passes only when `implementation-gate` succeeds. + +## Why the row repeats the condition rather than the rule living in a script + +The gate evaluates the same expression, over the same fixed classify outputs, with the same +evaluator as the job's `if:`. It cannot disagree with the job unless the text differs, and the +test compares the text. A script outside the workflow would need a checkout in the gate and a +second copy of the lane rules. + +## The guard + +`.github/scripts/test-required-ci-gate.py` parses `ci.yml` with Psych, like the other workflow +validators, and checks: + +1. Every job that does not itself depend on `implementation-gate` is in its `needs`. Any new job + that gates on `lifecycle-gate`, or is otherwise selected before the gate, fails the test until + it is added. `classify`, `preflight` and `lifecycle-gate` are named explicitly. +2. Every job in `needs` has exactly one row, each row reads its own job's result, and its + selection text equals the job's `if:`. A job whose `if:` uses a status function other than a + leading `always()` fails, because the gate cannot mirror it. +3. A simulation of the job graph, with GitHub's implicit and transitive `success()`, runs the + gate's own step and `ci-gate`'s step under `bash -e`, as the runner does for a step that names + no shell. It covers every classify lane, every combination of the classify flags the + conditions read, and every event. The required gate must be green when every selected job + succeeds, and red when any one of them fails or is cancelled. +4. The pre-#796 gate definition, kept as a fixture, reproduces the bug in the same simulation, so + the harness can see what it guards against. + +It runs in the `validate-action` CI job, which runs whenever `ci.yml` changes because a workflow +change selects the full lane, and as the Fledge task `ci-gate-test` in the `verify`, `ci` and +`repo` lanes. + +## From the change's testing.md + +# Testing + +## Requirement evidence + +| Requirement | Evidence | +|---|---| +| REQ-github-021 | `.github/scripts/test-required-ci-gate.py`: `test_workflow_holds_to_the_gate_contract`, `test_gate_needs_preflight_and_the_lifecycle_gate`, `test_every_job_gated_on_the_lifecycle_gate_is_required`, `test_issue_796_lifecycle_gate_failure_turns_the_required_gate_red`, `test_any_selected_job_failing_or_cancelled_turns_the_required_gate_red`, `test_every_path_is_green_when_every_selected_job_succeeds`, `test_every_classify_output_combination_is_green_when_everything_succeeds`, `test_deselected_jobs_are_skipped_and_do_not_block`, the seven `GuardMutationTests` and the six `GateScriptTests` | + +## What the tests do + +- **Contract.** `ci.yml` is parsed with Psych. Every job that does not depend on + `implementation-gate` must be in its `needs`, and `classify`, `preflight` and `lifecycle-gate` + are named. Every job in `needs` has one `GATES` row that reads its own result and whose + selection text equals the job's `if:`. +- **Guard mutations.** Each of these fails the contract: dropping `lifecycle-gate` or + `preflight` from `needs`, adding a job with `needs: [classify, lifecycle-gate]` or a new + site-selected job without adding it, adding a job to `needs` without a row, changing `test`'s + `if:` without its row, and an `if:` that calls `failure()`. +- **Simulation.** The job graph is evaluated with GitHub's implicit, transitive `success()`, and + the steps of `implementation-gate` and `ci-gate` run under `bash -e`, which is how the runner + invokes a step that names no shell (the job log prints `shell: /usr/bin/bash -e {0}`). On 15 + named lanes (full, full awaiting review, site-only, + VS Code-only, site and VS Code, specs/lifecycle-only with and without a review due, archive-only, + legacy archive-only, review-only, four push-to-`main` lanes and `workflow_dispatch`), the + required gate is green when every selected job succeeds. Forcing any one selected job to + `failure` or `cancelled` turns it red in all 252 cases. All 192 combinations of the six classify + flags the conditions read, under all three events, are green when everything succeeds. +- **Reproduction.** With the pre-#796 gate definition swapped in, a failed lifecycle gate on a + full pull request leaves `test`, `audit`, `coverage` and `spec-check` skipped, and both gates + green, which is what #795 showed at `bb1d80f2`. +- **Gate script.** Run directly, the step fails on a selected job that was skipped, a deselected + job that ran, a malformed selection, rows that do not cover `needs`, and any `failure` in `needs`. + +## Discrimination + +The same tests run against `ci.yml` from `origin/main` (`cddc39e4`) fail 65 cases in five tests: +the contract (`preflight` and `lifecycle-gate` missing, no `GATES` table), both named `needs` +checks, the CI wiring check, the #796 reproduction, and 60 injection cases (`preflight` or +`lifecycle-gate` failing or cancelled, on each of the 15 lanes). On this branch all pass. + +## Truth table + +Printed by `python3 .github/scripts/test-required-ci-gate.py --truth-table` and recorded in the +pull request. + +## Suite + +`fledge lanes run verify` passes: fmt, `cargo clippy -- -D warnings` and `cargo check`; the full +`cargo test` with 2504 unit and 437 integration tests and 0 failures; the release build; +`specsync check --strict --require-coverage 100 --force` with 62/62 specs and 100% file coverage; +the 52 release-candidate tests; and the 29 new `ci-gate-test` tests. +`python3 -S .github/scripts/validate-workflow-runtime-pins.py`, +`python3 -S .github/scripts/validate-release-version.py` and +`.github/scripts/test-classify-ci-paths.sh` pass on the changed workflow. +`fledge lanes run pre-push` and `fledge trust verify` pass. + +## Live check + +The pull request (#797) was opened before approval, so `Lifecycle gate` failed on the unapproved +draft at `17e63f5d` (run 36252382792): "meaningful changed paths are not covered by an active +change". `test`, `audit`, `coverage` and `spec-check` were skipped, as on #795. This time +`SpecSync implementation ready` failed, with one annotation per cause: + +```text +lifecycle-gate was selected and ended with: failure +test was selected but skipped, so a job it needs did not succeed +spec-check was selected but skipped, so a job it needs did not succeed +audit was selected but skipped, so a job it needs did not succeed +coverage was selected but skipped, so a job it needs did not succeed +a job this gate needs ended with: failure +``` + +`Required CI gate` failed with it. The new test ran in `validate-action` (29 tests, OK). The same +job log shows the step shell as `bash -e {0}`; the simulation first assumed +`bash --noprofile --norc -eo pipefail`, which is what `shell: bash` gets, and now uses `bash -e`. + +## Where these lessons go + +- `specs/github/context.md` diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/plan.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/plan.md new file mode 100644 index 00000000..a2ec5d8d --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/plan.md @@ -0,0 +1,25 @@ +--- +change: required-ci-gate-fails-when-the-lifecycle-gate-fails +artifact: plan +--- + +# Plan + +Four files change, plus the `github` spec and its companions. + +1. **`.github/workflows/ci.yml`.** Add `preflight` and `lifecycle-gate` to + `implementation-gate.needs`. Replace its step with the selection-aware `GATES` table and check. + Run the new test in `validate-action`. Correct the `ci-gate` comment, which said lifecycle + coherence is proven in spec-check alone. +2. **`.github/scripts/test-required-ci-gate.py`.** The structural guard, the job-graph + simulation, the pre-#796 reproduction, guard mutation tests and a `--truth-table` mode for the + pull request. +3. **`fledge.toml`.** Add the `ci-gate-test` task to the `verify`, `ci` and `repo` lanes. +4. **`docs/HLD.md`.** State what `SpecSync implementation ready` requires, next to the existing + note that branch protection requires `Required CI gate`. +5. **`github` spec.** Add the gate invariant and REQ-github-021 through the delta. Record the + test in `testing.md`, the decision in `context.md` and the task in `tasks.md`. +6. Show that the test fails on the pre-fix `ci.yml`, then run `fledge lanes run verify`, + `fledge lanes run pre-push` and `fledge trust verify`. +7. Open the pull request before approval. The unapproved draft fails `Lifecycle gate`, and with + this change `Required CI gate` should go red on it, which is the live proof. diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/research.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/research.md new file mode 100644 index 00000000..bbcdbd37 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/research.md @@ -0,0 +1,57 @@ +--- +change: required-ci-gate-fails-when-the-lifecycle-gate-fails +artifact: research +--- + +# Research + +## What each job runs on + +Read from `.github/workflows/ci.yml` on `cddc39e4`: + +| Job | Needs | `if:` | +|---|---|---| +| `classify` | none | none | +| `preflight` | none | none | +| `lifecycle-gate` | classify, preflight | none | +| `test`, `audit`, `coverage` | classify, lifecycle-gate | `full` | +| `spec-check` | classify, lifecycle-gate | not `archive_only`, not `review_only` | +| `fmt`, `hi-check`, `action-consumer` | classify | `full` | +| `validate-action` | classify | not `archive_only`, not `review_only` | +| `site` | classify | `full` or `site` | +| `vscode-extension` | classify | `full` or `vscode` | +| `corvid-pet` | 13 jobs | `always()`, pull request, `review_required` | +| `implementation-gate` | 12 jobs (no preflight, no lifecycle-gate) | `always()` | +| `ci-gate` | classify, implementation-gate | `always()` | +| `attest` | ci-gate | ci-gate success, push to `main` | + +A job whose `if:` has no status function gets an implicit `success()`, so it is skipped +whenever a dependency did not succeed. + +## Observed check runs + +| Commit | Lifecycle gate | test/audit/coverage/spec-check | Implementation ready | Required CI gate | +|---|---|---|---|---| +| #795 `bb1d80f2` (unapproved draft) | failure | skipped | success | success | +| #795 `f50bfdd5` (product tip) | success | success | success | success | +| #795 `96f948d9` (archive tip) | success | success | success | success | +| #790 head (specs/lifecycle-only) | success | test/audit/coverage skipped, spec-check success | success | success | +| #791 head (archive tip) | success | success | success | success | + +The first row is #796. The others show that `preflight` and `lifecycle-gate` succeed on every +lane a lifecycle pull request passes through, so requiring them costs nothing. + +## Classify lanes + +From `.github/scripts/classify-ci-paths.sh` and `select-ci-lane.sh`: `full` for product paths, +workflows, scripts and anything unrecognized (so `docs/**` and `*.md` run the full lane); `site` +or `vscode` alone for those trees; nothing selected for `specs/**` and `.specsync/changes/**`; +`archive_only` for a proven one-change workflow-v2 archive move; `legacy_archive_only` together +with `full` for workflow-v1; and `review_only` for a review-only tip. Pushes classify from a +name-only diff, so they are never archive-only or review-only. `review_required` is computed on +every event, but `corvid-pet` also requires `pull_request`. + +## `attest` never runs on `main` + +The last 15 pushes to `main` all show `Record attestation: skipped`, including runs where +`Required CI gate` succeeded. This is the transitive `success()` above and a separate defect. diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/review-attempts.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/review-attempts.json new file mode 100644 index 00000000..2c263c5b --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/review-attempts.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "reviews": [ + { + "schema_version": 2, + "change_id": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "reviewer": "user:0xLeif", + "provenance": { + "schema_version": 1, + "provider": "github_actions_check", + "required_check": "SpecSync scoped review" + }, + "verdict": "pass", + "implementation_commit": "3ffef4758ab6654507183cab5942b412dc6064fa", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "execution_digest": "751b6e677bfde7d7d922ca616de219af0101f79d5f5506beb2e0c5dbfb171ea6", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "timestamp": 1790441666 + } + ] +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/review.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/review.json new file mode 100644 index 00000000..83a4023a --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/review.json @@ -0,0 +1,16 @@ +{ + "schema_version": 2, + "change_id": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "reviewer": "user:0xLeif", + "provenance": { + "schema_version": 1, + "provider": "github_actions_check", + "required_check": "SpecSync scoped review" + }, + "verdict": "pass", + "implementation_commit": "3ffef4758ab6654507183cab5942b412dc6064fa", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "execution_digest": "751b6e677bfde7d7d922ca616de219af0101f79d5f5506beb2e0c5dbfb171ea6", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "timestamp": 1790441666 +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/state.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/state.json new file mode 100644 index 00000000..e74ee086 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/state.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "workflow_version": 2, + "workflow_origin_version": 2, + "id": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "slug": "required-ci-gate-fails-when-the-lifecycle-gate-fails", + "title": "Required CI gate fails when the lifecycle gate fails", + "description": "Required CI gate fails when the lifecycle gate fails", + "kind": "bug_fix", + "state": "archived", + "canonical_applied": true, + "base_commit": "cddc39e478dcc1f111940a3cfb02134bba9804cc", + "created_at": 1790434419, + "updated_at": 1790442196, + "affected_specs": [ + "github" + ], + "affected_paths": [ + ".github/workflows/ci.yml", + ".github/scripts/test-required-ci-gate.py", + "fledge.toml", + "docs/HLD.md" + ], + "no_spec_change": false, + "no_spec_change_rationale": null, + "acceptance_criteria": [ + "Required CI gate fails whenever Lifecycle preflight or Lifecycle gate fails or is cancelled. implementation-gate (SpecSync implementation ready) needs classify, preflight, lifecycle-gate and every other job that can finish before it, and accepts skipped for a job only when that job's own if: condition, evaluated again over the classify outputs, deselected it; a selected job that was skipped because something it needs did not succeed fails the gate. Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and review-only pull requests, pushes to main and workflow_dispatch runs stay green when every selected job succeeds. .github/scripts/test-required-ci-gate.py runs in the validate-action CI job and in the Fledge verify lane; it fails if preflight or lifecycle-gate is missing from implementation-gate.needs, if a job that gates on lifecycle-gate or can otherwise finish before the gate is missing, or if a gate row no longer matches its job's if:, and it simulates every classify path to show the required gate red when any one selected job fails or is cancelled and green otherwise, and reproduces #796 against the pre-fix gate." + ], + "selected_artifacts": [ + "context", + "testing", + "tasks", + "research", + "design", + "plan" + ], + "dependencies": [], + "answers": { + "architecture_risk": "yes", + "public_contract": "no" + } +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/tasks.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/tasks.md new file mode 100644 index 00000000..36202bb3 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/tasks.md @@ -0,0 +1,17 @@ +--- +change: required-ci-gate-fails-when-the-lifecycle-gate-fails +artifact: tasks +--- + +# Tasks + +- [x] Add `preflight` and `lifecycle-gate` to `implementation-gate.needs`. +- [x] Accept `skipped` only where the job's own `if:`, evaluated over classify outputs, deselected it; fail a selected job that was skipped. +- [x] Keep the old failure/cancelled check and fail when rows and `needs` disagree in number. +- [x] Guard test: missing `preflight`/`lifecycle-gate`, a job missing from `needs`, a row that drifts from its job's `if:`. +- [x] Simulate every classify lane, flag combination and event against the gate's own script. +- [x] Reproduce #796 against the pre-fix gate, and show the test fails on the pre-fix `ci.yml`. +- [x] Wire the test into `validate-action` and the Fledge `verify`, `ci` and `repo` lanes. +- [x] `docs/HLD.md`, the `github` delta, and the `github` companion notes. +- [x] `fledge lanes run verify`, `fledge lanes run pre-push` and `fledge trust verify`. +- [x] Confirm on the pull request that `Required CI gate` is red while the draft is unapproved (#797, run 36252382792). diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/testing.md b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/testing.md new file mode 100644 index 00000000..58bb9902 --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/testing.md @@ -0,0 +1,80 @@ +--- +change: required-ci-gate-fails-when-the-lifecycle-gate-fails +artifact: testing +--- + +# Testing + +## Requirement evidence + +| Requirement | Evidence | +|---|---| +| REQ-github-021 | `.github/scripts/test-required-ci-gate.py`: `test_workflow_holds_to_the_gate_contract`, `test_gate_needs_preflight_and_the_lifecycle_gate`, `test_every_job_gated_on_the_lifecycle_gate_is_required`, `test_issue_796_lifecycle_gate_failure_turns_the_required_gate_red`, `test_any_selected_job_failing_or_cancelled_turns_the_required_gate_red`, `test_every_path_is_green_when_every_selected_job_succeeds`, `test_every_classify_output_combination_is_green_when_everything_succeeds`, `test_deselected_jobs_are_skipped_and_do_not_block`, the seven `GuardMutationTests` and the six `GateScriptTests` | + +## What the tests do + +- **Contract.** `ci.yml` is parsed with Psych. Every job that does not depend on + `implementation-gate` must be in its `needs`, and `classify`, `preflight` and `lifecycle-gate` + are named. Every job in `needs` has one `GATES` row that reads its own result and whose + selection text equals the job's `if:`. +- **Guard mutations.** Each of these fails the contract: dropping `lifecycle-gate` or + `preflight` from `needs`, adding a job with `needs: [classify, lifecycle-gate]` or a new + site-selected job without adding it, adding a job to `needs` without a row, changing `test`'s + `if:` without its row, and an `if:` that calls `failure()`. +- **Simulation.** The job graph is evaluated with GitHub's implicit, transitive `success()`, and + the steps of `implementation-gate` and `ci-gate` run under `bash -e`, which is how the runner + invokes a step that names no shell (the job log prints `shell: /usr/bin/bash -e {0}`). On 15 + named lanes (full, full awaiting review, site-only, + VS Code-only, site and VS Code, specs/lifecycle-only with and without a review due, archive-only, + legacy archive-only, review-only, four push-to-`main` lanes and `workflow_dispatch`), the + required gate is green when every selected job succeeds. Forcing any one selected job to + `failure` or `cancelled` turns it red in all 252 cases. All 192 combinations of the six classify + flags the conditions read, under all three events, are green when everything succeeds. +- **Reproduction.** With the pre-#796 gate definition swapped in, a failed lifecycle gate on a + full pull request leaves `test`, `audit`, `coverage` and `spec-check` skipped, and both gates + green, which is what #795 showed at `bb1d80f2`. +- **Gate script.** Run directly, the step fails on a selected job that was skipped, a deselected + job that ran, a malformed selection, rows that do not cover `needs`, and any `failure` in `needs`. + +## Discrimination + +The same tests run against `ci.yml` from `origin/main` (`cddc39e4`) fail 65 cases in five tests: +the contract (`preflight` and `lifecycle-gate` missing, no `GATES` table), both named `needs` +checks, the CI wiring check, the #796 reproduction, and 60 injection cases (`preflight` or +`lifecycle-gate` failing or cancelled, on each of the 15 lanes). On this branch all pass. + +## Truth table + +Printed by `python3 .github/scripts/test-required-ci-gate.py --truth-table` and recorded in the +pull request. + +## Suite + +`fledge lanes run verify` passes: fmt, `cargo clippy -- -D warnings` and `cargo check`; the full +`cargo test` with 2504 unit and 437 integration tests and 0 failures; the release build; +`specsync check --strict --require-coverage 100 --force` with 62/62 specs and 100% file coverage; +the 52 release-candidate tests; and the 29 new `ci-gate-test` tests. +`python3 -S .github/scripts/validate-workflow-runtime-pins.py`, +`python3 -S .github/scripts/validate-release-version.py` and +`.github/scripts/test-classify-ci-paths.sh` pass on the changed workflow. +`fledge lanes run pre-push` and `fledge trust verify` pass. + +## Live check + +The pull request (#797) was opened before approval, so `Lifecycle gate` failed on the unapproved +draft at `17e63f5d` (run 36252382792): "meaningful changed paths are not covered by an active +change". `test`, `audit`, `coverage` and `spec-check` were skipped, as on #795. This time +`SpecSync implementation ready` failed, with one annotation per cause: + +```text +lifecycle-gate was selected and ended with: failure +test was selected but skipped, so a job it needs did not succeed +spec-check was selected but skipped, so a job it needs did not succeed +audit was selected but skipped, so a job it needs did not succeed +coverage was selected but skipped, so a job it needs did not succeed +a job this gate needs ended with: failure +``` + +`Required CI gate` failed with it. The new test ran in `validate-action` (29 tests, OK). The same +job log shows the step shell as `bash -e {0}`; the simulation first assumed +`bash --noprofile --norc -eo pipefail`, which is what `shell: bash` gets, and now uses `bash -e`. diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/verification-attempts.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/verification-attempts.json new file mode 100644 index 00000000..fa1d27ab --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/verification-attempts.json @@ -0,0 +1,155 @@ +{ + "schema_version": 1, + "attempts": [ + { + "timestamp": 1790441032, + "commit": "c8581870847247a8f96592dc2c1d594fa622242b", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "execution_digest": "751b6e677bfde7d7d922ca616de219af0101f79d5f5506beb2e0c5dbfb171ea6", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "passed": true, + "commands": [ + { + "command": "specsync check --spec github", + "success": true, + "exit_code": 0 + } + ], + "requirement_ids": [ + "REQ-github-021" + ] + }, + { + "timestamp": 1790441052, + "commit": "2c366b2659e59ed51453cae20faa9b448025ac4e", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "execution_digest": "751b6e677bfde7d7d922ca616de219af0101f79d5f5506beb2e0c5dbfb171ea6", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "passed": true, + "commands": [ + { + "command": "specsync check --spec github", + "success": true, + "exit_code": 0 + } + ], + "requirement_ids": [ + "REQ-github-021" + ] + }, + { + "timestamp": 1790441052, + "commit": "3ffef4758ab6654507183cab5942b412dc6064fa", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "execution_digest": "751b6e677bfde7d7d922ca616de219af0101f79d5f5506beb2e0c5dbfb171ea6", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "acceptance_input_digest": "97ce79f04fdc4f42af6b0b479b6376eec4c5027e4c9f2d660c0d0389284b092f", + "acceptance_manifest": { + "schema_version": 1, + "entries": [ + { + "path": ".github/scripts/test-required-ci-gate.py", + "kind": "file", + "mode": 33188, + "payload_digest": "fd8d688305dd7f0ff9afcae7df3f9e21c28acf20aa9e019f5d07ed3f17969886", + "entry_digest": "374959d5ca4582b4ca8026ae6edba8fa45df7ebc734c77e2abdfd8c6d29b6ab0", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": ".github/workflows/ci.yml", + "kind": "file", + "mode": 33188, + "payload_digest": "7e9aa330aa2f3fe6f2a2e96d402aeb800fd01be4c04f0cd8421a547d10718780", + "entry_digest": "264e8690a48bb67236874ecd860f89cfd1ff79026d1f52b59aecd0301d03e899", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": "docs/HLD.md", + "kind": "file", + "mode": 33188, + "payload_digest": "9910cecb9a6b26b55d490efef4beb256727a61d438f4555ebcf21d8512dcbedd", + "entry_digest": "d79c5bc3b533f39f7b5ece13de3cd60c6caa5879ec4493732862895b10fc65f2", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": "fledge.toml", + "kind": "file", + "mode": 33188, + "payload_digest": "78f64165a395e854e30229257f36ca9c557c8ad4fe39b3ce4c8df41c5f485131", + "entry_digest": "60c7c021098391dcb96a613dfda311b037c8f3416116f0ffa0368b47a5ce4480", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": "specs/github/context.md", + "kind": "file", + "mode": 33188, + "payload_digest": "ffce758891c4af10782a3d9bd921d6eda66d108926e01d2d18555b1216f86e26", + "entry_digest": "44294b2ce66eb191595c62e34b796c6e0fc14b19044f3a592691c6a970043d39", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/github.spec.md", + "kind": "file", + "mode": 33188, + "payload_digest": "fa43398ed65dd52866f35569f6b7f49bc3041fddb02f5c4947622b84f4becf15", + "entry_digest": "b160d264d9ad6e971ad10eb372ecb282509326475b9e87d72adb1ee35753cd6a", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/requirements.md", + "kind": "file", + "mode": 33188, + "payload_digest": "8c6463559c27e9beb6f36a867a4228cd55a7c7e124310465f1d1f442b3f15569", + "entry_digest": "97ba7deda278926ea27e1ca43d80196768d12146d294d7930e9002d73c838481", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/tasks.md", + "kind": "file", + "mode": 33188, + "payload_digest": "d956ef03115be8ea4938deb00471931254b1f197b58268415005c77cb92fadbb", + "entry_digest": "75042e3c7ce9bd7f96a1b6f610ed4c8669742b027433e603f66842b5a2ea3542", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/testing.md", + "kind": "file", + "mode": 33188, + "payload_digest": "5b36338d60cda1180fb06965132f3d5f2801e6a3ac0e745544e54c12cb8ed893", + "entry_digest": "1d44058708b3160aa64371081ae687e415ebab8bb7a6ba044743a6cfdc64c68a", + "owners": [ + "github" + ] + } + ] + }, + "passed": true, + "commands": [ + { + "command": "specsync check --spec github", + "success": true, + "exit_code": 0 + } + ], + "requirement_ids": [ + "REQ-github-021" + ] + } + ] +} diff --git a/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/verification.json b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/verification.json new file mode 100644 index 00000000..f89f4f3d --- /dev/null +++ b/.specsync/archive/changes/2026-09-26-required-ci-gate-fails-when-the-lifecycle-gate-fails/verification.json @@ -0,0 +1,114 @@ +{ + "timestamp": 1790441052, + "commit": "3ffef4758ab6654507183cab5942b412dc6064fa", + "contract_digest": "baa6fc958f3c065adeafb9385c73380bedc2abdb580462737d18a48a55704d24", + "execution_digest": "751b6e677bfde7d7d922ca616de219af0101f79d5f5506beb2e0c5dbfb171ea6", + "workspace_digest": "e27514201aa0d0f122fc272801f011eccfe56489ef7d529ee5a776307dee2945", + "acceptance_input_digest": "97ce79f04fdc4f42af6b0b479b6376eec4c5027e4c9f2d660c0d0389284b092f", + "acceptance_manifest": { + "schema_version": 1, + "entries": [ + { + "path": ".github/scripts/test-required-ci-gate.py", + "kind": "file", + "mode": 33188, + "payload_digest": "fd8d688305dd7f0ff9afcae7df3f9e21c28acf20aa9e019f5d07ed3f17969886", + "entry_digest": "374959d5ca4582b4ca8026ae6edba8fa45df7ebc734c77e2abdfd8c6d29b6ab0", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": ".github/workflows/ci.yml", + "kind": "file", + "mode": 33188, + "payload_digest": "7e9aa330aa2f3fe6f2a2e96d402aeb800fd01be4c04f0cd8421a547d10718780", + "entry_digest": "264e8690a48bb67236874ecd860f89cfd1ff79026d1f52b59aecd0301d03e899", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": "docs/HLD.md", + "kind": "file", + "mode": 33188, + "payload_digest": "9910cecb9a6b26b55d490efef4beb256727a61d438f4555ebcf21d8512dcbedd", + "entry_digest": "d79c5bc3b533f39f7b5ece13de3cd60c6caa5879ec4493732862895b10fc65f2", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": "fledge.toml", + "kind": "file", + "mode": 33188, + "payload_digest": "78f64165a395e854e30229257f36ca9c557c8ad4fe39b3ce4c8df41c5f485131", + "entry_digest": "60c7c021098391dcb96a613dfda311b037c8f3416116f0ffa0368b47a5ce4480", + "owners": [ + "@exact:delivery" + ] + }, + { + "path": "specs/github/context.md", + "kind": "file", + "mode": 33188, + "payload_digest": "ffce758891c4af10782a3d9bd921d6eda66d108926e01d2d18555b1216f86e26", + "entry_digest": "44294b2ce66eb191595c62e34b796c6e0fc14b19044f3a592691c6a970043d39", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/github.spec.md", + "kind": "file", + "mode": 33188, + "payload_digest": "fa43398ed65dd52866f35569f6b7f49bc3041fddb02f5c4947622b84f4becf15", + "entry_digest": "b160d264d9ad6e971ad10eb372ecb282509326475b9e87d72adb1ee35753cd6a", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/requirements.md", + "kind": "file", + "mode": 33188, + "payload_digest": "8c6463559c27e9beb6f36a867a4228cd55a7c7e124310465f1d1f442b3f15569", + "entry_digest": "97ba7deda278926ea27e1ca43d80196768d12146d294d7930e9002d73c838481", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/tasks.md", + "kind": "file", + "mode": 33188, + "payload_digest": "d956ef03115be8ea4938deb00471931254b1f197b58268415005c77cb92fadbb", + "entry_digest": "75042e3c7ce9bd7f96a1b6f610ed4c8669742b027433e603f66842b5a2ea3542", + "owners": [ + "github" + ] + }, + { + "path": "specs/github/testing.md", + "kind": "file", + "mode": 33188, + "payload_digest": "5b36338d60cda1180fb06965132f3d5f2801e6a3ac0e745544e54c12cb8ed893", + "entry_digest": "1d44058708b3160aa64371081ae687e415ebab8bb7a6ba044743a6cfdc64c68a", + "owners": [ + "github" + ] + } + ] + }, + "passed": true, + "commands": [ + { + "command": "specsync check --spec github", + "success": true, + "exit_code": 0 + } + ], + "requirement_ids": [ + "REQ-github-021" + ] +} diff --git a/docs/HLD.md b/docs/HLD.md index 0364f2f3..3b492d4c 100644 --- a/docs/HLD.md +++ b/docs/HLD.md @@ -514,7 +514,13 @@ Job names and dependencies come from [`.github/workflows/ci.yml`](../.github/wor [`.github/scripts/classify-ci-paths.sh`](../.github/scripts/classify-ci-paths.sh). A tip that only adds a review or only moves one workflow-v2 change into the archive skips the heavy product lane. Documentation-only paths still run the full lane, so every PR can report the required -gate. Branch protection on `main` requires the `Required CI gate` context. +gate. Branch protection on `main` requires the `Required CI gate` context, which passes only when +`SpecSync implementation ready` does. That job needs every job that can finish before it, the +lifecycle preflight and the lifecycle gate included. A job passes when it succeeded, or when it was +skipped because classify deselected it. A selected job that was skipped was skipped because +something it needs did not succeed, and it fails the gate. +[`.github/scripts/test-required-ci-gate.py`](../.github/scripts/test-required-ci-gate.py) holds +the workflow to that rule and simulates every classify path. [`docs/ci-confidence.md`](ci-confidence.md) explains who owns each check and why Trust does not re-run the test suite. diff --git a/fledge.toml b/fledge.toml index c1a5ce4e..365fdf50 100644 --- a/fledge.toml +++ b/fledge.toml @@ -19,6 +19,10 @@ cmd = "cargo check" [tasks.release-candidate-test] cmd = "python3 .github/scripts/test-validate-release-candidate.py" +# Required CI gate contract and per-classify-path simulation (#796). +[tasks.ci-gate-test] +cmd = "python3 .github/scripts/test-required-ci-gate.py" + [tasks.spec-check] cmd = "cargo run -- check --strict --require-coverage 100 --force" @@ -58,6 +62,7 @@ steps = [ "audit", "spec-check", "release-candidate-test", + "ci-gate-test", { parallel = ["docs-test", "docs-lint", "docs-build", "vscode-compile", "vscode-package"] }, ] @@ -80,6 +85,7 @@ steps = [ "build", "spec-check", "release-candidate-test", + "ci-gate-test", ] # Used by .trust.toml [lifecycle] for the GitHub Trust action only. @@ -105,5 +111,6 @@ steps = [ "audit", "spec-check", "release-candidate-test", + "ci-gate-test", { parallel = ["docs-test", "docs-lint", "docs-build", "vscode-compile", "vscode-package"] }, ] diff --git a/specs/github/context.md b/specs/github/context.md index 69ee110e..12e6bb85 100644 --- a/specs/github/context.md +++ b/specs/github/context.md @@ -72,6 +72,15 @@ spec: github.spec.md is retained only because `src/change.rs` reads it. Two of the live defects #499 removed were in the CI copy and not in SpecSync, which is the argument for not rebuilding it: a reimplementation of a shipped rule drifts from it, and only the copy is unshipped and untested by users. +- **A skipped job is not a pass unless classify deselected it (#796)**: `implementation-gate` + ("SpecSync implementation ready") needs every job that can finish before it, `preflight` and + `lifecycle-gate` included. GitHub reports `skipped` both for a job classify deselected and for a + job whose dependency failed, so each row of the gate's `GATES` table carries that job's own `if:` + (less a leading `always() &&`), evaluated again over the same classify outputs. `skipped` passes + only where that says the job was not selected; a selected job that was skipped fails the gate. + Before this, a failed lifecycle gate left `test`, `audit`, `coverage` and `spec-check` skipped + and `Required CI gate` green. `.github/scripts/test-required-ci-gate.py` holds each row to its + job's `if:` as text and simulates every classify lane against the gate's own script. ## Key Files @@ -82,6 +91,9 @@ spec: github.spec.md - `.github/scripts/validate-release-version.py` - Current package, Action, docs, CI consumer, and Trust candidate version consistency - `.github/scripts/validate-workflow-runtime-pins.py` - Exact hosted Bun runtime enforcement +- `.github/scripts/test-required-ci-gate.py` - Required CI gate contract: every job before the + gate is in its `needs`, each gate row mirrors its job's `if:`, and the gate's own script is run + over every classify lane - `fledge.toml` and `.trust.toml` - Keep full local verification separate from hosted Trust's residual lifecycle prerequisite - `docs/ci-confidence.md` - CI/Trust ownership, confidence tiers, and protected Tier B follow-up diff --git a/specs/github/github.spec.md b/specs/github/github.spec.md index d830ac91..199a1386 100644 --- a/specs/github/github.spec.md +++ b/specs/github/github.spec.md @@ -1,6 +1,6 @@ --- module: github -version: 33 +version: 34 status: stable files: - src/github.rs @@ -60,6 +60,13 @@ extension CI. Every path that can be merged can reach the required CI gate; a path the CI workflow cannot trigger can never report the gate and blocks its pull request. +The required CI gate fails whenever a job it depends on did not succeed, unless classify +deselected that job. A skipped job passes only when its own `if:` condition, evaluated over the +classify outputs, left it unselected. A selected job that was skipped was skipped because a job it +needs did not succeed, and it fails the gate: `skipped` is never read as green on its own. The +lifecycle preflight and the lifecycle gate are selected on every path, so either one failing turns +the required gate red. + Release qualification verifies exactly the tag protections this repository actually has, and names every protection it does not verify on every run, green runs included. A gate that demands an unprovisioned policy fails on every candidate and therefore verifies nothing — it is not a safe @@ -224,3 +231,4 @@ first use — so the workflow names no environment rather than publish a gate th | 2026-09-10 | document-shipped-specsync-6-0-0-and-set-the-action-default-to-the-stable-release: Set the Action omitted-input default on main to shipped 6.0.0 and document that @v6.0.0 still embeds 6.0.0-rc.14 | | 2026-09-10 | rc-qualification-requires-ubuntu-and-macos-only-windows-is-not-a-6-0-target: RC qualification requires Ubuntu and macOS only; Windows is not a 6.0 target | | 2026-09-10 | correct-remaining-specsync-6-0-docs-after-the-stable-ship: Correct remaining SpecSync 6.0 docs after the stable ship | +| 2026-09-26 | required-ci-gate-fails-when-the-lifecycle-gate-fails: Required CI gate fails when the lifecycle gate fails | diff --git a/specs/github/requirements.md b/specs/github/requirements.md index daefe7fe..647ee463 100644 --- a/specs/github/requirements.md +++ b/specs/github/requirements.md @@ -280,3 +280,25 @@ Acceptance Criteria - Incomplete check-run status yields overall `pending` when no failure is present. - Auth tokens are redacted from surfaced REST error messages. +### REQ-github-021 + +The required CI gate SHALL fail whenever the lifecycle preflight, the lifecycle gate, or any job +classify selected for the run fails, is cancelled, or is skipped because a job it needs did not +succeed. + +Acceptance Criteria + +- `implementation-gate` (SpecSync implementation ready) needs `classify`, `preflight`, + `lifecycle-gate` and every other job that can finish before it, and `ci-gate` (Required CI gate) + passes only when `implementation-gate` succeeds. +- A skipped job passes only when that job's own `if:` condition, evaluated again over the classify + outputs, deselected it. +- Full, site-only, VS Code-only, specs/lifecycle-only, archive-only, legacy archive-only and + review-only pull requests, pushes to `main` and `workflow_dispatch` runs stay green when every + selected job succeeds. +- `.github/scripts/test-required-ci-gate.py` fails when `preflight` or `lifecycle-gate` is missing + from the gate's `needs`, when a job that gates on `lifecycle-gate` or can otherwise finish before + the gate is missing, or when a gate row no longer matches its job's `if:`. +- The same test runs the gate's own script over every classify path and requires the required gate + to be red when any one selected job fails or is cancelled. + diff --git a/specs/github/tasks.md b/specs/github/tasks.md index 7fa61da2..ab4fd87c 100644 --- a/specs/github/tasks.md +++ b/specs/github/tasks.md @@ -74,3 +74,4 @@ Per-module role sign-offs were not collected. Release approval is governed by di crossing code edges or allowing cancelled republications to poison exact-SHA success (CHG-0077) - [x] Pin Action/`github-action.md` default to published `6.0.0-rc.14` and teach `validate-release-version.py` the candidate-window exception (#628) +- [x] Make `Required CI gate` fail when `preflight` or `lifecycle-gate` fails, and read `skipped` as a pass only where classify deselected the job, with a workflow guard and per-path simulation (#796, REQ-github-021) diff --git a/specs/github/testing.md b/specs/github/testing.md index 6e0e54f8..7a4558a3 100644 --- a/specs/github/testing.md +++ b/specs/github/testing.md @@ -11,6 +11,7 @@ spec: github.spec.md | Hosted Bun runtime | `python3 -S .github/scripts/validate-workflow-runtime-pins.py` | Pages, site CI, and VS Code extension CI each contain exactly one expected `setup-bun` Action ref with the supported exact Bun version under that step's `with` mapping; structural parsing covers block/flow mappings, quoted keys, arbitrary valid list-marker spacing, and `uses` after other keys, while mixed-case repositories, moving refs, duplicates, unexpected jobs, and missing inputs fail without Python site packages | | Immutable RC evidence | `python3 .github/scripts/test-validate-release-candidate.py` | Exactly one successful Ubuntu and macOS record must share the expected annotated RC identity, candidate SHA, schema, and Fledge lane; Windows is not required as of 6.0. Missing, duplicate, malformed, failed, cancelled, or mixed identity evidence fails closed | | Lifecycle coherence | `cargo run -- change audit --strict` | SpecSync itself validates active change workspaces and living SDD policy/spec coherence; CI no longer reimplements these rules against commit topology | +| Required CI gate | `python3 -S .github/scripts/test-required-ci-gate.py` (Fledge `ci-gate-test`) | Every job that can finish before `implementation-gate` is in its `needs`, `preflight` and `lifecycle-gate` included; each `GATES` row reads its own job's result and repeats that job's `if:`; simulated over every classify lane, flag combination and event, the required gate is green when every selected job succeeds and red when any one fails or is cancelled; the pre-#796 gate reproduces the bug (REQ-github-021) | ## Coverage Gaps @@ -75,6 +76,8 @@ spec: github.spec.md | Candidate content or marker changes | Prior platform evidence cannot authorize promotion or upload | Change the expected SHA/tag in validator fixtures and require failure; conflicting workflow history also fails | | Final publication | Final tag and artifacts use the already-qualified candidate SHA | Require authorization before promotion and independent final-tag/checkout identity checks before upload | | Lifecycle metadata rides with the product commit | No separate archive tip is required before merge | Require `cargo run -- change audit --strict` to pass on the pull request as a whole | +| Lifecycle gate or preflight fails on a pull request | `test`, `audit`, `coverage` and `spec-check` are skipped, and `SpecSync implementation ready` and `Required CI gate` fail (#796) | `test_issue_796_lifecycle_gate_failure_turns_the_required_gate_red` and the per-path injection test in `test-required-ci-gate.py` | +| A new CI job gates on `lifecycle-gate` or is selected before the gate | It must be in `implementation-gate.needs` with a `GATES` row matching its `if:` | `test_workflow_holds_to_the_gate_contract` and the guard mutation tests in `test-required-ci-gate.py` | | Action omitted-input default after stable 6.0.0 | Default on `main` and `@v6` is `6.0.0`; `@v6.0.0` tag still embeds `6.0.0-rc.14` | Read `action.yml` default and site inputs-table default; `git show v6:action.yml` default is `6.0.0`; `git rev-parse v6^{commit}` is not the `v6.0.0` commit; run `python3 -S .github/scripts/validate-release-version.py` |