From 7ca2c4e6f39ad27a8943d250e0d0b5c745a00c05 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Mon, 16 Nov 2026 16:50:15 -0500 Subject: [PATCH] One monitor, slice 2: one self-closing alert issue per failing gate, labelled with its owners The deploy and the nightly each keep one issue per gate command, from the results the gates recorded: a failure opens or reopens it, a pass closes it. A run that failed outside any gate keeps one fallback issue per workflow. Replaces the deployment-failure and stale-evidence issues. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 142 ++++++++--------- .github/workflows/evidence-nightly.yml | 201 ++++++------------------- docs/runbooks/evidence-sla-alarm.md | 4 +- scripts/gate-alerts.py | 157 +++++++++++++++++++ tests/test_gate_alerts.py | 67 +++++++++ 5 files changed, 334 insertions(+), 237 deletions(-) create mode 100755 scripts/gate-alerts.py create mode 100644 tests/test_gate_alerts.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index fb25a13b..44a5d8a6 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -139,7 +139,7 @@ jobs: # direction: renaming a watched job would silently drop it from an # inclusion list, whereas here it stays watched, and only a NEW dependent # job could cause a false cancel — which is loud and gets noticed. - DEPENDENT='["Deploy Infrastructure","Notify on Failure","Queue watchdog"]' + DEPENDENT='["Deploy Infrastructure","Gate alerts","Queue watchdog"]' stuck='' for _ in $(seq 1 $(( GRACE_MINUTES * 2 ))); do stuck=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \ @@ -236,8 +236,21 @@ jobs: # host time) are banned, and every sprintf format must be a literal with # no precision specifier. - name: Gates for the pr stage (make gates STAGE=pr) + # Each gate's result goes to this file, for the alert job. Set on this + # step only: the unit tests above run the runner with their own inputs. + env: + GATE_RESULTS_FILE: /tmp/gate-results.jsonl run: make gates STAGE=pr + - name: Upload the results the pr stage recorded, for the alert job + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: gate-results-pr + path: /tmp/gate-results.jsonl + if-no-files-found: ignore + retention-days: 7 + # Real-bug lint (pyflakes + bugbear, not cosmetic style) and lenient type # check. Config in pyproject.toml. Catches the class of latent bug the panel # review flagged (e.g. the unhashable-dict crash) before it reaches a deploy. @@ -609,8 +622,11 @@ jobs: actions: read # Detect-and-recover gate events (infrastructure/schemas/gate-modes.json) are # appended here by scripts/_gates.py and published with this run's evidence. + # Each gate's result (scripts/run-gates.sh) goes to GATE_RESULTS_FILE, for + # the gate alerts job. env: GATE_EVENTS_FILE: /tmp/gate-events.jsonl + GATE_RESULTS_FILE: /tmp/gate-results.jsonl needs: [test, compliance-check, security-scan, apps] # IMPORTANT: Only deploy if on main branch AND security check passed if: github.ref == 'refs/heads/main' && needs.compliance-check.outputs.compliant == 'true' @@ -1884,6 +1900,15 @@ jobs: }); } + - name: Upload the results the deploy stages recorded for the alert job + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: gate-results-deploy + path: /tmp/gate-results.jsonl + if-no-files-found: ignore + retention-days: 7 + - name: Remove the trust-root state snapshot if: always() run: rm -f infrastructure/tfstate-bootstrap.json @@ -1981,92 +2006,51 @@ jobs: retention-days: 5 # =========================================================================== - # JOB 4: ALERT WHEN THINGS BREAK - # =========================================================================== - # Sends notifications and creates tickets when deployment fails + # JOB 4: ALERT WHEN A GATE FAILS # =========================================================================== - # This job used to report SUCCESS while the deploy underneath it failed, and it - # did so twelve nights running. Two defects, both fixed here: - # - # 1. The issue-creating step was gated on `github.event_name == 'push'`. On a - # scheduled run the step was SKIPPED — and a job whose only step is skipped - # concludes `success`. So the run summary showed a green "Notify on Failure" - # next to a red deploy, and no issue was ever filed. The event gate is gone; - # a failed deploy alerts however it was triggered. - # - # 2. Even when it did fire, it opened a brand-new issue every time. The - # repository accumulated 15+ identical open "Deployment Failed" issues, - # which is functionally the same as no alert: nothing distinguishes the one - # that matters. It now reuses a single issue, reopening and commenting, and - # assigns it to the repository owner so it reaches a person. + # One issue per failing gate, labelled with its owners (design principle 9), + # from the results each gate recorded (scripts/gate-alerts.py). A failing gate + # opens or reopens its issue; a passing gate closes it, so it runs on every + # deploy run, not only failed ones. A run that failed outside any gate keeps + # one fallback issue. Every issue is reused, never duplicated: a new issue per + # failure once left 99 identical open issues, which is the same as no alert. + # Pull requests do not touch the issues: they describe production gates. notify: - name: Notify on Failure + name: Gate alerts runs-on: ubuntu-latest - needs: [compliance-check, apps, deploy] - if: always() && (needs.compliance-check.result == 'failure' || needs.apps.result == 'failure' || needs.deploy.result == 'failure') + needs: [test, compliance-check, apps, deploy] + if: always() && !cancelled() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' permissions: contents: read issues: write steps: - - name: Raise or update the deployment-failure issue - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - # Job results reach the script through the environment rather than by - # `${{ }}` interpolation into the script body — templating workflow context - # straight into executable script is the habit that causes injection bugs. + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # None to download (every gate job skipped) leaves every gate "not run": + # no issue changes, and a failed run still raises the fallback issue. + - name: Download the gate results + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: gate-results-* + path: gate-results + + # Job results reach the step through the environment, not by `${{ }}` + # interpolation into the script. + - name: Open, update or close the gate alert issues env: + GH_TOKEN: ${{ github.token }} + TEST_RESULT: ${{ needs.test.result }} COMPLIANCE_RESULT: ${{ needs.compliance-check.result }} + APPS_RESULT: ${{ needs.apps.result }} DEPLOY_RESULT: ${{ needs.deploy.result }} - with: - script: | - const TITLE = `Deployment failing: ${context.workflow}`; - const runUrl = `${context.payload.repository.html_url}/actions/runs/${context.runId}`; - const body = [ - `The deploy pipeline failed. Until it succeeds, \`main\` is not reaching production.`, - ``, - `| | |`, - `|---|---|`, - `| Run | [${context.runId}](${runUrl}) |`, - `| Trigger | \`${context.eventName}\` |`, - `| Commit | ${context.sha} |`, - `| Ref | ${context.ref} |`, - `| Compliance check | \`${process.env.COMPLIANCE_RESULT}\` |`, - `| Deploy | \`${process.env.DEPLOY_RESULT}\` |`, - ``, - `Review the failing step in the run log, and the compliance reports it uploaded.`, - ``, - `This issue is reused rather than duplicated — each failure adds a comment. Close it once a deploy succeeds.`, - ].join('\n'); - - const existing = await github.paginate(github.rest.issues.listForRepo, { - owner: context.repo.owner, - repo: context.repo.repo, - state: 'all', - per_page: 100, - labels: 'deploy-failure', - }); - const match = existing.find(i => i.title === TITLE && !i.pull_request); - - if (match) { - if (match.state === 'closed') { - await github.rest.issues.update({ - owner: context.repo.owner, repo: context.repo.repo, - issue_number: match.number, state: 'open', - }); - } - await github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, - issue_number: match.number, body, - }); - core.notice(`Updated issue #${match.number}`); - } else { - const created = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: TITLE, - body, - labels: ['deployment', 'deploy-failure', 'urgent'], - assignees: [context.repo.owner], - }); - core.notice(`Created issue #${created.data.number}`); - } + run: | + failed=false + for r in "$TEST_RESULT" "$COMPLIANCE_RESULT" "$APPS_RESULT" "$DEPLOY_RESULT"; do + [ "$r" = failure ] && failed=true + done + cat gate-results/*/gate-results.jsonl > /tmp/gate-results.jsonl 2>/dev/null || true + python3 scripts/gate-alerts.py deploy /tmp/gate-results.jsonl pr,pre-apply,post-apply,pre-publish \ + --run-failed "$failed" diff --git a/.github/workflows/evidence-nightly.yml b/.github/workflows/evidence-nightly.yml index b06d2ab1..96b0b4a4 100644 --- a/.github/workflows/evidence-nightly.yml +++ b/.github/workflows/evidence-nightly.yml @@ -340,15 +340,6 @@ jobs: env: GATE_EVENTS_FILE: /tmp/gate-events.jsonl GATE_RESULTS_FILE: /tmp/gate-results.jsonl - # Which gate stopped the run, handed to the alert job. A failed vulnerability - # gate and a skipped reconciliation are different problems: the first means - # stale evidence, the second means nothing compared the live site against its - # documentation. The alert has to be able to say which. - outputs: - vuln_gate: ${{ steps.vulngate.outcome }} - vuln_findings: ${{ steps.vulngate.outputs.findings }} - detection: ${{ steps.detection.outcome }} - reconcile: ${{ steps.reconcile.outcome }} permissions: contents: read # Sigstore keyless signing via the GitHub Actions OIDC provider, and the @@ -554,24 +545,12 @@ jobs: # vulnerability that is not remediated or dispositioned false-positive / # operational-requirement in data/decisions.json. When a newly # disclosed CVE lands, THIS is what stops it publishing, and the alert job - # is what tells a human to disposition it. - # - # The ids the gate names are exported as a JSON list so the alert can put - # each one in front of a person with its advisory and any open Dependabot - # PR. The gate's own exit status is what fails the step; capturing its - # output changes nothing about what passes. + # is what tells a human to disposition it: its issue links this run, whose + # log names each finding. - name: Vulnerability gate (fix or categorize every vuln) id: vulngate working-directory: infrastructure - run: | - set +e - bash ../scripts/run-gates.sh vuln > vuln-gate.out 2>&1 - rc=$? - set -e - cat vuln-gate.out - findings=$(sed -n 's/^ ✗ \([^ ]*\) .*/\1/p' vuln-gate.out | jq -Rsc 'split("\n") | map(select(length > 0))') - echo "findings=${findings}" >> "$GITHUB_OUTPUT" - exit "$rc" + run: bash ../scripts/run-gates.sh vuln - name: Build VDR vuln trend ledger (RA-5(6)) working-directory: infrastructure @@ -850,6 +829,15 @@ jobs: --content-type application/json --cache-control "no-cache" fi + - name: Upload the gate results for the alert job + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: gate-results-nightly + path: /tmp/gate-results.jsonl + if-no-files-found: ignore + retention-days: 7 + # =========================================================================== # JOB 3: ALERT — the job that has to work # =========================================================================== @@ -860,156 +848,57 @@ jobs: # Nothing was raised, nothing looked wrong, and the published evidence rotted. # # This job takes the opposite defaults: - # - no event gating whatsoever; a failed run alerts on every trigger - # - one issue, reopened and commented, instead of a new issue per failure - # (the deploy path has left 15+ identical open issues, which is the same - # as no alert at all) - # - assigned to the repository owner, so it reaches a person rather than a - # notifications tab - # - states the freshness deadline, so the message carries its own urgency + # - no event gating whatsoever; every run that was not deliberately dropped + # reports, so a passing gate also closes its issue + # - one issue per failing gate (scripts/gate-alerts.py), labelled with the + # owners the gate register names, reopened and commented rather than + # duplicated (the deploy path once left 99 identical open issues, which is + # the same as no alert at all); a run that failed outside any gate keeps + # one fallback issue # # RESIDUAL RISK, recorded rather than hidden: this alert rides on the same # scheduled workflow it is alerting about. If GitHub disables the cron — which # it does after 60 days with no repository activity — neither the refresh nor - # this alert runs, and the silence looks identical to success. An alert that - # cannot detect its own absence needs an out-of-band watchdog; that is a - # follow-up, not something this workflow can solve from the inside. + # this alert runs, and the silence looks identical to success. The evidence + # watchdog (infrastructure/watchdog.tf) pages on that from outside. # =========================================================================== alert: - name: Alert on stale evidence + name: Gate alerts runs-on: ubuntu-latest needs: [sast, pin, apps, refresh] # The one exclusion: a skipped `sast` means the run was deliberately dropped # (a post-deploy trigger for a deploy that did not qualify), not that it failed. - # A failed pin or apps job skips refresh, which is not success, so it alerts. - if: always() && needs.sast.result != 'skipped' && (needs.sast.result != 'success' || needs.refresh.result != 'success') + if: always() && !cancelled() && needs.sast.result != 'skipped' permissions: contents: read issues: write - # Read-only: find open Dependabot PRs for the packages the gate named. - pull-requests: read steps: - - name: Raise or update the stale-evidence issue - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - # Job results reach the script through the environment rather than by - # `${{ }}` interpolation into the script body. These particular values are - # a fixed vocabulary and could not carry an injection, but templating - # workflow context straight into executable script is the habit that - # causes those bugs, so the safe form is the one written down here. + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # None to download (every gate job skipped) leaves every gate "not run": + # no issue changes, and a failed run still raises the fallback issue. + - name: Download the gate results + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: gate-results-* + path: gate-results + + # Job results reach the step through the environment, not by `${{ }}` + # interpolation into the script. A failed pin or apps job skips refresh, + # which is not success, so the run counts as failed. + - name: Open, update or close the gate alert issues env: + GH_TOKEN: ${{ github.token }} SAST_RESULT: ${{ needs.sast.result }} REFRESH_RESULT: ${{ needs.refresh.result }} - VULN_GATE: ${{ needs.refresh.outputs.vuln_gate }} - VULN_FINDINGS: ${{ needs.refresh.outputs.vuln_findings }} - RECONCILE: ${{ needs.refresh.outputs.reconcile }} - DETECTION: ${{ needs.refresh.outputs.detection }} - with: - script: | - const TITLE = 'Evidence nightly failed — published compliance reporting is going stale'; - const runUrl = `${context.payload.repository.html_url}/actions/runs/${context.runId}`; - // Step outcomes: success | failure | cancelled | skipped, or empty when - // the job never reached the step. - const vulnGate = process.env.VULN_GATE || 'not reached'; - const reconcile = process.env.RECONCILE || 'not reached'; - const detection = process.env.DETECTION || 'not reached'; - const causes = []; - if (detection === 'failure') { - causes.push(`**Detection failed (G7).** A scanner or feed the VDR depends on was missing, malformed or stale, so nothing was built or published. The "Detection gate (G7)" step names each input and why; the status beacon's \`detection\` block records it.`); - } - if (reconcile === 'failure') { - causes.push(`**The live reconciliation found drift.** The live site and its published documentation disagree. The "Reconciliation gate" step names the failing invariant.`); - } else if (reconcile !== 'success') { - causes.push(`**The live reconciliation did not run (\`${reconcile}\`).** Nothing compared the live site against its published documentation in this run. That is a separate problem from stale evidence: treat drift as unchecked until a run shows this gate passing.`); - } - if (vulnGate === 'failure') { - const runbook = `${context.payload.repository.html_url}/blob/main/docs/runbooks/vuln-gate-failure.md`; - let findings = []; - try { findings = JSON.parse(process.env.VULN_FINDINGS || '[]'); } catch (e) { findings = []; } - // Open Dependabot PRs, matched to each finding by package name. - const openPRs = (await github.paginate(github.rest.pulls.list, { - owner: context.repo.owner, repo: context.repo.repo, state: 'open', per_page: 100, - })).filter(p => p.head.ref.startsWith('dependabot/')); - const lines = findings.map(id => { - const ghsa = (id.match(/GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}/) || [])[0]; - const pkg = (id.match(/pkg:[^/]+\/(.+)@[^@]+$/) || [])[1]; - const pr = pkg && openPRs.find(p => p.title.includes(` ${pkg} `)); - const parts = [`- \`${id}\``]; - if (ghsa) parts.push(`[advisory](https://github.com/advisories/${ghsa})`); - parts.push(pr ? `open Dependabot PR: #${pr.number}` : 'no open Dependabot PR'); - return parts.join(' — '); - }); - causes.push([ - `**The vulnerability gate failed** on ${findings.length || 'one or more'} finding(s) with no fix and no disposition:`, - ``, - ...(lines.length ? lines : ['- (the run log\'s "Vulnerability gate" step names them)']), - ``, - `Fix each (a Dependabot PR plus a deploy clears it: the nightly scans the *published* commit), or record a decision for it in data/decisions.json. Remediation clocks run from first detection. Runbook: ${runbook}`, - ].join('\n')); - } - if (causes.length === 0) { - causes.push(`**Both gates passed or were not reached.** Check, in order: the freshness assertion at the end (published but not served), the setup steps before the gates, and the SAST job.`); - } - const body = [ - `The nightly VDR refresh did not complete, so \`/.well-known/vdr-report.json\` is not being updated.`, - ``, - `Published compliance reporting must be under 24 hours old. Every hour this stays broken is an hour the site is serving evidence that does not meet that policy.`, - ``, - `| | |`, - `|---|---|`, - `| Run | [${context.runId}](${runUrl}) |`, - `| Trigger | \`${context.eventName}\` |`, - `| Commit | ${context.sha} |`, - `| SAST job | \`${process.env.SAST_RESULT}\` |`, - `| Refresh job | \`${process.env.REFRESH_RESULT}\` |`, - `| Detection gate (G7) | \`${detection}\` |`, - `| Vulnerability gate | \`${vulnGate}\` |`, - `| Reconciliation gate (live drift check) | \`${reconcile}\` |`, - ``, - ...causes.flatMap(c => [c, ``]), - `This issue is reused rather than duplicated — each failure adds a comment. It will not close itself; close it once a run succeeds.`, - ].join('\n'); - - // One issue, not one per night. Search by exact title among open issues. - const existing = await github.paginate(github.rest.issues.listForRepo, { - owner: context.repo.owner, - repo: context.repo.repo, - state: 'all', - per_page: 100, - labels: 'evidence-freshness', - }); - // Prefer an open issue, then the oldest. The lookup once created a - // second issue while an identical open one existed (#330, #336) for a - // reason the logs did not show, so record what it matched. - const matches = existing - .filter(i => i.title === TITLE && !i.pull_request) - .sort((a, b) => (a.state === b.state ? a.number - b.number : a.state === 'open' ? -1 : 1)); - core.info(`Issue lookup: ${existing.length} labelled, ${matches.length} matching [${matches.map(i => `#${i.number}:${i.state}`).join(', ')}]`); - const match = matches[0]; - - if (match) { - if (match.state === 'closed') { - await github.rest.issues.update({ - owner: context.repo.owner, repo: context.repo.repo, - issue_number: match.number, state: 'open', - }); - } - await github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, - issue_number: match.number, body, - }); - core.notice(`Updated issue #${match.number}`); - } else { - const created = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: TITLE, - body, - labels: ['compliance', 'evidence-freshness', 'urgent'], - assignees: [context.repo.owner], - }); - core.notice(`Created issue #${created.data.number}`); - } + run: | + failed=false + if [ "$SAST_RESULT" != success ] || [ "$REFRESH_RESULT" != success ]; then failed=true; fi + cat gate-results/*/gate-results.jsonl > /tmp/gate-results.jsonl 2>/dev/null || true + python3 scripts/gate-alerts.py nightly /tmp/gate-results.jsonl nightly --run-failed "$failed" # Independent of why the run failed, report how stale the published evidence # actually is right now. This is the number the policy is about, and it puts diff --git a/docs/runbooks/evidence-sla-alarm.md b/docs/runbooks/evidence-sla-alarm.md index 4057bb84..1b7d9b8c 100644 --- a/docs/runbooks/evidence-sla-alarm.md +++ b/docs/runbooks/evidence-sla-alarm.md @@ -16,11 +16,11 @@ Any alarm also fires if the watchdog stops publishing metrics, because missing d ## `nightly-failed` 1. Open the latest **Evidence Nightly** run in GitHub Actions. The status beacon at `/.well-known/vdr-status.json` gives its URL (`run_url`), the overall `result`, and each gate's outcome. -2. The GitHub issue labelled `evidence-freshness` names the gate and, for a vulnerability-gate failure, each finding with its advisory and any open Dependabot PR. +2. Each failing gate has its own GitHub issue, labelled `gate-alert` and with its owners' roles, titled `[nightly] gate failing: `. A run that failed outside any gate has `[nightly] run failed outside any gate`. The issue links the run; the gate's step log names the cause (for the vulnerability gate, each finding). - **Vulnerability gate failed:** follow [`vuln-gate-failure.md`](vuln-gate-failure.md). - **Reconciliation gate failed:** the live system and its documentation disagree. The step log names the failing invariant. Fix the drift through the reviewed deploy path; never weaken the gate. - **`not reached` everywhere, or no recent run at all:** check whether GitHub has disabled the workflow's schedule (**Actions → Evidence Nightly**; it does this after 60 days without repository activity) and re-enable it. Then run it once by hand. -3. Close the alert issue once a run passes. The alarm clears on its own within about two hours. +3. The next run in which the gate passes closes its issue. The alarm clears on its own within about two hours. ## `vdr-age` diff --git a/scripts/gate-alerts.py b/scripts/gate-alerts.py new file mode 100755 index 00000000..5841e03e --- /dev/null +++ b/scripts/gate-alerts.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""One alert issue per failing gate, routed to its owner (design principle 9). + +Reads the results a run's gates recorded (GATE_RESULTS_FILE, written by +scripts/run-gates.sh) and, for every gate the register lists for the given +stages, keeps one GitHub issue per gate command: + + fail open it (or reopen it) and comment with this run + pass close it, if open, with a comment naming this run + not run / not on the runner leave it alone: this run says nothing about it + +The unit is the gate command, the thing that ran and reported. Several register +checks can share one command (the reconcile.* invariants); they pass or fail +together, so they share one issue, labelled with every owner they name. One +issue per check would open several identical issues for a single failure. + +A run that failed while no gate failed (a build, apply or signing step) keeps a +single fallback issue for that workflow, closed by the next run that succeeds. + +Issues carry the `gate-alert` label and an owner label per register role; the +workflow passes CONTEXT (deploy or nightly), so each workflow's gates keep their +own issues. Usage, from the repository root: + + gate-alerts.py CONTEXT RESULTS_FILE STAGE[,STAGE...] --run-failed true|false [--dry-run] + +`--dry-run` prints the actions without calling GitHub. Otherwise it uses `gh` +with GH_TOKEN, in the repository GITHUB_REPOSITORY names. +""" + +import argparse +import json +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _gates # noqa: E402 + +LABEL = "gate-alert" +CONTEXTS = ("deploy", "nightly") + + +def title_for(context, run): + return f"[{context}] gate failing: {run}" + + +def fallback_title(context): + return f"[{context}] run failed outside any gate" + + +def plan(register, context, stages, results_path, run_failed): + """The actions this run implies: [{action, title, labels, checks}], where + action is open (a failure) or close (a pass). Pure: no GitHub calls.""" + groups = {} + for stage in stages: + for r in _gates.stage_results(register, stage, results_path): + run = register["checks"][r["check"]].get("run") + if run is None: + continue + g = groups.setdefault(run, {"checks": {}, "result": r["result"]}) + g["checks"][r["check"]] = r["owner"] + actions = [] + for run, g in groups.items(): + if g["result"] not in ("pass", "fail"): + continue + actions.append({ + "action": "open" if g["result"] == "fail" else "close", + "title": title_for(context, run), + "labels": [LABEL] + sorted(set(g["checks"].values())), + "checks": sorted(g["checks"].items()), + }) + gate_failed = any(a["action"] == "open" for a in actions) + if run_failed and not gate_failed: + actions.append({"action": "open", "title": fallback_title(context), + "labels": [LABEL, "role:platform-owner"], "checks": []}) + elif not run_failed: + actions.append({"action": "close", "title": fallback_title(context), "labels": [LABEL], "checks": []}) + return actions + + +def _body(a, context): + server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + repo, run_id = os.environ.get("GITHUB_REPOSITORY", ""), os.environ.get("GITHUB_RUN_ID", "") + run_url = f"{server}/{repo}/actions/runs/{run_id}" if repo and run_id else "(local run)" + sha = os.environ.get("GITHUB_SHA", "") + if a["action"] == "close": + return f"Passed in the {context} run {run_url} (commit {sha}). Closing." + lines = [f"Failed in the {context} run {run_url} (commit {sha}).", ""] + if a["checks"]: + lines += ["| Check | Owner |", "|---|---|"] + [f"| `{c}` | `{o}` |" for c, o in a["checks"]] + lines += ["", "The run log names the cause. Fix it, or record a decision for it; " + "the next run where this gate passes closes this issue."] + else: + lines += ["The run failed in a step that is not a registered gate (a build, apply or " + "signing step). The run log names it. The next successful run closes this issue."] + return "\n".join(lines) + + +def _gh(*args): + return subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout + + +def apply(actions, context): + found = json.loads(_gh("issue", "list", "--label", LABEL, "--state", "all", "--limit", "500", + "--json", "number,title,state")) + for a in actions: + # Prefer an open issue, then the oldest, so one gate never gets two. + matches = sorted((i for i in found if i["title"] == a["title"]), + key=lambda i: (i["state"] != "OPEN", i["number"])) + match = matches[0] if matches else None + body = _body(a, context) + if a["action"] == "close": + if match and match["state"] == "OPEN": + _gh("issue", "close", str(match["number"]), "--comment", body) + print(f"closed #{match['number']}: {a['title']}") + continue + for label in a["labels"]: + _gh("label", "create", label, "--force", "--color", "B60205" if label == LABEL else "C5DEF5") + if match: + if match["state"] != "OPEN": + _gh("issue", "reopen", str(match["number"])) + _gh("issue", "comment", str(match["number"]), "--body", body) + print(f"updated #{match['number']}: {a['title']}") + else: + args = ["issue", "create", "--title", a["title"], "--body", body] + for label in a["labels"]: + args += ["--label", label] + # Every role is held by one person here: assign the repository + # owner, so the issue reaches a person and not only a label. + if os.environ.get("GITHUB_REPOSITORY_OWNER"): + args += ["--assignee", os.environ["GITHUB_REPOSITORY_OWNER"]] + print(f"created: {a['title']} {_gh(*args).strip()}") + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("context", choices=CONTEXTS) + p.add_argument("results") + p.add_argument("stages") + p.add_argument("--run-failed", required=True, choices=("true", "false")) + p.add_argument("--dry-run", action="store_true") + a = p.parse_args(argv) + stages = a.stages.split(",") + for s in stages: + if s not in _gates.STAGES: + raise SystemExit(f"unknown stage {s!r}; one of {list(_gates.STAGES)}") + actions = plan(_gates.load_gate_modes(), a.context, stages, a.results, a.run_failed == "true") + if a.dry_run: + for x in actions: + print(f"{x['action']}: {x['title']} {x['labels']}") + return 0 + apply(actions, a.context) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_gate_alerts.py b/tests/test_gate_alerts.py new file mode 100644 index 00000000..b8c57173 --- /dev/null +++ b/tests/test_gate_alerts.py @@ -0,0 +1,67 @@ +"""One alert issue per failing gate command, routed to its owners (scripts/gate-alerts.py).""" +import importlib.util +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import _gates # noqa: E402 + +spec = importlib.util.spec_from_file_location("gate_alerts", REPO / "scripts" / "gate-alerts.py") +ga = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ga) + +RECONCILE = "scripts/run-gates.sh reconcile" +VULN = "scripts/run-gates.sh vuln" + + +def _results(tmp_path, **runs): + p = tmp_path / "results.jsonl" + p.write_text("".join(json.dumps({"run": r, "result": res, "at": "t"}) + "\n" for r, res in runs.items())) + return p + + +def _by_title(actions): + return {a["title"]: a for a in actions} + + +def test_a_failing_gate_opens_one_issue_for_its_command_with_every_owner(tmp_path): + reg = _gates.load_gate_modes() + acts = _by_title(ga.plan(reg, "nightly", ["nightly"], _results(tmp_path, **{RECONCILE: "fail"}), True)) + a = acts[ga.title_for("nightly", RECONCILE)] + assert a["action"] == "open" + sharing = {c: e["owner"] for c, e in reg["checks"].items() if e.get("run") == RECONCILE and "nightly" in e["stages"]} + assert len(sharing) > 1, "the test needs a command several checks share" + assert dict(a["checks"]) == sharing + assert a["labels"] == [ga.LABEL] + sorted(set(sharing.values())) + # A gate failed, so the run's failure is accounted for: no fallback issue. + assert ga.fallback_title("nightly") not in acts + + +def test_a_passing_gate_closes_its_issue_and_unreported_gates_are_left_alone(tmp_path): + reg = _gates.load_gate_modes() + acts = _by_title(ga.plan(reg, "nightly", ["nightly"], _results(tmp_path, **{VULN: "pass"}), False)) + assert acts[ga.title_for("nightly", VULN)]["action"] == "close" + # Not run this time (and checks with no run script): no action either way. + assert ga.title_for("nightly", RECONCILE) not in acts + assert all(a["action"] == "close" for a in acts.values()) + + +def test_a_run_that_failed_outside_any_gate_opens_the_fallback_and_success_closes_it(tmp_path): + reg = _gates.load_gate_modes() + failed = _by_title(ga.plan(reg, "deploy", ["pr", "pre-apply"], _results(tmp_path, **{VULN: "pass"}), True)) + assert failed[ga.fallback_title("deploy")]["action"] == "open" + ok = _by_title(ga.plan(reg, "deploy", ["pr", "pre-apply"], _results(tmp_path), False)) + assert ok[ga.fallback_title("deploy")]["action"] == "close" + + +def test_each_workflow_keeps_its_own_issues(): + assert ga.title_for("deploy", VULN) != ga.title_for("nightly", VULN) + assert ga.fallback_title("deploy") != ga.fallback_title("nightly") + + +def test_dry_run_names_the_actions_and_calls_nothing(tmp_path, capsys, monkeypatch): + monkeypatch.setattr(ga, "_gh", lambda *a: (_ for _ in ()).throw(AssertionError("called gh"))) + assert ga.main(["nightly", str(_results(tmp_path, **{VULN: "fail"})), "nightly", "--run-failed", "true", "--dry-run"]) == 0 + assert f"open: {ga.title_for('nightly', VULN)}" in capsys.readouterr().out