Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 63 additions & 79 deletions .github/workflows/deploy-with-opa.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ jobs:
# direction: renaming a watched job would silently drop it from an
# inclusion list, whereas here it stays watched, and only a NEW dependent
# job could cause a false cancel — which is loud and gets noticed.
DEPENDENT='["Deploy Infrastructure","Notify on Failure","Queue watchdog"]'
DEPENDENT='["Deploy Infrastructure","Gate alerts","Queue watchdog"]'
stuck=''
for _ in $(seq 1 $(( GRACE_MINUTES * 2 ))); do
stuck=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \
Expand Down Expand Up @@ -236,8 +236,21 @@ jobs:
# host time) are banned, and every sprintf format must be a literal with
# no precision specifier.
- name: Gates for the pr stage (make gates STAGE=pr)
# Each gate's result goes to this file, for the alert job. Set on this
# step only: the unit tests above run the runner with their own inputs.
env:
GATE_RESULTS_FILE: /tmp/gate-results.jsonl
run: make gates STAGE=pr

- name: Upload the results the pr stage recorded, for the alert job
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: gate-results-pr
path: /tmp/gate-results.jsonl
if-no-files-found: ignore
retention-days: 7

# Real-bug lint (pyflakes + bugbear, not cosmetic style) and lenient type
# check. Config in pyproject.toml. Catches the class of latent bug the panel
# review flagged (e.g. the unhashable-dict crash) before it reaches a deploy.
Expand Down Expand Up @@ -609,8 +622,11 @@ jobs:
actions: read
# Detect-and-recover gate events (infrastructure/schemas/gate-modes.json) are
# appended here by scripts/_gates.py and published with this run's evidence.
# Each gate's result (scripts/run-gates.sh) goes to GATE_RESULTS_FILE, for
# the gate alerts job.
env:
GATE_EVENTS_FILE: /tmp/gate-events.jsonl
GATE_RESULTS_FILE: /tmp/gate-results.jsonl
needs: [test, compliance-check, security-scan, apps]
# IMPORTANT: Only deploy if on main branch AND security check passed
if: github.ref == 'refs/heads/main' && needs.compliance-check.outputs.compliant == 'true'
Expand Down Expand Up @@ -1884,6 +1900,15 @@ jobs:
});
}

- name: Upload the results the deploy stages recorded for the alert job
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: gate-results-deploy
path: /tmp/gate-results.jsonl
if-no-files-found: ignore
retention-days: 7

- name: Remove the trust-root state snapshot
if: always()
run: rm -f infrastructure/tfstate-bootstrap.json
Expand Down Expand Up @@ -1981,92 +2006,51 @@ jobs:
retention-days: 5

# ===========================================================================
# JOB 4: ALERT WHEN THINGS BREAK
# ===========================================================================
# Sends notifications and creates tickets when deployment fails
# JOB 4: ALERT WHEN A GATE FAILS
# ===========================================================================
# This job used to report SUCCESS while the deploy underneath it failed, and it
# did so twelve nights running. Two defects, both fixed here:
#
# 1. The issue-creating step was gated on `github.event_name == 'push'`. On a
# scheduled run the step was SKIPPED — and a job whose only step is skipped
# concludes `success`. So the run summary showed a green "Notify on Failure"
# next to a red deploy, and no issue was ever filed. The event gate is gone;
# a failed deploy alerts however it was triggered.
#
# 2. Even when it did fire, it opened a brand-new issue every time. The
# repository accumulated 15+ identical open "Deployment Failed" issues,
# which is functionally the same as no alert: nothing distinguishes the one
# that matters. It now reuses a single issue, reopening and commenting, and
# assigns it to the repository owner so it reaches a person.
# One issue per failing gate, labelled with its owners (design principle 9),
# from the results each gate recorded (scripts/gate-alerts.py). A failing gate
# opens or reopens its issue; a passing gate closes it, so it runs on every
# deploy run, not only failed ones. A run that failed outside any gate keeps
# one fallback issue. Every issue is reused, never duplicated: a new issue per
# failure once left 99 identical open issues, which is the same as no alert.
# Pull requests do not touch the issues: they describe production gates.
notify:
name: Notify on Failure
name: Gate alerts
runs-on: ubuntu-latest
needs: [compliance-check, apps, deploy]
if: always() && (needs.compliance-check.result == 'failure' || needs.apps.result == 'failure' || needs.deploy.result == 'failure')
needs: [test, compliance-check, apps, deploy]
if: always() && !cancelled() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
permissions:
contents: read
issues: write

steps:
- name: Raise or update the deployment-failure issue
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
# Job results reach the script through the environment rather than by
# `${{ }}` interpolation into the script body — templating workflow context
# straight into executable script is the habit that causes injection bugs.
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

# None to download (every gate job skipped) leaves every gate "not run":
# no issue changes, and a failed run still raises the fallback issue.
- name: Download the gate results
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: gate-results-*
path: gate-results

# Job results reach the step through the environment, not by `${{ }}`
# interpolation into the script.
- name: Open, update or close the gate alert issues
env:
GH_TOKEN: ${{ github.token }}
TEST_RESULT: ${{ needs.test.result }}
COMPLIANCE_RESULT: ${{ needs.compliance-check.result }}
APPS_RESULT: ${{ needs.apps.result }}
DEPLOY_RESULT: ${{ needs.deploy.result }}
with:
script: |
const TITLE = `Deployment failing: ${context.workflow}`;
const runUrl = `${context.payload.repository.html_url}/actions/runs/${context.runId}`;
const body = [
`The deploy pipeline failed. Until it succeeds, \`main\` is not reaching production.`,
``,
`| | |`,
`|---|---|`,
`| Run | [${context.runId}](${runUrl}) |`,
`| Trigger | \`${context.eventName}\` |`,
`| Commit | ${context.sha} |`,
`| Ref | ${context.ref} |`,
`| Compliance check | \`${process.env.COMPLIANCE_RESULT}\` |`,
`| Deploy | \`${process.env.DEPLOY_RESULT}\` |`,
``,
`Review the failing step in the run log, and the compliance reports it uploaded.`,
``,
`This issue is reused rather than duplicated — each failure adds a comment. Close it once a deploy succeeds.`,
].join('\n');

const existing = await github.paginate(github.rest.issues.listForRepo, {
owner: context.repo.owner,
repo: context.repo.repo,
state: 'all',
per_page: 100,
labels: 'deploy-failure',
});
const match = existing.find(i => i.title === TITLE && !i.pull_request);

if (match) {
if (match.state === 'closed') {
await github.rest.issues.update({
owner: context.repo.owner, repo: context.repo.repo,
issue_number: match.number, state: 'open',
});
}
await github.rest.issues.createComment({
owner: context.repo.owner, repo: context.repo.repo,
issue_number: match.number, body,
});
core.notice(`Updated issue #${match.number}`);
} else {
const created = await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: TITLE,
body,
labels: ['deployment', 'deploy-failure', 'urgent'],
assignees: [context.repo.owner],
});
core.notice(`Created issue #${created.data.number}`);
}
run: |
failed=false
for r in "$TEST_RESULT" "$COMPLIANCE_RESULT" "$APPS_RESULT" "$DEPLOY_RESULT"; do
[ "$r" = failure ] && failed=true
done
cat gate-results/*/gate-results.jsonl > /tmp/gate-results.jsonl 2>/dev/null || true
python3 scripts/gate-alerts.py deploy /tmp/gate-results.jsonl pr,pre-apply,post-apply,pre-publish \
--run-failed "$failed"
Loading
Loading