diff --git a/docs/runbooks/evidence-sla-alarm.md b/docs/runbooks/evidence-sla-alarm.md index 1b7d9b8c..700195b2 100644 --- a/docs/runbooks/evidence-sla-alarm.md +++ b/docs/runbooks/evidence-sla-alarm.md @@ -8,6 +8,7 @@ The evidence SLA watchdog (`infrastructure/watchdog.tf`) emails the operator thr | `vdr-age` | the published VDR is more than 24h old | the reporting policy is breached; nightly runs have failed or stopped | | `runtime-age` | the runtime signal is more than 26h old | the daily runtime-check Lambda has failed or stopped | | `artifact-stale` | any artifact listed in `infrastructure/schemas/freshness-config.json` is past its window, missing, or has no readable timestamp | no deploy within the deploy-cadence window; the corroboration was not re-run; or one of the above | +| `failing-gates` | a nightly check in the gate register (`infrastructure/schemas/gate-modes.json`) with a run script did not pass in the beacon's `gate_results`, or is missing from it | a gate failed, including a detect-and-recover gate the run continued past; or the nightly did not reach it | The windows are governed values in `infrastructure/schemas/freshness-config.json`; the thresholds above are read from it. Change a window there, through review, never in the alarm. @@ -38,9 +39,16 @@ The watchdog's log line for the run lists the stale artifacts under `stale`. To - **`boundary-corroboration.json`:** re-run `tools/boundary-map/corroborate.py`, commit the result, and deploy. - **Nightly artifacts** (`vdr-*`): follow `nightly-failed` / `vdr-age` above. -## All four at once +## `failing-gates` -Check the watchdog: its log group `/aws/lambda/samaydlette-com-evidence-watchdog`, the dead-letter queue, and its EventBridge rule. Each run logs one JSON line with the values it published, the config version, and the stale artifacts. +The watchdog's log line lists each gate under `failing`, with its owner and its result (`fail`, `not run`, or `absent` from the beacon). The beacon's `gate_results` at `/.well-known/vdr-status.json` shows the same from outside. + +- **`fail`:** the `[nightly] gate failing: ` issue (label `gate-alert`) links the run; follow it as for `nightly-failed`. +- **`not run` or `absent`:** the nightly stopped before the gate, or the register lists a nightly check the runner does not report. `nightly-failed` usually fires too; start there. + +## All of them at once + +Check the watchdog: its log group `/aws/lambda/samaydlette-com-evidence-watchdog`, the dead-letter queue, and its EventBridge rule. Each run logs one JSON line with the values it published, the config and register versions, the stale artifacts and the failing gates. ## Setup and testing (operator) diff --git a/infrastructure/watchdog.tf b/infrastructure/watchdog.tf index 16f1777f..98a3268e 100644 --- a/infrastructure/watchdog.tf +++ b/infrastructure/watchdog.tf @@ -13,8 +13,10 @@ # (schemas/freshness-config.json: the VDR, the runtime signal, the # nightly's status beacon and every other signed output under # /.well-known/) straight from the site bucket +# and compares the beacon's gate results with the gate register +# (schemas/gate-modes.json) # -> CloudWatch metrics (Samaydlette/Evidence) -# -> four alarms -> SNS topic (CMK-encrypted) -> operator email +# -> five alarms -> SNS topic (CMK-encrypted) -> operator email # # The freshness windows are governed values (REQ-PUB-2, REQ-GOV-1). They are # read here with jsondecode(file(...)): the alarm thresholds, the objects the @@ -30,7 +32,7 @@ # Terraform lands in state and in the uploaded plan artifact of a public repo. # It is created once, out of band (docs/runbooks/evidence-sla-alarm.md). # -# Cost: four standard alarms plus four custom metrics, roughly $1.60/month. +# Cost: five standard alarms plus five custom metrics, roughly $2.00/month. # Every listed artifact feeds one StaleArtifacts count rather than a metric # each, so the cost stays flat as the list grows. # SNS email at this volume and 720 short invocations a month are within free @@ -78,6 +80,11 @@ locals { threshold = 0 summary = "At least one published /.well-known/ artifact is older than its governed freshness window, or unreadable. The watchdog's log names which." } + "failing-gates" = { + metric = "FailingGates" + threshold = 0 + summary = "At least one nightly gate did not pass in the latest run, including a detect-and-recover gate the run continued past. The gate-alert issue on GitHub and the watchdog's log name the gate and its owners." + } } } @@ -100,6 +107,11 @@ data "archive_file" "evidence_watchdog" { content = file("${path.module}/schemas/freshness-config.json") filename = "freshness-config.json" } + # The gate register, so the watchdog knows which nightly checks must report. + source { + content = file("${path.module}/schemas/gate-modes.json") + filename = "gate-modes.json" + } } resource "aws_iam_role" "evidence_watchdog" { diff --git a/infrastructure/watchdog/index.mjs b/infrastructure/watchdog/index.mjs index dd1b50af..e3f860d2 100644 --- a/infrastructure/watchdog/index.mjs +++ b/infrastructure/watchdog/index.mjs @@ -3,7 +3,7 @@ // ============================================================================= // Measures, from outside GitHub, whether the published compliance evidence is // fresh, and whether the unattended nightly refresh last succeeded. Runs hourly -// on an EventBridge schedule and publishes three CloudWatch metrics; alarms on +// on an EventBridge schedule and publishes five CloudWatch metrics; alarms on // those metrics page the operator by email through SNS. // // Why it exists: from 2026-09-09 to 2026-09-24 the nightly failed every night @@ -25,9 +25,12 @@ // missing, or is too old to trust; else 0 // StaleArtifacts how many listed artifacts are past their governed // window, unreadable, or without a usable timestamp +// FailingGates how many nightly checks in the gate register +// (gate-modes.json, packaged beside this file) with a +// run script did not pass in the beacon's gate_results // -// A missing or invalid config fails the run before anything is published; the -// alarms read the missing data as a breach and page. +// A missing or invalid config or register fails the run before anything is +// published; the alarms read the missing data as a breach and page. // // A read or parse failure is reported as a breach (UNREADABLE_AGE_HOURS / 1), // never skipped: an artifact the watchdog cannot read is not evidence that it @@ -38,9 +41,10 @@ import { readFile } from 'node:fs/promises'; import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3'; import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch'; -import { KEYS, computeMetrics, metricData, parseConfig } from './logic.mjs'; +import { KEYS, computeMetrics, metricData, parseConfig, parseRegister } from './logic.mjs'; const CONFIG_URL = new URL('./freshness-config.json', import.meta.url); +const REGISTER_URL = new URL('./gate-modes.json', import.meta.url); async function read(s3, bucket, key) { try { @@ -59,6 +63,7 @@ export async function handler(_event, _context, deps = {}) { // Fail closed: no valid governed windows, no metrics (and so, a page). const config = parseConfig(JSON.parse(await (deps.readConfig || (() => readFile(CONFIG_URL, 'utf8')))())); + const register = parseRegister(JSON.parse(await (deps.readRegister || (() => readFile(REGISTER_URL, 'utf8')))())); const s3 = deps.s3 || new S3Client({}); const cw = deps.cw || new CloudWatchClient({}); @@ -67,18 +72,19 @@ export async function handler(_event, _context, deps = {}) { const paths = config.artifacts.map((a) => a.path); const bodies = await Promise.all(paths.map((p) => read(s3, bucket, p))); const readings = Object.fromEntries(paths.map((p, i) => [p, bodies[i]])); - const { metrics, stale } = computeMetrics(config, readings, now); + const { metrics, stale, failing } = computeMetrics(config, readings, now, register); await cw.send(new PutMetricDataCommand({ Namespace: namespace, MetricData: metricData(metrics, now), })); - // One structured line per run: the numbers the alarms act on, and which - // artifacts are stale, for the log. + // One structured line per run: the numbers the alarms act on, which + // artifacts are stale and which gates are failing (with their owners), for the log. let beacon = null; try { beacon = JSON.parse(readings[KEYS.status]?.body); } catch { beacon = null; } - console.log(JSON.stringify({ msg: 'evidence watchdog', config_version: config.version, ...metrics, stale, + console.log(JSON.stringify({ msg: 'evidence watchdog', config_version: config.version, + gates_version: register.version, ...metrics, stale, failing, nightly: beacon ? { result: beacon.result, finished_at: beacon.finished_at } : null })); return metrics; } diff --git a/infrastructure/watchdog/logic.mjs b/infrastructure/watchdog/logic.mjs index 97692b70..a2102024 100644 --- a/infrastructure/watchdog/logic.mjs +++ b/infrastructure/watchdog/logic.mjs @@ -110,25 +110,59 @@ export function staleArtifacts(config, readings, now = Date.now()) { return stale; } -export function computeMetrics(config, readings, now = Date.now()) { +export function computeMetrics(config, readings, now = Date.now(), register = null) { const age = (path) => { const a = config.artifacts.find((x) => x.path === path); const h = ageHours(timestampOf(a, readings[path]), now); return h === null ? UNREADABLE_AGE_HOURS : h; }; const stale = staleArtifacts(config, readings, now); - return { - metrics: { - VdrAgeHours: age(KEYS.vdr), - RuntimeSignalAgeHours: age(KEYS.runtime), - NightlyFailed: nightlyFailed(parsedBody(readings[KEYS.status]), config.windows[KEYS.status], now), - StaleArtifacts: stale.length, - }, - stale, + const status = parsedBody(readings[KEYS.status]); + const failing = register ? failingGates(register, status) : []; + const metrics = { + VdrAgeHours: age(KEYS.vdr), + RuntimeSignalAgeHours: age(KEYS.runtime), + NightlyFailed: nightlyFailed(status, config.windows[KEYS.status], now), + StaleArtifacts: stale.length, }; + if (register) metrics.FailingGates = failing.length; + return { metrics, stale, failing }; } -const COUNTS = new Set(['NightlyFailed', 'StaleArtifacts']); +// The gate register (infrastructure/schemas/gate-modes.json, packaged beside +// this file as gate-modes.json). scripts/_gates.py is its full validator; this +// checks only the fields the watchdog reads, and throws on anything else, so a +// watchdog without a usable register publishes nothing and the alarms page. +export function parseRegister(reg) { + const bad = (msg) => { throw new Error(`gate register invalid: ${msg}`); }; + if (!reg || typeof reg !== 'object' || Array.isArray(reg)) bad('not an object'); + if (!nonEmpty(reg.version)) bad('version must be a non-empty string'); + if (!Array.isArray(reg.checks) || reg.checks.length === 0) bad('checks must be a non-empty list'); + for (const c of reg.checks) { + if (!c || typeof c !== 'object' || !nonEmpty(c.check)) bad('a check has no name'); + if (!Array.isArray(c.stages) || !c.stages.every(nonEmpty)) bad(`${c.check}: stages must be a list`); + if (!nonEmpty(c.owner)) bad(`${c.check}: owner must be a role`); + if ('run' in c && !nonEmpty(c.run)) bad(`${c.check}: run must be a non-empty string`); + } + return reg; +} + +// Every nightly check with a run script whose result in the beacon is not a +// pass: failed, not run, or absent. A gate in detect-and-recover mode can fail +// while the nightly still succeeds, so NightlyFailed alone does not page on it. +// A beacon without gate results says nothing passed: every such check counts. +// Checks with no run script cannot report, and are not counted. +export function failingGates(register, status) { + const results = new Map(); + for (const r of (status && Array.isArray(status.gate_results)) ? status.gate_results : []) { + if (r && typeof r === 'object' && typeof r.check === 'string') results.set(r.check, r.result); + } + return register.checks + .filter((c) => c.stages.includes('nightly') && 'run' in c && results.get(c.check) !== 'pass') + .map((c) => ({ check: c.check, owner: c.owner, result: results.get(c.check) ?? 'absent' })); +} + +const COUNTS = new Set(['NightlyFailed', 'StaleArtifacts', 'FailingGates']); export function metricData(metrics, now) { return Object.entries(metrics).map(([MetricName, Value]) => ({ diff --git a/tests/test_evidence_watchdog.py b/tests/test_evidence_watchdog.py index 0bdd0e27..d6904a92 100644 --- a/tests/test_evidence_watchdog.py +++ b/tests/test_evidence_watchdog.py @@ -84,8 +84,31 @@ def metrics(vdr=None, runtime=None, status=None): return compute(objects)["metrics"] -def status(result="success", finished_at="2026-09-25T06:00:00Z"): - return {"result": result, "finished_at": finished_at, "vuln_gate": "success", "reconcile": "success"} +REGISTER = REPO / "infrastructure" / "schemas" / "gate-modes.json" + + +def register(): + return json.loads(REGISTER.read_text()) + + +def nightly_runnable(): + return [c for c in register()["checks"] if "nightly" in c["stages"] and "run" in c] + + +def gate_results(**override): + """Every nightly check as the beacon reports it: pass, or not on the runner.""" + out = [] + for c in register()["checks"]: + if "nightly" in c["stages"]: + res = "pass" if "run" in c else "not on the runner" + out.append({"check": c["check"], "gate": c["gate"], "owner": c["owner"], + "result": override.get(c["check"], res)}) + return out + + +def status(result="success", finished_at="2026-09-25T06:00:00Z", results=None): + return {"result": result, "finished_at": finished_at, "vuln_gate": "success", "reconcile": "success", + "gate_results": gate_results() if results is None else results} def test_fresh_evidence_and_a_passing_nightly_page_nobody(): @@ -190,14 +213,16 @@ def test_the_watchdog_rejects_an_invalid_config(): assert _parse(cfg)["ok"] is False, cfg -def _fake_sdk(tmp_path, objects, cfg=None): +def _fake_sdk(tmp_path, objects, cfg=None, reg=None): """Copy the watchdog next to stand-in @aws-sdk packages that serve `objects` - from S3 and record every PutMetricData call. The governed config is - packaged beside the handler, as watchdog.tf packages it.""" + from S3 and record every PutMetricData call. The governed config and the + gate register are packaged beside the handler, as watchdog.tf packages them.""" for f in ("index.mjs", "logic.mjs"): shutil.copy(WATCHDOG / f, tmp_path / f) if cfg is not False: (tmp_path / "freshness-config.json").write_text(json.dumps(cfg or config())) + if reg is not False: + (tmp_path / "gate-modes.json").write_text(json.dumps(reg or register())) s3 = tmp_path / "node_modules" / "@aws-sdk" / "client-s3" cw = tmp_path / "node_modules" / "@aws-sdk" / "client-cloudwatch" for d in (s3, cw): @@ -224,8 +249,8 @@ def _fake_sdk(tmp_path, objects, cfg=None): """)) -def run_handler(tmp_path, objects, cfg=None): - _fake_sdk(tmp_path, objects, cfg) +def run_handler(tmp_path, objects, cfg=None, reg=None): + _fake_sdk(tmp_path, objects, cfg, reg) script = textwrap.dedent(f""" process.env.S3_BUCKET = 'site-bucket'; process.env.METRIC_NAMESPACE = 'Samaydlette/Evidence'; @@ -287,3 +312,82 @@ def test_alarm_thresholds_and_reads_come_from_the_governed_config(): # No window is a literal in the watchdog any more. assert "MAX_BEACON_AGE_HOURS" not in logic assert not re.search(r"threshold\s*=\s*(24|26)\b", tf) + + +def gates(beacon, reg=None): + script = textwrap.dedent(f""" + import {{ failingGates, parseRegister }} from '{WATCHDOG / "logic.mjs"}'; + console.log(JSON.stringify(failingGates(parseRegister({json.dumps(reg or register())}), {json.dumps(beacon)}))); + """) + return node(script, REPO) + + +def test_every_nightly_gate_passing_counts_none_failing(): + assert len(nightly_runnable()) > 1 + assert gates(status()) == [] + + +def test_a_failed_a_not_run_and_an_absent_gate_each_count(): + a, b, c = (x["check"] for x in nightly_runnable()[:3]) + results = [r for r in gate_results(**{a: "fail", b: "not run"}) if r["check"] != c] + out = gates(status(results=results)) + assert {(x["check"], x["result"]) for x in out} == {(a, "fail"), (b, "not run"), (c, "absent")} + owners = {x["check"]: x["owner"] for x in register()["checks"]} + assert all(x["owner"] == owners[x["check"]] for x in out) + + +def test_a_gate_failing_inside_a_successful_run_still_counts(): + # The case NightlyFailed cannot see: a detect-and-recover gate fails and the run continues. + a = nightly_runnable()[0]["check"] + beacon = status("success", results=gate_results(**{a: "fail"})) + assert metrics(status=beacon)["NightlyFailed"] == 0 + assert [x["check"] for x in gates(beacon)] == [a] + + +def test_checks_not_on_the_runner_are_not_counted(): + off = [c["check"] for c in register()["checks"] if "nightly" in c["stages"] and "run" not in c] + assert off, "the test needs a nightly check with no run script" + assert not {x["check"] for x in gates(status())} & set(off) + + +def test_no_beacon_or_no_gate_results_counts_every_runnable_gate(): + n = len(nightly_runnable()) + assert len(gates(None)) == n + assert len(gates({"result": "success"})) == n + + +def test_the_watchdog_rejects_an_invalid_register(): + for mutate in (lambda r: r.pop("version"), lambda r: r.update(checks=[]), + lambda r: r["checks"][0].pop("stages"), lambda r: r["checks"][0].update(run="")): + reg = register() + mutate(reg) + script = textwrap.dedent(f""" + import {{ parseRegister }} from '{WATCHDOG / "logic.mjs"}'; + try {{ parseRegister({json.dumps(reg)}); console.log('{{"ok": true}}'); }} + catch (e) {{ console.log('{{"ok": false}}'); }} + """) + assert node(script, REPO)["ok"] is False, reg + + +def test_handler_publishes_failing_gates_and_names_them_in_the_log(tmp_path): + a = nightly_runnable()[0] + objects = fresh_objects() + objects[".well-known/vdr-status.json"] = status(results=gate_results(**{a["check"]: "fail"})) + r = run_handler(tmp_path, objects) + names = {d["MetricName"]: d["Value"] for d in r["puts"][0]["MetricData"]} + assert names["FailingGates"] == 1 + line = json.loads(r["logs"][-1]) + assert line["failing"] == [{"check": a["check"], "owner": a["owner"], "result": "fail"}] + assert line["gates_version"] == register()["version"] + + +@pytest.mark.parametrize("reg", [False, "bad"]) +def test_handler_with_a_missing_or_invalid_register_publishes_nothing(tmp_path, reg): + r = run_handler(tmp_path, fresh_objects(), reg={"version": ""} if reg == "bad" else False) + assert r["error"] and r["puts"] == [] and r["reads"] == [] + + +def test_the_failing_gates_alarm_and_the_packaged_register(): + tf = (REPO / "infrastructure" / "watchdog.tf").read_text() + assert 'metric = "FailingGates"\n threshold = 0' in tf + assert 'content = file("${path.module}/schemas/gate-modes.json")\n filename = "gate-modes.json"' in tf