Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions docs/runbooks/evidence-sla-alarm.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ The evidence SLA watchdog (`infrastructure/watchdog.tf`) emails the operator thr
| `vdr-age` | the published VDR is more than 24h old | the reporting policy is breached; nightly runs have failed or stopped |
| `runtime-age` | the runtime signal is more than 26h old | the daily runtime-check Lambda has failed or stopped |
| `artifact-stale` | any artifact listed in `infrastructure/schemas/freshness-config.json` is past its window, missing, or has no readable timestamp | no deploy within the deploy-cadence window; the corroboration was not re-run; or one of the above |
| `failing-gates` | a nightly check in the gate register (`infrastructure/schemas/gate-modes.json`) with a run script did not pass in the beacon's `gate_results`, or is missing from it | a gate failed, including a detect-and-recover gate the run continued past; or the nightly did not reach it |

The windows are governed values in `infrastructure/schemas/freshness-config.json`; the thresholds above are read from it. Change a window there, through review, never in the alarm.

Expand Down Expand Up @@ -38,9 +39,16 @@ The watchdog's log line for the run lists the stale artifacts under `stale`. To
- **`boundary-corroboration.json`:** re-run `tools/boundary-map/corroborate.py`, commit the result, and deploy.
- **Nightly artifacts** (`vdr-*`): follow `nightly-failed` / `vdr-age` above.

## All four at once
## `failing-gates`

Check the watchdog: its log group `/aws/lambda/samaydlette-com-evidence-watchdog`, the dead-letter queue, and its EventBridge rule. Each run logs one JSON line with the values it published, the config version, and the stale artifacts.
The watchdog's log line lists each gate under `failing`, with its owner and its result (`fail`, `not run`, or `absent` from the beacon). The beacon's `gate_results` at `/.well-known/vdr-status.json` shows the same from outside.

- **`fail`:** the `[nightly] gate failing: <gate>` issue (label `gate-alert`) links the run; follow it as for `nightly-failed`.
- **`not run` or `absent`:** the nightly stopped before the gate, or the register lists a nightly check the runner does not report. `nightly-failed` usually fires too; start there.

## All of them at once

Check the watchdog: its log group `/aws/lambda/samaydlette-com-evidence-watchdog`, the dead-letter queue, and its EventBridge rule. Each run logs one JSON line with the values it published, the config and register versions, the stale artifacts and the failing gates.

## Setup and testing (operator)

Expand Down
16 changes: 14 additions & 2 deletions infrastructure/watchdog.tf
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
# (schemas/freshness-config.json: the VDR, the runtime signal, the
# nightly's status beacon and every other signed output under
# /.well-known/) straight from the site bucket
# and compares the beacon's gate results with the gate register
# (schemas/gate-modes.json)
# -> CloudWatch metrics (Samaydlette/Evidence)
# -> four alarms -> SNS topic (CMK-encrypted) -> operator email
# -> five alarms -> SNS topic (CMK-encrypted) -> operator email
#
# The freshness windows are governed values (REQ-PUB-2, REQ-GOV-1). They are
# read here with jsondecode(file(...)): the alarm thresholds, the objects the
Expand All @@ -30,7 +32,7 @@
# Terraform lands in state and in the uploaded plan artifact of a public repo.
# It is created once, out of band (docs/runbooks/evidence-sla-alarm.md).
#
# Cost: four standard alarms plus four custom metrics, roughly $1.60/month.
# Cost: five standard alarms plus five custom metrics, roughly $2.00/month.
# Every listed artifact feeds one StaleArtifacts count rather than a metric
# each, so the cost stays flat as the list grows.
# SNS email at this volume and 720 short invocations a month are within free
Expand Down Expand Up @@ -78,6 +80,11 @@ locals {
threshold = 0
summary = "At least one published /.well-known/ artifact is older than its governed freshness window, or unreadable. The watchdog's log names which."
}
"failing-gates" = {
metric = "FailingGates"
threshold = 0
summary = "At least one nightly gate did not pass in the latest run, including a detect-and-recover gate the run continued past. The gate-alert issue on GitHub and the watchdog's log name the gate and its owners."
}
}
}

Expand All @@ -100,6 +107,11 @@ data "archive_file" "evidence_watchdog" {
content = file("${path.module}/schemas/freshness-config.json")
filename = "freshness-config.json"
}
# The gate register, so the watchdog knows which nightly checks must report.
source {
content = file("${path.module}/schemas/gate-modes.json")
filename = "gate-modes.json"
}
}

resource "aws_iam_role" "evidence_watchdog" {
Expand Down
22 changes: 14 additions & 8 deletions infrastructure/watchdog/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// =============================================================================
// Measures, from outside GitHub, whether the published compliance evidence is
// fresh, and whether the unattended nightly refresh last succeeded. Runs hourly
// on an EventBridge schedule and publishes three CloudWatch metrics; alarms on
// on an EventBridge schedule and publishes five CloudWatch metrics; alarms on
// those metrics page the operator by email through SNS.
//
// Why it exists: from 2026-09-09 to 2026-09-24 the nightly failed every night
Expand All @@ -25,9 +25,12 @@
// missing, or is too old to trust; else 0
// StaleArtifacts how many listed artifacts are past their governed
// window, unreadable, or without a usable timestamp
// FailingGates how many nightly checks in the gate register
// (gate-modes.json, packaged beside this file) with a
// run script did not pass in the beacon's gate_results
//
// A missing or invalid config fails the run before anything is published; the
// alarms read the missing data as a breach and page.
// A missing or invalid config or register fails the run before anything is
// published; the alarms read the missing data as a breach and page.
//
// A read or parse failure is reported as a breach (UNREADABLE_AGE_HOURS / 1),
// never skipped: an artifact the watchdog cannot read is not evidence that it
Expand All @@ -38,9 +41,10 @@
import { readFile } from 'node:fs/promises';
import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3';
import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch';
import { KEYS, computeMetrics, metricData, parseConfig } from './logic.mjs';
import { KEYS, computeMetrics, metricData, parseConfig, parseRegister } from './logic.mjs';

const CONFIG_URL = new URL('./freshness-config.json', import.meta.url);
const REGISTER_URL = new URL('./gate-modes.json', import.meta.url);

async function read(s3, bucket, key) {
try {
Expand All @@ -59,6 +63,7 @@ export async function handler(_event, _context, deps = {}) {

// Fail closed: no valid governed windows, no metrics (and so, a page).
const config = parseConfig(JSON.parse(await (deps.readConfig || (() => readFile(CONFIG_URL, 'utf8')))()));
const register = parseRegister(JSON.parse(await (deps.readRegister || (() => readFile(REGISTER_URL, 'utf8')))()));

const s3 = deps.s3 || new S3Client({});
const cw = deps.cw || new CloudWatchClient({});
Expand All @@ -67,18 +72,19 @@ export async function handler(_event, _context, deps = {}) {
const paths = config.artifacts.map((a) => a.path);
const bodies = await Promise.all(paths.map((p) => read(s3, bucket, p)));
const readings = Object.fromEntries(paths.map((p, i) => [p, bodies[i]]));
const { metrics, stale } = computeMetrics(config, readings, now);
const { metrics, stale, failing } = computeMetrics(config, readings, now, register);

await cw.send(new PutMetricDataCommand({
Namespace: namespace,
MetricData: metricData(metrics, now),
}));

// One structured line per run: the numbers the alarms act on, and which
// artifacts are stale, for the log.
// One structured line per run: the numbers the alarms act on, which
// artifacts are stale and which gates are failing (with their owners), for the log.
let beacon = null;
try { beacon = JSON.parse(readings[KEYS.status]?.body); } catch { beacon = null; }
console.log(JSON.stringify({ msg: 'evidence watchdog', config_version: config.version, ...metrics, stale,
console.log(JSON.stringify({ msg: 'evidence watchdog', config_version: config.version,
gates_version: register.version, ...metrics, stale, failing,
nightly: beacon ? { result: beacon.result, finished_at: beacon.finished_at } : null }));
return metrics;
}
54 changes: 44 additions & 10 deletions infrastructure/watchdog/logic.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -110,25 +110,59 @@ export function staleArtifacts(config, readings, now = Date.now()) {
return stale;
}

export function computeMetrics(config, readings, now = Date.now()) {
export function computeMetrics(config, readings, now = Date.now(), register = null) {
const age = (path) => {
const a = config.artifacts.find((x) => x.path === path);
const h = ageHours(timestampOf(a, readings[path]), now);
return h === null ? UNREADABLE_AGE_HOURS : h;
};
const stale = staleArtifacts(config, readings, now);
return {
metrics: {
VdrAgeHours: age(KEYS.vdr),
RuntimeSignalAgeHours: age(KEYS.runtime),
NightlyFailed: nightlyFailed(parsedBody(readings[KEYS.status]), config.windows[KEYS.status], now),
StaleArtifacts: stale.length,
},
stale,
const status = parsedBody(readings[KEYS.status]);
const failing = register ? failingGates(register, status) : [];
const metrics = {
VdrAgeHours: age(KEYS.vdr),
RuntimeSignalAgeHours: age(KEYS.runtime),
NightlyFailed: nightlyFailed(status, config.windows[KEYS.status], now),
StaleArtifacts: stale.length,
};
if (register) metrics.FailingGates = failing.length;
return { metrics, stale, failing };
}

const COUNTS = new Set(['NightlyFailed', 'StaleArtifacts']);
// The gate register (infrastructure/schemas/gate-modes.json, packaged beside
// this file as gate-modes.json). scripts/_gates.py is its full validator; this
// checks only the fields the watchdog reads, and throws on anything else, so a
// watchdog without a usable register publishes nothing and the alarms page.
export function parseRegister(reg) {
const bad = (msg) => { throw new Error(`gate register invalid: ${msg}`); };
if (!reg || typeof reg !== 'object' || Array.isArray(reg)) bad('not an object');
if (!nonEmpty(reg.version)) bad('version must be a non-empty string');
if (!Array.isArray(reg.checks) || reg.checks.length === 0) bad('checks must be a non-empty list');
for (const c of reg.checks) {
if (!c || typeof c !== 'object' || !nonEmpty(c.check)) bad('a check has no name');
if (!Array.isArray(c.stages) || !c.stages.every(nonEmpty)) bad(`${c.check}: stages must be a list`);
if (!nonEmpty(c.owner)) bad(`${c.check}: owner must be a role`);
if ('run' in c && !nonEmpty(c.run)) bad(`${c.check}: run must be a non-empty string`);
}
return reg;
}

// Every nightly check with a run script whose result in the beacon is not a
// pass: failed, not run, or absent. A gate in detect-and-recover mode can fail
// while the nightly still succeeds, so NightlyFailed alone does not page on it.
// A beacon without gate results says nothing passed: every such check counts.
// Checks with no run script cannot report, and are not counted.
export function failingGates(register, status) {
const results = new Map();
for (const r of (status && Array.isArray(status.gate_results)) ? status.gate_results : []) {
if (r && typeof r === 'object' && typeof r.check === 'string') results.set(r.check, r.result);
}
return register.checks
.filter((c) => c.stages.includes('nightly') && 'run' in c && results.get(c.check) !== 'pass')
.map((c) => ({ check: c.check, owner: c.owner, result: results.get(c.check) ?? 'absent' }));
}

const COUNTS = new Set(['NightlyFailed', 'StaleArtifacts', 'FailingGates']);

export function metricData(metrics, now) {
return Object.entries(metrics).map(([MetricName, Value]) => ({
Expand Down
118 changes: 111 additions & 7 deletions tests/test_evidence_watchdog.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,31 @@ def metrics(vdr=None, runtime=None, status=None):
return compute(objects)["metrics"]


def status(result="success", finished_at="2026-09-25T06:00:00Z"):
return {"result": result, "finished_at": finished_at, "vuln_gate": "success", "reconcile": "success"}
REGISTER = REPO / "infrastructure" / "schemas" / "gate-modes.json"


def register():
return json.loads(REGISTER.read_text())


def nightly_runnable():
return [c for c in register()["checks"] if "nightly" in c["stages"] and "run" in c]


def gate_results(**override):
"""Every nightly check as the beacon reports it: pass, or not on the runner."""
out = []
for c in register()["checks"]:
if "nightly" in c["stages"]:
res = "pass" if "run" in c else "not on the runner"
out.append({"check": c["check"], "gate": c["gate"], "owner": c["owner"],
"result": override.get(c["check"], res)})
return out


def status(result="success", finished_at="2026-09-25T06:00:00Z", results=None):
return {"result": result, "finished_at": finished_at, "vuln_gate": "success", "reconcile": "success",
"gate_results": gate_results() if results is None else results}


def test_fresh_evidence_and_a_passing_nightly_page_nobody():
Expand Down Expand Up @@ -190,14 +213,16 @@ def test_the_watchdog_rejects_an_invalid_config():
assert _parse(cfg)["ok"] is False, cfg


def _fake_sdk(tmp_path, objects, cfg=None):
def _fake_sdk(tmp_path, objects, cfg=None, reg=None):
"""Copy the watchdog next to stand-in @aws-sdk packages that serve `objects`
from S3 and record every PutMetricData call. The governed config is
packaged beside the handler, as watchdog.tf packages it."""
from S3 and record every PutMetricData call. The governed config and the
gate register are packaged beside the handler, as watchdog.tf packages them."""
for f in ("index.mjs", "logic.mjs"):
shutil.copy(WATCHDOG / f, tmp_path / f)
if cfg is not False:
(tmp_path / "freshness-config.json").write_text(json.dumps(cfg or config()))
if reg is not False:
(tmp_path / "gate-modes.json").write_text(json.dumps(reg or register()))
s3 = tmp_path / "node_modules" / "@aws-sdk" / "client-s3"
cw = tmp_path / "node_modules" / "@aws-sdk" / "client-cloudwatch"
for d in (s3, cw):
Expand All @@ -224,8 +249,8 @@ def _fake_sdk(tmp_path, objects, cfg=None):
"""))


def run_handler(tmp_path, objects, cfg=None):
_fake_sdk(tmp_path, objects, cfg)
def run_handler(tmp_path, objects, cfg=None, reg=None):
_fake_sdk(tmp_path, objects, cfg, reg)
script = textwrap.dedent(f"""
process.env.S3_BUCKET = 'site-bucket';
process.env.METRIC_NAMESPACE = 'Samaydlette/Evidence';
Expand Down Expand Up @@ -287,3 +312,82 @@ def test_alarm_thresholds_and_reads_come_from_the_governed_config():
# No window is a literal in the watchdog any more.
assert "MAX_BEACON_AGE_HOURS" not in logic
assert not re.search(r"threshold\s*=\s*(24|26)\b", tf)


def gates(beacon, reg=None):
script = textwrap.dedent(f"""
import {{ failingGates, parseRegister }} from '{WATCHDOG / "logic.mjs"}';
console.log(JSON.stringify(failingGates(parseRegister({json.dumps(reg or register())}), {json.dumps(beacon)})));
""")
return node(script, REPO)


def test_every_nightly_gate_passing_counts_none_failing():
assert len(nightly_runnable()) > 1
assert gates(status()) == []


def test_a_failed_a_not_run_and_an_absent_gate_each_count():
a, b, c = (x["check"] for x in nightly_runnable()[:3])
results = [r for r in gate_results(**{a: "fail", b: "not run"}) if r["check"] != c]
out = gates(status(results=results))
assert {(x["check"], x["result"]) for x in out} == {(a, "fail"), (b, "not run"), (c, "absent")}
owners = {x["check"]: x["owner"] for x in register()["checks"]}
assert all(x["owner"] == owners[x["check"]] for x in out)


def test_a_gate_failing_inside_a_successful_run_still_counts():
# The case NightlyFailed cannot see: a detect-and-recover gate fails and the run continues.
a = nightly_runnable()[0]["check"]
beacon = status("success", results=gate_results(**{a: "fail"}))
assert metrics(status=beacon)["NightlyFailed"] == 0
assert [x["check"] for x in gates(beacon)] == [a]


def test_checks_not_on_the_runner_are_not_counted():
off = [c["check"] for c in register()["checks"] if "nightly" in c["stages"] and "run" not in c]
assert off, "the test needs a nightly check with no run script"
assert not {x["check"] for x in gates(status())} & set(off)


def test_no_beacon_or_no_gate_results_counts_every_runnable_gate():
n = len(nightly_runnable())
assert len(gates(None)) == n
assert len(gates({"result": "success"})) == n


def test_the_watchdog_rejects_an_invalid_register():
for mutate in (lambda r: r.pop("version"), lambda r: r.update(checks=[]),
lambda r: r["checks"][0].pop("stages"), lambda r: r["checks"][0].update(run="")):
reg = register()
mutate(reg)
script = textwrap.dedent(f"""
import {{ parseRegister }} from '{WATCHDOG / "logic.mjs"}';
try {{ parseRegister({json.dumps(reg)}); console.log('{{"ok": true}}'); }}
catch (e) {{ console.log('{{"ok": false}}'); }}
""")
assert node(script, REPO)["ok"] is False, reg


def test_handler_publishes_failing_gates_and_names_them_in_the_log(tmp_path):
a = nightly_runnable()[0]
objects = fresh_objects()
objects[".well-known/vdr-status.json"] = status(results=gate_results(**{a["check"]: "fail"}))
r = run_handler(tmp_path, objects)
names = {d["MetricName"]: d["Value"] for d in r["puts"][0]["MetricData"]}
assert names["FailingGates"] == 1
line = json.loads(r["logs"][-1])
assert line["failing"] == [{"check": a["check"], "owner": a["owner"], "result": "fail"}]
assert line["gates_version"] == register()["version"]


@pytest.mark.parametrize("reg", [False, "bad"])
def test_handler_with_a_missing_or_invalid_register_publishes_nothing(tmp_path, reg):
r = run_handler(tmp_path, fresh_objects(), reg={"version": ""} if reg == "bad" else False)
assert r["error"] and r["puts"] == [] and r["reads"] == []


def test_the_failing_gates_alarm_and_the_packaged_register():
tf = (REPO / "infrastructure" / "watchdog.tf").read_text()
assert 'metric = "FailingGates"\n threshold = 0' in tf
assert 'content = file("${path.module}/schemas/gate-modes.json")\n filename = "gate-modes.json"' in tf
Loading