From 139cd010422f746c7f5f1f4201b58f9e8ebc618f Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 13:32:52 -0500 Subject: [PATCH 01/97] Rule records: one per applicable FedRAMP rule, none claiming more than is shown data/frr-records.json holds a record for each of the 154 FedRAMP rules that apply to a 20x Program Class C provider, derived from the pinned corpus's own subset applicability metadata (scripts/_rules.applicable_provider_rules). scripts/_frr.py fails closed on a missing or extra record, an Implemented record without evidence, evidence that does not resolve (test, workflow step, freshness-governed artifact, in-force decision), a non-MAY rule left optional, or published text carrying an account ID or a foreign email. Records are AI-drafted from the 2026-11-05 assessment, corrected for W0/W1 and #496, for the operator's review. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 1 + data/frr-records.json | 1629 +++++++++++++++++ .../schemas/frr-records.schema.json | 53 + scripts/_frr.py | 261 +++ scripts/_rules.py | 58 + tests/fixtures/frr/known-bad.json | 24 + tests/test_frr_records.py | 196 ++ 8 files changed, 2223 insertions(+), 1 deletion(-) create mode 100644 data/frr-records.json create mode 100644 infrastructure/schemas/frr-records.schema.json create mode 100644 scripts/_frr.py create mode 100644 tests/fixtures/frr/known-bad.json create mode 100644 tests/test_frr_records.py diff --git a/CLAUDE.md b/CLAUDE.md index a7ccabb9..f0a24b76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ Hold two facts at once — they shape every decision: Confirm against the working tree on first run; correct this section if reality differs. - `infrastructure/` — Terraform (`main.tf`), OPA policy packages (`policy/`), JSON schemas (`schemas/ksi-signal.schema.json`, `schemas/ksi-catalog.json`), runtime Lambda (`lambda/index.js`). -- `scripts/` — the artifact **generators**: `build-ksi-signal.py` (the canonical inventory), the VDR builder, and the CR26 record builders (`build-sdr.py`, `build-cpo.py`, and `build-cr26-vdr.py`, which turns the VDR report into FedRAMP's VDT, AVI and history reports). CR26 reports are validated against FedRAMP's 2026-06-24 JSON schemas, vendored verbatim with their sha256s in `infrastructure/schemas/fedramp-2026-06-24/`, by `scripts/_fedramp_schema.py` before they are written; a report that does not validate is never written. Never edit a vendored schema; the loader refuses any file whose hash differs from `SOURCES.json`. The per-control view (status, origination, statement per NIST 800-53 control) is the control projection in `scripts/_controls.py`, computed in memory from the inventory and the KSI catalog; the OSCAL SSP that used to carry it was retired under CR26 (2026-11-05) and the SDR's `rev5_controls` publishes it. Every human decision about a finding is a record in `data/decisions.json` (the decision register, which replaced the POA&M under CR26), read through `scripts/_decisions.py` by the VDR builder, the vulnerability gate, reconciliation and the trust center; every in-force record carries `decided_by`, `decided_on` and an annual `review_by`, which the loader and `tests/test_decision_records.py` enforce. These are the source of truth for everything published. `scripts/_common.py` holds the helpers they share (`classify` — the inheritance rule four artifacts must agree on — plus `sha256_file`, `prop`, `load_hub`, `sid`); it is imported via an `__file__`-relative `sys.path.insert` so every script stays standalone-runnable from any cwd. **`scripts/staged/` is not wired into anything** — parked generators, see its README; never assume a script there runs. +- `scripts/` — the artifact **generators**: `build-ksi-signal.py` (the canonical inventory), the VDR builder, and the CR26 record builders (`build-sdr.py`, `build-cpo.py`, and `build-cr26-vdr.py`, which turns the VDR report into FedRAMP's VDT, AVI and history reports). CR26 reports are validated against FedRAMP's 2026-06-24 JSON schemas, vendored verbatim with their sha256s in `infrastructure/schemas/fedramp-2026-06-24/`, by `scripts/_fedramp_schema.py` before they are written; a report that does not validate is never written. Never edit a vendored schema; the loader refuses any file whose hash differs from `SOURCES.json`. The per-control view (status, origination, statement per NIST 800-53 control) is the control projection in `scripts/_controls.py`, computed in memory from the inventory and the KSI catalog; the OSCAL SSP that used to carry it was retired under CR26 (2026-11-05) and the SDR's `rev5_controls` publishes it. Every human decision about a finding is a record in `data/decisions.json` (the decision register, which replaced the POA&M under CR26), read through `scripts/_decisions.py` by the VDR builder, the vulnerability gate, reconciliation and the trust center; every in-force record carries `decided_by`, `decided_on` and an annual `review_by`, which the loader and `tests/test_decision_records.py` enforce. Whether each FedRAMP rule that applies (20x Program Class C, derived from the pinned corpus's applicability metadata by `_rules.applicable_provider_rules`) is implemented is a record in `data/frr-records.json`, read through `scripts/_frr.py`: one record per applicable rule, a missing or extra record fails, and a record may say "Implemented" only with evidence that resolves (a test, a workflow step, a freshness-governed published artifact, or an in-force decision). A change that implements a rule flips its record in the same PR, with the evidence. These are the source of truth for everything published. `scripts/_common.py` holds the helpers they share (`classify` — the inheritance rule four artifacts must agree on — plus `sha256_file`, `prop`, `load_hub`, `sid`); it is imported via an `__file__`-relative `sys.path.insert` so every script stays standalone-runnable from any cwd. **`scripts/staged/` is not wired into anything** — parked generators, see its README; never assume a script there runs. - `website/` — static site content. `website/.well-known/` is where the **published** artifacts are served from (KSI signal + bundle, SDR, CPO, VDR + trend, IIW CSV, runtime signal, signing pubkey, schema), but in git it tracks only `security.txt` and `vdr-trend.json`: everything else there is gitignored build output that the deploy signs and uploads (the website sync excludes `.well-known/*`). - `docs/` — `policies/`, `runbooks/`, `scn/`, and `assessment/` (e.g. `ground-truth.md`). - `website/research/` — plain static pages: `authorization-boundary.html` (scope; **read it first for the "why"**) and the long-form essay `staying-in-the-loop.html` (threshold dynamics, titled "Staying in the Loop"). The essay's filename is **not** referenced by `build-ksi-signal.py`, which discovers pages with `rglob("*.html")`; the hardcoded path lives in `tools/essay/paths.py`, so renaming means updating that. `/research/tuning-the-eigenvalue.html` and `/research/the-plumbing.html` are redirect stubs (the latter to the dashboard; the "Automated Security Compliance" paper is retired). diff --git a/README.md b/README.md index 172e001b..e7b5fe9e 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ The repository ships a documentation set that closes 27 KSI indicators with rati | [`docs/supply-chain.md`](docs/supply-chain.md) | KSI-SCR-MIT, KSI-SCR-MON | SCRM with Dependabot config in `.github/dependabot.yml` | | [`docs/training-log.md`](docs/training-log.md) | KSI-CED-RAT | Self-attested training log | | [`data/decisions.json`](data/decisions.json) | (cross-cutting — meta) | The decision register: every decision about a finding (false positive, accepted risk, operational requirement, remediation history), with who decided, when, and the review date. Replaced the POA&M under CR26 | +| [`data/frr-records.json`](data/frr-records.json) | SDR-CSO-FRR | One record per FedRAMP rule applicable to 20x Class C: status, implementation, evidence, and the gap and closing work when not implemented. Validated by `scripts/_frr.py`, which refuses a claim of "Implemented" without evidence that resolves | | [`docs/policies/`](docs/policies/) | NIST 800-53 Rev 5 *-1 controls (AC-1, AT-1, ... SR-1, PT-1) | Per-family policy and procedures docs, plus the FedRAMP 20x Secure Configuration Guide. Each file integrates the relevant 20x rules (SCN, VDR, MAS, SCG, CCM, ICP, FSI, UCM) and cites AWS authorization package AGENCYAMAZONEW for inherited families | | [`docs/continuous-monitoring-plan.md`](docs/continuous-monitoring-plan.md) | CA-7, KSI-MLA, FedRAMP 20x CCM | Continuous Monitoring strategy and mechanisms (deploy-time gate, runtime emitter, VDR aggregator, annual review) | | [`docs/privacy-threshold-analysis.md`](docs/privacy-threshold-analysis.md) | PT-1 (negative determination) | PTA: no PII processed; full PIA not required | diff --git a/data/frr-records.json b/data/frr-records.json new file mode 100644 index 00000000..7c943439 --- /dev/null +++ b/data/frr-records.json @@ -0,0 +1,1629 @@ +{ + "schema": "frr-records/1", + "$comment": "One record per FedRAMP rule applicable to 20x Program Class C. Validated by scripts/_frr.py; see infrastructure/schemas/frr-records.schema.json.", + "basis": "Drafted 2026-11-05 from a rule-by-rule assessment of the repository and the live site, corrected for the changes since (account IDs redacted, published claims fixed, CR26 vulnerability reports published). AI-drafted; for the operator's review.", + "records": [ + { + "frr_id": "AFC-CSO-ACK", + "status": "Not Implemented", + "implementation": "No FSI; no auto-responder.", + "evidence": [], + "gap": "No prompt automatic acknowledgment of FedRAMP messages. To close: Configure an auto-acknowledgment for mail from @fedramp.gov/@gsa.gov (e.g. SES receipt rule -> Lambda reply, or provider auto-reply rule) and include it in the nightly configuration check.", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-CRA", + "status": "Not Implemented", + "implementation": "No FSI, no Emergency/Emergency Test handling runbook in docs/incident-response.md or ir-policy.md.", + "evidence": [], + "gap": "No procedure to recognize Emergency / Emergency Test designated messages, extract the required actions and timeframe, and complete them; no record of completion. To close: Write an FSI runbook (classify by designation, open a tracked item with the message's deadline, complete, reply with completion) and wire Emergency-designated mail to page the operator; keep a completion log. Rehearse with a self-generated Emergency Test.", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-EMR", + "status": "Not Implemented", + "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", + "evidence": [], + "gap": "No routing of Emergency-designated messages to a senior security official; no mailbox configuration or automated validation (both artifacts missing). To close: Add a mail rule that routes subject-designated 'Emergency' FedRAMP messages to the senior security official (the accountable official in the CPO, plus SNS page), and include it in the nightly mailbox-configuration check.", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-IMA", + "status": "Not Implemented", + "implementation": "No FSI; no handling procedure for Important-designated messages.", + "evidence": [], + "gap": "No procedure to complete actions in Important messages within their stated timeframe. To close: Extend the FSI runbook to Important messages with tracked deadlines (same tracker as CRA).", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-INB", + "status": "Not Implemented", + "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", + "evidence": [], + "gap": "No dedicated FedRAMP Security Inbox address exists or is designated; the required artifact (an email address to receive messages from FedRAMP) cannot be produced. To close: Create a dedicated address on the system's domain (e.g. fedramp-security@samaydlette.com: MX record in dns.tf plus SES inbound or a mail provider, with SPF/DKIM/DMARC), document it as the FSI in the CPO certification_data (security_contact block) and trust center, and monitor it (alarm on delivery failure).", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-NOC", + "status": "Not Implemented", + "implementation": "No FSI exists, so no change-notification process exists; nothing in docs/incident-response.md or docs/policies/ir-policy.md covers notifying FedRAMP of an inbox change.", + "evidence": [], + "gap": "No process (manual or automated) to notify FedRAMP when the FSI address changes. To close: Make the FSI address a single value in the canonical inventory and add a CI check that fails a change to it unless the commit carries a 'FedRAMP notified on via ' record; document the runbook step (email FedRAMP from the old and new address) in ir-policy.md.", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-RCV", + "status": "Not Implemented", + "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", + "evidence": [], + "gap": "Nothing receives FedRAMP mail without disruption (no inbox, no anti-spam allowlisting, no monitoring, no backup reader for a one-person operation). To close: After INB/TFG: allowlist FedRAMP senders, add delivery-failure and unread-age alarms (SNS to operator, reusing the watchdog path), and name a backup reader or an out-of-office auto-forward so a single absent operator does not disrupt receipt. Provider half is fully buildable; only an actual FedRAMP message demonstrates it end-to-end.", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "AFC-CSO-TFG", + "status": "Not Implemented", + "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", + "evidence": [], + "gap": "No mailbox configuration treating @fedramp.gov / @gsa.gov senders as FedRAMP, and no automated validation of that configuration (both listed artifacts missing). To close: In the FSI mail rules, tag/route all mail from @fedramp.gov and @gsa.gov as FedRAMP (never spam-filtered/quarantined); export the rule set as code and add a nightly check (evidence-nightly.yml) that reads the live rules and compares them with the expected configuration, published to the trust center like check-manual-settings.py does for GitHub settings.", + "closes_with": [ + "W9" + ] + }, + { + "frr_id": "CCM-OCR-AFS", + "status": "Not Implemented", + "implementation": "No summary section in build-ocr.py or the live OCR; declared N/A (build-ocr.py:106-110).", + "evidence": [], + "gap": "No anonymized/desensitized summary of feedback, questions and answers, and no stated delivery method. To close: Add a 'feedback summary' section to the OCR (and an addendum file) built from the FBM channel, with a redaction step for names/agency identifiers; when nothing was received, state 'no feedback received on OCR '.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-OCR-AVL", + "status": "Partially Implemented", + "implementation": "Live https://samaydlette.com/.well-known/ongoing-certification-report.json + .md (fetched 2026-11-05), built by scripts/build-ocr.py in deploy-with-opa.yml step 'Build Ongoing Certification Report' (line ~1555); all eight summary sections present (build-ocr.py:91-105).", + "evidence": [], + "gap": "The OCR is rebuilt on every deploy rather than issued quarterly: there is no frozen, dated, signed issue per period, and no index of past issues.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-OCR-FBM", + "status": "Not Implemented", + "implementation": "build-ocr.py:106-110 and the live OCR state the feedback channel is 'N/A absent a consuming agency'; no feedback link in the OCR, CPO or trust center; security.txt contact is the only channel and is not tied to the OCR.", + "evidence": [], + "gap": "No asynchronous feedback/question mechanism for each OCR, and no documented way to access it (artifact missing). To close: Stand up a per-report asynchronous channel (e.g. a GitHub Discussions category or issue form per OCR issue, with a private alternative address for agencies), link it from each OCR, the CPO and the trust center.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-OCR-LSI", + "status": "Partially Implemented", + "implementation": "The OCR carries SCN summaries, decision titles and the accepted findings already public in the VDR, and passes the publication redaction gate (no AWS account ID) on every deploy.", + "evidence": [], + "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it. No documented sensitivity determination per issue and no screen beyond account IDs (ARNs, IPs, email addresses, secret-like strings). SCN text is copied verbatim.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-OCR-NRD", + "status": "Partially Implemented", + "implementation": "Live OCR json/md carry next_ongoing_certification_report_date 2027-02-04 (build-ocr.py:88).", + "evidence": [], + "gap": "The date is now+91 days recomputed on every deploy, so it is a moving value, not a committed target; it is published only inside the OCR, not with the other public Certification Data (CPO/trust center). To close: Derive the next date from the fixed cycle (last issued OCR + cycle), keep it stable between issues, and publish it in the CPO and trust-center.json as well.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-OCR-RPS", + "status": "Partially Implemented", + "implementation": "OCR is public at /.well-known/ (fetched).", + "evidence": [], + "gap": "Permissive rule, exercised; but no recorded determination that public release will not adversely affect the offering. To close: Record the public-release determination (who, when, basis) in each OCR issue or in the CPO.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-OCR-SOR", + "status": "Not Implemented", + "implementation": "build-ocr.py:34,54-55,88: rolling window recomputed per deploy.", + "evidence": [], + "gap": "No regular 3-month cycle and no chosen phase within the quarter. To close: Pick fixed issue dates (e.g. mid-quarter: 15 Feb/May/Aug/Nov), encode them in config, and have the build issue the OCR only on those dates.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-ACT", + "status": "Not Implemented", + "implementation": "No reviews held.", + "evidence": [], + "gap": "No additional agency-relevant information supplied in reviews. To close: Add a standing agenda item (e.g. upcoming roadmap, KSI trend, VDR trend) to the review template.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-MTG", + "status": "Not Implemented", + "implementation": "build-ocr.py:106-110 / live OCR declare the Quarterly Review N/A; no meeting record, agenda or schedule anywhere in the repo.", + "evidence": [], + "gap": "No synchronous Quarterly Review has been scheduled or held. To close: Schedule a recurring quarterly review (video call) 3-10 business days after each OCR issue, prepare an agenda from the OCR, hold it even if no agency attends, and keep minutes; publish the schedule.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-NID", + "status": "Partially Implemented", + "implementation": "No Quarterly Review has been held, so nothing has been disclosed; no content-review procedure exists.", + "evidence": [], + "gap": "No pre-meeting sensitivity review procedure for review content. To close: Add a review checklist (same screen as OCR-LSI) for slides/agenda before each review.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-NRD", + "status": "Partially Implemented", + "implementation": "Live OCR next_quarterly_review_date 2027-02-04 (build-ocr.py:89).", + "evidence": [], + "gap": "Same moving now+91 value as the OCR date (so the review date equals the report date, contradicting QTR-SAR); not published with the CPO/trust center; no review actually scheduled for that date. To close: Derive the review date from the OCR issue date + 3-10 business days, keep it stable, publish it in CPO and trust center, and tie it to the .ics.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-REG", + "status": "Not Implemented", + "implementation": "No registration link or .ics anywhere in website/ or the published artifacts.", + "evidence": [], + "gap": "No registration URL or calendar file (artifact missing). To close: Generate an .ics per review from the cycle config and publish it (e.g. /.well-known/quarterly-review.ics) plus a registration form link; reference it from the OCR and CPO.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-RTP", + "status": "Partially Implemented", + "implementation": "No reviews held, so no third parties invited; no attendance policy exists.", + "evidence": [], + "gap": "No documented invitation policy limiting third parties. To close: Document an attendee policy (agencies and FedRAMP; third parties only with stated relevance) in the review procedure.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-RTR", + "status": "Not Implemented", + "implementation": "No reviews held; no recording/transcript procedure.", + "evidence": [], + "gap": "No recording or transcript supplied to necessary parties. To close: Record or transcribe each review and distribute through the FBM channel; keep them with the OCR issue.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-SAR", + "status": "Not Implemented", + "implementation": "build-ocr.py:88-89 set the review date equal to the report date (0 business days after).", + "evidence": [], + "gap": "Review not scheduled 3-10 business days after the OCR release. To close: Compute the review date as OCR issue + N business days with 3<=N<=10 (holiday-aware) and assert it in a test.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CCM-QTR-SCR", + "status": "Not Implemented", + "implementation": "No review content exists.", + "evidence": [], + "gap": "Optional (MAY); not exercised.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "CCM-QTR-SRR", + "status": "Not Implemented", + "implementation": "No recordings exist.", + "evidence": [], + "gap": "Optional (MAY); not exercised. No recordings are released; if one ever is, agency information must be removed first.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "CDS-CSO-AVR", + "status": "Partially Implemented", + "implementation": "https://samaydlette.com/.well-known/availability.json + availability.md, scripts/build-availability.py (appends one point per deploy, L47-53; WINDOW_DAYS=45 L27).", + "evidence": [], + "gap": "Not an availability measurement: a point is written only when a deploy succeeds ('A deploy that produces this feed is, by definition, a successful one', L10-11), so an outage can never be recorded and there are no incidents field/records. Days without deploys are simply missing (27 points spanning 2026-08-06..11-05; ~10 in the last 30 days). Trim keeps the last 45 points, not 45 days, so history older than the window persists. Core services are not probed (Silk Reeling API/Cognito not checked at all). Served from the same CloudFront/S3 as the offering, so it is unavailable when the offering is (self-stated residual). To close: Run an external synthetic monitor on independent infrastructure (different provider/region/account and DNS name, e.g. a status subdomain on separate hosting) that probes the site, /.well-known/, and the Silk Reeling API/Cognito every few minutes; record incidents (start, end, affected service, cause); publish JSON + HTML status page from that independent host with 30+ days of continuous history.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-CSO-CBF", + "status": "Partially Implemented", + "implementation": "Each builder renders JSON and Markdown from the same in-memory dict in one run (build-cds-overview.py, build-cpo.py, build-availability.py, build-policy-index.py:109-119, build-sdr.py render_markdown L278, build-ocr.py); trust page renders from trust-center.json.", + "evidence": [], + "gap": "Human-readable twins are lossy, so the formats are not consistent in content: policy-index.md drops summary and version (build-policy-index.py:112-118); security-decision-record.md shows only KSI id/controls/explanation, not verification/validation or the 333 rev5 control statements. No test or reconcile invariant checks JSON/MD parity. Human-readable policies themselves (docs/policies/*.md) are not cross-checked against the SDR/control statements (ac-policy.md:3 says 'no end-user authentication' while the Cognito user pool exists). To close: Render every JSON field into the human view (or generate the MD from a schema-driven template), and add a parity test/reconcile invariant that parses each .md back and compares against the JSON.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-CSO-FID", + "status": "Partially Implemented", + "implementation": "offering.json:3 fedramp_id flows into certification-overview-package.json and certification-package-overview.json (build-cpo.py:80).", + "evidence": [], + "gap": "No FedRAMP ID is assigned. Even once assigned, the wiring would not put it in 'all certification data': build-ocr.py:83 hardcodes its own placeholder string; the SDR, VDR reports, availability feed, policy index, SCN records and ksi-signal carry no FedRAMP ID field. To close: Make offering.json the single source of fedramp_id and stamp it into every generated artifact (OCR, SDR, VDR/VDT/AVI, SCN, availability, policy index, trust-center.json); add a reconcile invariant that every published artifact carries the same FedRAMP ID.", + "closes_with": [ + "W3", + "sam" + ] + }, + { + "frr_id": "CDS-CSO-FRC", + "status": "Not Implemented", + "implementation": "No slot or process for FedRAMP Certification Reports: build-cpo.py PACKAGE_CONTENTS (L30-40) has no entry; no docs/ procedure.", + "evidence": [], + "gap": "No FedRAMP Certification Report has been received, and there is no mechanism to publish one unmodified within 2 weeks. To close: Add a package slot (store received report verbatim under data/, publish byte-identical with its hash, signed) and a runbook with a 2-week clock tracked as a decision/deadline item.", + "closes_with": [ + "W4", + "sam" + ] + }, + { + "frr_id": "CDS-CSO-HAD", + "status": "Not Implemented", + "implementation": "No snapshot mechanism: every artifact (including ongoing-certification-report.json) is overwritten in place each deploy; S3 versioning exists on the website bucket but versions are not exposed and are subject to lifecycle expiry (infrastructure/main.tf:148-200).", + "evidence": [], + "gap": "No snapshot of the certification data aligned to each OCR, and no retention for the duration of certification, and no access explanation. To close: At each OCR date, freeze the full package into an immutable, dated path (e.g. /.well-known/snapshots/2027-02-04/ with a manifest + signatures; S3 Object Lock), list snapshots in the trust center, and document access.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "CDS-CSO-IRP", + "status": "Partially Implemented", + "implementation": "policy-index.json and .md under /.well-known/ (scripts/build-policy-index.py): 28 policies with summary, owner, related practices, file, last update and version. Since W1 each policy's date and version (git:) come from its own last commit in full history; a shallow checkout is refused.", + "evidence": [], + "gap": "'Related FedRAMP Practices' are mostly Rev5 control families, not FedRAMP rules or KSIs. Procedures (runbooks, verification, deploy notes, architecture decisions) are not indexed. The policies themselves are served only from the repository, with no access explanation in the certification data, and the .md omits summary and version.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-CSO-PSM", + "status": "Not Implemented", + "implementation": "Permissive (MAY). Checked: no per-service materials are supplied; nothing in the rule requires them.", + "evidence": [], + "gap": "Optional (MAY); not exercised. Could split package views per service (website, Silk Reeling).", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "CDS-CSO-PUB", + "status": "Partially Implemented", + "implementation": "certification-overview-package.json and .md under /.well-known/ (scripts/build-cds-overview.py from data/cds/offering.json): service and deployment model, business category, sales and security contacts, product website, description, service list, documentation and trust-center URLs. The logo is a real PNG at /assets/images/brand/logo.png and /trust/ serves its index (W1).", + "evidence": [], + "gap": "No UEI (no SAM.gov registration) and no FedRAMP ID (none exists; the provider name and acronym stand in, CDS-CSO-FID). The next-OCR date is now+91 days on every deploy, so it recedes and never arrives (W4 fixes the calendar). The secure configuration guide is not served from the site (W10). The documentation overview and the trust-center link are one field. Current assessment service: none. To close: Register in SAM.gov for a UEI; issue the OCR on a fixed calendar; serve the secure configuration guide from the site; split the documentation overview from the trust-center link.", + "closes_with": [ + "W4", + "W10", + "sam" + ] + }, + { + "frr_id": "CDS-CSO-RIS", + "status": "Partially Implemented", + "implementation": "Every artifact the deploy publishes passes a publication redaction gate (scripts/check-published-redaction.py, W0) that fails on an AWS account ID; account IDs in ARNs are replaced with ACCOUNT before publication. The trust center and boundary map also refuse ARNs and email addresses.", + "evidence": [], + "gap": "No written disclosure standard says what is public and what is need-to-know; IAM role and policy detail, the API endpoint ID and the CloudFront domain are published without a recorded determination. On sufficiency, the package lacks an independent assessment and the rule records are not yet in the SDR.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-CSO-RPS", + "status": "Partially Implemented", + "implementation": "All data is shared publicly; COP trust_center_status says so (certification-overview-package.json), trust page intro explains the self-attested context.", + "evidence": [], + "gap": "The artifact requires an explanation of if/how information is shared and the provider's determination that public sharing will not likely have an adverse effect. There is a statement of public sharing but no documented adverse-effect determination, and the redaction practice is inconsistent (see CDS-CSO-RIS). To close: Publish a short sharing statement: what is public, to whom else data goes, and the recorded determination (decision record in data/decisions.json with decided_by/on and review date) that it is not adverse.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-CSO-SVC", + "status": "Partially Implemented", + "implementation": "https://samaydlette.com/.well-known/certification-overview-package.json services[] (2 entries) + .md table; source data/cds/offering.json services.", + "evidence": [], + "gap": "List exists and names the two services, but security category is a uniform label ('FIPS-199 Moderate (FedRAMP 20x Class C)') rather than each service's own category (the system categorization is C low / I moderate / A low; component C values range not-applicable..moderate). It does not say what is NOT included (e.g. the optional LLM feedback is mentioned, but third-party-hosted podcast audio, the MediaPipe CDN/model hosts, newsletter/feed are not classified in or out). No separate human-readable page on the trust center. To close: Give each service its own C/I/A category derived from the inventory components it contains, add an explicit 'not included' list, and render the list on the trust page.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "CDS-CSO-UTC", + "status": "Partially Implemented", + "implementation": "https://samaydlette.com/trust/index.html (renders https://samaydlette.com/.well-known/trust-center.json) + signed artifacts under /.well-known/.", + "evidence": [], + "gap": "A trust center exists, but it is not FedRAMP-compatible: it does not meet CDS-TRC-AAI (no agency access inventory), CDS-TRC-ACL (no viewer-level access logging or summaries), CDS-TRC-PAC (no documented programmatic index of all data), and it is not the 'complete and definitive source' (FRD-TRC): policies live on GitHub, COP/availability/policy-index/OCR are not listed on the trust page. To close: Close the CDS-TRC-* gaps below and make the trust center list and serve every piece of certification data (including policies) from one place.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-TRC-AAI", + "status": "Not Implemented", + "implementation": "Access is anonymous public GET; no identity, registration or access inventory exists anywhere in scripts/ or infrastructure/.", + "evidence": [], + "gap": "No inventory or history of federal agency users or systems with access; no way to give one to FedRAMP; no explanation artifact. To close: Add an access tier for agencies (e.g. Cognito or OIDC federation with agency IdPs / API keys per agency system) for the full package; record every grant/revoke in an append-only register (who, agency, when, by whom) and document how FedRAMP requests it. An empty inventory with the mechanism live would satisfy the rule today.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-TRC-ACL", + "status": "Partially Implemented", + "implementation": "S3 server access logging on the website bucket (infrastructure/logging.tf:114) to a log bucket retained 400 days (logging.tf:59).", + "evidence": [], + "gap": "S3 server access logs only see origin fetches; CloudFront edge hits are not logged (no logging_config on the distribution, bootstrap/cloudfront.tf:233-311), so most access to certification data is invisible. No access summaries are produced or stored; no explanation of how parties obtain their log information. To close: Enable CloudFront standard (or real-time) logging for /.well-known/ and /trust/, build a scheduled job that writes monthly access summaries (per artifact, per requester/agency once authenticated) retained >= 6 months, and document how a party requests its records.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-TRC-HMR", + "status": "Partially Implemented", + "implementation": "JSON + .md twins for COP, CPO, SDR, OCR, availability, policy index, VDR (https://samaydlette.com/.well-known/*.md return 200, text/markdown); trust page renders trust-center.json and boundary-map.json.", + "evidence": [], + "gap": "ksi-signal.json, iiw.csv and boundary-map.json have no full human-readable twin; SDR .md omits most of the record; the .md files are not linked from the trust page; policies are not viewable/downloadable from the trust center. To close: Generate complete human views for every artifact and link both formats for each from one trust-center index.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-TRC-PAC", + "status": "Partially Implemented", + "implementation": "All data is plain HTTPS GET under /.well-known/ (fetched live); the trust page 'Verify' section shows curl + cosign usage; docs/ksi-signal.md documents the signal schema.", + "evidence": [], + "gap": "No signed machine-readable manifest lists every certification artifact with URL, media type, schema and signature, and no programmatic-access documentation exists to cite. Human-readable policies are served only from the repository.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-TRC-SSM", + "status": "Not Implemented", + "implementation": "Checked: no account, API-key, or access-management features exist (public anonymous access only).", + "evidence": [], + "gap": "No features for parties to provision or manage their own users' and services' access, and no documentation URL. To close: With the agency access tier from CDS-TRC-AAI, let an agency administrator self-register and manage its own users/service credentials (Cognito groups or API keys scoped per agency), and document it. If staying fully public, document that anonymous access requires no provisioning (weaker).", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-TRC-USH", + "status": "Partially Implemented", + "implementation": "The trust center is a static public site on CloudFront; /trust/ and every directory URL serve their index through a CloudFront Function (W1), and /.well-known/*.json are served directly.", + "evidence": [], + "gap": "Availability of the trust center is not measured independently (CDS-CSO-AVR), there is a single origin with no failover (DEC-009), and the page's content needs JavaScript.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-UTC-AAD", + "status": "Not Implemented", + "implementation": "Checked: no access-request process, denial path or FedRAMP-notification procedure in docs/policies, docs/runbooks or data/decisions.json.", + "evidence": [], + "gap": "No procedure to notify FedRAMP within 5 business days of denying an agency access request, and no tracking of the clock. To close: Write the access-request/denial procedure (who decides, record as a decision kind, auto-compute the 5-business-day deadline, notify FedRAMP via its published channel) and add it to the policy index.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CDS-UTC-AGA", + "status": "Partially Implemented", + "implementation": "Everything is public, so any agency can obtain the package without asking.", + "evidence": [], + "gap": "Neither artifact exists: no URL/explanation of how to request the package (beyond the public data) and no explanation of how the provider decides whether to share. To close: Add a 'Request access' section to the trust page (contact, what is provided, turnaround) and a sharing-decision policy entry.", + "closes_with": [ + "W8" + ] + }, + { + "frr_id": "CMU-CSO-CAT", + "status": "Not Implemented", + "implementation": "Checked infrastructure/cognito.tf, silk-reeling.tf and bootstrap/cloudfront.tf: tenant-facing defaults use standard endpoints; no list of default modules exists.", + "evidence": [], + "gap": "No configuration that defaults agency tenants (Silk Reeling users/tenants) to validated modules, and no list of default modules with validation status. To close: Define the agency-tenant default (FIPS endpoints, TLS policy on CloudFront/API Gateway, KMS CMKs) in Terraform, and publish the generated list of default modules with CMVP status.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "CMU-CSO-CMD", + "status": "Partially Implemented", + "implementation": "certification-package-overview.json cryptographic_modules (4 entries, from offering.json); research/authorization-boundary.html#fips (L86-91); boundary-map.json fips_modules.", + "evidence": [], + "gap": "Not per service and not complete: Silk Reeling's Cognito, API Gateway TLS, Lambda runtime TLS, DynamoDB/SQS/SNS encryption, Secrets Manager, CloudTrail log-file validation, Route 53 DNSSEC signing are not listed. Validation status is imprecise: 'FIPS-validated cipher suites' for CloudFront TLS is not a CMVP status (no module name/cert number); S3 SSE-S3 has no validation status; no CMVP certificate numbers anywhere. Stale: lists an 'app secret' CMK (removed by SCN-2026-006) and omits the silk_reeling CMK (silk-reeling.tf:50). To close: Build the module list from the inventory per service: for each AWS service used, cite the module (e.g. AWS-LC FIPS, KMS HSM) with CMVP certificate number and whether it is validated or an update stream, from AWS's published FIPS documentation; generate it, don't hand-maintain it, and fail the build when a new crypto-using service appears unmapped.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "CMU-CSO-UVM", + "status": "Partially Implemented", + "implementation": "Customer-managed keys use AWS KMS (FIPS 140-3 HSM): infrastructure/main.tf:532,605,648; silk-reeling.tf:50,207.", + "evidence": [], + "gap": "Validated modules are not shown for the paths that would protect customer data in transit: CloudFront, API Gateway and Cognito are reached via standard (non-FIPS) endpoints with no evidence their TLS module is validated; buckets use SSE-S3 rather than SSE-KMS; Sigstore signing is non-validated (not customer data, but documented). To close: Confirm and document each customer-data path's module validation from AWS FIPS docs; use FIPS endpoints where offered (cognito-idp-fips, API Gateway via FIPS endpoint, KMS-fips), move data-bearing stores to SSE-KMS, and record any remaining non-validated path as a decision.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "CPO-CSO-MTD", + "status": "Partially Implemented", + "implementation": "certification-package-overview.json accountable_official (name, title, email) and metadata.last_updated / source_of_update (build-cpo.py:66-72).", + "evidence": [], + "gap": "Version is the constant 'live (regenerated each deploy)' (build-cpo.py:69), which identifies nothing; fields are not in the schema's names. To close: Use a real version (e.g. semver bumped on content change, or commit SHA + signal_id) and map to the schema's metadata fields.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "CPO-CSO-OSA", + "status": "Not Implemented", + "implementation": "certification-package-overview.json independent_assessment.status = 'N/A ... no FedRAMP Recognized independent assessment service engaged' (build-cpo.py:97-102).", + "evidence": [], + "gap": "No independent assessment, so no assessor-supplied overall summary. To close: Engage a FedRAMP Recognized assessment service, receive its IVV-IAS-OSA summary (signed assessor record, as _assessor.py already supports), and embed it unmodified in the CPO.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "CPO-CSO-OVR", + "status": "Partially Implemented", + "implementation": "https://samaydlette.com/.well-known/certification-package-overview.json + .md (scripts/build-cpo.py).", + "evidence": [], + "gap": "Does not validate against FedRAMP's CPO schema (vendored infrastructure/schemas/fedramp-2026-06-24/...certification-package-overview...: running scripts/_fedramp_schema.validate on the live file: wrong $schema, missing serviceIdentification, serviceProperties, contactInformation) even though its 'schema' field names it. Of the required content: CDS-CSO-PUB is only linked (and partial); IRP only linked; MAS-CSO-IIR is a count and a link; MAS-CSO-FLO and MAS-CSO-TPR are absent (only named in a 'reference' string); CMU-CSO-CMD is partial; IVV-CSO-ICP is 'N/A'. To close: Rebuild the CPO in FedRAMP's schema shape (serviceIdentification, serviceProperties, contactInformation, assessor, certifiedServices, thirdPartyInformationResources), validate it with write_validated() like the VDR reports, and include or reference the MAS flow/TPR data and the assessor's IVV material.", + "closes_with": [ + "W3", + "sam" + ] + }, + { + "frr_id": "CPO-CSX-CPM", + "status": "Partially Implemented", + "implementation": "The package (CPO, SDR, OCR, CDS artifacts) is rebuilt and published on every deploy to main; the VDR family is rebuilt twice daily.", + "evidence": [], + "gap": "Nothing guarantees a rebuild at least every 2 weeks: with no push to main for 14 days, the CPO, SDR and OCR go stale, and no freshness window covers them at that cadence.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "FRC-APP-AFC", + "status": "Not Implemented", + "implementation": "Checked: no application form completed or drafted in the repo.", + "evidence": [], + "gap": "Form not completed. Completing it 'in full' depends on prerequisites not met (Marketplace listing, fresh independent assessment, UEI). To close: After the prerequisites, the operator completes and submits the FedRAMP Certification Application Form personally; keep a copy in the package.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "FRC-APP-FCP", + "status": "Partially Implemented", + "implementation": "Package artifacts regenerate on every deploy (deploy-with-opa.yml build steps ~L1558-1600) and the runtime emitter re-validates daily (trust-center.json 'runtime' check).", + "evidence": [], + "gap": "Freshness is not bounded at 7 days: deploys are irregular (gaps 2026-08-27→09-24, 10-05→10-20 in the SDR metric history) and the governed freshness window for ksi-signal/OCR/COP/policy-index is 720h (infrastructure/schemas/freshness-config.json); the package is also incomplete (see FRC-CSO-PKG). To close: Rebuild and re-verify the full package on a schedule (at least weekly, e.g. a cron job reusing the deploy build) and set package freshness windows to <=168h so the watchdog alarms on staleness.", + "closes_with": [ + "W4" + ] + }, + { + "frr_id": "FRC-APP-FIA", + "status": "Not Implemented", + "implementation": "certification-package-overview.json independent_assessment: none engaged; SDR assessor_records.status 'no assessor record'.", + "evidence": [], + "gap": "No independent assessment by a FedRAMP Recognized assessment service, let alone within 3 months. To close: Engage a FedRAMP Recognized assessor, complete the assessment, and apply within 3 months of its completion.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "FRC-APP-MLF", + "status": "Not Implemented", + "implementation": "Not listed in the FedRAMP Marketplace (stated in docs/cr26-applicability.md and COP).", + "evidence": [], + "gap": "No listing; MKT-CSO-MLR depends on CDS-CSO-PUB (partial); MKT-CSO-PML request not sent; MKT-IIP-AGU agency use case and MKT-IIP-DCP continuous progress not shown. To close: Finish CDS-CSO-PUB, document an agency use case within FedRAMP scope, submit the Marketplace Provider Listing Request Form, and publish continuous-progress evidence.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "FRC-APP-NTP", + "status": "Partially Implemented", + "implementation": "No application exists, and the CPO names the operator as the accountable official, the person who would apply.", + "evidence": [], + "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it. To close: Operator applies personally; do not let the assessor or an advisor submit.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "FRC-APP-USA", + "status": "Not Implemented", + "implementation": "Permissive (MAY). Checked: no initial assessment exists, so there is nothing stale to freshen.", + "evidence": [], + "gap": "Optional (MAY); not exercised.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "FRC-CSO-FCP", + "status": "Partially Implemented", + "implementation": "Target stated: Class C, Moderate, 20x (system-profile categorization in ksi-signal.json; CPO class/impact). Conformance self-report: trust-center.json profile summary 18 implemented / 16 partial / 38 not yet.", + "evidence": [], + "gap": "The profile is identified, but 'apply all relevant FedRAMP Practices' is not met: most applicable rules are partially implemented or not implemented (see these records), and docs/cr26-applicability.md still records some families as not applicable.", + "closes_with": [ + "W3", + "W10" + ] + }, + { + "frr_id": "FRC-CSO-JSN", + "status": "Partially Implemented", + "implementation": "The VDT, AVI and VDR history are written through scripts/_fedramp_schema.write_validated against the vendored 2026-06-24 schemas and published (C3a, #496). The CPO, SDR and OCR are JSON but in their own field names.", + "evidence": [], + "gap": "Live CPO, SDR and OCR do not validate against the vendored FedRAMP schemas (validate(): cpo 4 errors, sdr 6, ocr 11 — wrong $schema, missing required top-level objects), and their builders do not call write_validated. No CORS: a GET with Origin: https://example.org returns no Access-Control-Allow-Origin, so browser apps on other domains cannot read the JSON. To close: Port build-cpo.py, build-sdr.py and build-ocr.py to the FedRAMP schemas and write them via write_validated(); add a CloudFront response-headers policy for /.well-known/* with Access-Control-Allow-Origin: * (GET/HEAD/OPTIONS) and add a live check of the headers in verify-published.sh.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "FRC-CSO-MRA", + "status": "Partially Implemented", + "implementation": "Reconciliation gate (scripts/reconcile.py; live reconcile-report.json: 10 invariants held) blocks publishing inconsistent artifacts; signed provenance; AI-drafted third-party records are held at 'not recorded' until decided (data/third-party-components.json _note).", + "evidence": [], + "gap": "The reconciliation gate checks the inventory but not the package's claims: the CPO, SDR and OCR do not validate against their schemas, and no check covers URL liveness, the CMK list against the inventory, a fixed OCR calendar, or one FedRAMP ID across artifacts. W1 corrected the logo, policy dates, AC policy and CMK list, each with a test.", + "closes_with": [ + "W3", + "W8" + ] + }, + { + "frr_id": "FRC-CSO-PKG", + "status": "Not Implemented", + "implementation": "Package components exist publicly (CPO, SDR, OCR under /.well-known/), but nothing has been supplied to FedRAMP.", + "evidence": [], + "gap": "No package supplied to FedRAMP; the components that exist are incomplete (CPO/SDR/OCR fail FedRAMP schemas; SDR lacks FRR records and independent V&V). To close: Complete CPO/SDR/OCR per their rules and schemas, then submit through the FedRAMP application (FRC-APP-*).", + "closes_with": [ + "W3", + "sam" + ] + }, + { + "frr_id": "FRC-CSO-POP", + "status": "Partially Implemented", + "implementation": "Checked: no application for either program exists (no FedRAMP ID, no Marketplace listing, no application in repo/docs); docs/cr26-applicability.md targets 20x Class C.", + "evidence": [], + "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it. Labels mix programs (CPO/SDR impact_level_canonical 'Moderate (FedRAMP Rev 5; 20x Class C)'; SDR carries rev5_controls), which could read as seeking both. To close: When applying, apply for 20x only and state certificationType '20x Class C' alone in the CPO.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "FRC-CSX-MAS", + "status": "Partially Implemented", + "implementation": "The KSI signal applies the KSIs to the inventoried components (ksi-signal.json), derived from Terraform state.", + "evidence": [], + "gap": "The inventory misses external services the site loads from (cdn.jsdelivr.net, storage.googleapis.com) and some flows, and human-process KSIs are not applied per component; MAS-CSO-IIR and MAS-CSO-FLO carry the detail.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "FRC-CSX-VVK", + "status": "Partially Implemented", + "implementation": "Terraform plans are checked against the OPA policies on every deploy, and the runtime Lambda re-evaluates the same Rego against live configuration for object stores and the distribution daily (ksi-signal-runtime.json).", + "evidence": [], + "gap": "Class C requires at least two automated methods for each KSI. The runtime check covers only some component types, and no record counts the automated methods per KSI to show two exist for each.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "FRC-CSX-VVR", + "status": "Partially Implemented", + "implementation": "scripts/_frr.py checks the rule records automatically on every build: the set matches the pinned corpus, and every piece of evidence cited resolves to a test, workflow step, published artifact or decision.", + "evidence": [], + "gap": "Resolution shows the evidence exists, not that it passed for the rule: the records are not yet tied to test results (conformance-results.json) or published in the SDR.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "IEC-CSO-AIR", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", + "evidence": [], + "gap": "No automation exists for incident reporting; nothing exists to automate. To close: Once the incident register and report builder exist, run them from a workflow (workflow_dispatch + on push to the incident register) that validates, signs, publishes and sends IIR/OIR/FIR automatically, and pages through the existing watchdog/SNS when a report deadline is near or missed.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IEC-CSO-DPR", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", + "evidence": [], + "gap": "Nothing assigns a PAIN rating to an incident, so there is no default-to-N5 behaviour and no record of whether a prompt estimate replaced it. The VDR PAIN classifier (scripts/build-vdr-report.py:1730 assign_pain) rates vulnerabilities, not incidents. To close: In the incident register, set pain=5 with basis 'default (IEC-CSO-DPR)' when a record becomes reportable, and allow it to drop only through an EFI estimate record (rating, reason, estimated_at). Drive the IIR/OIR/FIR clocks from the effective rating. Test both paths and show them in the simulated-incident log.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IEC-CSO-EFI", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", + "evidence": [], + "gap": "No Incident Rating step: nothing estimates likely adverse impact on agency customers for an incident or assigns N1-N5 using the minimal/narrow/disruptive/debilitating x 1 or >1 agency scale. The required artifact (an incident log showing the evaluation and its reason) does not exist. To close: Add an EFI record type to the incident register: rating N1-N5, the customer-effect word, single or multiple agencies, reason, estimated_at. Document the rubric in docs/incident-response.md (it can reuse the VDR effect words from infrastructure/schemas/vdr-pain-config.json). Exercise it in the simulated incidents.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IEC-CSO-EFR", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A. docs/policies/ir-policy.md:15 only restates the trigger.", + "evidence": [], + "gap": "No step that evaluates each incident for effect (actual or likely) on confidentiality/integrity of federal customer data, no recorded determination with its reason, and no incident log. The required artifact (an incident log showing one or more real or simulated incidents evaluated, with the reason for the determination) does not exist. The runbook explicitly scopes federal-data incidents out, so it can't make the determination the rule asks for. To close: Add an incident register (e.g. data/incidents.json + schema) whose records carry detected_at, evaluated_at, reportable (bool), reason, and the federal-customer-data types considered. Add an 'Evaluate FedRAMP reportability' step to docs/incident-response.md triage (before Contain), define the data types that would be federal customer data for this offering, and fail a check if any incident record lacks an evaluation. Exercise it with at least two simulated incidents (one reportable, one not) and publish the log.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IEC-CSO-FIR", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A. A schema test fixture (tests/fixtures/cr26/good/incident.json, reportType Final, CVE-2099-0142) checks the validator only; it is not an incident.", + "evidence": [], + "gap": "No Final Incident Report generation on resolution/recovery, no final updates to all previously reported items, no clock (Class C: within 6 hours for N3-N5, 1 business day for N1-N2, after recovery), no delivery. No sample FIR. To close: Add reportType=Final generation when the incident record gets resolved_at and recovery_complete_at, with final values for every earlier IIR/OIR field. Alarm on the FIR deadline. Close the simulated incident with a FIR and link it from the incident log and the Ongoing Certification Report.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IEC-CSO-IIR", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A. The vendored schema infrastructure/schemas/fedramp-2026-06-24/fedramp-incident-report-schema-2026-06-24.json (reportType Initial/Ongoing/Final, federalIncidentCoordinator, timeline, potentialImpact, functionalImpact, recoveryPlan, affectedAgencies) and its validator are in place, but nothing produces a report.", + "evidence": [], + "gap": "No Initial Incident Report generator, no delivery to all affected parties (FedRAMP + affected agencies), and no clock (Class C: within 1 hour for N3-N5, 24 hours for N2, 1 business day for N1). None of the 8 items exists in any record: federal incident response coordinator contact, tracking ID, description, timeline (start, detection time/source, evaluation time, milestones), historical/current PAIN with EFI explanation, functional impact incl. C/I and federal data types, recovery plan/milestones, likely affected agencies. No sample IIR exists, real or simulated. To close: Build scripts/build-incident-report.py that renders an IIR from the incident register into the vendored schema (validated before write) plus a human-readable version; name the federal incident response coordinator contact; add a delivery step (e.g. signed publication to a restricted location plus email to a configured recipient list holding FedRAMP's incident address and each agency's contact) that starts on 'reportable' and is alarmed against the PAIN-based deadline (watchdog). Run a timed simulated incident end to end and keep the IIR.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IEC-CSO-OIR", + "status": "Not Implemented", + "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", + "evidence": [], + "gap": "No Ongoing Incident Report mechanism or cadence (Class C: every 6 hours for N3-N5, 24 hours for N2, 1 business day for N1), no 'updates or lack of updates' logic, and none of the additional items recorded: observed activity, IOCs, related CVE, root cause, response/recovery activities. No sample OIR. To close: Extend the incident report builder for reportType=Ongoing: diff against the previous report so every earlier item is restated (changed or 'no update'), add the 5 OIR fields to the incident record, and schedule/alarm the next OIR from the effective PAIN. Produce OIRs in the simulated exercise.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "IVV-CSO-DUS", + "status": "Partially Implemented", + "implementation": "Searched docs/ and scripts/ for sampling: no use of representative samples documented anywhere.", + "evidence": [], + "gap": "Conditional rule; nothing states whether verification/validation uses full populations or samples (e.g. runtime emitter coverage), so an assessor cannot confirm it is not triggered. To close: Add a statement to the SDR/verification.md that checks run over the full inventory (with counts from the KSI signal), or document any sampling method if used.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "IVV-CSO-FIA", + "status": "Not Implemented", + "implementation": "Live certification-package-overview.json independent_assessment: 'N/A — no FedRAMP Recognized independent assessment service engaged'; SDR assessor_records.status 'no assessor record'; data/assessor-records/README.md: none engaged.", + "evidence": [], + "gap": "No annual independent verification and validation of all applicable rules by a FedRAMP Recognized assessor or FedRAMP. To close: Engage a FedRAMP Recognized independent assessment service; pin its key (infrastructure/keys/assessor-identity.json) and ingest its signed records; schedule the persistent annual cycle. Provider prerequisite: SDR records for every applicable FRR (today only 46 KSI records), see SEI/SEE.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "IVV-CSO-ICP", + "status": "Partially Implemented", + "implementation": "scripts/_assessor.py: signed-record intake (ECDSA P-256 verify against pinned identity, verbatim carry, verify_output() at line 153 fails the build if output differs); build-sdr.py:36,201 wires it in; SDR assessor_records = 'no assessor record'.", + "evidence": [], + "gap": "No independent-assessment results exist to include; CPO independent_assessment is N/A text. To close: When an assessment occurs, ingest its signed results through the existing port and list the assessment report in CPO package_contents unaltered.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "IVV-CSO-RAA", + "status": "Not Implemented", + "implementation": "No assessor engaged.", + "evidence": [], + "gap": "Optional (MAY); not exercised.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "IVV-CSO-SEE", + "status": "Partially Implemented", + "implementation": "SDR 'validation' field present on 46 KSI records; runtime drift check (scripts/check-runtime-divergence.py, deploy-with-opa.yml:1139) and evidence-nightly.yml produce live effectiveness signals.", + "evidence": [], + "gap": "'validation' is one identical sentence for all 46 KSIs; no per-rule effectiveness results; FRR rules absent from the SDR; nothing supplied to an assessor. To close: Emit per-rule validation results (runtime check outcomes, test runs, drill records with dates) into the SDR for every applicable rule.", + "closes_with": [ + "W3", + "W10" + ] + }, + { + "frr_id": "IVV-CSO-SEI", + "status": "Partially Implemented", + "implementation": "Live security-decision-record.json: 46 rule_records (KSI only) each with a 'verification' field; 333 rev5_controls with mechanisms; scripts/verify-published.sh + docs/verification.md let a third party verify signatures/provenance.", + "evidence": [], + "gap": "The 'verification' text is the same boilerplate sentence for all 46 KSIs (1 distinct value), so it is not per-measure implementation evidence; the SDR has no records for FRR rules (CCM, SCN, SCG, AFC, IVV, VDR, etc.), so 'all applicable FedRAMP rules' have no implementation evidence; nothing has been supplied to an assessor. To close: Generate per-rule verification entries (KSIs and every applicable FRR) that cite the specific check/artifact and result (e.g. OPA rule id + pass/fail, file + line, live check output), and package them for assessor delivery.", + "closes_with": [ + "W3", + "W10" + ] + }, + { + "frr_id": "IVV-CSO-STE", + "status": "Partially Implemented", + "implementation": "docs/verification.md and scripts/verify-published.sh (reproducible signature/provenance/schema checks from the public internet); SDR rev5_controls mechanisms text.", + "evidence": [], + "gap": "No assessor-oriented technical walkthrough per rule/KSI (how each check works, how to run it, expected output), and no assessor to supply it to. To close: Write an assessor guide mapping each rule to its mechanism, command to reproduce, and evidence location; offer a demonstration session.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "IVV-CSO-USR", + "status": "Not Implemented", + "implementation": "No sampling in use.", + "evidence": [], + "gap": "Optional (MAY); not exercised.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "IVV-CSX-AIA", + "status": "Not Implemented", + "implementation": "No FedRAMP independent assessment has taken place. The package is built so an assessor can verify each KSI.", + "evidence": [], + "gap": "Engaging an assessor is the operator's decision; until then no annual independent assessment covers the KSIs (operator's decision of 2026-11-05: assessor-ready, none engaged).", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "MAS-CSO-FLO", + "status": "Partially Implemented", + "implementation": "https://samaydlette.com/.well-known/boundary-map.json: 11 flows with protocol/auth/encryption/data, 137 edges derived from Terraform, per-node classification; flows declared in docs/boundary/boundary-classification.json and resolved against the inventory (unmatched endpoints check = 0); code scripts/build-boundary-graph.py; human view on /trust/ and authorization-boundary page.", + "evidence": [], + "gap": "Flows are not complete for ALL resources: missing user→Cognito Hosted UI authentication, browser→jsDelivr / storage.googleapis.com, Lambda/app→DynamoDB/SQS, CloudTrail/S3/CloudFront→log bucket, operator→AWS console (Duo/root MFA), nightly→GitHub Advisory DB / CISA KEV / detection feeds. Security categories are per component (classification) but flows carry no category. To close: Add the missing flows to the classification, and add a check that every inventoried external service and every data store appears in at least one flow; derive per-flow security category from its endpoints.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "MAS-CSO-IIR", + "status": "Partially Implemented", + "implementation": "Machine-readable: https://samaydlette.com/.well-known/ksi-signal.json components[] (214, from Terraform state + SBOMs) and iiw.csv; human-readable derivation: docs/ksi-signal.md and research/authorization-boundary.html; code: scripts/build-ksi-signal.py; reconcile invariant (a) 'every live in-boundary resource is in the inventory' held.", + "evidence": [], + "gap": "Incomplete: the Silk Reeling SPA loads MediaPipe code from cdn.jsdelivr.net and models from storage.googleapis.com and talks to Cognito Hosted UI (bootstrap/cloudfront.tf:185-186, 378-380); jsDelivr and Google Cloud Storage are not inventoried as information resources although they run code next to the camera feed. The npm package is listed, the delivery services are not. (Repo publicity for the 'code' artifact not verified from here.) To close: Derive external services from the CSP/connect-src allowlists and runtime egress as well as Terraform, so every origin the app loads code or data from becomes an inventoried component; or self-host MediaPipe assets and models.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "MAS-CSO-MDI", + "status": "Not Implemented", + "implementation": "Checked: no metadata inventory; CPO says 'MAS-CSO-MDI N/A (no federal customer data)'.", + "evidence": [], + "gap": "No machine-readable output, explanation or code covering metadata the offering collects: Cognito user records (email, MFA), CloudFront/S3/API Gateway/Lambda logs (IP addresses, user agents), CloudTrail, DynamoDB items, SNS subscriptions. To close: Classify each data store/log as metadata in the inventory (type, contents, retention, location), emit a metadata section from build-ksi-signal.py, and document the derivation.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "MAS-CSO-SUP", + "status": "Not Implemented", + "implementation": "Permissive (MAY). Checked: the package supplies no supplement; CPO package_contents lists only in-scope artifacts.", + "evidence": [], + "gap": "Optional (MAY); not exercised. If coverage material for other frameworks (CMMC, GovRAMP, TX-RAMP on the trust page) is ever put in the package, mark it as supplemental.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "MAS-CSO-TPR", + "status": "Partially Implemented", + "implementation": "ksi-signal.json external_service components (7) carry purpose, cia_impact, verification_mechanism, baseline_configuration; data/third-party-components.json holds responsibility split records; build-ksi-signal.py:1322-1345.", + "evidence": [], + "gap": "All 7 third-party records are undecided (decided_by/decided_on null), so the published upstream_offering/responsibility_split read 'not recorded'. No explicit justification-for-use, mitigation-measures or compensating-controls fields per resource. jsDelivr and Google Cloud Storage (MediaPipe) are missing entirely. Not included in the CPO (schema thirdPartyInformationResources). To close: Add the four required fields (usage/configuration, justification, mitigations, compensating controls) to the record schema, have the operator decide all records, add the MediaPipe hosts, and emit them into the CPO's thirdPartyInformationResources.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "MKT-CSO-MLR", + "status": "Partially Implemented", + "implementation": "CDS-CSO-PUB data exists (see CDS-CSO-PUB).", + "evidence": [], + "gap": "CDS-CSO-PUB is only partial (no UEI, no FedRAMP ID, a receding OCR date, no assessor) and no listing application or update request exists. Not applying is the operator's decision of 2026-11-05 (build to apply, don't apply).", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "MKT-CSO-PML", + "status": "Not Implemented", + "implementation": "Checked: no listing request sent (docs/cr26-applicability.md records MKT as N/A).", + "evidence": [], + "gap": "Marketplace Provider Listing Request Form not submitted. To close: Operator submits the FedRAMP Marketplace Provider Listing Request Form once PUB data and an agency use case are ready.", + "closes_with": [ + "sam" + ] + }, + { + "frr_id": "SCG-CSO-AUP", + "status": "Partially Implemented", + "implementation": "Live policy-index.json (listed in CPO package_contents) has an entry 'Secure Configuration Guide' with file 'docs/policies/secure-configuration-guide.md' (repo-relative path, no URL).", + "evidence": [], + "gap": "The certification package lists no secure configuration guide, and the guide is not served from the site; the OCR's reference to it points nowhere on samaydlette.com.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-CSO-PUB", + "status": "Partially Implemented", + "implementation": "A secure configuration guide exists in the repository only.", + "evidence": [], + "gap": "The guide is not published on the site in human-readable or machine-readable form, and its content predates the current inventory.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-CSO-RSC", + "status": "Partially Implemented", + "implementation": "docs/policies/secure-configuration-guide.md (94 lines): sections 1 (AWS root, l.7-17) and 2 (GitHub owner, l.19-40) give access/config/operation/decommission instructions.", + "evidence": [], + "gap": "Required item 1 is inaccurate: root 'MFA enforced via Duo' (l.11) vs DEC-025 (virtual TOTP on root and operator IAM user); l.29-36 prescribe 90-day rotation of a deployer IAM user access key although CI uses OIDC roles (deploy-with-opa.yml:328,639; DEC-001 closed/remediated); section 3 (l.49-62) still describes the retired Basic Auth credential, 'no authorizer' and 'access logging deferred', contradicting SCN-2026-002. Required item 2 is missing: l.67-76 declare it N/A by misreading it as customer settings; root-only and owner-only security settings (root MFA, account password policy, account-level S3 BPA, account closure; branch protection, Actions permissions, environment reviewers) are not explained with their implications. Recommended item 3 missing: privileged accounts (operator IAM user saydlette-dev, deploy/plan/evidence-nightly roles, Cognito user-pool admin) not covered. Artifacts: no samaydlette.com URL for the human-readable guide and no machine-readable version. To close: Rewrite the guide from the current inventory: correct MFA/credential facts, delete the access-key and Basic Auth material, add a root-only/owner-only settings table with security implications, add a privileged-accounts section, and generate both an HTML page on samaydlette.com and a JSON form from one source each deploy (with a test that fails when the guide names a retired mechanism).", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-CSO-SDF", + "status": "Partially Implemented", + "implementation": "Terraform-provisioned accounts ship hardened: infrastructure/cognito.tf:17 allow_admin_create_user_only, :21 mfa_configuration ON, :27 minimum_length 14; IAM roles least-privilege via Terraform.", + "evidence": [], + "gap": "The top-level administrative accounts (AWS root, GitHub owner) and the operator IAM user are not provisioned from code, and their MFA is virtual TOTP rather than the phishing-resistant default the system itself treats as the target (DEC-025, past due 2026-10-30); no explanation of how to access the defaults (artifact). To close: Move root/operator to phishing-resistant MFA (hardware key/passkey) and close DEC-025; record initial-provisioning defaults for the non-Terraform accounts in the guide and check them.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-ENH-API", + "status": "Partially Implemented", + "implementation": "Guide l.83 points to AWS and GitHub APIs as the upstream APIs.", + "evidence": [], + "gap": "Only a generic pointer; no explanation of which API calls view/adjust which security settings in this system. To close: Document, per setting in the SCG, the AWS/GitHub API or CLI call to read and change it (and that changes go via Terraform/PR).", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-ENH-CMP", + "status": "Partially Implemented", + "implementation": "scripts/check-manual-settings.py (deploy-with-opa.yml:1664) compares live GitHub environment reviewers / required checks to expected values and publishes to trust-center.json (github_settings 'ok'); scripts/check-runtime-divergence.py compares live AWS config to deploy-time signal.", + "evidence": [], + "gap": "No comparison for AWS root settings (MFA type, root access keys absent, password policy) or the operator IAM user; the comparison is against deploy-time state, not against the guide's recommended defaults; GitHub owner settings beyond branch checks 'need admin access and are not observed'. To close: Add checks for root/operator settings (IAM credential report: root MFA, no root keys, MFA type; account password policy; account S3 BPA) and GitHub owner settings, comparing to the machine-readable SCG defaults; publish results.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-ENH-EXP", + "status": "Partially Implemented", + "implementation": "Guide l.81 offers 'all configuration is in Terraform code'; trust-center.json publishes manual-settings results.", + "evidence": [], + "gap": "Terraform source is not an export of current security settings; root and GitHub-owner settings are not exported at all; no machine-readable export of current values. To close: Publish a signed JSON export of current security settings for top-level admin and privileged accounts (from the CMP checks) each nightly.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-ENH-MRG", + "status": "Not Implemented", + "implementation": "None: no machine-readable secure configuration guidance exists.", + "evidence": [], + "gap": "No machine-readable SCG that tools can compare against current settings. To close: Emit the recommended defaults as JSON (setting id, account, recommended value, implication, check id) and use it as the CMP comparison input.", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCG-ENH-VRH", + "status": "Partially Implemented", + "implementation": "Git history of docs/policies/secure-configuration-guide.md.", + "evidence": [], + "gap": "No version number or release history of the recommended defaults; git history mixes editorial and default changes, and the guide's stale content shows changes to defaults were not released into it. To close: Add a version and a changelog of default changes to the machine-readable SCG; bump on each default change (enforced by a test when the defaults JSON changes).", + "closes_with": [ + "W10" + ] + }, + { + "frr_id": "SCN-ADP-NTF", + "status": "Partially Implemented", + "implementation": "Each adaptive SCN record carries a new-risks summary (scn-register.csv new_risks_summary; SCN md section 5).", + "evidence": [], + "gap": "No notification has been sent to anyone: notify_by says 'No necessary parties'; FedRAMP is always a necessary party (FRD 'All Necessary Parties'), so this is not N/A; no sent-date is recorded for any SCN (artifact requires sent date and applied date). To close: Send each adaptive SCN through the NOM mechanism within 10 business days of date_finished, record sent_on in the register, and add a check that fails when date_finished + 10 business days passes without sent_on.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-ARI", + "status": "Partially Implemented", + "implementation": "docs/scn/SCN-2026-001-silk-reeling.md sections 3-5 (boundary delta, SIA by control family, verification result) add information beyond SCN-CSO-INF.", + "evidence": [], + "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-EMG", + "status": "Partially Implemented", + "implementation": "docs/incident-response.md:86-104 emergency-change procedure; scn-tag.yml:53-58 recognizes 'emergency'.", + "evidence": [], + "gap": "Never exercised; 'notify all necessary parties' has no channel; no gate that requires the retroactive SCN materials (only a warning). To close: Add a check that opens/blocks until a retroactive SCN record exists for each emergency-tagged commit within 5 business days; route notification through the NOM mechanism.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-EVA", + "status": "Partially Implemented", + "implementation": ".github/workflows/scn-tag.yml (required check, confirmed live by check-manual-settings 'main requires 4 status checks', EXPECTED_REQUIRED_CHECKS l.39) validates an SCN-Type line; docs/policies/cm-policy.md:5; six adaptive records with categorization rationale (docs/scn/SCN-2026-00[1-6]*.md section 1).", + "evidence": [], + "gap": "Evaluation is not forced: a missing tag passes as routine-recurring (scn-tag.yml:59), and only 8 of 273 commits since 2026-08-01 carry any SCN-Type, so most changes have no recorded evaluation; the gate has no 'not significant' or 'class change' outcome and requires no rationale; commit 437be7b carries 'SCN-Type: Routine Recurring', a value the case statement (l.49-66) rejects. Adaptive tags do not require an SCN record (e.g. #376 tagged adaptive has none). Artifact gap: no example (real or fictitious) for routine-recurring, transformative or class change. SCN-2026-001 (new non-FedRAMP external interconnection) was judged adaptive 'at the upper edge'; the rationale should be revisited. To close: Require an explicit SCN-Type plus a one-line rationale on every PR (fail when missing); add values not-significant and class-change (class-change blocks and points to reassessment); fail when adaptive/transformative lacks a docs/scn record; add documented examples for each type (fictitious allowed).", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-HIS", + "status": "Partially Implemented", + "implementation": "docs/scn/ holds all six SCNs since 2026-06 (README: rows never deleted).", + "evidence": [], + "gap": "The SCN register and records are in the repository only; nothing under /.well-known/ publishes them, and the CPO lists no SCN material or how FedRAMP obtains it.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-HRM", + "status": "Partially Implemented", + "implementation": "Human-readable: docs/scn/*.md and scn-register.csv.", + "evidence": [], + "gap": "No JSON form of SCNs or of the related audit records (CSV is not JSON); no explanation of how to request them or how sharing is decided. To close: Emit SCN JSON (schema-valid) and the evaluation log as JSON; add access + sharing statement to the CPO.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-INF", + "status": "Partially Implemented", + "implementation": "docs/scn/scn-register.csv columns cover all ten items (fedramp_id, assessor_name, related_vulnerability, scn_type, categorization_rationale, short_description, reason_for_change, customer_impact, plan_and_timeline, sia_reference, approver_name/title); per-change SIA in docs/scn/SCN-2026-00x-*.md.", + "evidence": [], + "gap": "Service Offering FedRAMP ID is the placeholder ''; not emitted in FedRAMP's SCN schema (tests/fixtures/cr26/good/scn.json exists but no builder); SCN-2026-006 verification still unrecorded (trust center scn_unverified); some impacted-control lists reference the retired OSCAL SSP/POA&M. To close: Generate per-SCN JSON in fedramp-significant-change-notifications-schema via _fedramp_schema.write_validated; fill the FedRAMP ID when assigned; record SCN-2026-006 verification.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-MAR", + "status": "Partially Implemented", + "implementation": "Git history + scn-tag.yml run logs; docs/scn/README.md 'How this relates' explains git history is the audit record; register and per-change records in docs/scn/.", + "evidence": [], + "gap": "Records of evaluation are incomplete because the default-to-routine path records no evaluation; Actions logs expire (default 90 days), so the gate's decisions are not durable; no explanation in the Certification Package of how FedRAMP obtains the records. To close: Write each evaluation (commit, type, rationale, evaluator, date) to an append-only signed log published with the certification data; state access in the CPO.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-CSO-NOM", + "status": "Partially Implemented", + "implementation": "docs/scn/README.md: notification = publishing updated certification data; register notify_by column says 'No necessary parties ... the public GitHub push ... is the notification'.", + "evidence": [], + "gap": "Mechanism documented only in the repo README, not in the Certification Package (CPO); no list of available notification mechanisms (artifact); a git push is not a notification any party receives. To close: Offer real mechanisms (signed SCN JSON feed + Atom/RSS at /.well-known/, email list) and list them in the CPO.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-RTR-NNR", + "status": "Partially Implemented", + "implementation": "docs/scn/ contains only adaptive SCNs; cm-policy.md:5 routine-recurring changes produce no SCN record.", + "evidence": [], + "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-TRF-NAF", + "status": "Partially Implemented", + "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", + "evidence": [], + "gap": "No post-finish notice step or 5-business-day deadline check. To close: Add post-finish notice fields and a deadline check (finish + 5 business days).", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-TRF-NAV", + "status": "Partially Implemented", + "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", + "evidence": [], + "gap": "No post-verification notice step carrying updates, new risks and the assessment report; no deadline check. To close: Add post-verification notice fields (updates, new risks, SAR link) and a deadline check (verification complete + 5 business days).", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-TRF-NFP", + "status": "Partially Implemented", + "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", + "evidence": [], + "gap": "No final-plan notice step or 10-business-day check. To close: Add final-plan notice fields and gate (>=10 business days before start).", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-TRF-NIP", + "status": "Partially Implemented", + "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", + "evidence": [], + "gap": "Only a sentence in policy; no template, no planned-start field, no 30-business-day check, no channel to notify parties. To close: Add a transformative SCN template with initial-plan and security-impact sections, sent_on fields and a gate that blocks merge of a transformative change until initial notice is >=30 business days old.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-TRF-TPR", + "status": "Not Implemented", + "implementation": "No transformative changes; no recorded policy on third-party review.", + "evidence": [], + "gap": "Neither a third-party review nor an explanation of why one was not engaged exists (artifact accepts either). To close: Add to the transformative template a required 'human validation needed? assessor engaged / reason not engaged' field.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SCN-TRF-UPD", + "status": "Partially Implemented", + "implementation": "Machine-generated documentation (CPO, SDR, OCR) is rebuilt every deploy (deploy-with-opa.yml), so it tracks changes automatically.", + "evidence": [], + "gap": "Hand-written service documentation is not updated after changes: the Secure Configuration Guide still describes the Basic Auth gate retired by SCN-2026-002 (adaptive) months later; no check ties a transformative change to documentation updates within 30 business days. To close: Add a documentation-update checklist/fields to the SCN record (doc, updated_on) and a check that fails 30 business days after finish if listed docs are unchanged; fix the SCG now.", + "closes_with": [ + "W5" + ] + }, + { + "frr_id": "SDR-CSO-FRR", + "status": "Partially Implemented", + "implementation": "data/frr-records.json holds one record per applicable rule (154, derived from the pinned corpus's applicability metadata), each with status, implementation, evidence, gap and the work that closes it; scripts/_frr.py fails the build on a missing or extra record, an Implemented record without evidence, or evidence that does not resolve.", + "evidence": [], + "gap": "The published SDR does not yet carry these records in fedRampRequirements (next W3 PR), and the independent verification and validation halves stay empty until an assessor is engaged (operator's decision of 2026-11-05). To close: Publish the records in the SDR's fedRampRequirements with rule-specific verification and validation drawn from the evidence each cites; flip each record as its workstream lands.", + "closes_with": [ + "W3", + "sam" + ] + }, + { + "frr_id": "SDR-CSO-MTD", + "status": "Partially Implemented", + "implementation": "security-decision-record.json metadata: last_updated (timestamp), source_of_update (build-sdr.py).", + "evidence": [], + "gap": "Version is the constant 'live (regenerated each deploy)', not an identifier; field names do not match the schema (version/lastUpdated/updateSource). To close: Emit a real version (semver bumped on content change, or commit SHA) and the schema's field names.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "SDR-CSX-KMT", + "status": "Not Implemented", + "implementation": "KSI results are published per run (ksi-signal-runtime.json) and VDR counts as a daily trend (vdr-trend.json).", + "evidence": [], + "gap": "The SDR carries no historical metrics per KSI: no 30-day summary and no summary up to a year. No per-KSI history is kept to build them from.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "SDR-CSX-KSI", + "status": "Partially Implemented", + "implementation": "security-decision-record.json under /.well-known/ carries a record per KSI with an explanation and artifacts, built from the KSI signal's checks.", + "evidence": [], + "gap": "Verification and validation are the same boilerplate for every KSI (build-sdr.py), not per-KSI statements of the measures, their cycle, and how their automation was verified and validated.", + "closes_with": [ + "W3" + ] + }, + { + "frr_id": "VDR-CSO-ADT", + "status": "Implemented", + "implementation": ".github/workflows/evidence-nightly.yml (cron '17 7 * * *' and '17 19 * * *'), deploy-with-opa.yml (every push to main), zap-dast.yml (weekly), infrastructure/main.tf:809 runtime emitter on EventBridge rate(1 day), watchdog rate(1 hour) (infrastructure/watchdog.tf:231). Detection, evaluation (policies/triage.rego via OPA), gating, signing and publication are all automated; live vdr-status.json finished_at 2026-11-05T14:05:40Z.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#Detection gate (G7)" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build VDR Report (FedRAMP 20x Class C)" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Vulnerability gate (fix or categorize every vuln)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-status.json" + } + ] + }, + { + "frr_id": "VDR-CSO-AKE", + "status": "Partially Implemented", + "implementation": "policies/triage.rego:385 block_rule: any KEV blocks (G1, gate-modes.json 'vdr.kev' mode block, even when dispositioned). deploy-with-opa.yml: Grype scan (1002) runs before Terraform Apply (1171).", + "evidence": [], + "gap": "The KEV check doesn't run before activation. In deploy-with-opa.yml the KEV-blocking VDR build ('Build VDR Report', line 1428) and 'Vulnerability gate' (1454) run *after* 'Terraform Apply' (1171) and 'Sync Website Content' (1200). A new Lambda package or SPA bundle with a KEV dependency is deployed and live before the gate fails the run; G7 at 1157 only checks that feeds exist and are fresh. No pre-apply check either for KEVs in third-party runtimes (e.g. the Lambda runtime version). To close: Add a pre-apply KEV check: after the Grype step and KEV fetch, intersect grype.json CVEs with kev-catalog.json and fail before 'Generate fresh Terraform plan'/'Terraform Apply' and before the S3 sync. Add a test proving a KEV fixture stops the deploy pre-apply.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-CSO-DAC", + "status": "Partially Implemented", + "implementation": "deploy-with-opa.yml on every push to main: 'Security Scan' job (tfsec + Checkov), 'Scan application and Lambda dependencies (Syft SBOM + Grype)' (line 1002), 'Detection gate (G7)' (1157), 'Build VDR Report' (1428), OPA plan check on the Terraform plan ('Run OPA Compliance Check').", + "evidence": [], + "gap": "Changed IaC and dependency trees are scanned on every change, but the changed *running* surfaces aren't: no DAST runs after a deploy (ZAP is weekly on a schedule, zap-dast.yml cron '23 7 * * 1'), and changed first-party code gets no SAST. A change to the Silk Reeling API or site pages isn't dynamically tested until the next Monday, and the API never is. To close: Trigger the ZAP workflow (baseline + authenticated API scan) via workflow_run after a successful deploy that changed website/ or the Silk Reeling app, and add code SAST to the deploy's Security Scan job.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-CSO-DET", + "status": "Partially Implemented", + "implementation": ".github/workflows/evidence-nightly.yml (twice-daily cron, 07:17/19:17): job 'SAST (Checkov + tfsec)' over all Terraform; step 'Scan application and Lambda dependencies (Syft SBOM + Grype)' over Silk Reeling Python/JS, compliance Lambda and a11y tooling; Dependabot alerts, CISA KEV and FIRST EPSS via scripts/fetch-detection-feeds.sh; 'Reconciliation gate' runs scripts/reconcile.py --live (live completeness (a) and tag (i) invariants). .github/workflows/zap-dast.yml: weekly ZAP baseline on https://samaydlette.com. infrastructure/lambda/index.js: daily runtime OPA (Wasm) re-validation. VDP: https://samaydlette.com/.well-known/security.txt (fetched). Live vdr-report.json methodology.detection (fetched 2026-11-05): all feeds ok.", + "evidence": [], + "gap": "Some techniques and resources aren't covered. (1) No DAST of the Cognito-gated Silk Reeling API Gateway/Lambda; ZAP is unauthenticated and targets the static site only (zap-dast.yml header: authenticated coverage is a 'Phase 2 follow-up'). (2) No SAST of first-party application code (Silk Reeling Python Lambda, SPA JS, compliance Lambda): no CodeQL/Semgrep/Bandit anywhere in .github/. (3) No penetration testing, ever. (4) No cloud-native threat/vulnerability detection (GuardDuty, Inspector, IAM Access Analyzer, Security Hub: none in infrastructure/*.tf). (5) 'Persistently verifying and validating that resources operate as intended' covers only 5 live components at runtime (see VDR-TFR-MVX) and nothing for non-machine resources (see VDR-TFR-NMV). (6) Inventory gaps: 5 of 35 live findings name IAM policies that don't resolve to an inventory component (irv_basis 'unresolved'). To close: Add authenticated ZAP (or API scan from the OpenAPI/route list) against the Silk Reeling API with a test Cognito user; add CodeQL or Semgrep on scripts/, the Lambdas and the SPA, ingested into the VDR; enable IAM Access Analyzer (free) and either Inspector Lambda scanning or GuardDuty, ingested as a VDR source; commission or perform and record an annual penetration test of the app; close the inventory gaps so every finding resolves.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-CSO-DFR", + "status": "Implemented", + "implementation": "Architecture: static content on private S3 behind CloudFront (infrastructure/main.tf), serverless Silk Reeling (API Gateway -> Lambda, Cognito-gated; infrastructure/silk-reeling.tf, cognito.tf); no servers/OS to patch; all infra as Terraform with OPA plan gating (deploy-with-opa.yml 'Run OPA Compliance Check'); runtime re-evaluation reuses the same Rego as Wasm (infrastructure/lambda/index.js header). The live inventory (ksi-signal.json) has no compute instances or containers.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#Run OPA Compliance Check" + }, + { + "kind": "decision", + "ref": "DEC-007" + }, + { + "kind": "decision", + "ref": "DEC-009" + }, + { + "kind": "decision", + "ref": "DEC-026" + } + ] + }, + { + "frr_id": "VDR-CSO-FAV", + "status": "Partially Implemented", + "implementation": "scripts/detection-gate.py (G7, 'fail the run when detection itself failed (T52, VDR-CSO-FAV)'); scripts/_gates.py + infrastructure/schemas/gate-modes.json (block by default, detect-and-recover events published in methodology.gate_events); build-vdr-report.py main(): missing/stale ZAP report blocks (detection.zap.missing/stale); infrastructure/watchdog.tf + infrastructure/watchdog/logic.mjs: hourly freshness alarms emailed via SNS; deploy-with-opa.yml job 'Notify on Failure' opens an issue; evidence-nightly.yml failure summary.", + "evidence": [], + "gap": "Detection and response failures are alarmed and stop the run, but they aren't treated *as vulnerabilities*: no failure becomes a tracked finding with a tracking ID, PAIN/IRV/LEV evaluation, a Class C deadline, a disposition, and an entry in the VDR/VDT. A stale feed, a skipped DAST or a watchdog breach leaves only a failed run, an email or an issue, so its history and time-to-recover aren't in the vulnerability record. To close: When a G7 check, a watchdog alarm or a gate in detect-and-recover mode fails, emit a synthetic finding (source 'process', e.g. process-detection-zap-stale) into the next VDR with first_detected = failure time, a governed PAIN for process failures, and normal deadline/disposition/remediation handling. Close it automatically when the check passes again (Remediated with time).", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-CSO-MSP", + "status": "Implemented", + "implementation": "Checked: security/zap/allowlist.json has entries [] ('No edge allowlist exists for the scanner today'); the VDR builder flags any allowlisted DAST finding (build-vdr-report.py:946-983, REQ-IRV-11). Scanner/reconcile AWS access is read-only (infrastructure/bootstrap/evidence-nightly.tf:142 read-only List*/Describe* policy). ZAP scans production unauthenticated as a stranger would; no security controls were disabled or opened for scanning.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_zap_allowlist.py::test_allowlist_entry_without_a_decision_fails_closed" + }, + { + "kind": "test", + "ref": "tests/test_zap_allowlist.py::test_allowlist_status_is_published_in_the_report" + } + ] + }, + { + "frr_id": "VDR-CSO-RES", + "status": "Partially Implemented", + "implementation": "scripts/build-vdr-report.py: ledger carries first_detected/first_evaluated/clock across builds (1142 ingest_previous_ledger, 1176 clock_facts), rating history and completed reductions (1225); policies/triage.rego decides IRV/LEV/disposition/deadline/blocking per finding; verified mitigations against the signed runtime signal (build-vdr-report.py:387 mitigation_state); scripts/vuln-gate.py forces fix-or-decide; data/decisions.json register (39 records); reconcile.py invariant (h) needs a decision for every finding; published nightly (live vdr-report.json, 35 findings, all dispositioned).", + "evidence": [], + "gap": "'Assess exploitation of' isn't mechanized: the 'exploitation' fact read by triage.rego (figure7 'exploitation-observed') is never set by any ingester (build-vdr-report.py:1345 reads f.get('exploitation'); nothing writes it), and nothing checks logs/CloudTrail for exploitation of detected vulnerabilities. Monitoring/reporting halves are partial; see VER-RPT-* (CR26 reports not yet live, supplementary info never filled). To close: Add an exploitation-assessment step: for each open IRV/LEV finding, query CloudFront/API Gateway/CloudTrail logs for indicators tied to the advisory (or record a dated operator assessment in the decision register), and set exploitation='active'|'none-observed' with basis on the finding. Merge the CR26 reports PR.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-CSO-SIR", + "status": "Not Implemented", + "implementation": "Checked: no sampling is used. Grype scans every SBOM'd dependency tree, Checkov/tfsec scan all Terraform, and the runtime emitter validates each object store and the distribution individually (live ksi-signal-runtime.json: 5 validations, one per component).", + "evidence": [], + "gap": "Optional (MAY); not exercised. Note that coverage gaps are a DET/MVX matter, not sampling.", + "closes_with": [ + "optional" + ] + }, + { + "frr_id": "VDR-TFR-KEV", + "status": "Partially Implemented", + "implementation": "scripts/build-vdr-report.py:1054 ingest_kev loads the CISA catalog; triage.rego:385 blocks any KEV (even when dispositioned or fully mitigated) via G1; detection-gate.py checks catalog freshness (14 days, detection-feeds.json). Live: kev feed ok, 1768 entries, 0 KEV findings.", + "evidence": [], + "gap": "CISA due dates aren't tracked: ingest_kev keeps only cveID and drops dueDate/dateAdded, so the report never states the BOD 26-04 due date or whether remediation met it. A KEV blocks publication, but nothing records a remediation deadline tied to the catalog. A fully mitigated KEV is still blocked (good), but its remediation due date isn't reported either. G1 runs after apply in deploys (see AKE). To close: Keep {cveID: dueDate, dateAdded} from the catalog; set remediation_due_at = min(Class C deadline, CISA dueDate) for KEV findings even when fully mitigated; publish kev_due_date and overdue state in the VDR/VDT providerExtensions; add tests.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-TFR-MVX", + "status": "Partially Implemented", + "implementation": "Runtime emitter daily (infrastructure/main.tf:809, variables.tf:144 rate(1 day)), live ksi-signal-runtime.json emitted 2026-11-05T14:01:20Z: 5 validations (4 object stores + CDN), all pass; divergence 'converged' over 46 KSIs. reconcile.py --live twice daily: live completeness and tags for every in-boundary resource. Watchdog alarms if the runtime signal is >26h old.", + "evidence": [], + "gap": "Of the ~30 cloud components in the live inventory (3 functions, API Gateway, Cognito, 6 IAM roles, 17 IAM policies, 4 KMS keys, log groups, alarms, DNS, SNS, SQS, DynamoDB, OIDC provider), only 5 have their configuration status verified and validated within 3 days; the rest get existence/tag checks only. Their full validation (52 validations) lives in the deploy-time signal, which may be up to 30 days old (freshness-config.json). Packages/html artifacts are covered by Grype/ZAP, not V&V. To close: Extend the runtime emitter (or a nightly OPA pass) to read live config for every machine component type and evaluate it against the same Rego, so ksi-signal-runtime.json carries one validation per machine component. Add a reconcile invariant that fails when any machine component lacks a runtime validation younger than 72h.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-TFR-NMV", + "status": "Not Implemented", + "implementation": "docs/security-review.md 'Quarterly continuous-monitoring reviews': one entry, 2026-Q2 dated 2026-05-08, 'Next review due: 2026-08-08'; no later entry anywhere in docs/ (grep for 2026-Q3). docs/training-log.md and docs/policies/*.md carry no dated quarterly verification. scripts/check-manual-settings.py observes three hand-applied *machine* settings on each deploy.", + "evidence": [], + "gap": "Non-machine resources (policies in docs/policies/, procedures/runbooks, the IR plan, the operator's training/role, decision records, the governance registry docs/governance/domains.json) have no verification-and-validation at least every 3 months. The only quarterly review on record is ~6 months old and overdue since 2026-08-08, and it reviewed machine signals (runtime emitter, VDR), not policies/people. No mechanism alarms when the 3-month window passes. To close: Create a non-machine resource register (each policy/procedure/role with owner, last_verified, evidence link) and a quarterly V&V record that checks each against what's actually practiced. Add a freshness entry (max 92 days) to infrastructure/schemas/freshness-config.json so the watchdog/trust center goes red when it lapses. Do the overdue Q3/Q4 review now.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-TFR-PCD", + "status": "Partially Implemented", + "implementation": "Nightly (twice daily) Grype over every dependency tree and Checkov/tfsec over all Terraform (evidence-nightly.yml 'SAST (Checkov + tfsec)', 'Scan application and Lambda dependencies'); weekly ZAP over the static site; detection-gate enforces feed freshness (infrastructure/schemas/detection-feeds.json).", + "evidence": [], + "gap": "Monthly detection doesn't reach every resource: the Silk Reeling API/app surface gets no dynamic detection and first-party code gets no SAST (see DET). The rule's 'all information resources' includes non-machine resources, which get no vulnerability detection at all (see NMV). Nothing maps inventory components to the detection that covered them in the month, so completeness can't be shown. To close: Add a per-component coverage record to the VDR (component -> detection techniques -> last run) and fail/alarm when any inventory component has no detection in 30 days. Close the API DAST and code SAST gaps; fold the quarterly non-machine review in.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-TFR-PDD", + "status": "Partially Implemented", + "implementation": "Twice daily: reconcile.py --live (evidence-nightly.yml:605) enumerates live AWS for completeness (a) and classification tags (i); Grype on all dependency trees; Checkov/tfsec on source. Daily: runtime emitter (infrastructure/lambda/index.js, rate(1 day)) re-evaluates live S3 buckets and the CloudFront distribution against the deploy Rego and computes divergence (live ksi-signal-runtime.json: 5 validations, divergence 'converged', 46 KSIs compared). Hourly watchdog for evidence availability.", + "evidence": [], + "gap": "Drift detection on live configuration covers only the 4 buckets and the distribution. Live drift in IAM roles/policies (privileges), the 3 Lambda functions (code/config), API Gateway, Cognito, KMS, DNS, Secrets isn't detected within 14 days: those are re-validated only on a human-initiated deploy (deploy-time ksi-signal.json, 52 validations, which freshness-config allows to be 30 days old) and by existence/tag checks. scripts/check-silk-runtime-config.py runs only in the deploy. No document says which resources are 'likely to drift'. To close: Extend buildRuntimeSignal (infrastructure/lambda/index.js:299) to read and evaluate live config for IAM roles/policies, Lambda (code SHA, env, runtime), API Gateway routes/authorizers, Cognito pool settings and KMS policies against the same Wasm policy; or run check-silk-runtime-config.py and a live OPA pass in the nightly. Record which component types are classed 'likely to drift' in the inventory.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-TFR-PSD", + "status": "Partially Implemented", + "implementation": "Every 12h: Grype over all dependency SBOMs, Checkov/tfsec over all IaC (evidence-nightly.yml). Daily: runtime OPA over live S3/CloudFront (infrastructure/lambda/index.js). Weekly: ZAP baseline.", + "evidence": [], + "gap": "Over a 3-day window, live machine resources other than S3/CloudFront get no vulnerability detection: Lambda functions, API Gateway, Cognito, IAM. DAST of the web surface runs every 7 days, not 3, and the API never. To close: Run the ZAP baseline (and the authenticated API scan once built) at least every 3 days, e.g. in the nightly or a 2-day cron with the --zap-max-age-days window set to 3. Extend runtime live-config detection to all machine component types (see PDD).", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VDR-TFR-PVR", + "status": "Implemented", + "implementation": "policies/triage.rego:87 timeframe() reads the Class C matrix from the pinned FedRAMP rules (data.rules ... VDR-TFR-PVR.varies_by_class.c, hash-checked by scripts/_rules.py) by PAIN x IRV x LEV; deadline from clock_start (triage.rego:363), clock moved only by a verified mitigation (build-vdr-report.py:1176 clock_facts); re-evaluation re-selects the timeframe; G3 'vdr.past-deadline' blocks publication (gate-modes.json mode block); likely-overdue margin (vdr-pain-config.json). Live vdr-report.json carries remediation_due_at per finding.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_triage_policy.py::test_timeframe_comes_from_data_rules" + }, + { + "kind": "test", + "ref": "tests/test_triage_policy.py::test_rules_missing_a_timeframe_cell_fail_closed" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build VDR Report (FedRAMP 20x Class C)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-report.json" + } + ] + }, + { + "frr_id": "VDR-TFR-RMN", + "status": "Implemented", + "implementation": "N1 findings get no timeframe (triage.rego:87 'null for a level with no timeframe (N1: routine operations)'), yet scripts/vuln-gate.py and reconcile.py invariant (h) still need every finding fixed or dispositioned by an in-force decision with review_by. Live: both N1 findings (tfsec-AVD-AWS-0162, zap-10036-2) carry decisions DEC-032/DEC-040.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_rules_as_data.py::test_t19_n1_has_no_timeframe" + }, + { + "kind": "test", + "ref": "tests/test_vuln_gate.py::test_remediate_disposition_does_not_pass" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Vulnerability gate (fix or categorize every vuln)" + }, + { + "kind": "decision", + "ref": "DEC-040" + } + ] + }, + { + "frr_id": "VER-EVA-AIA", + "status": "Partially Implemented", + "implementation": "triage.rego figure7 'floor': IRV + AV:N/PR:N/UI:N with direct impact is LEV regardless of EPSS (REQ-LEV-3); no score defaults to LEV.", + "evidence": [], + "gap": "No explicit 'automatable' assumption is recorded or evaluated. The floor covers only unauthenticated, no-interaction network vectors. An EPSS score below 0.10 makes a finding NLEV with no evidence that its exploitation can't be automated, so the rule's 'UNLESS they have evidence proving otherwise' isn't enforced. To close: Add an 'automatable' fact per finding (default true; CVSS 4.0 AU or SSVC Automatable when present), let only an evidence-bearing decision record set it false, and make automatable=true block the EPSS-only NLEV path (or document why EPSS < threshold is evidence). Publish automatable + basis.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-EVA-EFA", + "status": "Partially Implemented", + "implementation": "Criticality: inventory classification -> CR/IR/AR (build-vdr-report.py:1680). Reachability: IRV (triage.rego reach/irv). Exploitability: Figure 7/EPSS/CVSS. Known threats: CISA KEV + EPSS. Prevalence: consolidate_by_cve counts assets (display only). Privilege: CVSS PR in the LEV floor only.", + "evidence": [], + "gap": "No factor for Detectability (how easily an attacker learns of the vulnerability) or Proximate Vulnerabilities (interaction with earlier or partially mitigated findings). Prevalence and Privilege (privilege gained by exploitation) don't feed PAIN or LEV. For config findings (all current findings) only Reachability is actually evaluated (see ELX/EPA). To close: Add the missing factors to the evaluation record: detectability (e.g. public advisory + exposed surface), prevalence (asset count -> raise PAIN when >1 component), privilege (scope change/privileged component -> raise PAIN), proximate (link to other open/mitigated findings on the same component and re-evaluate together). Document each in the method profile and publish per finding.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-EVA-EFP", + "status": "Implemented", + "implementation": "False-positive is a governed disposition: decision records with justification, decided_by/on, review_by (data/decisions.json: 13 false-positive records, e.g. DEC-039 ZAP 10099 'pattern matched prose', DEC-040 'Server: AmazonS3 carries no version'); triage.rego:262 record_status rejects a false positive that names a control (it must be 'mitigated') and lapses at review_by; live report: 30 findings with final_disposition false-positive, each with a decision_id.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_dispositions_mitigations.py::test_t15_false_positive_naming_a_control_is_refused" + }, + { + "kind": "test", + "ref": "tests/test_vuln_gate.py::test_false_positive_passes" + }, + { + "kind": "decision", + "ref": "DEC-039" + }, + { + "kind": "decision", + "ref": "DEC-040" + } + ] + }, + { + "frr_id": "VER-EVA-EIR", + "status": "Partially Implemented", + "implementation": "policies/triage.rego:189 reach() / :199 irv(): outside-in observation (ZAP) -> IRV; unresolved component or package -> IRV; else the inventory's declared internet_reachable; verified 'not-reachable' mitigation -> NIRV; irv_basis published per finding.", + "evidence": [], + "gap": "In the live report, 5 findings and all 3 suppression records have irv_basis 'unresolved': their resource (e.g. aws_iam_group_policy.operators_assessment_readonly, aws_iam_role_policy_attachment.legacy, .checkov.yaml) doesn't resolve to an inventory component, so IRV=true was assumed, not evaluated. The suppression records name '.checkov.yaml' as the resource, not the affected AWS resource. To close: Add the bootstrap-stack IAM policies to the canonical inventory with declared reachability; make suppression records name their real resources (from the decision's applies_to.resources); make unresolved_components > 0 a reconciliation failure, or at least a recorded detect-and-recover event.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-EVA-ELX", + "status": "Partially Implemented", + "implementation": "policies/triage.rego:207 figure7 (fully mitigated -> KEV -> exploitation observed -> floor IRV+AV:N/PR:N/UI:N -> no direct impact -> EPSS>=0.10 -> no score = LEV), plus verified 'not-likely-exploitable' mitigations and governed LEV overrides; per-finding lev_basis published. EPSS feed live (400,564 scores).", + "evidence": [], + "gap": "For every current finding, exploitability rests on the default, not an evaluation: all 35 live findings have lev_basis 'no-score' (config/DAST findings have no vector or EPSS; methodology.check_vectors_in_force = 0, lev_overrides_in_force = 0). So, for example, two ZAP false positives and a CloudTrail SNS setting are all 'likely exploitable', with no context considered. Only CVE findings get a contextual LEV path. To close: Put governed CVSS vectors in force for the Checkov/tfsec check IDs that recur (infrastructure/schemas/check-vectors.json needs decided_by/decided_on), and add a per-finding LEV judgment record (in the decision register) for DAST/config findings so LEV reflects the system context. Keep the default for anything unevaluated.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-EVA-EPA", + "status": "Partially Implemented", + "implementation": "scripts/build-vdr-report.py:1730 assign_pain: CVSS-Environmental derivation for CVE findings, using asset CR/IR/AR from the inventory classification and a multi-agency scope term, bucketed into minimal/narrow/disruptive/debilitating x 1/>1 agency (_WORD_SCOPE_TO_N); governed config infrastructure/schemas/vdr-pain-config.json.", + "evidence": [], + "gap": "Findings without a CVSS vector, which is every current finding, use a severity proxy (SEVERITY_TO_PAIN, build-vdr-report.py:515: scanner severity -> N-level) with no system or agency context. N0 is never assigned: the scale is read from the PVR keys N1-N5 and fully mitigated maps to the lowest level N1 (triage.rego impact()), though EPA defines N0. The narrative 'potentialAgencyImpact' in the VDT is boilerplate ('Potential Agency Impact N2 (technical impact ...)'), not an estimate of effect on agencies. To close: Same governed per-check vectors as ELX so config findings go through the contextual derivation; add N0 to the scale for fully mitigated/no-effect cases; write a one-line agency-effect rationale per finding (or per check ID) into the record and the VDT potentialAgencyImpact.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-EVA-GRV", + "status": "Partially Implemented", + "implementation": "scripts/build-vdr-report.py:1777 consolidate_by_cve groups CVE findings across scanners/assets into cve_findings (live: 0 CVEs); decisions group check IDs/resources (e.g. DEC-032 covers CKV2_AWS_10, CKV_AWS_252, AVD-AWS-0162).", + "evidence": [], + "gap": "Grouping is for display only: deadlines, LEV/IRV and reporting still apply per instance (VDT emits one record per scanner finding; the 3 CloudTrail findings under DEC-032 are 3 VDT/AVI records). Config findings are never grouped by check or root cause, so the rules aren't 'applied to the consolidated groupings'. To close: Make a group (CVE, or check ID + root cause, or decision) the unit for clock, disposition and VDT record, listing its member instances/assets in providerExtensions, and derive the group's PAIN/IRV/LEV as the worst of its members.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-RPT-AVI", + "status": "Partially Implemented", + "implementation": "vdr-accepted-vulnerabilities.json and .md under /.well-known/ (scripts/build-cr26-vdr.py, #496): each accepted vulnerability's detail and acceptance rationale from its decision record; the build fails if a rationale is missing.", + "evidence": [], + "gap": "Supplementary information is never populated. For the three suppression records, the detection time is when the ledger first saw the suppression rather than when the weakness was found, and the resource is the suppression file, so IRV is unresolved. To close: Take the detection date from the decision; resolve suppressions to the decision's resources so IRV is evaluated; populate supplementary information.", + "closes_with": [ + "W2" + ] + }, + { + "frr_id": "VER-RPT-HLO", + "status": "Partially Implemented", + "implementation": "Live vdr-report.json methodology.detection lists each feed/scanner with status, count, source date and age; vdr-status.json shows gate results; security.txt is the VDP.", + "evidence": [], + "gap": "There's no high-level overview of *all* VDR activities in the period: no summary of DAST runs, the manual ZAP scans of 2026-10-26/27, VDP reports received (none recorded), penetration testing (none), assessments (the self-performed 2026-05-08 assessment), or remediations completed. The detection block is a machine feed inventory for one run, not a period overview. To close: Add an 'activities' section to the period report (VDT providerExtensions and the .md): each activity type (SCA, IaC SAST, DAST, VDP, pentest, assessment, runtime V&V) with runs in period, coverage, findings opened/closed, and an explicit 'none this period' when it didn't happen.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-RPT-NID", + "status": "Partially Implemented", + "implementation": "The trust center and boundary map refuse ARNs, account IDs and email addresses before publishing, and since W0 every published artifact passes the account-ID gate (scripts/check-published-redaction.py).", + "evidence": [], + "gap": "Everything is published publicly the moment it's detected, with no sensitivity review: no rule or step decides what is too specific to disclose before remediation, and the VDR has no redaction pass. A real IRV+LEV CVE would be published with package, version, component and deadline before it's fixed, which the FRD 'Responsibly' definition warns against. The 'sufficient information to all necessary parties' half is met, but with no restricted channel there's no way to give FedRAMP/agencies detail while withholding it from the public. To close: Define a disclosure policy: until remediation, public artifacts carry only tracking ID, rating, IRV/LEV, dates and status for IRV+LEV findings at N3+; full detail goes to a restricted artifact (signed, access-controlled, e.g. a Cognito-gated or presigned location) for necessary parties. Apply a redaction pass + test to vdr-report/VDT/AVI, and record the disclosure decision per finding.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-RPT-PER", + "status": "Partially Implemented", + "implementation": "vdr-report.json, vdr-trend.json, vdr-status.json and ksi-signal-runtime.json are refreshed twice daily and on each deploy; the VDT, AVI and history (#496) cover the period since the previous report.", + "evidence": [], + "gap": "No report summarizes the period's activity (detections, evaluations, reductions, remediations, scans and V&V runs with times), and V&V results are not cross-referenced from the period's report. To close: Add an activity summary for the period to each report (detections, evaluations, reductions, remediations, scans with times, V&V runs and failures), linking the runtime results for the period.", + "closes_with": [ + "W2", + "W7" + ] + }, + { + "frr_id": "VER-RPT-RPD", + "status": "Partially Implemented", + "implementation": "Checked: the provider discloses publicly (live /.well-known/vdr-report.json and .md; public data/decisions.json).", + "evidence": [], + "gap": "The permission depends on the provider having *determined* that disclosure won't likely lead to exploitation; no such determination is recorded per finding or as policy (see NID). To close: Record the determination: a disclosure policy plus a per-finding 'public_disclosure' field (full/summary/withheld, with reason) set by the triage policy and published.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-RPT-VDT", + "status": "Partially Implemented", + "implementation": "vdr-detail-report.json and .md under /.well-known/ (scripts/build-cr26-vdr.py, #496), validated against the vendored VDT schema before writing: tracking ID, detection time and source, evaluation time, IRV, LEV, current rating, PAIN reductions, projected next reduction, overdue status and final disposition.", + "evidence": [], + "gap": "Historical PAIN and rating history are not carried into the record; the projected next reduction is always the remediation deadline; supplementary information is never produced; overdue explanations for closed findings describe the suppression instead; ZAP detection times start when the scans entered the pipeline, not at the earlier manual scans. To close: Carry historical PAIN and rating history; let a decision set a planned next reduction; populate supplementary information from the decision; take detection dates from the decision where the scanner never saw the finding.", + "closes_with": [ + "W2" + ] + }, + { + "frr_id": "VER-TFR-EVU", + "status": "Partially Implemented", + "implementation": "Evaluation is automatic in the build that first ingests a finding: clock_facts sets first_evaluated = now on first sight (build-vdr-report.py:1176); detection feeds are fetched twice daily, so first_evaluated - detection is < 1 day. scripts/_rules.py:129 evaluation_days() reads the 5-day rule.", + "evidence": [], + "gap": "The timing is met in practice, but nothing enforces or reports it: evaluation_days() is never called, so no check that evaluation completed within 5 days of the source's detection (e.g. Dependabot/Grype detection times). The evaluation completed within that window is the default-driven one for config/DAST findings (see ELX/EPA/EFA), not a full VER-EVA evaluation. To close: Use _rules.evaluation_days() in the triage policy: flag (and count as a process vulnerability, see FAV) any finding whose first_evaluated is more than 5 days after its source detection time, or whose LEV/PAIN still rests on a default 5 days after detection.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-TFR-IRI", + "status": "Not Implemented", + "implementation": "Checked: policies/triage.rego block_rule 'top-impact-lev-irv' (G2) blocks only N5+LEV+IRV, as a pipeline block; docs/policies/ir-policy.md:13 states the IRI rule in prose.", + "evidence": [], + "gap": "Nothing treats an internet-reachable likely exploitable vulnerability at N4 or N5 as a FedRAMP Reportable Incident: no incident record is opened, no IIR/OIR/FIR clocks start, and nothing ends that status when it's partially mitigated to N3 or below. N4 IRV+LEV isn't even flagged beyond its 4-day deadline. Depends on the missing IEC machinery. To close: In triage.rego add an 'incident' decision (IRV and LEV and PAIN > N3) and in the builder open/update an incident record in the incident register (see IEC-CSO-EFR) with default PAIN-5 treatment, closing it when a verified partial mitigation brings it to N3 or below. Test with a synthetic finding.", + "closes_with": [ + "W6" + ] + }, + { + "frr_id": "VER-TFR-MAV", + "status": "Partially Implemented", + "implementation": "policies/triage.rego:372 accepted_ns = first_evaluated + 192 days (read from data.rules VER-TFR-MAV), :376 accepted_now; build-cr26-vdr.py treats accepted/risk-accepted/operational-requirement decisions and MAV-passed findings as accepted; tests/test_clock.py:136 test_t14_past_maximum_period...", + "evidence": [], + "gap": "Only the 'is not' half is mechanized: a finding becomes accepted only once day 192 has passed. Nothing categorizes a finding that *will not* be fully mitigated or remediated within 192 days (e.g. a vendor-dependency with no fix, or a 'remediate' decision whose plan date is past day 192) as accepted earlier. Never exercised live (oldest first_evaluated 2026-08-06, day ~91). To close: In triage.rego, mark accepted when the projected remediation/next-reduction date (or a decision's target date, or vendor-dependency with fix_state not-fixed past a governed horizon) falls after first_evaluated + 192 days; require an acceptance explanation at that point. Add tests.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-TFR-MHR", + "status": "Partially Implemented", + "implementation": "vdr-report.md under /.well-known/ is rebuilt twice daily (scripts/build-vdr-report.py render_markdown); since W1 its counts read Findings / Open / Dispositioned and accepted risks agree with the tables. vdr-detail-report.md and vdr-accepted-vulnerabilities.md are published (#496).", + "evidence": [], + "gap": "It is a current-state report, not a report of the month's activity: no detections, evaluations, reductions, remediations, scans or V&V runs for the period.", + "closes_with": [ + "W7" + ] + }, + { + "frr_id": "VER-TFR-MRH", + "status": "Partially Implemented", + "implementation": "vdr-history.json under /.well-known/ in FedRAMP's history schema (#496), rebuilt twice daily with 366-day retention, alongside vdr-report.json (per-finding rating history) and vdr-trend.json.", + "evidence": [], + "gap": "A record that leaves the VDR is labelled Remediated even when it vanished because coverage stopped or a suppression was added; closed decisions from before the history began are not backfilled. To close: Set a final disposition only from evidence (coverage continued and the scanner stopped reporting it, or a decision); otherwise record 'no longer reported'; backfill closed decisions.", + "closes_with": [ + "W2" + ] + }, + { + "frr_id": "VER-TFR-NRI", + "status": "Not Implemented", + "implementation": "Checked: docs/policies/ir-policy.md:13 adopts NRI in prose ('non-internet-reachable likely-exploitable vulnerabilities at PAIN N5 are reportable until N4 or below per VER-TFR-NRI'); no code implements it.", + "evidence": [], + "gap": "MAY rule: optional, but the published policy claims it's adopted and nothing implements it. NLEV-IRV N5 findings never open an incident. To close: Either implement alongside IRI (NIRV and LEV and PAIN = N5 -> incident until a partial mitigation brings it to N4) or remove the claim from ir-policy.md and record the decision not to use the MAY.", + "closes_with": [ + "W6" + ] + } + ] +} diff --git a/infrastructure/schemas/frr-records.schema.json b/infrastructure/schemas/frr-records.schema.json new file mode 100644 index 00000000..45342c6b --- /dev/null +++ b/infrastructure/schemas/frr-records.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://samaydlette.com/schemas/frr-records.schema.json", + "title": "FedRAMP rule records (SDR-CSO-FRR)", + "description": "One record per FedRAMP rule applicable to 20x Program Class C. scripts/_frr.py enforces this shape and what a schema cannot: the record set equals the applicable rules in the pinned corpus, every evidence reference resolves, and 'optional' is used only for MAY rules.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "basis", "records"], + "properties": { + "$comment": {"type": "string"}, + "schema": {"const": "frr-records/1"}, + "basis": {"type": "string", "minLength": 1}, + "records": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["frr_id", "status", "implementation"], + "properties": { + "frr_id": {"type": "string", "pattern": "^[A-Z]{3}-[A-Z]{3}-[A-Z]{3}$"}, + "status": {"enum": ["Implemented", "Partially Implemented", "Not Implemented"]}, + "implementation": {"type": "string", "minLength": 1}, + "evidence": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "ref"], + "properties": { + "kind": {"enum": ["test", "step", "artifact", "decision"]}, + "ref": {"type": "string", "minLength": 1} + } + } + }, + "gap": {"type": "string", "minLength": 1}, + "closes_with": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"enum": ["W2", "W3", "W4", "W5", "W6", "W7", "W8", "W9", "W10", "sam", "optional"]} + } + }, + "if": {"properties": {"status": {"const": "Implemented"}}}, + "then": { + "required": ["evidence"], + "properties": {"evidence": {"minItems": 1}}, + "not": {"anyOf": [{"required": ["gap"]}, {"required": ["closes_with"]}]} + }, + "else": {"required": ["gap", "closes_with"]} + } + } + } +} diff --git a/scripts/_frr.py b/scripts/_frr.py new file mode 100644 index 00000000..d18250f8 --- /dev/null +++ b/scripts/_frr.py @@ -0,0 +1,261 @@ +"""The FedRAMP rule records: data/frr-records.json (SDR-CSO-FRR). + +One record per FedRAMP rule that applies to this system's certification (20x, +Program path, Class C), saying whether the rule is implemented, how, what shows +it, and, when it isn't, what is missing and which piece of work closes it. The +Security Decision Record publishes these records; nothing else may say a rule +is met. + +The register is only allowed to claim what the repository can show: + +- The applicable rule set comes from the pinned corpus (_rules.applicable_provider_rules). + A missing or extra record fails, so a re-pinned corpus that adds a rule + cannot be published past. +- "Implemented" needs at least one piece of evidence, and every piece of + evidence on any record must resolve: + test tests/.py::[::] a test function that exists + step .yml# a step in .github/workflows/ + artifact .well-known/ a published, freshness-governed artifact + decision DEC-NNN an in-force decision record +- Anything short of implemented states its gap and what closes it: a + workstream (W2..W10), "sam" (an act only the operator can perform, such as an + application, a registration or engaging an assessor), or "optional" (a MAY + rule not exercised; refused for any other force). +- No record text may carry an AWS account ID, or an email address outside the + system's domain and FedRAMP's (the SDR is published). + +Fails closed, stdlib only (the deploy job installs no jsonschema); the JSON +schema beside it documents the same shape and is checked in tests. +""" + +import ast +import json +import re +import sys +from collections import Counter +from functools import lru_cache +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _decisions # noqa: E402 +import _freshness # noqa: E402 +import _redact # noqa: E402 +import _rules # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +DEFAULT_PATH = REPO / "data" / "frr-records.json" +SCHEMA_PATH = REPO / "infrastructure" / "schemas" / "frr-records.schema.json" +WORKFLOWS = REPO / ".github" / "workflows" +TESTS = REPO / "tests" + +STATUSES = ("Implemented", "Partially Implemented", "Not Implemented") +WORKSTREAMS = tuple(f"W{n}" for n in range(2, 11)) +CLOSERS = (*WORKSTREAMS, "sam", "optional") +EVIDENCE_KINDS = ("test", "step", "artifact", "decision") + +TOP_KEYS = {"$comment", "schema", "basis", "records"} +RECORD_KEYS = {"frr_id", "status", "implementation", "evidence", "gap", "closes_with"} +EVIDENCE_KEYS = {"kind", "ref"} + +TEST_RE = re.compile(r"^(tests/[A-Za-z0-9_/]+\.py)::([A-Za-z_][A-Za-z0-9_]*)(?:::([A-Za-z_][A-Za-z0-9_]*))?(?:\[[^\]]*\])?$") +STEP_RE = re.compile(r"^([A-Za-z0-9_.-]+\.ya?ml)#(.+)$") +STEP_LINE_RE = re.compile(r"^\s*-\s+name:\s*(.+?)\s*$") +EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)") +# Addresses a record may name: the system's own domain and FedRAMP's senders +# (AFC-CSO-TFG). Anything else, a personal mailbox above all, stays out of the SDR. +EMAIL_DOMAINS = ("samaydlette.com", "fedramp.gov", "gsa.gov") + + +def _fail(msg): + raise SystemExit(f"::error::FRR records: {msg} (fail-closed)") + + +# --- evidence resolution ---------------------------------------------------------- + +@lru_cache(maxsize=None) +def _test_names(path): + """{function name} and {(class, method)} defined in a test module.""" + tree = ast.parse(Path(path).read_text(), filename=str(path)) + funcs, methods = set(), set() + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + funcs.add(node.name) + elif isinstance(node, ast.ClassDef): + for sub in node.body: + if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)): + methods.add((node.name, sub.name)) + return funcs, methods + + +@lru_cache(maxsize=None) +def _step_names(path): + names = set() + for line in Path(path).read_text().splitlines(): + m = STEP_LINE_RE.match(line) + if m: + names.add(m.group(1).strip().strip("'\"")) + return names + + +def _resolve_test(ref, repo): + m = TEST_RE.match(ref) + if not m: + return "is not tests/.py::" + rel, first, second = m.groups() + path = repo / rel + if ".." in rel or not path.is_file(): + return f"{rel} does not exist" + funcs, methods = _test_names(str(path)) + if second is None: + ok = first in funcs and first.startswith("test") + else: + ok = (first, second) in methods and second.startswith("test") + return None if ok else "names no test function in that file" + + +def _resolve_step(ref, repo): + m = STEP_RE.match(ref) + if not m: + return "is not .yml#" + wf, step = m.groups() + path = repo / ".github" / "workflows" / wf + if not path.is_file(): + return f"workflow {wf} does not exist" + return None if step in _step_names(str(path)) else f"no step named {step!r} in {wf}" + + +def resolver(repo=REPO, freshness_path=None, decisions_path=None): + """A function evidence -> None when it resolves, else the reason it doesn't.""" + repo = Path(repo) + published = {a["path"] for a in _freshness.load_freshness_config( + freshness_path or repo / "infrastructure" / "schemas" / "freshness-config.json")["artifacts"]} + in_force = {r["id"] for r in _decisions.in_force(_decisions.load(decisions_path or repo / "data" / "decisions.json"))} + + def resolve(ev): + kind, ref = ev["kind"], ev["ref"] + if kind == "test": + return _resolve_test(ref, repo) + if kind == "step": + return _resolve_step(ref, repo) + if kind == "artifact": + return None if ref in published else "is not a freshness-governed published artifact" + if kind == "decision": + return None if ref in in_force else "is not an in-force decision record" + return f"unknown evidence kind {kind!r}" + return resolve + + +# --- validation ------------------------------------------------------------------- + +def _text(rec, key, rid, required=True): + v = rec.get(key) + if v is None and not required: + return None + if not isinstance(v, str) or not v.strip(): + _fail(f"{rid}: {key} must be a non-empty string") + if _redact.find(v): + _fail(f"{rid}: {key} carries an AWS account ID") + for m in EMAIL_RE.finditer(v): + if m.group(1).lower() not in EMAIL_DOMAINS: + _fail(f"{rid}: {key} names an email address outside {list(EMAIL_DOMAINS)}") + return v + + +def validate(doc, applicable, resolve, where="data/frr-records.json"): + """Structural and honesty checks. `applicable` is {rule ID: rule} from the + corpus; `resolve` maps evidence to None or a failure reason.""" + if not isinstance(doc, dict) or doc.get("schema") != "frr-records/1": + _fail(f"{where} is not an frr-records/1 document") + if set(doc) - TOP_KEYS: + _fail(f"{where}: unknown top-level field(s) {sorted(set(doc) - TOP_KEYS)}") + _text(doc, "basis", "basis") + recs = doc.get("records") + if not isinstance(recs, list): + _fail(f"{where}: records must be a list") + seen = set() + for rec in recs: + rid = rec.get("frr_id") if isinstance(rec, dict) else None + if not isinstance(rid, str) or not rid: + _fail(f"{where}: a record has no frr_id") + if rid in seen: + _fail(f"duplicate record for {rid}") + seen.add(rid) + if rid not in applicable: + _fail(f"{rid} is not a rule that applies to a 20x Class C provider (extra record)") + if set(rec) - RECORD_KEYS: + _fail(f"{rid}: unknown field(s) {sorted(set(rec) - RECORD_KEYS)}") + status = rec.get("status") + if status not in STATUSES: + _fail(f"{rid}: status {status!r} is not one of {list(STATUSES)}") + _text(rec, "implementation", rid) + evidence = rec.get("evidence", []) + if not isinstance(evidence, list): + _fail(f"{rid}: evidence must be a list") + for ev in evidence: + if not isinstance(ev, dict) or set(ev) != EVIDENCE_KEYS or ev.get("kind") not in EVIDENCE_KINDS \ + or not isinstance(ev.get("ref"), str) or not ev["ref"]: + _fail(f"{rid}: evidence {ev!r} must be {{kind: one of {list(EVIDENCE_KINDS)}, ref}}") + why = resolve(ev) + if why: + _fail(f"{rid}: {ev['kind']} evidence {ev['ref']!r} does not resolve: {why}") + closes = rec.get("closes_with") + if status == "Implemented": + if not evidence: + _fail(f"{rid}: Implemented needs evidence that resolves") + if rec.get("gap") is not None or closes is not None: + _fail(f"{rid}: an Implemented record has no gap and nothing closing it") + continue + _text(rec, "gap", rid) + if not isinstance(closes, list) or not closes or len(set(closes)) != len(closes) \ + or any(c not in CLOSERS for c in closes): + _fail(f"{rid}: closes_with must be a non-empty list drawn from {list(CLOSERS)}") + if "optional" in closes: + if applicable[rid]["force"] != "MAY": + _fail(f"{rid}: only a MAY rule can be left as optional (Class C force is {applicable[rid]['force']})") + if len(closes) != 1: + _fail(f"{rid}: an optional rule is closed by nothing else") + missing = sorted(set(applicable) - seen) + if missing: + _fail(f"no record for {len(missing)} applicable rule(s): {', '.join(missing)}") + return doc + + +def load(path=None, rules_dir=None, repo=REPO, freshness_path=None, decisions_path=None): + p = Path(path) if path else DEFAULT_PATH + if not p.is_file(): + _fail(f"{p} not found") + try: + doc = json.loads(p.read_text()) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + _fail(f"{p} is not valid JSON: {exc}") + applicable = _rules.applicable_provider_rules(rules_dir or _rules.RULES_DIR) + return validate(doc, applicable, resolver(repo, freshness_path, decisions_path), str(p)) + + +def summary(doc, applicable): + """Counts the trust center and the SDR can show: by status, and by status + within each force (MUST, SHOULD, ...).""" + by_status = Counter(r["status"] for r in doc["records"]) + by_force = {} + for r in doc["records"]: + f = applicable[r["frr_id"]]["force"] + by_force.setdefault(f, Counter())[r["status"]] += 1 + optional = sum(1 for r in doc["records"] if r.get("closes_with") == ["optional"]) + return {"rules": len(doc["records"]), + "by_status": {s: by_status.get(s, 0) for s in STATUSES}, + "by_force": {f: {s: c.get(s, 0) for s in STATUSES} for f, c in sorted(by_force.items())}, + "optional_not_exercised": optional} + + +def main(argv=None): + import argparse + ap = argparse.ArgumentParser(description="Validate data/frr-records.json against the pinned FedRAMP rules.") + ap.add_argument("--records", default=str(DEFAULT_PATH)) + args = ap.parse_args(argv) + doc = load(args.records) + print(json.dumps(summary(doc, _rules.applicable_provider_rules()), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/_rules.py b/scripts/_rules.py index 1c659b66..64d7043d 100644 --- a/scripts/_rules.py +++ b/scripts/_rules.py @@ -130,3 +130,61 @@ def evaluation_days(rules_dir=RULES_DIR): """VER-TFR-EVU: days from detection within which every vulnerability is evaluated.""" rules, _ = load_rules(rules_dir) return _class_c_or_flat(_rule(rules, "VER", "VER-TFR-EVU"), "VER-TFR-EVU") + + +# The certification this system builds toward: FedRAMP 20x, Program path, Class C. +CERT_TYPE = "20x" +CERT_PATH = "Program" +_SCOPES = ("all", "20x") # rule sections that can apply to a 20x certification + + +def _subset_applicability(family, scope, subset): + """A subset's applicability block. Subsets of the 20x section are described + in info["20x"].subsets where the family has one (CPO, FRC, IVV, SDR), and + otherwise, like the cross-program section's, in info.subsets (VDR).""" + info = family.get("info") or {} + sections = [(info.get(scope) or {}).get("subsets")] if scope != "all" else [] + for where in [*sections, info.get("subsets")]: + app = ((where or {}).get(subset) or {}).get("applicability") + if isinstance(app, dict): + return app + return None + + +def applicable_provider_rules(rules_dir=RULES_DIR): + """{rule ID: {"family", "name", "statement", "force"}} for every FedRAMP rule + a provider must address for a 20x Program Class C certification (SDR-CSO-FRR). + + Applicability is read from the corpus's own subset metadata (types, paths, + classes, affects), never from a list kept here, so a re-pinned corpus that + adds or retires a rule changes the set. A subset in a 20x-relevant section + with no applicability block fails closed: it can't be decided either way.""" + rules, _ = load_rules(rules_dir) + frr = rules.get("FRR") + if not isinstance(frr, dict) or not frr: + _fail("no FRR section") + out = {} + for fam_id, family in sorted(frr.items()): + for scope in _SCOPES: + for subset, members in sorted(((family.get("data") or {}).get(scope) or {}).items()): + app = _subset_applicability(family, scope, subset) + if app is None: + _fail(f"FRR/{fam_id}/{scope}/{subset} has no applicability metadata") + if not (CERT_TYPE in app.get("types", []) and CERT_PATH in app.get("paths", []) + and CERT_CLASS.upper() in app.get("classes", []) + and "Providers" in app.get("affects", [])): + continue + for rid, rule in members.items(): + if "Providers" not in (rule.get("affects") or []): + continue + cell = (rule.get("varies_by_class") or {}).get(CERT_CLASS, rule) + force, statement = cell.get("force"), cell.get("statement") + if not isinstance(force, str) or not isinstance(statement, str): + _fail(f"{rid} has no Class {CERT_CLASS.upper()} statement and force") + if rid in out: + _fail(f"{rid} appears twice in the corpus") + out[rid] = {"family": fam_id, "name": rule.get("name", ""), + "statement": statement, "force": force} + if not out: + _fail("no provider rules apply; the applicability metadata has changed shape") + return out diff --git a/tests/fixtures/frr/known-bad.json b/tests/fixtures/frr/known-bad.json new file mode 100644 index 00000000..892db523 --- /dev/null +++ b/tests/fixtures/frr/known-bad.json @@ -0,0 +1,24 @@ +{ + "$comment": "Each case changes the committed data/frr-records.json in one way that must fail scripts/_frr.py for the stated reason. Applied by tests/test_frr_records.py.", + "cases": [ + {"case": "missing record", "drop": "AFC-CSO-INB", "expect": "no record for 1 applicable rule(s): AFC-CSO-INB"}, + {"case": "extra record (Rev5 agency path)", "add": {"frr_id": "FRC-APS-ATO", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["sam"]}, "expect": "FRC-APS-ATO is not a rule that applies"}, + {"case": "duplicate record", "add": {"frr_id": "VDR-TFR-PVR", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W7"]}, "expect": "duplicate record for VDR-TFR-PVR"}, + {"case": "implemented without evidence", "replace": {"frr_id": "AFC-CSO-INB", "status": "Implemented", "implementation": "An inbox exists.", "evidence": []}, "expect": "AFC-CSO-INB: Implemented needs evidence that resolves"}, + {"case": "test that does not exist", "replace": {"frr_id": "AFC-CSO-INB", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "test", "ref": "tests/test_frr_records.py::test_the_inbox_receives_mail"}]}, "expect": "names no test function"}, + {"case": "test file that does not exist", "replace": {"frr_id": "AFC-CSO-INB", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "test", "ref": "tests/test_inbox.py::test_x"}]}, "expect": "tests/test_inbox.py does not exist"}, + {"case": "workflow step that does not exist", "replace": {"frr_id": "AFC-CSO-INB", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "step", "ref": "deploy-with-opa.yml#Check the FedRAMP Security Inbox"}]}, "expect": "no step named"}, + {"case": "artifact not published", "replace": {"frr_id": "AFC-CSO-INB", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "artifact", "ref": ".well-known/fedramp-inbox.json"}]}, "expect": "is not a freshness-governed published artifact"}, + {"case": "decision not in force", "replace": {"frr_id": "AFC-CSO-INB", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "decision", "ref": "DEC-999"}]}, "expect": "is not an in-force decision record"}, + {"case": "unresolved evidence on a partial record", "replace": {"frr_id": "AFC-CSO-INB", "status": "Partially Implemented", "implementation": "x", "evidence": [{"kind": "decision", "ref": "DEC-999"}], "gap": "x", "closes_with": ["W9"]}, "expect": "does not resolve"}, + {"case": "implemented with a gap", "replace": {"frr_id": "VDR-CSO-MSP", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "decision", "ref": "DEC-039"}], "gap": "x"}, "expect": "an Implemented record has no gap"}, + {"case": "partial without a gap", "replace": {"frr_id": "AFC-CSO-INB", "status": "Partially Implemented", "implementation": "x", "closes_with": ["W9"]}, "expect": "AFC-CSO-INB: gap must be a non-empty string"}, + {"case": "partial with nothing closing it", "replace": {"frr_id": "AFC-CSO-INB", "status": "Partially Implemented", "implementation": "x", "gap": "x"}, "expect": "closes_with must be a non-empty list"}, + {"case": "unknown workstream", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W11"]}, "expect": "closes_with must be a non-empty list"}, + {"case": "a MUST rule left as optional", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["optional"]}, "expect": "only a MAY rule can be left as optional"}, + {"case": "unknown status", "replace": {"frr_id": "AFC-CSO-INB", "status": "Met", "implementation": "x"}, "expect": "status 'Met' is not one of"}, + {"case": "unknown field", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "met": true}, "expect": "unknown field(s) ['met']"}, + {"case": "account ID in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "Bucket arn:aws:s3:us-east-1:111122223333:x", "gap": "x", "closes_with": ["W9"]}, "expect": "implementation carries an AWS account ID"}, + {"case": "personal email in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "Mail goes to someone@example.com today.", "closes_with": ["W9"]}, "expect": "gap names an email address outside"} + ] +} diff --git a/tests/test_frr_records.py b/tests/test_frr_records.py new file mode 100644 index 00000000..a1b3665a --- /dev/null +++ b/tests/test_frr_records.py @@ -0,0 +1,196 @@ +"""The FedRAMP rule records (SDR-CSO-FRR): one per applicable rule, and none +claims more than the repository shows.""" + +import copy +import hashlib +import importlib.util +import json +import shutil +import sys +from pathlib import Path + +import jsonschema +import pytest + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import _frr # noqa: E402 +import _rules # noqa: E402 + +RECORDS = REPO / "data" / "frr-records.json" +KNOWN_BAD = json.loads((REPO / "tests" / "fixtures" / "frr" / "known-bad.json").read_text())["cases"] + +# The 20x-specific rules a first assessment missed; the corpus metadata includes them. +CSX = {"SDR-CSX-KSI", "SDR-CSX-KMT", "CPO-CSX-CPM", "FRC-CSX-VVK", "FRC-CSX-VVR", "FRC-CSX-MAS", "IVV-CSX-AIA"} + + +@pytest.fixture(scope="module") +def applicable(): + return _rules.applicable_provider_rules() + + +@pytest.fixture(scope="module") +def resolve(): + return _frr.resolver() + + +def _committed(): + return json.loads(RECORDS.read_text()) + + +# --- which rules apply ------------------------------------------------------------- + +def test_applicable_rules_come_from_the_corpus_metadata(applicable): + assert len(applicable) == 154 + assert CSX <= set(applicable) + # Rev5 agency path, Class A only, and the Initial Implementation Phase (no class) are out. + for rid in ("FRC-APS-ATO", "FRC-CLA-ASF", "MKT-IIP-AGU", "FRC-CCL-UCC"): + assert rid not in applicable + # Only rules that bind providers. + assert "AFC-FRP-VRE" not in applicable + # Class C's own force and statement are used where the rule varies by class. + assert applicable["FRC-CSX-VVK"]["force"] == "MUST" + assert "at least 2 automated methods" in applicable["FRC-CSX-VVK"]["statement"] + assert applicable["FRC-CSX-VVR"]["force"] == "SHOULD" + + +def _corpus_copy(tmp_path, mutate): + """A copy of the pinned corpus with one change and a SOURCES.json that + matches it, so the hash check passes and the change itself is what's tested.""" + d = tmp_path / "rules" + shutil.copytree(_rules.RULES_DIR, d) + corpus = json.loads((d / _rules.RULES_FILE).read_text()) + mutate(corpus) + raw = json.dumps(corpus).encode() + (d / _rules.RULES_FILE).write_bytes(raw) + src = json.loads((d / "SOURCES.json").read_text()) + src["sha256"] = hashlib.sha256(raw).hexdigest() + (d / "SOURCES.json").write_text(json.dumps(src)) + return d + + +def test_a_subset_without_applicability_metadata_fails_closed(tmp_path): + def strip(c): + del c["FRR"]["AFC"]["info"]["subsets"]["CSO"]["applicability"] + with pytest.raises(SystemExit, match="FRR/AFC/all/CSO has no applicability metadata"): + _rules.applicable_provider_rules(_corpus_copy(tmp_path, strip)) + + +def test_a_rule_added_to_the_corpus_needs_a_record(tmp_path): + def add(c): + c["FRR"]["AFC"]["data"]["all"]["CSO"]["AFC-CSO-NEW"] = { + "name": "New", "statement": "Providers MUST do something new.", "force": "MUST", + "affects": ["Providers"], "updated": []} + rules_dir = _corpus_copy(tmp_path, add) + with pytest.raises(SystemExit, match="no record for 1 applicable rule.*AFC-CSO-NEW"): + _frr.load(RECORDS, rules_dir=rules_dir) + + +# --- the committed records --------------------------------------------------------- + +def test_the_committed_records_are_valid(applicable): + doc = _frr.load() + assert {r["frr_id"] for r in doc["records"]} == set(applicable) + + +def test_the_committed_records_match_their_schema(): + schema = json.loads(_frr.SCHEMA_PATH.read_text()) + jsonschema.Draft202012Validator.check_schema(schema) + jsonschema.validate(_committed(), schema) + + +def test_every_implemented_record_cites_evidence_that_resolves(resolve): + implemented = [r for r in _committed()["records"] if r["status"] == "Implemented"] + assert implemented + for r in implemented: + assert r["evidence"] + assert all(resolve(ev) is None for ev in r["evidence"]), r["frr_id"] + + +def test_summary_counts_add_up(applicable): + s = _frr.summary(_committed(), applicable) + assert s["rules"] == 154 == sum(s["by_status"].values()) + assert sum(sum(c.values()) for c in s["by_force"].values()) == 154 + assert s["optional_not_exercised"] == sum( + 1 for r in _committed()["records"] if r.get("closes_with") == ["optional"]) + + +# --- known-bad: each invariant fails for its own reason ---------------------------- + +def _apply(case): + doc = copy.deepcopy(_committed()) + recs = doc["records"] + if "drop" in case: + doc["records"] = [r for r in recs if r["frr_id"] != case["drop"]] + if "add" in case: + recs.append(case["add"]) + if "replace" in case: + rid = case["replace"]["frr_id"] + doc["records"] = [case["replace"] if r["frr_id"] == rid else r for r in recs] + return doc + + +@pytest.mark.parametrize("case", KNOWN_BAD, ids=[c["case"] for c in KNOWN_BAD]) +def test_known_bad_records_fail(case, applicable, resolve): + with pytest.raises(SystemExit) as exc: + _frr.validate(_apply(case), applicable, resolve) + assert case["expect"] in str(exc.value) + + +def test_known_bad_fixture_covers_every_check(): + """A new invariant in _frr.validate needs a known-bad case here.""" + expects = " ".join(c["expect"] for c in KNOWN_BAD) + for fragment in ("no record for", "is not a rule that applies", "duplicate record", + "Implemented needs evidence", "names no test function", "does not exist", + "no step named", "freshness-governed", "in-force decision", "has no gap", + "gap must be", "closes_with must be", "only a MAY rule", "is not one of", + "unknown field", "AWS account ID", "email address outside"): + assert fragment in expects, fragment + + +# --- evidence resolution ----------------------------------------------------------- + +@pytest.mark.parametrize("ref,ok", [ + ("tests/test_frr_records.py::test_summary_counts_add_up", True), + ("tests/test_frr_records.py::test_known_bad_records_fail[missing record]", True), + ("tests/test_frr_records.py::_committed", False), # not a test + ("tests/test_frr_records.py::applicable", False), # a fixture + ("tests/../scripts/_frr.py::main", False), # path escape + ("tests/test_frr_records.py", False), +]) +def test_test_references(ref, ok, resolve): + assert (resolve({"kind": "test", "ref": ref}) is None) is ok + + +def test_test_methods_in_a_class_resolve(tmp_path): + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_k.py").write_text("class TestK:\n def test_m(self):\n pass\n") + assert _frr._resolve_test("tests/test_k.py::TestK::test_m", tmp_path) is None + assert _frr._resolve_test("tests/test_k.py::TestK::test_n", tmp_path) is not None + assert _frr._resolve_test("tests/test_k.py::test_m", tmp_path) is not None + + +@pytest.mark.parametrize("ref,ok", [ + ("deploy-with-opa.yml#Run OPA Compliance Check", True), + ("deploy-with-opa.yml#Build VDR Report (FedRAMP 20x Class C)", True), + ("deploy-with-opa.yml#Run OPA Compliance", False), # a prefix is not a step + ("nope.yml#Run OPA Compliance Check", False), + ("../deploy-with-opa.yml#Run OPA Compliance Check", False), +]) +def test_step_references(ref, ok, resolve): + assert (resolve({"kind": "step", "ref": ref}) is None) is ok + + +def test_cli_prints_the_summary(): + spec = importlib.util.spec_from_file_location("frr_cli", REPO / "scripts" / "_frr.py") + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + assert m.main(["--records", str(RECORDS)]) == 0 + + +def test_the_system_and_fedramp_addresses_are_allowed(applicable, resolve): + doc = copy.deepcopy(_committed()) + for r in doc["records"]: + if r["frr_id"] == "AFC-CSO-ACK": + r["gap"] = "Acknowledge mail from x@fedramp.gov and x@gsa.gov at fedramp-security@samaydlette.com." + _frr.validate(doc, applicable, resolve) From caed675ddee08f1f47641afa55e93c0381b71638 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 13:45:00 -0500 Subject: [PATCH 02/97] Rule records: every MAY rule says why it was or wasn't exercised A MAY record now requires may_rationale and any other record refuses it; the text is screened like the rest. The two quarterly-review MAY rules the operator intends to exercise move from optional to W4. Co-Authored-By: Claude Opus 5.5 --- data/frr-records.json | 53 ++++++++++++------- .../schemas/frr-records.schema.json | 1 + scripts/_frr.py | 8 ++- tests/fixtures/frr/known-bad.json | 2 + tests/test_frr_records.py | 8 ++- 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/data/frr-records.json b/data/frr-records.json index 7c943439..cb17825c 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -141,7 +141,8 @@ "gap": "Permissive rule, exercised; but no recorded determination that public release will not adversely affect the offering. To close: Record the public-release determination (who, when, basis) in each OCR issue or in the CPO.", "closes_with": [ "W4" - ] + ], + "may_rationale": "Exercised. The system exists to demonstrate FedRAMP 20x in the open, and the OCR carries no federal customer or agency data, so it is published in full; each issue will record the no-adverse-effect determination." }, { "frr_id": "CCM-OCR-SOR", @@ -238,20 +239,22 @@ "status": "Not Implemented", "implementation": "No review content exists.", "evidence": [], - "gap": "Optional (MAY); not exercised.", + "gap": "No quarterly review has been held yet, so there is no content to publish.", "closes_with": [ - "optional" - ] + "W4" + ], + "may_rationale": "Will be exercised: content prepared for each quarterly review will be published with that quarter's OCR, for the same reason the OCR is public, after the same no-adverse-effect check." }, { "frr_id": "CCM-QTR-SRR", "status": "Not Implemented", "implementation": "No recordings exist.", "evidence": [], - "gap": "Optional (MAY); not exercised. No recordings are released; if one ever is, agency information must be removed first.", + "gap": "No quarterly review has been held yet, so there is no recording to publish.", "closes_with": [ - "optional" - ] + "W4" + ], + "may_rationale": "Will be exercised: each review's recording or transcript will be published after all agency comments, questions and names are removed, so the review is as open as the rest of the package." }, { "frr_id": "CDS-CSO-AVR", @@ -323,7 +326,8 @@ "gap": "Optional (MAY); not exercised. Could split package views per service (website, Silk Reeling).", "closes_with": [ "optional" - ] + ], + "may_rationale": "Not exercised. The two services (the website and Silk Reeling) share one boundary, one inventory and one pipeline; separate per-service packages would repeat the same evidence." }, { "frr_id": "CDS-CSO-PUB", @@ -355,7 +359,8 @@ "gap": "The artifact requires an explanation of if/how information is shared and the provider's determination that public sharing will not likely have an adverse effect. There is a statement of public sharing but no documented adverse-effect determination, and the redaction practice is inconsistent (see CDS-CSO-RIS). To close: Publish a short sharing statement: what is public, to whom else data goes, and the recorded determination (decision record in data/decisions.json with decided_by/on and review date) that it is not adverse.", "closes_with": [ "W8" - ] + ], + "may_rationale": "Exercised. The whole package is public: the system demonstrates 20x openly and holds no federal customer data. The determination that public sharing will not likely harm the offering is recorded with the sharing statement." }, { "frr_id": "CDS-CSO-SVC", @@ -586,7 +591,8 @@ "gap": "Optional (MAY); not exercised.", "closes_with": [ "optional" - ] + ], + "may_rationale": "Not exercised: no initial independent assessment exists, so there is nothing stale to freshen." }, { "frr_id": "FRC-CSO-FCP", @@ -779,7 +785,8 @@ "gap": "Optional (MAY); not exercised.", "closes_with": [ "optional" - ] + ], + "may_rationale": "Not exercised while no assessor is engaged. When one is, advice will be accepted where it improves the security posture or the evidence, and recorded with the assessment." }, { "frr_id": "IVV-CSO-SEE", @@ -821,7 +828,8 @@ "gap": "Optional (MAY); not exercised.", "closes_with": [ "optional" - ] + ], + "may_rationale": "Not exercised. The system is small enough to verify and validate every resource, and full coverage is a stronger claim than a sample." }, { "frr_id": "IVV-CSX-AIA", @@ -871,7 +879,8 @@ "gap": "Optional (MAY); not exercised. If coverage material for other frameworks (CMMC, GovRAMP, TX-RAMP on the trust page) is ever put in the package, mark it as supplemental.", "closes_with": [ "optional" - ] + ], + "may_rationale": "Not exercised. The package covers only the offering. The other-framework views on the trust page are derived from the same controls and evidence, not separate resources." }, { "frr_id": "MAS-CSO-TPR", @@ -1011,7 +1020,8 @@ "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it.", "closes_with": [ "W5" - ] + ], + "may_rationale": "Exercised. Change records add the boundary change, the security impact by control family and the verification result, so a reviewer can see why a change was classified as it was." }, { "frr_id": "SCN-CSO-EMG", @@ -1021,7 +1031,8 @@ "gap": "Never exercised; 'notify all necessary parties' has no channel; no gate that requires the retroactive SCN materials (only a warning). To close: Add a check that opens/blocks until a retroactive SCN record exists for each emergency-tagged commit within 5 business days; route notification through the NOM mechanism.", "closes_with": [ "W5" - ] + ], + "may_rationale": "Kept available for emergencies: a security fix may need to ship before notice can be given. When it is used, the retroactive records the rule requires are produced within its timeframe." }, { "frr_id": "SCN-CSO-EVA", @@ -1081,7 +1092,8 @@ "gap": "Mechanism documented only in the repo README, not in the Certification Package (CPO); no list of available notification mechanisms (artifact); a git push is not a notification any party receives. To close: Offer real mechanisms (signed SCN JSON feed + Atom/RSS at /.well-known/, email list) and list them in the CPO.", "closes_with": [ "W5" - ] + ], + "may_rationale": "Exercised. Notice is given by a published, signed feed rather than email, because there are no agency customers to address; the mechanism is documented in the certification package." }, { "frr_id": "SCN-RTR-NNR", @@ -1313,7 +1325,8 @@ "gap": "Optional (MAY); not exercised. Note that coverage gaps are a DET/MVX matter, not sampling.", "closes_with": [ "optional" - ] + ], + "may_rationale": "Not exercised. Every resource is scanned individually; the inventory is small and sampling would reduce assurance for no gain." }, { "frr_id": "VDR-TFR-KEV", @@ -1553,7 +1566,8 @@ "gap": "The permission depends on the provider having *determined* that disclosure won't likely lead to exploitation; no such determination is recorded per finding or as policy (see NID). To close: Record the determination: a disclosure policy plus a per-finding 'public_disclosure' field (full/summary/withheld, with reason) set by the triage policy and published.", "closes_with": [ "W7" - ] + ], + "may_rationale": "Exercised. Vulnerability reports are public to demonstrate the method; detail that could enable exploitation is withheld by a recorded disclosure policy until remediation." }, { "frr_id": "VER-RPT-VDT", @@ -1623,7 +1637,8 @@ "gap": "MAY rule: optional, but the published policy claims it's adopted and nothing implements it. NLEV-IRV N5 findings never open an incident. To close: Either implement alongside IRI (NIRV and LEV and PAIN = N5 -> incident until a partial mitigation brings it to N4) or remove the claim from ir-policy.md and record the decision not to use the MAY.", "closes_with": [ "W6" - ] + ], + "may_rationale": "Adopted, as the incident response policy states: treating a likely exploitable, non-internet-reachable N5 vulnerability as reportable is stricter than required and costs little at this scale." } ] } diff --git a/infrastructure/schemas/frr-records.schema.json b/infrastructure/schemas/frr-records.schema.json index 45342c6b..de042fad 100644 --- a/infrastructure/schemas/frr-records.schema.json +++ b/infrastructure/schemas/frr-records.schema.json @@ -33,6 +33,7 @@ } }, "gap": {"type": "string", "minLength": 1}, + "may_rationale": {"type": "string", "minLength": 1, "description": "Required on, and only on, MAY rules: why the provider did or didn't exercise the permission."}, "closes_with": { "type": "array", "minItems": 1, diff --git a/scripts/_frr.py b/scripts/_frr.py index d18250f8..c77cc768 100644 --- a/scripts/_frr.py +++ b/scripts/_frr.py @@ -21,6 +21,8 @@ workstream (W2..W10), "sam" (an act only the operator can perform, such as an application, a registration or engaging an assessor), or "optional" (a MAY rule not exercised; refused for any other force). +- Every MAY rule says why it was or wasn't exercised (may_rationale); no + other rule carries one. - No record text may carry an AWS account ID, or an email address outside the system's domain and FedRAMP's (the SDR is published). @@ -54,7 +56,7 @@ EVIDENCE_KINDS = ("test", "step", "artifact", "decision") TOP_KEYS = {"$comment", "schema", "basis", "records"} -RECORD_KEYS = {"frr_id", "status", "implementation", "evidence", "gap", "closes_with"} +RECORD_KEYS = {"frr_id", "status", "implementation", "evidence", "gap", "closes_with", "may_rationale"} EVIDENCE_KEYS = {"kind", "ref"} TEST_RE = re.compile(r"^(tests/[A-Za-z0-9_/]+\.py)::([A-Za-z_][A-Za-z0-9_]*)(?:::([A-Za-z_][A-Za-z0-9_]*))?(?:\[[^\]]*\])?$") @@ -188,6 +190,10 @@ def validate(doc, applicable, resolve, where="data/frr-records.json"): if status not in STATUSES: _fail(f"{rid}: status {status!r} is not one of {list(STATUSES)}") _text(rec, "implementation", rid) + if applicable[rid]["force"] == "MAY": + _text(rec, "may_rationale", rid) + elif "may_rationale" in rec: + _fail(f"{rid}: may_rationale is for MAY rules only (Class C force is {applicable[rid]['force']})") evidence = rec.get("evidence", []) if not isinstance(evidence, list): _fail(f"{rid}: evidence must be a list") diff --git a/tests/fixtures/frr/known-bad.json b/tests/fixtures/frr/known-bad.json index 892db523..b7e3a180 100644 --- a/tests/fixtures/frr/known-bad.json +++ b/tests/fixtures/frr/known-bad.json @@ -19,6 +19,8 @@ {"case": "unknown status", "replace": {"frr_id": "AFC-CSO-INB", "status": "Met", "implementation": "x"}, "expect": "status 'Met' is not one of"}, {"case": "unknown field", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "met": true}, "expect": "unknown field(s) ['met']"}, {"case": "account ID in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "Bucket arn:aws:s3:us-east-1:111122223333:x", "gap": "x", "closes_with": ["W9"]}, "expect": "implementation carries an AWS account ID"}, + {"case": "MAY rule without a rationale", "replace": {"frr_id": "VDR-CSO-SIR", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["optional"]}, "expect": "VDR-CSO-SIR: may_rationale must be a non-empty string"}, + {"case": "rationale on a MUST rule", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "may_rationale": "x"}, "expect": "may_rationale is for MAY rules only"}, {"case": "personal email in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "Mail goes to someone@example.com today.", "closes_with": ["W9"]}, "expect": "gap names an email address outside"} ] } diff --git a/tests/test_frr_records.py b/tests/test_frr_records.py index a1b3665a..9dfcd22c 100644 --- a/tests/test_frr_records.py +++ b/tests/test_frr_records.py @@ -144,7 +144,8 @@ def test_known_bad_fixture_covers_every_check(): "Implemented needs evidence", "names no test function", "does not exist", "no step named", "freshness-governed", "in-force decision", "has no gap", "gap must be", "closes_with must be", "only a MAY rule", "is not one of", - "unknown field", "AWS account ID", "email address outside"): + "unknown field", "AWS account ID", "email address outside", + "may_rationale must be", "for MAY rules only"): assert fragment in expects, fragment @@ -194,3 +195,8 @@ def test_the_system_and_fedramp_addresses_are_allowed(applicable, resolve): if r["frr_id"] == "AFC-CSO-ACK": r["gap"] = "Acknowledge mail from x@fedramp.gov and x@gsa.gov at fedramp-security@samaydlette.com." _frr.validate(doc, applicable, resolve) + + +def test_every_may_rule_says_why(applicable): + for r in _committed()["records"]: + assert ("may_rationale" in r) is (applicable[r["frr_id"]]["force"] == "MAY"), r["frr_id"] From 618dfaf3e5978302402404ff94a568476823b7de Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 13:52:18 -0500 Subject: [PATCH 03/97] Rule records: Not Applicable where FedRAMP's scoping says so A record may be Not Applicable, with its reason, only for a MAY permission not exercised or a rule in a subset FedRAMP scopes to a step the provider is not taking: Marketplace listing (MKT-CSO) and applying (FRC-APP). A test checks that scoping phrase is FedRAMP's own subset description. "optional" is no longer a closer. Co-Authored-By: Claude Opus 5.5 --- data/frr-records.json | 91 ++++++------------- .../schemas/frr-records.schema.json | 11 ++- scripts/_frr.py | 44 ++++++--- scripts/_rules.py | 2 +- tests/fixtures/frr/known-bad.json | 8 +- tests/test_frr_records.py | 19 +++- 6 files changed, 86 insertions(+), 89 deletions(-) diff --git a/data/frr-records.json b/data/frr-records.json index cb17825c..c7d069f9 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -320,13 +320,10 @@ }, { "frr_id": "CDS-CSO-PSM", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Permissive (MAY). Checked: no per-service materials are supplied; nothing in the rule requires them.", "evidence": [], - "gap": "Optional (MAY); not exercised. Could split package views per service (website, Silk Reeling).", - "closes_with": [ - "optional" - ], + "not_applicable": "A MAY permission that is not exercised; see the rationale.", "may_rationale": "Not exercised. The two services (the website and Silk Reeling) share one boundary, one inventory and one pipeline; separate per-service packages would repeat the same evidence." }, { @@ -535,63 +532,45 @@ }, { "frr_id": "FRC-APP-AFC", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Checked: no application form completed or drafted in the repo.", "evidence": [], - "gap": "Form not completed. Completing it 'in full' depends on prerequisites not met (Marketplace listing, fresh independent assessment, UEI). To close: After the prerequisites, the operator completes and submits the FedRAMP Certification Application Form personally; keep a copy in the package.", - "closes_with": [ - "sam" - ] + "not_applicable": "FedRAMP scopes the application rules to providers who have met all other relevant rules and are ready to apply. No application is made (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "FRC-APP-FCP", - "status": "Partially Implemented", + "status": "Not Applicable", "implementation": "Package artifacts regenerate on every deploy (deploy-with-opa.yml build steps ~L1558-1600) and the runtime emitter re-validates daily (trust-center.json 'runtime' check).", "evidence": [], - "gap": "Freshness is not bounded at 7 days: deploys are irregular (gaps 2026-08-27→09-24, 10-05→10-20 in the SDR metric history) and the governed freshness window for ksi-signal/OCR/COP/policy-index is 720h (infrastructure/schemas/freshness-config.json); the package is also incomplete (see FRC-CSO-PKG). To close: Rebuild and re-verify the full package on a schedule (at least weekly, e.g. a cron job reusing the deploy build) and set package freshness windows to <=168h so the watchdog alarms on staleness.", - "closes_with": [ - "W4" - ] + "not_applicable": "FedRAMP scopes the application rules to providers who have met all other relevant rules and are ready to apply. No application is made (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "FRC-APP-FIA", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "certification-package-overview.json independent_assessment: none engaged; SDR assessor_records.status 'no assessor record'.", "evidence": [], - "gap": "No independent assessment by a FedRAMP Recognized assessment service, let alone within 3 months. To close: Engage a FedRAMP Recognized assessor, complete the assessment, and apply within 3 months of its completion.", - "closes_with": [ - "sam" - ] + "not_applicable": "FedRAMP scopes the application rules to providers who have met all other relevant rules and are ready to apply. No application is made (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "FRC-APP-MLF", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Not listed in the FedRAMP Marketplace (stated in docs/cr26-applicability.md and COP).", "evidence": [], - "gap": "No listing; MKT-CSO-MLR depends on CDS-CSO-PUB (partial); MKT-CSO-PML request not sent; MKT-IIP-AGU agency use case and MKT-IIP-DCP continuous progress not shown. To close: Finish CDS-CSO-PUB, document an agency use case within FedRAMP scope, submit the Marketplace Provider Listing Request Form, and publish continuous-progress evidence.", - "closes_with": [ - "sam" - ] + "not_applicable": "FedRAMP scopes the application rules to providers who have met all other relevant rules and are ready to apply. No application is made (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "FRC-APP-NTP", - "status": "Partially Implemented", + "status": "Not Applicable", "implementation": "No application exists, and the CPO names the operator as the accountable official, the person who would apply.", "evidence": [], - "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it. To close: Operator applies personally; do not let the assessor or an advisor submit.", - "closes_with": [ - "sam" - ] + "not_applicable": "FedRAMP scopes the application rules to providers who have met all other relevant rules and are ready to apply. No application is made (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "FRC-APP-USA", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Permissive (MAY). Checked: no initial assessment exists, so there is nothing stale to freshen.", "evidence": [], - "gap": "Optional (MAY); not exercised.", - "closes_with": [ - "optional" - ], + "not_applicable": "FedRAMP scopes the application rules to providers who have met all other relevant rules and are ready to apply. No application is made (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is.", "may_rationale": "Not exercised: no initial independent assessment exists, so there is nothing stale to freshen." }, { @@ -779,13 +758,10 @@ }, { "frr_id": "IVV-CSO-RAA", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "No assessor engaged.", "evidence": [], - "gap": "Optional (MAY); not exercised.", - "closes_with": [ - "optional" - ], + "not_applicable": "A MAY permission that is not exercised; see the rationale.", "may_rationale": "Not exercised while no assessor is engaged. When one is, advice will be accepted where it improves the security posture or the evidence, and recorded with the assessment." }, { @@ -822,13 +798,10 @@ }, { "frr_id": "IVV-CSO-USR", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "No sampling in use.", "evidence": [], - "gap": "Optional (MAY); not exercised.", - "closes_with": [ - "optional" - ], + "not_applicable": "A MAY permission that is not exercised; see the rationale.", "may_rationale": "Not exercised. The system is small enough to verify and validate every resource, and full coverage is a stronger claim than a sample." }, { @@ -873,13 +846,10 @@ }, { "frr_id": "MAS-CSO-SUP", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Permissive (MAY). Checked: the package supplies no supplement; CPO package_contents lists only in-scope artifacts.", "evidence": [], - "gap": "Optional (MAY); not exercised. If coverage material for other frameworks (CMMC, GovRAMP, TX-RAMP on the trust page) is ever put in the package, mark it as supplemental.", - "closes_with": [ - "optional" - ], + "not_applicable": "A MAY permission that is not exercised; see the rationale.", "may_rationale": "Not exercised. The package covers only the offering. The other-framework views on the trust page are derived from the same controls and evidence, not separate resources." }, { @@ -894,23 +864,17 @@ }, { "frr_id": "MKT-CSO-MLR", - "status": "Partially Implemented", + "status": "Not Applicable", "implementation": "CDS-CSO-PUB data exists (see CDS-CSO-PUB).", "evidence": [], - "gap": "CDS-CSO-PUB is only partial (no UEI, no FedRAMP ID, a receding OCR date, no assessor) and no listing application or update request exists. Not applying is the operator's decision of 2026-11-05 (build to apply, don't apply).", - "closes_with": [ - "sam" - ] + "not_applicable": "FedRAMP scopes the Marketplace rules to providers seeking a listing in the FedRAMP Marketplace. No listing is sought (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "MKT-CSO-PML", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Checked: no listing request sent (docs/cr26-applicability.md records MKT as N/A).", "evidence": [], - "gap": "Marketplace Provider Listing Request Form not submitted. To close: Operator submits the FedRAMP Marketplace Provider Listing Request Form once PUB data and an agency use case are ready.", - "closes_with": [ - "sam" - ] + "not_applicable": "FedRAMP scopes the Marketplace rules to providers seeking a listing in the FedRAMP Marketplace. No listing is sought (operator's decision of 2026-11-05: build to apply, don't apply); the rule applies from the day one is." }, { "frr_id": "SCG-CSO-AUP", @@ -1319,13 +1283,10 @@ }, { "frr_id": "VDR-CSO-SIR", - "status": "Not Implemented", + "status": "Not Applicable", "implementation": "Checked: no sampling is used. Grype scans every SBOM'd dependency tree, Checkov/tfsec scan all Terraform, and the runtime emitter validates each object store and the distribution individually (live ksi-signal-runtime.json: 5 validations, one per component).", "evidence": [], - "gap": "Optional (MAY); not exercised. Note that coverage gaps are a DET/MVX matter, not sampling.", - "closes_with": [ - "optional" - ], + "not_applicable": "A MAY permission that is not exercised; see the rationale.", "may_rationale": "Not exercised. Every resource is scanned individually; the inventory is small and sampling would reduce assurance for no gain." }, { diff --git a/infrastructure/schemas/frr-records.schema.json b/infrastructure/schemas/frr-records.schema.json index de042fad..b8f881d6 100644 --- a/infrastructure/schemas/frr-records.schema.json +++ b/infrastructure/schemas/frr-records.schema.json @@ -18,7 +18,7 @@ "required": ["frr_id", "status", "implementation"], "properties": { "frr_id": {"type": "string", "pattern": "^[A-Z]{3}-[A-Z]{3}-[A-Z]{3}$"}, - "status": {"enum": ["Implemented", "Partially Implemented", "Not Implemented"]}, + "status": {"enum": ["Implemented", "Partially Implemented", "Not Implemented", "Not Applicable"]}, "implementation": {"type": "string", "minLength": 1}, "evidence": { "type": "array", @@ -33,12 +33,13 @@ } }, "gap": {"type": "string", "minLength": 1}, + "not_applicable": {"type": "string", "minLength": 1, "description": "Why the rule does not apply: FedRAMP scopes it to a step the provider is not taking, or it is a MAY permission not exercised."}, "may_rationale": {"type": "string", "minLength": 1, "description": "Required on, and only on, MAY rules: why the provider did or didn't exercise the permission."}, "closes_with": { "type": "array", "minItems": 1, "uniqueItems": true, - "items": {"enum": ["W2", "W3", "W4", "W5", "W6", "W7", "W8", "W9", "W10", "sam", "optional"]} + "items": {"enum": ["W2", "W3", "W4", "W5", "W6", "W7", "W8", "W9", "W10", "sam"]} } }, "if": {"properties": {"status": {"const": "Implemented"}}}, @@ -47,7 +48,11 @@ "properties": {"evidence": {"minItems": 1}}, "not": {"anyOf": [{"required": ["gap"]}, {"required": ["closes_with"]}]} }, - "else": {"required": ["gap", "closes_with"]} + "else": { + "if": {"properties": {"status": {"const": "Not Applicable"}}}, + "then": {"required": ["not_applicable"], "not": {"anyOf": [{"required": ["gap"]}, {"required": ["closes_with"]}]}}, + "else": {"required": ["gap", "closes_with"], "not": {"required": ["not_applicable"]}} + } } } } diff --git a/scripts/_frr.py b/scripts/_frr.py index c77cc768..da6a4717 100644 --- a/scripts/_frr.py +++ b/scripts/_frr.py @@ -17,10 +17,13 @@ step .yml# a step in .github/workflows/ artifact .well-known/ a published, freshness-governed artifact decision DEC-NNN an in-force decision record -- Anything short of implemented states its gap and what closes it: a - workstream (W2..W10), "sam" (an act only the operator can perform, such as an - application, a registration or engaging an assessor), or "optional" (a MAY - rule not exercised; refused for any other force). +- "Not Applicable" states why (not_applicable) and is allowed only where + FedRAMP's own scoping makes the rule conditional on something the provider + is not doing (a subset in CONDITIONAL_SUBSETS, such as seeking a Marketplace + listing) or the rule is a MAY permission not exercised. +- Anything else short of implemented states its gap and what closes it: a + workstream (W2..W10) or "sam" (an act only the operator can perform, such as + a registration or engaging an assessor). - Every MAY rule says why it was or wasn't exercised (may_rationale); no other rule carries one. - No record text may carry an AWS account ID, or an email address outside the @@ -50,13 +53,20 @@ WORKFLOWS = REPO / ".github" / "workflows" TESTS = REPO / "tests" -STATUSES = ("Implemented", "Partially Implemented", "Not Implemented") +STATUSES = ("Implemented", "Partially Implemented", "Not Implemented", "Not Applicable") +# Subsets FedRAMP scopes to a provider act, in the subset description: the rules +# apply once the provider takes that step, and not before. +CONDITIONAL_SUBSETS = { + ("MKT", "CSO"): "providers seeking a listing in the FedRAMP Marketplace", + ("FRC", "APP"): "providers who have met all other relevant rules and are ready to apply", +} WORKSTREAMS = tuple(f"W{n}" for n in range(2, 11)) -CLOSERS = (*WORKSTREAMS, "sam", "optional") +CLOSERS = (*WORKSTREAMS, "sam") EVIDENCE_KINDS = ("test", "step", "artifact", "decision") TOP_KEYS = {"$comment", "schema", "basis", "records"} -RECORD_KEYS = {"frr_id", "status", "implementation", "evidence", "gap", "closes_with", "may_rationale"} +RECORD_KEYS = {"frr_id", "status", "implementation", "evidence", "gap", "closes_with", "may_rationale", + "not_applicable"} EVIDENCE_KEYS = {"kind", "ref"} TEST_RE = re.compile(r"^(tests/[A-Za-z0-9_/]+\.py)::([A-Za-z_][A-Za-z0-9_]*)(?:::([A-Za-z_][A-Za-z0-9_]*))?(?:\[[^\]]*\])?$") @@ -211,15 +221,21 @@ def validate(doc, applicable, resolve, where="data/frr-records.json"): if rec.get("gap") is not None or closes is not None: _fail(f"{rid}: an Implemented record has no gap and nothing closing it") continue + if status == "Not Applicable": + rule = applicable[rid] + if rule["force"] != "MAY" and (rule["family"], rule.get("subset")) not in CONDITIONAL_SUBSETS: + _fail(f"{rid}: cannot be Not Applicable: FedRAMP scopes it to every 20x Class C provider " + f"(force {rule['force']})") + _text(rec, "not_applicable", rid) + if rec.get("gap") is not None or closes is not None: + _fail(f"{rid}: a Not Applicable record has no gap and nothing closing it") + continue + if "not_applicable" in rec: + _fail(f"{rid}: not_applicable is for Not Applicable records only") _text(rec, "gap", rid) if not isinstance(closes, list) or not closes or len(set(closes)) != len(closes) \ or any(c not in CLOSERS for c in closes): _fail(f"{rid}: closes_with must be a non-empty list drawn from {list(CLOSERS)}") - if "optional" in closes: - if applicable[rid]["force"] != "MAY": - _fail(f"{rid}: only a MAY rule can be left as optional (Class C force is {applicable[rid]['force']})") - if len(closes) != 1: - _fail(f"{rid}: an optional rule is closed by nothing else") missing = sorted(set(applicable) - seen) if missing: _fail(f"no record for {len(missing)} applicable rule(s): {', '.join(missing)}") @@ -246,11 +262,9 @@ def summary(doc, applicable): for r in doc["records"]: f = applicable[r["frr_id"]]["force"] by_force.setdefault(f, Counter())[r["status"]] += 1 - optional = sum(1 for r in doc["records"] if r.get("closes_with") == ["optional"]) return {"rules": len(doc["records"]), "by_status": {s: by_status.get(s, 0) for s in STATUSES}, - "by_force": {f: {s: c.get(s, 0) for s in STATUSES} for f, c in sorted(by_force.items())}, - "optional_not_exercised": optional} + "by_force": {f: {s: c.get(s, 0) for s in STATUSES} for f, c in sorted(by_force.items())}} def main(argv=None): diff --git a/scripts/_rules.py b/scripts/_rules.py index 64d7043d..ed703e1c 100644 --- a/scripts/_rules.py +++ b/scripts/_rules.py @@ -183,7 +183,7 @@ def applicable_provider_rules(rules_dir=RULES_DIR): _fail(f"{rid} has no Class {CERT_CLASS.upper()} statement and force") if rid in out: _fail(f"{rid} appears twice in the corpus") - out[rid] = {"family": fam_id, "name": rule.get("name", ""), + out[rid] = {"family": fam_id, "subset": subset, "name": rule.get("name", ""), "statement": statement, "force": force} if not out: _fail("no provider rules apply; the applicability metadata has changed shape") diff --git a/tests/fixtures/frr/known-bad.json b/tests/fixtures/frr/known-bad.json index b7e3a180..17c087d1 100644 --- a/tests/fixtures/frr/known-bad.json +++ b/tests/fixtures/frr/known-bad.json @@ -15,11 +15,15 @@ {"case": "partial without a gap", "replace": {"frr_id": "AFC-CSO-INB", "status": "Partially Implemented", "implementation": "x", "closes_with": ["W9"]}, "expect": "AFC-CSO-INB: gap must be a non-empty string"}, {"case": "partial with nothing closing it", "replace": {"frr_id": "AFC-CSO-INB", "status": "Partially Implemented", "implementation": "x", "gap": "x"}, "expect": "closes_with must be a non-empty list"}, {"case": "unknown workstream", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W11"]}, "expect": "closes_with must be a non-empty list"}, - {"case": "a MUST rule left as optional", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["optional"]}, "expect": "only a MAY rule can be left as optional"}, + {"case": "Not Applicable on a rule FedRAMP scopes in", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Applicable", "implementation": "x", "not_applicable": "No mail is expected."}, "expect": "AFC-CSO-INB: cannot be Not Applicable"}, + {"case": "Not Applicable without a reason", "replace": {"frr_id": "MKT-CSO-PML", "status": "Not Applicable", "implementation": "x"}, "expect": "MKT-CSO-PML: not_applicable must be a non-empty string"}, + {"case": "Not Applicable with a gap", "replace": {"frr_id": "MKT-CSO-PML", "status": "Not Applicable", "implementation": "x", "not_applicable": "x", "gap": "x", "closes_with": ["sam"]}, "expect": "a Not Applicable record has no gap"}, + {"case": "a reason for not applying on an applicable record", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "not_applicable": "x"}, "expect": "not_applicable is for Not Applicable records only"}, + {"case": "optional is no longer a closer", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["optional"]}, "expect": "closes_with must be a non-empty list"}, {"case": "unknown status", "replace": {"frr_id": "AFC-CSO-INB", "status": "Met", "implementation": "x"}, "expect": "status 'Met' is not one of"}, {"case": "unknown field", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "met": true}, "expect": "unknown field(s) ['met']"}, {"case": "account ID in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "Bucket arn:aws:s3:us-east-1:111122223333:x", "gap": "x", "closes_with": ["W9"]}, "expect": "implementation carries an AWS account ID"}, - {"case": "MAY rule without a rationale", "replace": {"frr_id": "VDR-CSO-SIR", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["optional"]}, "expect": "VDR-CSO-SIR: may_rationale must be a non-empty string"}, + {"case": "MAY rule without a rationale", "replace": {"frr_id": "VDR-CSO-SIR", "status": "Not Applicable", "implementation": "x", "not_applicable": "x"}, "expect": "VDR-CSO-SIR: may_rationale must be a non-empty string"}, {"case": "rationale on a MUST rule", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "may_rationale": "x"}, "expect": "may_rationale is for MAY rules only"}, {"case": "personal email in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "Mail goes to someone@example.com today.", "closes_with": ["W9"]}, "expect": "gap names an email address outside"} ] diff --git a/tests/test_frr_records.py b/tests/test_frr_records.py index 9dfcd22c..d35cb568 100644 --- a/tests/test_frr_records.py +++ b/tests/test_frr_records.py @@ -111,8 +111,6 @@ def test_summary_counts_add_up(applicable): s = _frr.summary(_committed(), applicable) assert s["rules"] == 154 == sum(s["by_status"].values()) assert sum(sum(c.values()) for c in s["by_force"].values()) == 154 - assert s["optional_not_exercised"] == sum( - 1 for r in _committed()["records"] if r.get("closes_with") == ["optional"]) # --- known-bad: each invariant fails for its own reason ---------------------------- @@ -143,7 +141,8 @@ def test_known_bad_fixture_covers_every_check(): for fragment in ("no record for", "is not a rule that applies", "duplicate record", "Implemented needs evidence", "names no test function", "does not exist", "no step named", "freshness-governed", "in-force decision", "has no gap", - "gap must be", "closes_with must be", "only a MAY rule", "is not one of", + "gap must be", "closes_with must be", "cannot be Not Applicable", "is not one of", + "not_applicable must be", "a Not Applicable record has no gap", "for Not Applicable records only", "unknown field", "AWS account ID", "email address outside", "may_rationale must be", "for MAY rules only"): assert fragment in expects, fragment @@ -200,3 +199,17 @@ def test_the_system_and_fedramp_addresses_are_allowed(applicable, resolve): def test_every_may_rule_says_why(applicable): for r in _committed()["records"]: assert ("may_rationale" in r) is (applicable[r["frr_id"]]["force"] == "MAY"), r["frr_id"] + + +def test_not_applicable_only_where_fedramp_scopes_it_out(applicable): + for r in _committed()["records"]: + if r["status"] == "Not Applicable": + rule = applicable[r["frr_id"]] + assert rule["force"] == "MAY" or (rule["family"], rule["subset"]) in _frr.CONDITIONAL_SUBSETS, r["frr_id"] + + +def test_conditional_subsets_say_so_in_the_corpus(): + """The scoping that makes a subset conditional is FedRAMP's text, not ours.""" + rules, _ = _rules.load_rules() + for (fam, subset), phrase in _frr.CONDITIONAL_SUBSETS.items(): + assert phrase in rules["FRR"][fam]["info"]["subsets"][subset]["description"] From ff9336e19cc08a0ae50c09b328904d53268c9d0a Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:32:49 -0500 Subject: [PATCH 04/97] CR26 vulnerability reports: rating history, decision-led fields, evidence-only history, period activity VER-RPT-VDT: carry the historical rating and rating history (providerExtensions; the schema has no field), take the next reduction from a decision's new planned_reduction before the remediation deadline, populate supplementary information from the decision, explain overdue status rather than the suppression, and date a record from a decision that names it (the manual ZAP scans behind DEC-039/040). VER-RPT-AVI: a .checkov.yaml suppression resolves to the resources its decision names, so its IRV is decided from the inventory; suppressed checks are dated by their decision. VER-TFR-MRH: a record that leaves the VDR is Remediated only from evidence (a closed decision, a false-positive suppression, or its scanner running again); otherwise it is recorded as no longer reported with the reason and no final disposition. Closed decisions are backfilled. VER-RPT-PER: each VDT/AVI carries an activity summary for its period, including scans with times and the runtime V&V run, linked. Ratings, deadlines, overdue status and blocking are unchanged on the live inputs. Co-Authored-By: Claude Opus 5.5 --- data/frr-records.json | 139 ++++- infrastructure/schemas/decisions.schema.json | 10 +- scripts/_decisions.py | 13 +- scripts/build-cr26-vdr.py | 567 ++++++++++++++++-- scripts/build-vdr-report.py | 100 ++- .../decisions/planned-reduction-closed.json | 28 + .../planned-reduction-malformed.json | 27 + tests/test_cr26_vdr.py | 309 +++++++++- tests/test_decisions.py | 2 + tests/test_vdr_component.py | 86 +++ 10 files changed, 1187 insertions(+), 94 deletions(-) create mode 100644 tests/fixtures/decisions/planned-reduction-closed.json create mode 100644 tests/fixtures/decisions/planned-reduction-malformed.json diff --git a/data/frr-records.json b/data/frr-records.json index c7d069f9..ede8fbb4 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -1481,11 +1481,36 @@ { "frr_id": "VER-RPT-AVI", "status": "Partially Implemented", - "implementation": "vdr-accepted-vulnerabilities.json and .md under /.well-known/ (scripts/build-cr26-vdr.py, #496): each accepted vulnerability's detail and acceptance rationale from its decision record; the build fails if a rationale is missing.", - "evidence": [], - "gap": "Supplementary information is never populated. For the three suppression records, the detection time is when the ledger first saw the suppression rather than when the weakness was found, and the resource is the suppression file, so IRV is unresolved. To close: Take the detection date from the decision; resolve suppressions to the decision's resources so IRV is evaluated; populate supplementary information.", + "implementation": "vdr-accepted-vulnerabilities.json and .md under /.well-known/ (scripts/build-cr26-vdr.py): each accepted vulnerability's detail and acceptance rationale from its decision record (the build fails if a rationale is missing), supplementary information from the decision (when decided, review date, resources, plan), and a detection time from the decision for suppressed checks no scanner reports. A .checkov.yaml suppression resolves to the resources its decision names (scripts/build-vdr-report.py resolve_decision_resources), so its IRV is decided from the inventory: CKV_AWS_68 and CKV_AWS_86 now resolve to the CloudFront distribution.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_supplementary_information_comes_from_the_decision" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_detection_time_comes_from_a_decision_that_names_the_record" + }, + { + "kind": "test", + "ref": "tests/test_vdr_component.py::test_a_suppressions_reachability_is_decided_from_the_resource_it_affects" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "step", + "ref": "evidence-nightly.yml#Build and sign CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-accepted-vulnerabilities.json" + } + ], + "gap": "CKV_AWS_117 (DEC-010) still has an unresolved IRV: DEC-010 names aws-lambda::compliance-monitor, which no inventory component carries, and the check is suppressed globally in .checkov.yaml, so it covers every Lambda, including the internet-reachable Silk Reeling backend. Its IRV is the fail-safe (true). DEC-007, DEC-009 and DEC-010 record no original_detection_date, so their suppressions are dated by decided_on. To close: the operator sets DEC-010's applies_to.resources to the Lambda components the suppression covers (component IDs, e.g. aws::function::opa_compliance) and records original detection dates for DEC-007/009/010.", "closes_with": [ - "W2" + "sam" ] }, { @@ -1511,11 +1536,23 @@ { "frr_id": "VER-RPT-PER", "status": "Partially Implemented", - "implementation": "vdr-report.json, vdr-trend.json, vdr-status.json and ksi-signal-runtime.json are refreshed twice daily and on each deploy; the VDT, AVI and history (#496) cover the period since the previous report.", - "evidence": [], - "gap": "No report summarizes the period's activity (detections, evaluations, reductions, remediations, scans and V&V runs with times), and V&V results are not cross-referenced from the period's report. To close: Add an activity summary for the period to each report (detections, evaluations, reductions, remediations, scans with times, V&V runs and failures), linking the runtime results for the period.", + "implementation": "vdr-report.json, vdr-trend.json, vdr-status.json and ksi-signal-runtime.json are refreshed twice daily and on each deploy. The VDT and AVI each cover the period since the previous report and carry an activity summary for it (providerExtensions.activitySummary, and an 'Activity this period' section in the .md): detections, evaluations, rating changes, reductions, records no longer reported, remediations, decisions recorded and closed, every scan and feed the build read with its time (VDR build, Grype, Dependabot, Checkov, tfsec, ZAP, KEV, EPSS), and the runtime V&V run the build read, linked by URI and signal ID with its validation and failure counts (counted only when its signature verifies).", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_each_period_report_summarizes_the_periods_activity" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_an_unverified_runtime_run_is_linked_but_not_counted" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_the_markdown_reports_carry_the_activity_and_link_the_runtime_results" + } + ], + "gap": "The runtime emitter publishes only its latest run, so a runtime V&V run replaced before the next VDR build is not summarized; and activity outside the pipeline (VDP reports, penetration tests, assessments) is not in the summary (see VER-RPT-HLO). To close: retain each runtime run's result (or have each report list every run since the previous report), and add the non-pipeline activities to the summary.", "closes_with": [ - "W2", "W7" ] }, @@ -1532,12 +1569,45 @@ }, { "frr_id": "VER-RPT-VDT", - "status": "Partially Implemented", - "implementation": "vdr-detail-report.json and .md under /.well-known/ (scripts/build-cr26-vdr.py, #496), validated against the vendored VDT schema before writing: tracking ID, detection time and source, evaluation time, IRV, LEV, current rating, PAIN reductions, projected next reduction, overdue status and final disposition.", - "evidence": [], - "gap": "Historical PAIN and rating history are not carried into the record; the projected next reduction is always the remediation deadline; supplementary information is never produced; overdue explanations for closed findings describe the suppression instead; ZAP detection times start when the scans entered the pipeline, not at the earlier manual scans. To close: Carry historical PAIN and rating history; let a decision set a planned next reduction; populate supplementary information from the decision; take detection dates from the decision where the scanner never saw the finding.", - "closes_with": [ - "W2" + "status": "Implemented", + "implementation": "vdr-detail-report.json and .md under /.well-known/ (scripts/build-cr26-vdr.py), validated against the vendored VDT schema before writing: tracking ID; detection time (the earliest evidence: the scanner's first report, or the original_detection_date of a decision naming that record, e.g. DEC-039/040 for the manual ZAP scans) and source; evaluation time; IRV; LEV; current rating, with the historical rating and full rating history under providerExtensions (the schema has no field for them); PAIN reductions; projected next reduction from the decision's planned_reduction, else the remediation deadline; overdue status whose explanation explains the overdue status (and likely-to-become-overdue); supplementary information from the decision (justification, dates, resources, plan, mitigation); final disposition. Each report carries an activity summary for its period.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_records_carry_their_historical_rating_and_rating_history" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_a_decision_sets_the_next_reduction_and_the_deadline_is_only_the_fallback" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_supplementary_information_comes_from_the_decision" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_overdue_explanations_explain_the_overdue_status_not_the_suppression" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_detection_time_comes_from_a_decision_that_names_the_record" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_outputs_validate_and_name_their_schema" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "step", + "ref": "evidence-nightly.yml#Build and sign CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-detail-report.json" + } ] }, { @@ -1582,12 +1652,41 @@ }, { "frr_id": "VER-TFR-MRH", - "status": "Partially Implemented", - "implementation": "vdr-history.json under /.well-known/ in FedRAMP's history schema (#496), rebuilt twice daily with 366-day retention, alongside vdr-report.json (per-finding rating history) and vdr-trend.json.", - "evidence": [], - "gap": "A record that leaves the VDR is labelled Remediated even when it vanished because coverage stopped or a suppression was added; closed decisions from before the history began are not backfilled. To close: Set a final disposition only from evidence (coverage continued and the scanner stopped reporting it, or a decision); otherwise record 'no longer reported'; backfill closed decisions.", - "closes_with": [ - "W2" + "status": "Implemented", + "implementation": "vdr-history.json under /.well-known/ in FedRAMP's history schema (scripts/build-cr26-vdr.py), rebuilt twice daily with 366-day retention, alongside vdr-report.json and vdr-trend.json. A record that leaves the VDR keeps the time it stopped being reported and gets a final disposition only from evidence (a closed decision, a false-positive suppression of its check, or its scanner running again over the same scope without reporting it), recorded in providerExtensions.dispositionBasis; otherwise noLongerReportedReason says why (suppressed, reported again, or unknown) and no final disposition is given. An unsupported Remediated carried from an earlier history is withdrawn. Closed (remediated) decisions from data/decisions.json are carried as history records (backfilledFromDecision) for 366 days from closing.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_history_keeps_a_finding_that_left_the_vdr_then_drops_it_after_a_year" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_a_record_that_vanished_without_evidence_has_no_final_disposition" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_an_unsupported_remediated_from_an_earlier_history_is_withdrawn" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_a_check_that_became_a_suppression_is_not_remediated" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_closed_decisions_are_backfilled_into_the_history_once" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "step", + "ref": "evidence-nightly.yml#Build and sign CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-history.json" + } ] }, { diff --git a/infrastructure/schemas/decisions.schema.json b/infrastructure/schemas/decisions.schema.json index a7fdcb71..9d1090dd 100644 --- a/infrastructure/schemas/decisions.schema.json +++ b/infrastructure/schemas/decisions.schema.json @@ -35,6 +35,13 @@ "closed_on": {"$ref": "#/$defs/date"}, "original_detection_date": {"$ref": "#/$defs/date"}, "planned_completion": {"type": "string"}, + "planned_reduction": { + "type": "object", + "description": "The next planned reduction in Potential Agency Impact (VER-RPT-VDT): the date it is expected and the rating it brings the finding to. Published as the finding's projectedNextReduction in place of the remediation deadline. In-force records only.", + "required": ["date", "target_pain"], + "properties": {"date": {"$ref": "#/$defs/date"}, "target_pain": {"type": "string", "pattern": "^N[0-5]$"}}, + "additionalProperties": false + }, "remediation_plan": {"type": "string"}, "vendor_dependency": {"type": ["string", "boolean", "object"]}, "point_of_contact": {"type": "string"}, @@ -57,7 +64,8 @@ "then": {"required": ["decided_by", "decided_on", "review_by"], "properties": {"disposition": {"not": {"const": "remediated"}}}}}, {"if": {"properties": {"status": {"const": "closed"}}}, - "then": {"properties": {"disposition": {"const": "remediated"}}}} + "then": {"properties": {"disposition": {"const": "remediated"}}, + "not": {"required": ["planned_reduction"]}}} ] } } diff --git a/scripts/_decisions.py b/scripts/_decisions.py index de3046a6..e847eea9 100644 --- a/scripts/_decisions.py +++ b/scripts/_decisions.py @@ -28,6 +28,8 @@ ID_RE = re.compile(r"^DEC-\d{3}$") FORMER_RE = re.compile(r"^POAM-\d{3}$") DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +PAIN_RE = re.compile(r"^N[0-5]$") +PLAN_KEYS = {"date", "target_pain"} # Dispositions of an in-force record. Config-scanner suppressions use # risk-accepted / false-positive; vulnerabilities use the triage vocabulary. @@ -39,7 +41,7 @@ # field is refused: a misspelt review_by would otherwise read as absent. TOP_KEYS = {"$comment", "_note", "schema", "decisions"} RECORD_KEYS = {"id", "former_id", "title", "status", "disposition", "justification", *DECISION_FIELDS, - "closed_on", "original_detection_date", "planned_completion", "remediation_plan", + "closed_on", "original_detection_date", "planned_completion", "planned_reduction", "remediation_plan", "vendor_dependency", "point_of_contact", "controls", "applies_to", "vuln_fields", "source"} APPLIES_KEYS = {"check_ids", "vuln_ids", "resources"} @@ -93,6 +95,15 @@ def validate(doc, where="data/decisions.json"): _fail(f"{rid}: {k} must be an ISO date") else: _fail(f"{rid}: status must be in-force or closed") + plan = r.get("planned_reduction") + if plan is not None: + # The next planned PAIN reduction (VER-RPT-VDT): when, and to what + # rating. A malformed plan would publish a projection nobody made. + if (not isinstance(plan, dict) or set(plan) != PLAN_KEYS + or not DATE_RE.match(str(plan.get("date"))) or not PAIN_RE.match(str(plan.get("target_pain")))): + _fail(f"{rid}: planned_reduction must be {{date: YYYY-MM-DD, target_pain: N0-N5}}") + if status != "in-force": + _fail(f"{rid}: only an in-force record plans a reduction") applies = r.get("applies_to") if not isinstance(applies, dict) or set(applies) - APPLIES_KEYS: _fail(f"{rid}: applies_to must be an object with only {sorted(APPLIES_KEYS)}") diff --git a/scripts/build-cr26-vdr.py b/scripts/build-cr26-vdr.py index 004ebeb3..3a322e47 100644 --- a/scripts/build-cr26-vdr.py +++ b/scripts/build-cr26-vdr.py @@ -23,15 +23,46 @@ # rationale is the decision record's justification, never invented: an accepted # finding with no justification stops the build. # +# What each record says, and where it comes from: +# detection time the earliest evidence: the scanner's first report, or the +# original_detection_date of a decision naming this exact +# record, when the weakness was found before the pipeline's +# scanner saw it (e.g. a manual scan). A .checkov.yaml +# suppression, which no scanner ever reports, falls back to +# the decision date when the decision records no detection +# date. The basis is in providerExtensions. +# next reduction the decision's planned_reduction (date + target rating) when +# one is recorded, else the remediation deadline. +# overdue explains the overdue status (deadline, and why it is still +# open), never the suppression. +# supplementary the decision behind the record: its justification (VDT only; +# the AVI carries it as the rationale), when it was decided and +# is reviewed, the resources it names, any remediation plan or +# mitigation. +# rating history the historical rating and every re-evaluation, from the VDR +# ledger. The schema has a field only for completed reductions, +# so the full history is in providerExtensions. +# # The schemas carry no field for the component, the CVE, the scanner check or the # decision. Those are the joins a reader needs to trace a finding to the system, # so each record carries them under `providerExtensions` (the schemas allow # additional properties). # # Report periods (VER-RPT-PER: summarize ALL activity since the previous report): -# each VDT/AVI runs from the previous published report's end to now. The MRH -# accumulates: a finding that leaves the VDR stays in the history, marked -# Remediated with the time it stopped being reported, for HISTORY_DAYS. +# each VDT/AVI runs from the previous published report's end to now, and carries +# an activity summary for that period (providerExtensions.activitySummary): +# detections, evaluations, rating changes and reductions, records that stopped +# being reported, decisions recorded and closed, every scan and feed the VDR +# build read with its time, and the runtime verification run it read, linked. +# +# The MRH accumulates. A record that leaves the VDR stays in the history for +# HISTORY_DAYS with the time it stopped being reported. It gets a final +# disposition only from evidence: a closed decision that says remediated, a +# false-positive suppression of its check, or its scanner having run again over +# the same scope without reporting it. Otherwise it is recorded as no longer +# reported, with the reason (a risk-accepted suppression, or unknown), and no +# final disposition. Closed (remediated) decisions are carried in the history +# too, including those from before the pipeline's ledger began. # # build-cr26-vdr.py --vdr vdr-report.json --decisions ../data/decisions.json \ # --previous-vdt prev-vdt.json --previous-avi prev-avi.json --previous-mrh prev-mrh.json @@ -59,6 +90,14 @@ "dependabot": "GitHub Dependabot (software composition analysis)", "opa": "OPA policy evaluation of the deployed configuration", } +SOURCE_BY_LABEL = {v: k for k, v in SOURCES.items()} +# The detection-gate feed that shows a source's scanner ran in a build. A source +# with no feed here (OPA) never earns a disposition from coverage. +COVERAGE_FEEDS = {"grype": "grype", "dependabot": "dependabot", "checkov": "checkov", + "checkov-suppression": "checkov", "tfsec": "tfsec"} +SCAN_FEEDS = {"grype": "Grype software composition scan", "dependabot": "Dependabot alerts", + "checkov": "Checkov IaC scan", "tfsec": "tfsec IaC scan", + "kev": "CISA KEV catalog", "epss": "FIRST EPSS scores"} VDT_OUT = "vdr-detail-report.json" AVI_OUT = "vdr-accepted-vulnerabilities.json" @@ -76,7 +115,7 @@ def _rating(pain): _fail(f"PAIN rating {pain!r} is not N0-N5") -def _ts(value, where): +def _dt(value, where): if not isinstance(value, str): _fail(f"{where}: missing timestamp") try: @@ -85,11 +124,28 @@ def _ts(value, where): _fail(f"{where}: {value!r} is not an ISO 8601 timestamp") if dt.tzinfo is None: _fail(f"{where}: {value!r} has no time zone") - return dt.astimezone(timezone.utc).isoformat() + return dt.astimezone(timezone.utc) + + +def _ts(value, where): + return _dt(value, where).isoformat() + + +def _maybe_dt(value): + """A timestamp, or a YYYY-MM-DD date read as midnight UTC; None if neither.""" + if not isinstance(value, str): + return None + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return (dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)).astimezone(timezone.utc) -def _justifications(register): - return {r["id"]: (r.get("justification") or "").strip() for r in register.get("decisions", [])} +def _day(value): + """A register date (YYYY-MM-DD) as midnight UTC, ISO 8601; None if absent.""" + dt = _maybe_dt(value) + return dt.isoformat() if dt else None def _description(f): @@ -104,16 +160,127 @@ def is_accepted(f): return disp in ACCEPTED_KINDS or bool(f.get("accepted")) -def detail(f): +# --- per-record fields -------------------------------------------------------------- + +def _detection(f, rec, tid): + """(detectedAt, basis): the earliest evidence of the weakness.""" + seen = _dt(f.get("first_detected"), f"{tid}.first_detected") + # Only a decision that names this exact record (a vulnerability ID such as + # zap-10099, or a suppressed check) dates it. A decision about a check across + # many resources dates the check, not a resource added after it. + if rec and tid in _decision_claims(rec): + known = _maybe_dt(rec.get("original_detection_date")) + if known and known < seen: + return known.isoformat(), (f"{rec['id']} original_detection_date: found before the " + "pipeline's scanner first reported it") + decided = _maybe_dt(rec.get("decided_on")) + if not known and f.get("source") == "checkov-suppression" and decided and decided < seen: + return decided.isoformat(), (f"{rec['id']} decided_on: the decision records no detection " + "date, and the weakness was known when it was decided (a " + "suppressed check is never reported by the scanner)") + return seen.isoformat(), "first reported by the scanner (VDR ledger)" + + +def _rating_history(f, tid): + out = [] + for h in f.get("rating_history") or []: + if not isinstance(h, dict): + continue + out.append({"at": _ts(h.get("at"), f"{tid}.rating_history"), "rating": _rating(h.get("pain")), + "isInternetReachable": h.get("irv"), "isLikelyExploitable": h.get("lev"), + "reason": h.get("reason")}) + return out + + +def _overdue(f, rec, final): + """overdueStatus whose explanation explains the overdue status itself.""" + over = bool(f.get("is_overdue")) + due = f.get("remediation_due_at") or (f.get("next_reduction") or {}).get("estimated_time") + did = rec["id"] if rec else f.get("decision_id") + if over: + if f.get("accepted") and f.get("accepted_at"): + why = ("Overdue: not fully mitigated or remediated within the VER-TFR-MAV maximum " + f"period, which ended {f['accepted_at'][:10]}") + elif due: + why = f"Overdue: its remediation deadline, {due[:10]}, passed without remediation" + else: + why = f"Overdue: {f.get('block_reason') or 'past its deadline'}" + if rec and rec.get("disposition") in ACCEPTED_KINDS: + why += (f". It is still open because {did} accepts it ({rec['disposition']}, decided " + f"{rec.get('decided_on', 'on a date not recorded')}), so no remediation is planned") + elif rec and rec.get("disposition") == "vendor-dependency": + why += f". It is waiting on a vendor fix ({did})" + elif f.get("block_reason") and due: + why += f" ({f['block_reason']})" + return {"isOverdue": True, "explanation": why + "."} + if f.get("will_be_overdue") and due: + return {"isOverdue": False, + "explanation": f"Not yet overdue, but likely to become overdue: remediation is due {due[:10]}."} + if final: + return {"isOverdue": False, + "explanation": f"Not overdue: closed as {final}{f' under {did}' if did else ''}."} + if due: + return {"isOverdue": False, "explanation": f"Not overdue: remediation is due {due[:10]}."} + return {"isOverdue": False} + + +def _supplementary(f, rec, accepted): + parts = [] + if rec: + head = (f"{rec['id']} ({rec['disposition']}; decided {rec.get('decided_on', 'on a date not recorded')}" + f"{', review by ' + rec['review_by'] if rec.get('review_by') else ''})") + parts.append(head + ("." if accepted else f": {rec['justification'].strip()}")) + res = (rec.get("applies_to") or {}).get("resources") or [] + if res: + parts.append(f"Resources named by the decision: {', '.join(res)}.") + if rec.get("remediation_plan"): + parts.append(f"Remediation plan: {rec['remediation_plan'].strip()}") + mit = (rec.get("vuln_fields") or {}).get("mitigation") + if mit: + parts.append(f"Mitigation record: {mit}.") + m = f.get("mitigation") + if isinstance(m, dict) and m.get("id"): + state = "verified" if m.get("valid") else f"void ({m.get('void_reason') or 'not verified'})" + parts.append(f"Mitigation {m['id']} ({m.get('effect')}): {state}.") + if f.get("supplementary_info"): + parts.append(str(f["supplementary_info"]).strip()) + return " ".join(p for p in parts if p) + + +def _projection(f, rec, rating, tid): + """(projectedNextReduction, basis), or (None, None) when none is planned.""" + plan = (rec or {}).get("planned_reduction") + if plan: + target = _rating(plan.get("target_pain")) + if target < rating: + return ({"estimatedAt": _ts(_day(plan.get("date")), f"{rec['id']}.planned_reduction"), + "targetRating": target}, f"{rec['id']} planned_reduction") + nxt = f.get("next_reduction") + if nxt and nxt.get("estimated_time"): + target = nxt.get("target_pain") + basis = "remediation deadline (no planned reduction recorded)" + if plan: + basis = f"remediation deadline ({rec['id']} planned_reduction is not below the current rating)" + # "remediated" removes the vulnerability: no impact remains (N0). + return ({"estimatedAt": _ts(nxt["estimated_time"], f"{tid}.next_reduction"), + "targetRating": 0 if target == "remediated" else _rating(target)}, basis) + return None, None + + +def detail(f, recs=None, accepted=False): """A VDR finding as a FedRAMP vulnerabilityDetail record.""" + recs = recs or {} tid = f.get("tracking_id") if not tid: _fail("a finding has no tracking_id") + rec = recs.get(f.get("decision_id")) pain = f.get("current_pain") or f.get("pain") - rec = { + rating = _rating(pain) + detected_at, detected_basis = _detection(f, rec, tid) + out = { "providerTrackingId": tid, "detection": { - "detectedAt": _ts(f.get("first_detected"), f"{tid}.first_detected"), + "detectedAt": detected_at, "detectionSource": SOURCES.get(f.get("source"), f.get("source") or "unknown"), }, "vulnerabilityDescription": _description(f) or tid, @@ -124,34 +291,31 @@ def detail(f): f"{tid}.first_evaluated"), "isInternetReachable": bool(f.get("internet_reachable")), "isLikelyExploitable": bool(f.get("likely_exploitable")), - "currentRating": _rating(pain), + "currentRating": rating, } - over = {"isOverdue": bool(f.get("is_overdue"))} - if f.get("overdue_explanation"): - over["explanation"] = f["overdue_explanation"] - elif over["isOverdue"]: - _fail(f"{tid}: overdue with no explanation") - rec["overdueStatus"] = over - if f.get("supplementary_info"): - rec["supplementaryRiskInformation"] = f["supplementary_info"] - reductions = [{"reducedAt": _ts(r.get("at"), f"{tid}.completed_reductions"), "rating": _rating(r.get("to"))} - for r in f.get("completed_reductions") or []] - if reductions: - rec["painReductionEvents"] = reductions disp = f.get("final_disposition") or f.get("current_disposition") or "open" + final = None if disp in FINAL: - rec["finalDisposition"] = FINAL[disp] + final = FINAL[disp] elif disp == "mitigated": - rec["finalDisposition"] = "Fully Mitigated" if f.get("fully_mitigated") else "Partially Mitigated" - nxt = f.get("next_reduction") - if "finalDisposition" not in rec and not is_accepted(f) and nxt and nxt.get("estimated_time"): - target = nxt.get("target_pain") - # "remediated" removes the vulnerability: no impact remains (N0). - rec["projectedNextReduction"] = { - "estimatedAt": _ts(nxt["estimated_time"], f"{tid}.next_reduction"), - "targetRating": 0 if target == "remediated" else _rating(target), - } - rec["providerExtensions"] = { + final = "Fully Mitigated" if f.get("fully_mitigated") else "Partially Mitigated" + out["overdueStatus"] = _overdue(f, rec, final) + supp = _supplementary(f, rec, accepted) + if supp: + out["supplementaryRiskInformation"] = supp + reductions = [{"reducedAt": _ts(r.get("at"), f"{tid}.completed_reductions"), "rating": _rating(r.get("to"))} + for r in f.get("completed_reductions") or []] + if reductions: + out["painReductionEvents"] = reductions + if final: + out["finalDisposition"] = final + projection_basis = None + if final is None and not is_accepted(f): + projection, projection_basis = _projection(f, rec, rating, tid) + if projection: + out["projectedNextReduction"] = projection + ext = { + "source": f.get("source"), "componentId": f.get("component_id"), "resource": f.get("resource") or None, "cve": f.get("cve"), @@ -160,20 +324,36 @@ def detail(f): "disposition": disp, "isKev": bool(f.get("is_kev")), "remediationDueAt": f.get("remediation_due_at"), + "detectedAtBasis": detected_basis, + "firstReportedAt": _ts(f.get("first_detected"), f"{tid}.first_detected"), } - return rec - - -def accepted_info(f, justifications): + if f.get("affected_resources"): + ext["affectedResources"] = list(f["affected_resources"]) + if projection_basis: + ext["projectionBasis"] = projection_basis + # VER-RPT-VDT asks for the historical and current ratings; the schema has a + # field only for the current one and for completed reductions. + if f.get("historical_pain"): + ext["historicalRating"] = _rating(f["historical_pain"]) + history = _rating_history(f, tid) + if history: + ext["ratingHistory"] = history + out["providerExtensions"] = ext + return out + + +def accepted_info(f, recs): did = f.get("decision_id") - why = justifications.get(did, "") if did else "" + why = ((recs.get(did) or {}).get("justification") or "").strip() if did else "" if not why and f.get("accepted") and f.get("acceptance_explanation"): why = f["acceptance_explanation"] if not why: _fail(f"accepted finding {f.get('tracking_id')} has no decision justification ({did or 'no decision_id'})") - return {"vulnerabilityDetail": detail(f), "acceptanceRationale": why} + return {"vulnerabilityDetail": detail(f, recs, accepted=True), "acceptanceRationale": why} +# --- periods ------------------------------------------------------------------------ + def _period_start(previous, fallback): """The previous report's end, if it is a valid past time; else `fallback`.""" try: @@ -199,15 +379,104 @@ def _load_previous(path): return doc +def activity_summary(vdr, mrh, register, frm, to): + """VER-RPT-PER: the vulnerability detection and response activity in + [frm, to], from the VDR report, the history and the decision register.""" + lo, hi = _dt(frm, "period.from"), _dt(to, "period.to") + + def within(value): + dt = _maybe_dt(value) + return bool(dt and lo <= dt <= hi) + + rows = [f for f in vdr.get("findings") or [] if isinstance(f, dict)] + \ + [r for r in vdr.get("risk_accepted") or [] if isinstance(r, dict)] + detections = [{"providerTrackingId": f["tracking_id"], "at": f["first_detected"]} + for f in rows if within(f.get("first_detected"))] + evaluations = [{"providerTrackingId": f["tracking_id"], "at": f["first_evaluated"]} + for f in rows if within(f.get("first_evaluated"))] + changes = [{"providerTrackingId": f["tracking_id"], "at": h["at"], "rating": h.get("pain"), + "reason": h.get("reason")} + for f in rows for h in f.get("rating_history") or [] + if isinstance(h, dict) and within(h.get("at")) and h.get("reason") != "first evaluation"] + reductions = [{"providerTrackingId": f["tracking_id"], "at": r["at"], "from": r.get("from"), "to": r.get("to")} + for f in rows for r in f.get("completed_reductions") or [] + if isinstance(r, dict) and within(r.get("at"))] + gone = [{"providerTrackingId": r["providerTrackingId"], + "at": r["providerExtensions"]["noLongerReportedAt"], + "finalDisposition": r.get("finalDisposition"), + "reason": r["providerExtensions"].get("noLongerReportedReason")} + for r in mrh.get("activeVulnerabilities") or [] + if within((r.get("providerExtensions") or {}).get("noLongerReportedAt"))] + decided = [{"decisionId": d["id"], "on": d["decided_on"], "disposition": d["disposition"]} + for d in register.get("decisions", []) if within(_day(d.get("decided_on")))] + closed = [{"decisionId": d["id"], "on": d["closed_on"], "disposition": d["disposition"]} + for d in register.get("decisions", []) + if d.get("status") == "closed" and within(_day(d.get("closed_on")))] + remediated = {g["providerTrackingId"] for g in gone if g["finalDisposition"] == "Remediated"} | \ + {c["decisionId"] for c in closed} + meth = vdr.get("methodology") or {} + det = meth.get("detection") or {} + scans = [{"activity": "VDR build: every finding re-evaluated (PAIN, IRV, LEV, deadlines)", + "at": vdr.get("emitted_at"), "timeBasis": "build time", "status": "ok", + "inPeriod": within(vdr.get("emitted_at"))}] + for feed in det.get("feeds") or []: + name = SCAN_FEEDS.get(feed.get("feed")) if isinstance(feed, dict) else None + if not name: + continue + if feed.get("feed") in ("kev", "epss"): + at, basis = feed.get("source_date"), "published by the source" + else: + at, basis = det.get("checked_at"), "run or fetched in this build (when the detection gate checked it)" + row = {"activity": f"{name} ({feed.get('file')})", "at": at, "timeBasis": basis, + "status": feed.get("status"), "count": feed.get("count"), "inPeriod": within(at)} + if feed.get("feed") == "grype" and feed.get("source_date"): + row["vulnerabilityDatabaseBuiltAt"] = feed["source_date"] + scans.append(row) + dast = meth.get("dast") + if isinstance(dast, dict): + scans.append({"activity": "OWASP ZAP dynamic scan (weekly)", "at": dast.get("generated_at"), + "timeBasis": "scan report generated", "status": dast.get("status"), + "count": dast.get("alerts"), "inPeriod": within(dast.get("generated_at"))}) + rt = meth.get("runtime_verification") + runtime = None + if isinstance(rt, dict): + runtime = {"uri": rt.get("uri"), "signalId": rt.get("signal_id"), "emittedAt": rt.get("emitted_at"), + "signatureVerified": bool(rt.get("verified")), "validations": rt.get("validations"), + "failed": rt.get("failed"), "divergence": rt.get("divergence"), + "inPeriod": within(rt.get("emitted_at")), + "note": ("The runtime emitter publishes only its latest run; this is the run the VDR " + "build read. A run it replaced before this build is not retained.")} + if not rt.get("verified"): + runtime["note"] = f"Runtime signal not verified ({rt.get('reason')}); its results are not reported." + in_period = [s for s in scans if s["inPeriod"]] + run_in_period = bool(runtime and runtime["inPeriod"] and runtime["signatureVerified"]) + return { + "period": {"from": frm, "to": to}, + "counts": {"detections": len(detections), "evaluations": len(evaluations), + "ratingChanges": len(changes), "reductions": len(reductions), + "noLongerReported": len(gone), "remediations": len(remediated), + "decisionsRecorded": len(decided), "decisionsClosed": len(closed), + "scansInPeriod": len(in_period), + "runtimeRunsInPeriod": 1 if run_in_period else 0, + "runtimeFailuresInPeriod": (runtime["failed"] or 0) if run_in_period else 0}, + "detections": detections, "evaluations": evaluations, "ratingChanges": changes, + "reductions": reductions, "noLongerReported": gone, "remediations": sorted(remediated), + "decisionsRecorded": decided, "decisionsClosed": closed, "scans": scans, + "runtimeVerification": runtime, + } + + +# --- build -------------------------------------------------------------------------- + def build(vdr, register, previous_vdt, previous_avi, previous_mrh, now): if not isinstance(vdr, dict) or not isinstance(vdr.get("findings"), list): _fail("vdr-report.json has no findings list") - just = _justifications(register) + recs = _decisions.records(register) scanner = list(vdr["findings"]) # Suppressed checks the VDR lists as risk-accepted are accepted vulnerabilities too. suppressed = [dict(r, final_disposition="risk-accepted") for r in vdr.get("risk_accepted") or []] - active = [detail(f) for f in scanner if not is_accepted(f)] - accepted = [accepted_info(f, just) for f in scanner + suppressed if is_accepted(f)] + active = [detail(f, recs) for f in scanner if not is_accepted(f)] + accepted = [accepted_info(f, recs) for f in scanner + suppressed if is_accepted(f)] times = [r["detection"]["detectedAt"] for r in active] + \ [a["vulnerabilityDetail"]["detection"]["detectedAt"] for a in accepted] earliest = min(times) if times else now.isoformat() @@ -223,18 +492,119 @@ def build(vdr, register, previous_vdt, previous_avi, previous_mrh, now): avi = {"$schema": fs.stamp("avi"), **cpo, "reportPeriod": {"from": _period_start(previous_avi, earliest), "to": to}, "acceptedVulnerabilities": accepted, "providerExtensions": dict(ext)} - mrh = build_history(active, accepted, previous_mrh, now) - mrh["providerExtensions"] = {**ext, **mrh["providerExtensions"]} for doc, period in ((vdt, "VDT"), (avi, "AVI")): if doc["reportPeriod"]["from"] > doc["reportPeriod"]["to"]: _fail(f"{period} period starts after it ends") + mrh = build_history(active, accepted, previous_mrh, now, vdr=vdr, register=register) + mrh["providerExtensions"] = {**ext, **mrh["providerExtensions"]} + for doc in (vdt, avi): + doc["providerExtensions"]["activitySummary"] = activity_summary( + vdr, mrh, register, doc["reportPeriod"]["from"], doc["reportPeriod"]["to"]) return vdt, avi, mrh -def build_history(active, accepted, previous, now): - """VER-TFR-MRH: every current record, plus records that left the VDR within - HISTORY_DAYS, closed as Remediated with the time they stopped being reported.""" - cutoff = (now - timedelta(days=HISTORY_DAYS)).isoformat() +# --- history ------------------------------------------------------------------------ + +def _keys(rec): + ext = rec.get("providerExtensions") or {} + return {k for k in (rec.get("providerTrackingId"), ext.get("scannerCheck"), ext.get("cve")) if k} + + +def _decision_claims(d): + a = d.get("applies_to") or {} + return set(a.get("check_ids") or []) | set(a.get("vuln_ids") or []) + + +def _coverage(source, vdr, last_reported): + """Evidence that `source`'s scanner ran again over the same scope after the + record was last reported, or None.""" + meth = (vdr or {}).get("methodology") or {} + if source == "zap": + dast = meth.get("dast") or {} + gen, last = _maybe_dt(dast.get("generated_at")), _maybe_dt(last_reported) + if dast.get("status") == "ok" and gen and last and gen > last: + return f"the ZAP scan of {gen.isoformat()} ran after it was last reported and does not report it" + return None + feed = COVERAGE_FEEDS.get(source) + det = meth.get("detection") or {} + rows = [r for r in det.get("feeds") or [] if isinstance(r, dict) and r.get("feed") == feed] + if feed and det.get("checked_at") and rows and all(r.get("status") == "ok" for r in rows): + return (f"{feed} ran in the VDR build checked {det['checked_at']} (status ok) " + "and no longer reports it") + return None + + +def _disposition_from_evidence(rec, vdr, register, last_reported): + """(finalDisposition or None, reason, basis) for a record that left the VDR.""" + ext = rec.get("providerExtensions") or {} + keys = _keys(rec) + did = ext.get("decisionId") + for d in (register or {}).get("decisions", []): + if d.get("status") == "closed" and (d["id"] == did or keys & _decision_claims(d)): + return ("Remediated", "remediated", + f"{d['id']} closed as remediated on {d.get('closed_on', 'a date not recorded')}") + check = ext.get("scannerCheck") + for section, final, reason in (("false_positives", "False Positive", "false-positive"), + ("risk_accepted", None, "suppressed")): + for s in (vdr or {}).get(section) or []: + if (isinstance(s, dict) and check and s.get("tool_id") == check + and s.get("tracking_id") != rec.get("providerTrackingId")): + return final, reason, (f"its check {check} is now suppressed in .checkov.yaml " + f"({s.get('decision_id') or 'no decision'}, {s.get('current_disposition')})") + source = ext.get("source") or SOURCE_BY_LABEL.get((rec.get("detection") or {}).get("detectionSource")) + if source == "checkov-suppression": + again = [f["tracking_id"] for f in (vdr or {}).get("findings") or [] + if isinstance(f, dict) and check and f.get("tool_id") == check] + if again: + return None, "reported-again", (f"its check {check} is no longer suppressed and the scanner " + f"reports it again as {', '.join(sorted(again))}") + covered = _coverage(source, vdr, last_reported) + if covered: + return "Remediated", "remediated", covered + return None, "unknown", ("no evidence of why: no closed decision names it, no suppression covers it, " + "and its scanner is not shown to have run over it again since it was last reported") + + +_BACKFILL_DETECTION = ( + ("original_detection_date", "original_detection_date"), + ("decided_on", "decided_on: no detection date is recorded; known by the decision date"), + ("closed_on", "closed_on: neither a detection nor a decision date is recorded; detected no later " + "than it was closed"), +) +_BACKFILL_SOURCE = {"original_risk_rating": "originalRiskRating", "adjusted_risk_rating": "adjustedRiskRating", + "identifier": "sourceIdentifier"} + + +def _backfill(d, now): + """A closed (remediated) decision as a history record.""" + at, basis = now.isoformat(), f"{d['id']}: no date is recorded" + for field, why in _BACKFILL_DETECTION: + if _day(d.get(field)): + at, basis = _day(d[field]), f"{d['id']} {why}" + break + src = d.get("source") or {} + ext = {"decisionId": d["id"], "disposition": "remediated", "backfilledFromDecision": True, + "remediatedOn": d.get("closed_on"), "detectedAtBasis": basis, + "dispositionBasis": f"{d['id']} closed as remediated on {d.get('closed_on', 'a date not recorded')}", + "affectedResources": list((d.get("applies_to") or {}).get("resources") or [])} + if d.get("former_id"): + ext["formerId"] = d["former_id"] + if d.get("remediation_plan"): + ext["remediationPlan"] = d["remediation_plan"] + ext.update({out: src[k] for k, out in _BACKFILL_SOURCE.items() if src.get(k)}) + return {"providerTrackingId": d["id"], + "detection": {"detectedAt": at, "detectionSource": src.get("detector") or "not recorded"}, + "vulnerabilityDescription": d["title"], + "supplementaryRiskInformation": d["justification"].strip(), + "finalDisposition": "Remediated", + "providerExtensions": ext} + + +def build_history(active, accepted, previous, now, vdr=None, register=None): + """VER-TFR-MRH: every current record; every record that left the VDR within + HISTORY_DAYS, with a final disposition only where evidence gives one; and + every closed (remediated) decision closed within HISTORY_DAYS.""" + cutoff = now - timedelta(days=HISTORY_DAYS) current = {r["providerTrackingId"] for r in active} | \ {a["vulnerabilityDetail"]["providerTrackingId"] for a in accepted} carried = [] @@ -244,27 +614,96 @@ def build_history(active, accepted, previous, now): if not isinstance(rec, dict) or rec.get("providerTrackingId") in current: continue ext = dict(rec.get("providerExtensions") or {}) + if ext.get("backfilledFromDecision"): + continue # rebuilt from the register below gone = ext.get("noLongerReportedAt") or now.isoformat() - if gone < cutoff: + if (_maybe_dt(gone) or now) < cutoff: continue closed = dict(rec) closed.pop("projectedNextReduction", None) - closed.setdefault("finalDisposition", "Remediated") - closed["overdueStatus"] = {"isOverdue": False} ext["noLongerReportedAt"] = gone - closed["providerExtensions"] = ext - carried.append(closed) + ext.setdefault("lastReportedAt", prev.get("generatedAt")) + final = closed.get("finalDisposition") + if final and (ext.get("dispositionBasis") or final != "Remediated"): + # Settled: the disposition it had when last reported, or one already + # earned from evidence. + ext.setdefault("dispositionBasis", "its final disposition when it was last reported") + ext.setdefault("noLongerReportedReason", "closed") + else: + # Re-decided each run until evidence settles it (a closed decision + # may come later). A Remediated with no evidence is withdrawn. + closed.pop("finalDisposition", None) + final, reason, basis = _disposition_from_evidence(closed, vdr, register, ext["lastReportedAt"]) + ext["noLongerReportedReason"] = reason + ext.pop("noLongerReportedBasis", None) + if final: + closed["finalDisposition"] = final + ext["dispositionBasis"] = basis + else: + ext["noLongerReportedBasis"] = basis + if closed.get("finalDisposition"): + closed["overdueStatus"] = {"isOverdue": False, "explanation": + f"Not overdue: no longer reported; {closed['finalDisposition']}."} + carried.append({**closed, "providerExtensions": ext}) + linked = {(r.get("providerExtensions") or {}).get("decisionId") for r in carried + active} | \ + {(a["vulnerabilityDetail"].get("providerExtensions") or {}).get("decisionId") for a in accepted} + claimed = set().union(*(_keys(r) for r in carried)) + backfilled = [] + for d in (register or {}).get("decisions", []): + if d.get("status") != "closed" or d["id"] in linked or _decision_claims(d) & claimed: + continue + closed_on = _maybe_dt(d.get("closed_on")) + if closed_on and closed_on < cutoff: + continue + backfilled.append(_backfill(d, now)) return {"$schema": fs.stamp("mrh"), "certificationPackageOverviewUri": fs.CPO_URI, "generatedAt": now.isoformat(), - "activeVulnerabilities": active + sorted(carried, key=lambda r: r["providerTrackingId"]), + "activeVulnerabilities": active + sorted(carried + backfilled, key=lambda r: r["providerTrackingId"]), "acceptedVulnerabilities": accepted, "providerExtensions": { - "retention": f"Records that leave the VDR stay {HISTORY_DAYS} days, closed as Remediated " - "with the time they stopped being reported (noLongerReportedAt)."}} - - -def render_markdown(title, rule, records, accepted=False): + "retention": (f"Records that leave the VDR stay {HISTORY_DAYS} days with the time they stopped " + "being reported (noLongerReportedAt). A final disposition is set only from " + "evidence (dispositionBasis): a closed decision, a false-positive suppression, " + "or the scanner running again without reporting it. Otherwise " + "noLongerReportedReason says why it is not reported, or 'unknown', and no final " + "disposition is given. Closed (remediated) decisions are included " + "(backfilledFromDecision) for the same period from their closing date.")}} + + +# --- markdown ----------------------------------------------------------------------- + +def _cell(value): + return str(value).replace("|", "\\|").replace("\n", " ") + + +def render_activity(summary): + c = summary["counts"] + out = ["## Activity this period", "", + f"{summary['period']['from']} to {summary['period']['to']}", "", + f"- Detections: {c['detections']}; evaluations: {c['evaluations']}; rating changes: " + f"{c['ratingChanges']}; reductions: {c['reductions']}", + f"- No longer reported: {c['noLongerReported']}; remediations: {c['remediations']}; " + f"decisions recorded: {c['decisionsRecorded']}; decisions closed: {c['decisionsClosed']}", + "", "| Scan or feed | Time | Status | Count | In period |", "|---|---|---|---|---|"] + for s in summary["scans"]: + out.append(f"| {_cell(s['activity'])} | {s.get('at') or 'not recorded'} ({s['timeBasis']}) | " + f"{s.get('status')} | {'' if s.get('count') is None else s['count']} | " + f"{'yes' if s['inPeriod'] else 'no'} |") + rt = summary.get("runtimeVerification") + out.append("") + if rt: + out.append(f"Runtime verification: [{rt['uri']}]({rt['uri']}) emitted {rt.get('emittedAt')}, " + f"{rt.get('validations')} validations, {rt.get('failed')} failed, divergence " + f"{rt.get('divergence')}; {'in' if rt['inPeriod'] else 'not in'} this period. {rt['note']}") + else: + out.append("Runtime verification: not recorded by the VDR build.") + return out + + +def render_markdown(title, rule, records, accepted=False, summary=None): out = [f"# {title}", "", f"_FedRAMP {rule}. Source of truth: the signed JSON beside this file._", ""] + if summary: + out += render_activity(summary) + ["", "## Vulnerabilities", ""] if not records: return "\n".join(out + ["_None._"]) out += ["| Tracking ID | Detected | Internet-reachable | Likely exploitable | Rating | Overdue | " @@ -276,8 +715,7 @@ def render_markdown(title, rule, records, accepted=False): out.append("| `{t}` | {at} | {irv} | {lev} | N{n} | {od} | {last} |".format( t=d["providerTrackingId"], at=d["detection"]["detectedAt"][:10], irv="yes" if d["isInternetReachable"] else "no", lev="yes" if d["isLikelyExploitable"] else "no", - n=d["currentRating"], od="yes" if d["overdueStatus"]["isOverdue"] else "no", - last=last.replace("|", "\\|").replace("\n", " "))) + n=d["currentRating"], od="yes" if d["overdueStatus"]["isOverdue"] else "no", last=_cell(last))) return "\n".join(out) @@ -312,10 +750,11 @@ def main(): fs.write_validated(avi, "avi", out / AVI_OUT, registry) fs.write_validated(mrh, "mrh", out / MRH_OUT, registry) (out / VDT_OUT.replace(".json", ".md")).write_text( - render_markdown("Vulnerability Detail Report", "VER-RPT-VDT", vdt["vulnerabilities"]) + "\n") + render_markdown("Vulnerability Detail Report", "VER-RPT-VDT", vdt["vulnerabilities"], + summary=vdt["providerExtensions"]["activitySummary"]) + "\n") (out / AVI_OUT.replace(".json", ".md")).write_text( render_markdown("Accepted Vulnerability Info", "VER-RPT-AVI", avi["acceptedVulnerabilities"], - accepted=True) + "\n") + accepted=True, summary=avi["providerExtensions"]["activitySummary"]) + "\n") print(f"CR26 vulnerability reports: VDT {len(vdt['vulnerabilities'])}, " f"AVI {len(avi['acceptedVulnerabilities'])}, history {len(mrh['activeVulnerabilities'])} active + " f"{len(mrh['acceptedVulnerabilities'])} accepted") diff --git a/scripts/build-vdr-report.py b/scripts/build-vdr-report.py index f037032d..8cc2f592 100755 --- a/scripts/build-vdr-report.py +++ b/scripts/build-vdr-report.py @@ -289,6 +289,7 @@ def classify_vulnerability(finding, register, today=None, mitigation=None): # The runtime emitter runs daily; two days allows one missed run. Moves to the # governed freshness config with the other windows. RUNTIME_SIGNAL_MAX_AGE_HOURS = 48 +RUNTIME_SIGNAL_URI = "https://samaydlette.com/.well-known/ksi-signal-runtime.json" def component_fingerprint(component): @@ -378,6 +379,8 @@ def load_runtime_signal(path, pubkey_path=None, now=None, max_age_hours=RUNTIME_ return out out["verified"] = True out["emitted_at"] = sig.get("emitted_at") + out["signal_id"] = sig.get("signal_id") + out["divergence"] = (sig.get("divergence") or {}).get("status") emitted = _ts(sig.get("emitted_at"), None) if sig.get("emitted_at") else None out["fresh"] = bool(emitted and timedelta(0) <= now - emitted <= timedelta(hours=max_age_hours)) if not out["fresh"]: @@ -386,6 +389,45 @@ def load_runtime_signal(path, pubkey_path=None, now=None, max_age_hours=RUNTIME_ return out +def runtime_verification_summary(runtime): + """The runtime V&V run this report read (VER-RPT-PER): when it ran, whether + its signature verified, how many validations it ran and how many failed. + Counts are published only from a verified signal.""" + runtime = runtime or {} + out = {"uri": RUNTIME_SIGNAL_URI, "verified": bool(runtime.get("verified")), + "fresh": bool(runtime.get("fresh")), "reason": runtime.get("reason"), + "emitted_at": runtime.get("emitted_at"), "signal_id": runtime.get("signal_id"), + "validations": None, "failed": None, "divergence": None} + if out["verified"]: + vals = [v for v in runtime.get("validations") or [] if isinstance(v, dict)] + out["validations"] = len(vals) + out["failed"] = sum(1 for v in vals if v.get("result") != "pass") + out["divergence"] = runtime.get("divergence") + return out + + +def dast_summary(zap_path): + """The DAST scan this report ingested: when ZAP generated it and how many + alert types it reported. None when no ZAP report was given.""" + if not zap_path: + return None + p = Path(zap_path) + if not p.is_file(): + return {"scanner": "zap", "status": "missing", "generated_at": None, "alerts": None} + try: + doc = json.loads(p.read_text()) + except (json.JSONDecodeError, UnicodeDecodeError): + return {"scanner": "zap", "status": "unreadable", "generated_at": None, "alerts": None} + if not isinstance(doc, dict): + return {"scanner": "zap", "status": "unreadable", "generated_at": None, "alerts": None} + # ZAP writes nanoseconds; datetime parses at most microseconds. + raw = re.sub(r"(\.\d{6})\d+", r"\1", doc["created"]) if isinstance(doc.get("created"), str) else None + created = _ts(raw, None) if raw else None + gen = created.astimezone(timezone.utc).isoformat() if created and created.tzinfo else None + alerts = sum(len(site.get("alerts") or []) for site in doc.get("site") or [] if isinstance(site, dict)) + return {"scanner": "zap", "status": "ok", "generated_at": gen, "alerts": alerts} + + def mitigation_state(finding, mitigations, inventory, runtime, today): """The mitigation covering this finding, evaluated, or None. When several name the finding, the first valid one (by ID) is used, else the first; a @@ -739,6 +781,9 @@ def ingest_suppressions(checkov_yaml): "title": f"{check_id} suppressed", "description": rationale, "resource": ".checkov.yaml", + # What the suppression affects: the resources its decision names. + "affected_resources": list(_decisions.records(DECISIONS).get( + RISK_ACCEPTED_BY_CHECK_ID.get(check_id), {}).get("applies_to", {}).get("resources", [])), "current_disposition": "risk-accepted", "detected_at": detected.get(RISK_ACCEPTED_BY_CHECK_ID.get(check_id)), "pain": evaluation["pain"], @@ -1642,6 +1687,42 @@ def resolve_component(finding, index, tf_components=()): return None +_DECISION_RESOURCE = re.compile(r"^aws-([a-z0-9-]+?)(?:::([\w.-]+))?$") + + +def resolve_decision_resource(name, inventory): + """The component a decision record's resource name denotes, or None. An + exact inventory identifier first; else the register's `aws-[::]` + form, matched to the one Terraform component of that resource type (and + name, when given). Ambiguous or unknown: None, never a guess.""" + if not isinstance(name, str) or not name: + return None + if name in inventory["index"]: + return inventory["index"][name] + m = _DECISION_RESOURCE.match(name) + if not m: + return None + tf_type = "aws_" + m.group(1).replace("-", "_") + want = m.group(2).replace("-", "_") if m.group(2) else None + hits = {cid for cid, t, n in inventory["tf"] if t == tf_type and (want is None or n == want)} + return hits.pop() if len(hits) == 1 else None + + +def resolve_decision_resources(names, inventory): + """The component a decision's resources resolve to: the one they all name, + or, when they name several, an internet-reachable one first (the + conservative choice for IRV). None when none resolves.""" + cids = [] + for n in names or []: + cid = resolve_decision_resource(n, inventory) + if cid and cid not in cids: + cids.append(cid) + if not cids: + return None + reachable = [c for c in cids if (inventory["classes"].get(c) or {}).get("internet_reachable") is True] + return (reachable or cids)[0] + + def tf_components_of(ksi_signal_path): """(component_id, tf_type, tf_name) for every Terraform-backed component.""" p = Path(ksi_signal_path) @@ -1954,8 +2035,14 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No sup_prepared = [] for s in suppressions: s = apply_check_vector(s, check_vectors) - s_cid = resolve_component(s, inv["index"], inv["tf"]) - s = {**s, "_classification": classification_for(s, inventory, index)} + # A .checkov.yaml suppression names a file, not a resource: it affects + # the resources its decision record names (VER-RPT-AVI), so those are + # what reachability and classification are decided from. + s_cid = (resolve_component(s, inv["index"], inv["tf"]) + or resolve_decision_resources(s.get("affected_resources"), inv)) + s = {**s, "_classification": ((inv["classes"].get(s_cid) or _FAILSAFE_CLASSIFICATION) + if s_cid and inventory is not None + else classification_for(s, inventory, index))} s_pain = assign_pain(s, index, config) if s.get("cvss_vector") else s.get("pain") items.append(triage_item(s, "suppression", pain=s_pain, is_kev=s.get("is_kev", False), epss_scores=epss_scores, reach=reach_facts(s, s_cid, inv))) @@ -2111,6 +2198,7 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "tool_id": s.get("tool_id"), "title": s.get("title", ""), "resource": s.get("resource", ""), + **({"affected_resources": s["affected_resources"]} if s.get("affected_resources") else {}), "first_detected": s_clk["first_detected"], "first_evaluated": s_clk["first_evaluated"], "completed_evaluation": now.isoformat(), @@ -2328,6 +2416,7 @@ def main(): check_vectors = load_check_vectors(args.check_vectors) lev_overrides = load_lev_overrides(args.lev_overrides) + runtime = load_runtime_signal(args.runtime_signal, args.runtime_pubkey) report, blocking = build_report(findings, suppressions, kev_cves, ledger, index=classification_index, config=pain_config, epss_scores=epss_scores, @@ -2336,7 +2425,7 @@ def main(): lev_overrides=lev_overrides, inventory=load_inventory(args.ksi_signal), mitigations=load_mitigations(args.mitigations), - runtime=load_runtime_signal(args.runtime_signal, args.runtime_pubkey)) + runtime=runtime) report["ksi_signal_id"] = ksi_signal_id report["false_positives"] = false_positives # Name the inventory component each finding affects, so the trust center can @@ -2394,6 +2483,11 @@ def main(): # REQ-GOV-2: the profile this report conforms to, and the results of that # profile's conformance tests it was built under. report["methodology"]["profile"] = load_conformance(args.conformance) if args.conformance else None + # VER-RPT-PER: the runs this report saw that the detection summary does not + # list: the runtime verification (its signed signal, verified above) and the + # DAST scan. Facts only; the CR26 period reports summarize them. + report["methodology"]["runtime_verification"] = runtime_verification_summary(runtime) + report["methodology"]["dast"] = dast_summary(args.zap) # G7: what each detection input was when this report was built (T52). report["methodology"]["detection"] = (json.loads(Path(args.detection).read_text()) if args.detection else None) diff --git a/tests/fixtures/decisions/planned-reduction-closed.json b/tests/fixtures/decisions/planned-reduction-closed.json new file mode 100644 index 00000000..5cc23945 --- /dev/null +++ b/tests/fixtures/decisions/planned-reduction-closed.json @@ -0,0 +1,28 @@ +{ + "_note": "Known-bad: a closed record that still plans a reduction. A closed (remediated) decision has nothing left to reduce; the build stops.", + "schema": "decisions/1", + "decisions": [ + { + "id": "DEC-001", + "title": "fixture", + "status": "closed", + "disposition": "remediated", + "justification": "fixture", + "decided_by": "Fixture", + "decided_on": "2026-01-01", + "review_by": "2027-01-01", + "applies_to": { + "check_ids": [], + "vuln_ids": [ + "CVE-2099-0001" + ], + "resources": [] + }, + "closed_on": "2026-06-01", + "planned_reduction": { + "date": "2026-12-01", + "target_pain": "N1" + } + } + ] +} diff --git a/tests/fixtures/decisions/planned-reduction-malformed.json b/tests/fixtures/decisions/planned-reduction-malformed.json new file mode 100644 index 00000000..4df6833d --- /dev/null +++ b/tests/fixtures/decisions/planned-reduction-malformed.json @@ -0,0 +1,27 @@ +{ + "_note": "Known-bad: a planned reduction whose target is not an N-rating. A malformed plan would publish a projected reduction nobody planned; the build stops.", + "schema": "decisions/1", + "decisions": [ + { + "id": "DEC-001", + "title": "fixture", + "status": "in-force", + "disposition": "remediate", + "justification": "fixture", + "decided_by": "Fixture", + "decided_on": "2026-01-01", + "review_by": "2027-01-01", + "applies_to": { + "check_ids": [], + "vuln_ids": [ + "CVE-2099-0001" + ], + "resources": [] + }, + "planned_reduction": { + "date": "2026-12-01", + "target_pain": "low" + } + } + ] +} diff --git a/tests/test_cr26_vdr.py b/tests/test_cr26_vdr.py index 6351753c..70f8ccc4 100644 --- a/tests/test_cr26_vdr.py +++ b/tests/test_cr26_vdr.py @@ -10,9 +10,14 @@ jsonschema; - each record carries the joins the schemas have no field for (component, CVE, check, decision) under providerExtensions; -- report periods run from the previous report's end (VER-RPT-PER); -- the history keeps a finding that left the VDR, closed as Remediated, for a - year, then drops it. +- report periods run from the previous report's end (VER-RPT-PER), and each + period report summarizes the period's activity, linking the runtime results; +- records carry the historical rating and rating history, the decision's + planned reduction (else the deadline), supplementary information from the + decision, an overdue explanation about the overdue status, and a detection + time from a decision that names the record; +- the history keeps a finding that left the VDR for a year, with a final + disposition only from evidence, and carries closed decisions. """ import importlib.util import json @@ -137,7 +142,8 @@ def test_periods_run_from_the_previous_report(): assert vdt["reportPeriod"] == {"from": "2026-11-04T15:00:00+00:00", "to": NOW.isoformat()} assert avi["reportPeriod"]["from"] == "2026-11-04T15:00:00+00:00" first, _, _ = _build() - assert first["reportPeriod"]["from"] == "2026-03-01T00:00:00+00:00" # earliest detection + # The earliest detection: the suppressed CKV_AWS_68, known since DEC-104 was decided. + assert first["reportPeriod"]["from"] == "2026-01-01T00:00:00+00:00" def test_a_previous_report_ending_in_the_future_stops_the_build(): @@ -146,16 +152,24 @@ def test_a_previous_report_ending_in_the_future_stops_the_build(): _build(prev_vdt=prev) +def _detection_ok(*feeds, checked_at="2026-11-06T14:59:00+00:00"): + return {"schema": "detection/1", "checked_at": checked_at, "result": "ok", + "feeds": [{"feed": f, "file": f"{f}.json", "status": "ok", "count": 0} for f in feeds]} + + def test_history_keeps_a_finding_that_left_the_vdr_then_drops_it_after_a_year(): _, _, mrh = _build() vdr = json.loads(json.dumps(VDR)) vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "grype-CVE-2099-0001-thumbnailer"] + # Grype ran in the build that no longer reports it: remediated, from evidence. + vdr["methodology"] = {"detection": _detection_ok("grype", "checkov")} _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer") assert gone["finalDisposition"] == "Remediated" + assert gone["providerExtensions"]["dispositionBasis"].startswith("grype ran in the VDR build checked") assert gone["providerExtensions"]["noLongerReportedAt"] == (NOW + timedelta(days=1)).isoformat() - assert "projectedNextReduction" not in gone and gone["overdueStatus"] == {"isOverdue": False} + assert "projectedNextReduction" not in gone and gone["overdueStatus"]["isOverdue"] is False assert fs.validate(later, "mrh") == [] # The time it left is kept, not reset, on later runs; after a year it is dropped. _, _, still = _build(vdr=vdr, prev_mrh=later, now=NOW + timedelta(days=200)) @@ -235,3 +249,288 @@ def test_an_unreadable_previous_report_stops_the_build_rather_than_dropping_hist "--out-dir", str(tmp_path)], capture_output=True, text=True) assert r.returncode != 0 and "unreadable" in r.stderr assert not list(tmp_path.glob("vdr-*")) + + +# --- what each record says (VER-RPT-VDT, VER-RPT-AVI) ------------------------------- + +def _register(edit): + """The fixture register with `edit(records_by_id)` applied, validated like the real one.""" + doc = json.loads((FIX / "decisions.json").read_text()) + edit({r["id"]: r for r in doc["decisions"]}) + return _decisions.validate(doc) + + +def _by_id(doc, accepted=False): + rows = doc["acceptedVulnerabilities"] if accepted else doc["vulnerabilities"] + return {(r["vulnerabilityDetail"] if accepted else r)["providerTrackingId"]: + (r["vulnerabilityDetail"] if accepted else r) for r in rows} + + +def test_records_carry_their_historical_rating_and_rating_history(): + vdr = json.loads(json.dumps(VDR)) + f = vdr["findings"][0] + f["historical_pain"] = "N4" + f["rating_history"] = [ + {"at": "2026-08-06T14:00:00+00:00", "pain": "N4", "lev": True, "irv": True, "reason": "first evaluation"}, + {"at": "2026-10-14T10:00:00+00:00", "pain": "N3", "lev": True, "irv": True, "reason": "re-evaluation: pain N4->N3"}] + vdt, _, _ = _build(vdr=vdr) + ext = _by_id(vdt)[f["tracking_id"]]["providerExtensions"] + assert ext["historicalRating"] == 4 + assert [(h["at"], h["rating"], h["reason"]) for h in ext["ratingHistory"]] == [ + ("2026-08-06T14:00:00+00:00", 4, "first evaluation"), + ("2026-10-14T10:00:00+00:00", 3, "re-evaluation: pain N4->N3")] + assert fs.validate(vdt, "vdt") == [] + + +def test_a_decision_sets_the_next_reduction_and_the_deadline_is_only_the_fallback(): + vdr = json.loads(json.dumps(VDR)) + vdr["findings"][0]["decision_id"] = "DEC-105" # open N3 finding, deadline 2026-11-20 + + def plan(target): + def edit(recs): + recs["DEC-105"]["planned_reduction"] = {"date": "2026-11-12", "target_pain": target} + return _register(edit) + vdt, _, _ = mod.build(vdr, plan("N1"), None, None, None, NOW) + rec = _by_id(vdt)["grype-CVE-2099-0001-thumbnailer"] + assert rec["projectedNextReduction"] == {"estimatedAt": "2026-11-12T00:00:00+00:00", "targetRating": 1} + assert rec["providerExtensions"]["projectionBasis"] == "DEC-105 planned_reduction" + # A plan that reduces nothing is not a reduction: the deadline stands, and says why. + vdt, _, _ = mod.build(vdr, plan("N3"), None, None, None, NOW) + rec = _by_id(vdt)["grype-CVE-2099-0001-thumbnailer"] + assert rec["projectedNextReduction"] == {"estimatedAt": "2026-11-20T00:00:00+00:00", "targetRating": 0} + assert "not below the current rating" in rec["providerExtensions"]["projectionBasis"] + # No plan: the remediation deadline. + vdt, _, _ = _build() + assert _by_id(vdt)["grype-CVE-2099-0001-thumbnailer"]["providerExtensions"]["projectionBasis"] == \ + "remediation deadline (no planned reduction recorded)" + + +def test_supplementary_information_comes_from_the_decision(): + vdt, avi, _ = _build() + fp = _by_id(vdt)["zap-10099-article"]["supplementaryRiskInformation"] + assert fp.startswith("DEC-103 (false-positive; decided 2026-01-01, review by 2027-01-01): " + "The rule matches article prose, not code.") + # The AVI carries the justification as the rationale; the supplementary + # information is the rest of the decision, not a copy of it. + acc = _by_id(avi, accepted=True)["CKV_AWS_68"]["supplementaryRiskInformation"] + assert acc.startswith("DEC-104 (risk-accepted; decided 2026-01-01, review by 2027-01-01).") + assert "Cost trade-off" not in acc + assert all(a["vulnerabilityDetail"].get("supplementaryRiskInformation") for a in avi["acceptedVulnerabilities"] + if a["vulnerabilityDetail"]["providerExtensions"]["decisionId"]) + + +def test_overdue_explanations_explain_the_overdue_status_not_the_suppression(): + vdr = json.loads(json.dumps(VDR)) + vdr["findings"][1]["remediation_due_at"] = "2026-09-23T13:51:28+00:00" # overdue, DEC-101 accepts it + vdt, avi, _ = _build(vdr=vdr) + every = list(_by_id(vdt).values()) + list(_by_id(avi, accepted=True).values()) + assert not any("documented suppression" in r["overdueStatus"].get("explanation", "") for r in every) + over = _by_id(avi, accepted=True)["checkov-CKV_AWS_310-website"]["overdueStatus"] + assert over == {"isOverdue": True, "explanation": ( + "Overdue: its remediation deadline, 2026-09-23, passed without remediation. It is still open because " + "DEC-101 accepts it (risk-accepted, decided 2026-01-01), so no remediation is planned.")} + assert _by_id(vdt)["zap-10099-article"]["overdueStatus"] == { + "isOverdue": False, "explanation": "Not overdue: closed as False Positive under DEC-103."} + mav = _by_id(avi, accepted=True)["dependabot-GHSA-xxxx-yyyy-zzzz"]["overdueStatus"] + assert mav["isOverdue"] and mav["explanation"].startswith("Overdue: not fully mitigated or remediated within") + + +def test_detection_time_comes_from_a_decision_that_names_the_record(): + def edit(recs): + # The ZAP finding was found in a manual scan before the pipeline's scanner saw it. + recs["DEC-103"]["applies_to"] = {"check_ids": [], "vuln_ids": ["zap-10099-article"], "resources": []} + recs["DEC-103"]["original_detection_date"] = "2026-07-01" + # A decision about a check across resources dates the check, not each resource. + recs["DEC-101"]["original_detection_date"] = "2026-02-01" + vdt, avi, _ = mod.build(VDR, _register(edit), None, None, None, NOW) + zap = _by_id(vdt)["zap-10099-article"] + assert zap["detection"]["detectedAt"] == "2026-07-01T00:00:00+00:00" + assert zap["providerExtensions"]["detectedAtBasis"].startswith("DEC-103 original_detection_date") + assert zap["providerExtensions"]["firstReportedAt"] == "2026-08-06T13:51:28+00:00" + acc = _by_id(avi, accepted=True) + assert acc["checkov-CKV_AWS_310-website"]["detection"]["detectedAt"] == "2026-08-06T13:51:28+00:00" + # A suppressed check is never reported by a scanner: its decision dates it. + assert acc["CKV_AWS_68"]["detection"]["detectedAt"] == "2026-01-01T00:00:00+00:00" + assert acc["CKV_AWS_68"]["providerExtensions"]["detectedAtBasis"].startswith("DEC-104 decided_on") + + +def test_a_suppression_names_the_resources_its_decision_affects(): + vdr = json.loads(json.dumps(VDR)) + vdr["risk_accepted"][0].update(affected_resources=["aws-cloudfront-distribution"], + component_id="aws::cdn_distribution::website", irv_basis="inventory") + _, avi, _ = _build(vdr=vdr) + ext = _by_id(avi, accepted=True)["CKV_AWS_68"]["providerExtensions"] + assert ext["affectedResources"] == ["aws-cloudfront-distribution"] + assert ext["componentId"] == "aws::cdn_distribution::website" + + +# --- history: a final disposition only from evidence (VER-TFR-MRH) ------------------- + +def _gone(vdr_edit, prev=None, now=NOW + timedelta(days=1), register=None): + """Build, drop the open grype finding, apply `vdr_edit`, rebuild with the first history.""" + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "grype-CVE-2099-0001-thumbnailer"] + vdr_edit(vdr) + _, _, later = mod.build(vdr, register or REGISTER, None, None, prev or mrh, now) + assert fs.validate(later, "mrh") == [] + return vdr, later, next(r for r in later["activeVulnerabilities"] + if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer") + + +def test_a_record_that_vanished_without_evidence_has_no_final_disposition(): + vdr, later, gone = _gone(lambda v: None) # no detection summary: coverage not shown + assert "finalDisposition" not in gone + assert gone["providerExtensions"]["noLongerReportedReason"] == "unknown" + assert gone["providerExtensions"]["noLongerReportedBasis"].startswith("no evidence of why") + # A failed scan is no evidence either. + _, _, failed = _gone(lambda v: v.update(methodology={"detection": { + **_detection_ok("grype"), "feeds": [{"feed": "grype", "status": "failed"}]}})) + assert "finalDisposition" not in failed + # A closed decision that arrives later settles it. + def closed(recs): + recs["DEC-105"].update(status="closed", disposition="remediated", closed_on="2026-11-07", + applies_to={"check_ids": [], "vuln_ids": ["CVE-2099-0001"], "resources": []}) + _, _, settled = _gone(lambda v: None, prev=later, now=NOW + timedelta(days=3), register=_register(closed)) + assert settled["finalDisposition"] == "Remediated" + assert settled["providerExtensions"]["dispositionBasis"] == "DEC-105 closed as remediated on 2026-11-07" + assert settled["providerExtensions"]["noLongerReportedAt"] == gone["providerExtensions"]["noLongerReportedAt"] + + +def test_an_unsupported_remediated_from_an_earlier_history_is_withdrawn(): + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "grype-CVE-2099-0001-thumbnailer"] + old = json.loads(json.dumps(mrh)) + for r in old["activeVulnerabilities"]: + if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer": + r["finalDisposition"] = "Remediated" # what the earlier builder wrote, with no evidence + r["providerExtensions"]["noLongerReportedAt"] = NOW.isoformat() + _, _, later = _build(vdr=vdr, prev_mrh=old, now=NOW + timedelta(days=1)) + gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer") + assert "finalDisposition" not in gone and gone["providerExtensions"]["noLongerReportedReason"] == "unknown" + + +def test_a_check_that_became_a_suppression_is_not_remediated(): + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "checkov-CKV_AWS_20-site"] + vdr["methodology"] = {"detection": _detection_ok("checkov")} + vdr["risk_accepted"].append({**vdr["risk_accepted"][0], "tracking_id": "CKV_AWS_20", "tool_id": "CKV_AWS_20", + "decision_id": "DEC-102"}) + _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) + gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "checkov-CKV_AWS_20-site") + assert "finalDisposition" not in gone + assert gone["providerExtensions"]["noLongerReportedReason"] == "suppressed" + # A false-positive suppression is a final disposition, from the decision. + vdr["risk_accepted"].pop() + vdr["false_positives"] = [{"tracking_id": "CKV_AWS_20", "tool_id": "CKV_AWS_20", "decision_id": "DEC-103", + "current_disposition": "false-positive"}] + _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) + gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "checkov-CKV_AWS_20-site") + assert gone["finalDisposition"] == "False Positive" + + +def test_a_vanished_zap_finding_is_remediated_only_after_a_later_scan(): + def dast(when): + return {"methodology": {"dast": {"scanner": "zap", "status": "ok", "generated_at": when}}} + last = NOW.isoformat() + assert mod._coverage("zap", dast("2026-11-04T00:00:00+00:00"), last) is None # the scan that reported it + assert mod._coverage("zap", dast("2026-11-06T00:00:00+00:00"), last).startswith("the ZAP scan of 2026-11-06") + assert mod._coverage("zap", {"methodology": {}}, last) is None + # A record that was a false positive when last reported keeps that disposition. + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "zap-10099-article"] + _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) + fp = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "zap-10099-article") + assert fp["finalDisposition"] == "False Positive" + + +def test_closed_decisions_are_backfilled_into_the_history_once(): + def edit(recs): + recs["DEC-105"].update(status="closed", disposition="remediated", closed_on="2026-06-18", + original_detection_date="2026-05-06", former_id="POAM-005", + source={"detector": "Manual review", "original_risk_rating": "moderate"}) + reg = _register(edit) + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["decision_id"] != "DEC-105"] + _, _, mrh = mod.build(vdr, reg, None, None, None, NOW) + back = [r for r in mrh["activeVulnerabilities"] if r["providerTrackingId"] == "DEC-105"] + assert len(back) == 1 + assert back[0]["finalDisposition"] == "Remediated" + assert back[0]["detection"] == {"detectedAt": "2026-05-06T00:00:00+00:00", "detectionSource": "Manual review"} + assert back[0]["providerExtensions"]["backfilledFromDecision"] is True + assert back[0]["providerExtensions"]["formerId"] == "POAM-005" + assert fs.validate(mrh, "mrh") == [] + # Carried forward: rebuilt from the register, never duplicated; dropped a year after closing. + _, _, again = mod.build(vdr, reg, None, None, mrh, NOW + timedelta(days=1)) + assert [r["providerTrackingId"] for r in again["activeVulnerabilities"]].count("DEC-105") == 1 + _, _, aged = mod.build(vdr, reg, None, None, again, datetime(2027, 6, 20, tzinfo=timezone.utc)) + assert "DEC-105" not in _ids(aged["activeVulnerabilities"]) + + +# --- the period's activity (VER-RPT-PER) --------------------------------------------- + +def _with_activity(): + vdr = json.loads(json.dumps(VDR)) + vdr["emitted_at"] = NOW.isoformat() + vdr["findings"][0].update(first_detected="2026-11-05T09:00:00+00:00", first_evaluated="2026-11-05T09:00:00+00:00", + completed_reductions=[{"at": "2026-11-05T10:00:00+00:00", "from": "N4", "to": "N3"}]) + vdr["methodology"] = { + "detection": {**_detection_ok("grype", "checkov", checked_at="2026-11-05T14:59:00+00:00"), + "feeds": [{"feed": "grype", "file": "grype-js.json", "status": "ok", "count": 1, + "source_date": "2026-11-05T06:00:00+00:00"}, + {"feed": "kev", "file": "kev-catalog.json", "status": "ok", "count": 9, + "source_date": "2026-11-04T20:00:00+00:00"}, + {"feed": "previous-vdr", "file": "previous-vdr-report.json", "status": "ok"}]}, + "dast": {"scanner": "zap", "status": "ok", "generated_at": "2026-11-05T12:47:20+00:00", "alerts": 4}, + "runtime_verification": {"uri": "https://samaydlette.com/.well-known/ksi-signal-runtime.json", + "verified": True, "fresh": True, "emitted_at": "2026-11-05T11:00:00Z", + "signal_id": "rt-1", "validations": 5, "failed": 1, "divergence": "converged"}} + return vdr + + +def test_each_period_report_summarizes_the_periods_activity(): + prev = {"reportPeriod": {"from": "2026-11-04T15:00:00+00:00", "to": "2026-11-05T03:00:00+00:00"}} + vdt, avi, _ = _build(vdr=_with_activity(), prev_vdt=prev, prev_avi=prev) + for doc in (vdt, avi): + s = doc["providerExtensions"]["activitySummary"] + assert s["period"] == doc["reportPeriod"] + assert s["detections"] == [{"providerTrackingId": "grype-CVE-2099-0001-thumbnailer", + "at": "2026-11-05T09:00:00+00:00"}] + assert s["reductions"] == [{"providerTrackingId": "grype-CVE-2099-0001-thumbnailer", + "at": "2026-11-05T10:00:00+00:00", "from": "N4", "to": "N3"}] + scans = {x["activity"]: x for x in s["scans"]} + assert scans["Grype software composition scan (grype-js.json)"]["at"] == "2026-11-05T14:59:00+00:00" + assert scans["Grype software composition scan (grype-js.json)"]["vulnerabilityDatabaseBuiltAt"] == \ + "2026-11-05T06:00:00+00:00" + assert scans["CISA KEV catalog (kev-catalog.json)"]["inPeriod"] is False # published before the period + assert scans["OWASP ZAP dynamic scan (weekly)"]["inPeriod"] is True + assert not any("previous-vdr" in k for k in scans) + rt = s["runtimeVerification"] + assert rt["uri"] == "https://samaydlette.com/.well-known/ksi-signal-runtime.json" and rt["signalId"] == "rt-1" + assert s["counts"]["runtimeRunsInPeriod"] == 1 and s["counts"]["runtimeFailuresInPeriod"] == 1 + assert s["counts"]["detections"] == 1 and s["counts"]["reductions"] == 1 + assert fs.validate(vdt, "vdt") == [] and fs.validate(avi, "avi") == [] + + +def test_an_unverified_runtime_run_is_linked_but_not_counted(): + vdr = _with_activity() + vdr["methodology"]["runtime_verification"].update(verified=False, validations=None, failed=None, + reason="runtime signal signature does not verify") + vdt, _, _ = _build(vdr=vdr) + s = vdt["providerExtensions"]["activitySummary"] + assert s["counts"]["runtimeRunsInPeriod"] == 0 and "not verified" in s["runtimeVerification"]["note"] + + +def test_the_markdown_reports_carry_the_activity_and_link_the_runtime_results(tmp_path): + vdr = tmp_path / "vdr.json" + vdr.write_text(json.dumps(_with_activity())) + r = subprocess.run([sys.executable, str(SCRIPT), "--vdr", str(vdr), "--decisions", str(FIX / "decisions.json"), + "--out-dir", str(tmp_path)], capture_output=True, text=True) + assert r.returncode == 0, r.stderr + for name in ("vdr-detail-report.md", "vdr-accepted-vulnerabilities.md"): + text = (tmp_path / name).read_text() + assert "## Activity this period" in text and "| OWASP ZAP dynamic scan (weekly) |" in text + assert "(https://samaydlette.com/.well-known/ksi-signal-runtime.json)" in text diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 56ab7059..c3fd3ac3 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -35,6 +35,8 @@ ("duplicate-claim.json", "'CKV_AWS_68' is claimed by both DEC-001 and DEC-002"), ("closed-not-remediated.json", "a closed record's disposition must be one of"), ("unknown-field.json", "unknown field(s) ['reviewby']"), + ("planned-reduction-malformed.json", "planned_reduction must be {date: YYYY-MM-DD, target_pain: N0-N5}"), + ("planned-reduction-closed.json", "only an in-force record plans a reduction"), ] diff --git a/tests/test_vdr_component.py b/tests/test_vdr_component.py index 11124867..a34e31aa 100644 --- a/tests/test_vdr_component.py +++ b/tests/test_vdr_component.py @@ -49,3 +49,89 @@ def test_file_level_findings_stay_unattributed(): assert resolve(".checkov.yaml") is None assert resolve("infrastructure/logging.tf") is None assert resolve("aws_s3_bucket.nonexistent") is None + + +# --- a .checkov.yaml suppression affects the resources its decision names -------- + +INV_SIGNAL = {"components": [ + {"component_id": "aws::cdn_distribution::website", "type": "cdn_distribution", + "attributes": {"tf_type": "aws_cloudfront_distribution", "tf_name": "website", + "classification": {"internet_reachable": True}}}, + {"component_id": "aws::function::opa_compliance", "type": "function", + "attributes": {"tf_type": "aws_lambda_function", "tf_name": "opa_compliance", + "classification": {"internet_reachable": False}}}, + {"component_id": "aws::function::silk_reeling", "type": "function", + "attributes": {"tf_type": "aws_lambda_function", "tf_name": "silk_reeling", + "classification": {"internet_reachable": True}}}, +]} + + +def _inventory(tmp_path): + p = tmp_path / "ksi-signal.json" + p.write_text(json.dumps(INV_SIGNAL)) + return vdr.load_inventory(p) + + +def test_a_decision_resource_resolves_only_when_it_names_one_component(tmp_path): + inv = _inventory(tmp_path) + # The register's aws-[::] form, matched to the one component of that type. + assert vdr.resolve_decision_resource("aws-cloudfront-distribution", inv) == "aws::cdn_distribution::website" + assert vdr.resolve_decision_resource("aws-lambda-function::silk-reeling", inv) == "aws::function::silk_reeling" + # An exact inventory identifier. + assert vdr.resolve_decision_resource("aws::function::opa_compliance", inv) == "aws::function::opa_compliance" + # Two Lambdas and no name, or a name no component carries: never a guess. + assert vdr.resolve_decision_resource("aws-lambda-function", inv) is None + assert vdr.resolve_decision_resource("aws-lambda::compliance-monitor", inv) is None + assert vdr.resolve_decision_resource(".checkov.yaml", inv) is None + # Several components: the internet-reachable one, the conservative choice for IRV. + assert vdr.resolve_decision_resources( + ["aws::function::opa_compliance", "aws-lambda-function::silk-reeling"], inv) == "aws::function::silk_reeling" + + +def test_a_suppression_carries_its_decisions_resources(): + supp = {s["tool_id"]: s for s in vdr.ingest_suppressions(str(REPO / ".checkov.yaml"))} + reg = {r["id"]: r for r in json.loads((REPO / "data" / "decisions.json").read_text())["decisions"]} + for check, s in supp.items(): + assert s["resource"] == ".checkov.yaml" + assert s["affected_resources"] == reg[s["decision_id"]]["applies_to"]["resources"], check + + +def test_a_suppressions_reachability_is_decided_from_the_resource_it_affects(tmp_path): + inv = _inventory(tmp_path) + sup = {"tracking_id": "CKV_AWS_68", "decision_id": "DEC-007", "source": "checkov-suppression", + "tool_id": "CKV_AWS_68", "title": "CKV_AWS_68 suppressed", "resource": ".checkov.yaml", + "affected_resources": ["aws-cloudfront-distribution"], "current_disposition": "risk-accepted", + "pain": "N2", "is_kev": False, "explanation": "x"} + unresolved = {**sup, "tracking_id": "CKV_AWS_117", "tool_id": "CKV_AWS_117", + "affected_resources": ["aws-lambda::compliance-monitor"]} + report, _ = vdr.build_report([], [sup, unresolved], set(), {}, inventory=inv) + by = {r["tracking_id"]: r for r in report["risk_accepted"]} + assert by["CKV_AWS_68"]["component_id"] == "aws::cdn_distribution::website" + assert by["CKV_AWS_68"]["irv_basis"] == "inventory" and by["CKV_AWS_68"]["internet_reachable"] is True + assert by["CKV_AWS_68"]["affected_resources"] == ["aws-cloudfront-distribution"] + # A resource that resolves to nothing stays unresolved, and fails safe. + assert by["CKV_AWS_117"]["component_id"] is None and by["CKV_AWS_117"]["irv_basis"] == "unresolved" + + +# --- the runs a VDR build read, for the period reports (VER-RPT-PER) ----------------- + +def test_runtime_results_are_counted_only_from_a_verified_signal(): + ok = {"verified": True, "fresh": True, "emitted_at": "2026-11-05T17:56:05Z", "signal_id": "s-1", + "divergence": "converged", "validations": [{"result": "pass"}, {"result": "fail"}, {"result": "pass"}]} + out = vdr.runtime_verification_summary(ok) + assert out["uri"].endswith("/.well-known/ksi-signal-runtime.json") + assert (out["validations"], out["failed"], out["divergence"], out["signal_id"]) == (3, 1, "converged", "s-1") + bad = vdr.runtime_verification_summary({"verified": False, "reason": "runtime signal signature does not verify", + "validations": [{"result": "pass"}]}) + assert bad["verified"] is False and bad["validations"] is None and bad["failed"] is None + assert "does not verify" in bad["reason"] + + +def test_the_dast_scan_time_is_the_reports_own(tmp_path): + zap = tmp_path / "zap.json" + zap.write_text(json.dumps({"@generated": "Thu, 5 Nov 2026 12:47:20", "created": "2026-11-05T12:47:20.585268837Z", + "site": [{"alerts": [{}, {}]}, {"alerts": [{}]}]})) + assert vdr.dast_summary(str(zap)) == {"scanner": "zap", "status": "ok", + "generated_at": "2026-11-05T12:47:20.585268+00:00", "alerts": 3} + assert vdr.dast_summary(str(tmp_path / "absent.json"))["status"] == "missing" + assert vdr.dast_summary(None) is None From bc86e2df7ca9c14e7001490581c9936eaa406542 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 15:51:41 -0500 Subject: [PATCH 05/97] docs(frr): records state the state after a successful deploy Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- data/frr-records.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f0a24b76..d092298d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ Hold two facts at once — they shape every decision: Confirm against the working tree on first run; correct this section if reality differs. - `infrastructure/` — Terraform (`main.tf`), OPA policy packages (`policy/`), JSON schemas (`schemas/ksi-signal.schema.json`, `schemas/ksi-catalog.json`), runtime Lambda (`lambda/index.js`). -- `scripts/` — the artifact **generators**: `build-ksi-signal.py` (the canonical inventory), the VDR builder, and the CR26 record builders (`build-sdr.py`, `build-cpo.py`, and `build-cr26-vdr.py`, which turns the VDR report into FedRAMP's VDT, AVI and history reports). CR26 reports are validated against FedRAMP's 2026-06-24 JSON schemas, vendored verbatim with their sha256s in `infrastructure/schemas/fedramp-2026-06-24/`, by `scripts/_fedramp_schema.py` before they are written; a report that does not validate is never written. Never edit a vendored schema; the loader refuses any file whose hash differs from `SOURCES.json`. The per-control view (status, origination, statement per NIST 800-53 control) is the control projection in `scripts/_controls.py`, computed in memory from the inventory and the KSI catalog; the OSCAL SSP that used to carry it was retired under CR26 (2026-11-05) and the SDR's `rev5_controls` publishes it. Every human decision about a finding is a record in `data/decisions.json` (the decision register, which replaced the POA&M under CR26), read through `scripts/_decisions.py` by the VDR builder, the vulnerability gate, reconciliation and the trust center; every in-force record carries `decided_by`, `decided_on` and an annual `review_by`, which the loader and `tests/test_decision_records.py` enforce. Whether each FedRAMP rule that applies (20x Program Class C, derived from the pinned corpus's applicability metadata by `_rules.applicable_provider_rules`) is implemented is a record in `data/frr-records.json`, read through `scripts/_frr.py`: one record per applicable rule, a missing or extra record fails, and a record may say "Implemented" only with evidence that resolves (a test, a workflow step, a freshness-governed published artifact, or an in-force decision). A change that implements a rule flips its record in the same PR, with the evidence. These are the source of truth for everything published. `scripts/_common.py` holds the helpers they share (`classify` — the inheritance rule four artifacts must agree on — plus `sha256_file`, `prop`, `load_hub`, `sid`); it is imported via an `__file__`-relative `sys.path.insert` so every script stays standalone-runnable from any cwd. **`scripts/staged/` is not wired into anything** — parked generators, see its README; never assume a script there runs. +- `scripts/` — the artifact **generators**: `build-ksi-signal.py` (the canonical inventory), the VDR builder, and the CR26 record builders (`build-sdr.py`, `build-cpo.py`, and `build-cr26-vdr.py`, which turns the VDR report into FedRAMP's VDT, AVI and history reports). CR26 reports are validated against FedRAMP's 2026-06-24 JSON schemas, vendored verbatim with their sha256s in `infrastructure/schemas/fedramp-2026-06-24/`, by `scripts/_fedramp_schema.py` before they are written; a report that does not validate is never written. Never edit a vendored schema; the loader refuses any file whose hash differs from `SOURCES.json`. The per-control view (status, origination, statement per NIST 800-53 control) is the control projection in `scripts/_controls.py`, computed in memory from the inventory and the KSI catalog; the OSCAL SSP that used to carry it was retired under CR26 (2026-11-05) and the SDR's `rev5_controls` publishes it. Every human decision about a finding is a record in `data/decisions.json` (the decision register, which replaced the POA&M under CR26), read through `scripts/_decisions.py` by the VDR builder, the vulnerability gate, reconciliation and the trust center; every in-force record carries `decided_by`, `decided_on` and an annual `review_by`, which the loader and `tests/test_decision_records.py` enforce. Whether each FedRAMP rule that applies (20x Program Class C, derived from the pinned corpus's applicability metadata by `_rules.applicable_provider_rules`) is implemented is a record in `data/frr-records.json`, read through `scripts/_frr.py`: one record per applicable rule, a missing or extra record fails, and a record may say "Implemented" only with evidence that resolves (a test, a workflow step, a freshness-governed published artifact, or an in-force decision). A change that implements a rule flips its record in the same PR, with the evidence. A record describes the state after a successful deploy of its commit, never "not yet deployed"; anything the deploy depends on (a bootstrap apply, a repo variable) must make the deploy fail when missing. These are the source of truth for everything published. `scripts/_common.py` holds the helpers they share (`classify` — the inheritance rule four artifacts must agree on — plus `sha256_file`, `prop`, `load_hub`, `sid`); it is imported via an `__file__`-relative `sys.path.insert` so every script stays standalone-runnable from any cwd. **`scripts/staged/` is not wired into anything** — parked generators, see its README; never assume a script there runs. - `website/` — static site content. `website/.well-known/` is where the **published** artifacts are served from (KSI signal + bundle, SDR, CPO, VDR + trend, IIW CSV, runtime signal, signing pubkey, schema), but in git it tracks only `security.txt` and `vdr-trend.json`: everything else there is gitignored build output that the deploy signs and uploads (the website sync excludes `.well-known/*`). - `docs/` — `policies/`, `runbooks/`, `scn/`, and `assessment/` (e.g. `ground-truth.md`). - `website/research/` — plain static pages: `authorization-boundary.html` (scope; **read it first for the "why"**) and the long-form essay `staying-in-the-loop.html` (threshold dynamics, titled "Staying in the Loop"). The essay's filename is **not** referenced by `build-ksi-signal.py`, which discovers pages with `rglob("*.html")`; the hardcoded path lives in `tools/essay/paths.py`, so renaming means updating that. `/research/tuning-the-eigenvalue.html` and `/research/the-plumbing.html` are redirect stubs (the latter to the dashboard; the "Automated Security Compliance" paper is retired). diff --git a/data/frr-records.json b/data/frr-records.json index c7d069f9..d84c52f9 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -1,7 +1,7 @@ { "schema": "frr-records/1", "$comment": "One record per FedRAMP rule applicable to 20x Program Class C. Validated by scripts/_frr.py; see infrastructure/schemas/frr-records.schema.json.", - "basis": "Drafted 2026-11-05 from a rule-by-rule assessment of the repository and the live site, corrected for the changes since (account IDs redacted, published claims fixed, CR26 vulnerability reports published). AI-drafted; for the operator's review.", + "basis": "Drafted 2026-11-05 from a rule-by-rule assessment of the repository and the live site, corrected for the changes since (account IDs redacted, published claims fixed, CR26 vulnerability reports published). AI-drafted; for the operator's review. Each record states the state after a successful deploy of the commit that contains it: what that deploy does counts as done, and a prerequisite it depends on must make the deploy fail when missing, so the record is accurate when published.", "records": [ { "frr_id": "AFC-CSO-ACK", From efbf8d81431441594e9a16156f72b48eb3ba20ba Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:30:58 -0500 Subject: [PATCH 06/97] =?UTF-8?q?feat(frr):=20deploy=20gate=20=E2=80=94=20?= =?UTF-8?q?every=20rule=20Implemented=20or=20Not=20Applicable;=20simulated?= =?UTF-8?q?=20counterparties=20flagged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/_frr.py --complete fails the unit-test job (which the deploy job needs) while any record is Partially or Not Implemented. A record relying on a simulated outside party carries simulated {counterparty, how}, allowed only on Implemented records with deploy-time evidence. Known-bad cases and tests for each invariant. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 6 ++ Makefile | 6 ++ .../schemas/frr-records.schema.json | 14 +++- scripts/_frr.py | 48 ++++++++++++- tests/fixtures/frr/known-bad.json | 5 +- tests/test_frr_records.py | 68 ++++++++++++++++++- 6 files changed, 142 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 933f590b..a327c2de 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -188,6 +188,12 @@ jobs: chmod 755 ./opa sudo mv opa /usr/local/bin + # The completeness gate (operator's decision of 2026-11-05): every + # applicable FedRAMP rule must be Implemented or Not Applicable, or nothing + # deploys. The deploy job needs this job, so this runs before any apply. + - name: FedRAMP rules complete (no gaps) + run: python3 scripts/_frr.py --complete + - name: Python unit tests run: python3 -m pytest tests/ -q diff --git a/Makefile b/Makefile index a5355c30..7fc38437 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,12 @@ check-full: check require-artifacts @$(MAKE) --no-print-directory reconcile @echo "✅ check-full: PASS" +# The completeness gate the deploy runs before anything is applied: every +# applicable FedRAMP rule Implemented or Not Applicable (scripts/_frr.py). +# Not part of `check`: work toward it lands on branches that are not yet complete. +check-complete: + python3 scripts/_frr.py --complete + require-artifacts: @missing=""; \ for f in ksi-signal.json vdr-report.json security-decision-record.json certification-package-overview.json; do \ diff --git a/infrastructure/schemas/frr-records.schema.json b/infrastructure/schemas/frr-records.schema.json index b8f881d6..a2773ac7 100644 --- a/infrastructure/schemas/frr-records.schema.json +++ b/infrastructure/schemas/frr-records.schema.json @@ -35,6 +35,16 @@ "gap": {"type": "string", "minLength": 1}, "not_applicable": {"type": "string", "minLength": 1, "description": "Why the rule does not apply: FedRAMP scopes it to a step the provider is not taking, or it is a MAY permission not exercised."}, "may_rationale": {"type": "string", "minLength": 1, "description": "Required on, and only on, MAY rules: why the provider did or didn't exercise the permission."}, + "simulated": { + "type": "object", + "additionalProperties": false, + "required": ["counterparty", "how"], + "description": "The outside party this proof of concept simulates; Implemented records only, with deploy-time evidence.", + "properties": { + "counterparty": {"enum": ["assessor", "fedramp", "agency", "incident-recipient", "mail", "sam-gov"]}, + "how": {"type": "string", "minLength": 1} + } + }, "closes_with": { "type": "array", "minItems": 1, @@ -50,8 +60,8 @@ }, "else": { "if": {"properties": {"status": {"const": "Not Applicable"}}}, - "then": {"required": ["not_applicable"], "not": {"anyOf": [{"required": ["gap"]}, {"required": ["closes_with"]}]}}, - "else": {"required": ["gap", "closes_with"], "not": {"required": ["not_applicable"]}} + "then": {"required": ["not_applicable"], "not": {"anyOf": [{"required": ["gap"]}, {"required": ["closes_with"]}, {"required": ["simulated"]}]}}, + "else": {"required": ["gap", "closes_with"], "not": {"anyOf": [{"required": ["not_applicable"]}, {"required": ["simulated"]}]}} } } } diff --git a/scripts/_frr.py b/scripts/_frr.py index da6a4717..f183654d 100644 --- a/scripts/_frr.py +++ b/scripts/_frr.py @@ -28,6 +28,15 @@ other rule carries one. - No record text may carry an AWS account ID, or an email address outside the system's domain and FedRAMP's (the SDR is published). +- A record that relies on a simulated outside party (an assessor, FedRAMP, an + agency, an incident recipient, the mail system, SAM.gov) says so in + `simulated` {counterparty, how}. Only an Implemented record may, and it must + cite deploy-time evidence (a workflow step or a published artifact): the + mechanism is real and exercised on every deploy; only the party is not. + +The deploy gate (`--complete`, operator's decision of 2026-11-05): the deploy +proceeds only when every record is Implemented or Not Applicable. Each record +describes the state after a successful deploy of the commit that contains it. Fails closed, stdlib only (the deploy job installs no jsonschema); the JSON schema beside it documents the same shape and is checked in tests. @@ -63,10 +72,17 @@ WORKSTREAMS = tuple(f"W{n}" for n in range(2, 11)) CLOSERS = (*WORKSTREAMS, "sam") EVIDENCE_KINDS = ("test", "step", "artifact", "decision") +# Outside parties this proof of concept stands in for (operator's decision of +# 2026-11-05): the mechanism is real and runs on every deploy; the party is not. +SIMULATED_COUNTERPARTIES = ("assessor", "fedramp", "agency", "incident-recipient", "mail", "sam-gov") +SIMULATED_KEYS = {"counterparty", "how"} +# What the completeness gate accepts: the deploy proceeds only when every +# applicable rule is one of these. +COMPLETE_STATUSES = ("Implemented", "Not Applicable") TOP_KEYS = {"$comment", "schema", "basis", "records"} RECORD_KEYS = {"frr_id", "status", "implementation", "evidence", "gap", "closes_with", "may_rationale", - "not_applicable"} + "not_applicable", "simulated"} EVIDENCE_KEYS = {"kind", "ref"} TEST_RE = re.compile(r"^(tests/[A-Za-z0-9_/]+\.py)::([A-Za-z_][A-Za-z0-9_]*)(?:::([A-Za-z_][A-Za-z0-9_]*))?(?:\[[^\]]*\])?$") @@ -214,6 +230,17 @@ def validate(doc, applicable, resolve, where="data/frr-records.json"): why = resolve(ev) if why: _fail(f"{rid}: {ev['kind']} evidence {ev['ref']!r} does not resolve: {why}") + if "simulated" in rec: + sim = rec["simulated"] + if not isinstance(sim, dict) or set(sim) != SIMULATED_KEYS \ + or sim.get("counterparty") not in SIMULATED_COUNTERPARTIES: + _fail(f"{rid}: simulated must be {{counterparty: one of {list(SIMULATED_COUNTERPARTIES)}, how}}") + _text(sim, "how", f"{rid} simulated") + if status != "Implemented": + _fail(f"{rid}: simulated is for Implemented records only") + if not any(ev["kind"] in ("step", "artifact") for ev in evidence): + _fail(f"{rid}: a simulated counterparty needs deploy-time evidence (a workflow step or a " + f"published artifact): the simulation runs on every deploy") closes = rec.get("closes_with") if status == "Implemented": if not evidence: @@ -254,6 +281,12 @@ def load(path=None, rules_dir=None, repo=REPO, freshness_path=None, decisions_pa return validate(doc, applicable, resolver(repo, freshness_path, decisions_path), str(p)) +def incomplete(doc): + """The records that keep the deploy from proceeding: every rule must be + Implemented or Not Applicable (operator's decision of 2026-11-05).""" + return [(r["frr_id"], r["status"]) for r in doc["records"] if r["status"] not in COMPLETE_STATUSES] + + def summary(doc, applicable): """Counts the trust center and the SDR can show: by status, and by status within each force (MUST, SHOULD, ...).""" @@ -262,8 +295,10 @@ def summary(doc, applicable): for r in doc["records"]: f = applicable[r["frr_id"]]["force"] by_force.setdefault(f, Counter())[r["status"]] += 1 + simulated = Counter(r["simulated"]["counterparty"] for r in doc["records"] if "simulated" in r) return {"rules": len(doc["records"]), "by_status": {s: by_status.get(s, 0) for s in STATUSES}, + "simulated": {c: simulated[c] for c in sorted(simulated)}, "by_force": {f: {s: c.get(s, 0) for s in STATUSES} for f, c in sorted(by_force.items())}} @@ -271,9 +306,20 @@ def main(argv=None): import argparse ap = argparse.ArgumentParser(description="Validate data/frr-records.json against the pinned FedRAMP rules.") ap.add_argument("--records", default=str(DEFAULT_PATH)) + ap.add_argument("--complete", action="store_true", + help="also fail unless every record is Implemented or Not Applicable (the deploy gate)") args = ap.parse_args(argv) doc = load(args.records) print(json.dumps(summary(doc, _rules.applicable_provider_rules()), indent=2)) + if args.complete: + open_ = incomplete(doc) + if open_: + for rid, status in open_: + print(f"::error::{rid} is {status}: every applicable rule must be Implemented or Not Applicable") + print(f"{len(open_)} of {len(doc['records'])} rule(s) not complete; the deploy does not proceed.", + file=sys.stderr) + return 1 + print("Every applicable rule is Implemented or Not Applicable.") return 0 diff --git a/tests/fixtures/frr/known-bad.json b/tests/fixtures/frr/known-bad.json index 17c087d1..479aaa1f 100644 --- a/tests/fixtures/frr/known-bad.json +++ b/tests/fixtures/frr/known-bad.json @@ -25,6 +25,9 @@ {"case": "account ID in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "Bucket arn:aws:s3:us-east-1:111122223333:x", "gap": "x", "closes_with": ["W9"]}, "expect": "implementation carries an AWS account ID"}, {"case": "MAY rule without a rationale", "replace": {"frr_id": "VDR-CSO-SIR", "status": "Not Applicable", "implementation": "x", "not_applicable": "x"}, "expect": "VDR-CSO-SIR: may_rationale must be a non-empty string"}, {"case": "rationale on a MUST rule", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "may_rationale": "x"}, "expect": "may_rationale is for MAY rules only"}, - {"case": "personal email in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "Mail goes to someone@example.com today.", "closes_with": ["W9"]}, "expect": "gap names an email address outside"} + {"case": "personal email in published text", "replace": {"frr_id": "AFC-CSO-INB", "status": "Not Implemented", "implementation": "x", "gap": "Mail goes to someone@example.com today.", "closes_with": ["W9"]}, "expect": "gap names an email address outside"}, + {"case": "simulated on a partial record", "replace": {"frr_id": "AFC-CSO-INB", "status": "Partially Implemented", "implementation": "x", "gap": "x", "closes_with": ["W9"], "simulated": {"counterparty": "mail", "how": "x"}}, "expect": "simulated is for Implemented records only"}, + {"case": "simulated with an unknown counterparty", "replace": {"frr_id": "VDR-CSO-MSP", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "step", "ref": "deploy-with-opa.yml#Run OPA Compliance Check"}], "simulated": {"counterparty": "the-moon", "how": "x"}}, "expect": "simulated must be {counterparty: one of"}, + {"case": "simulated without deploy-time evidence", "replace": {"frr_id": "VDR-CSO-MSP", "status": "Implemented", "implementation": "x", "evidence": [{"kind": "decision", "ref": "DEC-039"}], "simulated": {"counterparty": "assessor", "how": "x"}}, "expect": "a simulated counterparty needs deploy-time evidence"} ] } diff --git a/tests/test_frr_records.py b/tests/test_frr_records.py index d35cb568..e9977e6c 100644 --- a/tests/test_frr_records.py +++ b/tests/test_frr_records.py @@ -144,7 +144,8 @@ def test_known_bad_fixture_covers_every_check(): "gap must be", "closes_with must be", "cannot be Not Applicable", "is not one of", "not_applicable must be", "a Not Applicable record has no gap", "for Not Applicable records only", "unknown field", "AWS account ID", "email address outside", - "may_rationale must be", "for MAY rules only"): + "may_rationale must be", "for MAY rules only", "simulated is for Implemented records only", + "simulated must be {counterparty", "needs deploy-time evidence"): assert fragment in expects, fragment @@ -213,3 +214,68 @@ def test_conditional_subsets_say_so_in_the_corpus(): rules, _ = _rules.load_rules() for (fam, subset), phrase in _frr.CONDITIONAL_SUBSETS.items(): assert phrase in rules["FRR"][fam]["info"]["subsets"][subset]["description"] + + +# --- the completeness gate --------------------------------------------------------- + +def _doc_with(rid, **fields): + doc = copy.deepcopy(_committed()) + for r in doc["records"]: + if r["frr_id"] == rid: + r.clear() + r.update({"frr_id": rid, **fields}) + return doc + + +def _complete_doc(): + """The committed records with every open one made Implemented on a test that exists.""" + doc = copy.deepcopy(_committed()) + for r in doc["records"]: + if r["status"] not in _frr.COMPLETE_STATUSES: + for k in ("gap", "closes_with"): + r.pop(k, None) + r["status"] = "Implemented" + r["evidence"] = [{"kind": "test", "ref": "tests/test_frr_records.py::test_summary_counts_add_up"}] + return doc + + +@pytest.mark.parametrize("status", ["Partially Implemented", "Not Implemented"]) +def test_any_open_record_keeps_the_deploy_from_proceeding(status, tmp_path, applicable, resolve): + doc = _complete_doc() + rec = next(r for r in doc["records"] if r["frr_id"] == "AFC-CSO-INB") + rec.update({"status": status, "gap": "x", "closes_with": ["W9"]}) + rec.pop("simulated", None) + _frr.validate(doc, applicable, resolve) # honest, so it passes the honesty rule... + assert _frr.incomplete(doc) == [("AFC-CSO-INB", status)] + p = tmp_path / "frr.json" + p.write_text(json.dumps(doc)) + assert _frr.main(["--records", str(p), "--complete"]) == 1 # ...and still stops the deploy + + +def test_a_complete_record_set_lets_the_deploy_proceed(tmp_path, applicable, resolve): + doc = _complete_doc() + _frr.validate(doc, applicable, resolve) + assert _frr.incomplete(doc) == [] + p = tmp_path / "frr.json" + p.write_text(json.dumps(doc)) + assert _frr.main(["--records", str(p), "--complete"]) == 0 + + +def test_the_summary_counts_simulated_counterparties(applicable, resolve): + doc = _doc_with("VDR-CSO-MSP", status="Implemented", implementation="x", + evidence=[{"kind": "step", "ref": "deploy-with-opa.yml#Run OPA Compliance Check"}], + simulated={"counterparty": "assessor", "how": "A plainly fictional assessor."}) + _frr.validate(doc, applicable, resolve) + assert _frr.summary(doc, applicable)["simulated"] == {"assessor": 1} + + +def test_the_gate_runs_before_anything_is_applied(): + """The completeness gate is in the job the deploy job needs, so a gap + stops the run before terraform apply or any publish.""" + import yaml + wf = yaml.safe_load((REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text()) + steps = [s.get("run", "") for s in wf["jobs"]["test"]["steps"]] + assert any("scripts/_frr.py --complete" in r for r in steps) + assert "test" in wf["jobs"]["deploy"]["needs"] + assert not any("terraform apply" in s.get("run", "") or "aws s3" in s.get("run", "") + for s in wf["jobs"]["test"]["steps"]) From 0f5e4fb5ce5db991113a78fa0aa564725e467c6e Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:17:10 -0500 Subject: [PATCH 07/97] KSI catalog: FedRAMP's current KSI-to-control mappings, from the pinned rules The vendored ksi-catalog.json predated FedRAMP's 2026-10-13 mapping revision: 40 of 46 KSIs mapped fewer (or different) controls than the pinned Consolidated Rules. scripts/sync-ksi-catalog.py now derives the KSI block from the pinned rules (Class C statements where a KSI varies by class, titles from the NIST Rev5 catalog), and a test fails while they differ. Within the Moderate baseline no implementation status changes; SC-45 and SC-45.1 become shared. The projection reaches 59 controls beyond Moderate (was 10). s3.rego drops si-12, which no KSI maps any more. The SSP parity test runs on the catalog as it stood at the SSP's retirement. Co-Authored-By: Claude Opus 5.5 --- .venv | 1 + infrastructure/policy/s3.rego | 2 +- infrastructure/schemas/ksi-catalog.json | 1065 ++++++-- scripts/sync-ksi-catalog.py | 95 + .../migration/ksi-catalog-2026-11-05.json | 2206 +++++++++++++++++ tests/test_controls.py | 7 +- tests/test_inject_figures.py | 7 +- tests/test_ksi_catalog_corpus.py | 57 + 8 files changed, 3273 insertions(+), 167 deletions(-) create mode 120000 .venv create mode 100644 scripts/sync-ksi-catalog.py create mode 100644 tests/fixtures/migration/ksi-catalog-2026-11-05.json create mode 100644 tests/test_ksi_catalog_corpus.py diff --git a/.venv b/.venv new file mode 120000 index 00000000..7e6cd3e1 --- /dev/null +++ b/.venv @@ -0,0 +1 @@ +../wt-w3-frr/.venv \ No newline at end of file diff --git a/infrastructure/policy/s3.rego b/infrastructure/policy/s3.rego index 7808baef..0243ac65 100644 --- a/infrastructure/policy/s3.rego +++ b/infrastructure/policy/s3.rego @@ -25,7 +25,7 @@ import data.policy.gate # id: versioning_disabled # category: infrastructure # severity: MEDIUM -# nist_controls: [cp-9, cp-10, si-12] +# nist_controls: [cp-9, cp-10] # ksi_ids: [KSI-RPL-ABO] violations contains violation if { some r in gate.resources diff --git a/infrastructure/schemas/ksi-catalog.json b/infrastructure/schemas/ksi-catalog.json index 7353e477..4b8dc8a7 100644 --- a/infrastructure/schemas/ksi-catalog.json +++ b/infrastructure/schemas/ksi-catalog.json @@ -121,7 +121,7 @@ { "reference": "NIST SP 800-53B", "reference_url": "https://csrc.nist.gov/pubs/sp/800/53/b/upd1/final", - "description": "Section 2.5 states \"As the number of controls in [SP 800-53] grows in response to an increasingly sophisticated threat space, it is important for organizations to have the ability to describe key capabilities needed to protect organizational missions and business functions, and to subsequently select controls that\u2014if properly designed, developed, and implemented\u2014produce such capabilities. The use of capabilities simplifies how the protection problem is viewed conceptually. Using the construct of a capability provides a method of grouping controls that are employed for a common purpose or to achieve a common objective.\" This section later states \"Ultimately, authorization decisions (i.e., risk acceptance decisions) are made based on the degree to which the desired capabilities have been effectively achieved.\"" + "description": "Section 2.5 states \"As the number of controls in [SP 800-53] grows in response to an increasingly sophisticated threat space, it is important for organizations to have the ability to describe key capabilities needed to protect organizational missions and business functions, and to subsequently select controls that—if properly designed, developed, and implemented—produce such capabilities. The use of capabilities simplifies how the protection problem is viewed conceptually. Using the construct of a capability provides a method of grouping controls that are employed for a common purpose or to achieve a common objective.\" This section later states \"Ultimately, authorization decisions (i.e., risk acceptance decisions) are made based on the degree to which the desired capabilities have been effectively achieved.\"" }, { "reference": "NIST SP 800-53A", @@ -129,9 +129,9 @@ "description": "Section 3.5 states \"When organizations employ the concept of capabilities, automated and manual assessments account for all security and privacy controls that comprise the security and privacy capabilities. Assessors are aware of how the controls work together to provide such capabilities.\"" }, { - "reference": "FedRAMP Authorization Act (44 USC \u00a7 3609 (a) (1))", + "reference": "FedRAMP Authorization Act (44 USC § 3609 (a) (1))", "reference_url": "https://fedramp.gov/docs/authority/law/#a-roles-and-responsibilities", - "description": "requires that the Administrator of the General Services Administration shall \"in consultation with the [DHS] Secretary, develop, coordinate, and implement a process to support agency review, reuse, and standardization, where appropriate, of security assessments of cloud computing products and services...\" 44 USC \u00a7 3609 (c) (2) further states that \"the [GSA] Administrator shall establish a means for the automation of security assessments and reviews.\"", + "description": "requires that the Administrator of the General Services Administration shall \"in consultation with the [DHS] Secretary, develop, coordinate, and implement a process to support agency review, reuse, and standardization, where appropriate, of security assessments of cloud computing products and services...\" 44 USC § 3609 (c) (2) further states that \"the [GSA] Administrator shall establish a means for the automation of security assessments and reviews.\"", "delegation": "These responsibilities are delegated to the FedRAMP Director", "delegation_url": "https://www.gsa.gov/directives-library/gsa-delegations-of-authority-fedramp" } @@ -159,18 +159,6 @@ "moderate": true }, "controls": [ - { - "control_id": "cp-3", - "title": "Contingency Training" - }, - { - "control_id": "ir-2", - "title": "Incident Response Training" - }, - { - "control_id": "ps-6", - "title": "Access Agreements" - }, { "control_id": "at-2", "title": "Literacy Training and Awareness" @@ -183,6 +171,10 @@ "control_id": "at-2.3", "title": "Social Engineering and Mining" }, + { + "control_id": "at-3", + "title": "Role-based Training" + }, { "control_id": "at-3.5", "title": "Processing Personally Identifiable Information" @@ -191,13 +183,53 @@ "control_id": "at-4", "title": "Training Records" }, + { + "control_id": "cm-11", + "title": "User-installed Software" + }, + { + "control_id": "cp-3", + "title": "Contingency Training" + }, + { + "control_id": "cp-3.1", + "title": "Simulated Events" + }, + { + "control_id": "ir-2", + "title": "Incident Response Training" + }, + { + "control_id": "ir-2.1", + "title": "Simulated Events" + }, + { + "control_id": "ir-2.2", + "title": "Automated Training Environments" + }, { "control_id": "ir-2.3", "title": "Breach" }, { - "control_id": "at-3", - "title": "Role-based Training" + "control_id": "ir-9.2", + "title": "Training" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" + }, + { + "control_id": "pl-4", + "title": "Rules of Behavior" + }, + { + "control_id": "pl-4.1", + "title": "Social Media and External Site/Application Usage Restrictions" + }, + { + "control_id": "ps-6", + "title": "Access Agreements" }, { "control_id": "sr-11.1", @@ -224,14 +256,26 @@ "control_id": "au-2", "title": "Event Logging" }, + { + "control_id": "ca-7.4", + "title": "Risk Monitoring" + }, { "control_id": "cm-3", "title": "Configuration Change Control" }, + { + "control_id": "cm-3.1", + "title": "Automated Documentation, Notification, and Prohibition of Changes" + }, { "control_id": "cm-3.2", "title": "Testing, Validation, and Documentation of Changes" }, + { + "control_id": "cm-3.6", + "title": "Cryptography Management" + }, { "control_id": "cm-4.2", "title": "Verification of Controls" @@ -243,10 +287,6 @@ { "control_id": "cm-8.3", "title": "Automated Unauthorized Component Detection" - }, - { - "control_id": "ma-2", - "title": "Controlled Maintenance" } ] }, @@ -345,6 +385,14 @@ "control_id": "cm-4.2", "title": "Verification of Controls" }, + { + "control_id": "cm-6.2", + "title": "Respond to Unauthorized Changes" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" + }, { "control_id": "si-2", "title": "Flaw Remediation" @@ -370,6 +418,18 @@ "control_id": "cm-2", "title": "Baseline Configuration" }, + { + "control_id": "cm-8.4", + "title": "Accountability Information" + }, + { + "control_id": "cp-10.4", + "title": "Restore Within Time Period" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" + }, { "control_id": "si-3", "title": "Malicious Code Protection" @@ -390,8 +450,48 @@ "title": "Independent Assessors" }, { - "control_id": "ca-7.1", - "title": "Independent Assessment" + "control_id": "cm-11", + "title": "User-installed Software" + }, + { + "control_id": "cm-6.2", + "title": "Respond to Unauthorized Changes" + }, + { + "control_id": "cp-10.4", + "title": "Restore Within Time Period" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" + }, + { + "control_id": "sc-24", + "title": "Fail in Known State" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" + }, + { + "control_id": "si-4.22", + "title": "Unauthorized Network Services" + }, + { + "control_id": "si-7", + "title": "Software, Firmware, and Information Integrity" + }, + { + "control_id": "si-7.1", + "title": "Integrity Checks" + }, + { + "control_id": "si-7.5", + "title": "Automated Response to Integrity Violations" + }, + { + "control_id": "si-7.7", + "title": "Integration of Detection and Response" } ] }, @@ -415,6 +515,10 @@ { "control_id": "pl-10", "title": "Baseline Selection" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" } ] }, @@ -447,6 +551,14 @@ "control_id": "ca-9", "title": "Internal System Connections" }, + { + "control_id": "sc-10", + "title": "Network Disconnect" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" + }, { "control_id": "sc-7.3", "title": "Access Points" @@ -457,7 +569,7 @@ }, { "control_id": "sc-7.5", - "title": "Deny by Default \u2014 Allow by Exception" + "title": "Deny by Default — Allow by Exception" }, { "control_id": "sc-7.8", @@ -467,10 +579,6 @@ "control_id": "sc-8", "title": "Transmission Confidentiality and Integrity" }, - { - "control_id": "sc-10", - "title": "Network Disconnect" - }, { "control_id": "si-10", "title": "Information Input Validation" @@ -493,7 +601,16 @@ "low": true, "moderate": true }, - "controls": [] + "controls": [ + { + "control_id": "cp-2.5", + "title": "Continue Mission and Business Functions" + }, + { + "control_id": "sc-12.1", + "title": "Availability" + } + ] }, { "id": "KSI-CNA-RNT", @@ -516,9 +633,41 @@ "control_id": "cm-7.1", "title": "Periodic Review" }, + { + "control_id": "sc-7.10", + "title": "Prevent Exfiltration" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" + }, + { + "control_id": "sc-7.20", + "title": "Dynamic Isolation and Segregation" + }, + { + "control_id": "sc-7.21", + "title": "Isolation of System Components" + }, { "control_id": "sc-7.5", - "title": "Deny by Default \u2014 Allow by Exception" + "title": "Deny by Default — Allow by Exception" + }, + { + "control_id": "si-10", + "title": "Information Input Validation" + }, + { + "control_id": "si-4.11", + "title": "Analyze Communications Traffic Anomalies" + }, + { + "control_id": "si-4.18", + "title": "Analyze Traffic and Covert Exfiltration" + }, + { + "control_id": "si-4.4", + "title": "Inbound and Outbound Communications Traffic" }, { "control_id": "si-8", @@ -535,10 +684,18 @@ "moderate": true }, "controls": [ + { + "control_id": "ir-4.2", + "title": "Dynamic Reconfiguration" + }, { "control_id": "sc-5", "title": "Denial-of-service Protection" }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" + }, { "control_id": "si-8", "title": "Spam Protection" @@ -566,10 +723,22 @@ "control_id": "ac-17.3", "title": "Managed Access Control Points" }, + { + "control_id": "ac-4.21", + "title": "Physical or Logical Separation of Information Flows" + }, { "control_id": "ca-9", "title": "Internal System Connections" }, + { + "control_id": "cm-11", + "title": "User-installed Software" + }, + { + "control_id": "sc-10", + "title": "Network Disconnect" + }, { "control_id": "sc-4", "title": "Information in Shared System Resources" @@ -578,6 +747,14 @@ "control_id": "sc-7", "title": "Boundary Protection" }, + { + "control_id": "sc-7.20", + "title": "Dynamic Isolation and Segregation" + }, + { + "control_id": "sc-7.21", + "title": "Isolation of System Components" + }, { "control_id": "sc-7.7", "title": "Split Tunneling for Remote Devices" @@ -585,10 +762,6 @@ { "control_id": "sc-8", "title": "Transmission Confidentiality and Integrity" - }, - { - "control_id": "sc-10", - "title": "Network Disconnect" } ] } @@ -607,6 +780,10 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-2.13", + "title": "Disable Accounts for High-risk Individuals" + }, { "control_id": "ac-2.2", "title": "Automated Temporary and Emergency Account Management" @@ -615,18 +792,10 @@ "control_id": "ac-2.3", "title": "Disable Accounts" }, - { - "control_id": "ac-2.13", - "title": "Disable Accounts for High-risk Individuals" - }, { "control_id": "ac-6.7", "title": "Review of User Privileges" }, - { - "control_id": "ia-4.4", - "title": "Identify User Status" - }, { "control_id": "ia-12", "title": "Identity Proofing" @@ -642,6 +811,14 @@ { "control_id": "ia-12.5", "title": "Address Confirmation" + }, + { + "control_id": "ia-4.4", + "title": "Identify User Status" + }, + { + "control_id": "ia-8.4", + "title": "Use of Defined Profiles" } ] }, @@ -654,54 +831,70 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-2", + "title": "Account Management" + }, { "control_id": "ac-3", "title": "Access Enforcement" }, { - "control_id": "ia-5.1", - "title": "Password-based Authentication" + "control_id": "ia-2", + "title": "Identification and Authentication (Organizational Users)" }, { - "control_id": "ia-5.2", - "title": "Public Key-based Authentication" + "control_id": "ia-2.1", + "title": "Multi-factor Authentication to Privileged Accounts" }, { - "control_id": "ia-5.6", - "title": "Protection of Authenticators" + "control_id": "ia-2.2", + "title": "Multi-factor Authentication to Non-privileged Accounts" }, { - "control_id": "ia-6", - "title": "Authentication Feedback" + "control_id": "ia-2.6", + "title": "Access to Accounts —separate Device" }, { - "control_id": "ac-2", - "title": "Account Management" + "control_id": "ia-2.8", + "title": "Access to Accounts — Replay Resistant" }, { - "control_id": "ia-2", - "title": "Identification and Authentication (Organizational Users)" + "control_id": "ia-5", + "title": "Authenticator Management" }, { - "control_id": "ia-2.1", - "title": "Multi-factor Authentication to Privileged Accounts" + "control_id": "ia-5.1", + "title": "Password-based Authentication" }, { - "control_id": "ia-2.2", - "title": "Multi-factor Authentication to Non-privileged Accounts" + "control_id": "ia-5.2", + "title": "Public Key-based Authentication" }, { - "control_id": "ia-2.8", - "title": "Access to Accounts \u2014 Replay Resistant" + "control_id": "ia-5.6", + "title": "Protection of Authenticators" }, { - "control_id": "ia-5", - "title": "Authenticator Management" + "control_id": "ia-5.8", + "title": "Multiple System Accounts" + }, + { + "control_id": "ia-6", + "title": "Authentication Feedback" }, { "control_id": "ia-8", "title": "Identification and Authentication (Non-organizational Users)" }, + { + "control_id": "ia-8.2", + "title": "Acceptance of External Authenticators" + }, + { + "control_id": "ia-8.4", + "title": "Use of Defined Profiles" + }, { "control_id": "sc-23", "title": "Session Authenticity" @@ -717,26 +910,6 @@ "moderate": true }, "controls": [ - { - "control_id": "ac-2.5", - "title": "Inactivity Logout" - }, - { - "control_id": "ac-2.6", - "title": "Dynamic Privilege Management" - }, - { - "control_id": "ac-3", - "title": "Access Enforcement" - }, - { - "control_id": "ac-4", - "title": "Information Flow Enforcement" - }, - { - "control_id": "ac-6", - "title": "Least Privilege" - }, { "control_id": "ac-12", "title": "Session Termination" @@ -761,6 +934,30 @@ "control_id": "ac-17.3", "title": "Managed Access Control Points" }, + { + "control_id": "ac-18.4", + "title": "Restrict Configurations by Users" + }, + { + "control_id": "ac-2.11", + "title": "Usage Conditions" + }, + { + "control_id": "ac-2.5", + "title": "Inactivity Logout" + }, + { + "control_id": "ac-2.6", + "title": "Dynamic Privilege Management" + }, + { + "control_id": "ac-2.7", + "title": "Privileged User Accounts" + }, + { + "control_id": "ac-2.9", + "title": "Restrictions on Use of Shared and Group Accounts" + }, { "control_id": "ac-20", "title": "Use of External Systems" @@ -769,14 +966,62 @@ "control_id": "ac-20.1", "title": "Limits on Authorized Use" }, + { + "control_id": "ac-21", + "title": "Information Sharing" + }, + { + "control_id": "ac-3", + "title": "Access Enforcement" + }, + { + "control_id": "ac-4", + "title": "Information Flow Enforcement" + }, + { + "control_id": "ac-6", + "title": "Least Privilege" + }, + { + "control_id": "ac-6.3", + "title": "Network Access to Privileged Commands" + }, + { + "control_id": "ac-6.8", + "title": "Privilege Levels for Code Execution" + }, + { + "control_id": "au-12.3", + "title": "Changes by Authorized Individuals" + }, + { + "control_id": "au-6.7", + "title": "Permitted Actions" + }, { "control_id": "cm-2.7", "title": "Configure Systems and Components for High-risk Areas" }, + { + "control_id": "cm-5.1", + "title": "Automated Access Enforcement and Audit Records" + }, + { + "control_id": "cm-5.5", + "title": "Privilege Limitation for Production and Operation" + }, + { + "control_id": "cm-7.2", + "title": "Prevent Program Execution" + }, { "control_id": "cm-9", "title": "Configuration Management Plan" }, + { + "control_id": "ia-11", + "title": "Re-authentication" + }, { "control_id": "ia-2", "title": "Identification and Authentication (Organizational Users)" @@ -802,8 +1047,12 @@ "title": "Protection of Authenticators" }, { - "control_id": "ia-11", - "title": "Re-authentication" + "control_id": "ir-4.6", + "title": "Insider Threats" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" }, { "control_id": "ps-2", @@ -826,8 +1075,12 @@ "title": "Access Agreements" }, { - "control_id": "sc-4", - "title": "Information in Shared System Resources" + "control_id": "ra-5.5", + "title": "Privileged Access" + }, + { + "control_id": "sc-2", + "title": "Separation of System and User Functionality" }, { "control_id": "sc-20", @@ -845,13 +1098,25 @@ "control_id": "sc-23", "title": "Session Authenticity" }, + { + "control_id": "sc-3", + "title": "Security Function Isolation" + }, { "control_id": "sc-39", "title": "Process Isolation" }, + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + }, { "control_id": "si-3", "title": "Malicious Code Protection" + }, + { + "control_id": "si-4.19", + "title": "Risk for Individuals" } ] }, @@ -864,6 +1129,14 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-17", + "title": "Remote Access" + }, + { + "control_id": "ac-17.4", + "title": "Privileged Commands and Access" + }, { "control_id": "ac-2", "title": "Account Management" @@ -872,6 +1145,10 @@ "control_id": "ac-2.1", "title": "Automated System Account Management" }, + { + "control_id": "ac-2.11", + "title": "Usage Conditions" + }, { "control_id": "ac-2.2", "title": "Automated Temporary and Emergency Account Management" @@ -888,6 +1165,18 @@ "control_id": "ac-2.6", "title": "Dynamic Privilege Management" }, + { + "control_id": "ac-2.7", + "title": "Privileged User Accounts" + }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, + { + "control_id": "ac-21", + "title": "Information Sharing" + }, { "control_id": "ac-3", "title": "Access Enforcement" @@ -908,10 +1197,18 @@ "control_id": "ac-6.1", "title": "Authorize Access to Security Functions" }, + { + "control_id": "ac-6.10", + "title": "Prohibit Non-privileged Users from Executing Privileged Functions" + }, { "control_id": "ac-6.2", "title": "Non-privileged Access for Nonsecurity Functions" }, + { + "control_id": "ac-6.3", + "title": "Network Access to Privileged Commands" + }, { "control_id": "ac-6.5", "title": "Privileged Accounts" @@ -921,33 +1218,45 @@ "title": "Review of User Privileges" }, { - "control_id": "ac-6.9", - "title": "Log Use of Privileged Functions" + "control_id": "ac-6.8", + "title": "Privilege Levels for Code Execution" }, { - "control_id": "ac-6.10", - "title": "Prohibit Non-privileged Users from Executing Privileged Functions" + "control_id": "ac-6.9", + "title": "Log Use of Privileged Functions" }, { "control_id": "ac-7", "title": "Unsuccessful Logon Attempts" }, { - "control_id": "ac-20.1", - "title": "Limits on Authorized Use" + "control_id": "au-10", + "title": "Non-repudiation" }, { - "control_id": "ac-17", - "title": "Remote Access" + "control_id": "au-12.3", + "title": "Changes by Authorized Individuals" }, { "control_id": "au-9.4", "title": "Access by Subset of Privileged Users" }, + { + "control_id": "ca-3.6", + "title": "Transfer Authorizations" + }, { "control_id": "cm-5", "title": "Access Restrictions for Change" }, + { + "control_id": "cm-5.1", + "title": "Automated Access Enforcement and Audit Records" + }, + { + "control_id": "cm-5.5", + "title": "Privilege Limitation for Production and Operation" + }, { "control_id": "cm-7", "title": "Least Functionality" @@ -958,7 +1267,7 @@ }, { "control_id": "cm-7.5", - "title": "Authorized Software \u2014 Allow-by-exception" + "title": "Authorized Software — Allow-by-exception" }, { "control_id": "cm-9", @@ -976,6 +1285,10 @@ "control_id": "ia-7", "title": "Cryptographic Module Authentication" }, + { + "control_id": "ir-4.6", + "title": "Insider Threats" + }, { "control_id": "ps-2", "title": "Position Risk Designation" @@ -1012,6 +1325,10 @@ "control_id": "sc-23", "title": "Session Authenticity" }, + { + "control_id": "sc-3", + "title": "Security Function Isolation" + }, { "control_id": "sc-39", "title": "Process Isolation" @@ -1035,6 +1352,10 @@ "control_id": "ac-2.2", "title": "Automated Temporary and Emergency Account Management" }, + { + "control_id": "ac-2.9", + "title": "Restrictions on Use of Shared and Group Accounts" + }, { "control_id": "ac-4", "title": "Information Flow Enforcement" @@ -1043,6 +1364,14 @@ "control_id": "ac-6.5", "title": "Privileged Accounts" }, + { + "control_id": "au-10", + "title": "Non-repudiation" + }, + { + "control_id": "ia-2.5", + "title": "Individual Authentication with Group Authentication" + }, { "control_id": "ia-3", "title": "Device Identification and Authentication" @@ -1075,17 +1404,45 @@ "title": "Automated System Account Management" }, { - "control_id": "ac-2.3", - "title": "Disable Accounts" + "control_id": "ac-2.11", + "title": "Usage Conditions" + }, + { + "control_id": "ac-2.12", + "title": "Account Monitoring for Atypical Usage" }, { "control_id": "ac-2.13", "title": "Disable Accounts for High-risk Individuals" }, + { + "control_id": "ac-2.3", + "title": "Disable Accounts" + }, { "control_id": "ac-7", "title": "Unsuccessful Logon Attempts" }, + { + "control_id": "au-6.5", + "title": "Integrated Analysis of Audit Records" + }, + { + "control_id": "cm-5.1", + "title": "Automated Access Enforcement and Audit Records" + }, + { + "control_id": "cm-6.2", + "title": "Respond to Unauthorized Changes" + }, + { + "control_id": "ir-4.2", + "title": "Dynamic Reconfiguration" + }, + { + "control_id": "ir-4.6", + "title": "Insider Threats" + }, { "control_id": "ps-4", "title": "Personnel Termination" @@ -1093,6 +1450,30 @@ { "control_id": "ps-8", "title": "Personnel Sanctions" + }, + { + "control_id": "si-4", + "title": "System Monitoring" + }, + { + "control_id": "si-4.1", + "title": "System-wide Intrusion Detection System" + }, + { + "control_id": "si-4.19", + "title": "Risk for Individuals" + }, + { + "control_id": "si-4.20", + "title": "Privileged Users" + }, + { + "control_id": "si-4.22", + "title": "Unauthorized Network Services" + }, + { + "control_id": "si-4.4", + "title": "Inbound and Outbound Communications Traffic" } ] } @@ -1123,9 +1504,17 @@ "control_id": "ir-4.1", "title": "Automated Incident Handling Processes" }, + { + "control_id": "ir-5.1", + "title": "Automated Tracking, Data Collection, and Analysis" + }, { "control_id": "ir-8", "title": "Incident Response Plan" + }, + { + "control_id": "ir-9.3", + "title": "Post-spill Operations" } ] }, @@ -1146,6 +1535,10 @@ "control_id": "ir-4.1", "title": "Automated Incident Handling Processes" }, + { + "control_id": "ir-4.11", + "title": "Integrated Incident Response Team" + }, { "control_id": "ir-6", "title": "Incident Reporting" @@ -1174,6 +1567,10 @@ "control_id": "ir-8.1", "title": "Breaches" }, + { + "control_id": "ir-9.3", + "title": "Post-spill Operations" + }, { "control_id": "si-4.5", "title": "System-generated Alerts" @@ -1201,13 +1598,29 @@ "control_id": "ir-4.1", "title": "Automated Incident Handling Processes" }, + { + "control_id": "ir-4.11", + "title": "Integrated Incident Response Team" + }, + { + "control_id": "ir-4.4", + "title": "Information Correlation" + }, { "control_id": "ir-5", "title": "Incident Monitoring" }, + { + "control_id": "ir-5.1", + "title": "Automated Tracking, Data Collection, and Analysis" + }, { "control_id": "ir-8", "title": "Incident Response Plan" + }, + { + "control_id": "ra-5.8", + "title": "Review Historic Audit Logs" } ] } @@ -1226,6 +1639,14 @@ "moderate": true }, "controls": [ + { + "control_id": "au-12.3", + "title": "Changes by Authorized Individuals" + }, + { + "control_id": "au-6.7", + "title": "Permitted Actions" + }, { "control_id": "si-11", "title": "Error Handling" @@ -1241,6 +1662,10 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-19", + "title": "Access Control for Mobile Devices" + }, { "control_id": "ca-7", "title": "Continuous Monitoring" @@ -1253,6 +1678,14 @@ "control_id": "cm-6", "title": "Configuration Settings" }, + { + "control_id": "cm-6.1", + "title": "Automated Management, Application, and Verification" + }, + { + "control_id": "ra-3", + "title": "Risk Assessment" + }, { "control_id": "si-7.7", "title": "Integration of Detection and Response" @@ -1268,33 +1701,57 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-17.1", + "title": "Monitoring and Control" + }, { "control_id": "ac-2.4", "title": "Automated Audit Actions" }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, { "control_id": "ac-6.9", "title": "Log Use of Privileged Functions" }, { - "control_id": "ac-17.1", - "title": "Monitoring and Control" + "control_id": "au-10", + "title": "Non-repudiation" + }, + { + "control_id": "au-12", + "title": "Audit Record Generation" + }, + { + "control_id": "au-2", + "title": "Event Logging" + }, + { + "control_id": "au-7", + "title": "Audit Record Reduction and Report Generation" + }, + { + "control_id": "au-7.1", + "title": "Automatic Processing" }, { - "control_id": "ac-20.1", - "title": "Limits on Authorized Use" + "control_id": "cm-5.1", + "title": "Automated Access Enforcement and Audit Records" }, { - "control_id": "au-2", - "title": "Event Logging" + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" }, { - "control_id": "au-7.1", - "title": "Automatic Processing" + "control_id": "si-4.19", + "title": "Risk for Individuals" }, { - "control_id": "au-12", - "title": "Audit Record Generation" + "control_id": "si-4.20", + "title": "Privileged Users" }, { "control_id": "si-4.4", @@ -1327,6 +1784,14 @@ "control_id": "ac-20.1", "title": "Limits on Authorized Use" }, + { + "control_id": "au-11", + "title": "Audit Record Retention" + }, + { + "control_id": "au-12.1", + "title": "System-wide and Time-correlated Audit Trail" + }, { "control_id": "au-2", "title": "Event Logging" @@ -1347,6 +1812,10 @@ "control_id": "au-5", "title": "Response to Audit Logging Process Failures" }, + { + "control_id": "au-5.1", + "title": "Storage Capacity Warning" + }, { "control_id": "au-6.1", "title": "Automated Process Integration" @@ -1355,6 +1824,14 @@ "control_id": "au-6.3", "title": "Correlate Audit Record Repositories" }, + { + "control_id": "au-6.4", + "title": "Central Review and Analysis" + }, + { + "control_id": "au-6.5", + "title": "Integrated Analysis of Audit Records" + }, { "control_id": "au-7", "title": "Audit Record Reduction and Report Generation" @@ -1371,14 +1848,14 @@ "control_id": "au-9", "title": "Protection of Audit Information" }, - { - "control_id": "au-11", - "title": "Audit Record Retention" - }, { "control_id": "ir-4.1", "title": "Automated Incident Handling Processes" }, + { + "control_id": "si-4.16", + "title": "Correlate Monitoring Information" + }, { "control_id": "si-4.2", "title": "Automated Tools and Mechanisms for Real-time Analysis" @@ -1386,10 +1863,6 @@ { "control_id": "si-4.4", "title": "Inbound and Outbound Communications Traffic" - }, - { - "control_id": "si-7.7", - "title": "Integration of Detection and Response" } ] }, @@ -1422,10 +1895,22 @@ "control_id": "au-6.1", "title": "Automated Process Integration" }, + { + "control_id": "au-7", + "title": "Audit Record Reduction and Report Generation" + }, + { + "control_id": "ra-5.8", + "title": "Review Historic Audit Logs" + }, { "control_id": "si-4", "title": "System Monitoring" }, + { + "control_id": "si-4.19", + "title": "Risk for Individuals" + }, { "control_id": "si-4.4", "title": "Inbound and Outbound Communications Traffic" @@ -1447,13 +1932,21 @@ "moderate": true }, "controls": [ + { + "control_id": "cm-12", + "title": "Information Location" + }, + { + "control_id": "cm-12.1", + "title": "Automated Tools to Support Information Location" + }, { "control_id": "cm-2.2", "title": "Automation Support for Accuracy and Currency" }, { "control_id": "cm-7.5", - "title": "Authorized Software \u2014 Allow-by-exception" + "title": "Authorized Software — Allow-by-exception" }, { "control_id": "cm-8", @@ -1464,12 +1957,12 @@ "title": "Updates During Installation and Removal" }, { - "control_id": "cm-12", - "title": "Information Location" + "control_id": "cm-8.2", + "title": "Automated Maintenance" }, { - "control_id": "cm-12.1", - "title": "Automated Tools to Support Information Location" + "control_id": "cm-8.4", + "title": "Accountability Information" }, { "control_id": "cp-2.8", @@ -1504,6 +1997,10 @@ "control_id": "ca-2", "title": "Control Assessments" }, + { + "control_id": "ca-2.2", + "title": "Specialized Assessments" + }, { "control_id": "cp-2.1", "title": "Coordinate with Related Plans" @@ -1516,6 +2013,10 @@ "control_id": "ir-3.2", "title": "Coordination with Related Plans" }, + { + "control_id": "ir-4.11", + "title": "Integrated Incident Response Team" + }, { "control_id": "pm-3", "title": "Information Security and Privacy Resources" @@ -1555,6 +2056,10 @@ "control_id": "cm-3.4", "title": "Security and Privacy Representatives" }, + { + "control_id": "ia-5.7", + "title": "No Embedded Unencrypted Static Authenticators" + }, { "control_id": "pl-8", "title": "Security and Privacy Architectures" @@ -1563,6 +2068,10 @@ "control_id": "pm-7", "title": "Enterprise Architecture" }, + { + "control_id": "ra-9", + "title": "Criticality Analysis" + }, { "control_id": "sa-3", "title": "System Development Life Cycle" @@ -1571,14 +2080,14 @@ "control_id": "sa-8", "title": "Security and Privacy Engineering Principles" }, - { - "control_id": "sc-4", - "title": "Information in Shared System Resources" - }, { "control_id": "sc-18", "title": "Mobile Code" }, + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + }, { "control_id": "si-10", "title": "Information Input Validation" @@ -1642,10 +2151,6 @@ { "control_id": "cp-10.2", "title": "Transaction Recovery" - }, - { - "control_id": "si-12", - "title": "Information Management and Retention" } ] }, @@ -1752,6 +2257,10 @@ "moderate": true }, "controls": [ + { + "control_id": "cp-10", + "title": "System Recovery and Reconstitution" + }, { "control_id": "cp-2.1", "title": "Coordinate with Related Plans" @@ -1780,10 +2289,6 @@ "control_id": "cp-9.1", "title": "Testing for Reliability and Integrity" }, - { - "control_id": "cp-10", - "title": "System Recovery and Reconstitution" - }, { "control_id": "ir-3", "title": "Incident Response Testing" @@ -1791,6 +2296,14 @@ { "control_id": "ir-3.2", "title": "Coordination with Related Plans" + }, + { + "control_id": "ir-9.3", + "title": "Post-spill Operations" + }, + { + "control_id": "sc-12.1", + "title": "Availability" } ] } @@ -1803,23 +2316,31 @@ { "id": "KSI-SCR-MIT", "name": "Mitigating Supply Chain Risk", - "statement": "Persistently identify, review, and mitigate potential supply chain risks.", + "statement": "Potential supply chain risks are persistently identified, reviewed, and mitigated.", "impact": { "low": true, "moderate": true }, "controls": [ + { + "control_id": "ac-19", + "title": "Access Control for Mobile Devices" + }, { "control_id": "ac-20", "title": "Use of External Systems" }, { - "control_id": "ra-3.1", - "title": "Supply Chain Risk Assessment" + "control_id": "ca-7.4", + "title": "Risk Monitoring" }, { - "control_id": "sa-9", - "title": "External System Services" + "control_id": "ra-3", + "title": "Risk Assessment" + }, + { + "control_id": "ra-3.1", + "title": "Supply Chain Risk Assessment" }, { "control_id": "sa-10", @@ -1829,33 +2350,101 @@ "control_id": "sa-11", "title": "Developer Testing and Evaluation" }, + { + "control_id": "sa-11.1", + "title": "Static Code Analysis" + }, + { + "control_id": "sa-11.2", + "title": "Threat Modeling and Vulnerability Analyses" + }, + { + "control_id": "sa-15", + "title": "Development Process, Standards, and Tools" + }, { "control_id": "sa-15.3", "title": "Criticality Analysis" }, + { + "control_id": "sa-16", + "title": "Developer-provided Training" + }, + { + "control_id": "sa-17", + "title": "Developer Security and Privacy Architecture and Design" + }, + { + "control_id": "sa-21", + "title": "Developer Screening" + }, { "control_id": "sa-22", "title": "Unsupported System Components" }, { - "control_id": "si-7.1", - "title": "Integrity Checks" + "control_id": "sa-4", + "title": "Acquisition Process" }, { - "control_id": "sr-5", - "title": "Acquisition Strategies, Tools, and Methods" + "control_id": "sa-4.1", + "title": "Functional Properties of Controls" }, { - "control_id": "sr-6", - "title": "Supplier Assessments and Reviews" + "control_id": "sa-4.2", + "title": "Design and Implementation Information for Controls" }, { - "control_id": "ca-7.4", - "title": "Risk Monitoring" + "control_id": "sa-4.5", + "title": "System, Component, and Service Configurations" + }, + { + "control_id": "sa-4.9", + "title": "Functions, Ports, Protocols, and Services in Use" + }, + { + "control_id": "sa-9", + "title": "External System Services" + }, + { + "control_id": "sa-9.1", + "title": "Risk Assessments and Organizational Approvals" + }, + { + "control_id": "sa-9.2", + "title": "Identification of Functions, Ports, Protocols, and Services" + }, + { + "control_id": "sa-9.5", + "title": "Processing, Storage, and Service Location" }, { "control_id": "sc-18", "title": "Mobile Code" + }, + { + "control_id": "si-7.1", + "title": "Integrity Checks" + }, + { + "control_id": "si-7.15", + "title": "Code Authentication" + }, + { + "control_id": "sr-2", + "title": "Supply Chain Risk Management Plan" + }, + { + "control_id": "sr-3", + "title": "Supply Chain Controls and Processes" + }, + { + "control_id": "sr-5", + "title": "Acquisition Strategies, Tools, and Methods" + }, + { + "control_id": "sr-6", + "title": "Supplier Assessments and Reviews" } ] }, @@ -1868,10 +2457,18 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-19", + "title": "Access Control for Mobile Devices" + }, { "control_id": "ac-20", "title": "Use of External Systems" }, + { + "control_id": "ca-2.3", + "title": "Leveraging Results from External Organizations" + }, { "control_id": "ca-3", "title": "Information Exchange" @@ -1884,6 +2481,10 @@ "control_id": "ps-7", "title": "External Personnel Security" }, + { + "control_id": "ra-3", + "title": "Risk Assessment" + }, { "control_id": "ra-5", "title": "Vulnerability Monitoring and Scanning" @@ -1896,6 +2497,10 @@ "control_id": "si-5", "title": "Security Alerts, Advisories, and Directives" }, + { + "control_id": "sr-3", + "title": "Supply Chain Controls and Processes" + }, { "control_id": "sr-5", "title": "Acquisition Strategies, Tools, and Methods" @@ -1929,6 +2534,10 @@ "control_id": "ac-2.4", "title": "Automated Audit Actions" }, + { + "control_id": "ac-6.8", + "title": "Privilege Levels for Code Execution" + }, { "control_id": "cm-2", "title": "Baseline Configuration" @@ -1949,18 +2558,30 @@ "control_id": "cm-7.1", "title": "Periodic Review" }, - { - "control_id": "pl-9", - "title": "Central Management" - }, { "control_id": "pl-10", "title": "Baseline Selection" }, + { + "control_id": "pl-9", + "title": "Central Management" + }, { "control_id": "sa-5", "title": "System Documentation" }, + { + "control_id": "sc-45", + "title": "System Time Synchronization" + }, + { + "control_id": "sc-45.1", + "title": "Synchronization with Authoritative Time Source" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" + }, { "control_id": "si-5", "title": "Security Alerts, Advisories, and Directives" @@ -1996,6 +2617,10 @@ "control_id": "sc-12", "title": "Cryptographic Key Establishment and Management" }, + { + "control_id": "sc-12.1", + "title": "Availability" + }, { "control_id": "sc-17", "title": "Public Key Infrastructure Certificates" @@ -2012,29 +2637,41 @@ }, "controls": [ { - "control_id": "cm-7.1", - "title": "Periodic Review" + "control_id": "ca-7.4", + "title": "Risk Monitoring" }, { "control_id": "cm-12.1", "title": "Automated Tools to Support Information Location" }, { - "control_id": "ma-2", - "title": "Controlled Maintenance" + "control_id": "cm-5.5", + "title": "Privilege Limitation for Production and Operation" + }, + { + "control_id": "cm-7.1", + "title": "Periodic Review" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" }, { "control_id": "pl-8", "title": "Security and Privacy Architectures" }, { - "control_id": "sc-7", - "title": "Boundary Protection" + "control_id": "ra-3", + "title": "Risk Assessment" }, { "control_id": "sc-39", "title": "Process Isolation" }, + { + "control_id": "sc-7", + "title": "Boundary Protection" + }, { "control_id": "si-2.2", "title": "Automated Flaw Remediation Status" @@ -2058,9 +2695,25 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-19", + "title": "Access Control for Mobile Devices" + }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" + }, + { + "control_id": "sc-12.1", + "title": "Availability" + }, { "control_id": "sc-4", "title": "Information in Shared System Resources" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" } ] }, @@ -2073,6 +2726,10 @@ "moderate": true }, "controls": [ + { + "control_id": "ir-9", + "title": "Information Spillage Response" + }, { "control_id": "si-12.3", "title": "Information Disposal" @@ -2080,6 +2737,14 @@ { "control_id": "si-18.4", "title": "Individual Requests" + }, + { + "control_id": "si-8", + "title": "Spam Protection" + }, + { + "control_id": "si-8.2", + "title": "Automatic Updates" } ] }, @@ -2093,25 +2758,33 @@ }, "controls": [ { - "control_id": "ac-1", - "title": "Policy and Procedures" + "control_id": "ac-11", + "title": "Device Lock" + }, + { + "control_id": "ac-11.1", + "title": "Pattern-hiding Displays" }, { "control_id": "ac-17.2", "title": "Protection of Confidentiality and Integrity Using Encryption" }, { - "control_id": "cp-9.8", - "title": "Cryptographic Protection" + "control_id": "ac-19.5", + "title": "Full Device or Container-based Encryption" }, { - "control_id": "sc-8", - "title": "Transmission Confidentiality and Integrity" + "control_id": "ac-21", + "title": "Information Sharing" }, { - "control_id": "sc-8.1", + "control_id": "cp-9.8", "title": "Cryptographic Protection" }, + { + "control_id": "ir-9.4", + "title": "Exposure to Unauthorized Personnel" + }, { "control_id": "sc-13", "title": "Cryptographic Protection" @@ -2139,6 +2812,26 @@ { "control_id": "sc-28.1", "title": "Cryptographic Protection" + }, + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + }, + { + "control_id": "sc-7.18", + "title": "Fail Secure" + }, + { + "control_id": "sc-8", + "title": "Transmission Confidentiality and Integrity" + }, + { + "control_id": "sc-8.1", + "title": "Cryptographic Protection" + }, + { + "control_id": "si-4.18", + "title": "Analyze Traffic and Covert Exfiltration" } ] }, @@ -2151,10 +2844,30 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-10", + "title": "Concurrent Session Control" + }, + { + "control_id": "ac-17.4", + "title": "Privileged Commands and Access" + }, + { + "control_id": "ac-18", + "title": "Wireless Access" + }, + { + "control_id": "ac-19", + "title": "Access Control for Mobile Devices" + }, { "control_id": "sc-23", "title": "Session Authenticity" }, + { + "control_id": "si-4.22", + "title": "Unauthorized Network Services" + }, { "control_id": "si-7.1", "title": "Integrity Checks" @@ -2170,6 +2883,26 @@ "moderate": true }, "controls": [ + { + "control_id": "ac-17.4", + "title": "Privileged Commands and Access" + }, + { + "control_id": "ac-18", + "title": "Wireless Access" + }, + { + "control_id": "ac-19", + "title": "Access Control for Mobile Devices" + }, + { + "control_id": "au-9.3", + "title": "Cryptographic Protection" + }, + { + "control_id": "cm-14", + "title": "Signed Components" + }, { "control_id": "cm-2.2", "title": "Automation Support for Accuracy and Currency" @@ -2194,9 +2927,17 @@ "control_id": "si-7.1", "title": "Integrity Checks" }, + { + "control_id": "si-7.15", + "title": "Code Authentication" + }, { "control_id": "sr-10", "title": "Inspection of Systems or Components" + }, + { + "control_id": "sr-11", + "title": "Component Authenticity" } ] } diff --git a/scripts/sync-ksi-catalog.py b/scripts/sync-ksi-catalog.py new file mode 100644 index 00000000..199c3a8d --- /dev/null +++ b/scripts/sync-ksi-catalog.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Keep infrastructure/schemas/ksi-catalog.json equal to the pinned FedRAMP rules. + +The catalog's KSI block (IDs, names, statements and KSI->control mappings) is +FedRAMP's, so it is derived from the vendored, hash-checked Consolidated Rules +(scripts/_rules.py), never edited by hand. FedRAMP revises mappings between +releases (for example on 2026-10-13); re-pinning the rules and re-running this +script carries the change, and tests/test_ksi_catalog_corpus.py fails while +the two disagree. + +Control titles come from the vendored NIST SP 800-53 Rev5 catalog. The per-KSI impact flags are the catalog's +own (the corpus does not carry them) and are kept. + + sync-ksi-catalog.py rewrite the catalog + sync-ksi-catalog.py --check exit 1 if it differs from the corpus +""" + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _rules # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +CATALOG = REPO / "infrastructure" / "schemas" / "ksi-catalog.json" +NIST = REPO / "data" / "catalogs" / "NIST_SP-800-53_rev5_catalog.json" + + +def _fail(msg): + raise SystemExit(f"::error::KSI catalog: {msg} (fail-closed)") + + +def _titles(): + out = {} + + def walk(group): + for c in group.get("controls", []) or []: + out[c["id"]] = c["title"] + walk(c) + for g in group.get("groups", []) or []: + walk(g) + walk(json.loads(NIST.read_text())["catalog"]) + return out + + +def derive(catalog, rules_dir=_rules.RULES_DIR): + """The catalog with its KSI block rebuilt from the pinned corpus.""" + rules, _ = _rules.load_rules(rules_dir) + titles = _titles() + old_impact = {i["id"]: i.get("impact") for f in catalog.get("KSI", {}).values() + for i in f.get("indicators", [])} + ksi = {} + for fam_id, fam in rules["KSI"].items(): + indicators = [] + for kid, k in fam["indicators"].items(): + controls = [] + for c in k.get("controls", []) or []: + cid = str(c).lower() + if cid not in titles: + _fail(f"{kid} maps {cid}, which is not in the NIST SP 800-53 Rev5 catalog") + controls.append({"control_id": cid, "title": titles[cid]}) + if kid not in old_impact: + _fail(f"{kid} is new in the corpus; set its impact flags in the catalog first") + # Class C's statement where the KSI varies by class (five do). + statement = ((k.get("varies_by_class") or {}).get(_rules.CERT_CLASS) or k).get("statement") + if not isinstance(statement, str): + _fail(f"{kid} has no Class {_rules.CERT_CLASS.upper()} statement") + indicators.append({"id": kid, "name": k["name"], "statement": statement, + "impact": old_impact[kid], "controls": controls}) + ksi[fam_id] = {"id": fam["id"], "name": fam["name"], "indicators": indicators} + out = dict(catalog) + out["KSI"] = ksi + return out + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument("--check", action="store_true") + a = ap.parse_args(argv) + current = json.loads(CATALOG.read_text()) + want = derive(current) + if a.check: + if want != current: + print("::error::infrastructure/schemas/ksi-catalog.json differs from the pinned FedRAMP rules; " + "run scripts/sync-ksi-catalog.py and review the diff") + return 1 + return 0 + CATALOG.write_text(json.dumps(want, indent=2, ensure_ascii=False) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/migration/ksi-catalog-2026-11-05.json b/tests/fixtures/migration/ksi-catalog-2026-11-05.json new file mode 100644 index 00000000..7353e477 --- /dev/null +++ b/tests/fixtures/migration/ksi-catalog-2026-11-05.json @@ -0,0 +1,2206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "FedRAMP.schema.json", + "info": { + "name": "Key Security Indicators", + "short_name": "KSI", + "effective": { + "rev5": { + "is": "no" + }, + "20x": { + "is": "required", + "signup_url": "https://www.fedramp.gov/20x/phase-two/participate/", + "current_status": "Phase 2 Pilot", + "start_date": "2025-11-18", + "end_date": "2026-03-31", + "comments": [ + "Phase 1 pilot authorizations have one year from authorization to fully address this process but must demonstrate continuous quarterly progress.", + "Phase 2 Pilot participants must demonstrate significant progress towards addressing this process prior to submission for authorization review." + ] + } + }, + "releases": [ + { + "id": "26.06", + "published_date": "2026-06-24", + "description": "Consolidated Rules for 2026 (official launch). Key Security Indicators renumbered to mnemonic IDs; the AFR (Authorization by FedRAMP) family was retired and TPR (Third-Party) was renamed SCR (Supply Chain Risk). Rebuilt from the published markdown corpus by scripts/staged/build-ksi-catalog.py." + }, + { + "id": "25.12A", + "published_date": "2025-12-29", + "description": "This release updates a significant number of KSIs to improve clarity and expectations; in general the measures required to meet each KSI remain unchanged but these updates should make it easier to address them. In additional, some KSIs have been retired.", + "public_comment": false + }, + { + "id": "25.11C", + "published_date": "2025-12-01", + "description": "No effective/material changes; adjusted some KSI-AFR indcators to reference \"process\" instead of \"standard\" and added a note to KSI-AFR-02 addressing the potential infinite loop.", + "public_comment": false, + "machine_readable_link": "https://raw.githubusercontent.com/FedRAMP/docs/ced2160b22455ed26a11bf697be8f0ae3e1e5dff/data/FRMR.KSI.key-security-indicators.json" + }, + { + "id": "25.11B", + "published_date": "2025-11-24", + "description": "No material changes to content; updated JSON structure with additional information about Rev5 application added.", + "public_comment": false, + "machine_readable_link": "https://raw.githubusercontent.com/FedRAMP/docs/a37618fccf49d6b3406d90edf2125dc6cfcba140/data/FRMR.KSI.key-security-indicators.json" + }, + { + "id": "25.11A", + "published_date": "2025-11-18", + "description": "Updates Key Security Indicators for the FedRAMP 20x Phase Two pilot, including underlying structural changes to machine-readable docs; Renamed KSI \"indicator\" property to \"theme\" and KSI \"requirements\" property to \"indicators\" to match current naming conventions..", + "public_comment": true, + "related_rfcs": [ + { + "start_date": "2025-09-10", + "end_date": "2025-11-17", + "id": "0015", + "url": "https://www.fedramp.gov/rfcs/0015/", + "discussion_url": "https://github.com/FedRAMP/community/discussions/84", + "short_name": "rfc-0015-recommended-secure-configuration", + "full_name": "FedRAMP RFC-0015: Recommended Secure Configuration Standard" + } + ], + "machine_readable_link": "https://raw.githubusercontent.com/FedRAMP/docs/3291fa1952f5b68eaf1a815a8ef1846ae8ca9e2f/data/FRMR.KSI.key-security-indicators.json" + }, + { + "id": "25.10A", + "published_date": "2025-10-17", + "description": "Minor updates to improve clarity; switch from federal information to federal customer data; add impact level metadata; no substantive changes.", + "public_comment": false, + "machine_readable_link": null + }, + { + "id": "25.05D", + "published_date": "2025-08-24", + "description": "Minor non-breaking updates to align term definitions and highlighted terms across updated materials (no changes to KSIs, definitions are now in FRD-ALL).", + "public_comment": false, + "machine_readable_link": null + }, + { + "id": "25.05C", + "published_date": "2025-06-28", + "description": "Key Security Indicators in this release are unchanged from previously releases. 25.05C adds references for each KSI to underlying SP 800-53 controls.", + "public_comment": false, + "machine_readable_link": null + }, + { + "id": "25.05B", + "published_date": "2025-06-18", + "description": "Initial release of Key Security Indicators from 25.05; the previous 25.05A release contained errors during conversion to JSON that are fixed in this release, the KSIs should now be identical to the original 25.05 paper release of the KSIs released on 2025-05-30. FRR-KSI-AY rules were converted to FRR-KSI rules, but otherwise unchanged.", + "public_comment": false, + "machine_readable_link": null + }, + { + "id": "25.05A", + "published_date": "2025-05-30", + "description": "Initial release of Key Security Indicators", + "public_comment": true, + "related_rfcs": [ + { + "start_date": "2025-04-24", + "end_date": "2025-05-24", + "id": "0006", + "url": "https://www.fedramp.gov/rfcs/0006/", + "discussion_url": "https://github.com/FedRAMP/community/discussions/3", + "short_name": "rfc-0006-key-security-indicators", + "full_name": "FedRAMP RFC-0006: 20x Phase One Key Security Indicators" + } + ], + "machine_readable_link": null + } + ], + "front_matter": { + "authority": [ + { + "reference": "OMB Circular A-130", + "reference_url": "https://whitehouse.gov/wp-content/uploads/legacy_drupal_files/omb/circulars/A130/a130revised.pdf", + "description": "Appendix I states \"Agencies may also develop overlays for specific types of information or communities of interest (e.g., all web-based applications, all health care-related systems) as part of the security control selection process. Overlays provide a specification of security or privacy controls, control enhancements, supplemental guidance, and other supporting information as part of the tailoring process, that is intended to complement (and further refine) security control baselines. The overlay may be more stringent or less stringent than the original security control baseline and can be applied to multiple information systems.\"" + }, + { + "reference": "NIST SP 800-53B", + "reference_url": "https://csrc.nist.gov/pubs/sp/800/53/b/upd1/final", + "description": "Section 2.5 states \"As the number of controls in [SP 800-53] grows in response to an increasingly sophisticated threat space, it is important for organizations to have the ability to describe key capabilities needed to protect organizational missions and business functions, and to subsequently select controls that\u2014if properly designed, developed, and implemented\u2014produce such capabilities. The use of capabilities simplifies how the protection problem is viewed conceptually. Using the construct of a capability provides a method of grouping controls that are employed for a common purpose or to achieve a common objective.\" This section later states \"Ultimately, authorization decisions (i.e., risk acceptance decisions) are made based on the degree to which the desired capabilities have been effectively achieved.\"" + }, + { + "reference": "NIST SP 800-53A", + "reference_url": "https://csrc.nist.gov/pubs/sp/800/53/a/r5/final", + "description": "Section 3.5 states \"When organizations employ the concept of capabilities, automated and manual assessments account for all security and privacy controls that comprise the security and privacy capabilities. Assessors are aware of how the controls work together to provide such capabilities.\"" + }, + { + "reference": "FedRAMP Authorization Act (44 USC \u00a7 3609 (a) (1))", + "reference_url": "https://fedramp.gov/docs/authority/law/#a-roles-and-responsibilities", + "description": "requires that the Administrator of the General Services Administration shall \"in consultation with the [DHS] Secretary, develop, coordinate, and implement a process to support agency review, reuse, and standardization, where appropriate, of security assessments of cloud computing products and services...\" 44 USC \u00a7 3609 (c) (2) further states that \"the [GSA] Administrator shall establish a means for the automation of security assessments and reviews.\"", + "delegation": "These responsibilities are delegated to the FedRAMP Director", + "delegation_url": "https://www.gsa.gov/directives-library/gsa-delegations-of-authority-fedramp" + } + ], + "purpose": "Modern cloud services use automated or code-driven configuration management and control planes to ensure predictable, repeatable, reliable, and secure outcomes during deployment and operation. The majority of a service security assessment can take place continuously via automated validation for simple cloud-native services if the need for a traditional control-by-control narrative approach is removed.", + "expected_outcomes": [ + "Cloud service providers following commercial security best practices will be able to meet and validate FedRAMP security requirements with the application of simple changes and automated capabilities", + "Third-party independent assessors will have a simpler framework to assess security and implementation decisions based on engineering decisions in context", + "Federal agencies will be able to easily, quickly, and effectively review and consume security information about the service to make informed risk-based authorization to operate decisions based on their planned use case" + ] + } + }, + "FRR.KSI": {}, + "KSI": { + "CED": { + "id": "KSI-CED", + "name": "Cybersecurity Education", + "indicators": [ + { + "id": "KSI-CED-RAT", + "name": "Reviewing All Training", + "statement": "The effectiveness of relevant cybersecurity education and training is persistently reviewed, including at least general training for all employees, role-specific training for employees in high risk roles, training for development and engineering staff on secure software delivery, and training for staff involved with incident response or disaster recovery.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cp-3", + "title": "Contingency Training" + }, + { + "control_id": "ir-2", + "title": "Incident Response Training" + }, + { + "control_id": "ps-6", + "title": "Access Agreements" + }, + { + "control_id": "at-2", + "title": "Literacy Training and Awareness" + }, + { + "control_id": "at-2.2", + "title": "Insider Threat" + }, + { + "control_id": "at-2.3", + "title": "Social Engineering and Mining" + }, + { + "control_id": "at-3.5", + "title": "Processing Personally Identifiable Information" + }, + { + "control_id": "at-4", + "title": "Training Records" + }, + { + "control_id": "ir-2.3", + "title": "Breach" + }, + { + "control_id": "at-3", + "title": "Role-based Training" + }, + { + "control_id": "sr-11.1", + "title": "Anti-counterfeit Training" + } + ] + } + ] + }, + "CMT": { + "id": "KSI-CMT", + "name": "Change Management", + "indicators": [ + { + "id": "KSI-CMT-LMC", + "name": "Logging Changes", + "statement": "Modifications to the cloud service offering are logged and monitored.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "au-2", + "title": "Event Logging" + }, + { + "control_id": "cm-3", + "title": "Configuration Change Control" + }, + { + "control_id": "cm-3.2", + "title": "Testing, Validation, and Documentation of Changes" + }, + { + "control_id": "cm-4.2", + "title": "Verification of Controls" + }, + { + "control_id": "cm-6", + "title": "Configuration Settings" + }, + { + "control_id": "cm-8.3", + "title": "Automated Unauthorized Component Detection" + }, + { + "control_id": "ma-2", + "title": "Controlled Maintenance" + } + ] + }, + { + "id": "KSI-CMT-RMV", + "name": "Redeploying vs Modifying", + "statement": "Changes to machine-based information resources are executed through the redeployment of version controlled resources rather than direct modification wherever reasonable.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-2", + "title": "Baseline Configuration" + }, + { + "control_id": "cm-3", + "title": "Configuration Change Control" + }, + { + "control_id": "cm-5", + "title": "Access Restrictions for Change" + }, + { + "control_id": "cm-6", + "title": "Configuration Settings" + }, + { + "control_id": "cm-7", + "title": "Least Functionality" + }, + { + "control_id": "cm-8.1", + "title": "Updates During Installation and Removal" + }, + { + "control_id": "si-3", + "title": "Malicious Code Protection" + } + ] + }, + { + "id": "KSI-CMT-RVP", + "name": "Reviewing Change Procedures", + "statement": "The effectiveness of documented change management procedures is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-3", + "title": "Configuration Change Control" + }, + { + "control_id": "cm-3.2", + "title": "Testing, Validation, and Documentation of Changes" + }, + { + "control_id": "cm-3.4", + "title": "Security and Privacy Representatives" + }, + { + "control_id": "cm-5", + "title": "Access Restrictions for Change" + }, + { + "control_id": "cm-7.1", + "title": "Periodic Review" + }, + { + "control_id": "cm-9", + "title": "Configuration Management Plan" + } + ] + }, + { + "id": "KSI-CMT-VTD", + "name": "Validating Throughout Deployment", + "statement": "Persistent testing and validation of changes throughout deployment is automated.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-3", + "title": "Configuration Change Control" + }, + { + "control_id": "cm-3.2", + "title": "Testing, Validation, and Documentation of Changes" + }, + { + "control_id": "cm-4.2", + "title": "Verification of Controls" + }, + { + "control_id": "si-2", + "title": "Flaw Remediation" + } + ] + } + ] + }, + "CNA": { + "id": "KSI-CNA", + "name": "Cloud Native Architecture", + "indicators": [ + { + "id": "KSI-CNA-DFP", + "name": "Defining Functionality and Privileges", + "statement": "The functionality and privileges for infrastructure and services are strictly defined.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-2", + "title": "Baseline Configuration" + }, + { + "control_id": "si-3", + "title": "Malicious Code Protection" + } + ] + }, + { + "id": "KSI-CNA-EIS", + "name": "Enforcing Intended State", + "statement": "Automated services are used to persistently assess the security of all machine-based information resources and automatically enforce their intended operational state.", + "impact": { + "low": false, + "moderate": true + }, + "controls": [ + { + "control_id": "ca-2.1", + "title": "Independent Assessors" + }, + { + "control_id": "ca-7.1", + "title": "Independent Assessment" + } + ] + }, + { + "id": "KSI-CNA-IBP", + "name": "Implementing Best Practices", + "statement": "The use and configuration of third-party machine-based information resources is persistently compared against the original provider's best practices and guidance.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-17.3", + "title": "Managed Access Control Points" + }, + { + "control_id": "cm-2", + "title": "Baseline Configuration" + }, + { + "control_id": "pl-10", + "title": "Baseline Selection" + } + ] + }, + { + "id": "KSI-CNA-MAT", + "name": "Minimizing Attack Surface", + "statement": "Machine-based information resources are persistently reviewed to ensure they have a minimal attack surface and that lateral movement is minimized if compromised.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-17.3", + "title": "Managed Access Control Points" + }, + { + "control_id": "ac-18.1", + "title": "Authentication and Encryption" + }, + { + "control_id": "ac-18.3", + "title": "Disable Wireless Networking" + }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, + { + "control_id": "ca-9", + "title": "Internal System Connections" + }, + { + "control_id": "sc-7.3", + "title": "Access Points" + }, + { + "control_id": "sc-7.4", + "title": "External Telecommunications Services" + }, + { + "control_id": "sc-7.5", + "title": "Deny by Default \u2014 Allow by Exception" + }, + { + "control_id": "sc-7.8", + "title": "Route Traffic to Authenticated Proxy Servers" + }, + { + "control_id": "sc-8", + "title": "Transmission Confidentiality and Integrity" + }, + { + "control_id": "sc-10", + "title": "Network Disconnect" + }, + { + "control_id": "si-10", + "title": "Information Input Validation" + }, + { + "control_id": "si-11", + "title": "Error Handling" + }, + { + "control_id": "si-16", + "title": "Memory Protection" + } + ] + }, + { + "id": "KSI-CNA-OFA", + "name": "Optimizing for Availability", + "statement": "Machine-based information resources are persistently reviewed to ensure they are appropriately optimized for high availability and rapid recovery.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [] + }, + { + "id": "KSI-CNA-RNT", + "name": "Restricting Network Traffic", + "statement": "Machine-based information resources are persistently reviewed to ensure they are appropriately configured to limit inbound and outbound network traffic.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-17.3", + "title": "Managed Access Control Points" + }, + { + "control_id": "ca-9", + "title": "Internal System Connections" + }, + { + "control_id": "cm-7.1", + "title": "Periodic Review" + }, + { + "control_id": "sc-7.5", + "title": "Deny by Default \u2014 Allow by Exception" + }, + { + "control_id": "si-8", + "title": "Spam Protection" + } + ] + }, + { + "id": "KSI-CNA-RVP", + "name": "Reviewing Protections", + "statement": "The effectiveness of protection against denial of service attacks and other unwanted activity for machine-based information resources is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "sc-5", + "title": "Denial-of-service Protection" + }, + { + "control_id": "si-8", + "title": "Spam Protection" + }, + { + "control_id": "si-8.2", + "title": "Automatic Updates" + } + ] + }, + { + "id": "KSI-CNA-ULN", + "name": "Using Logical Networking", + "statement": "Logical networking and related capabilities are used and persistently reviewed to enforce traffic flow controls.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-12", + "title": "Session Termination" + }, + { + "control_id": "ac-17.3", + "title": "Managed Access Control Points" + }, + { + "control_id": "ca-9", + "title": "Internal System Connections" + }, + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + }, + { + "control_id": "sc-7", + "title": "Boundary Protection" + }, + { + "control_id": "sc-7.7", + "title": "Split Tunneling for Remote Devices" + }, + { + "control_id": "sc-8", + "title": "Transmission Confidentiality and Integrity" + }, + { + "control_id": "sc-10", + "title": "Network Disconnect" + } + ] + } + ] + }, + "IAM": { + "id": "KSI-IAM", + "name": "Identity and Access Management", + "indicators": [ + { + "id": "KSI-IAM-AAM", + "name": "Automating Account Management", + "statement": "The lifecycle and privileges of all accounts, roles, and groups are securely managed using automation.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2.2", + "title": "Automated Temporary and Emergency Account Management" + }, + { + "control_id": "ac-2.3", + "title": "Disable Accounts" + }, + { + "control_id": "ac-2.13", + "title": "Disable Accounts for High-risk Individuals" + }, + { + "control_id": "ac-6.7", + "title": "Review of User Privileges" + }, + { + "control_id": "ia-4.4", + "title": "Identify User Status" + }, + { + "control_id": "ia-12", + "title": "Identity Proofing" + }, + { + "control_id": "ia-12.2", + "title": "Identity Evidence" + }, + { + "control_id": "ia-12.3", + "title": "Identity Evidence Validation and Verification" + }, + { + "control_id": "ia-12.5", + "title": "Address Confirmation" + } + ] + }, + { + "id": "KSI-IAM-APM", + "name": "Adopting Passwordless Methods", + "statement": "Secure passwordless methods are used for user authentication and authorization when feasible, otherwise strong passwords with phishing-resistant MFA is used.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-3", + "title": "Access Enforcement" + }, + { + "control_id": "ia-5.1", + "title": "Password-based Authentication" + }, + { + "control_id": "ia-5.2", + "title": "Public Key-based Authentication" + }, + { + "control_id": "ia-5.6", + "title": "Protection of Authenticators" + }, + { + "control_id": "ia-6", + "title": "Authentication Feedback" + }, + { + "control_id": "ac-2", + "title": "Account Management" + }, + { + "control_id": "ia-2", + "title": "Identification and Authentication (Organizational Users)" + }, + { + "control_id": "ia-2.1", + "title": "Multi-factor Authentication to Privileged Accounts" + }, + { + "control_id": "ia-2.2", + "title": "Multi-factor Authentication to Non-privileged Accounts" + }, + { + "control_id": "ia-2.8", + "title": "Access to Accounts \u2014 Replay Resistant" + }, + { + "control_id": "ia-5", + "title": "Authenticator Management" + }, + { + "control_id": "ia-8", + "title": "Identification and Authentication (Non-organizational Users)" + }, + { + "control_id": "sc-23", + "title": "Session Authenticity" + } + ] + }, + { + "id": "KSI-IAM-ELP", + "name": "Ensuring Least Privilege", + "statement": "Identity and access management measures are used and persistently reviewed to ensure each user or device can only access the resources they need.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2.5", + "title": "Inactivity Logout" + }, + { + "control_id": "ac-2.6", + "title": "Dynamic Privilege Management" + }, + { + "control_id": "ac-3", + "title": "Access Enforcement" + }, + { + "control_id": "ac-4", + "title": "Information Flow Enforcement" + }, + { + "control_id": "ac-6", + "title": "Least Privilege" + }, + { + "control_id": "ac-12", + "title": "Session Termination" + }, + { + "control_id": "ac-14", + "title": "Permitted Actions Without Identification or Authentication" + }, + { + "control_id": "ac-17", + "title": "Remote Access" + }, + { + "control_id": "ac-17.1", + "title": "Monitoring and Control" + }, + { + "control_id": "ac-17.2", + "title": "Protection of Confidentiality and Integrity Using Encryption" + }, + { + "control_id": "ac-17.3", + "title": "Managed Access Control Points" + }, + { + "control_id": "ac-20", + "title": "Use of External Systems" + }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, + { + "control_id": "cm-2.7", + "title": "Configure Systems and Components for High-risk Areas" + }, + { + "control_id": "cm-9", + "title": "Configuration Management Plan" + }, + { + "control_id": "ia-2", + "title": "Identification and Authentication (Organizational Users)" + }, + { + "control_id": "ia-3", + "title": "Device Identification and Authentication" + }, + { + "control_id": "ia-4", + "title": "Identifier Management" + }, + { + "control_id": "ia-4.4", + "title": "Identify User Status" + }, + { + "control_id": "ia-5.2", + "title": "Public Key-based Authentication" + }, + { + "control_id": "ia-5.6", + "title": "Protection of Authenticators" + }, + { + "control_id": "ia-11", + "title": "Re-authentication" + }, + { + "control_id": "ps-2", + "title": "Position Risk Designation" + }, + { + "control_id": "ps-3", + "title": "Personnel Screening" + }, + { + "control_id": "ps-4", + "title": "Personnel Termination" + }, + { + "control_id": "ps-5", + "title": "Personnel Transfer" + }, + { + "control_id": "ps-6", + "title": "Access Agreements" + }, + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + }, + { + "control_id": "sc-20", + "title": "Secure Name/Address Resolution Service (Authoritative Source)" + }, + { + "control_id": "sc-21", + "title": "Secure Name/Address Resolution Service (Recursive or Caching Resolver)" + }, + { + "control_id": "sc-22", + "title": "Architecture and Provisioning for Name/Address Resolution Service" + }, + { + "control_id": "sc-23", + "title": "Session Authenticity" + }, + { + "control_id": "sc-39", + "title": "Process Isolation" + }, + { + "control_id": "si-3", + "title": "Malicious Code Protection" + } + ] + }, + { + "id": "KSI-IAM-JIT", + "name": "Authorizing Just-in-Time", + "statement": "A least-privileged, role and attribute-based, and just-in-time security authorization model is used and persistently reviewed for all user and non-user accounts and services.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2", + "title": "Account Management" + }, + { + "control_id": "ac-2.1", + "title": "Automated System Account Management" + }, + { + "control_id": "ac-2.2", + "title": "Automated Temporary and Emergency Account Management" + }, + { + "control_id": "ac-2.3", + "title": "Disable Accounts" + }, + { + "control_id": "ac-2.4", + "title": "Automated Audit Actions" + }, + { + "control_id": "ac-2.6", + "title": "Dynamic Privilege Management" + }, + { + "control_id": "ac-3", + "title": "Access Enforcement" + }, + { + "control_id": "ac-4", + "title": "Information Flow Enforcement" + }, + { + "control_id": "ac-5", + "title": "Separation of Duties" + }, + { + "control_id": "ac-6", + "title": "Least Privilege" + }, + { + "control_id": "ac-6.1", + "title": "Authorize Access to Security Functions" + }, + { + "control_id": "ac-6.2", + "title": "Non-privileged Access for Nonsecurity Functions" + }, + { + "control_id": "ac-6.5", + "title": "Privileged Accounts" + }, + { + "control_id": "ac-6.7", + "title": "Review of User Privileges" + }, + { + "control_id": "ac-6.9", + "title": "Log Use of Privileged Functions" + }, + { + "control_id": "ac-6.10", + "title": "Prohibit Non-privileged Users from Executing Privileged Functions" + }, + { + "control_id": "ac-7", + "title": "Unsuccessful Logon Attempts" + }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, + { + "control_id": "ac-17", + "title": "Remote Access" + }, + { + "control_id": "au-9.4", + "title": "Access by Subset of Privileged Users" + }, + { + "control_id": "cm-5", + "title": "Access Restrictions for Change" + }, + { + "control_id": "cm-7", + "title": "Least Functionality" + }, + { + "control_id": "cm-7.2", + "title": "Prevent Program Execution" + }, + { + "control_id": "cm-7.5", + "title": "Authorized Software \u2014 Allow-by-exception" + }, + { + "control_id": "cm-9", + "title": "Configuration Management Plan" + }, + { + "control_id": "ia-4", + "title": "Identifier Management" + }, + { + "control_id": "ia-4.4", + "title": "Identify User Status" + }, + { + "control_id": "ia-7", + "title": "Cryptographic Module Authentication" + }, + { + "control_id": "ps-2", + "title": "Position Risk Designation" + }, + { + "control_id": "ps-3", + "title": "Personnel Screening" + }, + { + "control_id": "ps-4", + "title": "Personnel Termination" + }, + { + "control_id": "ps-5", + "title": "Personnel Transfer" + }, + { + "control_id": "ps-6", + "title": "Access Agreements" + }, + { + "control_id": "ps-9", + "title": "Position Descriptions" + }, + { + "control_id": "ra-5.5", + "title": "Privileged Access" + }, + { + "control_id": "sc-2", + "title": "Separation of System and User Functionality" + }, + { + "control_id": "sc-23", + "title": "Session Authenticity" + }, + { + "control_id": "sc-39", + "title": "Process Isolation" + } + ] + }, + { + "id": "KSI-IAM-SNU", + "name": "Securing Non-User Authentication", + "statement": "Appropriately secure authentication methods are used and persistently reviewed for non-user accounts and services.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2", + "title": "Account Management" + }, + { + "control_id": "ac-2.2", + "title": "Automated Temporary and Emergency Account Management" + }, + { + "control_id": "ac-4", + "title": "Information Flow Enforcement" + }, + { + "control_id": "ac-6.5", + "title": "Privileged Accounts" + }, + { + "control_id": "ia-3", + "title": "Device Identification and Authentication" + }, + { + "control_id": "ia-5.2", + "title": "Public Key-based Authentication" + }, + { + "control_id": "ra-5.5", + "title": "Privileged Access" + } + ] + }, + { + "id": "KSI-IAM-SUS", + "name": "Responding to Suspicious Activity", + "statement": "Accounts with privileged access are disabled or otherwise secured in response to suspicious activity.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2", + "title": "Account Management" + }, + { + "control_id": "ac-2.1", + "title": "Automated System Account Management" + }, + { + "control_id": "ac-2.3", + "title": "Disable Accounts" + }, + { + "control_id": "ac-2.13", + "title": "Disable Accounts for High-risk Individuals" + }, + { + "control_id": "ac-7", + "title": "Unsuccessful Logon Attempts" + }, + { + "control_id": "ps-4", + "title": "Personnel Termination" + }, + { + "control_id": "ps-8", + "title": "Personnel Sanctions" + } + ] + } + ] + }, + "INR": { + "id": "KSI-INR", + "name": "Incident Response", + "indicators": [ + { + "id": "KSI-INR-AAR", + "name": "Generating After Action Reports", + "statement": "Incident after action reports are generated and lessons learned are persistently incorporated.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ir-3", + "title": "Incident Response Testing" + }, + { + "control_id": "ir-4", + "title": "Incident Handling" + }, + { + "control_id": "ir-4.1", + "title": "Automated Incident Handling Processes" + }, + { + "control_id": "ir-8", + "title": "Incident Response Plan" + } + ] + }, + { + "id": "KSI-INR-RIR", + "name": "Reviewing Incident Response Procedures", + "statement": "The effectiveness of documented incident response procedures is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ir-4", + "title": "Incident Handling" + }, + { + "control_id": "ir-4.1", + "title": "Automated Incident Handling Processes" + }, + { + "control_id": "ir-6", + "title": "Incident Reporting" + }, + { + "control_id": "ir-6.1", + "title": "Automated Reporting" + }, + { + "control_id": "ir-6.3", + "title": "Supply Chain Coordination" + }, + { + "control_id": "ir-7", + "title": "Incident Response Assistance" + }, + { + "control_id": "ir-7.1", + "title": "Automation Support for Availability of Information and Support" + }, + { + "control_id": "ir-8", + "title": "Incident Response Plan" + }, + { + "control_id": "ir-8.1", + "title": "Breaches" + }, + { + "control_id": "si-4.5", + "title": "System-generated Alerts" + } + ] + }, + { + "id": "KSI-INR-RPI", + "name": "Reviewing Past Incidents", + "statement": "Past incidents are persistently reviewed for patterns or vulnerabilities that were not previously apparent or identified.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ir-3", + "title": "Incident Response Testing" + }, + { + "control_id": "ir-4", + "title": "Incident Handling" + }, + { + "control_id": "ir-4.1", + "title": "Automated Incident Handling Processes" + }, + { + "control_id": "ir-5", + "title": "Incident Monitoring" + }, + { + "control_id": "ir-8", + "title": "Incident Response Plan" + } + ] + } + ] + }, + "MLA": { + "id": "KSI-MLA", + "name": "Monitoring, Logging, and Auditing", + "indicators": [ + { + "id": "KSI-MLA-ALA", + "name": "Authorizing Log Access", + "statement": "A least-privileged, role and attribute-based, and just-in-time access authorization model is used and persistently reviewed for access to log data based on organizationally defined data sensitivity.", + "impact": { + "low": false, + "moderate": true + }, + "controls": [ + { + "control_id": "si-11", + "title": "Error Handling" + } + ] + }, + { + "id": "KSI-MLA-EVC", + "name": "Evaluating Configurations", + "statement": "The configuration of machine-based information resources, especially infrastructure as code, is persistently evaluated and tested.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ca-7", + "title": "Continuous Monitoring" + }, + { + "control_id": "cm-2", + "title": "Baseline Configuration" + }, + { + "control_id": "cm-6", + "title": "Configuration Settings" + }, + { + "control_id": "si-7.7", + "title": "Integration of Detection and Response" + } + ] + }, + { + "id": "KSI-MLA-LET", + "name": "Logging Event Types", + "statement": "A list of information resources and event types that will be logged, monitored, and audited is maintained and persistently reviewed to ensure these activities occur.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2.4", + "title": "Automated Audit Actions" + }, + { + "control_id": "ac-6.9", + "title": "Log Use of Privileged Functions" + }, + { + "control_id": "ac-17.1", + "title": "Monitoring and Control" + }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, + { + "control_id": "au-2", + "title": "Event Logging" + }, + { + "control_id": "au-7.1", + "title": "Automatic Processing" + }, + { + "control_id": "au-12", + "title": "Audit Record Generation" + }, + { + "control_id": "si-4.4", + "title": "Inbound and Outbound Communications Traffic" + }, + { + "control_id": "si-4.5", + "title": "System-generated Alerts" + }, + { + "control_id": "si-7.7", + "title": "Integration of Detection and Response" + } + ] + }, + { + "id": "KSI-MLA-OSM", + "name": "Operating SIEM Capability", + "statement": "A Security Information and Event Management (SIEM) or similar system(s) is used and persistently reviewed for centralized, tamper-resistant logging of events, activities, and changes.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-17.1", + "title": "Monitoring and Control" + }, + { + "control_id": "ac-20.1", + "title": "Limits on Authorized Use" + }, + { + "control_id": "au-2", + "title": "Event Logging" + }, + { + "control_id": "au-3", + "title": "Content of Audit Records" + }, + { + "control_id": "au-3.1", + "title": "Additional Audit Information" + }, + { + "control_id": "au-4", + "title": "Audit Log Storage Capacity" + }, + { + "control_id": "au-5", + "title": "Response to Audit Logging Process Failures" + }, + { + "control_id": "au-6.1", + "title": "Automated Process Integration" + }, + { + "control_id": "au-6.3", + "title": "Correlate Audit Record Repositories" + }, + { + "control_id": "au-7", + "title": "Audit Record Reduction and Report Generation" + }, + { + "control_id": "au-7.1", + "title": "Automatic Processing" + }, + { + "control_id": "au-8", + "title": "Time Stamps" + }, + { + "control_id": "au-9", + "title": "Protection of Audit Information" + }, + { + "control_id": "au-11", + "title": "Audit Record Retention" + }, + { + "control_id": "ir-4.1", + "title": "Automated Incident Handling Processes" + }, + { + "control_id": "si-4.2", + "title": "Automated Tools and Mechanisms for Real-time Analysis" + }, + { + "control_id": "si-4.4", + "title": "Inbound and Outbound Communications Traffic" + }, + { + "control_id": "si-7.7", + "title": "Integration of Detection and Response" + } + ] + }, + { + "id": "KSI-MLA-RVL", + "name": "Reviewing Logs", + "statement": "Logs are persistently reviewed and audited.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2.4", + "title": "Automated Audit Actions" + }, + { + "control_id": "ac-6.9", + "title": "Log Use of Privileged Functions" + }, + { + "control_id": "au-2", + "title": "Event Logging" + }, + { + "control_id": "au-6", + "title": "Audit Record Review, Analysis, and Reporting" + }, + { + "control_id": "au-6.1", + "title": "Automated Process Integration" + }, + { + "control_id": "si-4", + "title": "System Monitoring" + }, + { + "control_id": "si-4.4", + "title": "Inbound and Outbound Communications Traffic" + } + ] + } + ] + }, + "PIY": { + "id": "KSI-PIY", + "name": "Policy and Inventory", + "indicators": [ + { + "id": "KSI-PIY-GIV", + "name": "Generating Inventories", + "statement": "Authoritative sources are used to automatically generate real-time inventories of all information resources when needed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-2.2", + "title": "Automation Support for Accuracy and Currency" + }, + { + "control_id": "cm-7.5", + "title": "Authorized Software \u2014 Allow-by-exception" + }, + { + "control_id": "cm-8", + "title": "System Component Inventory" + }, + { + "control_id": "cm-8.1", + "title": "Updates During Installation and Removal" + }, + { + "control_id": "cm-12", + "title": "Information Location" + }, + { + "control_id": "cm-12.1", + "title": "Automated Tools to Support Information Location" + }, + { + "control_id": "cp-2.8", + "title": "Identify Critical Assets" + } + ] + }, + { + "id": "KSI-PIY-RES", + "name": "Reviewing Executive Support", + "statement": "Executive support for achieving the provider's security goals is persistently reviewed and demonstrated.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [] + }, + { + "id": "KSI-PIY-RIS", + "name": "Reviewing Investments in Security", + "statement": "The effectiveness of the provider's investments in achieving security goals is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-5", + "title": "Separation of Duties" + }, + { + "control_id": "ca-2", + "title": "Control Assessments" + }, + { + "control_id": "cp-2.1", + "title": "Coordinate with Related Plans" + }, + { + "control_id": "cp-4.1", + "title": "Coordinate with Related Plans" + }, + { + "control_id": "ir-3.2", + "title": "Coordination with Related Plans" + }, + { + "control_id": "pm-3", + "title": "Information Security and Privacy Resources" + }, + { + "control_id": "sa-2", + "title": "Allocation of Resources" + }, + { + "control_id": "sa-3", + "title": "System Development Life Cycle" + }, + { + "control_id": "sr-2.1", + "title": "Establish SCRM Team" + } + ] + }, + { + "id": "KSI-PIY-RSD", + "name": "Reviewing Security in the SDLC", + "statement": "The effectiveness of building security and privacy considerations into the Software Development Lifecycle and aligning with CISA Secure By Design principles is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-5", + "title": "Separation of Duties" + }, + { + "control_id": "au-3.3", + "title": "Limit Personally Identifiable Information Elements" + }, + { + "control_id": "cm-3.4", + "title": "Security and Privacy Representatives" + }, + { + "control_id": "pl-8", + "title": "Security and Privacy Architectures" + }, + { + "control_id": "pm-7", + "title": "Enterprise Architecture" + }, + { + "control_id": "sa-3", + "title": "System Development Life Cycle" + }, + { + "control_id": "sa-8", + "title": "Security and Privacy Engineering Principles" + }, + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + }, + { + "control_id": "sc-18", + "title": "Mobile Code" + }, + { + "control_id": "si-10", + "title": "Information Input Validation" + }, + { + "control_id": "si-11", + "title": "Error Handling" + }, + { + "control_id": "si-16", + "title": "Memory Protection" + } + ] + }, + { + "id": "KSI-PIY-RVD", + "name": "Reviewing Vulnerability Disclosures", + "statement": "The effectiveness of the provider's vulnerability disclosure program is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ra-5.11", + "title": "Public Disclosure Program" + } + ] + } + ] + }, + "RPL": { + "id": "KSI-RPL", + "name": "Recovery Planning", + "indicators": [ + { + "id": "KSI-RPL-ABO", + "name": "Aligning Backups with Objectives", + "statement": "The alignment of machine-based information resource backups with defined recovery objectives is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-2.3", + "title": "Retention of Previous Configurations" + }, + { + "control_id": "cp-6", + "title": "Alternate Storage Site" + }, + { + "control_id": "cp-9", + "title": "System Backup" + }, + { + "control_id": "cp-10", + "title": "System Recovery and Reconstitution" + }, + { + "control_id": "cp-10.2", + "title": "Transaction Recovery" + }, + { + "control_id": "si-12", + "title": "Information Management and Retention" + } + ] + }, + { + "id": "KSI-RPL-ARP", + "name": "Aligning Recovery Plan", + "statement": "The alignment of recovery plans with defined recovery objectives is persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cp-2", + "title": "Contingency Plan" + }, + { + "control_id": "cp-2.1", + "title": "Coordinate with Related Plans" + }, + { + "control_id": "cp-2.3", + "title": "Resume Mission and Business Functions" + }, + { + "control_id": "cp-4.1", + "title": "Coordinate with Related Plans" + }, + { + "control_id": "cp-6", + "title": "Alternate Storage Site" + }, + { + "control_id": "cp-6.1", + "title": "Separation from Primary Site" + }, + { + "control_id": "cp-6.3", + "title": "Accessibility" + }, + { + "control_id": "cp-7", + "title": "Alternate Processing Site" + }, + { + "control_id": "cp-7.1", + "title": "Separation from Primary Site" + }, + { + "control_id": "cp-7.2", + "title": "Accessibility" + }, + { + "control_id": "cp-7.3", + "title": "Priority of Service" + }, + { + "control_id": "cp-8", + "title": "Telecommunications Services" + }, + { + "control_id": "cp-8.1", + "title": "Priority of Service Provisions" + }, + { + "control_id": "cp-8.2", + "title": "Single Points of Failure" + }, + { + "control_id": "cp-10", + "title": "System Recovery and Reconstitution" + }, + { + "control_id": "cp-10.2", + "title": "Transaction Recovery" + } + ] + }, + { + "id": "KSI-RPL-RRO", + "name": "Reviewing Recovery Objectives", + "statement": "The desired Recovery Time Objectives (RTO) and Recovery Point Objectives (RPO) are defined and persistently reviewed for alignment with the provider's business needs and capabilities.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cp-2.3", + "title": "Resume Mission and Business Functions" + }, + { + "control_id": "cp-10", + "title": "System Recovery and Reconstitution" + } + ] + }, + { + "id": "KSI-RPL-TRC", + "name": "Testing Recovery Capabilities", + "statement": "The capability to recover from incidents and contingencies aligned with defined recovery objectives is persistently tested.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cp-2.1", + "title": "Coordinate with Related Plans" + }, + { + "control_id": "cp-2.3", + "title": "Resume Mission and Business Functions" + }, + { + "control_id": "cp-4", + "title": "Contingency Plan Testing" + }, + { + "control_id": "cp-4.1", + "title": "Coordinate with Related Plans" + }, + { + "control_id": "cp-6", + "title": "Alternate Storage Site" + }, + { + "control_id": "cp-6.1", + "title": "Separation from Primary Site" + }, + { + "control_id": "cp-9.1", + "title": "Testing for Reliability and Integrity" + }, + { + "control_id": "cp-10", + "title": "System Recovery and Reconstitution" + }, + { + "control_id": "ir-3", + "title": "Incident Response Testing" + }, + { + "control_id": "ir-3.2", + "title": "Coordination with Related Plans" + } + ] + } + ] + }, + "SCR": { + "id": "KSI-SCR", + "name": "Supply Chain Risk", + "indicators": [ + { + "id": "KSI-SCR-MIT", + "name": "Mitigating Supply Chain Risk", + "statement": "Persistently identify, review, and mitigate potential supply chain risks.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-20", + "title": "Use of External Systems" + }, + { + "control_id": "ra-3.1", + "title": "Supply Chain Risk Assessment" + }, + { + "control_id": "sa-9", + "title": "External System Services" + }, + { + "control_id": "sa-10", + "title": "Developer Configuration Management" + }, + { + "control_id": "sa-11", + "title": "Developer Testing and Evaluation" + }, + { + "control_id": "sa-15.3", + "title": "Criticality Analysis" + }, + { + "control_id": "sa-22", + "title": "Unsupported System Components" + }, + { + "control_id": "si-7.1", + "title": "Integrity Checks" + }, + { + "control_id": "sr-5", + "title": "Acquisition Strategies, Tools, and Methods" + }, + { + "control_id": "sr-6", + "title": "Supplier Assessments and Reviews" + }, + { + "control_id": "ca-7.4", + "title": "Risk Monitoring" + }, + { + "control_id": "sc-18", + "title": "Mobile Code" + } + ] + }, + { + "id": "KSI-SCR-MON", + "name": "Monitoring Supply Chain Risk", + "statement": "Third party software information resources are automatically monitored for upstream vulnerabilities using mechanisms that may include contractual notification requirements or active monitoring services.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-20", + "title": "Use of External Systems" + }, + { + "control_id": "ca-3", + "title": "Information Exchange" + }, + { + "control_id": "ir-6.3", + "title": "Supply Chain Coordination" + }, + { + "control_id": "ps-7", + "title": "External Personnel Security" + }, + { + "control_id": "ra-5", + "title": "Vulnerability Monitoring and Scanning" + }, + { + "control_id": "sa-9", + "title": "External System Services" + }, + { + "control_id": "si-5", + "title": "Security Alerts, Advisories, and Directives" + }, + { + "control_id": "sr-5", + "title": "Acquisition Strategies, Tools, and Methods" + }, + { + "control_id": "sr-6", + "title": "Supplier Assessments and Reviews" + }, + { + "control_id": "sr-8", + "title": "Notification Agreements" + } + ] + } + ] + }, + "SVC": { + "id": "KSI-SVC", + "name": "Service Configuration", + "indicators": [ + { + "id": "KSI-SVC-ACM", + "name": "Automating Configuration Management", + "statement": "The configuration of machine-based information resources is managed using automation and persistently reviewed for drift.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-2.4", + "title": "Automated Audit Actions" + }, + { + "control_id": "cm-2", + "title": "Baseline Configuration" + }, + { + "control_id": "cm-2.2", + "title": "Automation Support for Accuracy and Currency" + }, + { + "control_id": "cm-2.3", + "title": "Retention of Previous Configurations" + }, + { + "control_id": "cm-6", + "title": "Configuration Settings" + }, + { + "control_id": "cm-7.1", + "title": "Periodic Review" + }, + { + "control_id": "pl-9", + "title": "Central Management" + }, + { + "control_id": "pl-10", + "title": "Baseline Selection" + }, + { + "control_id": "sa-5", + "title": "System Documentation" + }, + { + "control_id": "si-5", + "title": "Security Alerts, Advisories, and Directives" + }, + { + "control_id": "sr-10", + "title": "Inspection of Systems or Components" + } + ] + }, + { + "id": "KSI-SVC-ASM", + "name": "Automating Secret Management", + "statement": "Management, protection, and regular rotation of digital keys, certificates, and other secrets is automated and persistently reviewed.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-17.2", + "title": "Protection of Confidentiality and Integrity Using Encryption" + }, + { + "control_id": "ia-5.2", + "title": "Public Key-based Authentication" + }, + { + "control_id": "ia-5.6", + "title": "Protection of Authenticators" + }, + { + "control_id": "sc-12", + "title": "Cryptographic Key Establishment and Management" + }, + { + "control_id": "sc-17", + "title": "Public Key Infrastructure Certificates" + } + ] + }, + { + "id": "KSI-SVC-EIS", + "name": "Evaluating and Improving Security", + "statement": "Information resources are persistently evaluated for opportunities to improve security and those improvements are persistently made.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-7.1", + "title": "Periodic Review" + }, + { + "control_id": "cm-12.1", + "title": "Automated Tools to Support Information Location" + }, + { + "control_id": "ma-2", + "title": "Controlled Maintenance" + }, + { + "control_id": "pl-8", + "title": "Security and Privacy Architectures" + }, + { + "control_id": "sc-7", + "title": "Boundary Protection" + }, + { + "control_id": "sc-39", + "title": "Process Isolation" + }, + { + "control_id": "si-2.2", + "title": "Automated Flaw Remediation Status" + }, + { + "control_id": "si-4", + "title": "System Monitoring" + }, + { + "control_id": "sr-10", + "title": "Inspection of Systems or Components" + } + ] + }, + { + "id": "KSI-SVC-PRR", + "name": "Preventing Residual Risk", + "statement": "Plans, procedures, and the state of information resources are persistently reviewed after making changes to limit and remove unwanted residual elements that would likely negatively affect the confidentiality, integrity, or availability of federal customer data.", + "impact": { + "low": false, + "moderate": true + }, + "controls": [ + { + "control_id": "sc-4", + "title": "Information in Shared System Resources" + } + ] + }, + { + "id": "KSI-SVC-RUD", + "name": "Removing Unwanted Data", + "statement": "Unwanted federal customer data is removed promptly when requested by an agency in alignment with customer agreements, including from backups if appropriate; this typically applies when a customer spills information or when a customer seeks to remove information from a service due to a change in usage.", + "impact": { + "low": false, + "moderate": true + }, + "controls": [ + { + "control_id": "si-12.3", + "title": "Information Disposal" + }, + { + "control_id": "si-18.4", + "title": "Individual Requests" + } + ] + }, + { + "id": "KSI-SVC-SIN", + "name": "Securing Information", + "statement": "Information is encrypted or otherwise secured from unwanted access or modification.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "ac-1", + "title": "Policy and Procedures" + }, + { + "control_id": "ac-17.2", + "title": "Protection of Confidentiality and Integrity Using Encryption" + }, + { + "control_id": "cp-9.8", + "title": "Cryptographic Protection" + }, + { + "control_id": "sc-8", + "title": "Transmission Confidentiality and Integrity" + }, + { + "control_id": "sc-8.1", + "title": "Cryptographic Protection" + }, + { + "control_id": "sc-13", + "title": "Cryptographic Protection" + }, + { + "control_id": "sc-20", + "title": "Secure Name/Address Resolution Service (Authoritative Source)" + }, + { + "control_id": "sc-21", + "title": "Secure Name/Address Resolution Service (Recursive or Caching Resolver)" + }, + { + "control_id": "sc-22", + "title": "Architecture and Provisioning for Name/Address Resolution Service" + }, + { + "control_id": "sc-23", + "title": "Session Authenticity" + }, + { + "control_id": "sc-28", + "title": "Protection of Information at Rest" + }, + { + "control_id": "sc-28.1", + "title": "Cryptographic Protection" + } + ] + }, + { + "id": "KSI-SVC-VCM", + "name": "Validating Communications", + "statement": "The authenticity and integrity of communications between machine-based information resources is persistently validated using automation.", + "impact": { + "low": false, + "moderate": true + }, + "controls": [ + { + "control_id": "sc-23", + "title": "Session Authenticity" + }, + { + "control_id": "si-7.1", + "title": "Integrity Checks" + } + ] + }, + { + "id": "KSI-SVC-VRI", + "name": "Validating Resource Integrity", + "statement": "Use cryptographic methods to validate the integrity of machine-based information resources.", + "impact": { + "low": true, + "moderate": true + }, + "controls": [ + { + "control_id": "cm-2.2", + "title": "Automation Support for Accuracy and Currency" + }, + { + "control_id": "cm-8.3", + "title": "Automated Unauthorized Component Detection" + }, + { + "control_id": "sc-13", + "title": "Cryptographic Protection" + }, + { + "control_id": "sc-23", + "title": "Session Authenticity" + }, + { + "control_id": "si-7", + "title": "Software, Firmware, and Information Integrity" + }, + { + "control_id": "si-7.1", + "title": "Integrity Checks" + }, + { + "control_id": "sr-10", + "title": "Inspection of Systems or Components" + } + ] + } + ] + } + } +} diff --git a/tests/test_controls.py b/tests/test_controls.py index fa0aae40..75e0e53a 100644 --- a/tests/test_controls.py +++ b/tests/test_controls.py @@ -26,6 +26,11 @@ REPO = Path(__file__).resolve().parent.parent FIX_SIGNAL = REPO / "tests" / "fixtures" / "clean" / "ksi-signal.json" CATALOG = REPO / "infrastructure" / "schemas" / "ksi-catalog.json" +# The KSI catalog as it stood when the SSP was retired (FedRAMP's mappings before +# its 2026-10-13 revision). The parity test runs the projection on it, so the +# projection's code stays pinned to the SSP it replaced; the current catalog is +# FedRAMP's current mapping (tests/test_ksi_catalog_corpus.py). +CATALOG_AT_RETIREMENT = REPO / "tests" / "fixtures" / "migration" / "ksi-catalog-2026-11-05.json" FROZEN = REPO / "tests" / "fixtures" / "migration" / "ssp-control-status-2026-11-05.json" sys.path.insert(0, str(REPO / "scripts")) @@ -48,7 +53,7 @@ def _projection(): def test_the_projection_matches_the_last_ssp_control_for_control(): frozen = json.loads(FROZEN.read_text()) - irs = _projection() + irs = _controls.load(FIX_SIGNAL, CATALOG_AT_RETIREMENT) assert len(irs) == frozen["implemented_requirements"] == 333 got = {cid: list(v) for cid, v in _controls.status_map(irs).items()} assert got == frozen["controls"] diff --git a/tests/test_inject_figures.py b/tests/test_inject_figures.py index 2973c66d..76861df0 100644 --- a/tests/test_inject_figures.py +++ b/tests/test_inject_figures.py @@ -47,13 +47,14 @@ def test_figures_reproduce_published_numbers(): # hub split: hand-written + family-default = total assert int(f["hub_handwritten"]) + int(f["hub_generated"]) == int(f["hub_total"]) assert f["hub_handwritten"] == "116" - assert f["hub_total"] == "333" + assert f["hub_total"] == "382" # FedRAMP Moderate stack sums to the total assert int(f["moderate_implemented"]) + int(f["moderate_inherited"]) + int(f["moderate_na"]) == int(f["hub_total"]) # coverage is measured against the published baseline, not the hub total assert f["moderate_baseline"] == "323" assert f["moderate_coverage"] == "323/323" - assert f["moderate_beyond"] == "10" + # controls FedRAMP's KSI mappings reach beyond Moderate (2026-10-13 revision) + assert f["moderate_beyond"] == "59" assert int(f["moderate_baseline"]) + int(f["moderate_beyond"]) == int(f["hub_total"]) # spokes assert f["govramp_coverage"] == "339/339" @@ -126,7 +127,7 @@ def test_altered_projection_count_flows_to_html(monkeypatch): # hand-written count is unchanged (component def untouched), so generated grows assert f["hub_generated"] == str(baseline + 1 - int(f["hub_handwritten"])) assert f["moderate_coverage"] == "323/323" - assert f["moderate_beyond"] == "11" + assert f["moderate_beyond"] == "60" # and that new value actually lands in the marked HTML marked = '331' diff --git a/tests/test_ksi_catalog_corpus.py b/tests/test_ksi_catalog_corpus.py new file mode 100644 index 00000000..6fa3d396 --- /dev/null +++ b/tests/test_ksi_catalog_corpus.py @@ -0,0 +1,57 @@ +"""The KSI catalog is FedRAMP's: IDs, names, Class C statements and KSI->control +mappings equal the pinned Consolidated Rules (scripts/sync-ksi-catalog.py).""" + +import copy +import importlib.util +import json +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +spec = importlib.util.spec_from_file_location("sync_ksi", REPO / "scripts" / "sync-ksi-catalog.py") +sync = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sync) +BEFORE = REPO / "tests" / "fixtures" / "migration" / "ksi-catalog-2026-11-05.json" + + +def _catalog(): + return json.loads(sync.CATALOG.read_text()) + + +def test_the_committed_catalog_equals_the_pinned_rules(): + assert sync.main(["--check"]) == 0 + + +def test_a_mapping_that_drifts_from_the_rules_fails(): + """Known-bad: the catalog as committed before FedRAMP's 2026-10-13 mapping + revision was carried (40 of 46 KSIs differ).""" + stale = json.loads(BEFORE.read_text()) + assert sync.derive(stale) != stale + + +def test_a_dropped_control_is_detected(): + cat = _catalog() + bad = copy.deepcopy(cat) + bad["KSI"]["CED"]["indicators"][0]["controls"].pop() + assert sync.derive(bad) != bad + + +def test_class_c_statements_are_used_where_a_ksi_varies_by_class(): + by_id = {i["id"]: i for f in _catalog()["KSI"].values() for i in f["indicators"]} + assert len(by_id) == 46 + assert not by_id["KSI-CNA-EIS"]["statement"].startswith("**Optional:**") + + +def test_a_control_outside_the_nist_catalog_fails_closed(monkeypatch): + monkeypatch.setattr(sync, "_titles", lambda: {}) + with pytest.raises(SystemExit, match="not in the NIST SP 800-53 Rev5 catalog"): + sync.derive(_catalog()) + + +def test_a_new_ksi_needs_its_impact_set_first(): + cat = _catalog() + for f in cat["KSI"].values(): + f["indicators"] = [i for i in f["indicators"] if i["id"] != "KSI-CED-RAT"] + with pytest.raises(SystemExit, match="KSI-CED-RAT is new in the corpus"): + sync.derive(cat) From f1896430471d8f6d5788e279f2b11dcab8a1aae3 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:28:10 -0500 Subject: [PATCH 08/97] feat(sdr): Security Decision Record in FedRAMP's SDR schema Every applicable 20x Class C rule, every KSI and every Rev5 Moderate control, each no stronger than what backs it; semver versioning; consumers moved to the schema shape. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 3 +- infrastructure/schemas/freshness-config.json | 2 +- scripts/_assessor.py | 6 + scripts/build-sdr.py | 725 ++++++++++++------- scripts/reconcile.py | 11 +- tests/test_assessor_records.py | 28 +- tests/test_controls.py | 29 +- tests/test_evidence_freshness.py | 4 +- tests/test_evidence_watchdog.py | 2 +- tests/test_reconcile.py | 21 +- tests/test_sdr.py | 221 ++++++ website/assets/js/trust.js | 2 +- 12 files changed, 765 insertions(+), 289 deletions(-) create mode 100644 tests/test_sdr.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index a327c2de..24a24847 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1590,11 +1590,12 @@ jobs: - name: Build Security Decision Record (SDR — 20x SSP replacement) working-directory: infrastructure run: | - echo "Building Security Decision Record (SDR-CSO-FRR per KSI, SDR-CSF-CTF per Rev5 control) from the control projection..." + echo "Building the Security Decision Record (FedRAMP schema): every applicable rule, KSI and Moderate control..." curl -sfL -o previous-sdr.json "https://samaydlette.com/.well-known/security-decision-record.json" \ || echo '{}' > previous-sdr.json python3 ../scripts/build-sdr.py \ --catalog schemas/ksi-catalog.json --ksi-signal ksi-signal.json \ + --runtime-signal ksi-signal-runtime.json \ --vdr vdr-report.json --profile ../data/system-profile.json --previous-sdr previous-sdr.json # REQ-ASR-1 / T74: re-read the written SDR and fail if any assessor text # differs from the assessor's signed record, or a signed record is missing. diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index 7d02d3de..888561ba 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -195,7 +195,7 @@ "max_age_hours": 720, "timestamp": { "source": "json", - "field": "metadata.last_updated" + "field": "metadata.lastUpdated" }, "signed": true, "signature": ".well-known/security-decision-record.bundle" diff --git a/scripts/_assessor.py b/scripts/_assessor.py index 60a4d720..52cc5092 100644 --- a/scripts/_assessor.py +++ b/scripts/_assessor.py @@ -155,6 +155,12 @@ def verify_output(sdr, records): signed record is carried. Fails the build otherwise.""" signed = {r["record_id"]: _carried(r) for r in records} carried = [] + pe = sdr.get("providerExtensions") + if isinstance(pe, dict): + # The SDR in FedRAMP's schema keeps them under providerExtensions. + for entries in (pe.get("assessorCommentResponses") or {}).values(): + carried += entries or [] + carried += (pe.get("assessorRecords") or {}).get("records", []) or [] for rr in sdr.get("rule_records", []): carried += rr.get("assessor_comment_responses", []) or [] carried += (sdr.get("assessor_records") or {}).get("records", []) or [] diff --git a/scripts/build-sdr.py b/scripts/build-sdr.py index 70dc500b..d209947a 100644 --- a/scripts/build-sdr.py +++ b/scripts/build-sdr.py @@ -1,272 +1,506 @@ #!/usr/bin/env python3 # ============================================================================= -# SECURITY DECISION RECORD BUILDER (CR26 SDR — the 20x SSP replacement) +# SECURITY DECISION RECORD BUILDER (CR26 SDR, FedRAMP's 2026-06-24 schema) # ============================================================================= -# CR26 replaces the traditional System Security Plan with a persistently -# maintained Security Decision Record (SDR-CSO-FRR): for each applicable FedRAMP -# rule, an explanation of how it is followed, plus verification, validation, -# independent verification, independent validation, responses to assessor -# comments, and rule-specific artifacts. SDR-CSO-MTD carries version + last -# update + source; the 20x extension SDR-CSX-KSI carries per-KSI summaries, and -# SDR-CSX-KMT (Class C) carries metric summaries over 30 days / up to a year / -# all daily metric data. +# The SDR says, for this 20x Class C offering: # -# It carries both views CR26 defines over the same rule families: per KSI -# (SDR-CSX-KSI, the 20x record) and per applicable Rev5 control (SDR-CSF-CTF, -# the Rev5 record, in rev5_controls). Both come from the control projection -# (scripts/_controls.py) computed from the canonical inventory and the KSI -# catalog, so they cannot disagree. The OSCAL SSP that used to carry the -# per-control statements is retired (CR26); this record replaces it. +# fedRampRequirements every applicable FedRAMP rule (SDR-CSO-FRR), from the +# governed records in data/frr-records.json, which +# scripts/_frr.py holds to the pinned rules and to +# evidence that resolves; +# keySecurityIndicators every KSI (SDR-CSX-KSI): its measures (the OPA rules +# annotated with it), the automated methods that verify +# it (the deploy-time plan gate and the signed runtime +# re-validation), tests and evidence; +# securityControls every NIST 800-53 Rev5 control in the FedRAMP Moderate +# baseline, plus the controls FedRAMP's KSIs map beyond +# it: how 20x meets each Rev5 objective. A control's +# status is no stronger than what backs it. +# +# The record is 20x. The control view explains how the Rev5 objectives are met +# by the 20x evidence; it is not a second certification claim. +# +# Status rules for a control (the honesty rule, applied to Rev5): +# Implemented inherited from AWS (its FedRAMP authorization), or +# backed by a KSI whose measures are checked +# automatically, or by an implemented FedRAMP rule, or, +# for a policy-and-procedure control (XX-1), the +# family's published policy; +# (no status) not applicable, with the reason; +# Partially Implemented everything else the projection calls implemented: +# the mechanism is described but nothing checks it yet. +# +# Validated against the vendored schema before it is written; nothing that +# fails is written. Site-specific data is under providerExtensions. # # build-sdr.py --catalog schemas/ksi-catalog.json --ksi-signal ksi-signal.json \ -# --vdr vdr-report.json --previous-sdr prev.json +# --runtime-signal ksi-signal-runtime.json --vdr vdr-report.json --previous-sdr prev.json # ============================================================================= import argparse +import hashlib +import importlib.util import json import os +import re +import subprocess import sys from datetime import datetime, timezone from pathlib import Path -# Shared generator helpers. The insert is __file__-relative, so this script stays -# runnable standalone from any working directory (see scripts/_common.py). sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _common import prop # noqa: E402 import _assessor # noqa: E402 import _controls # noqa: E402 +import _fedramp_schema as fs # noqa: E402 +import _frr # noqa: E402 +import _rules # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +MODERATE_PROFILE = REPO / "data" / "profiles" / "FedRAMP_rev5_MODERATE-baseline_profile.json" +POLICY_DIR = REPO / "infrastructure" / "policy" +POLICIES = REPO / "docs" / "policies" +SITE = "https://samaydlette.com" +REPO_URL = "https://github.com/sam-aydlette/samaydlette.com/blob/main" +HISTORY_DAYS = 366 # SDR-CSX-KMT: daily metric data up to the past year +SEMVER = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") + +NO_ASSESSMENT = ("No independent assessment: no FedRAMP Recognized assessment service is engaged. " + "The provider-side verification and validation above is what an assessor would examine.") +WORKSTREAMS = { + "W2": "vulnerability report follow-ups", "W3": "certification package in FedRAMP's schemas", + "W4": "quarterly reporting cycle", "W5": "significant change notifications", + "W6": "incident reporting", "W7": "detection and evaluation", "W8": "trust center", + "W9": "FedRAMP Security Inbox", "W10": "scope, cryptography and secure configuration", + "sam": "an act of the operator (registration, assessor, application)", +} +EVIDENCE_WORDS = {"test": "test", "step": "pipeline step", "artifact": "published artifact", + "decision": "decision record"} + + +def _fail(msg): + raise SystemExit(f"::error::SDR: {msg} (fail-closed)") + + +# --- inputs --------------------------------------------------------------------------- + +def moderate_ids(path=MODERATE_PROFILE): + prof = json.loads(Path(path).read_text())["profile"] + return {i for imp in prof["imports"] for inc in imp.get("include-controls", []) for i in inc.get("with-ids", [])} + + +def policy_rules(policy_dir=POLICY_DIR): + """{KSI ID: [{"id", "title", "file"}]} from the OPA rule annotations + (custom.ksi_ids), read with `opa inspect` like check-policy-annotations.""" + out = subprocess.run(["opa", "inspect", "-a", str(policy_dir), "--format", "json"], + capture_output=True, text=True, check=True) + by_ksi = {} + for entry in json.loads(out.stdout).get("annotations", []) or []: + ann = entry.get("annotations") or {} + custom = ann.get("custom") or {} + if ann.get("scope") != "rule" or not custom.get("id"): + continue + f = Path((entry.get("location") or {}).get("file", "?")).name + for k in custom.get("ksi_ids") or []: + by_ksi.setdefault(k, []).append({"id": custom["id"], "title": ann.get("title", ""), "file": f}) + for rules in by_ksi.values(): + rules.sort(key=lambda r: (r["file"], r["id"])) + return by_ksi + + +def runtime_validations(path): + """Validations from the signed runtime signal, only when it verifies and is + fresh (the VDR builder's own loader); otherwise none, with the reason.""" + spec = importlib.util.spec_from_file_location("vdr_builder", REPO / "scripts" / "build-vdr-report.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + rt = mod.load_runtime_signal(path) + if not (rt["verified"] and rt["fresh"]): + return [], rt["reason"] or "runtime signal not usable" + return rt["validations"], None + + +# --- FedRAMP rules -------------------------------------------------------------------- + +def _evidence_line(ev): + return f"{EVIDENCE_WORDS[ev['kind']]} `{ev['ref']}`" -HISTORY_DAYS = 366 # SDR-CSX-KMT: all daily metric data up to the past year - -NA = "N/A — self-attested proof of concept; no FedRAMP Recognized independent assessment service engaged (IVV is not applicable to an unsponsored system)." - - -def _control_map(irs): - """control_id -> {status, origination} from the control projection.""" - m = {} - for ir in irs: - m[ir["control-id"]] = {"status": prop(ir, "implementation-status"), - "origination": prop(ir, "control-origination")} - return m - - -def build_records(catalog, cmap): - """One SDR-CSO-FRR record per in-scope KSI, derived from its controls' projected state.""" - records = [] - for fam in catalog.get("KSI", {}).values(): - for ind in fam.get("indicators", []) or []: - if ind.get("retired"): - continue - controls = [] - impl = inh = na = 0 - for c in ind.get("controls", []) or []: - cid = c.get("control_id") - st = cmap.get(cid, {}) - controls.append({"control_id": cid, "status": st.get("status"), - "origination": st.get("origination")}) - o = st.get("origination") - if st.get("status") == "not-applicable": - na += 1 - elif o == "inherited" or o == "shared": - inh += 1 - else: - impl += 1 - if controls: - expl = (f"Satisfied through {len(controls)} NIST 800-53 control(s) " - f"({impl} system-implemented, {inh} inherited/shared, {na} not-applicable); " - f"see rev5_controls in this record for each control's implementation statement.") - else: - expl = ("Process/organizational indicator with no direct 800-53 control mapping; " - "satisfied at the operator level and documented in the linked policy.") - records.append({ - "rule_id": ind["id"], - "statement": ind.get("statement", ""), - "controls": controls, - "explanation": expl, - "verification": ("Automated: the deploy-time OPA gate and the per-deploy KSI signal " - "record the implementing controls; the daily runtime emitter " - "re-validates the live configuration against the same compiled policy."), - "validation": ("Effectiveness reviewed continuously by the daily runtime revalidation " - "(drift detector) and annually by the structural security review."), - "independent_verification": NA, - "independent_validation": NA, - "assessor_comment_responses": [], - "artifacts": ["/.well-known/ksi-signal.json", "/.well-known/security-decision-record.json"], - }) - return records - - -# CR26 implementation-status vocabulary (SDR-CSF-CTF) for the projection's -# OSCAL tokens. -REV5_STATUS = {"implemented": "Implemented", "partial": "Partially Implemented", "planned": "Planned", - "alternative": "Alternative Implementation", "not-applicable": "Not Applicable"} - - -def build_rev5_controls(irs, catalog): - """SDR-CSF-CTF: a summary for each applicable Rev5 control: parameter values, - implementation status, the mechanisms (the projection's statement, including - any inheritance), verification, validation, independent verification and - validation, and control-specific artifacts. Verification and validation - point at the records of the KSIs that claim the control rather than - restating them; a control no KSI claims says so.""" - ksis_by_control, _ = _controls.contributing_ksis(catalog) + +def _assessment(rule_id, assessor_by_rule): + entries = assessor_by_rule.get(rule_id) or [] + if not entries: + return [NO_ASSESSMENT] + return [f"Assessor record {e['record_id']} ({e['kind']}): {e['text']}" for e in entries] + + +def build_requirements(records, applicable, assessor_by_rule): out = [] - for ir in irs: - cid = ir["control-id"] - status = prop(ir, "implementation-status") - origination = prop(ir, "control-origination") - ksis = sorted(ksis_by_control.get(cid, ())) - if status == "not-applicable": - verification = validation = "Not applicable." - elif origination == "inherited": - verification = validation = ("Inherited from AWS under its FedRAMP certification " - "(leveraged authorization); verified and validated there.") - elif ksis: - verification = validation = ("See the rule records (rule_records) of the contributing KSIs: " - + ", ".join(ksis) + ".") - else: - verification = ("No KSI claims this control; the statement is maintained in the control " - "projection (scripts/_controls.py, data/component-definitions/) and " - "reviewed with it.") - validation = "No automated validation." - remarks = [ir["statements"][0]["remarks"]] if ir.get("statements") else [] - live = prop(ir, "implementation-status-remarks") + for r in records["records"]: + rid, status = r["frr_id"], r["status"] + impl = [r["implementation"]] + if status == "Not Applicable": + impl.append(f"Not applicable: {r['not_applicable']}") + if r.get("may_rationale"): + impl.append(f"Use of this permission: {r['may_rationale']}") + val = [] + if r.get("evidence"): + val.append("Evidence: " + "; ".join(_evidence_line(e) for e in r["evidence"]) + ".") + val.append("Checked on every build by scripts/_frr.py: the rule set matches the pinned FedRAMP " + "rules and every piece of evidence cited resolves.") + if r.get("gap"): + val.append(f"Gap: {r['gap']}") + val.append("Closes with: " + "; ".join(f"{c} ({WORKSTREAMS[c]})" for c in r["closes_with"]) + ".") + rec = {"frrID": rid} + if status != "Not Applicable": + rec["frrImplementationStatus"] = status + rec["frrImplementation"] = impl + rec["frrValidation"] = val + rec["frrAssessment"] = _assessment(rid, assessor_by_rule) + rec["providerExtensions"] = {"status": status, "force": applicable[rid]["force"], + "evidence": r.get("evidence", []), + "closesWith": r.get("closes_with", [])} + out.append(rec) + return out + + +# --- KSIs ----------------------------------------------------------------------------- + +def ksi_status(k): + """Implemented only when the KSI passes and its measures are checked + automatically against the plan; a pass by inventory or document alone is + partial.""" + if k.get("status") != "pass": + return "Not Implemented" + if k.get("method") == "policy-validation" and (k.get("evidence") or {}).get("validation_ids"): + return "Implemented" + return "Partially Implemented" + + +def build_ksis(catalog, signal, rules_by_ksi, runtime, runtime_reason, assessor_by_rule): + statements = {i["id"]: i for f in catalog["KSI"].values() for i in f["indicators"]} + sig = {k["id"]: k for k in signal.get("ksis", [])} + missing = sorted(set(statements) - set(sig)) + if missing: + _fail(f"the KSI signal carries no result for {', '.join(missing)}") + emitted = (signal.get("emitted_at") or "")[:10] or None + out, status = [], {} + for kid in sorted(statements): + k, cat = sig[kid], statements[kid] + st = ksi_status(k) + status[kid] = st + ev = k.get("evidence") or {} + comps = set(ev.get("component_refs") or []) + rt = [v for v in runtime if comps & set(v.get("component_refs") or [])] + plan_ok = bool(ev.get("validation_ids")) + methods = int(plan_ok) + int(bool(rt)) + rules = rules_by_ksi.get(kid, []) + impl = [cat["statement"]] + if rules: + impl.append("Measures: " + "; ".join(f"{r['title']} (`{r['file']}#{r['id']}`)" for r in rules) + ".") + impl.append(f"Measured by {k.get('method')}: {len(comps)} inventory component(s), " + f"{len(ev.get('validation_ids') or [])} deploy-time validation(s), " + f"{len(ev.get('failed_validation_ids') or [])} failing.") + val = [] + if plan_ok: + val.append("Verified on every deploy: OPA evaluates the Terraform plan against the measures above " + "before apply, and the deploy stops on a violation.") + if rt: + val.append(f"Re-validated daily at runtime: the signed runtime signal re-evaluates the same compiled " + f"policy against live configuration for {len(rt)} of these components.") + elif runtime_reason: + val.append(f"Runtime re-validation not counted for this build: {runtime_reason}.") + val.append(f"Automated methods in place: {methods} (FRC-CSX-VVK requires 2 for Class C).") + if st != "Implemented": + val.append("Status is not Implemented because " + + ("the KSI does not pass." if st == "Not Implemented" else + f"its evidence is {k.get('method')}, not an automated check of its measures.")) + evidence = [{"evidenceType": "Report", + "evidenceDescription": "Deploy-time KSI signal: inventory, validations and per-KSI results, " + "signed with cosign by the deploy workflow.", + "evidenceLocation": f"{SITE}/.well-known/ksi-signal.json", + **({"lastUpdated": emitted} if emitted else {})}] + if rt: + evidence.append({"evidenceType": "Report", + "evidenceDescription": "Runtime KSI signal: daily re-validation of live configuration, " + "signed with the runtime KMS key.", + "evidenceLocation": f"{SITE}/.well-known/ksi-signal-runtime.json"}) out.append({ - "control_id": cid, - "implementation_status": REV5_STATUS.get(status, status), - "origination": origination, - "parameters": [{"param_id": sp["param-id"], "values": sp.get("values", [])} - for sp in ir.get("set-parameters", [])], - "mechanisms": " ".join(remarks + ([live] if live else [])), - "contributing_ksis": ksis, - "verification": verification, - "validation": validation, - "independent_verification": NA, - "independent_validation": NA, - "assessor_comment_responses": [], - "artifacts": [lk["href"] for lk in ir.get("links", [])], + "ksiId": kid, + "ksiImplementationStatus": st, + "ksiImplementation": impl, + "ksiValidation": val, + "ksiAssessment": _assessment(kid, assessor_by_rule), + "ksiTests": [f"infrastructure/policy/{r['file']}#{r['id']}" for r in rules], + "ksiEvidence": evidence, + "providerExtensions": {"method": k.get("method"), "automatedMethods": methods, + "controls": [c["control_id"] for c in cat["controls"]]}, }) + return out, status + + +# --- controls ------------------------------------------------------------------------- + +def nist_id(cid): + """ac-2.1 -> AC-2(1).""" + base, _, enh = cid.partition(".") + return base.upper() + (f"({enh})" if enh else "") + + +def control_basis(ir, ksis_for, ksi_st, frrs_for, frr_st, policies): + """(status, or None when not applicable; the basis sentence).""" + cid = ir["control-id"] + st = _controls.prop(ir, "implementation-status") + orig = _controls.prop(ir, "control-origination") + if st == "not-applicable": + return None, "Not applicable." + if st == "partial": + return "Partially Implemented", "A contributing KSI is failing." + checked = sorted(k for k in ksis_for.get(cid, ()) if ksi_st.get(k) == "Implemented") + rules = sorted(r for r in frrs_for.get(cid, ()) if frr_st.get(r) == "Implemented") + fam = cid.split("-")[0] + if orig == "inherited": + return "Implemented", "Inherited from AWS under its FedRAMP authorization." + if checked: + return "Implemented", "Backed by automatically checked KSI(s): " + ", ".join(checked) + "." + if rules: + return "Implemented", "Backed by implemented FedRAMP rule(s): " + ", ".join(rules) + "." + if re.fullmatch(r"[a-z]{2}-1", cid) and fam in policies: + return "Implemented", f"Policy and procedures, published: {policies[fam]}" + return "Partially Implemented", ("The mechanism is described, but no automated check, implemented " + "FedRAMP rule or published policy shows it yet.") + + +def _find_rule(rules, rid): + for fam in rules["FRR"].values(): + for groups in fam["data"].values(): + for members in groups.values(): + if rid in members: + return members[rid] + _fail(f"{rid} is not in the pinned rules") + + +def _sort_key(cid): + fam, _, rest = cid.partition("-") + num, _, enh = rest.partition(".") + return (fam, int(num), int(enh or 0)) + + +def build_controls(irs, catalog, ksi_st, applicable, records, moderate, policies): + ksis_for, _ = _controls.contributing_ksis(catalog) + frr_st = {r["frr_id"]: r["status"] for r in records["records"]} + rules, _ = _rules.load_rules() + frrs_for = {} + for rid in applicable: + for c in _find_rule(rules, rid).get("controls", []) or []: + frrs_for.setdefault(str(c).lower(), set()).add(rid) + missing = sorted(moderate - {ir["control-id"] for ir in irs}) + if missing: + _fail(f"{len(missing)} Moderate control(s) have no implementation record: {', '.join(missing[:10])}") + out = [] + for ir in sorted(irs, key=lambda i: _sort_key(i["control-id"])): + cid = ir["control-id"] + status, basis = control_basis(ir, ksis_for, ksi_st, frrs_for, frr_st, policies) + statement = ir["statements"][0]["remarks"] if ir.get("statements") else "" + orig = _controls.prop(ir, "control-origination") + ks = sorted(ksis_for.get(cid, ())) + fr = sorted(frrs_for.get(cid, ())) + desc = [f"**Not applicable.** {statement}"] if status is None else [f"**Basis:** {basis}", statement] + desc.append(f"**Origination:** {orig}.") + desc.append("**20x coverage:** " + (", ".join(ks) if ks else "no KSI maps this control") + + ("; FedRAMP rules: " + ", ".join(fr) if fr else "") + ".") + if cid not in moderate: + desc.append("Beyond the FedRAMP Moderate baseline; included because a FedRAMP KSI maps it.") + rec = {"controlId": nist_id(cid)} + params = [{"parameterId": sp["param-id"], "parameterValue": ", ".join(sp["values"])} + for sp in ir.get("set-parameters", []) if sp.get("values")] + if params: + rec["parameterValues"] = params + if status: + rec["controlImplementationStatus"] = status + rec["controlImplementationDescription"] = "\n\n".join(d for d in desc if d) + rec["providerExtensions"] = {"basis": basis, "origination": orig, "ksis": ks, "fedrampRules": fr, + "inModerateBaseline": cid in moderate, + "status": status or "Not Applicable"} + out.append(rec) return out -def build_metrics(records, controls_in_scope, vdr, signal, history, now): - """SDR-CSX-KMT: metric snapshot + appended daily history (toward one year).""" +def policy_documents(policy_dir=POLICIES): + """{control family: URL of its published policy} for families with one.""" + return {p.stem.removesuffix("-policy"): f"{REPO_URL}/docs/policies/{p.name}" + for p in sorted(Path(policy_dir).glob("??-policy.md"))} + + +# --- metrics, version ----------------------------------------------------------------- + +def build_metrics(counts, vdr, signal, history, now): + """SDR-CSX-KMT: today's point and daily history toward one year.""" sig_vals = signal.get("validations", []) if signal else [] - failing = sum(1 for v in sig_vals if v.get("result") == "fail") vsum = (vdr or {}).get("summary", {}) today = now.date().isoformat() - point = { - "date": today, - "ksi_in_scope": len(records), - "controls_in_scope": controls_in_scope, - "signal_validations": len(sig_vals), - "signal_failing": failing, - "vdr_total": vsum.get("total_findings", 0), - "vdr_blocking": vsum.get("blocking", 0), - "vdr_accepted": vsum.get("risk_accepted", 0), - "vdr_kev": vsum.get("kev", 0), - } - history = [h for h in history if h.get("date") != today] - history.append(point) + point = {"date": today, **counts, + "signal_validations": len(sig_vals), + "signal_failing": sum(1 for v in sig_vals if v.get("result") == "fail"), + "vdr_total": vsum.get("total_findings", 0), "vdr_blocking": vsum.get("blocking", 0), + "vdr_accepted": vsum.get("risk_accepted", 0), "vdr_kev": vsum.get("kev", 0)} + history = [h for h in history if h.get("date") != today] + [point] history = sorted(history, key=lambda h: h.get("date", ""))[-HISTORY_DAYS:] - last30 = history[-30:] - def _summ(window): - if not window: - return {} + def summ(window): return {"days": len(window), "max_blocking": max(h.get("vdr_blocking", 0) for h in window), "max_failing_validations": max(h.get("signal_failing", 0) for h in window), "current_accepted": window[-1].get("vdr_accepted", 0)} + return {"current": point, "summary_30d": summ(history[-30:]), "summary_1y": summ(history), + "daily_metric_data": history} + + +def previous_history(prev): + pe = prev.get("providerExtensions") or {} + hist = (pe.get("metrics") or {}).get("daily_metric_data") + if hist is None: # the record before the schema reshape kept it at the top level + hist = (prev.get("metrics") or {}).get("daily_metric_data") + return [h for h in hist or [] if isinstance(h, dict)] + + +def content_hash(doc): + """What the version tracks: everything but the metadata, the daily + metrics and the hash itself.""" + body = {k: v for k, v in doc.items() if k not in ("metadata", "providerExtensions")} + pe = {k: v for k, v in (doc.get("providerExtensions") or {}).items() if k not in ("metrics", "contentSha256")} + raw = json.dumps({"body": body, "pe": pe}, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(raw).hexdigest() - return { - "current": point, - "summary_30d": _summ(last30), - "summary_1y": _summ(history), - "daily_metric_data": history, # all daily points up to a year (SDR-CSX-KMT) - "note": ("Daily metric history accumulates one point per deploy/day; the full " - "one-year window builds over time from the first published SDR."), + +def next_version(prev, digest): + """Semver: unchanged content keeps the previous version; changed content + bumps the patch; a previous record without a semver starts at 1.0.0.""" + v = (prev.get("metadata") or {}).get("version", "") + m = SEMVER.match(v or "") + if not m: + return "1.0.0" + if (prev.get("providerExtensions") or {}).get("contentSha256") == digest: + return v + return f"{m.group(1)}.{m.group(2)}.{int(m.group(3)) + 1}" + + +# --- build ---------------------------------------------------------------------------- + +def _tally(items, key): + out = {} + for it in items: + s = it["providerExtensions"][key] + out[s] = out.get(s, 0) + 1 + return dict(sorted(out.items())) + + +def build(catalog, signal, records, applicable, irs, moderate, rules_by_ksi, runtime, runtime_reason, + assessor_records, prof, prev, vdr, now, policies): + ids = [r["frr_id"] for r in records["records"]] + sorted( + i["id"] for f in catalog["KSI"].values() for i in f["indicators"]) + pseudo = [{"rule_id": i, "assessor_comment_responses": []} for i in ids] + block = _assessor.attach(pseudo, assessor_records) + by_rule = {p["rule_id"]: p["assessor_comment_responses"] for p in pseudo if p["assessor_comment_responses"]} + + reqs = build_requirements(records, applicable, by_rule) + ksis, ksi_st = build_ksis(catalog, signal, rules_by_ksi, runtime, runtime_reason, by_rule) + controls = build_controls(irs, catalog, ksi_st, applicable, records, moderate, policies) + in_mod = [c for c in controls if c["providerExtensions"]["inModerateBaseline"]] + beyond = [c for c in controls if not c["providerExtensions"]["inModerateBaseline"]] + ksi_counts = {} + for s in ksi_st.values(): + ksi_counts[s] = ksi_counts.get(s, 0) + 1 + counts = {"rules": len(reqs), "ksis": len(ksis), "controls_moderate": len(in_mod), + "controls_beyond_moderate": len(beyond)} + doc = { + "$schema": fs.stamp("sdr"), + "certificationPackageOverviewUri": fs.CPO_URI, + "metadata": {}, + "fedRampRequirements": reqs, + "keySecurityIndicators": ksis, + "securityControls": controls, + "providerExtensions": { + "certification": "FedRAMP 20x, Program path, Class C", + "class": prof.get("fedramp_class"), + "impact_level": prof.get("impact_level"), + "ksi_signal_id": signal.get("signal_id"), + "rules": _rules.pin(), + "summary": {"fedRampRequirements": _tally(reqs, "status"), + "keySecurityIndicators": dict(sorted(ksi_counts.items())), + "securityControlsModerate": _tally(in_mod, "status"), + "securityControlsBeyondModerate": _tally(beyond, "status")}, + "assessorRecords": block, + "assessorCommentResponses": by_rule, + "metrics": build_metrics(counts, vdr, signal, previous_history(prev), now), + }, } + digest = content_hash(doc) + doc["providerExtensions"]["contentSha256"] = digest + doc["metadata"] = {"version": next_version(prev, digest), "lastUpdated": now.isoformat(), + "updateSource": "Automated: scripts/build-sdr.py in the deploy workflow, from " + "data/frr-records.json, the KSI signal and the control projection"} + return doc -def main(): +def main(argv=None): ap = argparse.ArgumentParser() ap.add_argument("--catalog", default="schemas/ksi-catalog.json") ap.add_argument("--ksi-signal", default="ksi-signal.json") + ap.add_argument("--runtime-signal", default="ksi-signal-runtime.json") ap.add_argument("--vdr", default="vdr-report.json") ap.add_argument("--profile", default="../data/system-profile.json") + ap.add_argument("--frr-records", default=str(_frr.DEFAULT_PATH)) ap.add_argument("--previous-sdr", default=None) ap.add_argument("--output", default="security-decision-record.json") ap.add_argument("--md-output", default="security-decision-record.md") - ap.add_argument("--assessor-records", default=str(_assessor.RECORDS_DIR), - help="directory of signed assessor records (REQ-ASR-1); empty = no assessor record") - ap.add_argument("--assessor-identity", default=str(_assessor.IDENTITY_PATH), - help="pinned assessor identity the records must verify against") - a = ap.parse_args() + ap.add_argument("--assessor-records", default=str(_assessor.RECORDS_DIR)) + ap.add_argument("--assessor-identity", default=str(_assessor.IDENTITY_PATH)) + a = ap.parse_args(argv) now = datetime.now(timezone.utc) catalog = json.loads(Path(a.catalog).read_text()) - signal = json.loads(Path(a.ksi_signal).read_text()) if Path(a.ksi_signal).exists() else {} + if not Path(a.ksi_signal).exists(): + _fail(f"{a.ksi_signal} not found") + signal = json.loads(Path(a.ksi_signal).read_text()) vdr = json.loads(Path(a.vdr).read_text()) if Path(a.vdr).exists() else {} prof = json.loads(Path(a.profile).read_text()) if Path(a.profile).exists() else {} - history = [] + prev = {} if a.previous_sdr and Path(a.previous_sdr).exists(): try: - history = json.loads(Path(a.previous_sdr).read_text()).get("metrics", {}).get("daily_metric_data", []) or [] - except Exception: - history = [] - - # The projection needs the inventory: without it there is no control state - # to record, so a missing signal stops the build (fail-closed). + prev = json.loads(Path(a.previous_sdr).read_text()) + except json.JSONDecodeError: + prev = {} + records = _frr.load(a.frr_records) + applicable = _rules.applicable_provider_rules() irs = _controls.load(a.ksi_signal, a.catalog) - cmap = _control_map(irs) - records = build_records(catalog, cmap) - # REQ-ASR-1: assessor records enter only through the signed port; any record - # that does not verify fails the build. They are carried verbatim. + runtime, reason = runtime_validations(a.runtime_signal) assessor_records = _assessor.load_records(a.assessor_records, a.assessor_identity) - assessor_block = _assessor.attach(records, assessor_records) - metrics = build_metrics(records, len(cmap), vdr, signal, history, now) - - sdr = { - "record_type": "security-decision-record", - "schema": "fedramp-security-decision-record-schema-2026-06-24.json", - "note": "The CR26 replacement for the System Security Plan: per-KSI records (20x) and " - "per-Rev5-control records (rev5_controls), both from one control projection.", - # SDR-CSO-MTD - "metadata": { - "version": "live (regenerated each deploy)", - "last_updated": now.isoformat(), - "source_of_update": "scripts/build-sdr.py, derived from the canonical KSI inventory and the control projection", - }, - "class": prof.get("fedramp_class"), - "impact_level": prof.get("impact_level"), - "ksi_signal_id": (signal or {}).get("signal_id"), - "rule_records": records, # SDR-CSO-FRR + SDR-CSX-KSI - "rev5_controls": build_rev5_controls(irs, catalog), # SDR-CSF-CTF - "metrics": metrics, # SDR-CSX-KMT - # REQ-ASR-1: verified, signed assessor records not tied to one rule, and - # the status ("no assessor record" when none exist). - "assessor_records": assessor_block, - } - _assessor.verify_output(sdr, assessor_records) # T74: output text == signed text - Path(a.output).write_text(json.dumps(sdr, indent=2) + "\n") + doc = build(catalog, signal, records, applicable, irs, moderate_ids(), policy_rules(), runtime, reason, + assessor_records, prof, prev, vdr, now, policy_documents()) + _assessor.verify_output(doc, assessor_records) # T74 + fs.write_validated(doc, "sdr", a.output) if a.md_output: - Path(a.md_output).write_text(render_markdown(sdr) + "\n") - print(f"security-decision-record: {len(records)} per-KSI rule records; " - f"{metrics['current']['controls_in_scope']} controls in scope; " - f"{len(metrics['daily_metric_data'])} day(s) of metric history") + Path(a.md_output).write_text(render_markdown(doc) + "\n") + s = doc["providerExtensions"]["summary"] + print(f"security-decision-record {doc['metadata']['version']}: rules {s['fedRampRequirements']}; " + f"KSIs {s['keySecurityIndicators']}; Moderate controls {s['securityControlsModerate']}") return 0 -def _assessor_lines(sdr): - block = sdr.get("assessor_records") or {"status": _assessor.NO_RECORD, "records": []} - entries = list(block.get("records", [])) - for r in sdr["rule_records"]: - entries += [dict(e, rule_id=r["rule_id"]) for e in r.get("assessor_comment_responses", [])] +# --- human-readable view (CDS-CSO-CBF: same data as the JSON) ------------------------- + +def _cell(text): + return str(text).replace("|", "\\|").replace("\n", " ") + + +def _assessor_lines(doc): + pe = doc["providerExtensions"] + entries = list(pe["assessorRecords"].get("records", [])) + for rid, es in pe.get("assessorCommentResponses", {}).items(): + entries += [dict(e, rule_id=rid) for e in es] if not entries: return ["**Assessor records:** No assessor record. Nothing on this page was written " "or signed by an assessor."] - out = [f"**Assessor records:** {block['status']}. Each is quoted verbatim from the " + out = [f"**Assessor records:** {pe['assessorRecords']['status']}. Each is quoted verbatim from the " "assessor's signed record; the signature is in the JSON."] for e in sorted(entries, key=lambda e: e["record_id"]): rule = f" (`{e['rule_id']}`)" if e.get("rule_id") else "" @@ -275,42 +509,49 @@ def _assessor_lines(sdr): return out -def render_markdown(sdr): - m = sdr["metadata"] - cur = sdr["metrics"]["current"] +def _summary_line(d): + return ", ".join(f"{k} {v}" for k, v in d.items()) or "none" + + +def render_markdown(doc): + m, pe = doc["metadata"], doc["providerExtensions"] + s = pe["summary"] out = ["# Security Decision Record", "", - f"_{sdr['note']}_", "", - f"- **Class:** {sdr['class']} | **Impact:** {sdr['impact_level']}", - f"- **Version:** {m['version']} | **Last updated:** {m['last_updated']}", - f"- **Source:** {m['source_of_update']}", - f"- **Bound to inventory signal:** `{sdr['ksi_signal_id']}`", - "", - "## Metrics (SDR-CSX-KMT)", "", - f"- KSIs in scope: **{cur['ksi_in_scope']}** | Controls in scope: **{cur['controls_in_scope']}**", - f"- Signal validations: {cur['signal_validations']} ({cur['signal_failing']} failing)", - f"- Vulnerabilities: {cur['vdr_total']} total, {cur['vdr_blocking']} blocking, " - f"{cur['vdr_accepted']} accepted, {cur['vdr_kev']} KEV", - f"- Daily metric history: {len(sdr['metrics']['daily_metric_data'])} day(s) on record " - "(builds toward one year)", - "", - "## Per-rule records (SDR-CSO-FRR / SDR-CSX-KSI)", "", - "Independent verification and validation are N/A for this self-attested system " - "(no FedRAMP Recognized assessor). Verification and validation below are the " - "provider-side evidence an assessor would consume.", "", - *_assessor_lines(sdr), "", - "| KSI | Controls | Explanation |", "|---|---|---|"] - for r in sdr["rule_records"]: - cids = ", ".join(c["control_id"] for c in r["controls"]) or "—" - out.append("| `{rid}` | {cids} | {ex} |".format( - rid=r["rule_id"], cids=cids, ex=r["explanation"].replace("|", "\\|"))) - rc = sdr.get("rev5_controls", []) - by_status = {} - for c in rc: - by_status[c["implementation_status"]] = by_status.get(c["implementation_status"], 0) + 1 - out += ["", "## Rev5 controls (SDR-CSF-CTF)", "", - f"{len(rc)} controls, each with its parameter values, implementation status, mechanisms, " - "verification and validation in `rev5_controls` of the JSON record. By status: " - + (", ".join(f"{k} {v}" for k, v in sorted(by_status.items())) or "none") + "."] + f"_{pe['certification']}. Every applicable FedRAMP rule, every Key Security Indicator, and every " + "NIST 800-53 Rev5 control in the Moderate baseline, with how 20x evidence meets it._", "", + f"- **Version:** {m['version']} | **Last updated:** {m['lastUpdated']}", + f"- **Source:** {m['updateSource']}", + f"- **Bound to inventory signal:** `{pe['ksi_signal_id']}`", + f"- **FedRAMP rules (pinned):** {pe['rules']['version']} at `{pe['rules']['commit'][:12]}`", + "", "## Summary", "", + f"- FedRAMP rules ({len(doc['fedRampRequirements'])}): {_summary_line(s['fedRampRequirements'])}", + f"- Key Security Indicators ({len(doc['keySecurityIndicators'])}): " + f"{_summary_line(s['keySecurityIndicators'])}", + f"- Rev5 Moderate controls ({sum(s['securityControlsModerate'].values())}): " + f"{_summary_line(s['securityControlsModerate'])}", + f"- Controls beyond Moderate mapped by KSIs ({sum(s['securityControlsBeyondModerate'].values())}): " + f"{_summary_line(s['securityControlsBeyondModerate'])}", + "", *_assessor_lines(doc), "", + "## FedRAMP rules", "", "| Rule | Status | Implementation | Gap / closes with |", "|---|---|---|---|"] + for r in doc["fedRampRequirements"]: + x = r["providerExtensions"] + gap = next((v for v in r["frrValidation"] if v.startswith("Gap: ")), "") + closes = next((v for v in r["frrValidation"] if v.startswith("Closes with: ")), "") + out.append(f"| `{r['frrID']}` | {x['status']} | {_cell(' '.join(r['frrImplementation']))} | " + f"{_cell((gap + ' ' + closes).strip()) or '—'} |") + out += ["", "## Key Security Indicators", "", + "| KSI | Status | Automated methods | Measures |", "|---|---|---|---|"] + for k in doc["keySecurityIndicators"]: + x = k["providerExtensions"] + out.append(f"| `{k['ksiId']}` | {k['ksiImplementationStatus']} | {x['automatedMethods']} | " + f"{_cell('; '.join(k['ksiTests'])) or '—'} |") + out += ["", "## NIST 800-53 Rev5 controls", "", + "| Control | Status | Basis | 20x coverage |", "|---|---|---|---|"] + for c in doc["securityControls"]: + x = c["providerExtensions"] + cov = ", ".join(x["ksis"] + x["fedrampRules"]) or "—" + tag = "" if x["inModerateBaseline"] else " (beyond Moderate)" + out.append(f"| {c['controlId']}{tag} | {x['status']} | {_cell(x['basis'])} | {_cell(cov)} |") out += ["", "---", "_Source of truth: the signed `security-decision-record.json`._"] return "\n".join(out) diff --git a/scripts/reconcile.py b/scripts/reconcile.py index 5f187a81..c2231d30 100644 --- a/scripts/reconcile.py +++ b/scripts/reconcile.py @@ -369,11 +369,18 @@ def check_c_suppressions(checkov_skips, register, vdr): # ---------------------------------------------------------------------------- # (d) impact level identical; nothing asserts Low # ---------------------------------------------------------------------------- +def sdr_fields(sdr): + """Where the SDR keeps its site-specific fields: under providerExtensions + since it follows FedRAMP's schema (the top level is FedRAMP's).""" + pe = sdr.get("providerExtensions") + return pe if isinstance(pe, dict) else sdr + + def extract_impacts(signal, sdr, cpo, vdr, dashboard_html): impacts = {} impacts["signal"] = normalize_impact( signal.get("categorization", {}).get("impact_level")) - impacts["sdr"] = normalize_impact(sdr.get("impact_level")) + impacts["sdr"] = normalize_impact(sdr_fields(sdr).get("impact_level")) impacts["cpo"] = normalize_impact(cpo.get("impact_level")) impacts["vdr"] = normalize_impact(vdr.get("impact_level") or ("class " + vdr.get("class", "")).strip()) @@ -406,7 +413,7 @@ def check_e_binding(signal, sdr, cpo, vdr): sid = signal.get("signal_id") if not sid: return ["(e) signal has no signal_id"] - for art, doc in (("vdr", vdr), ("sdr", sdr), ("cpo", cpo)): + for art, doc in (("vdr", vdr), ("sdr", sdr_fields(sdr)), ("cpo", cpo)): got = doc.get("ksi_signal_id") if got != sid: violations.append(f"(e) {art} signal_id {got!r} != inventory signal_id {sid!r}") diff --git a/tests/test_assessor_records.py b/tests/test_assessor_records.py index 894b6bca..dc0a816e 100644 --- a/tests/test_assessor_records.py +++ b/tests/test_assessor_records.py @@ -153,11 +153,25 @@ def test_t74_output_dropping_a_signed_record_fails(tmp_path): def test_sdr_markdown_states_no_assessor_record(): - sdr = {"note": "n", "class": "C", "impact_level": "low", "ksi_signal_id": "s", - "metadata": {"version": "v", "last_updated": "t", "source_of_update": "s"}, - "metrics": {"current": {"ksi_in_scope": 0, "controls_in_scope": 0, "signal_validations": 0, - "signal_failing": 0, "vdr_total": 0, "vdr_blocking": 0, - "vdr_accepted": 0, "vdr_kev": 0}, - "daily_metric_data": []}, - "rule_records": [], "assessor_records": {"status": "no assessor record", "records": []}} + sdr = {"metadata": {"version": "1.0.0", "lastUpdated": "t", "updateSource": "s"}, + "fedRampRequirements": [], "keySecurityIndicators": [], "securityControls": [], + "providerExtensions": {"certification": "c", "ksi_signal_id": "s", + "rules": {"version": "v", "commit": "0" * 40}, + "summary": {"fedRampRequirements": {}, "keySecurityIndicators": {}, + "securityControlsModerate": {}, "securityControlsBeyondModerate": {}}, + "assessorRecords": {"status": "no assessor record", "records": []}, + "assessorCommentResponses": {}}} assert "No assessor record" in sdr_mod.render_markdown(sdr) + + +def test_t74_reads_the_schema_shaped_sdr(tmp_path): + recs = _assessor.load_records(_dir_with(tmp_path, "genuine.json"), PIN) + rules = _rules("KSI-TEST-01") + block = _assessor.attach(rules, recs) + by_rule = {r["rule_id"]: r["assessor_comment_responses"] for r in rules if r["assessor_comment_responses"]} + sdr = {"providerExtensions": {"assessorRecords": block, "assessorCommentResponses": by_rule}} + _assessor.verify_output(sdr, recs) + bad = copy.deepcopy(sdr) + next(iter(bad["providerExtensions"]["assessorCommentResponses"].values()))[0]["text"] = "Softened." + with pytest.raises(SystemExit): + _assessor.verify_output(bad, recs) diff --git a/tests/test_controls.py b/tests/test_controls.py index 75e0e53a..c9a67ba0 100644 --- a/tests/test_controls.py +++ b/tests/test_controls.py @@ -83,34 +83,7 @@ def test_a_failing_contributing_ksi_downgrades_its_controls(): assert status == "partial" -# --- the SDR carries the per-control record (SDR-CSF-CTF) --------------------------- - -def test_the_sdr_summarizes_every_projected_control(): - sdr = _load("build_sdr", "build-sdr.py") - irs = _projection() - rows = sdr.build_rev5_controls(irs, json.loads(CATALOG.read_text())) - assert [r["control_id"] for r in rows] == [ir["control-id"] for ir in irs] - vocab = {"Implemented", "Partially Implemented", "Planned", "Alternative Implementation", "Not Applicable"} - for r, ir in zip(rows, irs, strict=True): - assert r["implementation_status"] in vocab, r["control_id"] - assert r["mechanisms"].startswith(ir["statements"][0]["remarks"]) - assert r["parameters"] == [{"param_id": sp["param-id"], "values": sp.get("values", [])} - for sp in ir.get("set-parameters", [])] - for k in ("verification", "validation", "independent_verification", "independent_validation"): - assert r[k].strip(), (r["control_id"], k) - # Parameter values the FedRAMP profile sets are carried, not dropped. - assert sum(len(r["parameters"]) for r in rows) > 200 - - -def test_sdr_verification_points_at_the_contributing_ksis(): - sdr = _load("build_sdr2", "build-sdr.py") - rows = sdr.build_rev5_controls(_projection(), json.loads(CATALOG.read_text())) - claimed = [r for r in rows if r["contributing_ksis"] and r["implementation_status"] != "Not Applicable" - and r["origination"] != "inherited"] - assert claimed - for r in claimed: - assert all(k in r["verification"] for k in r["contributing_ksis"]), r["control_id"] - +# --- the SDR carries the per-control record: tests/test_sdr.py ---------------------- def test_the_sdr_builder_fails_closed_without_the_inventory(tmp_path): r = subprocess.run([sys.executable, str(REPO / "scripts" / "build-sdr.py"), diff --git a/tests/test_evidence_freshness.py b/tests/test_evidence_freshness.py index 1472f2e8..110d05d2 100644 --- a/tests/test_evidence_freshness.py +++ b/tests/test_evidence_freshness.py @@ -141,9 +141,9 @@ def test_each_artifact_is_measured_by_its_own_field_and_window(tmp_path): assert freshness.main(["--url", path.as_uri()]) == 0 path.write_text(json.dumps({"generated_at": hours_ago(800)})) assert freshness.main(["--url", path.as_uri()]) == 1 - # A nested field (the SDR's metadata.last_updated). + # A nested field (the SDR's metadata.lastUpdated, FedRAMP's schema). sdr = tmp_path / "security-decision-record.json" - sdr.write_text(json.dumps({"metadata": {"last_updated": hours_ago(5)}})) + sdr.write_text(json.dumps({"metadata": {"lastUpdated": hours_ago(5)}})) assert freshness.main(["--url", sdr.as_uri()]) == 0 diff --git a/tests/test_evidence_watchdog.py b/tests/test_evidence_watchdog.py index 0bdd0e27..6ce98c98 100644 --- a/tests/test_evidence_watchdog.py +++ b/tests/test_evidence_watchdog.py @@ -131,7 +131,7 @@ def test_each_listed_artifact_past_its_own_window_is_counted(): objects[".well-known/trust-center.json"] = {"generated_at": "2026-09-24T11:00:00Z"} objects[".well-known/vdr-trend.json"] = {"updated_at": "2026-09-24T11:00:00Z"} # 31 days: past the deploy window. - objects[".well-known/security-decision-record.json"] = {"metadata": {"last_updated": "2026-08-25T00:00:00Z"}} + objects[".well-known/security-decision-record.json"] = {"metadata": {"lastUpdated": "2026-08-25T00:00:00Z"}} out = compute(objects) assert sorted(out["stale"]) == [".well-known/security-decision-record.json", ".well-known/vdr-trend.json"] assert out["metrics"]["StaleArtifacts"] == 2 diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py index 3abc7099..4dbe3634 100644 --- a/tests/test_reconcile.py +++ b/tests/test_reconcile.py @@ -54,7 +54,7 @@ def clean_set(): "resource_type": "AWS::S3::Bucket", "native_id": ARN_BUCKET}, ], } - sdr = {"record_type": "security-decision-record", "impact_level": "Moderate", "ksi_signal_id": SIGNAL_ID} + sdr = {"$schema": "x", "providerExtensions": {"impact_level": "Moderate", "ksi_signal_id": SIGNAL_ID}} cpo = {"package_type": "certification-package-overview", "impact_level": "Moderate", "ksi_signal_id": SIGNAL_ID} vdr = {"impact_level": "Moderate", "class": "C", "ksi_signal_id": SIGNAL_ID, @@ -152,10 +152,23 @@ def test_c_risk_accepted_needs_vdr_decision_id(): assert any("no non-null decision_id" in x for x in v) +def _fields(s, art): + """The dict an artifact keeps its impact level and signal binding in.""" + return rc.sdr_fields(s[art]) if art == "sdr" else s[art] + + +def test_d_reads_the_sdr_provider_extensions_not_its_top_level(): + s = clean_set() + s["sdr"]["impact_level"] = "Moderate" + s["sdr"]["providerExtensions"]["impact_level"] = "Low" + v = rc.check_d_impact(s["signal"], s["sdr"], s["cpo"], s["vdr"], s["dashboard_html"]) + assert v == ["(d) sdr impact level is 'low', expected 'moderate'"] + + @pytest.mark.parametrize("art", ["sdr", "cpo"]) def test_d_impact_catches_a_low_sdr_or_cpo(art): s = clean_set() - s[art]["impact_level"] = "Low" + _fields(s, art)["impact_level"] = "Low" v = rc.check_d_impact(s["signal"], s["sdr"], s["cpo"], s["vdr"], s["dashboard_html"]) assert v == [f"(d) {art} impact level is 'low', expected 'moderate'"] @@ -163,7 +176,7 @@ def test_d_impact_catches_a_low_sdr_or_cpo(art): @pytest.mark.parametrize("art", ["sdr", "cpo"]) def test_d_impact_catches_an_sdr_or_cpo_with_no_impact_level(art): s = clean_set() - del s[art]["impact_level"] + del _fields(s, art)["impact_level"] v = rc.check_d_impact(s["signal"], s["sdr"], s["cpo"], s["vdr"], s["dashboard_html"]) assert v == [f"(d) {art} carries no extractable impact level"] @@ -184,7 +197,7 @@ def test_d_impact_catches_low_dashboard(): @pytest.mark.parametrize("art", ["vdr", "sdr", "cpo"]) def test_e_binding_catches_mismatched_signal_id(art): s = clean_set() - s[art]["ksi_signal_id"] = "99999999-9999-4999-8999-999999999999" + _fields(s, art)["ksi_signal_id"] = "99999999-9999-4999-8999-999999999999" v = rc.check_e_binding(s["signal"], s["sdr"], s["cpo"], s["vdr"]) assert v == [f"(e) {art} signal_id '99999999-9999-4999-8999-999999999999' != inventory signal_id '{SIGNAL_ID}'"] diff --git a/tests/test_sdr.py b/tests/test_sdr.py new file mode 100644 index 00000000..39190cbf --- /dev/null +++ b/tests/test_sdr.py @@ -0,0 +1,221 @@ +"""The Security Decision Record in FedRAMP's schema: every applicable rule, +every KSI and every Moderate control, none stronger than what backs it.""" + +import copy +import importlib.util +import json +import shutil +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import _controls # noqa: E402 +import _fedramp_schema as fs # noqa: E402 +import _frr # noqa: E402 +import _rules # noqa: E402 + +spec = importlib.util.spec_from_file_location("build_sdr", REPO / "scripts" / "build-sdr.py") +sdr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sdr) + +CATALOG = REPO / "infrastructure" / "schemas" / "ksi-catalog.json" +FIX_SIGNAL = REPO / "tests" / "fixtures" / "clean" / "ksi-signal.json" +NOW = datetime(2026, 11, 5, 12, tzinfo=timezone.utc) +POLICIES = {"ac": "https://example.invalid/ac-policy.md"} + + +def _catalog(): + return json.loads(CATALOG.read_text()) + + +def _signal(method="policy-validation", status="pass", overrides=None): + """The clean fixture inventory plus a result for every KSI.""" + s = json.loads(FIX_SIGNAL.read_text()) + s["ksis"] = [] + for f in _catalog()["KSI"].values(): + for i in f["indicators"]: + s["ksis"].append({"id": i["id"], "status": status, "method": method, + "evidence": {"validation_ids": ["v-1"], "failed_validation_ids": [], + "component_refs": ["aws::function::app"]}}) + for k in s["ksis"]: + k.update((overrides or {}).get(k["id"], {})) + return s + + +def _build(signal=None, prev=None, records=None, irs=None, runtime=(), rules_by_ksi=None): + signal = signal or _signal() + return sdr.build(_catalog(), signal, records or _frr.load(), _rules.applicable_provider_rules(), + irs if irs is not None else _controls.project(signal, _catalog()), + sdr.moderate_ids(), rules_by_ksi or {}, list(runtime), None if runtime else "none supplied", + [], {"fedramp_class": "C", "impact_level": "Moderate"}, prev or {}, {}, NOW, POLICIES) + + +@pytest.fixture(scope="module") +def doc(): + return _build() + + +# --- shape --------------------------------------------------------------------------- + +def test_it_validates_against_fedramps_schema(doc): + assert fs.validate(doc, "sdr") == [] + + +def test_every_applicable_rule_is_listed_once(doc): + ids = [r["frrID"] for r in doc["fedRampRequirements"]] + assert sorted(ids) == sorted(_rules.applicable_provider_rules()) and len(ids) == len(set(ids)) + + +def test_not_applicable_rules_carry_no_status_and_say_why(doc): + na = [r for r in doc["fedRampRequirements"] if r["providerExtensions"]["status"] == "Not Applicable"] + assert na + for r in na: + assert "frrImplementationStatus" not in r + assert any(line.startswith("Not applicable: ") for line in r["frrImplementation"]) + + +def test_every_ksi_is_listed_with_tests_and_evidence(doc): + assert len(doc["keySecurityIndicators"]) == 46 + for k in doc["keySecurityIndicators"]: + assert k["ksiEvidence"] and k["ksiValidation"] and k["ksiAssessment"] + + +def test_every_moderate_control_is_addressed(doc): + got = {c["controlId"] for c in doc["securityControls"] if c["providerExtensions"]["inModerateBaseline"]} + assert got == {sdr.nist_id(c) for c in sdr.moderate_ids()} and len(got) == 323 + + +def test_every_projected_control_is_carried_with_its_parameters(doc): + irs = _controls.project(_signal(), _catalog()) + assert len(doc["securityControls"]) == len(irs) + assert sum(len(c.get("parameterValues", [])) for c in doc["securityControls"]) > 200 + + +def test_every_control_names_its_20x_coverage(doc): + for c in doc["securityControls"]: + assert "**20x coverage:**" in c["controlImplementationDescription"], c["controlId"] + + +def test_nist_ids(): + assert sdr.nist_id("ac-2") == "AC-2" and sdr.nist_id("ac-2.12") == "AC-2(12)" + + +# --- honesty rules ------------------------------------------------------------------- + +def _ir(cid, status="implemented", origination="sp-system"): + return {"control-id": cid, "props": [{"name": "implementation-status", "value": status}, + {"name": "control-origination", "value": origination}]} + + +@pytest.mark.parametrize("ir,ksis,ksi_st,frrs,frr_st,want", [ + (_ir("sc-13", origination="inherited"), {}, {}, {}, {}, "Implemented"), + (_ir("sc-13"), {"sc-13": {"KSI-A"}}, {"KSI-A": "Implemented"}, {}, {}, "Implemented"), + (_ir("sc-13"), {"sc-13": {"KSI-A"}}, {"KSI-A": "Partially Implemented"}, {}, {}, "Partially Implemented"), + (_ir("sc-13"), {}, {}, {"sc-13": {"R-1"}}, {"R-1": "Implemented"}, "Implemented"), + (_ir("sc-13"), {}, {}, {"sc-13": {"R-1"}}, {"R-1": "Partially Implemented"}, "Partially Implemented"), + (_ir("ac-1"), {}, {}, {}, {}, "Implemented"), # policy control, policy published + (_ir("sc-1"), {}, {}, {}, {}, "Partially Implemented"), # no sc policy in POLICIES + (_ir("ac-2"), {}, {}, {}, {}, "Partially Implemented"), # nothing backs it + (_ir("ac-2", status="partial"), {}, {}, {}, {}, "Partially Implemented"), + (_ir("ac-2", status="not-applicable"), {}, {}, {}, {}, None), +]) +def test_a_control_is_no_stronger_than_what_backs_it(ir, ksis, ksi_st, frrs, frr_st, want): + assert sdr.control_basis(ir, ksis, ksi_st, frrs, frr_st, POLICIES)[0] == want + + +@pytest.mark.parametrize("k,want", [ + ({"status": "pass", "method": "policy-validation", "evidence": {"validation_ids": ["v"]}}, "Implemented"), + ({"status": "pass", "method": "policy-validation", "evidence": {"validation_ids": []}}, "Partially Implemented"), + ({"status": "pass", "method": "inventory-attestation", "evidence": {}}, "Partially Implemented"), + ({"status": "pass", "method": "documentation", "evidence": {}}, "Partially Implemented"), + ({"status": "fail", "method": "policy-validation", "evidence": {"validation_ids": ["v"]}}, "Not Implemented"), +]) +def test_a_ksi_is_implemented_only_when_its_measures_are_checked(k, want): + assert sdr.ksi_status(k) == want + + +def test_runtime_revalidation_counts_as_a_second_automated_method(): + rt = [{"validation_id": "r-1", "component_refs": ["aws::function::app"], "result": "pass"}] + d = _build(runtime=rt) + assert {k["providerExtensions"]["automatedMethods"] for k in d["keySecurityIndicators"]} == {2} + d = _build() + assert {k["providerExtensions"]["automatedMethods"] for k in d["keySecurityIndicators"]} == {1} + + +# --- known-bad: each fails closed for its own reason -------------------------------- + +def test_a_moderate_control_missing_from_the_projection_fails(): + irs = [ir for ir in _controls.project(_signal(), _catalog()) if ir["control-id"] != "ac-2"] + with pytest.raises(SystemExit, match="1 Moderate control.*ac-2"): + _build(irs=irs) + + +def test_a_signal_without_a_ksi_result_fails(): + s = _signal() + s["ksis"] = [k for k in s["ksis"] if k["id"] != "KSI-CED-RAT"] + with pytest.raises(SystemExit, match="no result for KSI-CED-RAT"): + _build(signal=s) + + +def test_an_invalid_record_set_never_reaches_the_sdr(tmp_path): + bad = json.loads(_frr.DEFAULT_PATH.read_text()) + bad["records"] = bad["records"][1:] + p = tmp_path / "frr.json" + p.write_text(json.dumps(bad)) + with pytest.raises(SystemExit, match="no record for 1 applicable rule"): + _frr.load(p) + + +def test_a_document_failing_the_schema_is_not_written(tmp_path, doc): + bad = copy.deepcopy(doc) + bad["keySecurityIndicators"][0].pop("ksiTests") + with pytest.raises(SystemExit, match="does not validate"): + fs.write_validated(bad, "sdr", tmp_path / "sdr.json") + assert not (tmp_path / "sdr.json").exists() + + +# --- version and history ------------------------------------------------------------- + +def test_version_starts_at_1_0_0_after_the_unversioned_record(doc): + assert _build(prev={"metadata": {"version": "live (regenerated each deploy)"}})["metadata"]["version"] == "1.0.0" + + +def test_unchanged_content_keeps_its_version_and_a_change_bumps_it(doc): + again = _build(prev=doc) + assert again["metadata"]["version"] == doc["metadata"]["version"] + prev = copy.deepcopy(doc) + prev["metadata"]["version"] = "1.0.7" + prev["providerExtensions"]["contentSha256"] = "0" * 64 + assert _build(prev=prev)["metadata"]["version"] == "1.0.8" + + +def test_daily_metrics_carry_over_from_the_record_before_the_reshape(): + old = {"metrics": {"daily_metric_data": [{"date": "2026-11-01", "vdr_blocking": 0, "signal_failing": 0}]}} + hist = _build(prev=old)["providerExtensions"]["metrics"]["daily_metric_data"] + assert [h["date"] for h in hist] == ["2026-11-01", "2026-11-05"] + + +# --- the human view says the same thing as the JSON --------------------------------- + +def test_markdown_counts_match_the_json(doc): + md = sdr.render_markdown(doc) + s = doc["providerExtensions"]["summary"] + assert f"- Rev5 Moderate controls (323): {sdr._summary_line(s['securityControlsModerate'])}" in md + assert md.count("\n| `") == len(doc["fedRampRequirements"]) + len(doc["keySecurityIndicators"]) + assert "No assessor record" in md + + +def test_the_builder_runs_end_to_end(tmp_path): + if not shutil.which("opa"): + pytest.skip("opa not available") + sig = tmp_path / "ksi-signal.json" + sig.write_text(json.dumps(_signal())) + out = tmp_path / "sdr.json" + assert sdr.main(["--catalog", str(CATALOG), "--ksi-signal", str(sig), "--runtime-signal", str(tmp_path / "none"), + "--vdr", str(tmp_path / "none"), "--profile", str(REPO / "data" / "system-profile.json"), + "--output", str(out), "--md-output", str(tmp_path / "sdr.md")]) == 0 + assert fs.validate(json.loads(out.read_text()), "sdr") == [] diff --git a/website/assets/js/trust.js b/website/assets/js/trust.js index 058d0425..dde882b2 100644 --- a/website/assets/js/trust.js +++ b/website/assets/js/trust.js @@ -23,7 +23,7 @@ const ARTIFACTS = [ ['boundary-map.json', 'The authorization boundary map'], ['reconcile-report.json', 'The reconciliation gate’s report for this deploy'], ['ksi-signal.json', 'The canonical inventory and KSI results'], - ['security-decision-record.json', 'Security Decision Record: how each KSI and each Rev5 control is met (replaces the SSP)'], + ['security-decision-record.json', 'Security Decision Record: every applicable FedRAMP rule, KSI and Rev5 Moderate control, with status and evidence'], ['certification-package-overview.json', 'Certification Package Overview'], ['vdr-report.json', 'Vulnerability Detection and Response report'], ['vdr-detail-report.json', 'Vulnerability Detail Report (CR26 VER-RPT-VDT): findings not accepted'], From 1db3a03e0dfad20f162900fc3220e0ca269da735 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 15:52:33 -0500 Subject: [PATCH 09/97] feat(frr): SDR records state what the schema-shaped SDR delivers on deploy SDR-CSO-MTD Implemented; SDR-CSO-FRR, SDR-CSX-KSI, FRC-CSX-VVK and FRC-CSX-VVR gaps restated for the post-deploy state. Co-Authored-By: Claude Opus 5.5 --- data/frr-records.json | 87 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/data/frr-records.json b/data/frr-records.json index d84c52f9..6a767bed 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -639,9 +639,9 @@ { "frr_id": "FRC-CSX-VVK", "status": "Partially Implemented", - "implementation": "Terraform plans are checked against the OPA policies on every deploy, and the runtime Lambda re-evaluates the same Rego against live configuration for object stores and the distribution daily (ksi-signal-runtime.json).", + "implementation": "Terraform plans are checked against the OPA policies on every deploy, and the runtime Lambda re-evaluates the same Rego against live configuration daily. The SDR counts the automated methods in place for each KSI (plan gate, plus runtime only when the signed runtime signal verifies and is fresh).", "evidence": [], - "gap": "Class C requires at least two automated methods for each KSI. The runtime check covers only some component types, and no record counts the automated methods per KSI to show two exist for each.", + "gap": "On the latest inputs 34 KSIs have two automated methods, 3 have one, and 9 have none (KSI-CED-RAT, KSI-IAM-AAM, KSI-IAM-APM, KSI-IAM-ELP, KSI-IAM-JIT, KSI-IAM-SNU, KSI-IAM-SUS, KSI-SCR-MIT, KSI-SCR-MON). To close: a second automated method for the 3, and two for the 9.", "closes_with": [ "W7" ] @@ -651,9 +651,9 @@ "status": "Partially Implemented", "implementation": "scripts/_frr.py checks the rule records automatically on every build: the set matches the pinned corpus, and every piece of evidence cited resolves to a test, workflow step, published artifact or decision.", "evidence": [], - "gap": "Resolution shows the evidence exists, not that it passed for the rule: the records are not yet tied to test results (conformance-results.json) or published in the SDR.", + "gap": "Resolution shows the evidence exists, not that it passed for the rule: the records published in the SDR are not tied to test results (conformance-results.json). To close: fail the build when a test cited as evidence fails, and publish the result per record.", "closes_with": [ - "W3" + "W10" ] }, { @@ -1132,22 +1132,55 @@ { "frr_id": "SDR-CSO-FRR", "status": "Partially Implemented", - "implementation": "data/frr-records.json holds one record per applicable rule (154, derived from the pinned corpus's applicability metadata), each with status, implementation, evidence, gap and the work that closes it; scripts/_frr.py fails the build on a missing or extra record, an Implemented record without evidence, or evidence that does not resolve.", - "evidence": [], - "gap": "The published SDR does not yet carry these records in fedRampRequirements (next W3 PR), and the independent verification and validation halves stay empty until an assessor is engaged (operator's decision of 2026-11-05). To close: Publish the records in the SDR's fedRampRequirements with rule-specific verification and validation drawn from the evidence each cites; flip each record as its workstream lands.", + "implementation": "data/frr-records.json holds one record per applicable rule (154, derived from the pinned corpus's applicability metadata); scripts/_frr.py fails the build on a missing or extra record, an Implemented record without evidence, or evidence that does not resolve. The published SDR carries every record in fedRampRequirements with its status, implementation, evidence and gap.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_sdr.py::test_every_applicable_rule_is_listed_once" + }, + { + "kind": "test", + "ref": "tests/test_sdr.py::test_an_invalid_record_set_never_reaches_the_sdr" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build Security Decision Record (SDR — 20x SSP replacement)" + }, + { + "kind": "artifact", + "ref": ".well-known/security-decision-record.json" + } + ], + "gap": "The independent verification and validation halves stay empty: no assessor is engaged (operator's decision of 2026-11-05). To close: engage an assessor.", "closes_with": [ - "W3", "sam" ] }, { "frr_id": "SDR-CSO-MTD", - "status": "Partially Implemented", - "implementation": "security-decision-record.json metadata: last_updated (timestamp), source_of_update (build-sdr.py).", - "evidence": [], - "gap": "Version is the constant 'live (regenerated each deploy)', not an identifier; field names do not match the schema (version/lastUpdated/updateSource). To close: Emit a real version (semver bumped on content change, or commit SHA) and the schema's field names.", - "closes_with": [ - "W3" + "status": "Implemented", + "implementation": "The SDR carries metadata.version, metadata.lastUpdated and metadata.updateSource in FedRAMP's schema. The version is semver: unchanged content keeps the previous version and a content change bumps it, judged by a sha256 of the content excluding metadata.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_sdr.py::test_version_starts_at_1_0_0_after_the_unversioned_record" + }, + { + "kind": "test", + "ref": "tests/test_sdr.py::test_unchanged_content_keeps_its_version_and_a_change_bumps_it" + }, + { + "kind": "test", + "ref": "tests/test_sdr.py::test_it_validates_against_fedramps_schema" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build Security Decision Record (SDR — 20x SSP replacement)" + }, + { + "kind": "artifact", + "ref": ".well-known/security-decision-record.json" + } ] }, { @@ -1163,11 +1196,29 @@ { "frr_id": "SDR-CSX-KSI", "status": "Partially Implemented", - "implementation": "security-decision-record.json under /.well-known/ carries a record per KSI with an explanation and artifacts, built from the KSI signal's checks.", - "evidence": [], - "gap": "Verification and validation are the same boilerplate for every KSI (build-sdr.py), not per-KSI statements of the measures, their cycle, and how their automation was verified and validated.", + "implementation": "The SDR carries every KSI with its tests (the OPA rules annotated to it), its evidence, its status rule, and the number of automated methods in place.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_sdr.py::test_every_ksi_is_listed_with_tests_and_evidence" + }, + { + "kind": "test", + "ref": "tests/test_sdr.py::test_a_ksi_is_implemented_only_when_its_measures_are_checked" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build Security Decision Record (SDR — 20x SSP replacement)" + }, + { + "kind": "artifact", + "ref": ".well-known/security-decision-record.json" + } + ], + "gap": "Validation text is generated per method, not written per KSI: 6 distinct statements across 46 KSIs, and the 9 KSIs with no automated method state no measures or cycle. To close: per-KSI statements of the measures, their cycle, and how their automation was verified and validated.", "closes_with": [ - "W3" + "W7", + "W10" ] }, { From f8ffd97859bf8efcbd512b83baecc4c0fb23ee56 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:31:10 -0500 Subject: [PATCH 10/97] SCN: explicit evaluation gate, append-only evaluation log, published notifications and deadline checks Every change now carries an explicit significant-change evaluation (SCN-Type, SCN-Rationale, and SCN-Record for adaptive/transformative/emergency); a missing tag fails instead of defaulting to routine-recurring, and class-change blocks. The deploy appends every merged evaluation to a signed, append-only log and publishes every register row in FedRAMP's SCN schema with a Markdown view and an Atom feed, listed in the CPO. Business-day deadlines (US federal holidays) are read from the vendored rules; a scheduled workflow fails on anything overdue. Notification is publication only; nothing is sent. SCN-Type: not-significant SCN-Rationale: change-management process and compliance reporting only; no component, boundary, data flow or access change to the system. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 41 ++ .github/workflows/scn-deadlines.yml | 68 +++ .github/workflows/scn-tag.yml | 67 ++- data/frr-records.json | 365 ++++++++++---- docs/incident-response.md | 8 +- docs/policies/cm-policy.md | 4 +- docs/scn/README.md | 66 ++- docs/scn/TEMPLATE-transformative.md | 51 ++ docs/scn/evaluation-examples.md | 98 ++++ docs/scn/scn-register.csv | 14 +- infrastructure/schemas/freshness-config.json | 35 ++ scripts/_bizdays.py | 79 ++++ scripts/_scn.py | 408 ++++++++++++++++ scripts/build-cpo.py | 26 + scripts/build-scn.py | 410 ++++++++++++++++ scripts/check-scn-deadlines.py | 77 +++ scripts/scn-evaluate.py | 81 ++++ tests/fixtures/scn/bad/adaptive-no-record.md | 2 + tests/fixtures/scn/bad/class-change.md | 2 + .../scn/bad/index-previous-with-dropped.json | 17 + tests/fixtures/scn/bad/log-rewritten.json | 36 ++ tests/fixtures/scn/bad/log-truncated.json | 16 + tests/fixtures/scn/bad/missing-rationale.md | 4 + tests/fixtures/scn/bad/missing-type.md | 4 + .../fixtures/scn/bad/placeholder-rationale.md | 2 + .../scn/bad/register-adaptive-no-risks.csv | 2 + .../scn/bad/register-dates-out-of-order.csv | 2 + tests/fixtures/scn/bad/register-routine.csv | 2 + .../register-transformative-incomplete.csv | 2 + tests/fixtures/scn/bad/routine-with-record.md | 3 + .../scn/bad/transformative-too-soon.md | 3 + tests/fixtures/scn/bad/two-types.md | 3 + tests/fixtures/scn/bad/unknown-type.md | 2 + tests/fixtures/scn/good/adaptive.md | 3 + tests/fixtures/scn/good/emergency.md | 3 + tests/fixtures/scn/good/not-significant.md | 5 + .../scn/good/register-transformative.csv | 2 + .../good/routine-recurring-legacy-spelling.md | 2 + .../fixtures/scn/good/transformative-ready.md | 3 + tests/fixtures/scn/index.json | 10 + tests/fixtures/scn/log-previous.json | 11 + tests/test_scn.py | 446 ++++++++++++++++++ 42 files changed, 2318 insertions(+), 167 deletions(-) create mode 100644 .github/workflows/scn-deadlines.yml create mode 100644 docs/scn/TEMPLATE-transformative.md create mode 100644 docs/scn/evaluation-examples.md create mode 100644 scripts/_bizdays.py create mode 100644 scripts/_scn.py create mode 100644 scripts/build-scn.py create mode 100644 scripts/check-scn-deadlines.py create mode 100644 scripts/scn-evaluate.py create mode 100644 tests/fixtures/scn/bad/adaptive-no-record.md create mode 100644 tests/fixtures/scn/bad/class-change.md create mode 100644 tests/fixtures/scn/bad/index-previous-with-dropped.json create mode 100644 tests/fixtures/scn/bad/log-rewritten.json create mode 100644 tests/fixtures/scn/bad/log-truncated.json create mode 100644 tests/fixtures/scn/bad/missing-rationale.md create mode 100644 tests/fixtures/scn/bad/missing-type.md create mode 100644 tests/fixtures/scn/bad/placeholder-rationale.md create mode 100644 tests/fixtures/scn/bad/register-adaptive-no-risks.csv create mode 100644 tests/fixtures/scn/bad/register-dates-out-of-order.csv create mode 100644 tests/fixtures/scn/bad/register-routine.csv create mode 100644 tests/fixtures/scn/bad/register-transformative-incomplete.csv create mode 100644 tests/fixtures/scn/bad/routine-with-record.md create mode 100644 tests/fixtures/scn/bad/transformative-too-soon.md create mode 100644 tests/fixtures/scn/bad/two-types.md create mode 100644 tests/fixtures/scn/bad/unknown-type.md create mode 100644 tests/fixtures/scn/good/adaptive.md create mode 100644 tests/fixtures/scn/good/emergency.md create mode 100644 tests/fixtures/scn/good/not-significant.md create mode 100644 tests/fixtures/scn/good/register-transformative.csv create mode 100644 tests/fixtures/scn/good/routine-recurring-legacy-spelling.md create mode 100644 tests/fixtures/scn/good/transformative-ready.md create mode 100644 tests/fixtures/scn/index.json create mode 100644 tests/fixtures/scn/log-previous.json create mode 100644 tests/test_scn.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index a327c2de..f6c9c311 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1608,6 +1608,30 @@ jobs: --offering ../data/cds/offering.json --profile ../data/system-profile.json \ --overview certification-overview-package.json --ksi-signal ksi-signal.json + # Significant Change Notifications (SCN-CSO-HIS/HRM/NOM/INF) and the + # append-only evaluation log (SCN-CSO-MAR). The builder fails if this build + # would drop or rewrite anything the previous published copies hold, so they + # are fetched first: a 404 means nothing has been published yet, any other + # failure stops the deploy rather than restarting the record. + - name: Build Significant Change Notifications and the evaluation log + working-directory: infrastructure + env: + GH_TOKEN: ${{ github.token }} + run: | + fetch() { + code=$(curl -sSL -o "$2" -w '%{http_code}' "https://samaydlette.com/.well-known/$1") \ + || { echo "::error::cannot fetch the published $1"; exit 1; } + case "$code" in + 200) ;; + 404) rm -f "$2"; echo "$1 not published yet" ;; + *) echo "::error::fetching the published $1 returned HTTP $code"; exit 1 ;; + esac + } + fetch significant-change-notifications.json previous-scn-index.json + fetch scn-evaluations.json previous-scn-evaluations.json + python3 ../scripts/build-scn.py --previous-index previous-scn-index.json \ + --previous-log previous-scn-evaluations.json --commit "$GITHUB_SHA" + # The authorization-boundary map the trust center draws: the inventory's # components placed by docs/boundary/boundary-classification.json, connected # by what Terraform state says references what, with the data-flow table @@ -1700,6 +1724,10 @@ jobs: cosign sign-blob --yes --new-bundle-format --bundle policy-index.bundle policy-index.json cosign sign-blob --yes --new-bundle-format --bundle security-decision-record.bundle security-decision-record.json cosign sign-blob --yes --new-bundle-format --bundle certification-package-overview.bundle certification-package-overview.json + for a in significant-change-notifications scn-evaluations; do + cosign sign-blob --yes --new-bundle-format --bundle "$a.bundle" "$a.json" + done + cosign sign-blob --yes --new-bundle-format --bundle significant-change-notifications.atom.bundle significant-change-notifications.atom - name: Sign trust-center artifacts (boundary map, gate report, trust center) working-directory: infrastructure @@ -1728,6 +1756,8 @@ jobs: for a in vdr-detail-report vdr-accepted-vulnerabilities; do files="$files $a.md"; done files="$files ongoing-certification-report.json ongoing-certification-report.md" for a in certification-overview-package availability policy-index security-decision-record certification-package-overview; do files="$files $a.json $a.md"; done + for a in significant-change-notifications scn-evaluations; do files="$files $a.json $a.md"; done + files="$files significant-change-notifications.atom" for a in boundary-map reconcile-report trust-center conformance-results boundary-corroboration; do [ -f "$a.json" ] || continue files="$files $a.json" @@ -1787,6 +1817,14 @@ jobs: aws s3 cp "$a.bundle" "s3://$BUCKET_NAME/.well-known/$a.bundle" --content-type application/json --cache-control "public, max-age=300" aws s3 cp "$a.md" "s3://$BUCKET_NAME/.well-known/$a.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" done + # Significant Change Notifications + evaluation log (SCN-*) + bundles + human-readable, and the Atom feed + for a in significant-change-notifications scn-evaluations; do + aws s3 cp "$a.json" "s3://$BUCKET_NAME/.well-known/$a.json" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp "$a.bundle" "s3://$BUCKET_NAME/.well-known/$a.bundle" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp "$a.md" "s3://$BUCKET_NAME/.well-known/$a.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" + done + aws s3 cp significant-change-notifications.atom "s3://$BUCKET_NAME/.well-known/significant-change-notifications.atom" --content-type "application/atom+xml; charset=utf-8" --cache-control "public, max-age=300" + aws s3 cp significant-change-notifications.atom.bundle "s3://$BUCKET_NAME/.well-known/significant-change-notifications.atom.bundle" --content-type application/json --cache-control "public, max-age=300" # Trust center: the generated boundary map, the gate's own report (it exists # only because the gate above passed) and the trust center's data. for a in boundary-map reconcile-report trust-center conformance-results boundary-corroboration; do @@ -1877,6 +1915,9 @@ jobs: check policy-index.json .well-known/policy-index.json check security-decision-record.json .well-known/security-decision-record.json check certification-package-overview.json .well-known/certification-package-overview.json + check significant-change-notifications.json .well-known/significant-change-notifications.json + check significant-change-notifications.atom .well-known/significant-change-notifications.atom + check scn-evaluations.json .well-known/scn-evaluations.json check iiw.csv .well-known/iiw.csv check boundary-map.json .well-known/boundary-map.json check reconcile-report.json .well-known/reconcile-report.json diff --git a/.github/workflows/scn-deadlines.yml b/.github/workflows/scn-deadlines.yml new file mode 100644 index 00000000..424efff6 --- /dev/null +++ b/.github/workflows/scn-deadlines.yml @@ -0,0 +1,68 @@ +# ============================================================================= +# SCN DEADLINE CHECK (SCN-ADP-NTF, SCN-TRF-NAF/NAV/UPD, SCN-CSO-EMG) +# ============================================================================= +# Once a business day, compares the published evaluation log and notification +# index with the register on main and fails on anything late and outstanding: +# a change named as adaptive/transformative/emergency with no record by its +# deadline, a notice not yet published after its deadline, or a transformative +# change's documentation not updated within 30 business days. The rules are in +# scripts/_scn.py (tests/test_scn.py). A failure raises or updates one GitHub +# issue, the same paper trail the evidence nightly uses. +# +# Reads only public URLs and this checkout: no AWS credentials, no writes +# outside the issue. +# ============================================================================= + +name: SCN Deadlines + +on: + schedule: + - cron: "17 13 * * 1-5" + workflow_dispatch: + +permissions: + contents: read + +jobs: + deadlines: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + result: ${{ steps.check.outcome }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Check SCN deadlines + id: check + run: python3 scripts/check-scn-deadlines.py + + alert: + name: Raise the overdue-SCN issue + runs-on: ubuntu-latest + needs: [deadlines] + if: always() && needs.deadlines.result == 'failure' + permissions: + issues: write + steps: + - name: Raise or update the overdue-SCN issue + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const TITLE = 'Significant Change Notification overdue'; + const runUrl = `${context.payload.repository.html_url}/actions/runs/${context.runId}`; + const body = [ + 'The scheduled SCN deadline check failed: a significant change has a record or notice past its deadline.', + '', + `The "Check SCN deadlines" step in [this run](${runUrl}) names each item. Add the record to docs/scn/scn-register.csv and deploy, which publishes it; a notice is given when the deploy publishes it.`, + ].join('\n'); + const open = (await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, repo: context.repo.repo, state: 'open', per_page: 100, + })).filter(i => i.title === TITLE && !i.pull_request); + if (open.length) { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: open[0].number, body, + }); + } else { + await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title: TITLE, body }); + } diff --git a/.github/workflows/scn-tag.yml b/.github/workflows/scn-tag.yml index 7492dcbe..54a6d7b4 100644 --- a/.github/workflows/scn-tag.yml +++ b/.github/workflows/scn-tag.yml @@ -1,16 +1,19 @@ # ============================================================================= -# SCN TAG VALIDATOR +# SCN EVALUATION GATE (SCN-CSO-EVA) # ============================================================================= -# Enforces FedRAMP 20x Significant Change Notification rules SCN-CSO-EVA and -# SCN-CSO-MAR by reading the SCN-Type tag from either the PR description (on -# pull_request events) or the commit message (on push events). The git history -# is the SCN audit record (SCN-CSO-MAR), and this gate ensures every entry in -# that record carries a recognized SCN-Type categorization. +# Every change carries an explicit significant-change evaluation: an SCN-Type +# line, a one-line SCN-Rationale and, for adaptive, transformative and emergency +# changes, the SCN-Record it will be documented in. The pull request's +# description is read on pull_request events; on a push to main, the merged +# pull request's description (or the commit message for a direct push). # -# Categorization defaults to routine-recurring per docs/policies/cm-policy.md -# when no SCN-Type line is present. The validator only fails on a malformed -# SCN-Type value; missing tags are treated as the default. Operators must -# explicitly upgrade the categorization for Adaptive or Transformative changes. +# There is no default. A missing type, a missing or placeholder rationale, or an +# unrecognized value fails. class-change always fails: it is a new +# certification, not a significant change. A transformative change fails until +# its initial and final plans have been published 30 and 10 business days +# (SCN-TRF-NIP/NFP). The rules live in scripts/_scn.py and are unit-tested in +# tests/test_scn.py; the deploy writes every merged evaluation to the +# append-only log at /.well-known/scn-evaluations.json (SCN-CSO-MAR). # ============================================================================= name: SCN Tag @@ -28,41 +31,33 @@ permissions: jobs: scn-tag: runs-on: ubuntu-latest + timeout-minutes: 5 steps: - - name: Validate SCN-Type tag + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Untrusted text (a PR description, a commit message) reaches the script + # only through the environment, never by ${{ }} interpolation into shell. + - name: Validate the SCN evaluation env: PR_BODY: ${{ github.event.pull_request.body }} COMMIT_MSG: ${{ github.event.head_commit.message }} EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} run: | set -euo pipefail if [ "${EVENT_NAME}" = "pull_request" ]; then - content="${PR_BODY:-}" + SCN_TEXT="${PR_BODY:-}" source_label="PR description" else - content="${COMMIT_MSG:-}" + SCN_TEXT="${COMMIT_MSG:-}" source_label="commit message" + pr=$(printf '%s\n' "${SCN_TEXT}" | head -n1 | sed -nE 's/^Merge pull request #([0-9]+) from .*/\1/p') + if [ -n "${pr}" ]; then + SCN_TEXT=$(gh api "repos/${REPO}/pulls/${pr}" --jq '.body // ""') + source_label="description of merged PR #${pr}" + fi fi - # Extract the SCN-Type line if present. Match optional leading hashes - # (Markdown heading) and case-insensitive 'SCN-Type:' prefix. - tag=$(printf '%s\n' "${content}" | grep -iE '^[[:space:]]*##?[[:space:]]*SCN-Type:' | head -n1 | sed -E 's/^[[:space:]]*##?[[:space:]]*SCN-Type:[[:space:]]*//I' | tr -d '[:space:]' || true) - case "${tag}" in - adaptive|routine-recurring|transformative) - echo "SCN-Type: ${tag} (from ${source_label})" - ;; - emergency) - # Emergency-change procedure per CR26 SCN-CSO-EMG. The change itself - # is permitted to bypass the formal SCN flow; the retroactive - # documentation is required per the IR runbook. - echo "::warning::SCN-Type: emergency (from ${source_label}). Emergency-change procedure invoked per SCN-CSO-EMG; retroactive SCN documentation required within 5 business days. See docs/incident-response.md." - ;; - "") - # Per CM policy default: missing tag = routine-recurring. This is the - # documented default for sole-operator direct pushes; do not fail. - echo "::notice::No SCN-Type line in ${source_label}; defaulting to routine-recurring per docs/policies/cm-policy.md." - ;; - *) - echo "::error::SCN-Type '${tag}' in ${source_label} is not a recognized value. Must be one of: adaptive | routine-recurring | transformative | emergency. See SCN-CSO-EVA / SCN-CSO-EMG in docs/policies/cm-policy.md." - exit 1 - ;; - esac + export SCN_TEXT + python3 scripts/scn-evaluate.py --text-env SCN_TEXT --source "${source_label}" diff --git a/data/frr-records.json b/data/frr-records.json index d84c52f9..ebfb3e40 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -969,164 +969,343 @@ { "frr_id": "SCN-ADP-NTF", "status": "Partially Implemented", - "implementation": "Each adaptive SCN record carries a new-risks summary (scn-register.csv new_risks_summary; SCN md section 5).", + "implementation": "An adaptive record's notice (with its new-risks summary, which the register requires) is due 10 business days after date_finished, counted with US federal holidays (scripts/_bizdays.py); it is given by publication in the certification data, and the index records the deadline and the date first published. The scheduled SCN deadline check fails and raises an issue when the notice is not published by its deadline. Notification is publication only, the method FedRAMP's rules name (an update to the Certification Data); nothing is sent to FedRAMP or anyone else.", "evidence": [], - "gap": "No notification has been sent to anyone: notify_by says 'No necessary parties'; FedRAMP is always a necessary party (FRD 'All Necessary Parties'), so this is not N/A; no sent-date is recorded for any SCN (artifact requires sent date and applied date). To close: Send each adaptive SCN through the NOM mechanism within 10 business days of date_finished, record sent_on in the register, and add a check that fails when date_finished + 10 business days passes without sent_on.", + "gap": "Not yet shown working: no notice has been published. The first deploy after this change publishes all six adaptive records; SCN-2026-006 is due 2026-11-10 and the other five are already past their deadlines, so they will be published marked late. Closes when a deploy publishes them (the operator approves deploys).", "closes_with": [ - "W5" + "sam" ] }, { "frr_id": "SCN-CSO-ARI", - "status": "Partially Implemented", - "implementation": "docs/scn/SCN-2026-001-silk-reeling.md sections 3-5 (boundary delta, SIA by control family, verification result) add information beyond SCN-CSO-INF.", - "evidence": [], - "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "Each published notification adds, beyond the required items, the change's status, initiation/finish/verification dates, the pull requests that made it, the new-risks summary, its notices with deadlines and publication dates, and a link to the full record (boundary change, impact by control family, verification result).", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_build_publishes_every_register_row_in_the_fedramp_schema" + }, + { + "kind": "artifact", + "ref": ".well-known/significant-change-notifications.json" + } ], "may_rationale": "Exercised. Change records add the boundary change, the security impact by control family and the verification result, so a reviewer can see why a change was classified as it was." }, { "frr_id": "SCN-CSO-EMG", - "status": "Partially Implemented", - "implementation": "docs/incident-response.md:86-104 emergency-change procedure; scn-tag.yml:53-58 recognizes 'emergency'.", - "evidence": [], - "gap": "Never exercised; 'notify all necessary parties' has no channel; no gate that requires the retroactive SCN materials (only a warning). To close: Add a check that opens/blocks until a retroactive SCN record exists for each emergency-tagged commit within 5 business days; route notification through the NOM mechanism.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "docs/incident-response.md (Emergency-change procedure), linked from the Certification Package Overview: an emergency change carries SCN-Type emergency with its SCN-Record; the gate accepts it and states the record is due; the retroactive register row (emergency = yes) is due within 5 business days of the change merging, enforced by the scheduled SCN deadline check, and the published notification marks each retroactive notice against that deadline. Notification is publication. Not yet invoked.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_an_emergency_change_owes_its_record_within_five_business_days" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_emergency_notices_are_retroactive_within_five_business_days" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_good_evaluation_passes" + }, + { + "kind": "step", + "ref": "scn-deadlines.yml#Check SCN deadlines" + } ], "may_rationale": "Kept available for emergencies: a security fix may need to ship before notice can be given. When it is used, the retroactive records the rule requires are produced within its timeframe." }, { "frr_id": "SCN-CSO-EVA", - "status": "Partially Implemented", - "implementation": ".github/workflows/scn-tag.yml (required check, confirmed live by check-manual-settings 'main requires 4 status checks', EXPECTED_REQUIRED_CHECKS l.39) validates an SCN-Type line; docs/policies/cm-policy.md:5; six adaptive records with categorization rationale (docs/scn/SCN-2026-00[1-6]*.md section 1).", - "evidence": [], - "gap": "Evaluation is not forced: a missing tag passes as routine-recurring (scn-tag.yml:59), and only 8 of 273 commits since 2026-08-01 carry any SCN-Type, so most changes have no recorded evaluation; the gate has no 'not significant' or 'class change' outcome and requires no rationale; commit 437be7b carries 'SCN-Type: Routine Recurring', a value the case statement (l.49-66) rejects. Adaptive tags do not require an SCN record (e.g. #376 tagged adaptive has none). Artifact gap: no example (real or fictitious) for routine-recurring, transformative or class change. SCN-2026-001 (new non-FedRAMP external interconnection) was judged adaptive 'at the upper edge'; the rationale should be revisited. To close: Require an explicit SCN-Type plus a one-line rationale on every PR (fail when missing); add values not-significant and class-change (class-change blocks and points to reassessment); fail when adaptive/transformative lacks a docs/scn record; add documented examples for each type (fictitious allowed).", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "Every pull request (or direct push) must carry SCN-Type (not-significant, routine-recurring, adaptive, transformative, emergency, class-change), a one-line SCN-Rationale and, for adaptive/transformative/emergency, the SCN-Record that will document it; the required scn-tag check (scripts/scn-evaluate.py, rules in scripts/_scn.py) fails a change without them, with no default, and always fails class-change. A change named adaptive, transformative or emergency whose record is not in the register by its deadline fails the scheduled SCN deadline check. docs/scn/evaluation-examples.md gives one example per type (real for adaptive, fictitious otherwise), each checked against the gate. Changes merged before this gate mostly carry no evaluation; the evaluation log starts at the change that introduced it. SCN-2026-001's categorization (adaptive, at the upper edge, for a new external interconnection since removed by SCN-2026-006) is unchanged and left to the operator's review.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_evaluation_fails_for_its_reason" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_an_empty_description_fails_there_is_no_default" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_workflow_runs_the_tested_gate_with_no_default" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_every_type_has_a_documented_example" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_an_adaptive_change_with_no_record_is_overdue_after_ten_business_days" + }, + { + "kind": "step", + "ref": "scn-tag.yml#Validate the SCN evaluation" + } ] }, { "frr_id": "SCN-CSO-HIS", - "status": "Partially Implemented", - "implementation": "docs/scn/ holds all six SCNs since 2026-06 (README: rows never deleted).", - "evidence": [], - "gap": "The SCN register and records are in the repository only; nothing under /.well-known/ publishes them, and the CPO lists no SCN material or how FedRAMP obtains it.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "Every notification is published with the certification data at /.well-known/significant-change-notifications.json (+ .md, Atom feed), listed in the Certification Package Overview, and never pruned: a deploy that would drop a published notification or re-date a notice fails (retention exceeds 12 months). The register rows are never deleted.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_a_build_that_would_drop_a_published_notification_fails" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_a_published_notice_keeps_its_first_publication_date" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_cpo_lists_the_notification_mechanism" + }, + { + "kind": "artifact", + "ref": ".well-known/significant-change-notifications.json" + } ] }, { "frr_id": "SCN-CSO-HRM", - "status": "Partially Implemented", - "implementation": "Human-readable: docs/scn/*.md and scn-register.csv.", - "evidence": [], - "gap": "No JSON form of SCNs or of the related audit records (CSV is not JSON); no explanation of how to request them or how sharing is decided. To close: Emit SCN JSON (schema-valid) and the evaluation log as JSON; add access + sharing statement to the CPO.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "Notifications are published in JSON (FedRAMP SCN schema) and as a Markdown view; the related audit records (the evaluation log) likewise in JSON and Markdown. The index and the Certification Package Overview state that everything is public with no request needed, and how sharing is decided (everything is shared: no federal data, no agency customer).", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_build_publishes_every_register_row_in_the_fedramp_schema" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_log_appends_to_the_previous_published_version" + }, + { + "kind": "artifact", + "ref": ".well-known/significant-change-notifications.json" + }, + { + "kind": "artifact", + "ref": ".well-known/scn-evaluations.json" + } ] }, { "frr_id": "SCN-CSO-INF", - "status": "Partially Implemented", - "implementation": "docs/scn/scn-register.csv columns cover all ten items (fedramp_id, assessor_name, related_vulnerability, scn_type, categorization_rationale, short_description, reason_for_change, customer_impact, plan_and_timeline, sia_reference, approver_name/title); per-change SIA in docs/scn/SCN-2026-00x-*.md.", - "evidence": [], - "gap": "Service Offering FedRAMP ID is the placeholder ''; not emitted in FedRAMP's SCN schema (tests/fixtures/cr26/good/scn.json exists but no builder); SCN-2026-006 verification still unrecorded (trust center scn_unverified); some impacted-control lists reference the retired OSCAL SSP/POA&M. To close: Generate per-SCN JSON in fedramp-significant-change-notifications-schema via _fedramp_schema.write_validated; fill the FedRAMP ID when assigned; record SCN-2026-006 verification.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "docs/scn/scn-register.csv carries the ten items for every record and scripts/_scn.py refuses a row missing one; each deploy writes every row as a notification in FedRAMP's SCN schema (scripts/build-scn.py via _fedramp_schema.write_validated) inside /.well-known/significant-change-notifications.json, with the security impact analysis linked. The Service Offering FedRAMP ID is published as not assigned, with the reason (the offering is not certified). SCN-2026-006's verification is recorded (2026-10-27). Older records' plans name artifacts since retired (the OSCAL SSP/POA&M); they describe the plan as it was.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_build_publishes_every_register_row_in_the_fedramp_schema" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_committed_register_is_valid" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_register_fails_for_its_reason" + }, + { + "kind": "artifact", + "ref": ".well-known/significant-change-notifications.json" + } ] }, { "frr_id": "SCN-CSO-MAR", - "status": "Partially Implemented", - "implementation": "Git history + scn-tag.yml run logs; docs/scn/README.md 'How this relates' explains git history is the audit record; register and per-change records in docs/scn/.", - "evidence": [], - "gap": "Records of evaluation are incomplete because the default-to-routine path records no evaluation; Actions logs expire (default 90 days), so the gate's decisions are not durable; no explanation in the Certification Package of how FedRAMP obtains the records. To close: Write each evaluation (commit, type, rationale, evaluator, date) to an append-only signed log published with the certification data; state access in the CPO.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "Each deploy appends the evaluation of every first-parent commit on main since the last entry (commit, merge date, pull request, SCN type, rationale, record, who merged it) to /.well-known/scn-evaluations.json (+ .md), signed with Sigstore and freshness-governed. The log is append-only against the previous published version: a build that would drop or rewrite an entry, or whose last entry has left main's history, fails. A change merged without a valid evaluation is logged with the problem, not skipped. The Certification Package Overview states how to obtain the records (public, no request needed). The log starts at the change that introduced it; earlier evaluations exist only for the six records in docs/scn.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_log_appends_to_the_previous_published_version" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_append_only_check_fails_each_known_bad_log" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_collect_logs_every_first_parent_commit_from_the_builder_onward" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_collect_fails_when_the_last_logged_commit_left_history" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_cpo_lists_the_notification_mechanism" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build Significant Change Notifications and the evaluation log" + }, + { + "kind": "artifact", + "ref": ".well-known/scn-evaluations.json" + } ] }, { "frr_id": "SCN-CSO-NOM", - "status": "Partially Implemented", - "implementation": "docs/scn/README.md: notification = publishing updated certification data; register notify_by column says 'No necessary parties ... the public GitHub push ... is the notification'.", - "evidence": [], - "gap": "Mechanism documented only in the repo README, not in the Certification Package (CPO); no list of available notification mechanisms (artifact); a git push is not a notification any party receives. To close: Offer real mechanisms (signed SCN JSON feed + Atom/RSS at /.well-known/, email list) and list them in the CPO.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "Mechanisms: an Atom feed (/.well-known/significant-change-notifications.atom, one entry per notice, subscribable with any feed reader), the signed JSON index and its Markdown view. They are listed, with how to obtain the evaluation records, in the Certification Package Overview's significant_change_notifications section. Notification is publication only, the method FedRAMP's rules name (an update to the Certification Data); nothing is sent to FedRAMP or anyone else.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_cpo_lists_the_notification_mechanism" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_feed_is_atom_with_one_entry_per_notice" + }, + { + "kind": "artifact", + "ref": ".well-known/significant-change-notifications.atom" + } ], "may_rationale": "Exercised. Notice is given by a published, signed feed rather than email, because there are no agency customers to address; the mechanism is documented in the certification package." }, { "frr_id": "SCN-RTR-NNR", - "status": "Partially Implemented", - "implementation": "docs/scn/ contains only adaptive SCNs; cm-policy.md:5 routine-recurring changes produce no SCN record.", - "evidence": [], - "gap": "Met at assessment, but no test, workflow step, published artifact or decision record shows it.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "No formal notification for routine recurring changes: the register refuses a routine-recurring (or not-significant) row, and the evaluation gate refuses an SCN-Record on a routine-recurring or not-significant change.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_register_fails_for_its_reason" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_evaluation_fails_for_its_reason" + } ] }, { "frr_id": "SCN-TRF-NAF", - "status": "Partially Implemented", - "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", - "evidence": [], - "gap": "No post-finish notice step or 5-business-day deadline check. To close: Add post-finish notice fields and a deadline check (finish + 5 business days).", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "date_finished adds the after-finish notice, due 5 business days later and published with the record's current information; the scheduled SCN deadline check fails while it is unpublished past the deadline. No transformative change has occurred.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_notices_and_deadlines" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_notices_after_finish_and_verification_are_overdue_unpublished" + }, + { + "kind": "step", + "ref": "scn-deadlines.yml#Check SCN deadlines" + } ] }, { "frr_id": "SCN-TRF-NAV", - "status": "Partially Implemented", - "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", - "evidence": [], - "gap": "No post-verification notice step carrying updates, new risks and the assessment report; no deadline check. To close: Add post-verification notice fields (updates, new risks, SAR link) and a deadline check (verification complete + 5 business days).", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "verified_on adds the after-verification notice, due 5 business days later; the register requires the new-risks summary and sar_reference (the assessment report, or why there is none) once verified_on is set; the scheduled SCN deadline check fails while the notice is unpublished past the deadline. No transformative change has occurred.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_notices_and_deadlines" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_notices_after_finish_and_verification_are_overdue_unpublished" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_an_incomplete_transformative_row_names_every_template_field" + }, + { + "kind": "step", + "ref": "scn-deadlines.yml#Check SCN deadlines" + } ] }, { "frr_id": "SCN-TRF-NFP", - "status": "Partially Implemented", - "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", - "evidence": [], - "gap": "No final-plan notice step or 10-business-day check. To close: Add final-plan notice fields and gate (>=10 business days before start).", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "plan_final = yes publishes the final-plan notice, which carries the record's current information (every notice republishes all of it). The evaluation gate blocks the merge until it has been published at least 10 business days. No transformative change has occurred.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_evaluation_fails_for_its_reason" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_good_evaluation_passes" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_notices_and_deadlines" + }, + { + "kind": "step", + "ref": "scn-tag.yml#Validate the SCN evaluation" + } ] }, { "frr_id": "SCN-TRF-NIP", - "status": "Partially Implemented", - "implementation": "No transformative change has occurred (scn-register.csv: all adaptive); cm-policy.md:9 lists the transformative windows.", - "evidence": [], - "gap": "Only a sentence in policy; no template, no planned-start field, no 30-business-day check, no channel to notify parties. To close: Add a transformative SCN template with initial-plan and security-impact sections, sent_on fields and a gate that blocks merge of a transformative change until initial notice is >=30 business days old.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "A transformative record (docs/scn/TEMPLATE-transformative.md) requires its planned start and a summary of likely security impacts; its initial-plan notice is published with the next deploy and dated. The evaluation gate blocks the merge of the transformative change until the initial-plan notice has been published at least 30 business days (read from the published index; fails closed when the index cannot be read). No transformative change has occurred.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_evaluation_fails_for_its_reason" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_good_evaluation_passes" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_fails_closed_without_the_published_index" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_notices_and_deadlines" + }, + { + "kind": "step", + "ref": "scn-tag.yml#Validate the SCN evaluation" + } ] }, { "frr_id": "SCN-TRF-TPR", - "status": "Not Implemented", - "implementation": "No transformative changes; no recorded policy on third-party review.", - "evidence": [], - "gap": "Neither a third-party review nor an explanation of why one was not engaged exists (artifact accepts either). To close: Add to the transformative template a required 'human validation needed? assessor engaged / reason not engaged' field.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "A transformative record must state whether the change needs human validation (tpr_human_validation) and name the third-party assessor engaged or why none was (tpr_assessor_or_reason); the register refuses a transformative row without them, and the published notification carries both. No transformative change has occurred.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_an_incomplete_transformative_row_names_every_template_field" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_each_known_bad_register_fails_for_its_reason" + } ] }, { "frr_id": "SCN-TRF-UPD", - "status": "Partially Implemented", - "implementation": "Machine-generated documentation (CPO, SDR, OCR) is rebuilt every deploy (deploy-with-opa.yml), so it tracks changes automatically.", - "evidence": [], - "gap": "Hand-written service documentation is not updated after changes: the Secure Configuration Guide still describes the Basic Auth gate retired by SCN-2026-002 (adaptive) months later; no check ties a transformative change to documentation updates within 30 business days. To close: Add a documentation-update checklist/fields to the SCN record (doc, updated_on) and a check that fails 30 business days after finish if listed docs are unchanged; fix the SCG now.", - "closes_with": [ - "W5" + "status": "Implemented", + "implementation": "A finished transformative record must list the service documentation to update (docs_to_update) or say how it was determined none was needed; the published notification carries the 30-business-day deadline and docs_updated_on, and the scheduled SCN deadline check fails when the deadline passes without an update. No transformative change has occurred. (The Secure Configuration Guide's stale Basic Auth text follows an adaptive change, SCN-2026-002, which this rule does not cover.)", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_transformative_documentation_update_is_overdue_after_thirty_business_days" + }, + { + "kind": "step", + "ref": "scn-deadlines.yml#Check SCN deadlines" + } ] }, { diff --git a/docs/incident-response.md b/docs/incident-response.md index 6a3f8f34..88101566 100644 --- a/docs/incident-response.md +++ b/docs/incident-response.md @@ -91,19 +91,19 @@ CR26 `SCN-CSO-EMG` permits the operator to execute significant changes — inclu **What gets bypassed.** During an emergency the operator may: -- Push directly to `main` without the usual SCN-Type categorization deliberation (an `emergency` label is still required on the commit message — see below). +- Merge without the usual categorization deliberation (an `SCN-Type: emergency` evaluation is still required — see below). - Skip the standard advance-notification timing for Adaptive (`SCN-ADP-NTF`, 10 business days) or Transformative (`SCN-TRF-NIP`/`NFP`/`NAF`/`NAV`, 30 / 10 / 5 / 5 business days, plus the 30-business-day documentation update in `SCN-TRF-UPD`) changes. - Defer the threat-model review and per-control implementation-statement updates that a Transformative change would normally trigger. **What still applies.** During and after an emergency the operator must: -1. Tag the commit message with `SCN-Type: emergency` so the SCN audit record reflects the emergency disposition. The CI workflow recognizes `emergency` as a fourth valid SCN-Type value alongside `routine-recurring`, `adaptive`, and `transformative`. +1. Carry `SCN-Type: emergency`, a one-line `SCN-Rationale:` and the `SCN-Record:` the retroactive record will use in the pull request description (or the commit message of a direct push), so the evaluation log records the emergency disposition. The `scn-tag` check accepts it and says the record is due. 2. Follow the rest of this IR runbook: detect, contain, eradicate, recover, after-action. -3. Within 5 business days of incident closure, retroactively produce the SCN materials that would have been produced in advance for the equivalent non-emergency change (per `SCN-CSO-EMG`): summary of the change, summary of new risks identified, copy of the security impact analysis, plan for any verification or assessment of impacted KSIs. +3. Within 5 business days, retroactively produce the SCN materials that would have been produced in advance for the equivalent non-emergency change (per `SCN-CSO-EMG`): summary of the change, summary of new risks identified, copy of the security impact analysis, plan for any verification or assessment of impacted KSIs. Count the 5 business days from the day the emergency change merged. The materials are a row in `docs/scn/scn-register.csv` with `emergency` = `yes` and the change in `evaluation_prs`, plus its record; the next deploy publishes them, which is the notification to all necessary parties. The scheduled SCN deadline check (`.github/workflows/scn-deadlines.yml`) fails and raises an issue if the record is not in the register within 5 business days of the emergency change merging, and the published index marks each retroactive notice against that deadline. 4. Append an after-action entry to the section above with the emergency-procedure invocation noted in the timeline. 5. Reopen the threat model in `docs/architecture-decisions.md` if the emergency change altered the threat surface, and update before the next security review. -**Documentation evidence.** Emergency-change procedures are documented in this section, in `docs/policies/cm-policy.md` (referenced under SCN integration), and in the `.github/workflows/scn-tag.yml` validator (which accepts `emergency` as a valid tag). Together these satisfy `SCN-CSO-EMG`'s "procedures should be documented in the FedRAMP Certification package" requirement. +**Documentation evidence.** Emergency-change procedures are documented in this section, in `docs/policies/cm-policy.md` (referenced under SCN integration), in the `.github/workflows/scn-tag.yml` gate (which accepts `emergency`), and in the Certification Package Overview, which links this section. Together these satisfy `SCN-CSO-EMG`'s "procedures should be documented in the FedRAMP Certification package" requirement. ## Pattern review (KSI-INR-RPI) diff --git a/docs/policies/cm-policy.md b/docs/policies/cm-policy.md index 0ce6d58c..fe8a41ee 100644 --- a/docs/policies/cm-policy.md +++ b/docs/policies/cm-policy.md @@ -1,8 +1,8 @@ # CM — Configuration Management -Configuration management is GitOps. Every change to infrastructure or content reaches `main` through a commit that triggers the OPA gate, the Terraform plan, the apply, the canonical inventory build, the Sigstore signing, and the publish. The default workflow for the sole operator is **direct push to `main`**; pull requests are used only when collaboration or staged review is desired (rare). The commit (or PR description, when a PR is used) is the change request, the review when applicable, the approval, and the audit record. +Configuration management is GitOps. Every change to infrastructure or content reaches `main` through a commit that triggers the OPA gate, the Terraform plan, the apply, the canonical inventory build, the Sigstore signing, and the publish. Changes reach `main` through pull requests; branch protection requires the status checks, `scn-tag` among them, to pass. The PR description is the change request and carries the significant-change evaluation; the merge is the approval. -The **Significant Change Notification (SCN) category** for any change is **Routine Recurring by default; no label is required for routine work**. The operator upgrades the categorization explicitly only when the change meets Adaptive or Transformative criteria per `SCN-CSO-EVA`, by including a line like `SCN-Type: adaptive` or `SCN-Type: transformative` in the commit message (or in the PR description, if a PR is used). A fourth value, `SCN-Type: emergency`, is reserved for emergency-change scenarios per `SCN-CSO-EMG` (see [`docs/incident-response.md`](../incident-response.md) for procedures and retroactive-documentation requirements). The CI workflow validates the tag on every push to `main` and every PR — it fails the build only on a malformed value (a recognized `SCN-Type:` line with a value that isn't one of the four recognized values). Missing tags are treated as Routine Recurring per the default. Audit records of these evaluations are the git history (`SCN-CSO-MAR`). Adaptive and Transformative changes — which produce a security impact analysis — additionally get a durable, structured record in [`docs/scn/`](../scn/): a per-change SCN record carrying the `SCN-CSO-INF` fields plus a copy of the SIA, and a row in the machine-and-human-readable register [`docs/scn/scn-register.csv`](../scn/scn-register.csv) (`SCN-CSO-HRM`), retained for at least 12 months (`SCN-CSO-HIS`). +The **Significant Change Notification (SCN) evaluation** is explicit on every change (`SCN-CSO-EVA`). Each pull request description (or direct-push commit message) carries an `SCN-Type:` line (`not-significant`, `routine-recurring`, `adaptive`, `transformative`, `emergency` or `class-change`), a one-line `SCN-Rationale:`, and for adaptive, transformative and emergency changes the `SCN-Record:` that will document it. The required `scn-tag` check fails a change without them; there is no default. `class-change` always fails: a change of certification class is a new certification and needs a new assessment. A transformative change cannot merge until its initial plan has been published for 30 business days and its final plan for 10. The format and one example per type are in [`docs/scn/evaluation-examples.md`](../scn/evaluation-examples.md); emergency changes follow [`docs/incident-response.md`](../incident-response.md). Each deploy appends every merged evaluation to the signed, append-only evaluation log at `/.well-known/scn-evaluations.json` (`SCN-CSO-MAR`). Adaptive and transformative changes get a row in [`docs/scn/scn-register.csv`](../scn/scn-register.csv) and a record with the security impact analysis; each deploy publishes them in FedRAMP's SCN schema with an Atom feed, which is how notice is given (`SCN-CSO-NOM`, `SCN-CSO-HIS`, `SCN-CSO-HRM`). A scheduled check fails on any record or notice past its deadline. See [`docs/scn/README.md`](../scn/README.md). The canonical inventory at `/.well-known/ksi-signal.json` is the authoritative system inventory (CM-8 family). It is generated automatically from Terraform state, Lambda package locks, and content hashes — see [`docs/ksi-signal.md`](../ksi-signal.md). Each component carries a normalized identifier (PURL/ARN/SHA-256/HBOM ref), a security category (per FIPS 199), and an information-flow tag (per `MAS-CSO-FLO`). diff --git a/docs/scn/README.md b/docs/scn/README.md index 037ab83a..64310f7e 100644 --- a/docs/scn/README.md +++ b/docs/scn/README.md @@ -5,31 +5,47 @@ per the SCN rules in the CR26 corpus (`providers/20x/rules/significant-change-no ## Contents -- **[`scn-register.csv`](scn-register.csv)** — the machine-readable **and** human-readable - SCN register (`SCN-CSO-HRM`). One row per significant change, carrying the - `SCN-CSO-INF` required fields. CSV is the format the FedRAMP SCN beta used to satisfy - both readability requirements simultaneously. -- **`SCN---.md`** — the human-readable SCN record + copy of the security - impact analysis for each change (`SCN-CSO-INF`, including item 9, the SIA). - -## How this relates to the rest of the system - -- **Audit records (`SCN-CSO-MAR`):** the git history is the audit record of SCN - evaluation activity. Every commit to `main` carries an `SCN-Type:` categorization - validated by [`.github/workflows/scn-tag.yml`](../../.github/workflows/scn-tag.yml); - the default is `routine-recurring` per [`docs/policies/cm-policy.md`](../policies/cm-policy.md), - and the operator upgrades to `adaptive` / `transformative` / `emergency` explicitly. - The register and the per-change records here are the durable, structured complement to - that history for Adaptive and Transformative changes (which produce a SIA). -- **Historical notifications (`SCN-CSO-HIS`):** at least **12 months** of SCNs are kept - here with the FedRAMP Certification data. Rows are never deleted; superseded records - are retained. -- **Notification mechanism (`SCN-CSO-NOM`):** the system has no agency customers today, - so notification is met by publishing the updated certification data — the KSI signal, - SDR, CPO and VDR report at `/.well-known/` — alongside these records. -- **Emergency changes (`SCN-CSO-EMG`):** handled per the IR runbook - ([`docs/incident-response.md`](../incident-response.md)); retroactive SCN materials are - added here within the required window. +- **[`scn-register.csv`](scn-register.csv)** — the register, and the source every published + notification is built from. One row per adaptive or transformative change, carrying the + `SCN-CSO-INF` fields, the dates the notice deadlines count from (`date_started`, + `date_finished`, `verified_on`), and for transformative changes the plan, third-party-review and + documentation-update fields. `scripts/_scn.py` validates it (`tests/test_scn.py`); a + routine-recurring row is refused (`SCN-RTR-NNR`). +- **`SCN---.md`** — the human-readable record and copy of the security impact + analysis for each change (`SCN-CSO-INF` item 9). +- **[`evaluation-examples.md`](evaluation-examples.md)** — the evaluation format and one example + per type (`SCN-CSO-EVA`). +- **[`TEMPLATE-transformative.md`](TEMPLATE-transformative.md)** — the transformative record + template: notice windows, third-party review, verification and documentation update. + +## How the process runs + +- **Evaluation (`SCN-CSO-EVA`).** Every pull request carries `SCN-Type:`, `SCN-Rationale:` and, + for adaptive, transformative and emergency changes, `SCN-Record:`. The required `scn-tag` + check ([`.github/workflows/scn-tag.yml`](../../.github/workflows/scn-tag.yml)) fails without + them; there is no default. `class-change` always fails. A transformative change fails until its + initial plan has been published 30 business days and its final plan 10. +- **Audit records (`SCN-CSO-MAR`).** Each deploy appends the evaluation of every change merged + since the last one (commit, type, rationale, record, who merged it, when) to the signed, + append-only log at `/.well-known/scn-evaluations.json` (and `.md`). A deploy that would drop or + rewrite an entry fails. +- **Notification (`SCN-CSO-NOM`).** FedRAMP's rules name the notification method as an update to + the Certification Data. Each deploy publishes every register row in FedRAMP's SCN schema at + `/.well-known/significant-change-notifications.json` (signed), its human-readable view (`.md`) + and an Atom feed (`significant-change-notifications.atom`) anyone can subscribe to; the + Certification Package Overview lists them. A notice is given when it is first published, and + the index records that date beside its deadline. **Nothing is sent by email or to FedRAMP.** +- **Deadlines.** Business days exclude weekends and US federal holidays (`scripts/_bizdays.py`). + Adaptive: 10 after finishing (`SCN-ADP-NTF`). Transformative: initial plan 30 before starting, + final plan 10 before, 5 after finishing, 5 after verification, documentation within 30 + (`SCN-TRF-*`). Emergency: the retroactive record within 5 (`SCN-CSO-EMG`). The scheduled + [`scn-deadlines`](../../.github/workflows/scn-deadlines.yml) workflow fails and raises an issue + on anything overdue. +- **History (`SCN-CSO-HIS`).** Notifications are never pruned; a deploy that would drop one or + re-date a notice fails. Rows here are never deleted. + +The six records below predate the published index: their notices were first published by the +deploy that introduced it, after their deadlines, and the index says so. ## Index diff --git a/docs/scn/TEMPLATE-transformative.md b/docs/scn/TEMPLATE-transformative.md new file mode 100644 index 00000000..2c5ecdfd --- /dev/null +++ b/docs/scn/TEMPLATE-transformative.md @@ -0,0 +1,51 @@ +# SCN-YYYY-NNN — Transformative change: + +Copy this file to `SCN-YYYY-NNN-<slug>.md` and add the row to +[`scn-register.csv`](scn-register.csv) **before** the change starts. The +register row is what the deploy publishes and what the checks read; the columns +named below are required by [`scripts/_scn.py`](../../scripts/_scn.py) for a +transformative row. + +## Timeline and notices + +| Notice | Rule | When | Register column that drives it | +| --- | --- | --- | --- | +| Initial plan | SCN-TRF-NIP | published ≥ 30 business days before `date_started` | the row exists with `date_started` | +| Final plan | SCN-TRF-NFP | published ≥ 10 business days before `date_started` | `plan_final` = `yes` | +| After finishing | SCN-TRF-NAF | within 5 business days of `date_finished` | `date_finished` | +| After verification | SCN-TRF-NAV | within 5 business days of `verified_on` | `verified_on`, `sar_reference`, `new_risks_summary` | +| Documentation updated | SCN-TRF-UPD | within 30 business days of `date_finished` | `docs_to_update`, `docs_updated_on` (or `docs_update_not_required`) | + +A notice is given when a deploy publishes it: the index records each notice's +deadline and the date it was first published. The SCN gate blocks the merge of +the change until the initial and final plans have been published long enough. +Business days exclude weekends and US federal holidays. + +## 1. Categorization (SCN-CSO-EVA) + +Why this change is transformative (FRD-TRF), and why it is not a class change. + +## 2. Required information (SCN-CSO-INF) + +Register columns: `fedramp_id`, `assessor_name`, `related_vulnerability`, +`categorization_rationale`, `short_description`, `reason_for_change`, +`customer_impact`, `plan_and_timeline`, `sia_reference` (this file), +`approver_name`, `approver_title`. + +## 3. Likely security impacts and changes in risk (SCN-TRF-NIP) + +Summary in `new_risks_summary`; detail here. + +## 4. Third-party review (SCN-TRF-TPR) + +- `tpr_human_validation`: `yes` or `no` — does the scope or impact need human validation? +- `tpr_assessor_or_reason`: the assessor engaged to review the scope and impact before the change starts, or why none was engaged. + +## 5. Verification, assessment and validation (SCN-TRF-NAV) + +The plan, the result, and `sar_reference`: the security assessment report, or why there is none. + +## 6. Documentation updates (SCN-TRF-UPD) + +`docs_to_update` (semicolon-separated paths) and, once done, `docs_updated_on`. +If none is needed, `docs_update_not_required` says how that was determined. diff --git a/docs/scn/evaluation-examples.md b/docs/scn/evaluation-examples.md new file mode 100644 index 00000000..0afe8ffe --- /dev/null +++ b/docs/scn/evaluation-examples.md @@ -0,0 +1,98 @@ +# Significant-change evaluation: format and one example per type (SCN-CSO-EVA) + +Every pull request (or direct push) to `main` carries an evaluation in its +description. [`.github/workflows/scn-tag.yml`](../../.github/workflows/scn-tag.yml) +fails the change without one; there is no default. The rules are in +[`scripts/_scn.py`](../../scripts/_scn.py). + +``` +SCN-Type: <not-significant | routine-recurring | adaptive | transformative | emergency | class-change> +SCN-Rationale: <one line: why the change is this type> +SCN-Record: SCN-YYYY-NNN (adaptive, transformative and emergency only) +``` + +The deploy copies every merged evaluation into the append-only log at +`/.well-known/scn-evaluations.json` (SCN-CSO-MAR). + +The types follow FedRAMP's definitions (FRD-SGC, FRD-RTR, FRD-ADP, FRD-TRF). A +change that is not likely to substantively affect the security or privacy +posture is **not-significant**. Examples marked *fictitious* are permitted by the +SCN-CSO-EVA artifact; they show the reasoning and are not changes to this system. + +## not-significant + +*Fictitious.* A typo fix on the about page. + +``` +SCN-Type: not-significant +SCN-Rationale: content-only edit to a public page; no component, boundary, data flow or access change. +``` + +## routine-recurring + +*Fictitious.* The weekly Dependabot patch bump of a library already in the +inventory, with no breaking change. + +``` +SCN-Type: routine-recurring +SCN-Rationale: weekly patch bump of an inventoried library, part of ongoing vulnerability remediation; no breaking change. +``` + +No notification is made (SCN-RTR-NNR): the gate refuses an `SCN-Record` on this +type, and the register refuses a routine-recurring row. + +## adaptive + +*Real:* [SCN-2026-006](SCN-2026-006-remove-anthropic-interconnection.md), the +removal of the Anthropic API interconnection. + +``` +SCN-Type: adaptive +SCN-Rationale: removes an external interconnection, a boundary-crossing flow and a secret; no new service or data, no class change. +SCN-Record: SCN-2026-006 +``` + +The record goes into [`scn-register.csv`](scn-register.csv) and is published +within 10 business days of the change finishing (SCN-ADP-NTF). + +## transformative + +*Fictitious.* The offering begins accepting agency user accounts and storing +their data: a new capability that changes how customers use the service. + +``` +SCN-Type: transformative +SCN-Rationale: introduces federal customer data and agency accounts, which changes existing risk determinations. +SCN-Record: SCN-2027-001 +``` + +The record is written from [`TEMPLATE-transformative.md`](TEMPLATE-transformative.md) +first. The gate blocks the merge until the initial plan has been published 30 +business days and the final plan 10 (SCN-TRF-NIP, SCN-TRF-NFP). + +## emergency + +*Fictitious.* A leaked credential is rotated immediately. + +``` +SCN-Type: emergency +SCN-Rationale: rotates a leaked credential now; waiting for notice windows would itself raise risk. +SCN-Record: SCN-2026-099 +``` + +The retroactive record is due within 5 business days +([`docs/incident-response.md`](../incident-response.md)); the scheduled deadline +check ([`.github/workflows/scn-deadlines.yml`](../../.github/workflows/scn-deadlines.yml)) +fails until it is in the register. + +## class-change + +*Fictitious.* A change that would move the offering out of Class C. + +``` +SCN-Type: class-change +SCN-Rationale: the offering would hold data that takes it above Class C. +``` + +The gate always fails this type: a class change is a new certification and +needs a new assessment, not a significant change notification. diff --git a/docs/scn/scn-register.csv b/docs/scn/scn-register.csv index a0842d96..9de859a3 100644 --- a/docs/scn/scn-register.csv +++ b/docs/scn/scn-register.csv @@ -1,7 +1,7 @@ -scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_finished,notify_by -SCN-2026-001,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),adaptive,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,"POAM-019..024 (all risk-accepted, Low residual) plus the new-ecosystem SCA blind-spot, remediated by adding Syft/Grype SCA of the built Lambda+SPA artifact to CI feeding the VDR aggregator (RA-5/VDR-CSO-DET).",Sam Aydlette,System Owner / Authorizing Operator,Implemented — live in production (first prod deploy 2026-06-03 via PR #82); post-implementation verification COMPLETE 2026-06-08 at prod commit 354da59 (pass: KEV 0 / blocking 0; SSP data-flow + interconnection rendered; deployed Python/JS deps CVE-free; CM-8 inventory complete),2026-06-03,2026-06-06,No necessary parties (no sponsor/customers); the public GitHub push that publishes the certification data is the notification (SCN-CSO-NOM / SCN-ADP-NTF) -SCN-2026-002,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),POAM-021 / POAM-022 / POAM-023 (single-factor shared credential; authorizer-less API; unthrottled credential endpoint),adaptive,"Replaces the end-user authentication mechanism of an existing public-facing component (HTTP Basic Auth -> Cognito with mandatory TOTP MFA + API Gateway JWT authorizer). No class change, no federal customer data, no new external interconnection, so not Transformative; an authentication-model change is not routine-recurring maintenance. Resolves to Adaptive per SCN-CSO-EVA. Backfilled record: the change shipped 2026-06-22 via the gated PR flow; this row completes the register (SCN-CSO-HIS).","Silk Reeling auth replacement: Basic Auth retired; Cognito user pool (mandatory TOTP MFA, 14-char minimum, admin-only creation) with Hosted UI (code + PKCE); JWT authorizer on ANY /api/{proxy+}; $default stays open serving the SPA shell only; stage throttling 20 rps / burst 10; API access logging to a CMK-encrypted log group.",Remediate POAM-021/022/023: retire the single-factor shared credential and gate the data API at the edge with rate limiting.,App users sign in with an invited individual account and mandatory TOTP MFA instead of a shared password. No agency customers exist.,"Gated deploy (OPA + Checkov gate) -> Cognito pool + authorizer + throttling applied via Terraform -> post-implementation verification of live authorizer/MFA/throttling/logging -> KSI signal + OSCAL SSP + POA&M re-emitted. Controls/KSIs re-verified: IA-2(x)/IA-8, AC-7, SC-5, AU-2/AU-3, KSI-IAM-*.",docs/scn/SCN-2026-002-cognito-auth.md,"Shared-credential and unthrottled-endpoint risks retired with POAM-021/023 closure. Residual: TOTP is not phishing-resistant (WebAuthn/FIDO2 upgrade path documented, POAM-025 direction); Cognito availability becomes a sign-in dependency (AWS-managed).",Sam Aydlette,System Owner / Authorizing Operator,"Implemented - live in production since 2026-06-22 (Task 3; POAM-021/022/023 closed); post-implementation verification COMPLETE 2026-07-06 via read-only AWS CLI (JWT authorizer on data routes, MFA ON, 14-char policy, admin-only creation, 20/10 throttling, access logging live)",2026-06-20,2026-06-22,No necessary parties (no sponsor/customers); the public GitHub push that publishes the certification data is the notification (SCN-CSO-NOM / SCN-ADP-NTF) -SCN-2026-003,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),AU-2/AU-11 control-plane audit gap (no durable CloudTrail trail; 90-day Event History only),adaptive,"Net-new internal service component (CloudTrail trail + delivery bucket). No new ingress/egress, no auth change, no customer impact, no class change - not Transformative; a net-new component is not routine-recurring maintenance. Conservatively Adaptive per SCN-CSO-EVA (lower edge: purely internal detective tooling).","Multi-region CloudTrail management-events trail with log-file validation, delivered to a dedicated locked-down S3 bucket; AU-11 tiering 12mo active + Deep Archive to 30mo. Deploy role can create/configure but not stop/delete the trail.",Close the control-plane audit gap: SSP AU-family statements assumed CloudTrail coverage that did not durably exist.,None (internal detective control).,"Bootstrap apply (deploy-role cloudtrail-management grant, manual) -> merge -> gated deploy creates trail + bucket -> post-implementation verification (get-trail-status IsLogging, objects delivered, audit_log_trail component in the next signal).",docs/scn/SCN-2026-003-cloudtrail.md,Strictly additive detective coverage. Residuals documented inline: SSE-S3 not CMK (POAM-029-consistent); no CloudWatch Logs integration / SNS (quarterly review per KSI-MLA-RVL).,Sam Aydlette,System Owner / Authorizing Operator,Implemented - trail live and delivering; post-implementation verification COMPLETE 2026-07-09 (gate green end-to-end; AWSLogs/<account>/ objects delivered; audit_log_trail component in the published signal with data_sensitivity internal; POAM-029/032 dispositions in the published VDR),2026-07-08,2026-07-09,No necessary parties (no sponsor/customers); the public GitHub push that publishes the certification data is the notification (SCN-CSO-NOM / SCN-ADP-NTF) -SCN-2026-004,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),CA-7/IR-6 monitoring gap (evidence nightly failed 2026-09-09..24 unalerted; VDR 12 days stale vs 24h policy; live drift check skipped),adaptive,"Net-new internal components (watchdog Lambda, 3 CloudWatch alarms, CMK-encrypted SNS topic) plus one new outbound flow: alarm metadata by email to the operator email, out of boundary per ROT #3 (flow I). Not Transformative: no federal data, no critical third party handling significant information, no auth/customer-responsibility/class change. Backfilled: shipping PRs #357/#359/#361 carried no SCN-Type and defaulted to routine-recurring.","Evidence SLA watchdog pages the operator by email (SNS) when published evidence goes stale or the evidence nightly fails; nightly runs twice daily, writes a status beacon, and runs its live drift check independently of the vulnerability gate.",The evidence nightly failed every night 2026-09-09..24 with only a GitHub issue as signal; the published VDR breached the 24h reporting policy and the vulnerability gate short-circuited the live reconciliation.,None (no agency customers; internal detective control).,"Bootstrap grants (deploy role, operators group; operator-applied after plan review) -> merge -> gated deploy creates watchdog/topic/alarms -> out-of-band email subscription -> post-implementation verification (forced alarm, delivery metrics).",docs/scn/SCN-2026-004-evidence-alerting.md,"Additive detective control. New outbound flow carries alarm metadata only (no evidence content or credentials); SNS email is not end-to-end encrypted, recorded as acceptable for that content. Watchdog has no internet egress; unattended-path permissions unchanged. ~$0.80/month.",Sam Aydlette,System Owner / Authorizing Operator,"Implemented 2026-09-25; post-implementation verification COMPLETE 2026-09-25 (forced alarm: 2 delivered, 0 failed; all alarms OK). Inventory/SSP/boundary documentation aligned 2026-09-27.",2026-09-25,2026-09-25,No necessary parties (no sponsor/customers); the public GitHub push that publishes the certification data is the notification (SCN-CSO-NOM / SCN-ADP-NTF) -SCN-2026-005,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),AC-6/CM-5 exposure: pull-request CI runs received the full deploy role without the prod environment approval,adaptive,"Changes the privileged-identity model of the CI/CD path (one new read-only IAM role; deploy-role trust narrowed to the reviewer-gated prod environment). Same class as SCN-2026-002's access-model change. Not Transformative: no new external service, data flow or component beyond one IAM role; no federal data; no customer-responsibility or class change.","compliance-check (terraform plan + OPA, on pull requests and main pushes) moves to a read-only plan role; the deploy role then trusts only the reviewer-gated prod environment.","The deploy role trusted pull_request and main-ref subjects because compliance-check planned with it, so every PR run held the full deploy permission set (IAM PutRolePolicy/PassRole on a Lambda role, Lambda code updates) without the prod approval gate.",None (no agency customers).,#374 plan role (operator-applied) -> AWS_PLAN_ROLE_ARN -> #375 compliance-check switch (verified by its CI run) -> this change: deploy trust narrowed to environment:prod (operator-applied after plan review) -> post-implementation verification.,docs/scn/SCN-2026-005-ci-identity-split.md,Net reduction: deploy credentials exist only inside an approved deploy. Plan role reads the Terraform state object (not new exposure: the deploy role already could from the same PRs) and cannot write except the lock item; data-plane reads denied. No cost.,Sam Aydlette,System Owner / Authorizing Operator,"Implemented and verified 2026-09-28 (#374, #375, #378 applied; deploy run 36434974425, PR check 36451980170, main-push check 36483804056).",2026-09-28,,No necessary parties (no sponsor/customers); the public GitHub push that publishes the certification data is the notification (SCN-CSO-NOM / SCN-ADP-NTF) -SCN-2026-006,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),"POAM-020 (Anthropic API interconnection, risk-accepted); POAM-019 (Anthropic key rotation, false positive)",adaptive,"Removes an external interconnection, a boundary-crossing data flow and a secret; changes the authorization boundary and SSP interconnections, so more than routine-recurring. Not Transformative: no new service, data flow or component; no federal data; no customer-responsibility or class change.","Silk Reeling app stops calling the Anthropic API and writes feedback from the scores; the API-key secret, its two IAM grants and env var are removed. LLM feedback becomes a customer-configurable option, disabled in this deployment.",The interconnection was a risk-accepted SA-9/CA-3 item to a service without a FedRAMP authorization and its key had expired; the app produces useful feedback without it.,None (no agency customers). App users get feedback written from their scores.,"App default-off (silk-reeling-mirror ADR-0009) -> this change (terraform plan review, gated deploy) -> post-implementation verification -> close POAM-020 and retire POAM-019.",docs/scn/SCN-2026-006-remove-anthropic-interconnection.md,"Net reduction: no app data leaves the boundary, no third-party credential stored, two IAM grants removed. No new risk.",Sam Aydlette,System Owner / Authorizing Operator,Implemented — verified live 2026-10-27 (secret scheduled for deletion; IAM and key policies clear; published SSP and signal without the interconnection; verify-published PASS),2026-10-26,2026-10-27,No necessary parties (no sponsor/customers); the public GitHub push that publishes the certification data is the notification (SCN-CSO-NOM / SCN-ADP-NTF) +scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_started,date_finished,verified_on,emergency,evaluation_prs,plan_final,tpr_human_validation,tpr_assessor_or_reason,sar_reference,docs_to_update,docs_updated_on,docs_update_not_required +SCN-2026-001,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),adaptive,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,"POAM-019..024 (all risk-accepted, Low residual) plus the new-ecosystem SCA blind-spot, remediated by adding Syft/Grype SCA of the built Lambda+SPA artifact to CI feeding the VDR aggregator (RA-5/VDR-CSO-DET).",Sam Aydlette,System Owner / Authorizing Operator,Implemented — live in production (first prod deploy 2026-06-03 via PR #82); post-implementation verification COMPLETE 2026-06-08 at prod commit 354da59 (pass: KEV 0 / blocking 0; SSP data-flow + interconnection rendered; deployed Python/JS deps CVE-free; CM-8 inventory complete),2026-06-03,,2026-06-06,2026-06-08,no,#82,,,,,,, +SCN-2026-002,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),POAM-021 / POAM-022 / POAM-023 (single-factor shared credential; authorizer-less API; unthrottled credential endpoint),adaptive,"Replaces the end-user authentication mechanism of an existing public-facing component (HTTP Basic Auth -> Cognito with mandatory TOTP MFA + API Gateway JWT authorizer). No class change, no federal customer data, no new external interconnection, so not Transformative; an authentication-model change is not routine-recurring maintenance. Resolves to Adaptive per SCN-CSO-EVA. Backfilled record: the change shipped 2026-06-22 via the gated PR flow; this row completes the register (SCN-CSO-HIS).","Silk Reeling auth replacement: Basic Auth retired; Cognito user pool (mandatory TOTP MFA, 14-char minimum, admin-only creation) with Hosted UI (code + PKCE); JWT authorizer on ANY /api/{proxy+}; $default stays open serving the SPA shell only; stage throttling 20 rps / burst 10; API access logging to a CMK-encrypted log group.",Remediate POAM-021/022/023: retire the single-factor shared credential and gate the data API at the edge with rate limiting.,App users sign in with an invited individual account and mandatory TOTP MFA instead of a shared password. No agency customers exist.,"Gated deploy (OPA + Checkov gate) -> Cognito pool + authorizer + throttling applied via Terraform -> post-implementation verification of live authorizer/MFA/throttling/logging -> KSI signal + OSCAL SSP + POA&M re-emitted. Controls/KSIs re-verified: IA-2(x)/IA-8, AC-7, SC-5, AU-2/AU-3, KSI-IAM-*.",docs/scn/SCN-2026-002-cognito-auth.md,"Shared-credential and unthrottled-endpoint risks retired with POAM-021/023 closure. Residual: TOTP is not phishing-resistant (WebAuthn/FIDO2 upgrade path documented, POAM-025 direction); Cognito availability becomes a sign-in dependency (AWS-managed).",Sam Aydlette,System Owner / Authorizing Operator,"Implemented - live in production since 2026-06-22 (Task 3; POAM-021/022/023 closed); post-implementation verification COMPLETE 2026-07-06 via read-only AWS CLI (JWT authorizer on data routes, MFA ON, 14-char policy, admin-only creation, 20/10 throttling, access logging live)",2026-06-20,,2026-06-22,2026-07-06,no,,,,,,,, +SCN-2026-003,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),AU-2/AU-11 control-plane audit gap (no durable CloudTrail trail; 90-day Event History only),adaptive,"Net-new internal service component (CloudTrail trail + delivery bucket). No new ingress/egress, no auth change, no customer impact, no class change - not Transformative; a net-new component is not routine-recurring maintenance. Conservatively Adaptive per SCN-CSO-EVA (lower edge: purely internal detective tooling).","Multi-region CloudTrail management-events trail with log-file validation, delivered to a dedicated locked-down S3 bucket; AU-11 tiering 12mo active + Deep Archive to 30mo. Deploy role can create/configure but not stop/delete the trail.",Close the control-plane audit gap: SSP AU-family statements assumed CloudTrail coverage that did not durably exist.,None (internal detective control).,"Bootstrap apply (deploy-role cloudtrail-management grant, manual) -> merge -> gated deploy creates trail + bucket -> post-implementation verification (get-trail-status IsLogging, objects delivered, audit_log_trail component in the next signal).",docs/scn/SCN-2026-003-cloudtrail.md,Strictly additive detective coverage. Residuals documented inline: SSE-S3 not CMK (POAM-029-consistent); no CloudWatch Logs integration / SNS (quarterly review per KSI-MLA-RVL).,Sam Aydlette,System Owner / Authorizing Operator,Implemented - trail live and delivering; post-implementation verification COMPLETE 2026-07-09 (gate green end-to-end; AWSLogs/<account>/ objects delivered; audit_log_trail component in the published signal with data_sensitivity internal; POAM-029/032 dispositions in the published VDR),2026-07-08,,2026-07-09,2026-07-09,no,#225,,,,,,, +SCN-2026-004,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),CA-7/IR-6 monitoring gap (evidence nightly failed 2026-09-09..24 unalerted; VDR 12 days stale vs 24h policy; live drift check skipped),adaptive,"Net-new internal components (watchdog Lambda, 3 CloudWatch alarms, CMK-encrypted SNS topic) plus one new outbound flow: alarm metadata by email to the operator email, out of boundary per ROT #3 (flow I). Not Transformative: no federal data, no critical third party handling significant information, no auth/customer-responsibility/class change. Backfilled: shipping PRs #357/#359/#361 carried no SCN-Type and defaulted to routine-recurring.","Evidence SLA watchdog pages the operator by email (SNS) when published evidence goes stale or the evidence nightly fails; nightly runs twice daily, writes a status beacon, and runs its live drift check independently of the vulnerability gate.",The evidence nightly failed every night 2026-09-09..24 with only a GitHub issue as signal; the published VDR breached the 24h reporting policy and the vulnerability gate short-circuited the live reconciliation.,None (no agency customers; internal detective control).,"Bootstrap grants (deploy role, operators group; operator-applied after plan review) -> merge -> gated deploy creates watchdog/topic/alarms -> out-of-band email subscription -> post-implementation verification (forced alarm, delivery metrics).",docs/scn/SCN-2026-004-evidence-alerting.md,"Additive detective control. New outbound flow carries alarm metadata only (no evidence content or credentials); SNS email is not end-to-end encrypted, recorded as acceptable for that content. Watchdog has no internet egress; unattended-path permissions unchanged. ~$0.80/month.",Sam Aydlette,System Owner / Authorizing Operator,"Implemented 2026-09-25; post-implementation verification COMPLETE 2026-09-25 (forced alarm: 2 delivered, 0 failed; all alarms OK). Inventory/SSP/boundary documentation aligned 2026-09-27.",2026-09-25,,2026-09-25,2026-09-25,no,#357;#359;#361,,,,,,, +SCN-2026-005,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),AC-6/CM-5 exposure: pull-request CI runs received the full deploy role without the prod environment approval,adaptive,"Changes the privileged-identity model of the CI/CD path (one new read-only IAM role; deploy-role trust narrowed to the reviewer-gated prod environment). Same class as SCN-2026-002's access-model change. Not Transformative: no new external service, data flow or component beyond one IAM role; no federal data; no customer-responsibility or class change.","compliance-check (terraform plan + OPA, on pull requests and main pushes) moves to a read-only plan role; the deploy role then trusts only the reviewer-gated prod environment.","The deploy role trusted pull_request and main-ref subjects because compliance-check planned with it, so every PR run held the full deploy permission set (IAM PutRolePolicy/PassRole on a Lambda role, Lambda code updates) without the prod approval gate.",None (no agency customers).,#374 plan role (operator-applied) -> AWS_PLAN_ROLE_ARN -> #375 compliance-check switch (verified by its CI run) -> this change: deploy trust narrowed to environment:prod (operator-applied after plan review) -> post-implementation verification.,docs/scn/SCN-2026-005-ci-identity-split.md,Net reduction: deploy credentials exist only inside an approved deploy. Plan role reads the Terraform state object (not new exposure: the deploy role already could from the same PRs) and cannot write except the lock item; data-plane reads denied. No cost.,Sam Aydlette,System Owner / Authorizing Operator,"Implemented and verified 2026-09-28 (#374, #375, #378 applied; the deploy, a pull-request check and a main-push check each passed).",2026-09-28,,2026-09-28,2026-09-28,no,#374;#375;#378,,,,,,, +SCN-2026-006,<FR-CERT-ID>,N/A (sole-operator self-attestation; no 3PAO engaged),"POAM-020 (Anthropic API interconnection, risk-accepted); POAM-019 (Anthropic key rotation, false positive)",adaptive,"Removes an external interconnection, a boundary-crossing data flow and a secret; changes the authorization boundary and SSP interconnections, so more than routine-recurring. Not Transformative: no new service, data flow or component; no federal data; no customer-responsibility or class change.","Silk Reeling app stops calling the Anthropic API and writes feedback from the scores; the API-key secret, its two IAM grants and env var are removed. LLM feedback becomes a customer-configurable option, disabled in this deployment.",The interconnection was a risk-accepted SA-9/CA-3 item to a service without a FedRAMP authorization and its key had expired; the app produces useful feedback without it.,None (no agency customers). App users get feedback written from their scores.,"App default-off (silk-reeling-mirror ADR-0009) -> this change (terraform plan review, gated deploy) -> post-implementation verification -> close POAM-020 and retire POAM-019.",docs/scn/SCN-2026-006-remove-anthropic-interconnection.md,"Net reduction: no app data leaves the boundary, no third-party credential stored, two IAM grants removed. No new risk.",Sam Aydlette,System Owner / Authorizing Operator,Implemented and verified 2026-10-27 (secret scheduled for deletion; IAM and key policies clear; published SSP and signal without the interconnection; verify-published PASS),2026-10-26,,2026-10-27,2026-10-27,no,#449,,,,,,, diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index 7d02d3de..cd4b2f02 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -276,6 +276,41 @@ "signed": true, "signature": ".well-known/conformance-results.bundle", "note": "The conformance suite's results for the method profile (REQ-GOV-2), run, signed and published by the deploy." + }, + { + "path": ".well-known/significant-change-notifications.json", + "cadence": "deploy", + "max_age_hours": 720, + "timestamp": { + "source": "json", + "field": "generated_at" + }, + "signed": true, + "signature": ".well-known/significant-change-notifications.bundle", + "note": "Significant Change Notifications index (SCN-CSO-HIS/HRM/NOM), rebuilt every deploy from docs/scn/scn-register.csv; never pruned." + }, + { + "path": ".well-known/significant-change-notifications.atom", + "cadence": "deploy", + "max_age_hours": 720, + "timestamp": { + "source": "last_modified" + }, + "signed": true, + "signature": ".well-known/significant-change-notifications.atom.bundle", + "note": "Atom feed of the SCN notices (SCN-CSO-NOM); its <updated> is the last notice's date, so the object's last-modified time is used." + }, + { + "path": ".well-known/scn-evaluations.json", + "cadence": "deploy", + "max_age_hours": 720, + "timestamp": { + "source": "json", + "field": "generated_at" + }, + "signed": true, + "signature": ".well-known/scn-evaluations.bundle", + "note": "Append-only significant-change evaluation log (SCN-CSO-MAR), one entry per change merged to main." } ] } diff --git a/scripts/_bizdays.py b/scripts/_bizdays.py new file mode 100644 index 00000000..bb529a65 --- /dev/null +++ b/scripts/_bizdays.py @@ -0,0 +1,79 @@ +"""Business-day arithmetic for FedRAMP timeframes ("bizdays" in the rules). + +A business day is a weekday that is not a US federal holiday (5 U.S.C. 6103). +Holidays are computed, not listed, so no year runs out: + + New Year's Day January 1 + Birthday of Martin Luther King, Jr. third Monday in January + Washington's Birthday third Monday in February + Memorial Day last Monday in May + Juneteenth June 19 + Independence Day July 4 + Labor Day first Monday in September + Columbus Day second Monday in October + Veterans Day November 11 + Thanksgiving Day fourth Thursday in November + Christmas Day December 25 + +A fixed-date holiday that falls on a Saturday is observed the Friday before, +one on a Sunday the Monday after (5 U.S.C. 6103(b), Executive Order 11582). +New Year's Day on a Saturday is therefore observed on December 31 of the +year before. + +Stdlib only. +""" + +from datetime import date, timedelta +from functools import lru_cache + +_FIXED = ((1, 1), (6, 19), (7, 4), (11, 11), (12, 25)) + + +def _nth_weekday(year, month, weekday, n): + """The nth `weekday` (Mon=0) of the month; n=-1 is the last.""" + if n > 0: + first = date(year, month, 1) + return first + timedelta(days=(weekday - first.weekday()) % 7 + 7 * (n - 1)) + nxt = date(year + (month == 12), month % 12 + 1, 1) + last = nxt - timedelta(days=1) + return last - timedelta(days=(last.weekday() - weekday) % 7) + + +def _observed(d): + if d.weekday() == 5: + return d - timedelta(days=1) + if d.weekday() == 6: + return d + timedelta(days=1) + return d + + +@lru_cache(maxsize=64) +def federal_holidays(year): + """The observed US federal holidays that fall in `year`.""" + days = set() + for y in (year, year + 1): # next year's New Year's Day may be observed on Dec 31 + days |= {_observed(date(y, m, d)) for m, d in _FIXED} + days |= { + _nth_weekday(y, 1, 0, 3), # MLK + _nth_weekday(y, 2, 0, 3), # Washington's Birthday + _nth_weekday(y, 5, 0, -1), # Memorial Day + _nth_weekday(y, 9, 0, 1), # Labor Day + _nth_weekday(y, 10, 0, 2), # Columbus Day + _nth_weekday(y, 11, 3, 4), # Thanksgiving + } + return frozenset(d for d in days if d.year == year) + + +def is_business_day(d): + return d.weekday() < 5 and d not in federal_holidays(d.year) + + +def add_business_days(d, n): + """The date n business days after d (n < 0: before). n == 0 returns d.""" + step = 1 if n >= 0 else -1 + left = abs(n) + while left: + d += timedelta(days=step) + if is_business_day(d): + left -= 1 + return d diff --git a/scripts/_scn.py b/scripts/_scn.py new file mode 100644 index 00000000..c4645b04 --- /dev/null +++ b/scripts/_scn.py @@ -0,0 +1,408 @@ +"""Significant Change Notification rules as code (SCN-*). + +One module for everything the SCN gate, the deploy-time builder and the +scheduled deadline check share: + +- the evaluation every change carries (SCN-CSO-EVA): an `SCN-Type:` line, an + `SCN-Rationale:` line and, for a change that needs a notification, an + `SCN-Record:` line naming its record in docs/scn/scn-register.csv; +- the register (docs/scn/scn-register.csv) and its validation (SCN-CSO-INF, + SCN-RTR-NNR, the transformative template fields); +- the notices each record owes and their deadlines in business days, read from + the vendored FedRAMP rules (SCN-ADP-NTF, SCN-TRF-NIP/NFP/NAF/NAV/UPD), plus + the operator's 5-business-day retroactive window for emergency changes + (SCN-CSO-EMG, docs/incident-response.md); +- the evaluation log (SCN-CSO-MAR) and the published notification index + (SCN-CSO-HIS): both only ever grow, and a build that would drop or rewrite a + published entry fails. + +Notification is publication: FedRAMP's rules name the method for every SCN +notice as an update to the FedRAMP Certification Data, and that is all this +code does. It sends nothing to anyone. + +Stdlib only; dates are ISO strings in every file. +""" + +import csv +import re +import sys +from datetime import date +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _bizdays # noqa: E402 +import _rules # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +REGISTER = REPO / "docs" / "scn" / "scn-register.csv" +BASE = "https://samaydlette.com/.well-known/" +INDEX_NAME = "significant-change-notifications" +LOG_NAME = "scn-evaluations" +FEED_NAME = "significant-change-notifications.atom" +SOURCE_URL = "https://github.com/sam-aydlette/samaydlette.com/blob/main/" + +# Every value an evaluation may carry. `class-change` is a valid evaluation +# that the gate refuses: a change of certification class is not a significant +# change at all but a new certification, and it needs a new assessment. +TYPES = ("not-significant", "routine-recurring", "adaptive", "transformative", "emergency", "class-change") +# Evaluations that owe a notification record. +NEEDS_RECORD = ("adaptive", "transformative", "emergency") +# The types a record may have (SCN-RTR-NNR: no formal notification for routine +# recurring changes, and none for changes that are not significant). +RECORD_TYPES = ("adaptive", "transformative") +# Operator procedure (docs/incident-response.md): retroactive SCN materials for +# an emergency change within 5 business days. +EMERGENCY_BIZDAYS = 5 + +SCN_ID_RE = re.compile(r"^SCN-\d{4}-\d{3}$") +_LINE = r"^[ \t>]*(?:#+[ \t]*)?(?:[-*][ \t]+)?(?:\*\*)?{key}(?:\*\*)?[ \t]*:(?:\*\*)?[ \t]*(.*?)[ \t]*$" +TYPE_LINE = re.compile(_LINE.format(key="SCN-Type"), re.I | re.M) +RATIONALE_LINE = re.compile(_LINE.format(key="SCN-Rationale"), re.I | re.M) +RECORD_LINE = re.compile(_LINE.format(key="SCN-Record"), re.I | re.M) +PLACEHOLDER = re.compile(r"^(<.*>|tbd|todo|n/?a|none|-+|\.+|same|see above|see below)$", re.I) +MIN_RATIONALE = 15 +MAX_RATIONALE = 500 + +REGISTER_COLUMNS = ( + "scn_id", "fedramp_id", "assessor_name", "related_vulnerability", "scn_type", "categorization_rationale", + "short_description", "reason_for_change", "customer_impact", "plan_and_timeline", "sia_reference", + "new_risks_summary", "approver_name", "approver_title", "status", "date_initiated", "date_started", + "date_finished", "verified_on", "emergency", "evaluation_prs", "plan_final", "tpr_human_validation", + "tpr_assessor_or_reason", "sar_reference", "docs_to_update", "docs_updated_on", "docs_update_not_required", +) +INF_REQUIRED = ("scn_id", "scn_type", "categorization_rationale", "short_description", "reason_for_change", + "customer_impact", "plan_and_timeline", "sia_reference", "approver_name", "approver_title", + "status", "date_initiated") +DATE_COLUMNS = ("date_initiated", "date_started", "date_finished", "verified_on", "docs_updated_on") +YES_NO = ("yes", "no") + + +class EvaluationError(ValueError): + pass + + +# --- the evaluation on every change ---------------------------------------------- + +def _clean(v): + return v.strip().strip("`*_").strip() + + +def normalize_type(raw): + return re.sub(r"[\s_]+", "-", _clean(raw).lower()) + + +def parse_evaluation(text): + """{type, rationale, record} from a PR body or commit message. + + Raises EvaluationError naming exactly what is missing or wrong. There is no + default: a change with no evaluation is not evaluated.""" + text = text or "" + types = TYPE_LINE.findall(text) + if not types: + raise EvaluationError("no 'SCN-Type:' line: every change needs an explicit significant-change evaluation") + if len(types) > 1: + raise EvaluationError(f"{len(types)} 'SCN-Type:' lines: give exactly one evaluation") + kind = normalize_type(types[0]) + if kind not in TYPES: + raise EvaluationError(f"SCN-Type '{_clean(types[0])}' is not one of: {' | '.join(TYPES)}") + rationales = RATIONALE_LINE.findall(text) + if not rationales: + raise EvaluationError("no 'SCN-Rationale:' line: say in one line why the change is this type") + if len(rationales) > 1: + raise EvaluationError(f"{len(rationales)} 'SCN-Rationale:' lines: give exactly one") + rationale = _clean(rationales[0]) + if PLACEHOLDER.match(rationale) or len(rationale) < MIN_RATIONALE: + raise EvaluationError(f"SCN-Rationale is empty or a placeholder (at least {MIN_RATIONALE} characters " + "explaining the categorization)") + records = [_clean(r) for r in RECORD_LINE.findall(text)] + if len(records) > 1: + raise EvaluationError(f"{len(records)} 'SCN-Record:' lines: name one record") + record = records[0] if records else None + if kind in NEEDS_RECORD: + if not record: + raise EvaluationError(f"SCN-Type {kind} needs an 'SCN-Record: SCN-YYYY-NNN' line naming its record " + "in docs/scn/scn-register.csv (the next free ID if the record is not written yet)") + if not SCN_ID_RE.match(record): + raise EvaluationError(f"SCN-Record '{record}' is not an SCN ID (SCN-YYYY-NNN)") + elif record: + raise EvaluationError(f"SCN-Type {kind} makes no notification (SCN-RTR-NNR), so it names no SCN-Record") + return {"type": kind, "rationale": rationale[:MAX_RATIONALE], "record": record} + + +def gate(text, index=None, today=None): + """(errors, notices, evaluation) for one change's evaluation text. + + `index` is the published notification index, needed only for a + transformative change: it may merge only once its initial plan has been + published 30 business days and its final plan 10 (SCN-TRF-NIP/NFP).""" + today = today or date.today() + try: + ev = parse_evaluation(text) + except EvaluationError as exc: + return [str(exc)], [], None + errors, notices = [], [] + if ev["type"] == "class-change": + errors.append("SCN-Type class-change blocks the merge: a change of certification class is a new " + "certification, not a significant change. Stop and plan a new assessment " + "(docs/policies/cm-policy.md).") + elif ev["type"] == "emergency": + notices.append(f"emergency change (SCN-CSO-EMG): the retroactive record {ev['record']} must be in the " + f"register and published within {EMERGENCY_BIZDAYS} business days " + "(docs/incident-response.md); the scheduled SCN deadline check enforces it") + elif ev["type"] == "transformative": + errors += transformative_lead_errors(ev["record"], index, today) + elif ev["type"] == "adaptive": + notices.append(f"adaptive change: the notice for {ev['record']} must be published within " + f"{timeframe('SCN-ADP-NTF')} business days of finishing (SCN-ADP-NTF)") + return errors, notices, ev + + +def transformative_lead_errors(record, index, today): + if index is None: + return ["cannot read the published notification index, so the transformative lead times cannot be " + "checked (fail closed)"] + scn = next((s for s in index.get("notifications", []) if s.get("scnId") == record), None) + if scn is None: + return [f"{record} has no published notification: a transformative change starts no sooner than " + f"{timeframe('SCN-TRF-NIP')} business days after its initial plan is published (SCN-TRF-NIP)"] + published = {n["stage"]: n.get("publishedOn") for n in scn.get("notices", [])} + errors = [] + for stage, rule in (("initial-plan", "SCN-TRF-NIP"), ("final-plan", "SCN-TRF-NFP")): + on = published.get(stage) + if not on: + errors.append(f"{record}: no {stage} notice published ({rule})") + continue + ready = _bizdays.add_business_days(date.fromisoformat(on), timeframe(rule)) + if today < ready: + errors.append(f"{record}: the {stage} notice was published {on}; the change may start on or after " + f"{ready.isoformat()} ({timeframe(rule)} business days, {rule})") + return errors + + +# --- FedRAMP timeframes ----------------------------------------------------------- + +def timeframe(rule_id): + """The rule's business-day timeframe, from the vendored rules (never typed in).""" + rules, _ = _rules.load_rules() + family, subset = rule_id.split("-")[0], rule_id.split("-")[1] + try: + r = rules["FRR"][family]["data"]["all"][subset][rule_id] + except KeyError: + raise SystemExit(f"::error::SCN: rule {rule_id} not in the vendored rules (fail-closed)") from None + if r.get("timeframe_type") != "bizdays" or not isinstance(r.get("timeframe_num"), int): + raise SystemExit(f"::error::SCN: rule {rule_id} has no business-day timeframe (fail-closed)") + return r["timeframe_num"] + + +# --- the register ------------------------------------------------------------------ + +def read_register(path=REGISTER): + with Path(path).open(newline="", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + header = tuple(reader.fieldnames or ()) + rows = [{k: (v or "").strip() for k, v in r.items()} for r in reader] + return header, rows + + +def _date(v): + try: + return date.fromisoformat(v) if v else None + except ValueError: + return None + + +def validate_register(header, rows, root=REPO): + """Every error in the register; empty means it can be published.""" + errors = [] + if header != REGISTER_COLUMNS: + missing = [c for c in REGISTER_COLUMNS if c not in header] + extra = [c for c in header if c not in REGISTER_COLUMNS] + errors.append(f"register columns differ from the template (missing {missing}, unexpected {extra})") + return errors + seen = set() + for r in rows: + sid = r["scn_id"] or "<no id>" + where = f"{sid}:" + if not SCN_ID_RE.match(r["scn_id"]): + errors.append(f"{where} scn_id is not SCN-YYYY-NNN") + if sid in seen: + errors.append(f"{where} duplicate scn_id") + seen.add(sid) + for c in INF_REQUIRED: + if not r[c]: + errors.append(f"{where} {c} is empty (SCN-CSO-INF)") + if r["scn_type"] and r["scn_type"] not in RECORD_TYPES: + errors.append(f"{where} scn_type '{r['scn_type']}': only adaptive and transformative changes get a " + "notification record (SCN-RTR-NNR)") + for c in DATE_COLUMNS: + if r[c] and _date(r[c]) is None: + errors.append(f"{where} {c} '{r[c]}' is not an ISO date") + order = [(c, _date(r[c])) for c in ("date_initiated", "date_started", "date_finished", "verified_on")] + known = [(c, d) for c, d in order if d] + for (c1, d1), (c2, d2) in zip(known, known[1:], strict=False): + if d2 < d1: + errors.append(f"{where} {c2} {d2} is before {c1} {d1}") + if r["sia_reference"] and not (Path(root) / r["sia_reference"]).is_file(): + errors.append(f"{where} sia_reference {r['sia_reference']} does not exist") + if r["emergency"] not in YES_NO: + errors.append(f"{where} emergency must be yes or no") + if r["emergency"] == "yes" and not r["evaluation_prs"]: + errors.append(f"{where} an emergency record names the change(s) it documents in evaluation_prs") + if r["scn_type"] == "adaptive" and not r["new_risks_summary"]: + errors.append(f"{where} new_risks_summary is empty (SCN-ADP-NTF)") + if r["scn_type"] == "transformative": + errors += [f"{where} {m}" for m in _transformative_errors(r)] + return errors + + +def _transformative_errors(r): + errs = [] + if not r["date_started"]: + errs.append("date_started (the planned start) is required: the notice windows count back from it " + "(SCN-TRF-NIP/NFP)") + if r["plan_final"] not in YES_NO: + errs.append("plan_final must be yes or no (SCN-TRF-NFP)") + if r["tpr_human_validation"] not in YES_NO: + errs.append("tpr_human_validation must be yes or no: does the change need human validation? (SCN-TRF-TPR)") + if not r["tpr_assessor_or_reason"]: + errs.append("tpr_assessor_or_reason: name the third-party assessor engaged, or why none was (SCN-TRF-TPR)") + if not r["new_risks_summary"]: + errs.append("new_risks_summary is empty (SCN-TRF-NIP, SCN-TRF-NAV)") + if r["verified_on"] and not r["sar_reference"]: + errs.append("sar_reference: link the security assessment report, or say why there is none (SCN-TRF-NAV)") + if r["date_finished"] and not (r["docs_to_update"] or r["docs_update_not_required"]): + errs.append("docs_to_update (the service documentation to update) or docs_update_not_required " + "(how that was determined) is required once finished (SCN-TRF-UPD)") + if r["docs_updated_on"] and not r["docs_to_update"]: + errs.append("docs_updated_on without docs_to_update") + return errs + + +# --- notices and deadlines ---------------------------------------------------------- + +def notices_due(row): + """The notices a record owes now: [{stage, rule, deadline, kind}]. + + kind "by": publish on or before the deadline. kind "before-start": publish + at least the rule's business days before the change starts, i.e. on or + before the deadline. An emergency record's notices are retroactive: each is + due within EMERGENCY_BIZDAYS of the change finishing (or starting).""" + out = [] + t = row["scn_type"] + start, finish, verified = _date(row["date_started"]), _date(row["date_finished"]), _date(row["verified_on"]) + emergency = row["emergency"] == "yes" + retro_from = finish or start or _date(row["date_initiated"]) + + def add(stage, rule, deadline, kind): + if emergency: + deadline, kind = _bizdays.add_business_days(retro_from, EMERGENCY_BIZDAYS), "retroactive" + rule = f"{rule}; SCN-CSO-EMG" + out.append({"stage": stage, "rule": rule, "deadline": deadline, "kind": kind}) + + if t == "adaptive" and finish: + add("after-finish", "SCN-ADP-NTF", _bizdays.add_business_days(finish, timeframe("SCN-ADP-NTF")), "by") + if t == "transformative": + if start: + add("initial-plan", "SCN-TRF-NIP", _bizdays.add_business_days(start, -timeframe("SCN-TRF-NIP")), + "before-start") + if row["plan_final"] == "yes": + add("final-plan", "SCN-TRF-NFP", _bizdays.add_business_days(start, -timeframe("SCN-TRF-NFP")), + "before-start") + if finish: + add("after-finish", "SCN-TRF-NAF", _bizdays.add_business_days(finish, timeframe("SCN-TRF-NAF")), "by") + if verified: + add("after-verification", "SCN-TRF-NAV", + _bizdays.add_business_days(verified, timeframe("SCN-TRF-NAV")), "by") + return out + + +def docs_update_deadline(row): + finish = _date(row["date_finished"]) + if row["scn_type"] != "transformative" or not finish: + return None + return _bizdays.add_business_days(finish, timeframe("SCN-TRF-UPD")) + + +def merge_notices(row, previous, today): + """This build's notices for a record. A notice already published keeps the + date it was first published; a new one is published today.""" + prev = {n["stage"]: n for n in previous or []} + out = [] + for n in notices_due(row): + on = prev.get(n["stage"], {}).get("publishedOn") or today.isoformat() + out.append({"stage": n["stage"], "rule": n["rule"], "deadline": n["deadline"].isoformat(), + "publishedOn": on, "onTime": date.fromisoformat(on) <= n["deadline"]}) + # A stage published before is never dropped, even if the record no longer owes it. + for stage, n in prev.items(): + if stage not in {o["stage"] for o in out}: + out.append(n) + return out + + +# --- append-only records ---------------------------------------------------------- + +def append_only_errors(previous, current, key): + """Errors if `current` does not start with every entry of `previous`, + unchanged and in order (`key` names an entry in messages).""" + errors = [] + if len(current) < len(previous): + errors.append(f"{len(previous) - len(current)} published entr(ies) would be dropped") + for i, (p, c) in enumerate(zip(previous, current, strict=False)): + if p != c: + errors.append(f"published entry {i} ({p.get(key)}) would be rewritten") + return errors + + +def retention_errors(previous_index, current_index): + """SCN-CSO-HIS: every notification published before is still published, + with the notices it had and their publication dates.""" + if not previous_index: + return [] + cur = {s["scnId"]: s for s in current_index.get("notifications", [])} + errors = [] + for s in previous_index.get("notifications", []): + sid = s.get("scnId") + if sid not in cur: + errors.append(f"{sid} was published and would be dropped") + continue + now = {n["stage"]: n.get("publishedOn") for n in cur[sid].get("notices", [])} + for n in s.get("notices", []): + if now.get(n["stage"]) != n.get("publishedOn"): + errors.append(f"{sid} {n['stage']} notice published {n.get('publishedOn')} would be " + f"dropped or re-dated") + return errors + + +# --- the scheduled deadline check ------------------------------------------------- + +def overdue(rows, index, log_entries, today): + """Everything late and still outstanding. A notice published late is a + fact the index records (onTime false), not an open item.""" + out = [] + by_id = {r["scn_id"]: r for r in rows} + published = {s["scnId"]: {n["stage"] for n in s.get("notices", [])} + for s in (index or {}).get("notifications", [])} + for e in log_entries: + if e.get("scn_type") not in NEEDS_RECORD or not e.get("scn_record"): + continue + ref, merged = e["scn_record"], date.fromisoformat(e["merged_on"]) + if ref in by_id: + continue + if e["scn_type"] == "transformative": + due = merged + elif e["scn_type"] == "emergency": + due = _bizdays.add_business_days(merged, EMERGENCY_BIZDAYS) + else: + due = _bizdays.add_business_days(merged, timeframe("SCN-ADP-NTF")) + if today > due: + out.append(f"{ref}: named by the {e['scn_type']} change {e.get('commit', '')[:12]} merged {merged}, " + f"but not in the register (due {due.isoformat()})") + for r in rows: + for n in notices_due(r): + if n["stage"] not in published.get(r["scn_id"], set()) and today > n["deadline"]: + out.append(f"{r['scn_id']}: {n['stage']} notice ({n['rule']}) not published; due " + f"{n['deadline'].isoformat()}") + upd = docs_update_deadline(r) + if upd and not r["docs_updated_on"] and not r["docs_update_not_required"] and today > upd: + out.append(f"{r['scn_id']}: service documentation not updated; due {upd.isoformat()} (SCN-TRF-UPD)") + return out diff --git a/scripts/build-cpo.py b/scripts/build-cpo.py index 732986d3..fb5162ef 100644 --- a/scripts/build-cpo.py +++ b/scripts/build-cpo.py @@ -37,8 +37,28 @@ ("availability.json", "Availability feed (CDS-CSO-AVR)", True), ("policy-index.json", "Policy and procedure index (CDS-CSO-IRP)", True), ("iiw.csv", "FedRAMP Integrated Inventory Workbook (Appendix M)", True), + ("significant-change-notifications.json", + "Significant Change Notifications, FedRAMP SCN schema (SCN-CSO-HIS/HRM)", True), + ("significant-change-notifications.atom", "SCN notification feed, Atom (SCN-CSO-NOM)", True), + ("scn-evaluations.json", "Significant-change evaluation log, append-only (SCN-CSO-MAR)", True), ] BASE = "https://samaydlette.com/.well-known/" +# SCN-CSO-NOM / SCN-CSO-MAR / SCN-CSO-HRM: how notifications are given and how +# the records are obtained; scripts/build-scn.py publishes the same statement. +SCN_NOTIFICATION = { + "mechanisms": [ + {"mechanism": "Atom feed", "url": BASE + "significant-change-notifications.atom"}, + {"mechanism": "Signed JSON index", "url": BASE + "significant-change-notifications.json"}, + {"mechanism": "Human-readable view", "url": BASE + "significant-change-notifications.md"}, + ], + "method": "Notification is publication: each notice is an update to the Certification Data at the URLs " + "above, dated when first published. Nothing is sent by email or to FedRAMP directly.", + "evaluation_records": BASE + "scn-evaluations.json", + "access": "Public; no request needed. Everything is published to everyone (no federal data, no agency " + "customer).", + "emergency_procedure": "https://github.com/sam-aydlette/samaydlette.com/blob/main/docs/incident-response.md" + "#emergency-change-procedure-cr26-scn-cso-emg", +} def main(): @@ -100,6 +120,7 @@ def main(): "the Security Decision Record; the independent half is unmet by design.", "current_service": off.get("current_independent_assessment_service"), }, + "significant_change_notifications": SCN_NOTIFICATION, "package_contents": [ {"artifact": fn, "description": desc, "url": BASE + fn, "signed": signed} for fn, desc, signed in PACKAGE_CONTENTS @@ -135,6 +156,11 @@ def render_markdown(c): out.append("| {mod} | {val} | {role} |".format( mod=m.get("module", ""), val=m.get("validation", ""), role=m.get("role", ""))) out += ["", "## Independent assessment (IVV-CSO-ICP)", "", c["independent_assessment"]["status"], ""] + scn = c["significant_change_notifications"] + out += ["## Significant Change Notifications (SCN-CSO-NOM)", "", scn["method"], ""] + out += [f"- {m['mechanism']}: {m['url']}" for m in scn["mechanisms"]] + out += [f"- Evaluation records (SCN-CSO-MAR): {scn['evaluation_records']}", + f"- Access: {scn['access']}", f"- Emergency changes (SCN-CSO-EMG): {scn['emergency_procedure']}", ""] out += ["## Package contents", "", "| Artifact | Description | Signed |", "|---|---|---|"] for p in c["package_contents"]: out.append("| [`{a}`]({u}) | {d} | {s} |".format( diff --git a/scripts/build-scn.py b/scripts/build-scn.py new file mode 100644 index 00000000..a49b803f --- /dev/null +++ b/scripts/build-scn.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +# ============================================================================= +# SIGNIFICANT CHANGE NOTIFICATIONS + EVALUATION LOG (SCN-*) +# ============================================================================= +# Run by the deploy, after the build and before signing. Writes, into --out-dir: +# +# scn/<SCN-ID>.json one notification per register row, +# each validated against FedRAMP's +# SCN schema (write_validated) +# significant-change-notifications.json the index: every notification, +# its notices and their dates (HIS) +# significant-change-notifications.md the human-readable view (HRM) +# significant-change-notifications.atom the subscribable feed (NOM) +# scn-evaluations.json / .md the evaluation log (MAR, HRM) +# +# Both the index and the log only ever grow. The previous published copies are +# fetched by the deploy and passed in; a build that would drop or rewrite a +# published entry fails (SCN-CSO-HIS, SCN-CSO-MAR). A notice keeps the date it +# was first published, which is the date it was given: FedRAMP's rules name the +# notification method for every SCN notice as an update to the Certification +# Data, and this publication is that update. Nothing is sent to anyone. +# +# The evaluation log takes one entry per first-parent commit on main after the +# last one it holds: the SCN-Type, rationale and record from the merged pull +# request's description (or the commit message for a direct push), who merged +# it and when. With no previous log, it starts at the commit that introduced +# this builder, so a lost copy is rebuilt from git rather than restarted. +# +# build-scn.py --previous-index prev-index.json --previous-log prev-log.json +# ============================================================================= + +import argparse +import json +import os +import re +import subprocess +import sys +from datetime import date, datetime, timezone +from pathlib import Path +from xml.sax.saxutils import escape + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _fedramp_schema # noqa: E402 +import _redact # noqa: E402 +import _scn # noqa: E402 + +REPO = _scn.REPO +GENESIS_PATH = "scripts/build-scn.py" +MERGE_RE = re.compile(r"^Merge pull request #(\d+) from ") +RUN_ID_RE = re.compile(r"(actions/runs/|run )\d{6,}") + +MECHANISMS = [ + {"mechanism": "Atom feed", "url": _scn.BASE + _scn.FEED_NAME, + "description": "One entry per notice (adaptive after-finish; transformative initial plan, final plan, " + "after finish, after verification). Any feed reader can subscribe; no account needed."}, + {"mechanism": "Signed JSON index", "url": _scn.BASE + _scn.INDEX_NAME + ".json", + "description": "Every notification in FedRAMP's SCN schema with each notice's publication date and " + "deadline; signed with Sigstore (.bundle beside it)."}, + {"mechanism": "Human-readable view", "url": _scn.BASE + _scn.INDEX_NAME + ".md", + "description": "The same notifications as a Markdown document."}, +] +ACCESS = ("All notifications and the evaluation log are public at the URLs above, in JSON and human-readable " + "form; no request is needed. The register they are built from is docs/scn/scn-register.csv in the " + "public repository.") +SHARING = ("Everything is published to everyone: the system holds no federal data and no agency customer, so " + "there is nothing in a notification to withhold. A notification that ever needed to carry sensitive " + "detail would publish a summary and say how to request the rest.") +NOTIFICATION = ("Notification is publication. FedRAMP's rules name the method for each SCN notice as an update to " + "the FedRAMP Certification Data; publishing here is that update. Nothing is sent by email or to " + "FedRAMP directly. A notice's publishedOn is the first deploy that published it.") + + +def _today(): + return datetime.now(timezone.utc).date() + + +def _load(path): + if not path or not Path(path).is_file() or Path(path).stat().st_size == 0: + return None + return json.loads(Path(path).read_text()) + + +# --- the evaluation log -------------------------------------------------------------- + +def _git(*args, root=REPO): + return subprocess.run(["git", *args], cwd=root, check=True, capture_output=True, text=True).stdout.strip() + + +def gh_pull(number): + """The pull request's description and merger, via the GitHub API (gh, GH_TOKEN).""" + repo = os.environ.get("GITHUB_REPOSITORY", "sam-aydlette/samaydlette.com") + out = subprocess.run(["gh", "api", f"repos/{repo}/pulls/{number}"], check=True, capture_output=True, text=True) + pr = json.loads(out.stdout) + return {"body": pr.get("body") or "", "merged_by": (pr.get("merged_by") or {}).get("login"), + "author": (pr.get("user") or {}).get("login")} + + +def _scrub(text): + """Free text from a PR description, made safe to publish.""" + return LONG_NUMBER.sub("<number>", RUN_ID_RE.sub(r"\1<run>", _redact.redact_text(text))) + + +# A standalone number of 9 to 11 digits in published text is most likely a +# GitHub run ID; 12 digits is an AWS account ID, which the publication gate +# catches. Neither belongs in a notification. +LONG_NUMBER = re.compile(r"(?<![0-9A-Za-z])[0-9]{9,11}(?![0-9A-Za-z])") + + +def long_number_errors(name, text): + return [f"{name}: a {len(m)}-digit number (a run ID?) would be published" for m in LONG_NUMBER.findall(text)] + + +def evaluation_entry(commit, subject, body, merged_on, author, lookup): + m = MERGE_RE.match(subject) + if m: + pr = lookup(int(m.group(1))) + text, evaluator, number = pr["body"], pr.get("merged_by") or pr.get("author"), int(m.group(1)) + else: + text, evaluator, number = f"{subject}\n{body}", author, None + entry = {"commit": commit, "merged_on": merged_on, "pull_request": number, "evaluator": evaluator or "unknown"} + try: + ev = _scn.parse_evaluation(text) + entry.update(scn_type=ev["type"], rationale=_scrub(ev["rationale"]), scn_record=ev["record"], problem=None) + except _scn.EvaluationError as exc: + entry.update(scn_type=None, rationale=None, scn_record=None, problem=f"no valid evaluation: {exc}") + return entry + + +def collect(previous_entries, root=REPO, lookup=gh_pull, head="HEAD"): + """Entries for the first-parent commits after the last one logged.""" + if previous_entries: + last = previous_entries[-1]["commit"] + try: + _git("merge-base", "--is-ancestor", last, head, root=root) + except subprocess.CalledProcessError: + raise SystemExit(f"::error::SCN log: last logged commit {last[:12]} is not an ancestor of {head}; " + "history was rewritten (fail-closed)") from None + span = f"{last}..{head}" + shas = _git("rev-list", "--first-parent", "--reverse", span, root=root).split() + else: + first = _git("log", "--first-parent", "--reverse", "--format=%H", head, "--", GENESIS_PATH, root=root).split() + if not first: + raise SystemExit(f"::error::SCN log: no commit introduces {GENESIS_PATH} (fail-closed)") + parent = _git("rev-list", "--max-count=1", f"{first[0]}^1", root=root) if _has_parent(first[0], root) \ + else None + span = f"{parent}..{head}" if parent else head + shas = _git("rev-list", "--first-parent", "--reverse", span, root=root).split() + out = [] + for sha in shas: + subject, body, when, author = _git("show", "-s", "--format=%s%x00%b%x00%cI%x00%an", sha, root=root) \ + .split("\x00") + out.append(evaluation_entry(sha, subject, body, when[:10], author, lookup)) + return out + + +def _has_parent(sha, root): + try: + _git("rev-parse", "--verify", "--quiet", f"{sha}^1", root=root) + return True + except subprocess.CalledProcessError: + return False + + +def build_log(previous, new_entries, now, commit): + prev = (previous or {}).get("entries", []) + entries = prev + new_entries + errors = _scn.append_only_errors(prev, entries, "commit") + if errors: + raise SystemExit("::error::SCN evaluation log is append-only: " + "; ".join(errors)) + return { + "$comment": "SCN-CSO-MAR: the significant-change evaluation of every change merged to main, one entry per " + "first-parent commit. Append-only: each deploy adds the commits since the last entry, and " + "a deploy that would drop or rewrite an entry fails. Built by scripts/build-scn.py.", + "generated_at": now.isoformat(), + "commit": commit, + "genesis_commit": (previous or {}).get("genesis_commit") or (entries[0]["commit"] if entries else None), + "access": ACCESS, + "entries": entries, + "unrecorded": sum(1 for e in entries if e["problem"]), + } + + +def render_log_md(log): + out = ["# Significant-change evaluation log (SCN-CSO-MAR)", "", + f"_Generated {log['generated_at']} from commit `{log['commit']}`. The signed JSON beside this file is " + "the record; this is its rendering._", "", + f"{len(log['entries'])} evaluations; {log['unrecorded']} without a valid evaluation.", "", + "| Merged | Commit | PR | Type | Record | Evaluator | Rationale |", "|---|---|---|---|---|---|---|"] + for e in reversed(log["entries"]): + rationale = e["rationale"] or f"_{e['problem']}_" + out.append("| {d} | `{c}` | {p} | {t} | {r} | {who} | {why} |".format( + d=e["merged_on"], c=e["commit"][:12], p=f"#{e['pull_request']}" if e["pull_request"] else "direct", + t=e["scn_type"] or "-", r=e["scn_record"] or "", who=e["evaluator"], why=rationale.replace("|", "\\|"))) + return "\n".join(out) + + +# --- notifications ------------------------------------------------------------------- + +def _fedramp_id(row): + v = row["fedramp_id"] + return None if not v or v.startswith("<") else v + + +def scn_document(row, previous_notices, today): + start = row["date_started"] or row["date_initiated"] + plan = {"summary": row["plan_and_timeline"], "plannedStart": start} + if row["date_finished"]: + plan["plannedCompletion"] = row["date_finished"] + doc = { + "$schema": _fedramp_schema.stamp("scn"), + "certificationPackageOverviewUri": _fedramp_schema.CPO_URI, + "scnId": row["scn_id"], + "serviceOfferingFedrampId": _fedramp_id(row), + "serviceOfferingFedrampIdNote": None if _fedramp_id(row) else + "No FedRAMP ID: the offering is not FedRAMP certified (a self-attested demonstration).", + "assessorName": row["assessor_name"], + "relatedVulnerability": row["related_vulnerability"], + "changeType": row["scn_type"].capitalize(), + "changeTypeExplanation": row["categorization_rationale"], + "changeDescription": row["short_description"], + "reason": row["reason_for_change"], + "customerImpact": row["customer_impact"], + "planAndTimeline": plan, + "impactAnalysis": _scn.SOURCE_URL + row["sia_reference"], + "approver": {"name": row["approver_name"], "title": row["approver_title"]}, + "newRisksSummary": row["new_risks_summary"], + "emergency": row["emergency"] == "yes", + "notices": _scn.merge_notices(row, previous_notices, today), + # SCN-CSO-ARI: what else a reviewer needs to judge the change. + "additionalInformation": { + "status": row["status"], + "dateInitiated": row["date_initiated"], + "dateFinished": row["date_finished"] or None, + "verifiedOn": row["verified_on"] or None, + "changes": [p.strip() for p in row["evaluation_prs"].split(";") if p.strip()], + "record": _scn.SOURCE_URL + row["sia_reference"], + }, + } + if row["scn_type"] == "transformative": + doc["transformative"] = { + "planFinal": row["plan_final"] == "yes", + "thirdPartyReview": {"humanValidationNeeded": row["tpr_human_validation"] == "yes", + "assessorOrReasonNotEngaged": row["tpr_assessor_or_reason"]}, + "securityAssessmentReport": row["sar_reference"] or None, + "documentationUpdate": { + "documents": [d.strip() for d in row["docs_to_update"].split(";") if d.strip()], + "updatedOn": row["docs_updated_on"] or None, + "notRequiredBecause": row["docs_update_not_required"] or None, + "deadline": (_scn.docs_update_deadline(row) or date.min).isoformat() + if row["date_finished"] else None, + }, + } + return doc + + +def build_index(rows, previous_index, now, commit, out_dir): + prev = {s["scnId"]: s.get("notices", []) for s in (previous_index or {}).get("notifications", [])} + scn_dir = Path(out_dir) / "scn" + scn_dir.mkdir(parents=True, exist_ok=True) + docs = [] + for row in rows: + doc = scn_document(row, prev.get(row["scn_id"]), now.date()) + path = scn_dir / f"{row['scn_id']}.json" + _fedramp_schema.write_validated(doc, "scn", path) + docs.append(json.loads(path.read_text())) + late = [f"{d['scnId']} {n['stage']}" for d in docs for n in d["notices"] if not n["onTime"]] + index = { + "$comment": "SCN-CSO-HIS / SCN-CSO-HRM / SCN-CSO-NOM: every Significant Change Notification, each in " + "FedRAMP's SCN schema, with each notice's deadline and the date it was published. Never " + "pruned: a deploy that would drop a notification or re-date a notice fails. Built by " + "scripts/build-scn.py from docs/scn/scn-register.csv.", + "generated_at": now.isoformat(), + "commit": commit, + "certificationPackageOverviewUri": _fedramp_schema.CPO_URI, + "notification": NOTIFICATION, + "notificationMechanisms": MECHANISMS, + "access": ACCESS, + "sharing": SHARING, + "retention": "All notifications are kept, which exceeds the 12 months SCN-CSO-HIS requires.", + "evaluationLog": _scn.BASE + _scn.LOG_NAME + ".json", + "lateNotices": late, + "notifications": docs, + } + errors = _scn.retention_errors(previous_index, index) + if errors: + raise SystemExit("::error::SCN notifications are retained (SCN-CSO-HIS): " + "; ".join(errors)) + return index + + +def render_index_md(index): + out = ["# Significant Change Notifications", "", + f"_Generated {index['generated_at']} from commit `{index['commit']}`. The signed JSON beside this " + "file is the record; this is its rendering._", "", + index["notification"], "", + "## How to receive them (SCN-CSO-NOM)", ""] + out += [f"- **{m['mechanism']}:** {m['url']} — {m['description']}" for m in index["notificationMechanisms"]] + out += ["", f"**Access.** {index['access']}", "", f"**Sharing.** {index['sharing']}", "", + f"**Retention.** {index['retention']}", "", + f"**Evaluation log (SCN-CSO-MAR):** {index['evaluationLog']}", ""] + if index["lateNotices"]: + out += ["**Notices published after their deadline:** " + ", ".join(index["lateNotices"]), ""] + for d in reversed(index["notifications"]): + out += [f"## {d['scnId']} — {d['changeType']}{' (emergency)' if d['emergency'] else ''}", "", + d["changeDescription"], "", + f"- **Service Offering FedRAMP ID:** {d['serviceOfferingFedrampId'] or d['serviceOfferingFedrampIdNote']}", + f"- **Assessor:** {d.get('assessorName') or '-'}", + f"- **Related vulnerability:** {d.get('relatedVulnerability') or '-'}", + f"- **Why this type:** {d['changeTypeExplanation']}", + f"- **Reason:** {d['reason']}", + f"- **Customer impact:** {d['customerImpact']}", + f"- **Plan and timeline:** {d['planAndTimeline']['summary']}", + f"- **Impact analysis:** {d['impactAnalysis']}", + f"- **New risks:** {d['newRisksSummary'] or '-'}", + f"- **Approver:** {d['approver']['name']}, {d['approver']['title']}", + f"- **Status:** {d['additionalInformation']['status']}"] + if "transformative" in d: + t = d["transformative"] + tpr = t["thirdPartyReview"] + out += [f"- **Human validation needed:** {'yes' if tpr['humanValidationNeeded'] else 'no'}; " + f"{tpr['assessorOrReasonNotEngaged']}", + f"- **Security assessment report:** {t['securityAssessmentReport'] or '-'}", + f"- **Documentation update:** {', '.join(t['documentationUpdate']['documents']) or '-'}; " + f"updated {t['documentationUpdate']['updatedOn'] or 'not yet'}"] + out += ["", "| Notice | Rule | Deadline | Published | On time |", "|---|---|---|---|---|"] + out += [f"| {n['stage']} | {n['rule']} | {n['deadline']} | {n['publishedOn']} | " + f"{'yes' if n['onTime'] else 'no'} |" for n in d["notices"]] + out.append("") + return "\n".join(out) + + +def render_feed(index): + """Atom (RFC 4287): one entry per notice, dated when it was published.""" + entries = [] + for d in index["notifications"]: + for n in d["notices"]: + entries.append((n["publishedOn"], d, n)) + entries.sort(key=lambda e: (e[0], e[1]["scnId"], e[2]["stage"]), reverse=True) + updated = (entries[0][0] if entries else index["generated_at"][:10]) + "T00:00:00Z" + out = ['<?xml version="1.0" encoding="utf-8"?>', + '<feed xmlns="http://www.w3.org/2005/Atom">', + " <title>samaydlette.com Significant Change Notifications", + " tag:samaydlette.com,2026:significant-change-notifications", + f" {updated}", + f' ', + f' ', + " samaydlette.com"] + for on, d, n in entries: + title = f"{d['scnId']} ({d['changeType']}) {n['stage']} notice: {d['changeDescription'][:120]}" + summary = (f"{d['changeDescription']} Rule {n['rule']}; deadline {n['deadline']}; published {on}" + f"{'' if n['onTime'] else ' (after the deadline)'}. Customer impact: {d['customerImpact']}") + out += [" ", + f" tag:samaydlette.com,2026:scn/{d['scnId']}/{n['stage']}", + f" {escape(title)}", + f" {on}T00:00:00Z", + f' ', + f" {escape(summary)}", + " "] + out.append("") + return "\n".join(out) + "\n" + + +def main(argv=None): + ap = argparse.ArgumentParser(description="Build the SCN notifications and evaluation log.") + ap.add_argument("--register", default=str(_scn.REGISTER)) + ap.add_argument("--previous-index", help="the published index (absent on the first publish)") + ap.add_argument("--previous-log", help="the published evaluation log (absent on the first publish)") + ap.add_argument("--out-dir", default=".") + ap.add_argument("--repo-root", default=str(REPO)) + ap.add_argument("--commit", default=os.environ.get("GITHUB_SHA", "")) + ap.add_argument("--new-evaluations", help="JSON list of entries to append instead of reading git (tests)") + a = ap.parse_args(argv) + + now = datetime.now(timezone.utc).replace(microsecond=0) + header, rows = _scn.read_register(a.register) + errors = _scn.validate_register(header, rows, a.repo_root) + if errors: + for e in errors: + print(f"::error::scn-register: {e}", file=sys.stderr) + return 1 + out = Path(a.out_dir) + prev_index, prev_log = _load(a.previous_index), _load(a.previous_log) + + index = build_index(rows, prev_index, now, a.commit, out) + if a.new_evaluations: + new = json.loads(Path(a.new_evaluations).read_text()) + else: + new = collect((prev_log or {}).get("entries", []), root=a.repo_root) + log = build_log(prev_log, new, now, a.commit) + + outputs = { + f"{_scn.INDEX_NAME}.json": json.dumps(index, indent=2) + "\n", + f"{_scn.INDEX_NAME}.md": render_index_md(index) + "\n", + _scn.FEED_NAME: render_feed(index), + f"{_scn.LOG_NAME}.json": json.dumps(log, indent=2) + "\n", + f"{_scn.LOG_NAME}.md": render_log_md(log) + "\n", + } + errors = [e for name, text in outputs.items() for e in long_number_errors(name, text)] + if errors: + for e in errors: + print(f"::error::{e}", file=sys.stderr) + return 1 + for name, text in outputs.items(): + (out / name).write_text(text) + print(f"scn: {len(index['notifications'])} notifications ({len(index['lateNotices'])} notice(s) published " + f"late); evaluation log {len(log['entries'])} entries (+{len(new)}, {log['unrecorded']} unrecorded)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-scn-deadlines.py b/scripts/check-scn-deadlines.py new file mode 100644 index 00000000..4e5f8e5c --- /dev/null +++ b/scripts/check-scn-deadlines.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Scheduled SCN deadline check (SCN-ADP-NTF, SCN-TRF-NAF/NAV/UPD, SCN-CSO-EMG). + +Reads the published evaluation log and notification index and the register in +this checkout, and fails (exit 1) on anything late and still outstanding: + +- an adaptive, transformative or emergency change whose named record is not in + the register by its deadline (adaptive: 10 business days after merge, the + earliest it can have finished; emergency: 5; transformative: at merge); +- a notice a register record owes that is not yet published after its + deadline (a record merged but never deployed is not a notification); +- a transformative change whose service documentation is not updated within + 30 business days of finishing. + +A notice published late is not an open item: the index records it (onTime +false) for good. Exit 0 nothing overdue, 1 overdue, 2 cannot read an input. + + check-scn-deadlines.py # the published copies + check-scn-deadlines.py --index-file i.json --log-file l.json --today 2026-11-05 +""" + +import argparse +import json +import sys +import urllib.error +import urllib.request +from datetime import date, datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _scn # noqa: E402 + + +def _read(path, url): + if path: + return json.loads(Path(path).read_text()) + try: + with urllib.request.urlopen(url, timeout=30) as r: # noqa: S310 (fixed https URL) + return json.loads(r.read()) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return {} + raise + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--register", default=str(_scn.REGISTER)) + ap.add_argument("--index-file") + ap.add_argument("--log-file") + ap.add_argument("--today", help="ISO date (tests)") + a = ap.parse_args(argv) + today = date.fromisoformat(a.today) if a.today else datetime.now(timezone.utc).date() + try: + index = _read(a.index_file, _scn.BASE + _scn.INDEX_NAME + ".json") + log = _read(a.log_file, _scn.BASE + _scn.LOG_NAME + ".json") + header, rows = _scn.read_register(a.register) + except (OSError, ValueError, urllib.error.URLError) as exc: + print(f"::error::SCN deadline check cannot read its inputs: {exc}") + return 2 + errors = _scn.validate_register(header, rows) + if errors: + for e in errors: + print(f"::error::scn-register: {e}") + return 2 + late = _scn.overdue(rows, index, log.get("entries", []), today) + for item in late: + print(f"::error::SCN overdue: {item}") + if late: + return 1 + print(f"SCN deadlines: nothing overdue on {today} ({len(rows)} records, " + f"{len(log.get('entries', []))} evaluations)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scn-evaluate.py b/scripts/scn-evaluate.py new file mode 100644 index 00000000..808c5cae --- /dev/null +++ b/scripts/scn-evaluate.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""The SCN gate (SCN-CSO-EVA): every change carries an explicit evaluation. + +Run by .github/workflows/scn-tag.yml over a pull request's description (or a +direct push's commit message). Passes only when the text carries: + + SCN-Type: not-significant | routine-recurring | adaptive | transformative | emergency | class-change + SCN-Rationale: one line on why the change is that type + SCN-Record: SCN-YYYY-NNN (adaptive, transformative and emergency only) + +There is no default: a missing or placeholder evaluation fails. class-change +always fails: a change of certification class is a new certification. A +transformative change fails until its initial plan has been published for 30 +business days and its final plan for 10 (SCN-TRF-NIP/NFP), read from the +published notification index. Exit 0 pass, 1 fail. + + scn-evaluate.py --text-env PR_BODY --source "PR description" + scn-evaluate.py --text-file body.md --index-file index.json --today 2026-11-05 +""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import date +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _scn # noqa: E402 + +INDEX_URL = _scn.BASE + _scn.INDEX_NAME + ".json" + + +def fetch_index(url): + try: + with urllib.request.urlopen(url, timeout=30) as r: # noqa: S310 (fixed https URL) + return json.loads(r.read()) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return {"notifications": []} + return None + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): + return None + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--text-env", help="environment variable holding the text") + src.add_argument("--text-file", help="file holding the text") + ap.add_argument("--source", default="change description", help="what the text is, for messages") + ap.add_argument("--index-file", help="the notification index (default: fetch the published one)") + ap.add_argument("--index-url", default=INDEX_URL) + ap.add_argument("--today", help="ISO date (tests)") + a = ap.parse_args(argv) + + text = os.environ.get(a.text_env, "") if a.text_env else Path(a.text_file).read_text() + today = date.fromisoformat(a.today) if a.today else date.today() + needs_index = _scn.TYPE_LINE.search(text or "") and \ + _scn.normalize_type(_scn.TYPE_LINE.search(text).group(1)) == "transformative" + index = None + if needs_index: + index = json.loads(Path(a.index_file).read_text()) if a.index_file else fetch_index(a.index_url) + + errors, notices, ev = _scn.gate(text, index, today) + for n in notices: + print(f"::notice::{n}") + if errors: + for e in errors: + print(f"::error::SCN evaluation ({a.source}): {e}") + print("See docs/policies/cm-policy.md and docs/scn/evaluation-examples.md for the format and examples.") + return 1 + print(f"SCN-Type: {ev['type']} (from {a.source}); rationale: {ev['rationale']}" + + (f"; record {ev['record']}" if ev["record"] else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/scn/bad/adaptive-no-record.md b/tests/fixtures/scn/bad/adaptive-no-record.md new file mode 100644 index 00000000..0d71319a --- /dev/null +++ b/tests/fixtures/scn/bad/adaptive-no-record.md @@ -0,0 +1,2 @@ +SCN-Type: adaptive +SCN-Rationale: adds a new internal component (a CloudWatch alarm) with no new ingress or egress. diff --git a/tests/fixtures/scn/bad/class-change.md b/tests/fixtures/scn/bad/class-change.md new file mode 100644 index 00000000..894d2a21 --- /dev/null +++ b/tests/fixtures/scn/bad/class-change.md @@ -0,0 +1,2 @@ +SCN-Type: class-change +SCN-Rationale: the offering would start holding federal customer data, which changes its certification class. diff --git a/tests/fixtures/scn/bad/index-previous-with-dropped.json b/tests/fixtures/scn/bad/index-previous-with-dropped.json new file mode 100644 index 00000000..813ae5c2 --- /dev/null +++ b/tests/fixtures/scn/bad/index-previous-with-dropped.json @@ -0,0 +1,17 @@ +{ + "$comment": "Known-bad previous index: holds SCN-2025-001, which the current register does not, so a build over it would drop a published notification (SCN-CSO-HIS).", + "notifications": [ + { + "scnId": "SCN-2025-001", + "notices": [ + { + "stage": "after-finish", + "rule": "SCN-ADP-NTF", + "deadline": "2025-12-01", + "publishedOn": "2025-11-20", + "onTime": true + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/scn/bad/log-rewritten.json b/tests/fixtures/scn/bad/log-rewritten.json new file mode 100644 index 00000000..f6de6894 --- /dev/null +++ b/tests/fixtures/scn/bad/log-rewritten.json @@ -0,0 +1,36 @@ +{ + "genesis_commit": "1111111111111111111111111111111111111111", + "entries": [ + { + "commit": "1111111111111111111111111111111111111111", + "merged_on": "2026-11-06", + "pull_request": 501, + "evaluator": "sam-aydlette", + "scn_type": "not-significant", + "rationale": "compliance-process change; no component, boundary or access change.", + "scn_record": null, + "problem": null + }, + { + "commit": "2222222222222222222222222222222222222222", + "merged_on": "2026-11-09", + "pull_request": 502, + "evaluator": "sam-aydlette", + "scn_type": "routine-recurring", + "rationale": "adds a new internal alarm with no new ingress or egress.", + "scn_record": null, + "problem": null + }, + { + "commit": "3333333333333333333333333333333333333333", + "merged_on": "2026-11-10", + "pull_request": 503, + "evaluator": "sam-aydlette", + "scn_type": "not-significant", + "rationale": "content-only change to a public page.", + "scn_record": null, + "problem": null + } + ], + "$comment": "Known-bad: rewrites entry 1 of log-previous.json (adaptive -> routine-recurring). The append-only check must fail." +} \ No newline at end of file diff --git a/tests/fixtures/scn/bad/log-truncated.json b/tests/fixtures/scn/bad/log-truncated.json new file mode 100644 index 00000000..2e053028 --- /dev/null +++ b/tests/fixtures/scn/bad/log-truncated.json @@ -0,0 +1,16 @@ +{ + "genesis_commit": "1111111111111111111111111111111111111111", + "entries": [ + { + "commit": "1111111111111111111111111111111111111111", + "merged_on": "2026-11-06", + "pull_request": 501, + "evaluator": "sam-aydlette", + "scn_type": "not-significant", + "rationale": "compliance-process change; no component, boundary or access change.", + "scn_record": null, + "problem": null + } + ], + "$comment": "Known-bad: drops entry 1 of log-previous.json. The append-only check must fail." +} \ No newline at end of file diff --git a/tests/fixtures/scn/bad/missing-rationale.md b/tests/fixtures/scn/bad/missing-rationale.md new file mode 100644 index 00000000..4ebdb1fa --- /dev/null +++ b/tests/fixtures/scn/bad/missing-rationale.md @@ -0,0 +1,4 @@ +## Changed +Bumps a dependency. + +SCN-Type: routine-recurring diff --git a/tests/fixtures/scn/bad/missing-type.md b/tests/fixtures/scn/bad/missing-type.md new file mode 100644 index 00000000..c996bb61 --- /dev/null +++ b/tests/fixtures/scn/bad/missing-type.md @@ -0,0 +1,4 @@ +## Changed +Bumps a dependency. + +SCN-Rationale: dependency version bump with no behaviour change. diff --git a/tests/fixtures/scn/bad/placeholder-rationale.md b/tests/fixtures/scn/bad/placeholder-rationale.md new file mode 100644 index 00000000..9fa83cc9 --- /dev/null +++ b/tests/fixtures/scn/bad/placeholder-rationale.md @@ -0,0 +1,2 @@ +SCN-Type: routine-recurring +SCN-Rationale: diff --git a/tests/fixtures/scn/bad/register-adaptive-no-risks.csv b/tests/fixtures/scn/bad/register-adaptive-no-risks.csv new file mode 100644 index 00000000..c7150d7d --- /dev/null +++ b/tests/fixtures/scn/bad/register-adaptive-no-risks.csv @@ -0,0 +1,2 @@ +scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_started,date_finished,verified_on,emergency,evaluation_prs,plan_final,tpr_human_validation,tpr_assessor_or_reason,sar_reference,docs_to_update,docs_updated_on,docs_update_not_required +SCN-2026-103,,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),adaptive,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,,Sam Aydlette,System Owner / Authorizing Operator,Implemented — live in production (first prod deploy 2026-06-03 via PR #82); post-implementation verification COMPLETE 2026-06-08 at prod commit 354da59 (pass: KEV 0 / blocking 0; SSP data-flow + interconnection rendered; deployed Python/JS deps CVE-free; CM-8 inventory complete),2026-06-03,,2026-06-06,2026-06-08,no,#82,,,,,,, diff --git a/tests/fixtures/scn/bad/register-dates-out-of-order.csv b/tests/fixtures/scn/bad/register-dates-out-of-order.csv new file mode 100644 index 00000000..07948b55 --- /dev/null +++ b/tests/fixtures/scn/bad/register-dates-out-of-order.csv @@ -0,0 +1,2 @@ +scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_started,date_finished,verified_on,emergency,evaluation_prs,plan_final,tpr_human_validation,tpr_assessor_or_reason,sar_reference,docs_to_update,docs_updated_on,docs_update_not_required +SCN-2026-104,,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),adaptive,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,"POAM-019..024 (all risk-accepted, Low residual) plus the new-ecosystem SCA blind-spot, remediated by adding Syft/Grype SCA of the built Lambda+SPA artifact to CI feeding the VDR aggregator (RA-5/VDR-CSO-DET).",Sam Aydlette,System Owner / Authorizing Operator,Implemented — live in production (first prod deploy 2026-06-03 via PR #82); post-implementation verification COMPLETE 2026-06-08 at prod commit 354da59 (pass: KEV 0 / blocking 0; SSP data-flow + interconnection rendered; deployed Python/JS deps CVE-free; CM-8 inventory complete),2026-06-03,,2026-06-01,2026-06-08,no,#82,,,,,,, diff --git a/tests/fixtures/scn/bad/register-routine.csv b/tests/fixtures/scn/bad/register-routine.csv new file mode 100644 index 00000000..ec1175ff --- /dev/null +++ b/tests/fixtures/scn/bad/register-routine.csv @@ -0,0 +1,2 @@ +scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_started,date_finished,verified_on,emergency,evaluation_prs,plan_final,tpr_human_validation,tpr_assessor_or_reason,sar_reference,docs_to_update,docs_updated_on,docs_update_not_required +SCN-2026-101,,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),routine-recurring,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,"POAM-019..024 (all risk-accepted, Low residual) plus the new-ecosystem SCA blind-spot, remediated by adding Syft/Grype SCA of the built Lambda+SPA artifact to CI feeding the VDR aggregator (RA-5/VDR-CSO-DET).",Sam Aydlette,System Owner / Authorizing Operator,Implemented — live in production (first prod deploy 2026-06-03 via PR #82); post-implementation verification COMPLETE 2026-06-08 at prod commit 354da59 (pass: KEV 0 / blocking 0; SSP data-flow + interconnection rendered; deployed Python/JS deps CVE-free; CM-8 inventory complete),2026-06-03,,2026-06-06,2026-06-08,no,#82,,,,,,, diff --git a/tests/fixtures/scn/bad/register-transformative-incomplete.csv b/tests/fixtures/scn/bad/register-transformative-incomplete.csv new file mode 100644 index 00000000..41a48d43 --- /dev/null +++ b/tests/fixtures/scn/bad/register-transformative-incomplete.csv @@ -0,0 +1,2 @@ +scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_started,date_finished,verified_on,emergency,evaluation_prs,plan_final,tpr_human_validation,tpr_assessor_or_reason,sar_reference,docs_to_update,docs_updated_on,docs_update_not_required +SCN-2026-102,,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),transformative,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,"POAM-019..024 (all risk-accepted, Low residual) plus the new-ecosystem SCA blind-spot, remediated by adding Syft/Grype SCA of the built Lambda+SPA artifact to CI feeding the VDR aggregator (RA-5/VDR-CSO-DET).",Sam Aydlette,System Owner / Authorizing Operator,Implemented — live in production (first prod deploy 2026-06-03 via PR #82); post-implementation verification COMPLETE 2026-06-08 at prod commit 354da59 (pass: KEV 0 / blocking 0; SSP data-flow + interconnection rendered; deployed Python/JS deps CVE-free; CM-8 inventory complete),2026-06-03,,,,no,#82,,,,,,, diff --git a/tests/fixtures/scn/bad/routine-with-record.md b/tests/fixtures/scn/bad/routine-with-record.md new file mode 100644 index 00000000..273ae797 --- /dev/null +++ b/tests/fixtures/scn/bad/routine-with-record.md @@ -0,0 +1,3 @@ +SCN-Type: routine-recurring +SCN-Rationale: weekly dependency bump with no breaking change. +SCN-Record: SCN-2026-007 diff --git a/tests/fixtures/scn/bad/transformative-too-soon.md b/tests/fixtures/scn/bad/transformative-too-soon.md new file mode 100644 index 00000000..5d436194 --- /dev/null +++ b/tests/fixtures/scn/bad/transformative-too-soon.md @@ -0,0 +1,3 @@ +SCN-Type: transformative +SCN-Rationale: the offering starts accepting agency user accounts, a new service used differently by customers. +SCN-Record: SCN-2099-001 diff --git a/tests/fixtures/scn/bad/two-types.md b/tests/fixtures/scn/bad/two-types.md new file mode 100644 index 00000000..9f92a1ed --- /dev/null +++ b/tests/fixtures/scn/bad/two-types.md @@ -0,0 +1,3 @@ +SCN-Type: routine-recurring +SCN-Type: adaptive +SCN-Rationale: weekly dependency bump with no breaking change. diff --git a/tests/fixtures/scn/bad/unknown-type.md b/tests/fixtures/scn/bad/unknown-type.md new file mode 100644 index 00000000..474927c4 --- /dev/null +++ b/tests/fixtures/scn/bad/unknown-type.md @@ -0,0 +1,2 @@ +SCN-Type: minor +SCN-Rationale: a small change to the website content only. diff --git a/tests/fixtures/scn/good/adaptive.md b/tests/fixtures/scn/good/adaptive.md new file mode 100644 index 00000000..fb9b18de --- /dev/null +++ b/tests/fixtures/scn/good/adaptive.md @@ -0,0 +1,3 @@ +SCN-Type: adaptive +SCN-Rationale: adds a new internal component (a CloudWatch alarm) with no new ingress or egress. +SCN-Record: SCN-2026-007 diff --git a/tests/fixtures/scn/good/emergency.md b/tests/fixtures/scn/good/emergency.md new file mode 100644 index 00000000..8485df5e --- /dev/null +++ b/tests/fixtures/scn/good/emergency.md @@ -0,0 +1,3 @@ +SCN-Type: emergency +SCN-Rationale: rotates a leaked credential immediately; the retroactive record follows within 5 business days. +SCN-Record: SCN-2026-008 diff --git a/tests/fixtures/scn/good/not-significant.md b/tests/fixtures/scn/good/not-significant.md new file mode 100644 index 00000000..656c1068 --- /dev/null +++ b/tests/fixtures/scn/good/not-significant.md @@ -0,0 +1,5 @@ +## Changed +Fixes a typo on the about page. + +**SCN-Type:** not-significant +**SCN-Rationale:** content-only edit to a public page; no component, boundary, data flow or access change. diff --git a/tests/fixtures/scn/good/register-transformative.csv b/tests/fixtures/scn/good/register-transformative.csv new file mode 100644 index 00000000..51fb364a --- /dev/null +++ b/tests/fixtures/scn/good/register-transformative.csv @@ -0,0 +1,2 @@ +scn_id,fedramp_id,assessor_name,related_vulnerability,scn_type,categorization_rationale,short_description,reason_for_change,customer_impact,plan_and_timeline,sia_reference,new_risks_summary,approver_name,approver_title,status,date_initiated,date_started,date_finished,verified_on,emergency,evaluation_prs,plan_final,tpr_human_validation,tpr_assessor_or_reason,sar_reference,docs_to_update,docs_updated_on,docs_update_not_required +SCN-2099-003,,N/A (sole-operator self-attestation; no 3PAO engaged),None (proactive feature addition),transformative,"Net-new service component with new public ingress and new external egress; no FedRAMP Certification class change and no federal customer data introduced, so not transformative and not a class change; not regular/routine maintenance, so not routine-recurring. Resolves to Adaptive per SCN-CSO-EVA. Sits at the upper edge of Adaptive due to the new non-FedRAMP-authorized external interconnection (Anthropic); the absence of federal customer data is what keeps it out of Transformative.","Adds the gated ""Silk Reeling Mirror"" app: a public, HTTP-Basic-Auth-gated API Gateway HTTP API -> Python 3.13 ZIP Lambda serving a browser SPA; the Lambda calls the Anthropic API to generate movement-feedback text.",New self-service capability addition to the site.,New customer/operator-responsibility control: operator-set HTTP Basic Auth credential (POAM-021). SAML/IdP federation is the recommended upgrade. No change to existing customers' configuration responsibilities.,"Gated deploy (create_silk_reeling=true) -> OPA + Checkov + tfsec gate -> Syft/Grype SCA over the built artifact -> VDR report regenerated -> post-implementation secure-configuration verification (Adaptive requirement) -> KSI signal + OSCAL SSP re-emitted. Controls/KSIs to re-verify: RA-5/VDR-*, SA-9/CA-3, SC-7, AC-3/IA-2, AU-2/AU-3, SC-12/SC-28.",docs/scn/SCN-2026-001-silk-reeling.md,Fictitious example.,Sam Aydlette,System Owner / Authorizing Operator,Fictitious example,2026-09-01,2026-11-02,2026-11-13,2026-11-20,no,#9999,yes,yes,No third-party assessor engaged: none is under contract; the operator validated the scope.,N/A: no assessor engaged; provider verification in docs/scn/SCN-2099-003.md,docs/secure-configuration-guide.md,, diff --git a/tests/fixtures/scn/good/routine-recurring-legacy-spelling.md b/tests/fixtures/scn/good/routine-recurring-legacy-spelling.md new file mode 100644 index 00000000..c1992bd0 --- /dev/null +++ b/tests/fixtures/scn/good/routine-recurring-legacy-spelling.md @@ -0,0 +1,2 @@ +SCN-Type: Routine Recurring +SCN-Rationale: weekly Dependabot patch bump of a library already in the inventory, no breaking change. diff --git a/tests/fixtures/scn/good/transformative-ready.md b/tests/fixtures/scn/good/transformative-ready.md new file mode 100644 index 00000000..ec1ce069 --- /dev/null +++ b/tests/fixtures/scn/good/transformative-ready.md @@ -0,0 +1,3 @@ +SCN-Type: transformative +SCN-Rationale: the offering starts accepting agency user accounts, a new service used differently by customers. +SCN-Record: SCN-2099-002 diff --git a/tests/fixtures/scn/index.json b/tests/fixtures/scn/index.json new file mode 100644 index 00000000..b286cf31 --- /dev/null +++ b/tests/fixtures/scn/index.json @@ -0,0 +1,10 @@ +{ + "notifications": [ + {"scnId": "SCN-2099-001", "notices": [ + {"stage": "initial-plan", "rule": "SCN-TRF-NIP", "deadline": "2099-01-01", "publishedOn": "2026-10-20", "onTime": true}, + {"stage": "final-plan", "rule": "SCN-TRF-NFP", "deadline": "2099-01-01", "publishedOn": "2026-11-02", "onTime": true}]}, + {"scnId": "SCN-2099-002", "notices": [ + {"stage": "initial-plan", "rule": "SCN-TRF-NIP", "deadline": "2099-01-01", "publishedOn": "2026-09-01", "onTime": true}, + {"stage": "final-plan", "rule": "SCN-TRF-NFP", "deadline": "2099-01-01", "publishedOn": "2026-10-15", "onTime": true}]} + ] +} diff --git a/tests/fixtures/scn/log-previous.json b/tests/fixtures/scn/log-previous.json new file mode 100644 index 00000000..63ea19de --- /dev/null +++ b/tests/fixtures/scn/log-previous.json @@ -0,0 +1,11 @@ +{ + "genesis_commit": "1111111111111111111111111111111111111111", + "entries": [ + {"commit": "1111111111111111111111111111111111111111", "merged_on": "2026-11-06", "pull_request": 501, + "evaluator": "sam-aydlette", "scn_type": "not-significant", + "rationale": "compliance-process change; no component, boundary or access change.", "scn_record": null, "problem": null}, + {"commit": "2222222222222222222222222222222222222222", "merged_on": "2026-11-09", "pull_request": 502, + "evaluator": "sam-aydlette", "scn_type": "adaptive", + "rationale": "adds a new internal alarm with no new ingress or egress.", "scn_record": "SCN-2026-007", "problem": null} + ] +} diff --git a/tests/test_scn.py b/tests/test_scn.py new file mode 100644 index 00000000..de58c2de --- /dev/null +++ b/tests/test_scn.py @@ -0,0 +1,446 @@ +"""Significant Change Notifications (SCN-*): the evaluation gate, the register, +the notice deadlines in business days, the append-only evaluation log, the +never-pruned notification index and feed, and the scheduled deadline check. + +Every gate invariant has a committed known-bad fixture under +tests/fixtures/scn/bad/ that fails for its own reason.""" + +import importlib.util +import json +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import date +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "scn" +sys.path.insert(0, str(REPO / "scripts")) +import _bizdays # noqa: E402 +import _fedramp_schema # noqa: E402 +import _scn # noqa: E402 + +TODAY = date(2026, 11, 5) + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +build = _load("build_scn", REPO / "scripts" / "build-scn.py") +evaluate = _load("scn_evaluate", REPO / "scripts" / "scn-evaluate.py") +deadlines = _load("scn_deadlines", REPO / "scripts" / "check-scn-deadlines.py") + + +# --- business days ------------------------------------------------------------------ + +def test_federal_holidays_2026_are_the_observed_eleven(): + assert sorted(_bizdays.federal_holidays(2026)) == [date(2026, m, d) for m, d in ( + (1, 1), (1, 19), (2, 16), (5, 25), (6, 19), (7, 3), (9, 7), (10, 12), (11, 11), (11, 26), (12, 25))] + + +@pytest.mark.parametrize("day,observed", [ + (date(2026, 7, 4), date(2026, 7, 3)), # Saturday -> Friday + (date(2027, 6, 19), date(2027, 6, 18)), # Saturday -> Friday + (date(2027, 7, 4), date(2027, 7, 5)), # Sunday -> Monday + (date(2027, 12, 25), date(2027, 12, 24)), + (date(2028, 1, 1), date(2027, 12, 31)), # New Year's on a Saturday: observed the year before +]) +def test_weekend_holidays_move_to_the_observed_day(day, observed): + assert observed in _bizdays.federal_holidays(observed.year) + assert day not in _bizdays.federal_holidays(day.year) + + +def test_business_day_arithmetic_skips_weekends_and_holidays(): + assert _bizdays.add_business_days(date(2026, 11, 25), 1) == date(2026, 11, 27) # Thanksgiving + assert _bizdays.add_business_days(date(2026, 11, 6), 1) == date(2026, 11, 9) # weekend + assert _bizdays.add_business_days(date(2026, 11, 10), 1) == date(2026, 11, 12) # Veterans Day + assert _bizdays.add_business_days(date(2027, 1, 4), -1) == date(2026, 12, 31) # New Year's Day + assert _bizdays.add_business_days(date(2026, 6, 6), 10) == date(2026, 6, 22) # Juneteenth in the span + assert _bizdays.add_business_days(date(2026, 11, 5), 0) == date(2026, 11, 5) + + +def test_timeframes_come_from_the_vendored_rules(): + assert {r: _scn.timeframe(r) for r in ("SCN-ADP-NTF", "SCN-TRF-NIP", "SCN-TRF-NFP", "SCN-TRF-NAF", + "SCN-TRF-NAV", "SCN-TRF-UPD")} == { + "SCN-ADP-NTF": 10, "SCN-TRF-NIP": 30, "SCN-TRF-NFP": 10, "SCN-TRF-NAF": 5, "SCN-TRF-NAV": 5, + "SCN-TRF-UPD": 30} + + +# --- the evaluation gate (SCN-CSO-EVA) ---------------------------------------------------- + +BAD = { + "missing-type.md": "no 'SCN-Type:' line", + "missing-rationale.md": "no 'SCN-Rationale:' line", + "placeholder-rationale.md": "empty or a placeholder", + "unknown-type.md": "is not one of", + "class-change.md": "class-change blocks the merge", + "adaptive-no-record.md": "needs an 'SCN-Record: SCN-YYYY-NNN' line", + "routine-with-record.md": "names no SCN-Record", + "two-types.md": "2 'SCN-Type:' lines", + "transformative-too-soon.md": "may start on or after", +} + + +@pytest.mark.parametrize("name,why", sorted(BAD.items())) +def test_each_known_bad_evaluation_fails_for_its_reason(name, why): + index = json.loads((FIX / "index.json").read_text()) + errors, _, _ = _scn.gate((FIX / "bad" / name).read_text(), index, TODAY) + assert errors and any(why in e for e in errors), errors + + +@pytest.mark.parametrize("name,kind", [ + ("not-significant.md", "not-significant"), ("routine-recurring-legacy-spelling.md", "routine-recurring"), + ("adaptive.md", "adaptive"), ("emergency.md", "emergency"), ("transformative-ready.md", "transformative")]) +def test_each_good_evaluation_passes(name, kind): + index = json.loads((FIX / "index.json").read_text()) + errors, _, ev = _scn.gate((FIX / "good" / name).read_text(), index, TODAY) + assert errors == [] and ev["type"] == kind + + +def test_an_empty_description_fails_there_is_no_default(): + errors, _, ev = _scn.gate("", None, TODAY) + assert ev is None and "no 'SCN-Type:' line" in errors[0] + + +def test_transformative_fails_closed_without_the_published_index(): + errors, _, _ = _scn.gate((FIX / "good" / "transformative-ready.md").read_text(), None, TODAY) + assert "fail closed" in errors[0] + + +def test_the_cli_exits_nonzero_on_every_known_bad_fixture(capsys): + for name in BAD: + rc = evaluate.main(["--text-file", str(FIX / "bad" / name), "--index-file", str(FIX / "index.json"), + "--today", TODAY.isoformat()]) + assert rc == 1, name + assert evaluate.main(["--text-file", str(FIX / "good" / "adaptive.md"), "--today", TODAY.isoformat()]) == 0 + assert "::error::" in capsys.readouterr().out + + +def test_the_workflow_runs_the_tested_gate_with_no_default(): + text = (REPO / ".github" / "workflows" / "scn-tag.yml").read_text() + assert "python3 scripts/scn-evaluate.py --text-env SCN_TEXT" in text + assert "defaulting" not in text and "routine-recurring)" not in text + # The job keeps the name branch protection requires. + assert re.search(r"^ scn-tag:$", text, re.M) + # Untrusted text only through the environment. + assert "${{ github.event.pull_request.body }}" in text.split("env:", 1)[1].split("run:", 1)[0] + + +def test_every_type_has_a_documented_example(): + doc = (REPO / "docs" / "scn" / "evaluation-examples.md").read_text() + for t in _scn.TYPES: + block = doc.split(f"## {t}\n", 1) + assert len(block) == 2, t + example = block[1].split("## ", 1)[0] + errors, _, ev = _scn.gate(example.split("```")[1], json.loads((FIX / "index.json").read_text()), TODAY) + assert ev and ev["type"] == t, (t, errors) + + +# --- the register (SCN-CSO-INF, SCN-RTR-NNR, transformative template) ----------------------- + +def test_the_committed_register_is_valid(): + header, rows = _scn.read_register() + assert _scn.validate_register(header, rows) == [] + assert len(rows) >= 6 + + +@pytest.mark.parametrize("name,why", [ + ("register-routine.csv", "only adaptive and transformative changes get a notification record (SCN-RTR-NNR)"), + ("register-transformative-incomplete.csv", "tpr_human_validation must be yes or no"), + ("register-adaptive-no-risks.csv", "new_risks_summary is empty (SCN-ADP-NTF)"), + ("register-dates-out-of-order.csv", "is before date_initiated"), +]) +def test_each_known_bad_register_fails_for_its_reason(name, why): + header, rows = _scn.read_register(FIX / "bad" / name) + errors = _scn.validate_register(header, rows) + assert any(why in e for e in errors), errors + + +def test_an_incomplete_transformative_row_names_every_template_field(): + header, rows = _scn.read_register(FIX / "bad" / "register-transformative-incomplete.csv") + text = " ".join(_scn.validate_register(header, rows)) + for field in ("date_started", "plan_final", "tpr_human_validation", "tpr_assessor_or_reason"): + assert field in text + + +def test_transformative_notices_and_deadlines(): + header, rows = _scn.read_register(FIX / "good" / "register-transformative.csv") + assert _scn.validate_register(header, rows) == [] + due = {n["stage"]: n["deadline"] for n in _scn.notices_due(rows[0])} + assert due == { + "initial-plan": date(2026, 9, 18), # 30 business days before 2026-11-02 (Labor Day, Columbus Day) + "final-plan": date(2026, 10, 19), # 10 before + "after-finish": date(2026, 11, 20), # 5 after 2026-11-13 + "after-verification": date(2026, 11, 30), # 5 after 2026-11-20, over Thanksgiving + } + assert _scn.docs_update_deadline(rows[0]) == _bizdays.add_business_days(date(2026, 11, 13), 30) + + +def test_emergency_notices_are_retroactive_within_five_business_days(): + _, rows = _scn.read_register(FIX / "good" / "register-transformative.csv") + row = dict(rows[0], emergency="yes") + for n in _scn.notices_due(row): + assert n["deadline"] == date(2026, 11, 20) and n["kind"] == "retroactive" + assert "SCN-CSO-EMG" in n["rule"] + + +def test_adaptive_notice_is_due_ten_business_days_after_finishing(): + _, rows = _scn.read_register() + by_id = {r["scn_id"]: r for r in rows} + assert _scn.notices_due(by_id["SCN-2026-001"]) == [ + {"stage": "after-finish", "rule": "SCN-ADP-NTF", "deadline": date(2026, 6, 22), "kind": "by"}] + + +# --- publication: index, feed, retention (SCN-CSO-HIS/HRM/NOM/INF) -------------------------- + +def _build(tmp_path, *extra, new_entries=None): + (tmp_path / "new.json").write_text(json.dumps(new_entries or [])) + return build.main(["--out-dir", str(tmp_path), "--commit", "abc123", "--new-evaluations", + str(tmp_path / "new.json"), *extra]) + + +def test_build_publishes_every_register_row_in_the_fedramp_schema(tmp_path): + assert _build(tmp_path) == 0 + _, rows = _scn.read_register() + index = json.loads((tmp_path / "significant-change-notifications.json").read_text()) + assert [d["scnId"] for d in index["notifications"]] == [r["scn_id"] for r in rows] + for d in index["notifications"]: + assert _fedramp_schema.validate(d, "scn") == [] + assert (tmp_path / "scn" / f"{d['scnId']}.json").is_file() + assert d["certificationPackageOverviewUri"] == _fedramp_schema.CPO_URI + # SCN-CSO-ARI: more than the required fields. + assert d["additionalInformation"]["record"].startswith(_scn.SOURCE_URL) + # Human-readable view (SCN-CSO-HRM) carries each one. + md = (tmp_path / "significant-change-notifications.md").read_text() + assert all(d["scnId"] in md for d in index["notifications"]) + assert {m["mechanism"] for m in index["notificationMechanisms"]} == { + "Atom feed", "Signed JSON index", "Human-readable view"} + + +def test_a_new_notice_is_dated_today_and_late_ones_say_so(tmp_path): + assert _build(tmp_path) == 0 + index = json.loads((tmp_path / "significant-change-notifications.json").read_text()) + n = index["notifications"][0]["notices"][0] + assert n["publishedOn"] == build._today().isoformat() + assert n["onTime"] is (date.fromisoformat(n["publishedOn"]) <= date.fromisoformat(n["deadline"])) + assert "SCN-2026-001 after-finish" in index["lateNotices"] + + +def test_a_published_notice_keeps_its_first_publication_date(tmp_path): + prev = {"notifications": [{"scnId": "SCN-2026-001", "notices": [ + {"stage": "after-finish", "rule": "SCN-ADP-NTF", "deadline": "2026-06-22", "publishedOn": "2026-06-15", + "onTime": True}]}]} + (tmp_path / "prev.json").write_text(json.dumps(prev)) + assert _build(tmp_path, "--previous-index", str(tmp_path / "prev.json")) == 0 + index = json.loads((tmp_path / "significant-change-notifications.json").read_text()) + n = index["notifications"][0]["notices"][0] + assert n["publishedOn"] == "2026-06-15" and n["onTime"] is True + + +def test_a_build_that_would_drop_a_published_notification_fails(tmp_path): + with pytest.raises(SystemExit, match="SCN-2025-001 was published and would be dropped"): + _build(tmp_path, "--previous-index", str(FIX / "bad" / "index-previous-with-dropped.json")) + + +def test_the_feed_is_atom_with_one_entry_per_notice(tmp_path): + assert _build(tmp_path) == 0 + index = json.loads((tmp_path / "significant-change-notifications.json").read_text()) + root = ET.parse(tmp_path / "significant-change-notifications.atom").getroot() + ns = {"a": "http://www.w3.org/2005/Atom"} + assert root.tag == "{http://www.w3.org/2005/Atom}feed" + ids = [e.findtext("a:id", namespaces=ns) for e in root.findall("a:entry", ns)] + assert len(ids) == sum(len(d["notices"]) for d in index["notifications"]) == len(set(ids)) + + +def test_a_run_id_in_published_text_fails_the_build(tmp_path): + reg = tmp_path / "reg.csv" + reg.write_text((REPO / "docs" / "scn" / "scn-register.csv").read_text().replace( + "Implemented and verified 2026-10-27", "Implemented and verified 2026-10-27 (run 36434974425)")) + assert _build(tmp_path, "--register", str(reg)) == 1 + + +def test_the_published_outputs_are_listed_signed_and_governed(): + deploy = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() + step = deploy[deploy.index("- name: Build Significant Change Notifications"):] + assert "build-scn.py --previous-index previous-scn-index.json" in step + assert deploy.index("- name: Build Significant Change Notifications") < deploy.index( + "- name: Sign CDS certification-data artifacts") + sign = deploy[deploy.index("- name: Sign CDS certification-data artifacts"):] + sign = sign[:sign.index("\n - name:")] + assert "for a in significant-change-notifications scn-evaluations" in sign + assert "significant-change-notifications.atom.bundle significant-change-notifications.atom" in sign + cfg = {a["path"] for a in json.loads((REPO / "infrastructure" / "schemas" / "freshness-config.json") + .read_text())["artifacts"]} + assert {".well-known/significant-change-notifications.json", ".well-known/significant-change-notifications.atom", + ".well-known/scn-evaluations.json"} <= cfg + + +def test_the_cpo_lists_the_notification_mechanism(tmp_path): + cpo = _load("build_cpo_scn", REPO / "scripts" / "build-cpo.py") + out, md = tmp_path / "cpo.json", tmp_path / "cpo.md" + sys_argv = sys.argv + sys.argv = ["build-cpo.py", "--offering", str(REPO / "data" / "cds" / "offering.json"), + "--profile", str(REPO / "data" / "system-profile.json"), "--overview", str(tmp_path / "x.json"), + "--ksi-signal", str(tmp_path / "y.json"), "--output", str(out), "--md-output", str(md)] + try: + assert cpo.main() == 0 + finally: + sys.argv = sys_argv + doc = json.loads(out.read_text()) + urls = {m["url"] for m in doc["significant_change_notifications"]["mechanisms"]} + assert _scn.BASE + _scn.FEED_NAME in urls + assert {"significant-change-notifications.json", "significant-change-notifications.atom", + "scn-evaluations.json"} <= {p["artifact"] for p in doc["package_contents"]} + assert "Significant Change Notifications (SCN-CSO-NOM)" in md.read_text() + + +# --- the evaluation log (SCN-CSO-MAR) ------------------------------------------------------- + +def test_the_log_appends_to_the_previous_published_version(tmp_path): + prev = json.loads((FIX / "log-previous.json").read_text()) + new = [dict(prev["entries"][0], commit="3" * 40, merged_on="2026-11-10", pull_request=503)] + (tmp_path / "prev.json").write_text(json.dumps(prev)) + assert _build(tmp_path, "--previous-log", str(tmp_path / "prev.json"), new_entries=new) == 0 + log = json.loads((tmp_path / "scn-evaluations.json").read_text()) + assert log["entries"] == prev["entries"] + new + assert log["genesis_commit"] == prev["genesis_commit"] + assert "#503" in (tmp_path / "scn-evaluations.md").read_text() + + +@pytest.mark.parametrize("name,why", [("log-rewritten.json", "entry 1 (2222"), ("log-truncated.json", "dropped")]) +def test_the_append_only_check_fails_each_known_bad_log(name, why): + prev = json.loads((FIX / "log-previous.json").read_text())["entries"] + cur = json.loads((FIX / "bad" / name).read_text())["entries"] + errors = _scn.append_only_errors(prev, cur, "commit") + assert any(why in e for e in errors), errors + + +def _git(repo, *args): + return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip() + + +@pytest.fixture +def history(tmp_path): + """main: an old commit, then a merge of PR #7 that adds the builder, then a + direct push, then a merge of PR #8 with no evaluation.""" + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "t@example.invalid") + _git(repo, "config", "user.name", "tester") + (repo / "a").write_text("a") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "before the log") + for n, path in ((7, build.GENESIS_PATH), (8, "c")): + _git(repo, "checkout", "-qb", f"b{n}") + (repo / path).parent.mkdir(parents=True, exist_ok=True) + (repo / path).write_text(str(n)) + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", f"work {n}") + _git(repo, "checkout", "-q", "main") + _git(repo, "merge", "-q", "--no-ff", f"b{n}", "-m", f"Merge pull request #{n} from x/b{n}") + if n == 7: + (repo / "d").write_text("d") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "direct push\n\nSCN-Type: not-significant\n" + "SCN-Rationale: content-only change to a public page here.") + return repo + + +def _lookup(n): + bodies = {7: "SCN-Type: adaptive\nSCN-Rationale: adds a new internal alarm, no ingress.\nSCN-Record: SCN-2026-007", + 8: "Forgot the evaluation."} + return {"body": bodies[n], "merged_by": "sam-aydlette", "author": "someone"} + + +def test_collect_logs_every_first_parent_commit_from_the_builder_onward(history): + entries = build.collect([], root=history, lookup=_lookup) + assert [(e["pull_request"], e["scn_type"]) for e in entries] == [ + (7, "adaptive"), (None, "not-significant"), (8, None)] + assert entries[0]["scn_record"] == "SCN-2026-007" and entries[0]["evaluator"] == "sam-aydlette" + assert entries[1]["evaluator"] == "tester" + assert entries[2]["problem"].startswith("no valid evaluation") + # Next deploy: only what came after the last entry. + assert build.collect(entries, root=history, lookup=_lookup) == [] + assert build.collect(entries[:1], root=history, lookup=_lookup) == entries[1:] + + +def test_collect_fails_when_the_last_logged_commit_left_history(history): + with pytest.raises(SystemExit, match="not an ancestor"): + build.collect([{"commit": "f" * 40}], root=history, lookup=_lookup) + + +def test_free_text_is_scrubbed_before_it_is_published(): + e = build.evaluation_entry("a" * 40, "direct", "SCN-Type: not-significant\nSCN-Rationale: fixes run 36434974425 " + "for account 123456789012 docs", "2026-11-05", "x", _lookup) + assert "36434974425" not in e["rationale"] and "123456789012" not in e["rationale"] + + +# --- the scheduled deadline check ----------------------------------------------------------- + +def _entry(kind, record, merged): + return {"commit": "9" * 40, "merged_on": merged, "scn_type": kind, "scn_record": record} + + +def test_an_adaptive_change_with_no_record_is_overdue_after_ten_business_days(): + _, rows = _scn.read_register() + index = {"notifications": [{"scnId": r["scn_id"], "notices": [{"stage": "after-finish"}]} for r in rows]} + log = [_entry("adaptive", "SCN-2026-007", "2026-11-02")] + assert _scn.overdue(rows, index, log, date(2026, 11, 17)) == [] # due 2026-11-17 (Veterans Day) + late = _scn.overdue(rows, index, log, date(2026, 11, 18)) + assert len(late) == 1 and "SCN-2026-007" in late[0] and "not in the register" in late[0] + + +def test_an_emergency_change_owes_its_record_within_five_business_days(): + _, rows = _scn.read_register() + index = {"notifications": [{"scnId": r["scn_id"], "notices": [{"stage": "after-finish"}]} for r in rows]} + log = [_entry("emergency", "SCN-2026-099", "2026-11-20")] + assert _scn.overdue(rows, index, log, date(2026, 11, 30)) == [] # Thanksgiving: due 11-30 + assert _scn.overdue(rows, index, log, date(2026, 12, 1)) + + +def test_an_unpublished_notice_is_overdue_until_a_deploy_publishes_it(): + _, rows = _scn.read_register() + late = _scn.overdue(rows, {}, [], TODAY) + assert any("SCN-2026-001: after-finish notice (SCN-ADP-NTF) not published" in x for x in late) + published = {"notifications": [{"scnId": r["scn_id"], "notices": [{"stage": "after-finish"}]} for r in rows]} + assert _scn.overdue(rows, published, [], TODAY) == [] + + +def test_transformative_notices_after_finish_and_verification_are_overdue_unpublished(): + _, rows = _scn.read_register(FIX / "good" / "register-transformative.csv") + index = {"notifications": [{"scnId": "SCN-2099-003", "notices": [ + {"stage": "initial-plan"}, {"stage": "final-plan"}]}]} + assert _scn.overdue(rows, index, [], date(2026, 11, 20)) == [] + late = _scn.overdue(rows, index, [], date(2026, 11, 23)) + assert len(late) == 1 and "after-finish notice (SCN-TRF-NAF)" in late[0] + late = _scn.overdue(rows, index, [], date(2026, 12, 1)) + assert any("after-verification notice (SCN-TRF-NAV)" in x for x in late) + + +def test_transformative_documentation_update_is_overdue_after_thirty_business_days(): + _, rows = _scn.read_register(FIX / "good" / "register-transformative.csv") + stages = ["initial-plan", "final-plan", "after-finish", "after-verification"] + index = {"notifications": [{"scnId": "SCN-2099-003", "notices": [{"stage": s} for s in stages]}]} + due = _scn.docs_update_deadline(rows[0]) + assert _scn.overdue(rows, index, [], due) == [] + assert "SCN-TRF-UPD" in _scn.overdue(rows, index, [], _bizdays.add_business_days(due, 1))[0] + done = [dict(rows[0], docs_updated_on="2026-12-01")] + assert _scn.overdue(done, index, [], _bizdays.add_business_days(due, 1)) == [] + + +def test_the_deadline_cli(tmp_path): + _, rows = _scn.read_register() + index = {"notifications": [{"scnId": r["scn_id"], "notices": [{"stage": "after-finish"}]} for r in rows]} + (tmp_path / "i.json").write_text(json.dumps(index)) + (tmp_path / "l.json").write_text(json.dumps({"entries": [_entry("adaptive", "SCN-2026-007", "2026-11-02")]})) + args = ["--index-file", str(tmp_path / "i.json"), "--log-file", str(tmp_path / "l.json")] + assert deadlines.main([*args, "--today", "2026-11-17"]) == 0 + assert deadlines.main([*args, "--today", "2026-11-18"]) == 1 From e911b5b49912a0d5f03dc1dfe2ab418d1a515e74 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:31:53 -0500 Subject: [PATCH 11/97] SCN log: accept only a full commit SHA from the published log (CodeGuard) SCN-Type: not-significant SCN-Rationale: input validation in the new SCN log builder; no system component, boundary or access change. Co-Authored-By: Claude Opus 5.5 --- scripts/build-scn.py | 6 +++++- tests/test_scn.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/build-scn.py b/scripts/build-scn.py index a49b803f..3f147272 100644 --- a/scripts/build-scn.py +++ b/scripts/build-scn.py @@ -46,6 +46,7 @@ REPO = _scn.REPO GENESIS_PATH = "scripts/build-scn.py" +SHA_RE = re.compile(r"^[0-9a-f]{40}$") MERGE_RE = re.compile(r"^Merge pull request #(\d+) from ") RUN_ID_RE = re.compile(r"(actions/runs/|run )\d{6,}") @@ -129,7 +130,10 @@ def evaluation_entry(commit, subject, body, merged_on, author, lookup): def collect(previous_entries, root=REPO, lookup=gh_pull, head="HEAD"): """Entries for the first-parent commits after the last one logged.""" if previous_entries: - last = previous_entries[-1]["commit"] + last = previous_entries[-1].get("commit", "") + # Read from the published log: a full SHA or nothing, never an option. + if not SHA_RE.match(last): + raise SystemExit("::error::SCN log: the published log's last entry has no valid commit (fail-closed)") try: _git("merge-base", "--is-ancestor", last, head, root=root) except subprocess.CalledProcessError: diff --git a/tests/test_scn.py b/tests/test_scn.py index de58c2de..500f0b27 100644 --- a/tests/test_scn.py +++ b/tests/test_scn.py @@ -375,6 +375,8 @@ def test_collect_logs_every_first_parent_commit_from_the_builder_onward(history) def test_collect_fails_when_the_last_logged_commit_left_history(history): with pytest.raises(SystemExit, match="not an ancestor"): build.collect([{"commit": "f" * 40}], root=history, lookup=_lookup) + with pytest.raises(SystemExit, match="no valid commit"): + build.collect([{"commit": "--output=/tmp/x"}], root=history, lookup=_lookup) def test_free_text_is_scrubbed_before_it_is_published(): From c3016bedc79ab6a134cba991db0719e9b881946f Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 15:51:25 -0500 Subject: [PATCH 12/97] SCN-ADP-NTF: record the post-deploy state; disclose the five late historical notices The record describes the system after a successful deploy of this commit: the deploy builds and publishes every notice (any failure fails the deploy; the round-trip check compares the served index and feed), so the mechanism is in force. SCN-2026-001..005 were published after their deadlines and are named. SCN-Type: not-significant SCN-Rationale: rule-record text and a test only; no component, boundary, data flow or access change. Co-Authored-By: Claude Opus 5.5 --- data/frr-records.json | 37 +++++++++++++++++++++++++++++++------ tests/test_scn.py | 7 +++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/data/frr-records.json b/data/frr-records.json index ebfb3e40..535a0498 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -968,12 +968,37 @@ }, { "frr_id": "SCN-ADP-NTF", - "status": "Partially Implemented", - "implementation": "An adaptive record's notice (with its new-risks summary, which the register requires) is due 10 business days after date_finished, counted with US federal holidays (scripts/_bizdays.py); it is given by publication in the certification data, and the index records the deadline and the date first published. The scheduled SCN deadline check fails and raises an issue when the notice is not published by its deadline. Notification is publication only, the method FedRAMP's rules name (an update to the Certification Data); nothing is sent to FedRAMP or anyone else.", - "evidence": [], - "gap": "Not yet shown working: no notice has been published. The first deploy after this change publishes all six adaptive records; SCN-2026-006 is due 2026-11-10 and the other five are already past their deadlines, so they will be published marked late. Closes when a deploy publishes them (the operator approves deploys).", - "closes_with": [ - "sam" + "status": "Implemented", + "implementation": "Each adaptive record's notice, carrying its new-risks summary (required by the register), is due 10 business days after date_finished, counted with US federal holidays (scripts/_bizdays.py). It is given by publication in the certification data, the method FedRAMP's rules name; nothing is sent to FedRAMP or anyone else. Every deploy builds and publishes all notices (a build or publish failure fails the deploy, and the post-publish round-trip compares the served index and feed), and the index records each notice's deadline and the date it was first published. The scheduled SCN deadline check fails and raises an issue when a notice is unpublished past its deadline. History: the six adaptive notices were first published by the deploy that introduced this mechanism. SCN-2026-001, SCN-2026-002, SCN-2026-003, SCN-2026-004 and SCN-2026-005 were published after their deadlines (2026-06-22, 2026-07-07, 2026-07-23, 2026-10-09, 2026-10-13) and are marked late (onTime false, lateNotices). SCN-2026-006 (deadline 2026-11-10) is on time only if that deploy ran by its deadline; the published index records which.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_scn.py::test_adaptive_notice_is_due_ten_business_days_after_finishing" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_a_new_notice_is_dated_today_and_late_ones_say_so" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_an_unpublished_notice_is_overdue_until_a_deploy_publishes_it" + }, + { + "kind": "test", + "ref": "tests/test_scn.py::test_the_published_outputs_are_listed_signed_and_governed" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build Significant Change Notifications and the evaluation log" + }, + { + "kind": "step", + "ref": "scn-deadlines.yml#Check SCN deadlines" + }, + { + "kind": "artifact", + "ref": ".well-known/significant-change-notifications.json" + } ] }, { diff --git a/tests/test_scn.py b/tests/test_scn.py index 500f0b27..9212d63e 100644 --- a/tests/test_scn.py +++ b/tests/test_scn.py @@ -276,6 +276,13 @@ def test_the_published_outputs_are_listed_signed_and_governed(): sign = sign[:sign.index("\n - name:")] assert "for a in significant-change-notifications scn-evaluations" in sign assert "significant-change-notifications.atom.bundle significant-change-notifications.atom" in sign + # A successful deploy means they are out: the build step has no `|| true` + # escape, and the post-publish round-trip compares the served bytes. + assert "|| true" not in step[:step.index("\n - name:")] and "continue-on-error" not in step[:step.index("\n - name:")] + rt = deploy[deploy.index("- name: Post-publish round-trip verify"):] + rt = rt[:rt.index("\n - name:")] + for f in ("significant-change-notifications.json", "significant-change-notifications.atom", "scn-evaluations.json"): + assert f"check {f} .well-known/{f}" in rt cfg = {a["path"] for a in json.loads((REPO / "infrastructure" / "schemas" / "freshness-config.json") .read_text())["artifacts"]} assert {".well-known/significant-change-notifications.json", ".well-known/significant-change-notifications.atom", From c8480a8456fbecd08df1a03a19a0076ab1ec71d5 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:36:22 -0500 Subject: [PATCH 13/97] Incident evaluation and reporting: register, IIR/OIR/FIR, clocks, IRI/NRI triggers - data/incidents.json + schema + fail-closed loader (scripts/_incidents.py): every incident carries its FedRAMP reportability evaluation and reason; default N5 until an IEC-CSO-EFI estimate (checked against the rule's rubric). - scripts/build-incident-report.py renders Initial/Ongoing/Final reports in FedRAMP's incident report schema (validated before write) and the incident log; Class C clocks and the rating scale are read from the pinned rules. - policies/triage.rego incident_triggers: VER-TFR-IRI and VER-TFR-NRI, with thresholds parsed from the rule text (fail closed on a changed text). - Reconciliation invariant (m): an open incident's findings are in the VDR. - incident-reports.yml publishes unattended (publish-only; nothing is sent); bootstrap role scoped to .well-known/incident-*; the deploy also publishes; watchdog metric and alarm for overdue reports; freshness entry. - Dated exercise (2026-11-05, exercise: true): IIR -> 2x OIR -> FIR, plus a not-reportable evaluation; run end to end in tests. - FRR records: EFR, DPR, EFI, VER-TFR-IRI, VER-TFR-NRI Implemented; IIR, OIR, FIR, AIR Partially Implemented (publish-only; bootstrap apply pending). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 28 ++ .github/workflows/incident-reports.yml | 245 +++++++++ CLAUDE.md | 1 + data/frr-records.json | 213 ++++++-- data/incidents.json | 109 ++++ docs/boundary/boundary-classification.json | 6 +- docs/cr26-applicability.md | 2 +- docs/incident-response.md | 29 +- docs/policies/ir-policy.md | 19 +- docs/runbooks/evidence-sla-alarm.md | 9 +- infrastructure/bootstrap/incident-reports.tf | 97 ++++ infrastructure/schemas/freshness-config.json | 12 + infrastructure/schemas/gate-modes.json | 3 + infrastructure/schemas/incidents.schema.json | 107 ++++ infrastructure/watchdog.tf | 13 +- infrastructure/watchdog/index.mjs | 3 + infrastructure/watchdog/logic.mjs | 22 +- policies/triage.rego | 98 ++++ policies/triage_test.rego | 102 ++++ scripts/_freshness.py | 1 + scripts/_gates.py | 1 + scripts/_incidents.py | 412 +++++++++++++++ scripts/_rules.py | 81 +++ scripts/build-incident-report.py | 413 +++++++++++++++ scripts/build-ksi-signal.py | 3 + scripts/build-ocr.py | 26 +- scripts/reconcile.py | 39 +- .../fixtures/incidents/bad/duplicate-id.json | 74 +++ .../fixtures/incidents/bad/empty-reason.json | 46 ++ .../bad/evaluated-before-detected.json | 46 ++ .../incidents/bad/final-before-recovery.json | 53 ++ .../incidents/bad/final-not-last.json | 62 +++ .../incidents/bad/first-not-initial.json | 46 ++ .../fixtures/incidents/bad/future-report.json | 53 ++ .../bad/initial-without-description.json | 46 ++ .../fixtures/incidents/bad/not-evaluated.json | 45 ++ .../bad/not-reportable-with-report.json | 46 ++ .../incidents/bad/notification-sends.json | 46 ++ .../incidents/bad/personal-email.json | 46 ++ .../incidents/bad/rating-off-rubric.json | 54 ++ .../bad/report-before-evaluation.json | 46 ++ .../fixtures/incidents/bad/unknown-field.json | 47 ++ .../incidents/bad/vuln-basis-no-finding.json | 44 ++ .../incidents/bad/vuln-basis-not-adopted.json | 46 ++ tests/fixtures/incidents/good.json | 46 ++ tests/fixtures/incidents/reconcile-m-bad.json | 46 ++ tests/test_evidence_watchdog.py | 37 ++ tests/test_incidents.py | 470 ++++++++++++++++++ tests/test_published_redaction.py | 8 + 49 files changed, 3465 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/incident-reports.yml create mode 100644 data/incidents.json create mode 100644 infrastructure/bootstrap/incident-reports.tf create mode 100644 infrastructure/schemas/incidents.schema.json create mode 100644 scripts/_incidents.py create mode 100644 scripts/build-incident-report.py create mode 100644 tests/fixtures/incidents/bad/duplicate-id.json create mode 100644 tests/fixtures/incidents/bad/empty-reason.json create mode 100644 tests/fixtures/incidents/bad/evaluated-before-detected.json create mode 100644 tests/fixtures/incidents/bad/final-before-recovery.json create mode 100644 tests/fixtures/incidents/bad/final-not-last.json create mode 100644 tests/fixtures/incidents/bad/first-not-initial.json create mode 100644 tests/fixtures/incidents/bad/future-report.json create mode 100644 tests/fixtures/incidents/bad/initial-without-description.json create mode 100644 tests/fixtures/incidents/bad/not-evaluated.json create mode 100644 tests/fixtures/incidents/bad/not-reportable-with-report.json create mode 100644 tests/fixtures/incidents/bad/notification-sends.json create mode 100644 tests/fixtures/incidents/bad/personal-email.json create mode 100644 tests/fixtures/incidents/bad/rating-off-rubric.json create mode 100644 tests/fixtures/incidents/bad/report-before-evaluation.json create mode 100644 tests/fixtures/incidents/bad/unknown-field.json create mode 100644 tests/fixtures/incidents/bad/vuln-basis-no-finding.json create mode 100644 tests/fixtures/incidents/bad/vuln-basis-not-adopted.json create mode 100644 tests/fixtures/incidents/good.json create mode 100644 tests/fixtures/incidents/reconcile-m-bad.json create mode 100644 tests/test_incidents.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index f6c9c311..06b22820 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1538,6 +1538,23 @@ jobs: cosign sign-blob --yes --new-bundle-format --bundle "$a.bundle" "$a.json" done + # Incident reports (IEC-CSO-IIR/OIR/FIR) and the incident log, from the + # incident register and this build's VDR (VER-TFR-IRI/NRI triggers). The + # same builder runs unattended in incident-reports.yml on every change to + # the register; the deploy republishes them so a deploy never serves a log + # older than the register it was built from. Each report is validated + # against FedRAMP's schema before it is written. No clock check here: an + # overdue report must not block the deploy that may be the fix; the + # incident-reports workflow and the evidence watchdog alarm on it. + - name: Build incident reports (IIR, OIR, FIR) + working-directory: infrastructure + run: | + set -euo pipefail + python3 ../scripts/build-incident-report.py --incidents ../data/incidents.json --vdr vdr-report.json --out-dir . + for f in incident-*.json; do + cosign sign-blob --yes --new-bundle-format --bundle "${f%.json}.bundle" "$f" + done + # ------------------------------------------------------------------------- # PROJECT THE CANONICAL INVENTORY INTO FEDRAMP IIW (APPENDIX M) SHAPE # ------------------------------------------------------------------------- @@ -1762,6 +1779,8 @@ jobs: [ -f "$a.json" ] || continue files="$files $a.json" done + # Incident log and reports (built every deploy, so the glob always matches). + for f in incident-*.json incident-*.md; do files="$files $f"; done # shellcheck disable=SC2086 # a word list by design python3 ../scripts/check-published-redaction.py $files @@ -1832,6 +1851,14 @@ jobs: aws s3 cp "$a.json" "s3://$BUCKET_NAME/.well-known/$a.json" --content-type application/json --cache-control "public, max-age=300" aws s3 cp "$a.bundle" "s3://$BUCKET_NAME/.well-known/$a.bundle" --content-type application/json --cache-control "public, max-age=300" done + # Incident log (IEC-CSO-EFR) + every Initial/Ongoing/Final report + signatures + aws s3 cp incident-reports.json "s3://$BUCKET_NAME/.well-known/incident-reports.json" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp incident-reports.bundle "s3://$BUCKET_NAME/.well-known/incident-reports.bundle" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp incident-reports.md "s3://$BUCKET_NAME/.well-known/incident-reports.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" + for f in incident-INC-*.json; do + aws s3 cp "$f" "s3://$BUCKET_NAME/.well-known/$f" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp "${f%.json}.bundle" "s3://$BUCKET_NAME/.well-known/${f%.json}.bundle" --content-type application/json --cache-control "public, max-age=300" + done echo "✅ All signed artifacts published to /.well-known/ (post-gate)" # ------------------------------------------------------------------------- @@ -1924,6 +1951,7 @@ jobs: check trust-center.json .well-known/trust-center.json check conformance-results.json .well-known/conformance-results.json if [ -f boundary-corroboration.json ]; then check boundary-corroboration.json .well-known/boundary-corroboration.json; fi + for f in incident-*.json; do check "$f" ".well-known/$f"; done # SLSA/in-toto provenance attestations (S2/S3): served bytes match the build. for a in ksi-signal.json security-decision-record.json vdr-report.json iiw.csv; do check "$a.intoto.jsonl" ".well-known/$a.intoto.jsonl" diff --git a/.github/workflows/incident-reports.yml b/.github/workflows/incident-reports.yml new file mode 100644 index 00000000..fbf92b77 --- /dev/null +++ b/.github/workflows/incident-reports.yml @@ -0,0 +1,245 @@ +# ============================================================================= +# GitHub Actions Workflow: Incident Reports (IEC-CSO-AIR) +# ============================================================================= +# FedRAMP's Incident Evaluation and Communication rules put an Initial Incident +# Report one hour after a FedRAMP Reportable Incident is identified (Class C, +# N3-N5), and an Ongoing report every six hours after that. The deploy path is +# behind a human reviewer and cannot keep those clocks, so reporting runs here, +# unattended, the moment the incident register changes (IEC-CSO-AIR: automate +# reporting; no hand-crafted email): +# +# push to data/incidents.json on main, or by hand, or twice a day, or after +# each nightly VDR refresh +# -> validate the register (scripts/_incidents.py, fail-closed) +# -> fetch the published VDR, verified against the deploy or nightly +# identity, for the vulnerabilities that are reportable incidents +# (VER-TFR-IRI / VER-TFR-NRI, decided by policies/triage.rego) +# -> build every IIR/OIR/FIR (validated against FedRAMP's incident report +# schema) and the incident log (scripts/build-incident-report.py) +# -> sign, redaction-gate, publish under /.well-known/incident-*, invalidate +# -> check the clocks: fail, and raise an issue, when a report is overdue +# +# The evidence watchdog (infrastructure/watchdog.tf) reads the published log +# hourly and pages by email when a listed report passes its deadline, on AWS's +# clock, whether or not this workflow runs. +# +# NOTIFICATION IS PUBLISH-ONLY. Nothing here sends a report to FedRAMP or to an +# agency. The operator has not authorized sending, and this system has no +# FedRAMP certification and no agency customers; the register's policy and +# every report say so. Sending is a separate decision and a separate change. +# +# THE TRADE, as for the evidence nightly: an unattended production write. Its +# role (infrastructure/bootstrap/incident-reports.tf, operator-applied) can put +# objects under `.well-known/incident-*` and invalidate the CDN, and nothing +# else: no Terraform, no website sync, no delete, no read of anything. +# ============================================================================= + +name: Incident Reports + +on: + push: + branches: [main] + paths: [data/incidents.json] + workflow_dispatch: + # Keeps the published log inside its 24-hour freshness window when nothing + # changes, and re-reads the VDR for new VER-TFR-IRI/NRI triggers. + schedule: + - cron: '47 7 * * *' + - cron: '47 19 * * *' + # Right after each nightly VDR refresh, so a vulnerability that has just + # become a reportable incident is in the log within minutes of the VDR. + workflow_run: + workflows: [Evidence Nightly (VDR refresh)] + types: [completed] + branches: [main] + +concurrency: + group: incident-reports + cancel-in-progress: false + +permissions: + contents: read + +env: + AWS_REGION: us-east-2 + # Same pins as the deploy and the nightly (tests/test_incidents.py checks). + OPA_VERSION: 1.18.2 + OPA_SHA256: 9903e5125ac281104f2c4b7371d10cc3b74a98933743fcbfc174f9bf0ab20de8 + SITE_BUCKET: samaydlette.com + SITE_ORIGIN: https://samaydlette.com + COSIGN_OIDC_ISSUER: https://token.actions.githubusercontent.com + +jobs: + report: + name: Build, sign and publish incident reports + runs-on: ubuntu-latest + timeout-minutes: 20 + # A nightly that did not succeed published no new VDR; a pull request never + # reaches the role. The event check closes a fork branch named `main`. + if: >- + github.event_name != 'workflow_run' || ( + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event != 'pull_request' && + github.event.workflow_run.head_repository.full_name == github.repository) + # No required reviewer: the clocks run while nobody is awake. The role this + # environment can assume writes only .well-known/incident-*. + environment: incident-reports + outputs: + clock: ${{ steps.clock.outcome }} + publish: ${{ steps.publish.outcome }} + permissions: + contents: read + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install cosign + uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 + + - name: Setup OPA (triage policy) + run: | + curl -fL --retry 5 --retry-delay 5 --connect-timeout 10 --max-time 120 \ + -o opa "https://openpolicyagent.org/downloads/v${{ env.OPA_VERSION }}/opa_linux_amd64_static" + echo "${{ env.OPA_SHA256 }} opa" | sha256sum -c - + chmod 755 ./opa + sudo mv opa /usr/local/bin + opa version + + # The VDR arrives over the public internet, so it is verified before a byte + # of it is used: it must be signed by the deploy or the nightly on main. + - name: Fetch and verify the published VDR report + run: | + set -euo pipefail + mkdir -p out + curl -sfL --retry 3 --retry-delay 5 -o vdr-report.json "${SITE_ORIGIN}/.well-known/vdr-report.json" + curl -sfL --retry 3 --retry-delay 5 -o vdr-report.bundle "${SITE_ORIGIN}/.well-known/vdr-report.bundle" + ok=0 + for wf in deploy-with-opa evidence-nightly; do + if cosign verify-blob --bundle vdr-report.bundle \ + --certificate-identity "https://github.com/${{ github.repository }}/.github/workflows/$wf.yml@refs/heads/main" \ + --certificate-oidc-issuer "${COSIGN_OIDC_ISSUER}" vdr-report.json 2>/dev/null; then ok=1; break; fi + done + [ "$ok" -eq 1 ] || { echo "::error::the published VDR does not verify; refusing to derive incidents from it"; exit 1; } + + - name: Build incident reports (IIR, OIR, FIR) + id: build + run: python3 scripts/build-incident-report.py --incidents data/incidents.json --vdr vdr-report.json --out-dir out + + - name: Sign incident reports + working-directory: out + run: | + set -euo pipefail + for f in incident-*.json; do + cosign sign-blob --yes --new-bundle-format --bundle "${f%.json}.bundle" "$f" + done + + - name: Publication redaction gate (no AWS account ID) + working-directory: out + run: python3 ../scripts/check-published-redaction.py incident-*.json incident-*.md + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + timeout-minutes: 6 + with: + # Defined in infrastructure/bootstrap/incident-reports.tf. Unset until + # the operator applies it and sets the variable; this step then fails + # and the alert job says so. + role-to-assume: ${{ vars.AWS_INCIDENT_REPORTS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + retry-max-attempts: 5 + + - name: Publish incident reports to /.well-known/ + id: publish + working-directory: out + run: | + set -euo pipefail + put() { + aws s3api put-object --bucket "${SITE_BUCKET}" --key ".well-known/$1" --body "$1" \ + --content-type "$2" --cache-control "public, max-age=300" --query 'ETag' --output text > /dev/null + echo "published .well-known/$1" + } + # Every report first, the log that lists them last. + for f in incident-INC-*.json; do + put "$f" "application/json" + put "${f%.json}.bundle" "application/json" + done + put incident-reports.md "text/markdown; charset=utf-8" + put incident-reports.bundle "application/json" + put incident-reports.json "application/json" + + - name: Invalidate CloudFront cache (incident paths only) + env: + DISTRIBUTION_ID: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }} + run: | + set -euo pipefail + aws cloudfront create-invalidation --distribution-id "${DISTRIBUTION_ID}" \ + --paths "/.well-known/incident-*" --query 'Invalidation.Id' --output text + + - name: Assert the published incident log is fresh at the edge + run: | + set -euo pipefail + python3 scripts/check-evidence-freshness.py \ + --url "${SITE_ORIGIN}/.well-known/incident-reports.json" \ + --expect-sha256 "$(sha256sum out/incident-reports.json | awk '{print $1}')" \ + --retries 12 --retry-delay 20 + + # Last, so an overdue report is published (the log says what is late) and + # still fails the run. The watchdog pages on the same list independently. + - name: Incident report clocks (fail when a report is due and missing) + id: clock + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: python3 scripts/build-incident-report.py --check out/incident-reports.json + + alert: + name: Alert on incident reporting + runs-on: ubuntu-latest + needs: [report] + if: always() && needs.report.result == 'failure' + permissions: + contents: read + issues: write + steps: + - name: Raise or update the incident-reporting issue + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CLOCK: ${{ needs.report.outputs.clock }} + PUBLISH: ${{ needs.report.outputs.publish }} + with: + script: | + const TITLE = 'Incident reporting: a report is overdue or was not published'; + const runUrl = `${context.payload.repository.html_url}/actions/runs/${context.runId}`; + const clock = process.env.CLOCK || 'not reached'; + const publish = process.env.PUBLISH || 'not reached'; + const runbook = `${context.payload.repository.html_url}/blob/main/docs/incident-response.md#reporting-fedramp-reportable-incidents-iec`; + const body = [ + `The incident reports workflow failed ([run](${runUrl}), trigger \`${context.eventName}\`).`, + ``, + `| | |`, `|---|---|`, + `| Report clocks | \`${clock}\` |`, + `| Publish | \`${publish}\` |`, + ``, + clock === 'failure' + ? `**A FedRAMP incident report is past its Class C deadline.** The "Incident report clocks" step names each one. Record the report (or open the incident) in data/incidents.json and push; this workflow publishes it.` + : `**The reports were not built or published.** Check the failing step. If "Configure AWS credentials" failed, the incident-reports role or the AWS_INCIDENT_REPORTS_ROLE_ARN variable is missing (infrastructure/bootstrap/incident-reports.tf).`, + ``, + `Runbook: ${runbook}. This issue is reused; each failure adds a comment.`, + ].join('\n'); + const existing = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, repo: context.repo.repo, state: 'all', per_page: 100, + labels: 'incident-reporting', + }); + const match = existing.filter(i => i.title === TITLE && !i.pull_request) + .sort((a, b) => (a.state === b.state ? a.number - b.number : a.state === 'open' ? -1 : 1))[0]; + if (match) { + if (match.state === 'closed') { + await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, + issue_number: match.number, state: 'open' }); + } + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, + issue_number: match.number, body }); + } else { + await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title: TITLE, + body, labels: ['compliance', 'incident-reporting', 'urgent'], assignees: [context.repo.owner] }); + } diff --git a/CLAUDE.md b/CLAUDE.md index d092298d..96af2dfe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,7 @@ Confirm against the working tree on first run; correct this section if reality d - `tools/boundary-map/corroborate.py` (local only) — independent corroboration of the map by RAMPART on TAP, which collects the account from the AWS APIs. The operator runs it and commits `docs/boundary/corroboration.json`; the deploy signs and publishes it as `/.well-known/boundary-corroboration.json`, and the trust center reports its result and age (30-day window). Resources only TAP sees go to the decision queue. - `scripts/build-trust-center.py` — the trust center's data (`/.well-known/trust-center.json`, signed): the "is this picture true?" checks, the queue of decisions that need a person, the policy catalog, the escalation rate and the decision log, all derived from this build's artifacts and the registers. Built after the reconciliation gate; fails closed if its inputs are bound to another inventory. A register field that is not recorded (who decided, when, review date) is published as a gap, never filled in. - `.github/workflows/deploy-with-opa.yml`, `infrastructure/Makefile`, `.checkov.yaml` — the pipeline, entry points, and IaC scan config. +- `data/incidents.json` + `scripts/_incidents.py` + `scripts/build-incident-report.py` — the incident register and FedRAMP incident reporting (IEC-CSO-*; VER-TFR-IRI/NRI via `policies/triage.rego` `incident_triggers`). Every incident carries its reportability evaluation; reportable ones get Initial/Ongoing/Final reports in FedRAMP's schema on the Class C clocks read from the pinned rules, published with the incident log at `/.well-known/incident-reports.json`. `.github/workflows/incident-reports.yml` publishes them unattended (role in `infrastructure/bootstrap/incident-reports.tf`, `.well-known/incident-*` only); the watchdog pages on an overdue report. Notification is publish-only: nothing is sent to FedRAMP or an agency. Exercises are records with `exercise: true`. - `.github/workflows/evidence-nightly.yml` — the unattended VDR refresh, **twice a day**, plus once after every successful deploy to `main` (`workflow_run`, so a deploy that fixes a failing nightly also clears its beacon): re-scan (Syft + Grype against a fresh vulnerability DB), rebuild the VDR + trend, run the same vulnerability and reconciliation gates, republish six objects under `.well-known/vdr-*` (report, bundle, markdown, trend, trend bundle, provenance attestation), and on every run, pass or fail, write a status beacon (`.well-known/vdr-status.json`, outcomes only, unsigned). No `terraform apply`, no website sync. Its alerting is the `alert` job (a GitHub issue), not the deploy workflow's `notify`. - `infrastructure/watchdog.tf` + `infrastructure/watchdog/` — the **evidence SLA watchdog**: an hourly Lambda reads every artifact listed in the governed freshness config from the bucket (AWS APIs only, no internet egress), publishes the VDR's and runtime signal's ages, the nightly's result and a count of artifacts past their window as CloudWatch metrics, and four alarms email the operator via a CMK-encrypted SNS topic. This is the page; the GitHub issue is the paper trail. The email subscription is created out of band so no address enters Terraform state. Runbook: `docs/runbooks/evidence-sla-alarm.md`. diff --git a/data/frr-records.json b/data/frr-records.json index 535a0498..4e1103cc 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -658,72 +658,163 @@ }, { "frr_id": "IEC-CSO-AIR", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", - "evidence": [], - "gap": "No automation exists for incident reporting; nothing exists to automate. To close: Once the incident register and report builder exist, run them from a workflow (workflow_dispatch + on push to the incident register) that validates, signs, publishes and sends IIR/OIR/FIR automatically, and pages through the existing watchdog/SNS when a report deadline is near or missed.", + "status": "Partially Implemented", + "implementation": ".github/workflows/incident-reports.yml runs on every push of data/incidents.json to main, by hand, twice a day and after each nightly VDR refresh: it validates the register, verifies the published VDR, builds every report against FedRAMP's schema, signs, redaction-gates and publishes under /.well-known/incident-* with its own least-privilege role (infrastructure/bootstrap/incident-reports.tf), and fails with a GitHub issue when a report is overdue; the evidence watchdog pages on the same deadlines hourly. A person records the facts in the register; nothing else is done by hand. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "evidence": [ + { + "kind": "step", + "ref": "incident-reports.yml#Build incident reports (IIR, OIR, FIR)" + }, + { + "kind": "step", + "ref": "incident-reports.yml#Incident report clocks (fail when a report is due and missing)" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_air_workflow_validates_builds_signs_publishes_and_checks_the_clock_in_order" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } + ], + "gap": "Not yet running: the role in infrastructure/bootstrap/incident-reports.tf is operator-applied, and the incident-reports GitHub Environment and AWS_INCIDENT_REPORTS_ROLE_ARN variable must be set. Reporting reaches no affected party automatically because delivery is publish-only by the operator's choice. To close: apply the bootstrap role and set the environment and variable; then the operator's delivery decision (IEC-CSO-IIR).", "closes_with": [ - "W6" + "sam" ] }, { "frr_id": "IEC-CSO-DPR", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", - "evidence": [], - "gap": "Nothing assigns a PAIN rating to an incident, so there is no default-to-N5 behaviour and no record of whether a prompt estimate replaced it. The VDR PAIN classifier (scripts/build-vdr-report.py:1730 assign_pain) rates vulnerabilities, not incidents. To close: In the incident register, set pain=5 with basis 'default (IEC-CSO-DPR)' when a record becomes reportable, and allow it to drop only through an EFI estimate record (rating, reason, estimated_at). Drive the IIR/OIR/FIR clocks from the effective rating. Test both paths and show them in the simulated-incident log.", - "closes_with": [ - "W6" + "status": "Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). A reportable incident is rated at the default the IEC-CSO-DPR text states (N5, read from the rule) until an IEC-CSO-EFI estimate record exists, and the report clocks use the rating in force at each start time. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. ", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_an_unrated_incident_is_n5_and_its_initial_report_is_due_in_an_hour" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_exercise_shows_the_default_rating_then_the_estimate" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } ] }, { "frr_id": "IEC-CSO-EFI", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", - "evidence": [], - "gap": "No Incident Rating step: nothing estimates likely adverse impact on agency customers for an incident or assigns N1-N5 using the minimal/narrow/disruptive/debilitating x 1 or >1 agency scale. The required artifact (an incident log showing the evaluation and its reason) does not exist. To close: Add an EFI record type to the incident register: rating N1-N5, the customer-effect word, single or multiple agencies, reason, estimated_at. Document the rubric in docs/incident-response.md (it can reuse the VDR effect words from infrastructure/schemas/vdr-pain-config.json). Exercise it in the simulated incidents.", - "closes_with": [ - "W6" + "status": "Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). An estimate record carries rating, customer effect (minimal/narrow/disruptive/debilitating), one or several agencies, reason and time; the loader refuses a rating that is not the one the IEC-CSO-EFI bullets give for that effect and agency count (the rubric is parsed from the rule text). Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. ", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_each_known_bad_register_fails_closed_for_its_reason" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_rating_scale_and_default_come_from_the_rules" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_exercise_shows_the_default_rating_then_the_estimate" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } ] }, { "frr_id": "IEC-CSO-EFR", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A. docs/policies/ir-policy.md:15 only restates the trigger.", - "evidence": [], - "gap": "No step that evaluates each incident for effect (actual or likely) on confidentiality/integrity of federal customer data, no recorded determination with its reason, and no incident log. The required artifact (an incident log showing one or more real or simulated incidents evaluated, with the reason for the determination) does not exist. The runbook explicitly scopes federal-data incidents out, so it can't make the determination the rule asks for. To close: Add an incident register (e.g. data/incidents.json + schema) whose records carry detected_at, evaluated_at, reportable (bool), reason, and the federal-customer-data types considered. Add an 'Evaluate FedRAMP reportability' step to docs/incident-response.md triage (before Contain), define the data types that would be federal customer data for this offering, and fail a check if any incident record lacks an evaluation. Exercise it with at least two simulated incidents (one reportable, one not) and publish the log.", - "closes_with": [ - "W6" + "status": "Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). Every incident record must carry an evaluation (evaluated_at, reportable true/false, basis, reason, the federal customer data considered); a record without one does not load. docs/incident-response.md triage now evaluates reportability before containment. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. The published incident log is the artifact the rule asks for.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_each_known_bad_register_fails_closed_for_its_reason" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_exercise_logs_a_not_reportable_evaluation_with_its_reason" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } ] }, { "frr_id": "IEC-CSO-FIR", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A. A schema test fixture (tests/fixtures/cr26/good/incident.json, reportType Final, CVE-2099-0142) checks the validator only; it is not an incident.", - "evidence": [], - "gap": "No Final Incident Report generation on resolution/recovery, no final updates to all previously reported items, no clock (Class C: within 6 hours for N3-N5, 1 business day for N1-N2, after recovery), no delivery. No sample FIR. To close: Add reportType=Final generation when the incident record gets resolved_at and recovery_complete_at, with final values for every earlier IIR/OIR field. Alarm on the FIR deadline. Close the simulated incident with a FIR and link it from the incident log and the Ongoing Certification Report.", + "status": "Partially Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). A Final report is allowed only after resolved_at and recovery_complete_at, restates final values for every earlier item, carries resolvedAt, and is due an FIR period after recovery (6 hours at N3-N5, 1 business day at N1-N2); overdue fails the workflow and pages. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_the_final_report_is_due_after_recovery_and_ongoing_reports_stop" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_ongoing_and_final_reports_restate_every_earlier_item" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } + ], + "gap": "Published, not delivered (see IEC-CSO-IIR). To close: the operator's delivery decision, as for IEC-CSO-IIR.", "closes_with": [ - "W6" + "sam" ] }, { "frr_id": "IEC-CSO-IIR", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A. The vendored schema infrastructure/schemas/fedramp-2026-06-24/fedramp-incident-report-schema-2026-06-24.json (reportType Initial/Ongoing/Final, federalIncidentCoordinator, timeline, potentialImpact, functionalImpact, recoveryPlan, affectedAgencies) and its validator are in place, but nothing produces a report.", - "evidence": [], - "gap": "No Initial Incident Report generator, no delivery to all affected parties (FedRAMP + affected agencies), and no clock (Class C: within 1 hour for N3-N5, 24 hours for N2, 1 business day for N1). None of the 8 items exists in any record: federal incident response coordinator contact, tracking ID, description, timeline (start, detection time/source, evaluation time, milestones), historical/current PAIN with EFI explanation, functional impact incl. C/I and federal data types, recovery plan/milestones, likely affected agencies. No sample IIR exists, real or simulated. To close: Build scripts/build-incident-report.py that renders an IIR from the incident register into the vendored schema (validated before write) plus a human-readable version; name the federal incident response coordinator contact; add a delivery step (e.g. signed publication to a restricted location plus email to a configured recipient list holding FedRAMP's incident address and each agency's contact) that starts on 'reportable' and is alarmed against the PAIN-based deadline (watchdog). Run a timed simulated incident end to end and keep the IIR.", + "status": "Partially Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). The Initial report carries all eight items (coordinator, tracking ID, description, timeline with start/detection/evaluation/milestones, historical and current rating with the EFI explanation, functional impact, recovery plan, likely affected agencies). Its clock (1 hour at N3-N5, 24 hours at N2, 1 business day at N1) runs from the evaluation; the incident-reports workflow fails and the evidence watchdog pages (IncidentReportsOverdue) once it passes. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_exercise_runs_initial_ongoing_final_all_on_time" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_every_report_validates_against_fedramps_schema_and_is_listed_with_its_hash" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_an_unrated_incident_is_n5_and_its_initial_report_is_due_in_an_hour" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } + ], + "gap": "Reports are published, not delivered: nothing notifies FedRAMP or an agency, and the operator has not decided to send. The unattended publisher needs infrastructure/bootstrap/incident-reports.tf applied and the AWS_INCIDENT_REPORTS_ROLE_ARN variable and incident-reports environment set before it runs. To close: the operator decides the delivery channel (e.g. SES from the system's domain to FedRAMP's incident address) and it is built behind that decision.", "closes_with": [ - "W6" + "sam" ] }, { "frr_id": "IEC-CSO-OIR", - "status": "Not Implemented", - "implementation": "No incident record, incident log, evaluation step or report generator exists in scripts/, policies/ or .github/workflows/ (grep for incident/IEC finds only build-ocr.py's static attestation text, scripts/_fedramp_schema.py:41 registering the vendored incident-report schema, and test fixtures tests/fixtures/cr26/{good,bad}/incident*.json). docs/incident-response.md 'Scope' lists federal-customer-data incidents as out of scope; docs/cr26-applicability.md records IEC as N/A.", - "evidence": [], - "gap": "No Ongoing Incident Report mechanism or cadence (Class C: every 6 hours for N3-N5, 24 hours for N2, 1 business day for N1), no 'updates or lack of updates' logic, and none of the additional items recorded: observed activity, IOCs, related CVE, root cause, response/recovery activities. No sample OIR. To close: Extend the incident report builder for reportType=Ongoing: diff against the previous report so every earlier item is restated (changed or 'no update'), add the 5 OIR fields to the incident record, and schedule/alarm the next OIR from the effective PAIN. Produce OIRs in the simulated exercise.", + "status": "Partially Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). Each Ongoing report restates every earlier item with its latest value and marks each 'updated' or 'no update', and adds observed activity, indicators of compromise, related CVEs, root cause and response/recovery activities. The next one is due an OIR period (6 hours at N3-N5, 24 at N2, 1 business day at N1) after the previous report until the incident is resolved; overdue fails the workflow and pages. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_ongoing_reports_are_due_every_oir_period_until_resolved" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_ongoing_and_final_reports_restate_every_earlier_item" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } + ], + "gap": "Published, not delivered (see IEC-CSO-IIR). To close: the operator's delivery decision, as for IEC-CSO-IIR.", "closes_with": [ - "W6" + "sam" ] }, { @@ -1756,12 +1847,25 @@ }, { "frr_id": "VER-TFR-IRI", - "status": "Not Implemented", - "implementation": "Checked: policies/triage.rego block_rule 'top-impact-lev-irv' (G2) blocks only N5+LEV+IRV, as a pipeline block; docs/policies/ir-policy.md:13 states the IRI rule in prose.", - "evidence": [], - "gap": "Nothing treats an internet-reachable likely exploitable vulnerability at N4 or N5 as a FedRAMP Reportable Incident: no incident record is opened, no IIR/OIR/FIR clocks start, and nothing ends that status when it's partially mitigated to N3 or below. N4 IRV+LEV isn't even flagged beyond its 4-day deadline. Depends on the missing IEC machinery. To close: In triage.rego add an 'incident' decision (IRV and LEV and PAIN > N3) and in the builder open/update an incident record in the incident register (see IEC-CSO-EFR) with default PAIN-5 treatment, closing it when a verified partial mitigation brings it to N3 or below. Test with a synthetic finding.", - "closes_with": [ - "W6" + "status": "Implemented", + "implementation": "policies/triage.rego (incident_triggers) names every VDR finding that is internet-reachable, likely exploitable and above the N-rating the VER-TFR-IRI Class C text states (read from the rule; a changed text fails closed), unless it is a false positive or fully mitigated; once partially mitigated to N3 or below it no longer triggers. scripts/build-incident-report.py publishes each trigger in the incident log and, while no unresolved incident in data/incidents.json names it, puts an Initial report due from the finding's first evaluation at the default N5 clock: the incident-reports workflow fails and the evidence watchdog pages when that passes. Opening the incident record is the operator's act; its reports then follow IEC-CSO-IIR/OIR/FIR. Reconciliation invariant (m) fails a build whose open incident names a finding the VDR does not carry. Adopted in data/incidents.json policy.adopted_vulnerability_rules.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_an_irv_lev_n4_finding_with_no_incident_is_due_an_initial_report" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_a_partially_mitigated_finding_at_n3_is_no_longer_an_incident" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_reconcile_m_fails_the_known_bad_register" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } ] }, { @@ -1796,12 +1900,21 @@ }, { "frr_id": "VER-TFR-NRI", - "status": "Not Implemented", - "implementation": "Checked: docs/policies/ir-policy.md:13 adopts NRI in prose ('non-internet-reachable likely-exploitable vulnerabilities at PAIN N5 are reportable until N4 or below per VER-TFR-NRI'); no code implements it.", - "evidence": [], - "gap": "MAY rule: optional, but the published policy claims it's adopted and nothing implements it. NLEV-IRV N5 findings never open an incident. To close: Either implement alongside IRI (NIRV and LEV and PAIN = N5 -> incident until a partial mitigation brings it to N4) or remove the claim from ir-policy.md and record the decision not to use the MAY.", - "closes_with": [ - "W6" + "status": "Implemented", + "implementation": "Adopted (docs/policies/ir-policy.md; data/incidents.json policy.adopted_vulnerability_rules). policies/triage.rego names every likely exploitable, not internet-reachable VDR finding at the N-rating the VER-TFR-NRI Class C text states (N5), until partially mitigated to N4 or below; the incident log and clocks treat it exactly as VER-TFR-IRI does. An unadopted rule's triggers are still shown but start no clock.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_incidents.py::test_an_nirv_lev_n5_finding_opens_under_nri_and_n4_does_not" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_an_unadopted_rule_is_shown_but_starts_no_clock" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-reports.json" + } ], "may_rationale": "Adopted, as the incident response policy states: treating a likely exploitable, non-internet-reachable N5 vulnerability as reportable is stricter than required and costs little at this scale." } diff --git a/data/incidents.json b/data/incidents.json new file mode 100644 index 00000000..f00b35da --- /dev/null +++ b/data/incidents.json @@ -0,0 +1,109 @@ +{ + "$comment": "The incident register (IEC-CSO-EFR/DPR/EFI/IIR/OIR/FIR). Every incident is recorded here once, with its FedRAMP reportability evaluation and reason; a reportable one carries its Incident Rating estimates and the Initial, Ongoing and Final reports issued, each with when it was issued and what it set. scripts/_incidents.py validates it (fail-closed) and scripts/build-incident-report.py renders the reports and the incident log; .github/workflows/incident-reports.yml publishes them on every change to this file. The report clocks are FedRAMP's Class C timeframes, read from the pinned rules, never set here. Everything in this file is published. Exercises are marked exercise: true. Shape: infrastructure/schemas/incidents.schema.json.", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Sam Aydlette, System Owner and incident response lead; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only. Each report is signed and published under https://samaydlette.com/.well-known/ (incident-reports.json lists every report and its sha256). Nothing is sent to FedRAMP or to any agency: the operator has not authorized sending, and this system has no FedRAMP certification and no agency customers.", + "adopted_vulnerability_rules": ["VER-TFR-IRI", "VER-TFR-NRI"], + "federal_customer_data": "None. This system holds no federal customer data (FRD-FCD) and has no agency customers. Were an agency to use the Silk Reeling application, what its users supply to it (pose landmarks) would be the federal customer data to consider; the operator's public site content, account information and the published evidence are not.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-001", + "title": "Exercise: internet-reachable likely exploitable N5 vulnerability in the Silk Reeling API", + "exercise": true, + "started_at": "2026-11-05T02:40:00Z", + "detected_at": "2026-11-05T03:00:00Z", + "detection_source": "Exercise inject: a simulated VDR finding that policies/triage.rego flags under VER-TFR-IRI", + "evaluation": { + "evaluated_at": "2026-11-05T03:10:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Simulated timeline. An internet-reachable, likely exploitable vulnerability rated N5 in a dependency of the Silk Reeling API. The operator treats such a vulnerability as a FedRAMP Reportable Incident (VER-TFR-IRI) until it is partially mitigated to N3 or below. The scenario assumes one agency whose users supply data to the application.", + "federal_customer_data_considered": "Scenario assumption: pose landmarks supplied by one agency's users of the Silk Reeling application. In reality the system holds no federal customer data." + }, + "findings": ["CVE-2099-0001"], + "estimates": [ + { + "estimated_at": "2026-11-05T04:30:00Z", + "rating": 3, + "customer_effect": "disruptive", + "agencies": "single", + "reason": "Exploitation could expose the landmark data of many of the assumed agency's users without interrupting the service for long; no other agency is assumed." + } + ], + "reports": [ + { + "type": "Initial", + "at": "2026-11-05T03:40:00Z", + "set": { + "description": "A request-parsing dependency of the Silk Reeling API has a vulnerability (simulated CVE-2099-0001) that an unauthenticated client could exploit over the internet to read other users' submitted data.", + "functional_impact": "Potential loss of confidentiality of pose landmarks submitted by the assumed agency's users; no integrity or availability effect is expected.", + "recovery_plan": "Throttle the affected endpoint now; upgrade the dependency to the fixed release and redeploy through the reviewed pipeline. Targets: partial mitigation within 3 hours, remediation within 1 day.", + "affected_agencies": ["Hypothetical agency A (exercise assumption)"], + "related_cves": ["CVE-2099-0001"], + "milestones": [ + {"at": "2026-11-05T03:00:00Z", "description": "Triage flagged the finding under VER-TFR-IRI (simulated)"} + ] + } + }, + { + "type": "Ongoing", + "at": "2026-11-05T09:20:00Z", + "set": { + "observed_activity": "No exploitation attempts in the API Gateway access logs since the vulnerable release was deployed (simulated review).", + "indicators_of_compromise": ["None observed (simulated)"], + "root_cause": "A vulnerable release of a request-parsing dependency (simulated CVE-2099-0001).", + "response_and_recovery_activities": "Endpoint throttled at 04:15 (simulated), a partial mitigation; Incident Rating estimated at N3; fixed release identified.", + "milestones": [ + {"at": "2026-11-05T04:15:00Z", "description": "Partial mitigation: affected endpoint throttled (simulated)"} + ] + } + }, + { + "type": "Ongoing", + "at": "2026-11-05T15:00:00Z", + "set": { + "response_and_recovery_activities": "Fixed release built and passing the pipeline's gates; deploy awaiting approval (simulated)." + } + }, + { + "type": "Final", + "at": "2026-11-05T17:45:00Z", + "set": { + "response_and_recovery_activities": "Fixed release deployed at 16:30 and the finding no longer appears in the VDR (simulated); recovery verified at 17:00.", + "recovery_plan": "Complete.", + "milestones": [ + {"at": "2026-11-05T16:30:00Z", "description": "Fixed release deployed (simulated)"} + ] + } + } + ], + "resolved_at": "2026-11-05T16:30:00Z", + "recovery_complete_at": "2026-11-05T17:00:00Z" + }, + { + "id": "INC-2026-002", + "title": "Exercise: unauthorized change to a page of the public site", + "exercise": true, + "started_at": "2026-11-05T10:00:00Z", + "detected_at": "2026-11-05T10:20:00Z", + "detection_source": "Exercise inject: simulated drift of a website HTML artifact between the deploy-time and runtime KSI signals", + "evaluation": { + "evaluated_at": "2026-11-05T10:35:00Z", + "reportable": false, + "basis": "IEC-CSO-EFR", + "reason": "Simulated timeline. The changed content is the operator's public site content, which is not federal customer data (FRD-FCD), and the change is not likely to affect the confidentiality or integrity of any federal customer data. Not a FedRAMP Reportable Incident; handled under the incident response runbook without FedRAMP reports.", + "federal_customer_data_considered": "None present: public website content only." + }, + "estimates": [], + "reports": [], + "resolved_at": "2026-11-05T11:30:00Z", + "recovery_complete_at": "2026-11-05T11:45:00Z" + } + ] +} diff --git a/docs/boundary/boundary-classification.json b/docs/boundary/boundary-classification.json index 69d1e17c..48d2e591 100644 --- a/docs/boundary/boundary-classification.json +++ b/docs/boundary/boundary-classification.json @@ -20,7 +20,7 @@ {"key": "audit", "zone": "system", "label": "Audit and shared keys", "match": ["aws::audit_log_trail::*", "aws::object_store::cloudtrail", "aws::kms_key::at_rest"]}, {"key": "trust-root", "zone": "system", "label": "CI/CD trust root (operator-applied bootstrap stack)", - "match": ["aws::oidc_provider::*", "aws::iam_role::deploy", "aws::iam_role::evidence_nightly", "aws::iam_role::plan", "aws::iam_group::*", "aws::iam_policy::*", "aws::object_store::tfstate", "aws::kv_table::*"]}, + "match": ["aws::oidc_provider::*", "aws::iam_role::deploy", "aws::iam_role::evidence_nightly", "aws::iam_role::incident_reports", "aws::iam_role::plan", "aws::iam_group::*", "aws::iam_policy::*", "aws::object_store::tfstate", "aws::kv_table::*"]}, {"key": "github", "zone": "external", "label": "GitHub repository, Actions and OIDC", "match": ["ext::github-repo", "ext::github-oidc"]}, {"key": "sigstore", "zone": "external", "label": "Sigstore (Fulcio CA and Rekor log)", @@ -69,6 +69,8 @@ {"id": "J", "label": "Evidence Nightly → S3 (.well-known/vdr-*)", "path": [{"component": "ext::github-oidc"}, {"component": "aws::iam_role::evidence_nightly"}, {"component": "aws::object_store::website"}], "protocol": "HTTPS 443 (TLS 1.2+, SigV4)", "auth": "GitHub OIDC → STS; the role can write only .well-known/vdr-* (unattended)", "encryption": "In transit TLS; at rest SSE-S3", "data": "Refreshed VDR and trend (Sigstore-signed)"}, {"id": "K", "label": "Watchdog → S3 (evidence freshness read)", "path": [{"component": "aws::event_schedule::evidence_watchdog"}, {"component": "aws::function::evidence_watchdog"}, {"component": "aws::object_store::website"}], - "protocol": "AWS-internal", "auth": "Watchdog Lambda role: s3:GetObject on the published evidence keys only", "encryption": "In transit: TLS", "data": "Published evidence timestamps; results go out as CloudWatch metrics that drive flow I's alarms"} + "protocol": "AWS-internal", "auth": "Watchdog Lambda role: s3:GetObject on the published evidence keys only", "encryption": "In transit: TLS", "data": "Published evidence timestamps; results go out as CloudWatch metrics that drive flow I's alarms"}, + {"id": "L", "label": "Incident Reports → S3 (.well-known/incident-*)", "path": [{"component": "ext::github-oidc"}, {"component": "aws::iam_role::incident_reports"}, {"component": "aws::object_store::website"}], + "protocol": "HTTPS 443 (TLS 1.2+, SigV4)", "auth": "GitHub OIDC → STS; the role can write only .well-known/incident-* (unattended)", "encryption": "In transit TLS; at rest SSE-S3", "data": "Incident log and IIR/OIR/FIR reports (Sigstore-signed)"} ] } diff --git a/docs/cr26-applicability.md b/docs/cr26-applicability.md index 0d20e28d..642739e1 100644 --- a/docs/cr26-applicability.md +++ b/docs/cr26-applicability.md @@ -25,6 +25,6 @@ The POA&M is retired (2026-11-04): CR26 has no POA&M. Open findings and their de - **IVV — Independent Verification and Validation.** `IVV-CSO-FIA` (Class C MUST) requires a persistent independent assessment by a **FedRAMP Recognized independent assessment service** (the artifact formerly called a 3PAO) or by FedRAMP. This system is self-attested and has engaged no such assessor, so the *independent* half cannot be met. The provider-side verification and validation an assessor would consume **are** produced (in the Security Decision Record); the independent half is unmet by design and recorded N/A. - **MKT — Marketplace Listing.** `MKT-IIP-AGU` requires demonstrating the offering is for agency use within 44 USC § 3506 scope, and the rule text is explicit that *"services used by private companies to meet other compliance requirements (such as CMMC) that do not also meet one of the above use cases are outside the scope of FedRAMP."* There is no agency use case and no intent to list, so MKT is N/A by statutory scope. - **AFC — Addressing FedRAMP Communication.** `AFC-CSO-INB` and the rest require maintaining a FedRAMP Security Inbox and handling FedRAMP correspondence. With no listing and no sponsor, FedRAMP has no reason to correspond, so there is no channel to maintain. AFC's grace period already ended **2026-07-01**; it is recorded N/A absent a sponsor. -- **IEC — Incident Evaluation and Communication.** A FedRAMP Reportable Incident under `IEC-CSO-EFR` is triggered by an effect on the confidentiality or integrity of **federal customer data**. This system holds none, so no incident is FedRAMP-reportable and the `IEC-CSO-IIR` reporting clocks never start. The evaluation step still runs — the VDR aggregator assigns a PAIN rating to every finding — so the machinery is in place the moment federal data ever enters scope. +- **IEC — Incident Evaluation and Communication.** A FedRAMP Reportable Incident under `IEC-CSO-EFR` is triggered by an effect on the confidentiality or integrity of **federal customer data**. This system holds none, so in practice no incident is FedRAMP-reportable. Every incident is still evaluated and recorded in `data/incidents.json`; reportable ones get Initial, Ongoing and Final reports in FedRAMP's schema, on the Class C clocks, published under `/.well-known/` (publish-only: nothing is sent). Exercised 2026-11-05 with simulated incidents marked as exercises; see `docs/incident-response.md`. These four are N/A because of *what this system is*, not because of an engineering gap. If the system were ever sponsored, listed, assessed, or handling federal data, each would activate, and the mechanisms above (the SDR's V&V evidence, the CDS overview package, the PAIN evaluation) are the inputs those activated obligations would consume. diff --git a/docs/incident-response.md b/docs/incident-response.md index 88101566..bae342b8 100644 --- a/docs/incident-response.md +++ b/docs/incident-response.md @@ -15,9 +15,10 @@ In-scope incidents: Out of scope: -- Any incident involving federal customer data, end-user accounts, or downstream agencies — this system has none of those - Service-level incidents around availability promises to anyone, since none exist +Federal customer data, end-user accounts and downstream agencies: this system has none, so no incident here is expected to be a FedRAMP Reportable Incident. Every incident is still evaluated for it (step 2 below) and the determination is recorded, because the evaluation is what the rules ask for, and because a wrong assumption would otherwise go unrecorded. + ## Detection Active detection sources: @@ -32,14 +33,30 @@ Active detection sources: ## Triage and response procedures 1. **Confirm.** Compare the runtime KSI signal at `/.well-known/ksi-signal-runtime.json` against the deploy-time signal. Any unexplained `result: "fail"` validation, or any component drift, is high-confidence signal of a real change. -2. **Contain.** +2. **Record and evaluate FedRAMP reportability (IEC-CSO-EFR).** Add the incident to [`data/incidents.json`](../data/incidents.json) with `detected_at`, `detection_source` and an `evaluation`: when it was evaluated, `reportable` true or false, the `basis` (`IEC-CSO-EFR`, or `VER-TFR-IRI` / `VER-TFR-NRI` for a vulnerability), the `reason`, and the federal customer data considered. Does it affect, or is it likely to affect, the confidentiality or integrity of federal customer data (data an agency or its users supply to the service, not account information or telemetry)? If yes it is a FedRAMP Reportable Incident: continue with [Reporting](#reporting-fedramp-reportable-incidents-iec) in parallel with containment. Push the record; the incident log republishes within minutes. +3. **Contain.** - **Credential compromise (AWS):** rotate the IAM access keys via the IAM console; revoke any active sessions; rotate the GitHub Actions secret holding the credentials. - **Credential compromise (GitHub):** revoke any compromised PATs; rotate the workflow's `GITHUB_TOKEN` is automatic per-run; reset OAuth grants if external apps are involved. - **Content compromise:** revert via `git`, push to `main`, and approve the `prod` deploy. S3 versioning preserves the prior state; CloudFront invalidation refreshes the edge. - **Supply-chain compromise:** pin the offending dependency to a known-good version, redeploy, and file a Dependabot suppression if the alert source is the GitHub Advisory Database. -3. **Eradicate.** Identify the root cause (credential, code, or config) and patch it. Confirm the patch holds via a fresh approved deploy. -4. **Recover.** Verify the runtime signal returns to clean. Verify `cosign verify-blob` still passes against the new signal and bundle. Confirm the site is reachable end-to-end. -5. **Lessons learned.** Append an after-action entry to the section below. +4. **Eradicate.** Identify the root cause (credential, code, or config) and patch it. Confirm the patch holds via a fresh approved deploy. +5. **Recover.** Verify the runtime signal returns to clean. Verify `cosign verify-blob` still passes against the new signal and bundle. Confirm the site is reachable end-to-end. +6. **Lessons learned.** Append an after-action entry to the section below. + +## Reporting FedRAMP Reportable Incidents (IEC) + +_AI-drafted (2026-11-05) to describe what is now built; for the operator's review._ + +Reporting is automated from the incident register. A person records facts; the pipeline renders, validates, signs, publishes and keeps the clocks. + +- **Register.** [`data/incidents.json`](../data/incidents.json), read through `scripts/_incidents.py` (fails closed; shape in `infrastructure/schemas/incidents.schema.json`). Each report is an entry in the incident's `reports` list (`Initial`, then `Ongoing`, then `Final`) with the time it is issued and the items it sets: description, functional impact, recovery plan, likely affected agencies, observed activity, indicators of compromise, related CVEs, root cause, response and recovery activities, milestones. A report restates everything set before it. +- **Rating.** Until an `estimates` entry exists, the incident is rated N5 (`IEC-CSO-DPR`). An estimate (`IEC-CSO-EFI`) records the customer effect (minimal, narrow, disruptive, debilitating), whether one agency or more than one is likely affected, the reason and the time; the register refuses a rating that is not the one the rules give for that combination (N1 minimal, N2 narrow, N3 disruptive on one agency, N4 debilitating on one or disruptive on several, N5 debilitating on several). +- **Clocks.** Read from FedRAMP's Class C rules: the Initial report is due from the evaluation, each Ongoing report from the previous report until the incident is resolved, and the Final report from `recovery_complete_at` (hours at N3-N5; 24 hours or one business day at N1-N2; business days skip weekends, and holidays are counted, which can only make a deadline earlier). The rating in force when each clock starts sets its length. +- **Build and publish.** `scripts/build-incident-report.py` writes each report in FedRAMP's incident report schema (`incident---.json`, validated before it is written) and the incident log (`incident-reports.json` and `.md`: every incident, its evaluation and reason, its rating history, each report with its sha256 and what changed, and each clock). `.github/workflows/incident-reports.yml` runs it on every push of the register to `main`, by hand, twice a day and after each nightly VDR refresh; it signs, redaction-gates and publishes under `/.well-known/incident-*` with a role that can write nothing else. Every deploy republishes the same set. +- **Alarms.** The workflow's last step fails, and raises a GitHub issue, when a report is past its deadline. The evidence SLA watchdog reads the published log hourly and pages by email (`incident-report-overdue`, [`runbooks/evidence-sla-alarm.md`](runbooks/evidence-sla-alarm.md)). +- **Vulnerabilities as incidents.** The operator adopts `VER-TFR-IRI` and `VER-TFR-NRI` (`policy.adopted_vulnerability_rules`). The VDR triage policy names each finding they cover; until an unresolved incident in the register lists it under `findings`, its Initial report is due from the finding's first evaluation at N5, so the alarms fire. Open the incident with basis `VER-TFR-IRI` or `VER-TFR-NRI` and the finding's tracking ID or CVE. Reconciliation invariant (m) fails a build whose open incident names a finding the VDR no longer carries: close it with a Final report. +- **Notification is publish-only.** Nothing is sent to FedRAMP or to any agency; the operator has not authorized sending, and the system has no certification and no agency customers. Every report says so. FedRAMP and the affected agencies are "all affected parties" under the rules; reaching them is a decision for the operator. +- **Exercises.** A simulated incident is recorded with `exercise: true` and is published marked as one. Exercised 2026-11-05: `INC-2026-001` (a simulated internet-reachable N5 vulnerability: Initial, two Ongoing and a Final report on a simulated timeline, all on time; rated N5 by default, then estimated N3) and `INC-2026-002` (a simulated page change, evaluated not reportable with its reason). `tests/test_incidents.py` runs the same exercise through the builder on every `make check`. ## After-action reports (KSI-INR-AAR) @@ -117,5 +134,5 @@ Single-tier. Author → author. External escalations: - **GitHub Support** — if the issue is GitHub-side (account, repo, workflow infrastructure) - The mailbox in `security.txt` for acknowledging external reporters -There is no actual FedRAMP authorization in scope, so the FedRAMP Security Inbox (FSI) and Incident Communications Procedures (ICP) do not apply. +There is no actual FedRAMP certification in scope, so the FedRAMP Security Inbox (FSI) does not apply. Incident reports under the FedRAMP Incident Evaluation and Communication rules are generated and published as described above; they are not sent. diff --git a/docs/policies/ir-policy.md b/docs/policies/ir-policy.md index 81be6cb0..03c4e0db 100644 --- a/docs/policies/ir-policy.md +++ b/docs/policies/ir-policy.md @@ -1,15 +1,24 @@ # IR — Incident Response -Incident response procedures are documented in [`incident-response.md`](../incident-response.md), which satisfies KSI-INR-RIR through KSI-INR-AAR and the IR-9 family (information spillage). The runbook covers detection sources, triage, containment, eradication, recovery, and after-action reporting. The IR lead is the operator; escalation is single-tier. +Incident response procedures are documented in [`incident-response.md`](../incident-response.md), which satisfies KSI-INR-RIR through KSI-INR-AAR and the IR-9 family (information spillage). The runbook covers detection sources, triage (including the FedRAMP reportability evaluation), containment, eradication, recovery, FedRAMP incident reporting, and after-action reporting. The IR lead is the operator; escalation is single-tier. External communication paths: - **Public reporter:** [`/.well-known/security.txt`](../../website/.well-known/security.txt) is the published vulnerability-reporting mailbox. -- **Incident Communications Procedures (ICP, 20x):** the ICP rules apply at the procedure-documented level even though the system has no actual FedRAMP-customer reporting obligation. The runbook describes what would be communicated and to whom for a real FedRAMP-certified service. +- **FedRAMP incident reports (IEC):** every incident is recorded in the incident register, [`data/incidents.json`](../../data/incidents.json), with its reportability evaluation. Initial, Ongoing and Final Incident Reports for a FedRAMP Reportable Incident are generated from it, validated against FedRAMP's incident report schema, signed, and published under `/.well-known/` (the incident log is `/.well-known/incident-reports.json`) by an unattended workflow on every change to the register. +- **Notification is publish-only.** Nothing is sent to FedRAMP or to any agency. The operator has not authorized sending, and the system has no FedRAMP certification and no agency customers. Choosing a delivery channel is an operator decision; until it is made, publication is the only channel and every report says so. - **FedRAMP Security Inbox (FSI, 20x):** the system is not FedRAMP-certified; no actual FSI obligation applies. Notification path is documented in the runbook for reference. -**20x rule integration.** KSI-INR (Incident Response), ICP (Incident Communications Procedures), FSI (FedRAMP Security Inbox). +**20x rule integration.** KSI-INR (Incident Response), IEC (Incident Evaluation and Communication), FSI (FedRAMP Security Inbox). -**Review cadence.** Annually with the [security review](../security-review.md); after every actual incident (currently zero). VDR thresholds (Class C) treat internet-reachable likely-exploitable vulnerabilities above PAIN N3 (i.e. N4 or N5) as FedRAMP Reportable Incidents per `VER-TFR-IRI`, until they are partially mitigated to N3 or below; non-internet-reachable likely-exploitable vulnerabilities at PAIN N5 are reportable until N4 or below per `VER-TFR-NRI`. +**Reportability (IEC-CSO-EFR).** Every incident is evaluated, promptly and before containment, for whether it affects or is likely to affect the confidentiality or integrity of **federal customer data**; such an incident is a FedRAMP Reportable Incident. The determination, its reason and the federal customer data considered are recorded in the register and published in the incident log. This system holds no federal customer data, so in practice the evaluation is "not reportable"; the machinery runs either way, and it was exercised on 2026-11-05 with one simulated reportable incident and one simulated non-reportable one, both marked as exercises. -**Reportability trigger (IEC).** A FedRAMP Reportable Incident under `IEC-CSO-EFR` is triggered by an effect on the confidentiality or integrity of **federal customer data**. This system holds none, so no finding here is FedRAMP-reportable in practice and the `IEC-CSO-IIR` reporting clocks (Class C: 1 hour for PAIN 3/4/5) never start. The evaluation step still runs — the VDR aggregator assigns the PAIN rating to every finding — so the machinery is in place the moment federal data ever enters scope. +**Rating (IEC-CSO-DPR, IEC-CSO-EFI).** A reportable incident is rated at the default the rules set (N5) until the operator records an Incident Rating estimate: a likely minimal, narrow, disruptive or debilitating customer effect on one or more than one agency, with a reason. The register refuses an estimate whose rating is not the one the rules give for that effect. + +**Report clocks.** The Initial, Ongoing and Final report deadlines are FedRAMP's Class C timeframes, read from the pinned rules and never restated. An overdue report fails the incident reports workflow (raising a GitHub issue) and pages the operator through the evidence SLA watchdog. + +**Vulnerabilities as incidents (VER-TFR-IRI, VER-TFR-NRI).** The operator adopts both rules. An internet-reachable likely exploitable vulnerability above PAIN N3 (i.e. N4 or N5) is a FedRAMP Reportable Incident until it is partially mitigated to N3 or below (`VER-TFR-IRI`), and a non-internet-reachable likely exploitable vulnerability at PAIN N5 is one until it is N4 or below (`VER-TFR-NRI`, a MAY the operator has chosen to use). The VDR triage policy names each such finding; the incident log shows it, and until an incident in the register names it, its Initial report is due and the alarms above apply. + +**Review cadence.** Annually with the [security review](../security-review.md); after every actual incident (currently zero; the two recorded incidents are exercises). + +_AI-drafted (2026-11-05) to describe what the incident register, report builder and workflow now do; for the operator's review._ diff --git a/docs/runbooks/evidence-sla-alarm.md b/docs/runbooks/evidence-sla-alarm.md index 4057bb84..79f3f7c5 100644 --- a/docs/runbooks/evidence-sla-alarm.md +++ b/docs/runbooks/evidence-sla-alarm.md @@ -7,11 +7,12 @@ The evidence SLA watchdog (`infrastructure/watchdog.tf`) emails the operator thr | `nightly-failed` | the evidence nightly's last run failed, or no run has reported in 26h | a gate failed; or the cron is disabled or stuck | | `vdr-age` | the published VDR is more than 24h old | the reporting policy is breached; nightly runs have failed or stopped | | `runtime-age` | the runtime signal is more than 26h old | the daily runtime-check Lambda has failed or stopped | +| `incident-report-overdue` | an incident report listed as due in `/.well-known/incident-reports.json` is past its Class C deadline, or the incident log is missing or unreadable | an Initial, Ongoing or Final incident report was not issued in time; or a vulnerability that is a FedRAMP Reportable Incident (VER-TFR-IRI/NRI) has no incident opened | | `artifact-stale` | any artifact listed in `infrastructure/schemas/freshness-config.json` is past its window, missing, or has no readable timestamp | no deploy within the deploy-cadence window; the corroboration was not re-run; or one of the above | The windows are governed values in `infrastructure/schemas/freshness-config.json`; the thresholds above are read from it. Change a window there, through review, never in the alarm. -Any alarm also fires if the watchdog stops publishing metrics, because missing data counts as a breach. If all four fire together, check the watchdog itself first (a missing or invalid freshness config stops it before it publishes anything). +Any alarm also fires if the watchdog stops publishing metrics, because missing data counts as a breach. If all five fire together, check the watchdog itself first (a missing or invalid freshness config stops it before it publishes anything). ## `nightly-failed` @@ -30,6 +31,10 @@ The policy is already breached, so treat this as the priority. It usually follow Check the compliance Lambda (`samaydlette-com-opa-compliance`): its CloudWatch logs, its EventBridge rule, and the shared dead-letter queue `samaydlette-com-opa-compliance-dlq` for failed invocations. +## `incident-report-overdue` + +The published incident log (`/.well-known/incident-reports.json`, `obligations_due`) names each report still due, for which incident or finding, and its deadline. Follow [`docs/incident-response.md`](../incident-response.md#reporting-fedramp-reportable-incidents-iec): record the report (or, for a finding with no incident, open the incident) in `data/incidents.json` and push it; the **Incident Reports** workflow validates, builds, signs and publishes within minutes, and the alarm clears once the republished log no longer lists the report as due. If the log itself is missing or unreadable, run the workflow by hand and check its publish step. + ## `artifact-stale` The watchdog's log line for the run lists the stale artifacts under `stale`. To see the same from outside, run `python3 scripts/check-evidence-freshness.py --all https://samaydlette.com`. @@ -38,7 +43,7 @@ The watchdog's log line for the run lists the stale artifacts under `stale`. To - **`boundary-corroboration.json`:** re-run `tools/boundary-map/corroborate.py`, commit the result, and deploy. - **Nightly artifacts** (`vdr-*`): follow `nightly-failed` / `vdr-age` above. -## All four at once +## All five at once Check the watchdog: its log group `/aws/lambda/samaydlette-com-evidence-watchdog`, the dead-letter queue, and its EventBridge rule. Each run logs one JSON line with the values it published, the config version, and the stale artifacts. diff --git a/infrastructure/bootstrap/incident-reports.tf b/infrastructure/bootstrap/incident-reports.tf new file mode 100644 index 00000000..d38fb120 --- /dev/null +++ b/infrastructure/bootstrap/incident-reports.tf @@ -0,0 +1,97 @@ +# ============================================================================= +# BOOTSTRAP: GitHub OIDC role for unattended incident reporting (IEC-CSO-AIR) +# ============================================================================= +# Operator-applied (not CI), like everything else in this stack. +# +# WHY A ROLE OF ITS OWN. FedRAMP's Class C incident report clocks are hours (an +# Initial Incident Report within one hour for N3-N5, an Ongoing report every +# six). The deploy role sits behind the `prod` reviewer and cannot keep them, +# and the evidence nightly's role is scoped to `.well-known/vdr-*` on purpose: +# widening it to carry incident reports would make the VDR refresh's grant +# about something else. `.github/workflows/incident-reports.yml` runs in its +# own GitHub Environment with no required reviewer and assumes this role. +# +# THE WHOLE PERMISSION SET: PutObject under `.well-known/incident-*` and a CDN +# invalidation. No delete (an unattended job never unpublishes evidence), no +# read of any kind (the workflow verifies its publish through the public URL), +# no Terraform state, no website sync. The bucket default-encrypts with SSE-S3, +# so no KMS grant is involved. If a future change needs more, re-examine +# whether that work belongs on the unattended path before widening this. +# ============================================================================= + +variable "incident_reports_environment" { + type = string + description = "GitHub Environment the incident reports workflow runs in. No required reviewer; the role it grants is scoped to .well-known/incident-*." + default = "incident-reports" +} + +# TRUST: this repository, this GitHub Environment, nothing else. A pull request +# cannot assume it, and neither can a workflow that does not declare the +# environment. +data "aws_iam_policy_document" "incident_reports_trust" { + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.github.arn] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${var.github_repo}:environment:${var.incident_reports_environment}"] + } + } +} + +resource "aws_iam_role" "incident_reports" { + name = "github-actions-incident-reports-oidc" + description = "Unattended incident reporting (IEC-CSO-AIR). PutObject under .well-known/incident-* and a CDN invalidation; nothing else." + assume_role_policy = data.aws_iam_policy_document.incident_reports_trust.json + max_session_duration = 3600 + + tags = merge(local.bootstrap_tags, local.bootstrap_cls.identity_secrets_internal, { Name = "github-actions-incident-reports-oidc" }) +} + +data "aws_iam_policy_document" "incident_reports_publish" { + statement { + sid = "PublishIncidentReportsOnly" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["arn:aws:s3:::${var.domain_name}/.well-known/incident-*"] + } +} + +resource "aws_iam_role_policy" "incident_reports_publish" { + name = "incident-reports-publish" + role = aws_iam_role.incident_reports.id + policy = data.aws_iam_policy_document.incident_reports_publish.json +} + +data "aws_iam_policy_document" "incident_reports_cdn" { + statement { + sid = "InvalidateIncidentPaths" + effect = "Allow" + actions = ["cloudfront:CreateInvalidation"] + resources = [aws_cloudfront_distribution.website.arn] + } +} + +resource "aws_iam_role_policy" "incident_reports_cdn" { + name = "incident-reports-cdn" + role = aws_iam_role.incident_reports.id + policy = data.aws_iam_policy_document.incident_reports_cdn.json +} + +output "incident_reports_role_arn" { + value = aws_iam_role.incident_reports.arn + description = "Set as the repository variable AWS_INCIDENT_REPORTS_ROLE_ARN, which incident-reports.yml passes to configure-aws-credentials." +} diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index cd4b2f02..c81d4307 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -86,6 +86,18 @@ "signed": false, "note": "The nightly's status beacon. Unsigned and not evidence; listed because the watchdog pages on its age. The nightly runs twice a day, so 26 hours means runs have been missed." }, + { + "path": ".well-known/incident-reports.json", + "cadence": "nightly", + "max_age_hours": 24, + "timestamp": { + "source": "json", + "field": "emitted_at" + }, + "signed": true, + "signature": ".well-known/incident-reports.bundle", + "note": "The incident log (IEC-CSO-EFR) and, through it, every incident report it lists with its sha256. Rebuilt by .github/workflows/incident-reports.yml on every change to data/incidents.json and twice a day, and by every deploy. The report deadlines themselves are hours, so the watchdog also pages on each listed report past its deadline (IncidentReportsOverdue)." + }, { "path": ".well-known/ksi-signal-runtime.json", "cadence": "runtime", diff --git a/infrastructure/schemas/gate-modes.json b/infrastructure/schemas/gate-modes.json index d1e87bc2..cb7d7bc6 100644 --- a/infrastructure/schemas/gate-modes.json +++ b/infrastructure/schemas/gate-modes.json @@ -50,6 +50,9 @@ "decided_by": "Sam Aydlette", "decided_on": "2026-11-04", "review_by": "2027-11-04"}, {"check": "reconcile.l", "gate": "G6", "mode": "block", "method_default": "block", "decided_by": "Sam Aydlette", "decided_on": "2026-11-04", "review_by": "2027-11-04"}, + {"check": "reconcile.m", "gate": "G5", "mode": "block", "method_default": "block", + "reason": "Proposed with the incident register (W6, AI-drafted): an open incident naming a finding the VDR does not carry blocks, like every other cross-artifact invariant. Confirm at review.", + "decided_by": "Operator (proposed; confirm at review)", "decided_on": "2026-11-05", "review_by": "2027-11-05"}, { "check": "reconcile.i.unreadable", "gate": "G7", diff --git a/infrastructure/schemas/incidents.schema.json b/infrastructure/schemas/incidents.schema.json new file mode 100644 index 00000000..d933d0d4 --- /dev/null +++ b/infrastructure/schemas/incidents.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "incidents.schema.json", + "title": "Incident register", + "description": "data/incidents.json: every incident, its FedRAMP reportability evaluation (IEC-CSO-EFR), its Incident Rating estimates (IEC-CSO-EFI; until one exists the IEC-CSO-DPR default applies) and the Initial, Ongoing and Final reports issued (IEC-CSO-IIR/OIR/FIR). scripts/_incidents.py enforces these rules at build time without a jsonschema dependency, plus the ones a schema cannot state: IDs are unique; times are in order (started, detected, evaluated, each report); an estimate's rating is the IEC-CSO-EFI rating for its customer effect and agency count; the first report is Initial and sets the description, and a Final report is last and follows recovery; a vulnerability basis is an adopted rule and names a finding; nothing is dated after now; and no account ID or outside email address appears.", + "type": "object", + "required": ["schema", "policy", "incidents"], + "properties": { + "$comment": {"type": "string"}, + "schema": {"const": "incidents/1"}, + "policy": {"$ref": "#/$defs/policy"}, + "incidents": {"type": "array", "items": {"$ref": "#/$defs/incident"}} + }, + "additionalProperties": false, + "$defs": { + "text": {"type": "string", "minLength": 1, "pattern": "\\S"}, + "date": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$"}, + "time": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+-]\\d{2}:\\d{2})$"}, + "texts": {"type": "array", "items": {"$ref": "#/$defs/text"}, "uniqueItems": true}, + "policy": { + "type": "object", + "required": ["federal_incident_coordinator", "notification", "notification_text", "adopted_vulnerability_rules", + "federal_customer_data", "decided_by", "decided_on", "review_by"], + "properties": { + "federal_incident_coordinator": {"$ref": "#/$defs/text", "description": "Who coordinates federal incident response and how to reach them (IEC-CSO-IIR item 1)."}, + "notification": {"enum": ["published-only"], "description": "How reports reach all affected parties. Only publication under /.well-known/ is built; sending is the operator's decision."}, + "notification_text": {"$ref": "#/$defs/text"}, + "adopted_vulnerability_rules": {"type": "array", "items": {"enum": ["VER-TFR-IRI", "VER-TFR-NRI"]}, "uniqueItems": true, "description": "Which of VER-TFR-IRI (SHOULD) and VER-TFR-NRI (MAY) the operator applies."}, + "federal_customer_data": {"$ref": "#/$defs/text", "description": "What would be federal customer data (FRD-FCD) for this offering."}, + "decided_by": {"$ref": "#/$defs/text"}, + "decided_on": {"$ref": "#/$defs/date"}, + "review_by": {"$ref": "#/$defs/date"} + }, + "additionalProperties": false + }, + "incident": { + "type": "object", + "required": ["id", "title", "exercise", "started_at", "detected_at", "detection_source", "evaluation", "estimates", "reports"], + "properties": { + "id": {"type": "string", "pattern": "^INC-\\d{4}-\\d{3}$"}, + "title": {"$ref": "#/$defs/text"}, + "exercise": {"type": "boolean", "description": "true for a simulated incident run to exercise the process."}, + "started_at": {"$ref": "#/$defs/time"}, + "detected_at": {"$ref": "#/$defs/time"}, + "detection_source": {"$ref": "#/$defs/text"}, + "evaluation": { + "type": "object", + "required": ["evaluated_at", "reportable", "basis", "reason", "federal_customer_data_considered"], + "properties": { + "evaluated_at": {"$ref": "#/$defs/time"}, + "reportable": {"type": "boolean"}, + "basis": {"enum": ["IEC-CSO-EFR", "VER-TFR-IRI", "VER-TFR-NRI"]}, + "reason": {"$ref": "#/$defs/text"}, + "federal_customer_data_considered": {"$ref": "#/$defs/text"} + }, + "additionalProperties": false + }, + "findings": {"$ref": "#/$defs/texts", "description": "VDR tracking IDs or CVE IDs the incident is about."}, + "estimates": {"type": "array", "items": { + "type": "object", + "required": ["estimated_at", "rating", "customer_effect", "agencies", "reason"], + "properties": { + "estimated_at": {"$ref": "#/$defs/time"}, + "rating": {"type": "integer", "minimum": 1, "maximum": 5}, + "customer_effect": {"enum": ["minimal", "narrow", "disruptive", "debilitating"]}, + "agencies": {"enum": ["single", "multiple"]}, + "reason": {"$ref": "#/$defs/text"} + }, + "additionalProperties": false + }}, + "reports": {"type": "array", "items": { + "type": "object", + "required": ["type", "at"], + "properties": { + "type": {"enum": ["Initial", "Ongoing", "Final"]}, + "at": {"$ref": "#/$defs/time"}, + "set": { + "type": "object", + "minProperties": 1, + "properties": { + "description": {"$ref": "#/$defs/text"}, + "functional_impact": {"$ref": "#/$defs/text"}, + "recovery_plan": {"$ref": "#/$defs/text"}, + "observed_activity": {"$ref": "#/$defs/text"}, + "root_cause": {"$ref": "#/$defs/text"}, + "response_and_recovery_activities": {"$ref": "#/$defs/text"}, + "affected_agencies": {"$ref": "#/$defs/texts"}, + "indicators_of_compromise": {"$ref": "#/$defs/texts"}, + "related_cves": {"type": "array", "items": {"type": "string", "pattern": "^CVE-[0-9]{4}-[0-9]{4,}$"}, "uniqueItems": true}, + "milestones": {"type": "array", "items": { + "type": "object", "required": ["at", "description"], + "properties": {"at": {"$ref": "#/$defs/time"}, "description": {"$ref": "#/$defs/text"}}, + "additionalProperties": false + }} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }}, + "resolved_at": {"$ref": "#/$defs/time"}, + "recovery_complete_at": {"$ref": "#/$defs/time"} + }, + "additionalProperties": false + } + } +} diff --git a/infrastructure/watchdog.tf b/infrastructure/watchdog.tf index 16f1777f..19764d91 100644 --- a/infrastructure/watchdog.tf +++ b/infrastructure/watchdog.tf @@ -14,7 +14,7 @@ # nightly's status beacon and every other signed output under # /.well-known/) straight from the site bucket # -> CloudWatch metrics (Samaydlette/Evidence) -# -> four alarms -> SNS topic (CMK-encrypted) -> operator email +# -> five alarms -> SNS topic (CMK-encrypted) -> operator email # # The freshness windows are governed values (REQ-PUB-2, REQ-GOV-1). They are # read here with jsondecode(file(...)): the alarm thresholds, the objects the @@ -30,7 +30,8 @@ # Terraform lands in state and in the uploaded plan artifact of a public repo. # It is created once, out of band (docs/runbooks/evidence-sla-alarm.md). # -# Cost: four standard alarms plus four custom metrics, roughly $1.60/month. +# Cost: five standard alarms plus five custom metrics, roughly $2.00/month (the +# fifth pair, for overdue incident reports, adds about $0.40). # Every listed artifact feeds one StaleArtifacts count rather than a metric # each, so the cost stays flat as the list grows. # SNS email at this volume and 720 short invocations a month are within free @@ -78,6 +79,14 @@ locals { threshold = 0 summary = "At least one published /.well-known/ artifact is older than its governed freshness window, or unreadable. The watchdog's log names which." } + # The Class C incident report clocks (IEC-CSO-IIR/OIR/FIR) are hours, not + # days, so this reads the deadlines the incident log lists rather than the + # log's age, and pages once one has passed without its report. + "incident-report-overdue" = { + metric = "IncidentReportsOverdue" + threshold = 0 + summary = "An incident report is past its FedRAMP Class C deadline and has not been issued, or the incident log is unreadable. /.well-known/incident-reports.json lists what is due; issue it by adding the report to data/incidents.json." + } } } diff --git a/infrastructure/watchdog/index.mjs b/infrastructure/watchdog/index.mjs index dd1b50af..e744b5b3 100644 --- a/infrastructure/watchdog/index.mjs +++ b/infrastructure/watchdog/index.mjs @@ -25,6 +25,9 @@ // missing, or is too old to trust; else 0 // StaleArtifacts how many listed artifacts are past their governed // window, unreadable, or without a usable timestamp +// IncidentReportsOverdue how many incident reports the published incident +// log lists as due whose Class C deadline has passed +// (1 if the log is missing or unreadable) // // A missing or invalid config fails the run before anything is published; the // alarms read the missing data as a breach and page. diff --git a/infrastructure/watchdog/logic.mjs b/infrastructure/watchdog/logic.mjs index 97692b70..8e896f03 100644 --- a/infrastructure/watchdog/logic.mjs +++ b/infrastructure/watchdog/logic.mjs @@ -13,10 +13,11 @@ export const KEYS = { vdr: '.well-known/vdr-report.json', runtime: '.well-known/ksi-signal-runtime.json', status: '.well-known/vdr-status.json', + incidents: '.well-known/incident-reports.json', }; // Every artifact a metric below depends on. The config must list them. -export const REQUIRED = [KEYS.vdr, KEYS.runtime, KEYS.status, '.well-known/boundary-corroboration.json']; +export const REQUIRED = [KEYS.vdr, KEYS.runtime, KEYS.status, KEYS.incidents, '.well-known/boundary-corroboration.json']; // Reported when an artifact cannot be read or has no usable timestamp. Far past // any threshold, so it reads as a breach on every alarm. @@ -98,6 +99,22 @@ export function nightlyFailed(status, maxBeaconAgeHours, now = Date.now()) { return status.result === 'success' ? 0 : 1; } +// How many incident reports are past their Class C deadline and not issued +// (IEC-CSO-IIR/OIR/FIR). The incident log lists each report still due with its +// deadline (scripts/build-incident-report.py, obligations_due); this counts the +// ones whose time has passed on AWS's clock, so a missed one-hour Initial +// report pages within the hour even if nothing republishes the log. A log that +// is missing, unreadable or has no list is a breach (1), never zero. +export function incidentReportsOverdue(log, now = Date.now()) { + if (!log || typeof log !== 'object' || !Array.isArray(log.obligations_due)) return 1; + let overdue = 0; + for (const d of log.obligations_due) { + const due = Date.parse(d && d.due_at); + if (Number.isNaN(due) || due < now) overdue += 1; + } + return overdue; +} + // Every listed artifact past its window, or unreadable, or without a usable // timestamp. One metric for all of them keeps the cost flat as the list grows; // the log line names which. @@ -123,12 +140,13 @@ export function computeMetrics(config, readings, now = Date.now()) { RuntimeSignalAgeHours: age(KEYS.runtime), NightlyFailed: nightlyFailed(parsedBody(readings[KEYS.status]), config.windows[KEYS.status], now), StaleArtifacts: stale.length, + IncidentReportsOverdue: incidentReportsOverdue(parsedBody(readings[KEYS.incidents]), now), }, stale, }; } -const COUNTS = new Set(['NightlyFailed', 'StaleArtifacts']); +const COUNTS = new Set(['NightlyFailed', 'StaleArtifacts', 'IncidentReportsOverdue']); export function metricData(metrics, now) { return Object.entries(metrics).map(([MetricName, Value]) => ({ diff --git a/policies/triage.rego b/policies/triage.rego index 8058d105..071839f0 100644 --- a/policies/triage.rego +++ b/policies/triage.rego @@ -444,3 +444,101 @@ decide(item) := { } decisions := [decide(item) | some item in input.items] + +# ============================================================================= +# VULNERABILITIES TREATED AS FEDRAMP REPORTABLE INCIDENTS (VER-TFR-IRI, -NRI) +# ============================================================================= +# Entry points, over the findings of a built VDR report (input.findings): +# data.vdr.triage.incident_triggers one entry per finding a rule makes a +# FedRAMP Reportable Incident +# data.vdr.triage.incident_errors non-empty => the caller fails closed +# +# The N-rating comparisons are read from the Class C rule text in data.rules, +# not restated here: "N-rating > 3 ... until ... at N3 or below" (IRI) and +# "N-rating = 5 ... until ... at N4 or below" (NRI). A rule stays in force for a +# finding until its current rating (after partial mitigation) is at or below +# the "until" level, which for both rules is the trigger level's complement. +# A false positive is not a vulnerability and a fully mitigated finding is not +# exploitable, so neither triggers. Whether the operator has adopted each rule +# (IRI is a SHOULD, NRI a MAY at Class C) is the incident register's policy, +# applied by scripts/build-incident-report.py. +ver_tfr := data.rules.FRR.VER.data.all.TFR + +class_c_statement(rid) := s if { + s := ver_tfr[rid].varies_by_class.c.statement +} else := ver_tfr[rid].statement + +rule_digit(pattern, rid) := to_number(m[0][1]) if { + m := regex.find_all_string_submatch_n(pattern, class_c_statement(rid), 1) + count(m) == 1 +} + +iri_above := rule_digit(`N-rating > ([0-9])`, "VER-TFR-IRI") + +iri_until := rule_digit(`at N([0-9]) or below`, "VER-TFR-IRI") + +nri_equals := rule_digit(`N-rating = ([0-9])`, "VER-TFR-NRI") + +nri_until := rule_digit(`at N([0-9]) or below`, "VER-TFR-NRI") + +incident_errors contains "VER-TFR-IRI Class C statement no longer reads 'N-rating > n ... at Nn or below'" if { + not iri_above +} + +incident_errors contains "VER-TFR-IRI Class C statement no longer reads 'N-rating > n ... at Nn or below'" if { + not iri_until +} + +incident_errors contains "VER-TFR-NRI Class C statement no longer reads 'N-rating = n ... at Nn or below'" if { + not nri_equals +} + +incident_errors contains "VER-TFR-NRI Class C statement no longer reads 'N-rating = n ... at Nn or below'" if { + not nri_until +} + +# The trigger and the exit must be complements; anything else is a changed rule +# that needs reading before it is trusted. +incident_errors contains "VER-TFR-IRI: the trigger and the 'until' level are not complements" if { + iri_above != iri_until +} + +incident_errors contains "VER-TFR-NRI: the trigger and the 'until' level are not complements" if { + nri_until != nri_equals - 1 +} + +incident_errors contains msg if { + some f in input.findings + not regex.match(`^N[0-9]$`, sprintf("%v", [f.pain])) + msg := sprintf("%v: impact %v is not an N-rating", [f.tracking_id, f.pain]) +} + +pain_number(p) := to_number(substring(p, 1, -1)) + +still_a_vulnerability(f) if { + f.fully_mitigated != true + f.final_disposition != "false-positive" +} + +incident_rule(f) := "VER-TFR-IRI" if { + f.internet_reachable == true + f.likely_exploitable == true + pain_number(f.pain) > iri_above +} else := "VER-TFR-NRI" if { + f.internet_reachable == false + f.likely_exploitable == true + pain_number(f.pain) == nri_equals +} + +incident_triggers := [t | + some f in input.findings + still_a_vulnerability(f) + rule := incident_rule(f) + t := { + "tracking_id": f.tracking_id, + "cve": object.get(f, "cve", null), + "rule": rule, + "pain": f.pain, + "first_evaluated": object.get(f, "first_evaluated", null), + } +] diff --git a/policies/triage_test.rego b/policies/triage_test.rego index c8af8841..4f6aadea 100644 --- a/policies/triage_test.rego +++ b/policies/triage_test.rego @@ -254,3 +254,105 @@ test_observed_reachability_beats_not_reachable_mitigation if { }) d.irv_basis == "observed" } + +# ---- VER-TFR-IRI / VER-TFR-NRI: vulnerabilities treated as incidents ------------------ +iri_text := concat(" ", [ + "Providers with Class C Certifications SHOULD treat internet-reachable likely exploitable vulnerabilities", + "where Potential Agency Impact N-rating > 3 as a FedRAMP Reportable Incident", + "until they are partially mitigated vulnerabilities at N3 or below.", +]) + +nri_text := concat(" ", [ + "Providers with Class C Certifications MAY treat likely exploitable vulnerabilities that are NOT internet-reachable", + "where Potential Agency Impact N-rating = 5 as a FedRAMP Reportable Incident", + "until they are partially mitigated vulnerabilities at N4 or below.", +]) + +ver_rules(iri, nri) := {"FRR": {"VER": {"data": {"all": {"TFR": { + "VER-TFR-IRI": {"varies_by_class": {"c": {"statement": iri}}}, + "VER-TFR-NRI": nri, +}}}}}} + +good_ver := ver_rules(iri_text, {"varies_by_class": {"c": {"statement": nri_text}}}) + +vf := { + "tracking_id": "t1", "cve": "CVE-2099-0001", "pain": "N4", "internet_reachable": true, + "likely_exploitable": true, "fully_mitigated": false, "final_disposition": "open", + "first_evaluated": "2026-11-01T00:00:00+00:00", +} + +triggers(patch, r) := ts if { + ts := triage.incident_triggers with data.rules as r + with input as {"findings": [object.union(vf, patch)]} +} + +ierrs(findings, r) := es if { + es := triage.incident_errors with data.rules as r with input as {"findings": findings} +} + +test_iri_opens_above_n3 if { + ts := triggers({}, good_ver) + ts == [{ + "tracking_id": "t1", "cve": "CVE-2099-0001", "rule": "VER-TFR-IRI", "pain": "N4", + "first_evaluated": "2026-11-01T00:00:00+00:00", + }] + triggers({"pain": "N5"}, good_ver)[0].rule == "VER-TFR-IRI" +} + +test_iri_ends_at_n3_or_below if { + triggers({"pain": "N3"}, good_ver) == [] + triggers({"pain": "N1"}, good_ver) == [] +} + +test_nri_opens_only_at_n5 if { + triggers({"internet_reachable": false, "pain": "N5"}, good_ver)[0].rule == "VER-TFR-NRI" + triggers({"internet_reachable": false, "pain": "N4"}, good_ver) == [] +} + +test_not_likely_exploitable_never_triggers if { + triggers({"likely_exploitable": false, "pain": "N5"}, good_ver) == [] + triggers({"likely_exploitable": false, "internet_reachable": false, "pain": "N5"}, good_ver) == [] +} + +test_false_positive_and_fully_mitigated_never_trigger if { + triggers({"final_disposition": "false-positive", "pain": "N5"}, good_ver) == [] + triggers({"fully_mitigated": true, "pain": "N5"}, good_ver) == [] +} + +test_accepted_still_triggers if { + triggers({"final_disposition": "accepted", "pain": "N5"}, good_ver)[0].rule == "VER-TFR-IRI" +} + +test_missing_cve_is_null if { + ts := triage.incident_triggers with data.rules as good_ver + with input as {"findings": [object.remove(vf, ["cve", "first_evaluated"])]} + ts[0].cve == null + ts[0].first_evaluated == null +} + +test_rules_read_cleanly if { + ierrs([vf], good_ver) == set() +} + +test_changed_iri_text_fails_closed if { + es := ierrs([vf], ver_rules("treat them as incidents", {"statement": nri_text})) + "VER-TFR-IRI Class C statement no longer reads 'N-rating > n ... at Nn or below'" in es +} + +test_changed_nri_text_fails_closed if { + es := ierrs([vf], ver_rules(iri_text, {"statement": "treat them as incidents"})) + "VER-TFR-NRI Class C statement no longer reads 'N-rating = n ... at Nn or below'" in es +} + +test_non_complement_fails_closed if { + bad_iri := replace(iri_text, "at N3 or below", "at N2 or below") + bad_nri := replace(nri_text, "at N4 or below", "at N3 or below") + es := ierrs([vf], ver_rules(bad_iri, {"statement": bad_nri})) + "VER-TFR-IRI: the trigger and the 'until' level are not complements" in es + "VER-TFR-NRI: the trigger and the 'until' level are not complements" in es +} + +test_bad_rating_fails_closed if { + es := ierrs([object.union(vf, {"pain": "high"})], good_ver) + "t1: impact high is not an N-rating" in es +} diff --git a/scripts/_freshness.py b/scripts/_freshness.py index 8272426c..06024ef4 100644 --- a/scripts/_freshness.py +++ b/scripts/_freshness.py @@ -33,6 +33,7 @@ ".well-known/vdr-report.json", ".well-known/ksi-signal-runtime.json", ".well-known/vdr-status.json", + ".well-known/incident-reports.json", ".well-known/boundary-corroboration.json", ) diff --git a/scripts/_gates.py b/scripts/_gates.py index d9c48274..0335c57a 100644 --- a/scripts/_gates.py +++ b/scripts/_gates.py @@ -62,6 +62,7 @@ "reconcile.e": "G6", "reconcile.f": "G6", "reconcile.l": "G6", + "reconcile.m": "G5", "reconcile.i.unreadable": "G7", "runtime-divergence.observer-fault": "G7", "detection.zap.missing": "G7", diff --git a/scripts/_incidents.py b/scripts/_incidents.py new file mode 100644 index 00000000..14fd68c2 --- /dev/null +++ b/scripts/_incidents.py @@ -0,0 +1,412 @@ +"""The incident register: data/incidents.json (IEC-CSO-EFR/DPR/EFI/IIR/OIR/FIR). + +Every incident the operator records lives here, once, with the evaluation the +FedRAMP Incident Evaluation and Communication rules ask for: + +- evaluation when it was evaluated, whether it is a FedRAMP Reportable + Incident (IEC-CSO-EFR) and why; the basis is IEC-CSO-EFR, or + VER-TFR-IRI / VER-TFR-NRI for a vulnerability treated as one +- estimates Incident Rating records (IEC-CSO-EFI): rating, customer effect, + one or several agencies, reason, when. Until the first one, a + reportable incident is rated at the default (IEC-CSO-DPR) +- reports the Initial, Ongoing and Final reports, each with when it was + issued and what it set; a report restates everything set before +- resolved_at / recovery_complete_at, which start the Final report's clock + +The clocks (when each report is due) come from FedRAMP's Class C timeframes +through scripts/_rules.py, never from this file. scripts/build-incident-report.py +renders the reports and the incident log from it; reconciliation checks the +findings an incident names against the VDR. + +Fails closed, stdlib only (the deploy job installs no jsonschema), following +scripts/_decisions.py: a missing or invalid register stops the build. The JSON +schema beside it (infrastructure/schemas/incidents.schema.json) documents the +same shape and is checked in tests. +""" + +import json +import re +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _redact # noqa: E402 +import _rules # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +DEFAULT_PATH = REPO / "data" / "incidents.json" +SCHEMA_PATH = REPO / "infrastructure" / "schemas" / "incidents.schema.json" + +ID_RE = re.compile(r"^INC-\d{4}-\d{3}$") +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$") +CVE_RE = re.compile(r"^CVE-[0-9]{4}-[0-9]{4,}$") +EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)") +# Addresses a published report may name: the system's own domain and FedRAMP's. +EMAIL_DOMAINS = ("samaydlette.com", "fedramp.gov", "gsa.gov") + +BASES = ("IEC-CSO-EFR", "VER-TFR-IRI", "VER-TFR-NRI") +VULN_RULES = ("VER-TFR-IRI", "VER-TFR-NRI") +REPORT_TYPES = ("Initial", "Ongoing", "Final") +EFFECTS = ("minimal", "narrow", "disruptive", "debilitating") +AGENCY_SCOPES = ("single", "multiple") +# Delivery the pipeline performs. Only publication is built; sending to FedRAMP +# or an agency is the operator's decision and is not implemented. +NOTIFICATION_MODES = ("published-only",) + +TOP_KEYS = {"$comment", "schema", "policy", "incidents"} +POLICY_KEYS = {"federal_incident_coordinator", "notification", "notification_text", "adopted_vulnerability_rules", + "federal_customer_data", "decided_by", "decided_on", "review_by"} +INCIDENT_KEYS = {"id", "title", "exercise", "started_at", "detected_at", "detection_source", "evaluation", + "findings", "estimates", "reports", "resolved_at", "recovery_complete_at"} +EVALUATION_KEYS = {"evaluated_at", "reportable", "basis", "reason", "federal_customer_data_considered"} +ESTIMATE_KEYS = {"estimated_at", "rating", "customer_effect", "agencies", "reason"} +REPORT_KEYS = {"type", "at", "set"} +# What a report can set, by the field it fills in the FedRAMP incident report. +TEXT_FIELDS = { + "description": "incidentDescription", + "functional_impact": "functionalImpact", + "recovery_plan": "recoveryPlan", + "observed_activity": "observedActivity", + "root_cause": "rootCause", + "response_and_recovery_activities": "responseAndRecoveryActivities", +} +LIST_FIELDS = { + "affected_agencies": "affectedAgencies", + "indicators_of_compromise": "indicatorsOfCompromise", + "related_cves": "relatedCveIds", +} +SET_KEYS = set(TEXT_FIELDS) | set(LIST_FIELDS) | {"milestones"} +MILESTONE_KEYS = {"at", "description"} + + +def _fail(msg): + raise SystemExit(f"::error::incident register: {msg} (fail-closed)") + + +def parse_ts(value, where): + """An aware datetime from an ISO 8601 timestamp with an offset, or fail.""" + if not isinstance(value, str) or not TS_RE.match(value): + _fail(f"{where} must be an ISO 8601 timestamp with a UTC offset, got {value!r}") + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + _fail(f"{where} is not a valid timestamp: {value!r}") + + +def _text(value, where): + if not isinstance(value, str) or not value.strip(): + _fail(f"{where} must be a non-empty string") + return value + + +def _strings(value, where, pattern=None): + if not isinstance(value, list) or not all(isinstance(x, str) and x.strip() for x in value): + _fail(f"{where} must be a list of non-empty strings") + if pattern is not None: + for x in value: + if not pattern.match(x): + _fail(f"{where}: {x!r} does not match {pattern.pattern}") + if len(set(value)) != len(value): + _fail(f"{where} lists an entry twice") + return value + + +def _published_text_ok(doc, where): + """Everything in the register is published: no AWS account ID, and no email + address outside the system's domain and FedRAMP's.""" + text = json.dumps(doc) + if _redact.find(text): + _fail(f"{where} contains an AWS account ID; the register is published") + for m in EMAIL_RE.finditer(text): + if not m.group(1).lower().endswith(EMAIL_DOMAINS): + _fail(f"{where} names an email address outside {', '.join(EMAIL_DOMAINS)}; the register is published") + + +def _validate_policy(p, where): + if not isinstance(p, dict) or set(p) - POLICY_KEYS or POLICY_KEYS - set(p): + _fail(f"{where}.policy must have exactly {sorted(POLICY_KEYS)}") + for k in ("federal_incident_coordinator", "notification_text", "decided_by"): + _text(p[k], f"{where}.policy.{k}") + for k in ("decided_on", "review_by"): + if not isinstance(p[k], str) or not DATE_RE.match(p[k]): + _fail(f"{where}.policy.{k} must be an ISO date") + if p["notification"] not in NOTIFICATION_MODES: + _fail(f"{where}.policy.notification {p['notification']!r} is not one of {list(NOTIFICATION_MODES)}; " + f"no other delivery is built") + adopted = _strings(p["adopted_vulnerability_rules"], f"{where}.policy.adopted_vulnerability_rules") + if set(adopted) - set(VULN_RULES): + _fail(f"{where}.policy.adopted_vulnerability_rules may name only {list(VULN_RULES)}") + _text(p["federal_customer_data"], f"{where}.policy.federal_customer_data") + + +def _validate_estimate(e, where, rubric, floor): + if not isinstance(e, dict) or set(e) != ESTIMATE_KEYS: + _fail(f"{where} must have exactly {sorted(ESTIMATE_KEYS)}") + at = parse_ts(e["estimated_at"], f"{where}.estimated_at") + if at < floor: + _fail(f"{where}.estimated_at is before the incident was detected") + if e["customer_effect"] not in EFFECTS or e["agencies"] not in AGENCY_SCOPES: + _fail(f"{where}: customer_effect must be one of {list(EFFECTS)} and agencies one of {list(AGENCY_SCOPES)}") + want = rubric[(e["customer_effect"], e["agencies"])] + if e["rating"] != want or isinstance(e["rating"], bool): + _fail(f"{where}: a likely {e['customer_effect']} effect on {e['agencies']} agenc" + f"{'y' if e['agencies'] == 'single' else 'ies'} is N{want} under IEC-CSO-EFI, not {e['rating']!r}") + _text(e["reason"], f"{where}.reason") + return at + + +def _validate_set(s, where): + if not isinstance(s, dict) or not s or set(s) - SET_KEYS: + _fail(f"{where}.set must be a non-empty object with only {sorted(SET_KEYS)}") + for k in TEXT_FIELDS: + if k in s: + _text(s[k], f"{where}.set.{k}") + for k in LIST_FIELDS: + if k in s: + _strings(s[k], f"{where}.set.{k}", CVE_RE if k == "related_cves" else None) + milestones = s.get("milestones", []) + if not isinstance(milestones, list): + _fail(f"{where}.set.milestones must be a list") + for i, m in enumerate(milestones): + if not isinstance(m, dict) or set(m) != MILESTONE_KEYS: + _fail(f"{where}.set.milestones[{i}] must have exactly {sorted(MILESTONE_KEYS)}") + parse_ts(m["at"], f"{where}.set.milestones[{i}].at") + _text(m["description"], f"{where}.set.milestones[{i}].description") + + +def _validate_incident(inc, where, policy, rubric, now): + rid = inc.get("id") if isinstance(inc, dict) else None + if not isinstance(rid, str) or not ID_RE.match(rid): + _fail(f"{where}: incident id {rid!r} is not INC-YYYY-NNN") + where = rid + required = {"id", "title", "exercise", "started_at", "detected_at", "detection_source", "evaluation", + "estimates", "reports"} + if set(inc) - INCIDENT_KEYS or required - set(inc): + _fail(f"{where}: unknown field(s) {sorted(set(inc) - INCIDENT_KEYS)}, missing {sorted(required - set(inc))}") + _text(inc["title"], f"{where}.title") + _text(inc["detection_source"], f"{where}.detection_source") + if not isinstance(inc["exercise"], bool): + _fail(f"{where}.exercise must be true or false") + started = parse_ts(inc["started_at"], f"{where}.started_at") + detected = parse_ts(inc["detected_at"], f"{where}.detected_at") + if detected < started: + _fail(f"{where}: detected before it started") + + ev = inc["evaluation"] + if not isinstance(ev, dict) or set(ev) != EVALUATION_KEYS: + _fail(f"{where}.evaluation must have exactly {sorted(EVALUATION_KEYS)} (IEC-CSO-EFR: when, the " + f"determination, its basis and reason, and the federal customer data considered)") + evaluated = parse_ts(ev["evaluated_at"], f"{where}.evaluation.evaluated_at") + if evaluated < detected: + _fail(f"{where}: evaluated before it was detected") + if not isinstance(ev["reportable"], bool): + _fail(f"{where}.evaluation.reportable must be true or false") + _text(ev["reason"], f"{where}.evaluation.reason") + _text(ev["federal_customer_data_considered"], f"{where}.evaluation.federal_customer_data_considered") + if ev["basis"] not in BASES: + _fail(f"{where}.evaluation.basis must be one of {list(BASES)}") + findings = _strings(inc.get("findings", []), f"{where}.findings") + if ev["basis"] in VULN_RULES: + if ev["basis"] not in policy["adopted_vulnerability_rules"]: + _fail(f"{where}: basis {ev['basis']} is not an adopted rule in policy.adopted_vulnerability_rules") + if not ev["reportable"]: + _fail(f"{where}: a vulnerability treated as a FedRAMP Reportable Incident under {ev['basis']} is reportable") + if not findings: + _fail(f"{where}: basis {ev['basis']} names no finding") + + if not isinstance(inc["estimates"], list) or not isinstance(inc["reports"], list): + _fail(f"{where}: estimates and reports must be lists") + if not ev["reportable"] and (inc["estimates"] or inc["reports"]): + _fail(f"{where}: not a FedRAMP Reportable Incident, so it carries no rating or report") + last = detected + for i, e in enumerate(inc["estimates"]): + at = _validate_estimate(e, f"{where}.estimates[{i}]", rubric, detected) + if at < last: + _fail(f"{where}.estimates are not in time order") + last = at + + resolved = parse_ts(inc["resolved_at"], f"{where}.resolved_at") if "resolved_at" in inc else None + recovered = (parse_ts(inc["recovery_complete_at"], f"{where}.recovery_complete_at") + if "recovery_complete_at" in inc else None) + if recovered and not resolved: + _fail(f"{where}: recovery is complete but the incident is not resolved") + if resolved and resolved < detected or recovered and resolved and recovered < resolved: + _fail(f"{where}: resolved_at / recovery_complete_at are out of order") + + reports = inc["reports"] + prev = evaluated + for i, r in enumerate(reports): + w = f"{where}.reports[{i}]" + if not isinstance(r, dict) or set(r) - REPORT_KEYS or {"type", "at"} - set(r): + _fail(f"{w} must have type, at and optionally set") + if r["type"] not in REPORT_TYPES: + _fail(f"{w}.type must be one of {list(REPORT_TYPES)}") + want = "Initial" if i == 0 else None + if want and r["type"] != want: + _fail(f"{where}: the first report is the Initial Incident Report (IEC-CSO-IIR)") + if i > 0 and r["type"] == "Initial": + _fail(f"{where}: only the first report is Initial") + if r["type"] == "Final" and i != len(reports) - 1: + _fail(f"{where}: nothing follows the Final Incident Report") + at = parse_ts(r["at"], f"{w}.at") + if at < prev: + _fail(f"{w}.at is before the evaluation or the previous report") + prev = at + if "set" in r: + _validate_set(r["set"], w) + elif r["type"] == "Initial": + _fail(f"{w}: the Initial report sets at least the description") + if reports and "description" not in reports[0].get("set", {}): + _fail(f"{where}: the Initial report must describe the incident (IEC-CSO-IIR)") + if reports and reports[-1]["type"] == "Final": + if not recovered: + _fail(f"{where}: a Final Incident Report needs resolved_at and recovery_complete_at (IEC-CSO-FIR)") + if parse_ts(reports[-1]["at"], "") < recovered: + _fail(f"{where}: the Final report is dated before recovery completed") + + if now is not None: + stamps = [started, detected, evaluated, resolved, recovered] + stamps += [parse_ts(e["estimated_at"], "") for e in inc["estimates"]] + stamps += [parse_ts(r["at"], "") for r in reports] + if any(t and t > now for t in stamps): + _fail(f"{where} records a time after now; a report cannot be issued in the future") + + +def validate(doc, where="data/incidents.json", now=None, rules_dir=_rules.RULES_DIR): + """Structural and rule checks, enforced at build time without a jsonschema + dependency. `now` (aware datetime), when given, also refuses future times.""" + if not isinstance(doc, dict) or doc.get("schema") != "incidents/1": + _fail(f"{where} is not an incidents/1 document") + if set(doc) - TOP_KEYS or {"schema", "policy", "incidents"} - set(doc): + _fail(f"{where}: top-level fields must be {sorted(TOP_KEYS)}") + _validate_policy(doc["policy"], where) + if not isinstance(doc["incidents"], list): + _fail(f"{where}: incidents must be a list") + rubric = _rules.incident_rating_rubric(rules_dir) + seen = set() + for i, inc in enumerate(doc["incidents"]): + _validate_incident(inc, f"{where} incidents[{i}]", doc["policy"], rubric, now) + if inc["id"] in seen: + _fail(f"{where}: duplicate id {inc['id']}") + seen.add(inc["id"]) + _published_text_ok(doc, where) + return doc + + +def load(path=None, now=None, rules_dir=_rules.RULES_DIR): + p = Path(path) if path else DEFAULT_PATH + if not p.is_file(): + _fail(f"{p} not found") + try: + doc = json.loads(p.read_text()) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + _fail(f"{p} is not valid JSON: {exc}") + return validate(doc, str(p), now=now, rules_dir=rules_dir) + + +# ---- ratings and clocks ------------------------------------------------------- + +def add_clock(start, unit, n): + """`start` plus n hours, or n business days. A business day here is Monday to + Friday in UTC; federal holidays are counted as business days, which can only + make a deadline earlier, never later.""" + if unit == "hours": + return start + timedelta(hours=n) + if unit == "bizdays": + t, left = start.astimezone(timezone.utc), n + while left: + t += timedelta(days=1) + if t.weekday() < 5: + left -= 1 + return t + _fail(f"unknown clock unit {unit!r}") + + +def rating_at(inc, t, default): + """(N-rating, basis) in force at time t: the latest Incident Rating estimate + made by then (IEC-CSO-EFI), else the default (IEC-CSO-DPR).""" + best = None + for e in inc["estimates"]: + if parse_ts(e["estimated_at"], "") <= t: + best = e + if best is None: + return default, "default (IEC-CSO-DPR)" + return best["rating"], f"estimated (IEC-CSO-EFI) at {best['estimated_at']}" + + +def rating_history(inc, default, until=None): + """[(when, rating, basis)] from evaluation through `until` (inclusive).""" + out = [(inc["evaluation"]["evaluated_at"], default, "default (IEC-CSO-DPR)")] + for e in inc["estimates"]: + if until is None or parse_ts(e["estimated_at"], "") <= until: + out.append((e["estimated_at"], e["rating"], f"estimated (IEC-CSO-EFI): {e['reason']}")) + return out + + +def _obligation(kind, due, report, index, now): + met_at = parse_ts(report["at"], "") if report else None + return {"report": kind, "due_at": due.isoformat(), "report_index": index, + "issued_at": report["at"] if report else None, + "on_time": (met_at <= due) if report else None, + "overdue": (report is None and now > due)} + + +def obligations(inc, timeframes, default, now): + """Every report the Class C clocks call for, in order, and whether each was + issued by its deadline. The last entry with report_index None is the one + still due. Not reportable: no obligations.""" + if not inc["evaluation"]["reportable"]: + return [] + reports = inc["reports"] + resolved = parse_ts(inc["resolved_at"], "") if "resolved_at" in inc else None + recovered = parse_ts(inc["recovery_complete_at"], "") if "recovery_complete_at" in inc else None + evaluated = parse_ts(inc["evaluation"]["evaluated_at"], "") + + def due(kind, start): + return add_clock(start, *timeframes[kind][rating_at(inc, start, default)[0]]) + + out = [_obligation("Initial", due("iir", evaluated), reports[0] if reports else None, 0 if reports else None, now)] + if not reports: + return out + i, prev = 1, parse_ts(reports[0]["at"], "") + while True: + oir_due = due("oir", prev) + if resolved is not None and resolved <= oir_due: + # Resolved before the next Ongoing report fell due: any further + # Ongoing reports are voluntary, and the Final report's clock starts + # when recovery completes (IEC-CSO-FIR). + while i < len(reports) and reports[i]["type"] == "Ongoing": + i += 1 + if recovered is not None: + final = reports[i] if i < len(reports) else None + out.append(_obligation("Final", due("fir", recovered), final, i if final else None, now)) + return out + nxt = reports[i] if i < len(reports) and reports[i]["type"] == "Ongoing" else None + out.append(_obligation("Ongoing", oir_due, nxt, i if nxt else None, now)) + if nxt is None: + return out + prev, i = parse_ts(nxt["at"], ""), i + 1 + + +def status(inc): + if not inc["evaluation"]["reportable"]: + return "not-reportable" + if inc["reports"] and inc["reports"][-1]["type"] == "Final": + return "closed" + if "resolved_at" in inc: + return "resolved" + return "open" + + +def open_findings(reg, include_exercises=False): + """{finding or CVE: incident id} named by incidents that are not yet resolved.""" + out = {} + for inc in reg["incidents"]: + if inc["exercise"] and not include_exercises: + continue + if status(inc) == "open": + for f in inc.get("findings", []): + out[f] = inc["id"] + return out diff --git a/scripts/_rules.py b/scripts/_rules.py index ed703e1c..26bb5ebe 100644 --- a/scripts/_rules.py +++ b/scripts/_rules.py @@ -9,6 +9,7 @@ import hashlib import json +import re from functools import lru_cache from pathlib import Path @@ -188,3 +189,83 @@ def applicable_provider_rules(rules_dir=RULES_DIR): if not out: _fail("no provider rules apply; the applicability metadata has changed shape") return out + + +# ---- Incident Evaluation and Communication (IEC), Class C --------------------- +# The report clocks and the Incident Rating scale are FedRAMP's, read here from +# the pinned corpus like the remediation matrix above, never typed in. + +INCIDENT_REPORTS = {"iir": "IEC-CSO-IIR", "oir": "IEC-CSO-OIR", "fir": "IEC-CSO-FIR"} +_CLOCK_UNITS = ("hours", "bizdays") + + +def _iec(rules, rule_id): + try: + return rules["FRR"]["IEC"]["data"]["all"]["CSO"][rule_id] + except (KeyError, TypeError): + _fail(f"{rule_id} not found at FRR/IEC/data/all/CSO") + + +def incident_report_timeframes(rules_dir=RULES_DIR): + """{"iir"|"oir"|"fir": {1..5: (unit, n)}} for Class C, unit "hours" or + "bizdays". Every impact level must carry every clock: a report with no + deadline cannot be alarmed.""" + rules, _ = load_rules(rules_dir) + out = {} + for kind, rid in INCIDENT_REPORTS.items(): + try: + table = _iec(rules, rid)["varies_by_class"][CERT_CLASS]["pain_timeframes"] + except (KeyError, TypeError): + _fail(f"{rid} has no Class {CERT_CLASS.upper()} pain_timeframes") + clocks = {} + for n in ("1", "2", "3", "4", "5"): + cell = (table.get(n) or {}).get(kind) + if not isinstance(cell, dict) or cell.get("timeframe_type") not in _CLOCK_UNITS \ + or not isinstance(cell.get("timeframe_num"), int) or isinstance(cell["timeframe_num"], bool) \ + or cell["timeframe_num"] <= 0: + _fail(f"{rid} Class C N{n} {kind} is not a positive number of hours or business days") + clocks[int(n)] = (cell["timeframe_type"], cell["timeframe_num"]) + out[kind] = clocks + return out + + +_RATING_RE = re.compile(r"^\*\*N([1-5])\*\* for (.+)\.$") +_CLAUSE_RE = re.compile(r"likely (minimal|narrow|disruptive|debilitating) customer effect on " + r"(1 or more agencies|more than 1 agency|1 agency)") +_SCOPES_BY_TEXT = {"1 or more agencies": ("single", "multiple"), "1 agency": ("single",), + "more than 1 agency": ("multiple",)} + + +def incident_rating_rubric(rules_dir=RULES_DIR): + """{(customer effect, "single"|"multiple" agencies): N-rating 1..5} from the + IEC-CSO-EFI bullets. Every (effect, scope) pair must be rated exactly once, + or the text has changed shape and the rubric is not trusted.""" + rules, _ = load_rules(rules_dir) + bullets = _iec(rules, "IEC-CSO-EFI").get("following_information_bullets") + if not isinstance(bullets, list) or not bullets: + _fail("IEC-CSO-EFI has no rating bullets") + out = {} + for b in bullets: + m = _RATING_RE.match(b) if isinstance(b, str) else None + clauses = _CLAUSE_RE.findall(m.group(2)) if m else [] + if not clauses: + _fail(f"IEC-CSO-EFI bullet {b!r} is not in the expected form") + for effect, scope_text in clauses: + for scope in _SCOPES_BY_TEXT[scope_text]: + if (effect, scope) in out: + _fail(f"IEC-CSO-EFI rates {effect}/{scope} twice") + out[(effect, scope)] = int(m.group(1)) + effects = ("minimal", "narrow", "disruptive", "debilitating") + if set(out) != {(e, s) for e in effects for s in ("single", "multiple")}: + _fail("IEC-CSO-EFI does not rate every customer effect for one and for several agencies") + return out + + +def default_incident_rating(rules_dir=RULES_DIR): + """IEC-CSO-DPR: the N-rating a FedRAMP Reportable Incident has until it is estimated.""" + rules, _ = load_rules(rules_dir) + stmt = _iec(rules, "IEC-CSO-DPR").get("statement", "") + m = re.search(r"\(PAIN\) of ([1-5])\b", stmt) + if not m: + _fail("IEC-CSO-DPR no longer states a default PAIN rating") + return int(m.group(1)) diff --git a/scripts/build-incident-report.py b/scripts/build-incident-report.py new file mode 100644 index 00000000..7c67def4 --- /dev/null +++ b/scripts/build-incident-report.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +# ============================================================================= +# INCIDENT REPORTS: IIR, OIR and FIR from the incident register +# ============================================================================= +# The incident register (data/incidents.json, read through scripts/_incidents.py) +# records each incident's FedRAMP reportability evaluation (IEC-CSO-EFR), its +# Incident Rating (IEC-CSO-EFI, or the IEC-CSO-DPR default) and the reports +# issued. This renders, for every FedRAMP Reportable Incident, each report in +# FedRAMP's 2026-06-24 incident report schema: +# +# incident---initial.json IEC-CSO-IIR +# incident---ongoing.json IEC-CSO-OIR (restates every earlier item) +# incident---final.json IEC-CSO-FIR (final values for every item) +# +# each validated before it is written (scripts/_fedramp_schema.py), plus the +# incident log: +# +# incident-reports.json every incident, its evaluation and reason, its rating +# history, each report (file, sha256, what changed) and +# each report clock (due, issued, on time) +# incident-reports.md the same, human-readable, with every report's content +# +# The clocks are FedRAMP's Class C timeframes, read through scripts/_rules.py. +# +# Vulnerabilities as incidents (VER-TFR-IRI, VER-TFR-NRI). Given the VDR report, +# policies/triage.rego names each finding a rule makes a FedRAMP Reportable +# Incident. One the operator has adopted that rule for and that no unresolved +# incident names is published as a trigger with no incident, its Initial +# report due from the finding's first evaluation at the default rating; the +# clock check fails, and the evidence watchdog pages, once that time passes. +# +# Notification. Delivery to FedRAMP and agencies is not built: the register's +# policy says "published-only" and every report says so. Publication under +# /.well-known/ is the only channel until the operator decides otherwise. +# +# build-incident-report.py --incidents ../data/incidents.json --vdr vdr-report.json --out-dir . +# build-incident-report.py --check incident-reports.json # exit 1 if a report is overdue +# ============================================================================= + +import argparse +import hashlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _fedramp_schema as fs # noqa: E402 +import _incidents # noqa: E402 +import _rules # noqa: E402 + +INDEX = "incident-reports.json" +INDEX_MD = "incident-reports.md" +EXERCISE_PREFIX = "EXERCISE: a simulated incident; no real system, data or agency was affected. " + +# The items an Initial report carries (IEC-CSO-IIR) and the further items an +# Ongoing report adds (IEC-CSO-OIR), in the rules' order, by register field. +IIR_ITEMS = [("coordinator", "Federal incident response coordinator"), + ("tracking_id", "Provider tracking identifier"), + ("description", "Description of the incident"), + ("timeline", "Timeline"), + ("rating", "Potential Agency Impact N-rating (historical and current)"), + ("functional_impact", "Functional impact to federal agency customers"), + ("recovery_plan", "Estimated recovery plan, milestones and timelines"), + ("affected_agencies", "Likely affected customer agencies")] +OIR_ITEMS = [("observed_activity", "Observed incident activity"), + ("indicators_of_compromise", "Indicators of compromise"), + ("related_cves", "Related CVE identifiers"), + ("root_cause", "Root cause"), + ("response_and_recovery_activities", "Response and recovery activities")] + + +def _fail(msg): + raise SystemExit(f"::error::incident reports: {msg} (fail-closed)") + + +def _iso(dt): + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def vulnerability_triggers(vdr): + """The VDR findings policies/triage.rego treats as FedRAMP Reportable Incidents.""" + import _triage + findings = [{k: f.get(k) for k in ("tracking_id", "cve", "pain", "internet_reachable", "likely_exploitable", + "fully_mitigated", "final_disposition", "first_evaluated")} + for f in vdr.get("findings", [])] + out = _triage.evaluate('{"triggers": data.vdr.triage.incident_triggers, ' + '"errors": data.vdr.triage.incident_errors}', {"findings": findings}) + if out.get("errors"): + _fail("; ".join(sorted(out["errors"]))) + if not isinstance(out.get("triggers"), list): + _fail("the triage policy returned no incident triggers") + return out["triggers"] + + +def _milestones(inc, upto, default): + """Every milestone known at `upto`: the register's, plus the ones the + register's own fields mark (evaluation, each rating, resolution, recovery).""" + ms = [] + for r in inc["reports"]: + if _incidents.parse_ts(r["at"], "") > upto: + break + ms += [(m["at"], m["description"]) for m in r.get("set", {}).get("milestones", [])] + ev = inc["evaluation"] + ms.append((ev["evaluated_at"], f"FedRAMP reportability evaluated ({ev['basis']}): reportable; " + f"rated N{default} by default (IEC-CSO-DPR)")) + for e in inc["estimates"]: + ms.append((e["estimated_at"], f"Incident Rating estimated N{e['rating']} (IEC-CSO-EFI)")) + for k, text in (("resolved_at", "Incident resolved"), ("recovery_complete_at", "Recovery complete")): + if k in inc: + ms.append((inc[k], text)) + known = [(at, d) for at, d in ms if _incidents.parse_ts(at, "") <= upto] + return [{"occurredAt": at, "milestoneDescription": d} + for at, d in sorted(known, key=lambda x: _incidents.parse_ts(x[0], ""))] + + +def _state(inc, k): + """The register fields as known after report k, and which report last set each.""" + state, set_by = {}, {} + for i, r in enumerate(inc["reports"][:k + 1]): + for key, val in r.get("set", {}).items(): + if key != "milestones": + state[key], set_by[key] = val, i + return state, set_by + + +def _rating_notes(inc, at, default): + rating, basis = _incidents.rating_at(inc, at, default) + hist = [f"N{r} from {when} ({b})" for when, r, b in _incidents.rating_history(inc, default, at)] + if basis.startswith("default"): + lead = (f"No Incident Rating estimate had been made; rated N{rating} by default as IEC-CSO-DPR requires.") + else: + e = [x for x in inc["estimates"] if _incidents.parse_ts(x["estimated_at"], "") <= at][-1] + scope = "one agency" if e["agencies"] == "single" else "more than one agency" + lead = (f"Estimated N{rating} under IEC-CSO-EFI: a likely {e['customer_effect']} customer effect on " + f"{scope}. {e['reason']}") + return rating, f"{lead} History: {'; '.join(hist)}." + + +def report_document(inc, k, policy, default): + """Report k of incident `inc` in the FedRAMP incident report schema. Every + item set by report k or earlier is restated with its latest value.""" + r = inc["reports"][k] + at = _incidents.parse_ts(r["at"], "") + state, _ = _state(inc, k) + rating, notes = _rating_notes(inc, at, default) + doc = { + "$schema": fs.stamp("incident"), + "certificationPackageOverviewUri": fs.CPO_URI, + "reportType": r["type"], + "providerTrackingId": inc["id"], + "federalIncidentCoordinator": policy["federal_incident_coordinator"], + "incidentDescription": (EXERCISE_PREFIX if inc["exercise"] else "") + state["description"], + "timeline": { + "startedAt": inc["started_at"], + "detectedAt": inc["detected_at"], + "detectionSource": inc["detection_source"], + "evaluationCompletedAt": inc["evaluation"]["evaluated_at"], + "milestones": _milestones(inc, at, default), + }, + "potentialImpact": { + "historicalRating": [n for _, n, _ in _incidents.rating_history(inc, default, at)], + "currentRating": rating, + "evaluationNotes": notes, + }, + } + for key, field in _incidents.TEXT_FIELDS.items(): + if key in state and key != "description": + doc[field] = state[key] + for key, field in _incidents.LIST_FIELDS.items(): + if key in state: + doc[field] = state[key] + if r["type"] == "Final": + doc["resolvedAt"] = inc["recovery_complete_at"] + return doc + + +def _changes(inc, k): + """{item: "updated" | "no update" | "not yet available"} for report k.""" + state, set_by = _state(inc, k) + out = {} + for key, _ in IIR_ITEMS + OIR_ITEMS: + if key in ("coordinator", "tracking_id"): + out[key] = "updated" if k == 0 else "no update" + elif key == "timeline": + out[key] = "updated" if k == 0 or inc["reports"][k].get("set", {}).get("milestones") else "no update" + elif key == "rating": + prev = _incidents.parse_ts(inc["reports"][k - 1]["at"], "") if k else None + now = _incidents.parse_ts(inc["reports"][k]["at"], "") + moved = k == 0 or any(prev < _incidents.parse_ts(e["estimated_at"], "") <= now for e in inc["estimates"]) + out[key] = "updated" if moved else "no update" + elif key not in state: + out[key] = "not yet available" + else: + out[key] = "updated" if set_by[key] == k else "no update" + return out + + +def build(reg, vdr, now, out_dir, rules_dir=_rules.RULES_DIR): + """Write every report and the incident log; return the log.""" + out_dir = Path(out_dir) + timeframes = _rules.incident_report_timeframes(rules_dir) + default = _rules.default_incident_rating(rules_dir) + policy = reg["policy"] + registry = fs.load_registry() + incidents, due = [], [] + for inc in reg["incidents"]: + reports = [] + for k, r in enumerate(inc["reports"]): + name = f"incident-{inc['id']}-{k + 1:02d}-{r['type'].lower()}.json" + fs.write_validated(report_document(inc, k, policy, default), "incident", out_dir / name, registry) + reports.append({"type": r["type"], "issued_at": r["at"], "file": name, + "sha256": hashlib.sha256((out_dir / name).read_bytes()).hexdigest(), + "changes": _changes(inc, k)}) + obligations = _incidents.obligations(inc, timeframes, default, now) + pending = [o for o in obligations if o["report_index"] is None] + for o in pending: + due.append({"incident": inc["id"], "finding": None, "report": o["report"], "due_at": o["due_at"], + "overdue": o["overdue"]}) + last = inc["reports"][-1]["at"] if inc["reports"] else inc["evaluation"]["evaluated_at"] + rating, basis = _incidents.rating_at(inc, _incidents.parse_ts(last, ""), default) + incidents.append({ + "id": inc["id"], "title": inc["title"], "exercise": inc["exercise"], + "status": _incidents.status(inc), + "detected_at": inc["detected_at"], "detection_source": inc["detection_source"], + "evaluation": inc["evaluation"], "findings": inc.get("findings", []), + "rating": ({"current": rating, "basis": basis, + "history": [{"from": w, "rating": n, "basis": b} + for w, n, b in _incidents.rating_history(inc, default)]} + if inc["evaluation"]["reportable"] else None), + "resolved_at": inc.get("resolved_at"), "recovery_complete_at": inc.get("recovery_complete_at"), + "reports": reports, "obligations": obligations, + "next_report_due_at": pending[0]["due_at"] if pending else None, + }) + + triggers = [] + covered = _incidents.open_findings(reg) + if vdr is not None: + for t in vulnerability_triggers(vdr): + adopted = t["rule"] in policy["adopted_vulnerability_rules"] + inc_id = covered.get(t["tracking_id"]) or (covered.get(t["cve"]) if t["cve"] else None) + entry = {**t, "adopted": adopted, "incident": inc_id} + triggers.append(entry) + if adopted and inc_id is None: + start = _incidents.parse_ts(t["first_evaluated"], "") if t["first_evaluated"] else now + unit, n = timeframes["iir"][default] + at = _incidents.add_clock(start, unit, n) + due.append({"incident": None, "finding": t["tracking_id"], "report": "Initial", + "due_at": at.isoformat(), "overdue": now > at}) + overdue = [d for d in due if d["overdue"]] + log = { + "schema": "incident-reports/1", + "emitted_at": _iso(now), + "generator": "scripts/build-incident-report.py", + "description": ("The incident log: every incident recorded in data/incidents.json, its FedRAMP " + "reportability evaluation and reason (IEC-CSO-EFR), its Incident Rating (IEC-CSO-EFI, " + "or the IEC-CSO-DPR default), each Initial, Ongoing and Final report issued, and each " + "Class C report clock. Exercises are marked exercise: true."), + "rules": _rules.pin(rules_dir), + "notification": {"mode": policy["notification"], "text": policy["notification_text"]}, + "policy": {k: policy[k] for k in ("federal_incident_coordinator", "adopted_vulnerability_rules", + "federal_customer_data", "decided_by", "decided_on", "review_by")}, + "vdr": ({"emitted_at": vdr.get("emitted_at"), "report_id": vdr.get("report_id")} if vdr is not None + else None), + "vulnerability_triggers": triggers, + "obligations_due": due, + "clock_status": "overdue" if overdue else "ok", + "summary": { + "incidents": len(incidents), + "reportable": sum(1 for i in incidents if i["evaluation"]["reportable"]), + "open": sum(1 for i in incidents if i["status"] in ("open", "resolved")), + "exercises": sum(1 for i in incidents if i["exercise"]), + "reports": sum(len(i["reports"]) for i in incidents), + "triggers_without_incident": sum(1 for t in triggers if t["adopted"] and t["incident"] is None), + "overdue": len(overdue), + }, + "incidents": incidents, + } + (out_dir / INDEX).write_text(json.dumps(log, indent=2) + "\n") + (out_dir / INDEX_MD).write_text(render_markdown(log, out_dir) + "\n") + return log + + +def _item_text(key, doc): + if key == "coordinator": + return doc["federalIncidentCoordinator"] + if key == "tracking_id": + return doc["providerTrackingId"] + if key == "description": + return doc["incidentDescription"] + if key == "timeline": + t = doc["timeline"] + lines = [f"started {t['startedAt']}; detected {t['detectedAt']} by {t['detectionSource']}; " + f"reportability evaluated {t['evaluationCompletedAt']}"] + lines += [f"{m['occurredAt']}: {m['milestoneDescription']}" for m in t["milestones"]] + return "\n".join(f" - {x}" for x in lines) + if key == "rating": + p = doc["potentialImpact"] + return f"N{p['currentRating']} (history {', '.join(f'N{n}' for n in p['historicalRating'])}). " \ + f"{p['evaluationNotes']}" + field = _incidents.TEXT_FIELDS.get(key) or _incidents.LIST_FIELDS.get(key) + val = doc.get(field) + if isinstance(val, list): + return ", ".join(val) if val else "none" + return val or "not yet available" + + +def render_markdown(log, out_dir): + s = log["summary"] + out = ["# Incident log and reports", "", + "_Generated by scripts/build-incident-report.py from data/incidents.json. The JSON beside this file " + "(incident-reports.json, signed) is the record; this is a rendering of it._", "", + f"- **Emitted:** {log['emitted_at']}", + f"- **Incidents:** {s['incidents']} ({s['reportable']} FedRAMP Reportable; {s['exercises']} exercises); " + f"**reports issued:** {s['reports']}; **overdue:** {s['overdue']}", + f"- **Notification:** {log['notification']['mode']}. {log['notification']['text']}", + f"- **Federal incident response coordinator:** {log['policy']['federal_incident_coordinator']}", + f"- **Vulnerabilities treated as incidents:** {', '.join(log['policy']['adopted_vulnerability_rules'])}", + ""] + out += ["## Vulnerabilities that are FedRAMP Reportable Incidents (VER-TFR-IRI / VER-TFR-NRI)", ""] + if log["vdr"] is None: + out += ["No VDR report was supplied to this build.", ""] + elif not log["vulnerability_triggers"]: + out += [f"None in the VDR report emitted {log['vdr']['emitted_at']}.", ""] + else: + out += ["| Finding | Rule | PAIN | Adopted | Incident |", "|---|---|---|---|---|"] + out += [f"| `{t['tracking_id']}` | {t['rule']} | {t['pain']} | {t['adopted']} | {t['incident'] or 'NONE'} |" + for t in log["vulnerability_triggers"]] + out.append("") + if log["obligations_due"]: + out += ["## Reports due", "", "| Incident / finding | Report | Due | Overdue |", "|---|---|---|---|"] + out += [f"| {d['incident'] or d['finding']} | {d['report']} | {d['due_at']} | {d['overdue']} |" + for d in log["obligations_due"]] + out.append("") + for entry in log["incidents"]: + ev = entry["evaluation"] + out += [f"## {entry['id']}: {entry['title']}" + (" (EXERCISE)" if entry["exercise"] else ""), "", + f"- **Status:** {entry['status']}", + f"- **Detected:** {entry['detected_at']} by {entry['detection_source']}", + f"- **Reportability (IEC-CSO-EFR, evaluated {ev['evaluated_at']}, basis {ev['basis']}):** " + f"{'FedRAMP Reportable Incident' if ev['reportable'] else 'not a FedRAMP Reportable Incident'}. " + f"{ev['reason']}", + f"- **Federal customer data considered:** {ev['federal_customer_data_considered']}"] + if entry["findings"]: + out.append(f"- **Findings named:** {', '.join(f'`{f}`' for f in entry['findings'])}") + if entry["rating"]: + out.append(f"- **Rating:** N{entry['rating']['current']} ({entry['rating']['basis']})") + if entry["obligations"]: + out += ["", "| Report | Due | Issued | On time |", "|---|---|---|---|"] + out += [f"| {o['report']} | {o['due_at']} | {o['issued_at'] or 'not yet'} | " + f"{'yes' if o['on_time'] else 'no' if o['on_time'] is False else ('OVERDUE' if o['overdue'] else 'pending')} |" + for o in entry["obligations"]] + out.append("") + for k, rep in enumerate(entry["reports"]): + doc = json.loads((Path(out_dir) / rep["file"]).read_text()) + out += [f"### {rep['type']} report, issued {rep['issued_at']} ({rep['file']})", "", + f"_Notification: {log['notification']['mode']}; see above._", ""] + items = IIR_ITEMS + (OIR_ITEMS if k else + [x for x in OIR_ITEMS if rep["changes"][x[0]] != "not yet available"]) + for key, label in items: + change = rep["changes"][key] + text = _item_text(key, doc) + sep = "\n" if text.startswith(" - ") else " " + out.append(f"- **{label}** ({change}):{sep}{text}") + if rep["type"] == "Final": + out.append(f"- **Resolved and recovered:** {doc['resolvedAt']}") + out.append("") + return "\n".join(out).rstrip() + + +def check(index_path, now): + """Exit status for the clock check: 1 when a report is due and missing.""" + log = json.loads(Path(index_path).read_text()) + late = [d for d in log.get("obligations_due", []) + if _incidents.parse_ts(d["due_at"], "obligations_due.due_at") < now] + for d in late: + who = d["incident"] or f"finding {d['finding']} (no incident opened)" + print(f"::error::{who}: the {d['report']} incident report was due at {d['due_at']} and has not been issued") + if not late: + print(f"incident report clocks: ok ({len(log.get('obligations_due', []))} report(s) pending, none overdue)") + return 1 if late else 0 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--incidents", default=str(_incidents.DEFAULT_PATH)) + ap.add_argument("--vdr", default=None, help="the built VDR report, for VER-TFR-IRI/NRI triggers") + ap.add_argument("--out-dir", default=".") + ap.add_argument("--now", default=None, help="ISO 8601 time to evaluate the clocks at (default: now)") + ap.add_argument("--check", default=None, metavar="INDEX", + help="check a built incident-reports.json and exit 1 if a report is overdue") + a = ap.parse_args(argv) + now = _incidents.parse_ts(a.now, "--now") if a.now else datetime.now(timezone.utc) + if a.check: + return check(a.check, now) + reg = _incidents.load(a.incidents, now=now) + vdr = None + if a.vdr: + try: + vdr = json.loads(Path(a.vdr).read_text()) + except (OSError, json.JSONDecodeError) as exc: + _fail(f"cannot read the VDR report {a.vdr}: {exc}") + if not isinstance(vdr, dict) or not isinstance(vdr.get("findings"), list): + _fail(f"{a.vdr} is not a VDR report") + log = build(reg, vdr, now, a.out_dir) + s = log["summary"] + print(f"incident reports: {s['incidents']} incident(s), {s['reports']} report(s), " + f"{s['triggers_without_incident']} trigger(s) without an incident, clocks {log['clock_status']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build-ksi-signal.py b/scripts/build-ksi-signal.py index 1d570b1c..08f2b07e 100755 --- a/scripts/build-ksi-signal.py +++ b/scripts/build-ksi-signal.py @@ -556,6 +556,9 @@ ("iam_role", "evidence_nightly"): { "function": "Nightly evidence workflow role", }, + ("iam_role", "incident_reports"): { + "function": "Incident reports workflow role (unattended IIR/OIR/FIR publication)", + }, ("object_store", "logs"): { "function": "Access-log bucket for the site bucket and CDN", "diagram_label": "S3 access-log bucket", diff --git a/scripts/build-ocr.py b/scripts/build-ocr.py index cad56d46..08f126ea 100644 --- a/scripts/build-ocr.py +++ b/scripts/build-ocr.py @@ -30,6 +30,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import _decisions # noqa: E402 +import _incidents # noqa: E402 QUARTER_DAYS = 91 @@ -50,7 +51,22 @@ def _scn_rows(path): return list(csv.DictReader(f)) -def build_ocr(vdr, decisions, scn_rows, signal_id, now): +def incident_summary(incidents, period_start): + """CCM-OCR-AVL item 7, from the incident register: the FedRAMP Reportable + Incidents detected this period, or an attestation of none. Exercises are + not incidents and are never counted.""" + log = "The incident log, /.well-known/incident-reports.json, records every incident evaluated, exercises included." + real = [i for i in (incidents or {}).get("incidents", []) + if not i["exercise"] and i["evaluation"]["reportable"] and i["detected_at"][:10] >= period_start] + if real: + return (f"{len(real)} FedRAMP Reportable Incident(s) this period: " + + "; ".join(f"{i['id']} ({_incidents.status(i)})" for i in real) + f". {log}") + return ("No FedRAMP Reportable Incidents occurred during this reporting period. The system holds " + "no federal customer data, so the IEC-CSO-EFR trigger does not apply; this is an " + f"attestation of none, not an omission. {log}") + + +def build_ocr(vdr, decisions, scn_rows, signal_id, now, incidents=None): period_start = (now - timedelta(days=QUARTER_DAYS)).date().isoformat() period_end = now.date().isoformat() @@ -97,10 +113,7 @@ def build_ocr(vdr, decisions, scn_rows, signal_id, now): "No new customer recommendations this period; the Secure Configuration " "Guidance at /.well-known/ remains current."], "agencies_using_product": [], # 6 (none) - "reportable_incidents_or_attestation": # 7 - "No FedRAMP Reportable Incidents occurred during this reporting period. " - "The system holds no federal customer data, so the IEC-CSO-EFR trigger " - "does not apply; this is an attestation of none, not an omission.", + "reportable_incidents_or_attestation": incident_summary(incidents, period_start), # 7 "lessons_learned_from_incidents": [], # 8 (none) }, "relationship_obligations_status": ( @@ -162,6 +175,7 @@ def main(): ap.add_argument("--vdr", default="vdr-report.json") ap.add_argument("--decisions", default="../data/decisions.json") ap.add_argument("--scn-register", default="../docs/scn/scn-register.csv") + ap.add_argument("--incidents", default="../data/incidents.json") ap.add_argument("--ksi-signal", default="ksi-signal.json") ap.add_argument("--output", default="ongoing-certification-report.json") ap.add_argument("--md-output", default="ongoing-certification-report.md") @@ -174,7 +188,7 @@ def main(): signal_id = sig.get("signal_id") if sig else None scn_rows = _scn_rows(a.scn_register) - report = build_ocr(vdr, decisions, scn_rows, signal_id, now) + report = build_ocr(vdr, decisions, scn_rows, signal_id, now, _incidents.load(a.incidents)) Path(a.output).write_text(json.dumps(report, indent=2) + "\n") if a.md_output: Path(a.md_output).write_text(render_markdown(report) + "\n") diff --git a/scripts/reconcile.py b/scripts/reconcile.py index 5f187a81..402194d3 100644 --- a/scripts/reconcile.py +++ b/scripts/reconcile.py @@ -42,6 +42,10 @@ # every boundary component of the inventory (no more, no # fewer), leaves none unclassified, and carries no ARN, # account ID or email address +# (m) incident findings — every finding or CVE an unresolved incident in the +# incident register (data/incidents.json) names is in +# this build's VDR report; exercises are exempt (their +# findings are simulated) # # Usage: # reconcile.py --artifacts-dir infrastructure [--live] [--expect-commit SHA] @@ -64,6 +68,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import _decisions # noqa: E402 import _gates # noqa: E402 +import _incidents # noqa: E402 import _redact # noqa: E402 REPO = Path(__file__).resolve().parent.parent @@ -525,9 +530,26 @@ def check_l_boundary_map(signal, boundary_map, expected_commit=None): return violations +def check_m_incident_findings(vdr, incidents): + """(m) Every finding an unresolved, non-exercise incident names (a VDR + tracking ID or a CVE ID) is in this build's VDR report. An incident about a + vulnerability the VDR does not carry is reporting on something the + vulnerability evidence says is not there: one of the two is wrong.""" + if incidents is None: + return [] + ids, cves = set(), set() + for f in _all_vdr_findings(vdr): + ids.add(f.get("tracking_id")) + if f.get("cve"): + cves.add(f["cve"]) + return [f"(m) incident {inc_id} names {ref!r}, which is not in the VDR report" + for ref, inc_id in sorted(_incidents.open_findings(incidents).items()) + if ref not in ids and ref not in cves] + + def run_all(signal, sdr, cpo, vdr, dashboard_html, checkov_text, register, live_arns=None, expected_commit=None, live_tags=None, - boundary_map=None): + boundary_map=None, incidents=None): violations = [] violations += check_d_impact(signal, sdr, cpo, vdr, dashboard_html) violations += check_e_binding(signal, sdr, cpo, vdr) @@ -537,6 +559,7 @@ def run_all(signal, sdr, cpo, vdr, dashboard_html, checkov_text, register, violations += check_h_finding_coverage(vdr, register) violations += check_j_decision_id_validity(signal, sdr, cpo, vdr, register) violations += check_l_boundary_map(signal, boundary_map, expected_commit) + violations += check_m_incident_findings(vdr, incidents) if live_arns is not None: violations += check_a_completeness(signal, live_arns) if live_tags is not None: @@ -562,6 +585,8 @@ def main(): help="path to .checkov.yaml (overridable for hermetic tests)") ap.add_argument("--decisions", default=str(_decisions.DEFAULT_PATH), help="the decision register (overridable for hermetic tests); missing or invalid fails closed") + ap.add_argument("--incidents", default=str(_incidents.DEFAULT_PATH), + help="the incident register (overridable for hermetic tests); missing or invalid fails closed") args = ap.parse_args() gates = _gates.load_gate_modes() @@ -583,6 +608,7 @@ def main(): dashboard_html = Path(args.dashboard).read_text() if Path(args.dashboard).exists() else None checkov_text = Path(args.checkov).read_text() register = _decisions.load(args.decisions) + incidents = _incidents.load(args.incidents) live_arns = None live_tags = None @@ -617,7 +643,7 @@ def main(): violations = run_all( signal, sdr, cpo, vdr, dashboard_html, checkov_text, register, live_arns=live_arns, expected_commit=args.expect_commit, live_tags=live_tags, - boundary_map=boundary_map, + boundary_map=boundary_map, incidents=incidents, ) # Each violation names its invariant, "(x) ..."; its check's governed mode @@ -638,12 +664,12 @@ def main(): return 1 optional = ",l" if boundary_map is not None else "" if live_arns is not None and live_tags is not None: - checks = "a-f,h-j" + optional + checks = "a-f,h-j" + optional + ",m" elif live_arns is not None: - checks = "a-f,h,j" + optional + " (i deferred: no live tags)" + checks = "a-f,h,j" + optional + ",m (i deferred: no live tags)" else: - checks = "b-f,h,j" + optional + " (a,i deferred: no --live)" - print(f"reconciliation OK — invariants {checks} hold across signal/VDR/SDR/CPO/decisions/dashboard") + checks = "b-f,h,j" + optional + ",m (a,i deferred: no --live)" + print(f"reconciliation OK — invariants {checks} hold across signal/VDR/SDR/CPO/decisions/incidents/dashboard") if args.report: Path(args.report).write_text(json.dumps(build_report( signal, args.expect_commit, @@ -666,6 +692,7 @@ def main(): "j": "Every decision reference resolves", # (k) is retired: the bundle it bound is no longer published by this site. "l": "The boundary map describes exactly this build's inventory", + "m": "Every finding an open incident names is in the VDR report", } diff --git a/tests/fixtures/incidents/bad/duplicate-id.json b/tests/fixtures/incidents/bad/duplicate-id.json new file mode 100644 index 00000000..54fc3da0 --- /dev/null +++ b/tests/fixtures/incidents/bad/duplicate-id.json @@ -0,0 +1,74 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: two incidents share an ID", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + }, + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/empty-reason.json b/tests/fixtures/incidents/bad/empty-reason.json new file mode 100644 index 00000000..db9cbf11 --- /dev/null +++ b/tests/fixtures/incidents/bad/empty-reason.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the reportability determination states no reason", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": " ", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/evaluated-before-detected.json b/tests/fixtures/incidents/bad/evaluated-before-detected.json new file mode 100644 index 00000000..84050cc7 --- /dev/null +++ b/tests/fixtures/incidents/bad/evaluated-before-detected.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the evaluation is dated before detection", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T09:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/final-before-recovery.json b/tests/fixtures/incidents/bad/final-before-recovery.json new file mode 100644 index 00000000..8b094b83 --- /dev/null +++ b/tests/fixtures/incidents/bad/final-before-recovery.json @@ -0,0 +1,53 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: a Final report with no recovery recorded", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + }, + { + "type": "Final", + "at": "2026-11-02T12:00:00Z", + "set": { + "root_cause": "x" + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/final-not-last.json b/tests/fixtures/incidents/bad/final-not-last.json new file mode 100644 index 00000000..67d2550e --- /dev/null +++ b/tests/fixtures/incidents/bad/final-not-last.json @@ -0,0 +1,62 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: a report follows the Final report", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + }, + { + "type": "Final", + "at": "2026-11-02T12:00:00Z", + "set": { + "root_cause": "x" + } + }, + { + "type": "Ongoing", + "at": "2026-11-02T13:00:00Z", + "set": { + "root_cause": "y" + } + } + ], + "resolved_at": "2026-11-02T11:00:00Z", + "recovery_complete_at": "2026-11-02T11:10:00Z" + } + ] +} diff --git a/tests/fixtures/incidents/bad/first-not-initial.json b/tests/fixtures/incidents/bad/first-not-initial.json new file mode 100644 index 00000000..9dd86033 --- /dev/null +++ b/tests/fixtures/incidents/bad/first-not-initial.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the first report is not the Initial report", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Ongoing", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/future-report.json b/tests/fixtures/incidents/bad/future-report.json new file mode 100644 index 00000000..e4e76502 --- /dev/null +++ b/tests/fixtures/incidents/bad/future-report.json @@ -0,0 +1,53 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: a report is dated after now (the test runs with now = 2026-11-03)", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + }, + { + "type": "Ongoing", + "at": "2026-11-04T00:00:00Z", + "set": { + "root_cause": "x" + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/initial-without-description.json b/tests/fixtures/incidents/bad/initial-without-description.json new file mode 100644 index 00000000..e50f60d6 --- /dev/null +++ b/tests/fixtures/incidents/bad/initial-without-description.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the Initial report does not describe the incident", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "root_cause": "x" + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/not-evaluated.json b/tests/fixtures/incidents/bad/not-evaluated.json new file mode 100644 index 00000000..5814ef50 --- /dev/null +++ b/tests/fixtures/incidents/bad/not-evaluated.json @@ -0,0 +1,45 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the incident has no reportability reason (IEC-CSO-EFR)", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/not-reportable-with-report.json b/tests/fixtures/incidents/bad/not-reportable-with-report.json new file mode 100644 index 00000000..bff5c9f1 --- /dev/null +++ b/tests/fixtures/incidents/bad/not-reportable-with-report.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: an incident evaluated as not reportable carries a report", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": false, + "basis": "IEC-CSO-EFR", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/notification-sends.json b/tests/fixtures/incidents/bad/notification-sends.json new file mode 100644 index 00000000..1e40df8f --- /dev/null +++ b/tests/fixtures/incidents/bad/notification-sends.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the policy claims a delivery mode that is not built", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "email", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/personal-email.json b/tests/fixtures/incidents/bad/personal-email.json new file mode 100644 index 00000000..b1bdd4c1 --- /dev/null +++ b/tests/fixtures/incidents/bad/personal-email.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: a published field names a personal email address", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "someone@example.org", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/rating-off-rubric.json b/tests/fixtures/incidents/bad/rating-off-rubric.json new file mode 100644 index 00000000..a40bbe99 --- /dev/null +++ b/tests/fixtures/incidents/bad/rating-off-rubric.json @@ -0,0 +1,54 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: an estimate's rating is not the IEC-CSO-EFI rating for its effect and agency count", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [ + { + "estimated_at": "2026-11-02T11:00:00Z", + "rating": 2, + "customer_effect": "disruptive", + "agencies": "single", + "reason": "x" + } + ], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/report-before-evaluation.json b/tests/fixtures/incidents/bad/report-before-evaluation.json new file mode 100644 index 00000000..b1bee965 --- /dev/null +++ b/tests/fixtures/incidents/bad/report-before-evaluation.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: a report is dated before the evaluation", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T09:45:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/unknown-field.json b/tests/fixtures/incidents/bad/unknown-field.json new file mode 100644 index 00000000..6cf3e552 --- /dev/null +++ b/tests/fixtures/incidents/bad/unknown-field.json @@ -0,0 +1,47 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: an incident carries a field the register does not define", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ], + "severity": "high" + } + ] +} diff --git a/tests/fixtures/incidents/bad/vuln-basis-no-finding.json b/tests/fixtures/incidents/bad/vuln-basis-no-finding.json new file mode 100644 index 00000000..03de43ae --- /dev/null +++ b/tests/fixtures/incidents/bad/vuln-basis-no-finding.json @@ -0,0 +1,44 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: a VER-TFR-IRI incident names no finding", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/bad/vuln-basis-not-adopted.json b/tests/fixtures/incidents/bad/vuln-basis-not-adopted.json new file mode 100644 index 00000000..2cb36efc --- /dev/null +++ b/tests/fixtures/incidents/bad/vuln-basis-not-adopted.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE: the basis is VER-TFR-NRI, which the policy has not adopted", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-NRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/good.json b/tests/fixtures/incidents/good.json new file mode 100644 index 00000000..55c20493 --- /dev/null +++ b/tests/fixtures/incidents/good.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-GOOD FIXTURE: a minimal valid incident register (one open, reportable, non-exercise incident).", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-0001" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/fixtures/incidents/reconcile-m-bad.json b/tests/fixtures/incidents/reconcile-m-bad.json new file mode 100644 index 00000000..5c830fee --- /dev/null +++ b/tests/fixtures/incidents/reconcile-m-bad.json @@ -0,0 +1,46 @@ +{ + "$comment": "KNOWN-BAD FIXTURE for reconcile invariant (m): an open incident names CVE-2099-9999, which the VDR report fixture does not carry.", + "schema": "incidents/1", + "policy": { + "federal_incident_coordinator": "Test coordinator; contact through https://samaydlette.com/.well-known/security.txt", + "notification": "published-only", + "notification_text": "Published only (test fixture).", + "adopted_vulnerability_rules": [ + "VER-TFR-IRI" + ], + "federal_customer_data": "Test fixture.", + "decided_by": "Operator", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + "incidents": [ + { + "id": "INC-2026-101", + "title": "Fixture incident", + "exercise": false, + "started_at": "2026-11-02T09:00:00Z", + "detected_at": "2026-11-02T09:30:00Z", + "detection_source": "test", + "evaluation": { + "evaluated_at": "2026-11-02T10:00:00Z", + "reportable": true, + "basis": "VER-TFR-IRI", + "reason": "Fixture: an internet-reachable LEV at N4.", + "federal_customer_data_considered": "Fixture." + }, + "findings": [ + "CVE-2099-9999" + ], + "estimates": [], + "reports": [ + { + "type": "Initial", + "at": "2026-11-02T10:30:00Z", + "set": { + "description": "Fixture description." + } + } + ] + } + ] +} diff --git a/tests/test_evidence_watchdog.py b/tests/test_evidence_watchdog.py index 0bdd0e27..9ae0184d 100644 --- a/tests/test_evidence_watchdog.py +++ b/tests/test_evidence_watchdog.py @@ -55,6 +55,7 @@ def fresh_objects(): node[leaf] = FRESH objects[a["path"]] = doc objects[".well-known/vdr-status.json"] = status() + objects[".well-known/incident-reports.json"]["obligations_due"] = [] return objects @@ -287,3 +288,39 @@ def test_alarm_thresholds_and_reads_come_from_the_governed_config(): # No window is a literal in the watchdog any more. assert "MAX_BEACON_AGE_HOURS" not in logic assert not re.search(r"threshold\s*=\s*(24|26)\b", tf) + + +# ---- incident report deadlines (IEC-CSO-IIR/OIR/FIR) -------------------------------- + +def _incident_log(*due): + return {"emitted_at": FRESH, "obligations_due": [{"incident": "INC-2026-009", "report": "Ongoing", + "due_at": d} for d in due]} + + +def test_no_report_due_pages_nobody(): + out = compute(fresh_objects()) + assert out["metrics"]["IncidentReportsOverdue"] == 0 + + +def test_a_report_past_its_deadline_pages_even_while_the_log_is_fresh(): + objects = fresh_objects() + # NOW is 12:00. One report due at 11:00 has not been issued; one due at 13:00 is not late yet. + objects[".well-known/incident-reports.json"] = _incident_log("2026-09-25T11:00:00Z", "2026-09-25T13:00:00Z") + out = compute(objects) + assert out["metrics"]["IncidentReportsOverdue"] == 1 + assert out["metrics"]["StaleArtifacts"] == 0 + + +def test_a_missing_or_malformed_incident_log_is_a_breach_not_zero(): + objects = fresh_objects() + objects[".well-known/incident-reports.json"] = {"emitted_at": FRESH} + assert compute(objects)["metrics"]["IncidentReportsOverdue"] == 1 + objects[".well-known/incident-reports.json"] = _incident_log("not a date") + assert compute(objects)["metrics"]["IncidentReportsOverdue"] == 1 + del objects[".well-known/incident-reports.json"] + assert compute(objects)["metrics"]["IncidentReportsOverdue"] == 1 + + +def test_the_incident_alarm_pages_on_the_first_overdue_report(): + tf = (REPO / "infrastructure" / "watchdog.tf").read_text() + assert 'metric = "IncidentReportsOverdue"\n threshold = 0' in tf diff --git a/tests/test_incidents.py b/tests/test_incidents.py new file mode 100644 index 00000000..5df3f472 --- /dev/null +++ b/tests/test_incidents.py @@ -0,0 +1,470 @@ +"""Incident evaluation and reporting (IEC-CSO-EFR/DPR/EFI/IIR/OIR/FIR/AIR, +VER-TFR-IRI, VER-TFR-NRI). + +- the register (data/incidents.json) loads, matches its schema, and every + known-bad register fails closed for its own reason; +- the Class C report clocks and the Incident Rating scale are read from the + pinned FedRAMP rules, and a changed rule fails closed; +- the clocks: default N5 until estimated (DPR), the IIR/OIR/FIR deadlines, + business days, overdue detection; +- the exercise runs end to end, IIR -> OIR -> FIR, every report validates + against FedRAMP's schema, restates every earlier item and was on time; +- a VDR finding that VER-TFR-IRI/NRI makes reportable with no incident is due + an Initial report, and the clock check fails once it is late; +- reconciliation invariant (m) and the unattended reporting workflow. +""" + +import copy +import hashlib +import importlib.util +import json +import re +import shutil +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import _fedramp_schema as fs # noqa: E402 +import _incidents # noqa: E402 +import _rules # noqa: E402 + +FIX = REPO / "tests" / "fixtures" / "incidents" +REGISTER = REPO / "data" / "incidents.json" +SCHEMA = REPO / "infrastructure" / "schemas" / "incidents.schema.json" +RULES_DIR = REPO / "infrastructure" / "schemas" / "fedramp-rules" +WF = REPO / ".github" / "workflows" +NOW = datetime(2026, 11, 3, tzinfo=timezone.utc) + + +def _load(name, rel): + spec = importlib.util.spec_from_file_location(name, REPO / rel) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +bir = _load("bir", "scripts/build-incident-report.py") +rec = _load("rec_incidents", "scripts/reconcile.py") + + +def ts(s): + return datetime.fromisoformat(s.replace("Z", "+00:00")) + + +def good(): + return json.loads((FIX / "good.json").read_text()) + + +# ---- the register ---------------------------------------------------------------- + +def test_the_committed_register_loads_and_matches_its_schema(): + _incidents.load(REGISTER, now=datetime.now(timezone.utc)) + jsonschema = pytest.importorskip("jsonschema") + jsonschema.validate(json.loads(REGISTER.read_text()), json.loads(SCHEMA.read_text())) + jsonschema.validate(good(), json.loads(SCHEMA.read_text())) + + +def test_the_committed_register_adopts_both_vulnerability_rules_and_publishes_only(): + """docs/policies/ir-policy.md adopts VER-TFR-NRI (a MAY) and IRI; nothing is sent.""" + p = json.loads(REGISTER.read_text())["policy"] + assert set(p["adopted_vulnerability_rules"]) == {"VER-TFR-IRI", "VER-TFR-NRI"} + assert p["notification"] == "published-only" + assert "not authorized" in p["notification_text"] + + +BAD = { + "not-evaluated": "evaluation must have exactly", + "empty-reason": "reason must be a non-empty string", + "rating-off-rubric": "is N3 under IEC-CSO-EFI", + "first-not-initial": "first report is the Initial Incident Report", + "initial-without-description": "must describe the incident", + "final-before-recovery": "needs resolved_at and recovery_complete_at", + "final-not-last": "nothing follows the Final Incident Report", + "not-reportable-with-report": "carries no rating or report", + "vuln-basis-not-adopted": "is not an adopted rule", + "vuln-basis-no-finding": "names no finding", + "evaluated-before-detected": "evaluated before it was detected", + "report-before-evaluation": "before the evaluation or the previous report", + "notification-sends": "no other delivery is built", + "personal-email": "names an email address outside", + "unknown-field": "unknown field", + "duplicate-id": "duplicate id", + "future-report": "records a time after now", +} + + +def test_every_known_bad_fixture_is_listed(): + assert {p.stem for p in (FIX / "bad").glob("*.json")} == set(BAD) + + +@pytest.mark.parametrize("name,why", sorted(BAD.items())) +def test_each_known_bad_register_fails_closed_for_its_reason(name, why): + _incidents.load(FIX / "good.json", now=NOW) # the base it was made from is valid + with pytest.raises(SystemExit) as exc: + _incidents.load(FIX / "bad" / f"{name}.json", now=NOW) + assert why in str(exc.value), str(exc.value) + + +@pytest.mark.parametrize("name", ["not-evaluated", "empty-reason", "notification-sends", "unknown-field"]) +def test_the_schema_refuses_the_structural_ones(name): + jsonschema = pytest.importorskip("jsonschema") + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(json.loads((FIX / "bad" / f"{name}.json").read_text()), json.loads(SCHEMA.read_text())) + + +def test_a_missing_or_unparseable_register_fails_closed(tmp_path): + with pytest.raises(SystemExit): + _incidents.load(tmp_path / "absent.json") + (tmp_path / "x.json").write_text("{") + with pytest.raises(SystemExit): + _incidents.load(tmp_path / "x.json") + + +# ---- rules as data -------------------------------------------------------------- + +def _corpus(): + return json.loads((RULES_DIR / "fedramp-consolidated-rules.json").read_text()) + + +def test_report_clocks_equal_the_pinned_rules(): + cso = _corpus()["FRR"]["IEC"]["data"]["all"]["CSO"] + got = _rules.incident_report_timeframes(RULES_DIR) + for kind, rid in (("iir", "IEC-CSO-IIR"), ("oir", "IEC-CSO-OIR"), ("fir", "IEC-CSO-FIR")): + table = cso[rid]["varies_by_class"]["c"]["pain_timeframes"] + assert got[kind] == {int(n): (c[kind]["timeframe_type"], c[kind]["timeframe_num"]) for n, c in table.items()} + # The Class C values the reports rely on, as published on 2026-06-24. + assert got["iir"][5] == ("hours", 1) and got["oir"][5] == ("hours", 6) and got["fir"][1] == ("bizdays", 1) + + +def test_rating_scale_and_default_come_from_the_rules(): + rubric = _rules.incident_rating_rubric(RULES_DIR) + assert rubric[("debilitating", "multiple")] == 5 and rubric[("disruptive", "multiple")] == 4 + assert rubric[("debilitating", "single")] == 4 and rubric[("minimal", "single")] == 1 + assert _rules.default_incident_rating(RULES_DIR) == 5 + + +def test_no_clock_is_typed_into_the_incident_code(): + for rel in ("scripts/_incidents.py", "scripts/build-incident-report.py"): + src = (REPO / rel).read_text() + assert not re.search(r"timedelta\(hours=(1|6|24)\)", src), rel + + +def _mutated_rules(tmp_path, mutate): + d = tmp_path / "rules" + d.mkdir() + corpus = _corpus() + mutate(corpus) + raw = json.dumps(corpus).encode() + (d / "fedramp-consolidated-rules.json").write_bytes(raw) + src = json.loads((RULES_DIR / "SOURCES.json").read_text()) + src["sha256"] = hashlib.sha256(raw).hexdigest() + (d / "SOURCES.json").write_text(json.dumps(src)) + return d + + +def test_a_changed_rating_scale_fails_closed(tmp_path): + def drop(c): + c["FRR"]["IEC"]["data"]["all"]["CSO"]["IEC-CSO-EFI"]["following_information_bullets"].pop() + with pytest.raises(SystemExit, match="does not rate every customer effect"): + _rules.incident_rating_rubric(_mutated_rules(tmp_path, drop)) + + +def test_a_missing_report_clock_fails_closed(tmp_path): + def drop(c): + c["FRR"]["IEC"]["data"]["all"]["CSO"]["IEC-CSO-OIR"]["varies_by_class"]["c"]["pain_timeframes"]["5"] = {} + with pytest.raises(SystemExit, match="IEC-CSO-OIR Class C N5 oir"): + _rules.incident_report_timeframes(_mutated_rules(tmp_path, drop)) + + +# ---- clocks (IEC-CSO-DPR, IIR, OIR, FIR) ------------------------------------------ + +TF = _rules.incident_report_timeframes(RULES_DIR) + + +def _open(evaluated="2026-11-02T10:00:00Z", reports=(), estimates=()): + inc = copy.deepcopy(good()["incidents"][0]) + inc["evaluation"]["evaluated_at"] = evaluated + inc["reports"] = list(reports) + inc["estimates"] = list(estimates) + return inc + + +def test_an_unrated_incident_is_n5_and_its_initial_report_is_due_in_an_hour(): + inc = _open() + assert _incidents.rating_at(inc, ts("2026-11-02T10:00:00Z"), 5) == (5, "default (IEC-CSO-DPR)") + (o,) = _incidents.obligations(inc, TF, 5, ts("2026-11-02T10:30:00Z")) + assert o["report"] == "Initial" and o["due_at"] == "2026-11-02T11:00:00+00:00" and not o["overdue"] + (o,) = _incidents.obligations(inc, TF, 5, ts("2026-11-02T11:01:00Z")) + assert o["overdue"] + + +def test_a_prompt_estimate_replaces_the_default_and_its_clock(): + est = {"estimated_at": "2026-11-06T10:00:00Z", "rating": 1, "customer_effect": "minimal", + "agencies": "single", "reason": "x"} + # Evaluated on a Friday, rated N1 at once: one business day, so Monday. + inc = _open(evaluated="2026-11-06T10:00:00Z", estimates=[est]) + (o,) = _incidents.obligations(inc, TF, 5, ts("2026-11-06T12:00:00Z")) + assert o["due_at"] == "2026-11-09T10:00:00+00:00" + + +def test_business_days_skip_the_weekend(): + assert _incidents.add_clock(ts("2026-11-06T23:00:00Z"), "bizdays", 1) == ts("2026-11-09T23:00:00Z") + assert _incidents.add_clock(ts("2026-11-04T08:00:00Z"), "bizdays", 1) == ts("2026-11-05T08:00:00Z") + + +def test_ongoing_reports_are_due_every_oir_period_until_resolved(): + reports = [{"type": "Initial", "at": "2026-11-02T10:30:00Z", "set": {"description": "d"}}] + obs = _incidents.obligations(_open(reports=reports), TF, 5, ts("2026-11-02T17:00:00Z")) + assert [o["report"] for o in obs] == ["Initial", "Ongoing"] + assert obs[1]["due_at"] == "2026-11-02T16:30:00+00:00" and obs[1]["overdue"] + + +def test_the_final_report_is_due_after_recovery_and_ongoing_reports_stop(): + inc = _open(reports=[{"type": "Initial", "at": "2026-11-02T10:30:00Z", "set": {"description": "d"}}]) + inc["resolved_at"], inc["recovery_complete_at"] = "2026-11-02T12:00:00Z", "2026-11-02T13:00:00Z" + obs = _incidents.obligations(inc, TF, 5, ts("2026-11-02T20:00:00Z")) + assert [o["report"] for o in obs] == ["Initial", "Final"] + assert obs[1]["due_at"] == "2026-11-02T19:00:00+00:00" and obs[1]["overdue"] + + +def test_a_not_reportable_incident_has_no_clock(): + inc = _open() + inc["evaluation"]["reportable"] = False + assert _incidents.obligations(inc, TF, 5, NOW) == [] + + +# ---- the exercise, end to end (IIR -> OIR -> FIR) --------------------------------- + +@pytest.fixture(scope="module") +def exercise(tmp_path_factory): + out = tmp_path_factory.mktemp("incident-reports") + reg = _incidents.load(REGISTER, now=ts("2026-11-05T19:00:00Z")) + log = bir.build(reg, None, ts("2026-11-05T19:00:00Z"), out) + return reg, log, out + + +def test_exercise_runs_initial_ongoing_final_all_on_time(exercise): + _, log, _ = exercise + inc = next(i for i in log["incidents"] if i["id"] == "INC-2026-001") + assert inc["exercise"] is True and inc["status"] == "closed" + assert [r["type"] for r in inc["reports"]] == ["Initial", "Ongoing", "Ongoing", "Final"] + assert [o["report"] for o in inc["obligations"]] == ["Initial", "Ongoing", "Ongoing", "Final"] + assert all(o["on_time"] for o in inc["obligations"]) + assert log["obligations_due"] == [] and log["clock_status"] == "ok" + + +def test_exercise_shows_the_default_rating_then_the_estimate(exercise): + _, log, out = exercise + initial = json.loads((out / "incident-INC-2026-001-01-initial.json").read_text()) + final = json.loads((out / "incident-INC-2026-001-04-final.json").read_text()) + assert initial["potentialImpact"]["currentRating"] == 5 + assert "IEC-CSO-DPR" in initial["potentialImpact"]["evaluationNotes"] + assert final["potentialImpact"]["historicalRating"] == [5, 3] and final["potentialImpact"]["currentRating"] == 3 + assert initial["incidentDescription"].startswith("EXERCISE") + + +def test_exercise_logs_a_not_reportable_evaluation_with_its_reason(exercise): + _, log, _ = exercise + inc = next(i for i in log["incidents"] if i["id"] == "INC-2026-002") + assert inc["status"] == "not-reportable" and inc["reports"] == [] and inc["rating"] is None + assert "not federal customer data" in inc["evaluation"]["reason"] + + +def test_every_report_validates_against_fedramps_schema_and_is_listed_with_its_hash(exercise): + _, log, out = exercise + files = sorted(p.name for p in out.glob("incident-INC-*.json")) + listed = [r for i in log["incidents"] for r in i["reports"]] + assert sorted(r["file"] for r in listed) == files + for r in listed: + doc = json.loads((out / r["file"]).read_text()) + assert fs.validate(doc, "incident") == [] + assert hashlib.sha256((out / r["file"]).read_bytes()).hexdigest() == r["sha256"] + + +def test_ongoing_and_final_reports_restate_every_earlier_item(exercise): + _, log, out = exercise + inc = next(i for i in log["incidents"] if i["id"] == "INC-2026-001") + first = json.loads((out / inc["reports"][0]["file"]).read_text()) + for r in inc["reports"][1:]: + doc = json.loads((out / r["file"]).read_text()) + assert set(first) <= set(doc), r["file"] + # Every item is either updated or explicitly "no update". + assert set(r["changes"].values()) <= {"updated", "no update"}, r["changes"] + assert inc["reports"][2]["changes"]["root_cause"] == "no update" + assert inc["reports"][2]["changes"]["response_and_recovery_activities"] == "updated" + final = json.loads((out / inc["reports"][-1]["file"]).read_text()) + assert final["reportType"] == "Final" and final["resolvedAt"] == "2026-11-05T17:00:00Z" + + +def test_reports_are_deterministic_and_say_they_are_published_only(exercise, tmp_path): + reg, log, out = exercise + again = bir.build(reg, None, ts("2026-11-05T19:30:00Z"), tmp_path) + for a, b in zip(log["incidents"], again["incidents"], strict=True): + assert [r["sha256"] for r in a["reports"]] == [r["sha256"] for r in b["reports"]] + md = (out / "incident-reports.md").read_text() + assert "published-only" in md and "not authorized" in md and "(no update)" in md + assert log["notification"]["mode"] == "published-only" + + +def test_the_cli_builds_and_checks(tmp_path): + assert bir.main(["--incidents", str(REGISTER), "--out-dir", str(tmp_path), + "--now", "2026-11-05T19:00:00Z"]) == 0 + assert bir.main(["--check", str(tmp_path / "incident-reports.json"), "--now", "2026-11-05T19:00:00Z"]) == 0 + + +# ---- vulnerabilities as incidents (VER-TFR-IRI / VER-TFR-NRI) ---------------------- + +pytestmark_opa = pytest.mark.skipif(shutil.which("opa") is None, reason="opa not available") + + +def _vdr(**patch): + f = {"tracking_id": "grype-CVE-2099-0777-api", "cve": "CVE-2099-0777", "pain": "N4", + "internet_reachable": True, "likely_exploitable": True, "fully_mitigated": False, + "final_disposition": "open", "first_evaluated": "2026-11-02T10:00:00+00:00"} + f.update(patch) + return {"emitted_at": "2026-11-02T12:00:00+00:00", "report_id": "r", "findings": [f]} + + +def _empty_register(adopted=("VER-TFR-IRI", "VER-TFR-NRI")): + reg = good() + reg["incidents"] = [] + reg["policy"]["adopted_vulnerability_rules"] = list(adopted) + return _incidents.validate(reg) + + +@pytestmark_opa +def test_an_irv_lev_n4_finding_with_no_incident_is_due_an_initial_report(tmp_path): + log = bir.build(_empty_register(), _vdr(), ts("2026-11-02T10:30:00Z"), tmp_path) + (t,) = log["vulnerability_triggers"] + assert t["rule"] == "VER-TFR-IRI" and t["adopted"] and t["incident"] is None + (d,) = log["obligations_due"] + assert d["report"] == "Initial" and d["due_at"] == "2026-11-02T11:00:00+00:00" and not d["overdue"] + assert bir.check(tmp_path / "incident-reports.json", ts("2026-11-02T10:59:00Z")) == 0 + assert bir.check(tmp_path / "incident-reports.json", ts("2026-11-02T11:01:00Z")) == 1 + + +@pytestmark_opa +def test_an_nirv_lev_n5_finding_opens_under_nri_and_n4_does_not(tmp_path): + log = bir.build(_empty_register(), _vdr(internet_reachable=False, pain="N5"), NOW, tmp_path) + assert log["vulnerability_triggers"][0]["rule"] == "VER-TFR-NRI" + log = bir.build(_empty_register(), _vdr(internet_reachable=False, pain="N4"), NOW, tmp_path) + assert log["vulnerability_triggers"] == [] + + +@pytestmark_opa +def test_a_partially_mitigated_finding_at_n3_is_no_longer_an_incident(tmp_path): + log = bir.build(_empty_register(), _vdr(pain="N3"), NOW, tmp_path) + assert log["vulnerability_triggers"] == [] and log["obligations_due"] == [] + + +@pytestmark_opa +def test_an_unadopted_rule_is_shown_but_starts_no_clock(tmp_path): + log = bir.build(_empty_register(adopted=("VER-TFR-IRI",)), _vdr(internet_reachable=False, pain="N5"), + NOW, tmp_path) + assert log["vulnerability_triggers"][0]["adopted"] is False and log["obligations_due"] == [] + + +@pytestmark_opa +def test_an_open_incident_naming_the_finding_covers_it(tmp_path): + reg = good() + reg["incidents"][0]["findings"] = ["CVE-2099-0777"] + log = bir.build(_incidents.validate(reg), _vdr(), ts("2026-11-02T10:40:00Z"), tmp_path) + assert log["vulnerability_triggers"][0]["incident"] == "INC-2026-101" + assert [d["incident"] for d in log["obligations_due"]] == ["INC-2026-101"] # its next Ongoing report + + +# ---- reconciliation (m) ---------------------------------------------------------- + +def _cr26_vdr(): + return json.loads((REPO / "tests" / "fixtures" / "cr26" / "vdr-report.json").read_text()) + + +def test_reconcile_m_passes_when_every_named_finding_is_in_the_vdr(): + assert rec.check_m_incident_findings(_cr26_vdr(), _incidents.validate(good())) == [] + + +def test_reconcile_m_fails_the_known_bad_register(): + reg = _incidents.load(FIX / "reconcile-m-bad.json") + v = rec.check_m_incident_findings(_cr26_vdr(), reg) + assert v == ["(m) incident INC-2026-101 names 'CVE-2099-9999', which is not in the VDR report"] + + +def test_reconcile_m_exempts_exercises_and_resolved_incidents(): + reg = json.loads((FIX / "reconcile-m-bad.json").read_text()) + reg["incidents"][0]["exercise"] = True + assert rec.check_m_incident_findings(_cr26_vdr(), _incidents.validate(reg)) == [] + reg["incidents"][0]["exercise"] = False + reg["incidents"][0]["resolved_at"] = "2026-11-02T11:00:00Z" + assert rec.check_m_incident_findings(_cr26_vdr(), _incidents.validate(reg)) == [] + + +def test_reconcile_m_holds_for_the_committed_register(): + assert rec.check_m_incident_findings(_cr26_vdr(), _incidents.load(REGISTER)) == [] + + +# ---- the unattended reporting workflow (IEC-CSO-AIR) -------------------------------- + +def _wf(): + return (WF / "incident-reports.yml").read_text() + + +def test_air_workflow_runs_on_register_change_by_hand_and_on_a_schedule(): + yaml = pytest.importorskip("yaml") + on = yaml.safe_load(_wf())[True] + assert on["push"]["paths"] == ["data/incidents.json"] and on["push"]["branches"] == ["main"] + assert "workflow_dispatch" in on and on["schedule"] + + +def test_air_workflow_validates_builds_signs_publishes_and_checks_the_clock_in_order(): + names = re.findall(r"^ - name: (.+)$", _wf(), re.M) + order = ["Fetch and verify the published VDR report", "Build incident reports (IIR, OIR, FIR)", + "Sign incident reports", "Publication redaction gate (no AWS account ID)", + "Publish incident reports to /.well-known/", + "Incident report clocks (fail when a report is due and missing)"] + assert [n for n in names if n in order] == order + + +def test_air_workflow_writes_only_incident_objects_and_sends_nothing(): + text = _wf() + keys = re.findall(r'--key "\.well-known/([^"]+)"', text) + assert keys == ["$1"] + puts = [p.strip('"') for p in re.findall(r"^\s*put (\S+) ", text, re.M)] + assert puts and all(p.startswith(("incident-", "$f", "${f")) for p in puts), puts + assert "for f in incident-INC-*.json; do" in text + assert "environment: incident-reports" in text and "AWS_INCIDENT_REPORTS_ROLE_ARN" in text + assert not re.search(r"\b(ses|sendmail|send-email|smtp)\b", text, re.I) + assert "aws s3 sync" not in text and "aws s3 rm" not in text and not re.search(r"^\s*terraform ", text, re.M) + + +def test_air_role_is_scoped_to_incident_objects(): + tf = (REPO / "infrastructure" / "bootstrap" / "incident-reports.tf").read_text() + assert '"arn:aws:s3:::${var.domain_name}/.well-known/incident-*"' in tf + actions = set(re.findall(r'"((?:s3|cloudfront):[A-Za-z*]+)"', tf)) + assert actions == {"s3:PutObject", "cloudfront:CreateInvalidation"} + assert 'environment:${var.incident_reports_environment}' in tf + + +def test_all_three_workflows_pin_the_same_opa(): + def pins(name): + text = (WF / name).read_text() + return {k: re.search(rf"^ {k}: (\S+)", text, re.M).group(1) for k in ("OPA_VERSION", "OPA_SHA256")} + assert pins("incident-reports.yml") == pins("deploy-with-opa.yml") == pins("evidence-nightly.yml") + + +# ---- the Ongoing Certification Report's incident summary (CCM-OCR-AVL item 7) ------- + +def test_ocr_attests_none_when_only_exercises_are_recorded(): + ocr = _load("ocr_inc", "scripts/build-ocr.py") + text = ocr.incident_summary(_incidents.load(REGISTER), "2026-08-01") + assert text.startswith("No FedRAMP Reportable Incidents occurred") and "incident-reports.json" in text + + +def test_ocr_lists_a_real_reportable_incident_in_the_period(): + ocr = _load("ocr_inc2", "scripts/build-ocr.py") + text = ocr.incident_summary(_incidents.validate(good()), "2026-08-01") + assert text.startswith("1 FedRAMP Reportable Incident(s) this period: INC-2026-101 (open)") + assert ocr.incident_summary(_incidents.validate(good()), "2026-11-03").startswith("No FedRAMP") diff --git a/tests/test_published_redaction.py b/tests/test_published_redaction.py index 53fb8e84..c63752c0 100644 --- a/tests/test_published_redaction.py +++ b/tests/test_published_redaction.py @@ -198,6 +198,14 @@ def test_deploy_gates_exactly_what_it_publishes(tmp_path): optional = ("boundary-map", "reconcile-report", "trust-center", "conformance-results", "boundary-corroboration") for a in optional: (tmp_path / f"{a}.json").write_text("{}") + # The incident reports are named by incident, so the steps use globs; with + # the files the builder writes present, both expand to the same set. + incident_files = ("incident-reports.json", "incident-reports.md", "incident-INC-2026-001-01-initial.json") + for f in incident_files: + (tmp_path / f).write_text("{}") + assert "incident-INC-*.json" in publish + publish = {n for n in publish if "*" not in n} | {f for f in incident_files if f.startswith("incident-INC-")} + assert {"incident-reports.json", "incident-reports.md"} <= publish assert _gate_files(gate_step, tmp_path) == publish # One absent (the operator-run corroboration): it is neither uploaded nor scanned. (tmp_path / "boundary-corroboration.json").unlink() From e56396610a05c4a8a54fbef6c0baf2366eb2952e Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 15:56:11 -0500 Subject: [PATCH 14/97] Incident reports: deploy preflight and proof of the publisher; records state the deployed system The deploy fails before applying anything unless AWS_INCIDENT_REPORTS_ROLE_ARN names the bootstrap incident-reports role and the incident-reports environment deploys only from main (scripts/check-incident-publisher.py; known-bad fixture with the variable missing). After the deploy job, the deploy calls incident-reports.yml (now also workflow_call) on its own role, passing only the CloudFront secret, so a green deploy proves the publisher publishes. IEC-CSO-IIR/OIR/FIR/AIR gaps now state only what holds after a successful deploy: published, not delivered to FedRAMP or agencies, by the operator's decision. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 35 ++++- .github/workflows/incident-reports.yml | 9 ++ data/frr-records.json | 18 ++- scripts/check-incident-publisher.py | 139 ++++++++++++++++++ tests/fixtures/incidents/preflight-good.json | 7 + .../incidents/preflight-missing-variable.json | 7 + tests/test_incidents.py | 92 ++++++++++++ 7 files changed, 300 insertions(+), 7 deletions(-) create mode 100644 scripts/check-incident-publisher.py create mode 100644 tests/fixtures/incidents/preflight-good.json create mode 100644 tests/fixtures/incidents/preflight-missing-variable.json diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 06b22820..7cf23ab4 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -612,6 +612,19 @@ jobs: with: fetch-depth: 0 + # The incident report records (IEC-CSO-IIR/OIR/FIR/AIR) describe the system + # after a successful deploy, which includes the unattended publisher in + # .github/workflows/incident-reports.yml. Fail before anything is applied if + # its prerequisites are missing: the role variable set and naming the + # bootstrap role, and the incident-reports environment main-only (checked + # when the job token can read it; said so when it cannot). The + # incident-publisher job below then proves the role publishes. + - name: Incident report publisher preflight + env: + INCIDENT_REPORTS_ROLE_ARN: ${{ vars.AWS_INCIDENT_REPORTS_ROLE_ARN }} + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/check-incident-publisher.py + - name: Setup Terraform uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 with: @@ -2249,11 +2262,27 @@ jobs: # which is functionally the same as no alert: nothing distinguishes the one # that matters. It now reuses a single issue, reopening and commenting, and # assigns it to the repository owner so it reaches a person. + # A deploy is not done until the unattended incident report publisher has run + # on its own role from the incident-reports environment: the IEC records say + # it publishes, so a deploy where it cannot fails here. + incident-publisher: + name: Prove the incident report publisher + needs: [deploy] + if: needs.deploy.result == 'success' + permissions: + contents: read + id-token: write + issues: write + uses: ./.github/workflows/incident-reports.yml + # Least privilege: the one secret the publisher uses, not `inherit`. + secrets: + CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }} + notify: name: Notify on Failure runs-on: ubuntu-latest - needs: [compliance-check, deploy] - if: always() && (needs.compliance-check.result == 'failure' || needs.deploy.result == 'failure') + needs: [compliance-check, deploy, incident-publisher] + if: always() && (needs.compliance-check.result == 'failure' || needs.deploy.result == 'failure' || needs.incident-publisher.result == 'failure') permissions: contents: read issues: write @@ -2267,6 +2296,7 @@ jobs: env: COMPLIANCE_RESULT: ${{ needs.compliance-check.result }} DEPLOY_RESULT: ${{ needs.deploy.result }} + PUBLISHER_RESULT: ${{ needs.incident-publisher.result }} with: script: | const TITLE = `Deployment failing: ${context.workflow}`; @@ -2282,6 +2312,7 @@ jobs: `| Ref | ${context.ref} |`, `| Compliance check | \`${process.env.COMPLIANCE_RESULT}\` |`, `| Deploy | \`${process.env.DEPLOY_RESULT}\` |`, + `| Incident report publisher | \`${process.env.PUBLISHER_RESULT}\` |`, ``, `Review the failing step in the run log, and the compliance reports it uploaded.`, ``, diff --git a/.github/workflows/incident-reports.yml b/.github/workflows/incident-reports.yml index fbf92b77..e8b2abee 100644 --- a/.github/workflows/incident-reports.yml +++ b/.github/workflows/incident-reports.yml @@ -41,6 +41,15 @@ on: branches: [main] paths: [data/incidents.json] workflow_dispatch: + # Called by every deploy (deploy-with-opa.yml, job incident-publisher) after + # the deploy job succeeds, so a green deploy proves this publisher assumes its + # role from the incident-reports environment and publishes; a deploy whose + # publisher cannot fails. + workflow_call: + # Only the secret the publish step uses; the caller passes it by name. + secrets: + CLOUDFRONT_DISTRIBUTION_ID: + required: true # Keeps the published log inside its 24-hour freshness window when nothing # changes, and re-reads the VDR for new VER-TFR-IRI/NRI triggers. schedule: diff --git a/data/frr-records.json b/data/frr-records.json index 4e1103cc..823b1694 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -659,7 +659,7 @@ { "frr_id": "IEC-CSO-AIR", "status": "Partially Implemented", - "implementation": ".github/workflows/incident-reports.yml runs on every push of data/incidents.json to main, by hand, twice a day and after each nightly VDR refresh: it validates the register, verifies the published VDR, builds every report against FedRAMP's schema, signs, redaction-gates and publishes under /.well-known/incident-* with its own least-privilege role (infrastructure/bootstrap/incident-reports.tf), and fails with a GitHub issue when a report is overdue; the evidence watchdog pages on the same deadlines hourly. A person records the facts in the register; nothing else is done by hand. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "implementation": ".github/workflows/incident-reports.yml runs on every push of data/incidents.json to main, by hand, twice a day and after each nightly VDR refresh: it validates the register, verifies the published VDR, builds every report against FedRAMP's schema, signs, redaction-gates and publishes under /.well-known/incident-* with its own least-privilege role (infrastructure/bootstrap/incident-reports.tf), and fails with a GitHub issue when a report is overdue; the evidence watchdog pages on the same deadlines hourly. A person records the facts in the register; nothing else is done by hand. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). Every deploy checks the publisher's prerequisites before it applies anything (scripts/check-incident-publisher.py: the role variable names the bootstrap role and the incident-reports environment deploys only from main) and, after the deploy job, runs this workflow on its own role (deploy-with-opa.yml job incident-publisher), so the deploy fails if the publisher cannot publish.", "evidence": [ { "kind": "step", @@ -669,6 +669,14 @@ "kind": "step", "ref": "incident-reports.yml#Incident report clocks (fail when a report is due and missing)" }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Incident report publisher preflight" + }, + { + "kind": "test", + "ref": "tests/test_incidents.py::test_deploy_proves_the_publisher_by_calling_it_after_the_deploy" + }, { "kind": "test", "ref": "tests/test_incidents.py::test_air_workflow_validates_builds_signs_publishes_and_checks_the_clock_in_order" @@ -678,7 +686,7 @@ "ref": ".well-known/incident-reports.json" } ], - "gap": "Not yet running: the role in infrastructure/bootstrap/incident-reports.tf is operator-applied, and the incident-reports GitHub Environment and AWS_INCIDENT_REPORTS_ROLE_ARN variable must be set. Reporting reaches no affected party automatically because delivery is publish-only by the operator's choice. To close: apply the bootstrap role and set the environment and variable; then the operator's delivery decision (IEC-CSO-IIR).", + "gap": "Everything up to publication is automated; what is left is the last step of the rule, reaching all affected parties. Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). A person still records the incident's facts in the register, which the rule allows (it asks to minimize human intervention, not remove it). To close: the operator's delivery decision (IEC-CSO-IIR); the delivery is then built into this workflow.", "closes_with": [ "sam" ] @@ -762,7 +770,7 @@ "ref": ".well-known/incident-reports.json" } ], - "gap": "Published, not delivered (see IEC-CSO-IIR). To close: the operator's delivery decision, as for IEC-CSO-IIR.", + "gap": "Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). To close: the operator's delivery decision, as for IEC-CSO-IIR.", "closes_with": [ "sam" ] @@ -789,7 +797,7 @@ "ref": ".well-known/incident-reports.json" } ], - "gap": "Reports are published, not delivered: nothing notifies FedRAMP or an agency, and the operator has not decided to send. The unattended publisher needs infrastructure/bootstrap/incident-reports.tf applied and the AWS_INCIDENT_REPORTS_ROLE_ARN variable and incident-reports environment set before it runs. To close: the operator decides the delivery channel (e.g. SES from the system's domain to FedRAMP's incident address) and it is built behind that decision.", + "gap": "Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). To close: the operator decides a delivery channel (e.g. SES from the system's domain to FedRAMP's incident address) and it is built behind that decision.", "closes_with": [ "sam" ] @@ -812,7 +820,7 @@ "ref": ".well-known/incident-reports.json" } ], - "gap": "Published, not delivered (see IEC-CSO-IIR). To close: the operator's delivery decision, as for IEC-CSO-IIR.", + "gap": "Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). To close: the operator's delivery decision, as for IEC-CSO-IIR.", "closes_with": [ "sam" ] diff --git a/scripts/check-incident-publisher.py b/scripts/check-incident-publisher.py new file mode 100644 index 00000000..e31d3cfa --- /dev/null +++ b/scripts/check-incident-publisher.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Deploy preflight: the unattended incident report publisher is in place. + +The FedRAMP incident report records (IEC-CSO-IIR/OIR/FIR/AIR) describe the +system as it is after a successful deploy, so a deploy must not succeed +without the prerequisites they rely on. Run early in the deploy job, before +anything is applied, this fails the deploy unless: + + 1. the repository variable AWS_INCIDENT_REPORTS_ROLE_ARN is set, and + 2. it names the role infrastructure/bootstrap/incident-reports.tf creates + (github-actions-incident-reports-oidc, an IAM role ARN), and + 3. the `incident-reports` GitHub Environment exists and deploys only from + main (a custom branch policy naming exactly `main`). + +Check 3 reads GitHub's API with the job token. A missing environment (404) +fails. If the token cannot read the environment at all (any other error), the +check is reported as not observed and the deploy continues on checks 1-2: +the deploy's later incident-publisher job assumes the role from that +environment, so a wrong environment still fails the deploy there. + +Whether the role itself works is proved later in the same run: the +`incident-publisher` job calls .github/workflows/incident-reports.yml, which +assumes the role, publishes, and fails the deploy if it cannot. + +Prints no ARN or account ID. Exit 0 pass, 1 fail. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.request +from typing import Any + +EXPECTED_ROLE_NAME = "github-actions-incident-reports-oidc" +ENVIRONMENT = "incident-reports" +EXPECTED_BRANCHES = {"main"} +ROLE_ARN_RE = re.compile(r"^arn:aws(-[a-z]+)*:iam::[0-9]{12}:role/(?:[\w+=,.@-]+/)*([\w+=,.@-]+)$") + + +def check_variable(role_arn: str | None) -> list[str]: + if not role_arn or not role_arn.strip(): + return ["the repository variable AWS_INCIDENT_REPORTS_ROLE_ARN is not set: apply " + "infrastructure/bootstrap/incident-reports.tf and set it from the incident_reports_role_arn output"] + m = ROLE_ARN_RE.match(role_arn.strip()) + if not m: + return ["AWS_INCIDENT_REPORTS_ROLE_ARN is not an IAM role ARN"] + if m.group(2) != EXPECTED_ROLE_NAME: + return [f"AWS_INCIDENT_REPORTS_ROLE_ARN names a role other than {EXPECTED_ROLE_NAME}"] + return [] + + +def check_environment(env: dict[str, Any] | None, branch_policies: list[str] | None) -> tuple[list[str], str]: + """(problems, note). env None = the environment does not exist.""" + if env is None: + return [f"the {ENVIRONMENT} GitHub Environment does not exist"], "" + policy = env.get("deployment_branch_policy") or {} + if not policy.get("custom_branch_policies") or policy.get("protected_branches"): + return [f"the {ENVIRONMENT} environment does not restrict deployments to a custom branch list"], "" + if branch_policies is None: + return [], "branch policies not observed" + if set(branch_policies) != EXPECTED_BRANCHES: + return [f"the {ENVIRONMENT} environment deploys from {sorted(branch_policies) or 'no branch'}, " + "not only main"], "" + return [], f"{ENVIRONMENT} deploys only from main" + + +def _get(url: str, token: str | None) -> Any: + if not url.startswith("https://"): + raise ValueError("refusing non-HTTPS URL") + headers = {"Accept": "application/vnd.github+json", "User-Agent": "samaydlette-preflight"} + if token: + headers["Authorization"] = f"Bearer {token}" + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=15) as r: # noqa: S310 + return json.loads(r.read().decode()) + + +def read_environment(repo: str, token: str | None) -> tuple[str, dict[str, Any] | None, list[str] | None]: + """("ok"|"missing"|"unreadable", environment, branch policy names).""" + api = f"https://api.github.com/repos/{repo}/environments/{ENVIRONMENT}" + try: + env = _get(api, token) + except urllib.error.HTTPError as exc: + return ("missing" if exc.code == 404 else "unreadable"), None, None + except Exception: # noqa: BLE001 (any other failure: not observed) + return "unreadable", None, None + try: + pols = _get(f"{api}/deployment-branch-policies", token) + names = [p.get("name", "") for p in pols.get("branch_policies", [])] + except Exception: # noqa: BLE001 + names = None + return "ok", env, names + + +def evaluate(role_arn: str | None, env_state: str, env: dict[str, Any] | None, + branch_policies: list[str] | None) -> tuple[list[str], list[str]]: + """(problems, notes) for one run; pure, for tests.""" + problems = check_variable(role_arn) + notes = [] + if env_state == "unreadable": + notes.append(f"the {ENVIRONMENT} environment could not be read with the job token; not observed " + "(the incident-publisher job still fails the deploy if the role cannot be assumed from it)") + else: + p, note = check_environment(env if env_state == "ok" else None, branch_policies) + problems += p + if note: + notes.append(note) + return problems, notes + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--fixture", default=None, + help="JSON {role_arn, env_state, environment, branch_policies} instead of live reads (tests)") + a = ap.parse_args(argv) + if a.fixture: + with open(a.fixture) as fh: + fx = json.load(fh) + problems, notes = evaluate(fx.get("role_arn"), fx["env_state"], fx.get("environment"), + fx.get("branch_policies")) + else: + state, env, pols = read_environment(os.environ.get("GITHUB_REPOSITORY", "sam-aydlette/samaydlette.com"), + os.environ.get("GITHUB_TOKEN")) + problems, notes = evaluate(os.environ.get("INCIDENT_REPORTS_ROLE_ARN"), state, env, pols) + for n in notes: + print(f"incident publisher preflight: {n}") + for p in problems: + print(f"::error::incident publisher preflight: {p}") + if not problems: + print("incident publisher preflight: role variable names the incident-reports role; ok") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/incidents/preflight-good.json b/tests/fixtures/incidents/preflight-good.json new file mode 100644 index 00000000..c7738649 --- /dev/null +++ b/tests/fixtures/incidents/preflight-good.json @@ -0,0 +1,7 @@ +{ + "_comment": "Good: variable names the incident-reports role (example account), environment deploys only from main.", + "role_arn": "arn:aws:iam::000000000000:role/github-actions-incident-reports-oidc", + "env_state": "ok", + "environment": {"name": "incident-reports", "deployment_branch_policy": {"protected_branches": false, "custom_branch_policies": true}}, + "branch_policies": ["main"] +} diff --git a/tests/fixtures/incidents/preflight-missing-variable.json b/tests/fixtures/incidents/preflight-missing-variable.json new file mode 100644 index 00000000..6ce86a33 --- /dev/null +++ b/tests/fixtures/incidents/preflight-missing-variable.json @@ -0,0 +1,7 @@ +{ + "_comment": "Known-bad: the repository variable AWS_INCIDENT_REPORTS_ROLE_ARN is unset; the environment is correct. check-incident-publisher.py must fail for the variable alone.", + "role_arn": "", + "env_state": "ok", + "environment": {"name": "incident-reports", "deployment_branch_policy": {"protected_branches": false, "custom_branch_policies": true}}, + "branch_policies": ["main"] +} diff --git a/tests/test_incidents.py b/tests/test_incidents.py index 5df3f472..36cc70ba 100644 --- a/tests/test_incidents.py +++ b/tests/test_incidents.py @@ -455,6 +455,98 @@ def pins(name): assert pins("incident-reports.yml") == pins("deploy-with-opa.yml") == pins("evidence-nightly.yml") +# ---- deploy preflight and proof of the publisher (the records hold once deployed) ---- + +preflight = _load("preflight", "scripts/check-incident-publisher.py") +ENV_OK = {"name": "incident-reports", + "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}} +ARN_OK = "arn:aws:iam::000000000000:role/github-actions-incident-reports-oidc" + + +def test_preflight_role_name_is_the_bootstrap_role(): + tf = (REPO / "infrastructure" / "bootstrap" / "incident-reports.tf").read_text() + assert f'"{preflight.EXPECTED_ROLE_NAME}"' in tf + assert f"environment: {preflight.ENVIRONMENT}" in _wf() + + +def test_preflight_passes_a_set_variable_and_a_main_only_environment(): + assert preflight.evaluate(ARN_OK, "ok", ENV_OK, ["main"]) == ([], ["incident-reports deploys only from main"]) + assert preflight.main(["--fixture", str(FIX / "preflight-good.json")]) == 0 + + +def test_preflight_fails_the_known_bad_fixture_for_the_missing_variable(capsys): + fx = json.loads((FIX / "preflight-missing-variable.json").read_text()) + problems, _ = preflight.evaluate(fx["role_arn"], fx["env_state"], fx["environment"], fx["branch_policies"]) + assert len(problems) == 1 and "AWS_INCIDENT_REPORTS_ROLE_ARN is not set" in problems[0] + assert preflight.main(["--fixture", str(FIX / "preflight-missing-variable.json")]) == 1 + assert "::error::" in capsys.readouterr().out + + +@pytest.mark.parametrize("arn,why", [ + (None, "is not set"), + ("arn:aws:iam::000000000000:role/github-actions-oidc", "names a role other than"), + ("arn:aws:iam::000000000000:user/github-actions-incident-reports-oidc", "is not an IAM role ARN"), + ("github-actions-incident-reports-oidc", "is not an IAM role ARN"), +]) +def test_preflight_fails_a_variable_that_does_not_name_the_role(arn, why): + problems, _ = preflight.evaluate(arn, "ok", ENV_OK, ["main"]) + assert len(problems) == 1 and why in problems[0] + + +def test_preflight_never_prints_the_arn_or_account(capsys, tmp_path): + fx = tmp_path / "fx.json" + fx.write_text(json.dumps({"role_arn": "arn:aws:iam::123456789012:role/other", "env_state": "ok", + "environment": ENV_OK, "branch_policies": ["main"]})) + preflight.main(["--fixture", str(fx)]) + assert "123456789012" not in capsys.readouterr().out + + +@pytest.mark.parametrize("state,env,branches,why", [ + ("missing", None, None, "does not exist"), + ("ok", {"deployment_branch_policy": None}, None, "custom branch list"), + ("ok", {"deployment_branch_policy": {"protected_branches": True, "custom_branch_policies": False}}, None, + "custom branch list"), + ("ok", ENV_OK, ["main", "feat/*"], "not only main"), + ("ok", ENV_OK, [], "not only main"), +]) +def test_preflight_fails_a_missing_or_unrestricted_environment(state, env, branches, why): + problems, _ = preflight.evaluate(ARN_OK, state, env, branches) + assert len(problems) == 1 and why in problems[0] + + +def test_preflight_says_so_when_the_environment_cannot_be_read_and_checks_the_variable(): + problems, notes = preflight.evaluate(ARN_OK, "unreadable", None, None) + assert problems == [] and "not observed" in notes[0] + problems, _ = preflight.evaluate("", "unreadable", None, None) + assert problems and "is not set" in problems[0] + + +def _deploy(): + return (WF / "deploy-with-opa.yml").read_text() + + +def test_deploy_runs_the_preflight_before_anything_is_applied(): + yaml = pytest.importorskip("yaml") + steps = yaml.safe_load(_deploy())["jobs"]["deploy"]["steps"] + names = [st.get("name") for st in steps] + i = names.index("Incident report publisher preflight") + assert i < names.index("Configure AWS credentials") < names.index("Terraform Apply") + assert "INCIDENT_REPORTS_ROLE_ARN: ${{ vars.AWS_INCIDENT_REPORTS_ROLE_ARN }}" in _deploy() + assert "run: python3 scripts/check-incident-publisher.py" in _deploy() + + +def test_deploy_proves_the_publisher_by_calling_it_after_the_deploy(): + yaml = pytest.importorskip("yaml") + jobs = yaml.safe_load(_deploy())["jobs"] + job = jobs["incident-publisher"] + assert job["uses"] == "./.github/workflows/incident-reports.yml" and job["needs"] == ["deploy"] + assert job["permissions"]["id-token"] == "write" + assert job["secrets"] == {"CLOUDFRONT_DISTRIBUTION_ID": "${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }}"} + assert list(yaml.safe_load(_wf())[True]["workflow_call"]["secrets"]) == ["CLOUDFRONT_DISTRIBUTION_ID"] + assert "incident-publisher" in jobs["notify"]["needs"] + assert "needs.incident-publisher.result == 'failure'" in jobs["notify"]["if"] + assert "workflow_call" in yaml.safe_load(_wf())[True] + # ---- the Ongoing Certification Report's incident summary (CCM-OCR-AVL item 7) ------- def test_ocr_attests_none_when_only_exercises_are_recorded(): From 15d59e73f3c5db89643062bb0c95b99b97196d43 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:35:50 -0500 Subject: [PATCH 15/97] FedRAMP Security Inbox, simulated (AFC-CSO-*) The inbox is simulated for this proof of concept by the operator's decision: nothing is received or sent. scripts/_fsi.py is the inbox logic (classification, designators, Class C deadline, register, header-safe acknowledgement sent once, paging). Every deploy runs it over committed synthetic messages before the SDR build, fails on any unexpected outcome, and publishes the signed, dated result as fedramp-inbox-simulation.json, labelled as a simulation. The published contact says so and points to LinkedIn; the AFC-CSO-NOC check applies to that contact. AFC records are Implemented with simulated {counterparty: mail}. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 38 +- data/cds/offering.json | 14 + data/frr-records.json | 262 +++++++++--- data/fsi-simulation/messages.json | 366 +++++++++++++++++ docs/cr26-applicability.md | 5 +- docs/incident-response.md | 2 +- docs/policies/ir-policy.md | 2 +- docs/runbooks/fedramp-security-inbox.md | 78 ++++ infrastructure/schemas/freshness-config.json | 12 + scripts/_fsi.py | 322 +++++++++++++++ scripts/build-cpo.py | 17 + scripts/build-fsi-simulation.py | 221 +++++++++++ scripts/check-fsi-notice.py | 123 ++++++ .../fsi/offering-changed-with-notice.json | 21 + .../fsi/offering-changed-without-history.json | 13 + .../fsi/offering-changed-without-notice.json | 17 + .../fsi/simulation-wrong-expectation.json | 366 +++++++++++++++++ tests/test_fedramp_inbox.py | 374 ++++++++++++++++++ tests/test_frr_records.py | 6 +- tests/test_fsi_notice.py | 94 +++++ 20 files changed, 2291 insertions(+), 62 deletions(-) create mode 100644 data/fsi-simulation/messages.json create mode 100644 docs/runbooks/fedramp-security-inbox.md create mode 100644 scripts/_fsi.py create mode 100644 scripts/build-fsi-simulation.py create mode 100644 scripts/check-fsi-notice.py create mode 100644 tests/fixtures/fsi/offering-changed-with-notice.json create mode 100644 tests/fixtures/fsi/offering-changed-without-history.json create mode 100644 tests/fixtures/fsi/offering-changed-without-notice.json create mode 100644 tests/fixtures/fsi/simulation-wrong-expectation.json create mode 100644 tests/test_fedramp_inbox.py create mode 100644 tests/test_fsi_notice.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 7cf23ab4..2af5f99a 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -249,6 +249,24 @@ jobs: - name: Exception register expiry check run: python3 scripts/check-exceptions.py + # AFC-CSO-NOC: the FedRAMP Security Inbox contact (data/cds/offering.json) + # cannot change without a history entry recording that FedRAMP was + # notified. Compared with the merge base on a pull request and with the + # previous commit on a push, so the history is also append-only. + - name: FedRAMP Security Inbox change notice (AFC-CSO-NOC) + env: + BASE_REF: ${{ github.event_name == 'pull_request' && github.base_ref || '' }} + BEFORE_SHA: ${{ github.event_name == 'push' && github.event.before || '' }} + run: | + base="" + if [ -n "$BASE_REF" ] && git fetch --no-tags --depth=1 origin "$BASE_REF"; then + git show FETCH_HEAD:data/cds/offering.json > fsi-base.json 2>/dev/null && base="--base fsi-base.json" + elif [ -n "$BEFORE_SHA" ] && git fetch --no-tags --depth=1 origin "$BEFORE_SHA"; then + git show "$BEFORE_SHA":data/cds/offering.json > fsi-base.json 2>/dev/null && base="--base fsi-base.json" + fi + # shellcheck disable=SC2086 # $base is an optional flag pair + python3 scripts/check-fsi-notice.py $base + # Known Go-vs-Wasm builtin divergences (%q, %v-on-sets, host time) are # banned at lint time; the parity test catches what the corpus exercises, # this catches the rest before it exists. It also requires every sprintf @@ -1617,6 +1635,19 @@ jobs: --previous previous-availability.json --ksi-signal ksi-signal.json python3 ../scripts/build-policy-index.py --output policy-index.json --md-output policy-index.md + # SIMULATION. The FedRAMP Security Inbox is simulated for this proof of + # concept (operator's decision): nothing real is received or sent. This + # runs the inbox logic (scripts/_fsi.py) over committed synthetic messages + # and fails the deploy if any outcome differs from the expected one + # (AFC-CSO-INB/TFG/RCV/CRA/EMR/IMA/ACK, simulated). + - name: FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent) + working-directory: infrastructure + run: | + python3 ../scripts/build-fsi-simulation.py \ + --messages ../data/fsi-simulation/messages.json \ + --offering ../data/cds/offering.json \ + --output fedramp-inbox-simulation.json + - name: Build Security Decision Record (SDR — 20x SSP replacement) working-directory: infrastructure run: | @@ -1744,6 +1775,8 @@ jobs: run: | echo "Signing ongoing-certification-report.json with cosign keyless..." cosign sign-blob --yes --new-bundle-format --bundle ongoing-certification-report.bundle ongoing-certification-report.json + echo "Signing fedramp-inbox-simulation.json (SIMULATION) with cosign keyless..." + cosign sign-blob --yes --new-bundle-format --bundle fedramp-inbox-simulation.bundle fedramp-inbox-simulation.json - name: Sign CDS certification-data artifacts working-directory: infrastructure @@ -1784,7 +1817,7 @@ jobs: files="ksi-signal.json schemas/ksi-signal.schema.json vdr-report.json vdr-report.md vdr-trend.json iiw.csv" for a in vdr-detail-report vdr-accepted-vulnerabilities vdr-history; do files="$files $a.json"; done for a in vdr-detail-report vdr-accepted-vulnerabilities; do files="$files $a.md"; done - files="$files ongoing-certification-report.json ongoing-certification-report.md" + files="$files ongoing-certification-report.json ongoing-certification-report.md fedramp-inbox-simulation.json" for a in certification-overview-package availability policy-index security-decision-record certification-package-overview; do files="$files $a.json $a.md"; done for a in significant-change-notifications scn-evaluations; do files="$files $a.json $a.md"; done files="$files significant-change-notifications.atom" @@ -1843,6 +1876,8 @@ jobs: aws s3 cp ongoing-certification-report.json "s3://$BUCKET_NAME/.well-known/ongoing-certification-report.json" --content-type application/json --cache-control "public, max-age=300" aws s3 cp ongoing-certification-report.bundle "s3://$BUCKET_NAME/.well-known/ongoing-certification-report.bundle" --content-type application/json --cache-control "public, max-age=300" aws s3 cp ongoing-certification-report.md "s3://$BUCKET_NAME/.well-known/ongoing-certification-report.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" + aws s3 cp fedramp-inbox-simulation.json "s3://$BUCKET_NAME/.well-known/fedramp-inbox-simulation.json" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp fedramp-inbox-simulation.bundle "s3://$BUCKET_NAME/.well-known/fedramp-inbox-simulation.bundle" --content-type application/json --cache-control "public, max-age=300" # CDS certification-data artifacts (overview package, availability feed, policy index) + bundles + human-readable for a in certification-overview-package availability policy-index security-decision-record certification-package-overview; do aws s3 cp "$a.json" "s3://$BUCKET_NAME/.well-known/$a.json" --content-type application/json --cache-control "public, max-age=300" @@ -1950,6 +1985,7 @@ jobs: check "$a.json" ".well-known/$a.json" done check ongoing-certification-report.json .well-known/ongoing-certification-report.json + check fedramp-inbox-simulation.json .well-known/fedramp-inbox-simulation.json check certification-overview-package.json .well-known/certification-overview-package.json check availability.json .well-known/availability.json check policy-index.json .well-known/policy-index.json diff --git a/data/cds/offering.json b/data/cds/offering.json index 27be34e5..ce1c5cd5 100644 --- a/data/cds/offering.json +++ b/data/cds/offering.json @@ -7,6 +7,20 @@ "uei": "", "sales_contact": { "name": "Sam Aydlette", "url": "https://samaydlette.com/pages/contact.html" }, "security_contact": { "name": "Sam Aydlette", "email": "sam.aydlette@gmail.com", "source": "/.well-known/security.txt" }, + "fedramp_security_inbox": { + "_note": "The FedRAMP Security Inbox contact (AFC-CSO-INB), written in this one place. By the operator's decision the inbox is SIMULATED for this proof of concept: the inbox logic (scripts/_fsi.py) runs on every deploy over synthetic messages and publishes /.well-known/fedramp-inbox-simulation.json, and nothing real is received or sent. The CPO publishes this contact, and scripts/check-fsi-notice.py fails any change to it that does not record notifying FedRAMP (AFC-CSO-NOC). To change it: add a history entry with fedramp_notified, then change contact.", + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox, no mail is received or sent, and the inbox is permanently at zero because it does not exist. Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/", + "simulation": "https://samaydlette.com/.well-known/fedramp-inbox-simulation.json", + "runbook": "https://github.com/sam-aydlette/samaydlette.com/blob/main/docs/runbooks/fedramp-security-inbox.md", + "history": [ + { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox, no mail is received or sent, and the inbox is permanently at zero because it does not exist. Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/", + "since": "2026-11-05", + "fedramp_notified": null, + "basis": "Established. No earlier FedRAMP Security Inbox or Marketplace Security Email is on file with FedRAMP for this unsponsored system, so there was no prior contact to change from." + } + ] + }, "product_website": "https://samaydlette.com", "logo_url": "https://samaydlette.com/assets/images/brand/logo.png", "service_description": "A static personal website on AWS that doubles as a working continuous-compliance pipeline: every deploy derives signed, machine-readable compliance artifacts from one canonical inventory and publishes them under /.well-known/. Includes the Silk Reeling Mirror application, a Cognito-gated Tai Chi coaching API.", diff --git a/data/frr-records.json b/data/frr-records.json index 823b1694..2ebc5672 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -5,83 +5,235 @@ "records": [ { "frr_id": "AFC-CSO-ACK", - "status": "Not Implemented", - "implementation": "No FSI; no auto-responder.", - "evidence": [], - "gap": "No prompt automatic acknowledgment of FedRAMP messages. To close: Configure an auto-acknowledgment for mail from @fedramp.gov/@gsa.gov (e.g. SES receipt rule -> Lambda reply, or provider auto-reply rule) and include it in the nightly configuration check.", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). The acknowledgement logic (scripts/_fsi.py, build_ack and receive) acknowledges each FedRAMP message once, even when it is redelivered, with headers inbound content cannot inject, and never to auto-replies, bulk mail, bounces, mail without a From address, or mail failing SPF/DKIM/DMARC (FedRAMP's own mail passes them, AFC-FRP-VRE; this keeps a forged sender from using the inbox as a reflector; such mail is still registered and paged). In the simulation each acknowledgement is built and checked for header safety, then filed nowhere. Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_nothing_from_the_inbound_message_can_inject_a_header" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_a_redelivered_message_is_acknowledged_and_paged_once" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_unauthenticated_fedramp_mail_is_registered_and_paged_but_not_acknowledged" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_no_acknowledgement_to_auto_replies_bulk_mail_or_bounces" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "AFC-CSO-CRA", - "status": "Not Implemented", - "implementation": "No FSI, no Emergency/Emergency Test handling runbook in docs/incident-response.md or ir-policy.md.", - "evidence": [], - "gap": "No procedure to recognize Emergency / Emergency Test designated messages, extract the required actions and timeframe, and complete them; no record of completion. To close: Write an FSI runbook (classify by designation, open a tracked item with the message's deadline, complete, reply with completion) and wire Emergency-designated mail to page the operator; keep a completion log. Rehearse with a self-generated Emergency Test.", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). Emergency and Emergency Test messages (subject designator, AFC-FRP-CDS) get a register item with a deadline: by default 3:00 p.m. ET on the 2nd business day (AFC-FRP-ERT, Class C; the receipt day counts and holidays are not skipped, both erring earlier), replaced at triage by the timeframe the message states; the register reports an item past its deadline until it is closed (scripts/_fsi.py, deadlines, triage, close, overdue). In the simulation the Emergency Test received on a Friday is due 3:00 p.m. ET the following Monday, is reported once that passes, and stops being reported when closed. Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_class_c_default_deadline_is_3pm_eastern_on_the_second_business_day" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_emergency_records_carry_a_deadline_and_important_ones_ask_for_the_messages_own" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_overdue_counts_untriaged_unclosed_and_failed_legs" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "AFC-CSO-EMR", - "status": "Not Implemented", - "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", - "evidence": [], - "gap": "No routing of Emergency-designated messages to a senior security official; no mailbox configuration or automated validation (both artifacts missing). To close: Add a mail rule that routes subject-designated 'Emergency' FedRAMP messages to the senior security official (the accountable official in the CPO, plus SNS page), and include it in the nightly mailbox-configuration check.", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). Every FedRAMP message, Emergency-designated ones included, produces a page to the senior security official (the accountable official named in the CPO, the operator) naming the designation and the sender's domain (scripts/_fsi.py, receive and page_subject); an Emergency from an address other than FedRAMP's designated Emergency senders (AFC-FRP-UFS) is flagged, not demoted. In the simulation the page is recorded, not delivered: the operator's phone stays blissfully quiet. Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_an_emergency_from_an_address_other_than_fedramp_security_is_flagged_not_demoted" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_the_page_subject_is_ascii_and_names_the_domain_not_the_address" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "AFC-CSO-IMA", - "status": "Not Implemented", - "implementation": "No FSI; no handling procedure for Important-designated messages.", - "evidence": [], - "gap": "No procedure to complete actions in Important messages within their stated timeframe. To close: Extend the FSI runbook to Important messages with tracked deadlines (same tracker as CRA).", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). Important-designated messages open a register item that cannot be triaged without recording the timeframe the message states; the register reports it while untriaged after 24 hours, triaged without a timeframe, or past that timeframe and not closed (scripts/_fsi.py, triage and overdue). The simulation checks each of these. Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_an_important_message_cannot_be_triaged_without_its_stated_timeframe" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_overdue_counts_untriaged_unclosed_and_failed_legs" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "AFC-CSO-INB", - "status": "Not Implemented", - "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", - "evidence": [], - "gap": "No dedicated FedRAMP Security Inbox address exists or is designated; the required artifact (an email address to receive messages from FedRAMP) cannot be produced. To close: Create a dedicated address on the system's domain (e.g. fedramp-security@samaydlette.com: MX record in dns.tf plus SES inbound or a mail provider, with SPF/DKIM/DMARC), document it as the FSI in the CPO certification_data (security_contact block) and trust center, and monitor it (alarm on delivery failure).", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). The published FedRAMP Security Inbox contact (data/cds/offering.json, fedramp_security_inbox.contact, published in the CPO's certification_data) says, plainly, that the inbox does not exist and to hit the operator up on LinkedIn. The logic a real inbox would run receives, classifies, registers, acknowledges and pages. Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_the_simulation_passes_and_publishes_a_labelled_result_without_addresses" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_the_cpo_publishes_the_simulated_contact_not_an_email_address" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "AFC-CSO-NOC", - "status": "Not Implemented", - "implementation": "No FSI exists, so no change-notification process exists; nothing in docs/incident-response.md or docs/policies/ir-policy.md covers notifying FedRAMP of an inbox change.", - "evidence": [], - "gap": "No process (manual or automated) to notify FedRAMP when the FSI address changes. To close: Make the FSI address a single value in the canonical inventory and add a CI check that fails a change to it unless the commit carries a 'FedRAMP notified on via ' record; document the runbook step (email FedRAMP from the old and new address) in ir-policy.md.", - "closes_with": [ - "W9" + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). The published contact has one source (data/cds/offering.json, fedramp_security_inbox.contact) with an append-only history, and the notice rule applies to it unchanged. scripts/check-fsi-notice.py, run in the unit-test job on every pull request and push, fails a change to the contact unless a new history entry carries a dated fedramp_notified marker naming the channel (FedRAMP's '[CSP] Notification of Changes' form), and fails a rewritten history against the merge base. Known-bad fixtures in tests/fixtures/fsi/. Procedure: docs/runbooks/fedramp-security-inbox.md. The contact was established 2026-11-05 with nothing earlier on file with FedRAMP, so there has been no change to notify.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_fsi_notice.py::test_a_contact_change_without_the_fedramp_notified_marker_fails" + }, + { + "kind": "test", + "ref": "tests/test_fsi_notice.py::test_a_contact_change_without_a_history_entry_fails" + }, + { + "kind": "test", + "ref": "tests/test_fsi_notice.py::test_rewriting_history_fails_against_the_base" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox change notice (AFC-CSO-NOC)" + } ] }, { "frr_id": "AFC-CSO-RCV", - "status": "Not Implemented", - "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", - "evidence": [], - "gap": "Nothing receives FedRAMP mail without disruption (no inbox, no anti-spam allowlisting, no monitoring, no backup reader for a one-person operation). To close: After INB/TFG: allowlist FedRAMP senders, add delivery-failure and unread-age alarms (SNS to operator, reusing the watchdog path), and name a backup reader or an out-of-office auto-forward so a single absent operator does not disrupt receipt. Provider half is fully buildable; only an actual FedRAMP message demonstrates it end-to-end.", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). Nothing in the inbox logic asks anything of FedRAMP or drops FedRAMP mail: every FedRAMP message is registered and paged whatever its verdicts (the simulation fails the deploy otherwise), a failed acknowledgement is recorded, reported overdue and sent once on retry, and overdue items are reported by the register (scripts/_fsi.py, receive and overdue). Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_a_failed_acknowledgement_is_recorded_retried_and_counted_overdue" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_overdue_counts_untriaged_unclosed_and_failed_legs" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_each_register_check_fails_the_simulation_when_its_outcome_differs" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "AFC-CSO-TFG", - "status": "Not Implemented", - "implementation": "No inbox exists: no MX record in infrastructure/dns.tf, no mail infrastructure in infrastructure/*.tf; live /.well-known/security.txt Contact is a personal gmail address with no FSI designation; docs/incident-response.md:120 and docs/policies/ir-policy.md:9 declare FSI does not apply; docs/cr26-applicability.md:27 records AFC as N/A.", - "evidence": [], - "gap": "No mailbox configuration treating @fedramp.gov / @gsa.gov senders as FedRAMP, and no automated validation of that configuration (both listed artifacts missing). To close: In the FSI mail rules, tag/route all mail from @fedramp.gov and @gsa.gov as FedRAMP (never spam-filtered/quarantined); export the rule set as code and add a nightly check (evidence-nightly.yml) that reads the live rules and compares them with the expected configuration, published to the trust center like check-manual-settings.py does for GitHub settings.", - "closes_with": [ - "W9" - ] + "status": "Implemented", + "implementation": "SIMULATED FOR THIS PROOF OF CONCEPT: there is no FedRAMP Security Inbox, no mail is received or sent, and FedRAMP is directed to the operator on LinkedIn (https://www.linkedin.com/in/sa2/). Classification (scripts/_fsi.py, classify): a message whose From address or envelope sender is exactly @fedramp.gov or @gsa.gov is FedRAMP, whatever its spam verdict or SPF/DKIM/DMARC result; look-alike domains and display-name spoofs are not; failed authentication is recorded so the operator can confirm the sender, never used to filter. The simulation includes a forged @gsa.gov message (still FedRAMP), a FedRAMP envelope sender behind an unreadable From (still FedRAMP), a look-alike domain and a display-name spoof (neither is FedRAMP). Automated validation of the configuration is the deploy-time simulation itself. Every deploy runs this logic (scripts/_fsi.py) over committed synthetic messages (data/fsi-simulation/messages.json) in the step 'FedRAMP Security Inbox simulation', fails the deploy if any outcome differs from the expected one, and publishes the signed, dated result, labelled as a simulation, as /.well-known/fedramp-inbox-simulation.json.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + }, + { + "kind": "artifact", + "ref": ".well-known/fedramp-inbox-simulation.json" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_fedramp_senders_are_fedramp_even_when_unauthenticated" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_look_alike_domains_and_display_name_spoofs_are_not_fedramp" + }, + { + "kind": "test", + "ref": "tests/test_fedramp_inbox.py::test_a_fedramp_envelope_sender_counts_even_if_the_from_header_is_unparseable" + } + ], + "simulated": { + "counterparty": "mail", + "how": "The mail system and FedRAMP's messages are simulated: synthetic messages run through the real inbox logic on every deploy, outbound acknowledgements and pages are recorded, never sent, and the published contact is 'Hit me up on LinkedIn'." + } }, { "frr_id": "CCM-OCR-AFS", diff --git a/data/fsi-simulation/messages.json b/data/fsi-simulation/messages.json new file mode 100644 index 00000000..4e556857 --- /dev/null +++ b/data/fsi-simulation/messages.json @@ -0,0 +1,366 @@ +{ + "_note": "SIMULATION INPUT. Synthetic messages for the simulated FedRAMP Security Inbox (this proof of concept receives and sends no mail, by the operator's decision). scripts/build-fsi-simulation.py runs scripts/_fsi.py over these on every deploy and fails the deploy if any observed outcome differs from 'expected'. Addresses here are synthetic test values and are never published; the published result carries sender domains only.", + "inbox": "fedramp-security-inbox@simulation.invalid", + "register_check": { + "now": "2026-11-06T17:00:00Z", + "expected_overdue": [ + "sim-ack-send-fails", + "sim-emergency-other-sender", + "sim-emergency-test", + "sim-forged-gsa" + ], + "expected_retry": { + "acknowledgements_sent_again": 1, + "pages_sent_again": 0 + }, + "important_case": "sim-important", + "important_respond_by": "2026-11-10T22:00:00Z", + "later": "2026-11-09T21:00:00Z", + "emergency_case": "sim-emergency-test", + "untriaged_important_case": "sim-envelope-only" + }, + "messages": [ + { + "case": "emergency-test", + "description": "Emergency Test from FedRAMP's Emergency sender (AFC-FRP-UFS), authenticated", + "message_id": "sim-emergency-test", + "received_at": "2026-11-06T14:05:00Z", + "from": "FedRAMP Security ", + "envelope": "fedramp_security@fedramp.gov", + "subject": "EMERGENCY TEST: FedRAMP Security Inbox exercise", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "EMERGENCY_TEST", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": true, + "ack": "sent", + "in_reply_to": true, + "page": true, + "triage_by": "2026-11-06T16:05:00Z", + "respond_by": "2026-11-09T20:00:00Z" + } + }, + { + "case": "important", + "description": "Important message whose subject and Message-ID try to inject a Bcc header", + "message_id": "sim-important", + "received_at": "2026-11-06T14:10:00Z", + "from": "FedRAMP PMO ", + "envelope": "info@fedramp.gov", + "subject": "IMPORTANT: action requested\r\nBcc: injected@attacker.invalid", + "headers": { + "Message-ID": "\r\nBcc: injected@attacker.invalid" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "IMPORTANT", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "sent", + "in_reply_to": false, + "page": true, + "triage_by": "2026-11-07T14:10:00Z", + "respond_by": null + } + }, + { + "case": "general", + "description": "General communication from @gsa.gov, no designator", + "message_id": "sim-general", + "received_at": "2026-11-06T14:15:00Z", + "from": "info@gsa.gov", + "envelope": "info@gsa.gov", + "subject": "FedRAMP program update", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "sent", + "in_reply_to": true, + "page": true, + "triage_by": "2026-11-09T14:15:00Z", + "respond_by": null + } + }, + { + "case": "forged-gsa", + "description": "Forged Emergency claiming to be from the gsa.gov Emergency sender; SPF, DKIM and DMARC fail. Still FedRAMP by default (AFC-CSO-TFG) and paged, but not acknowledged", + "message_id": "sim-forged-gsa", + "received_at": "2026-11-06T14:20:00Z", + "from": "FedRAMP Security ", + "envelope": "bounce@attacker.invalid", + "subject": "EMERGENCY: reply with your credentials", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "FAIL", + "dkim": "FAIL", + "dmarc": "FAIL" + }, + "expected": { + "kind": "fedramp", + "designation": "EMERGENCY", + "sender_domain": "gsa.gov", + "authenticated": false, + "emergency_sender_ok": true, + "ack": "suppressed:unauthenticated", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-06T16:20:00Z", + "respond_by": "2026-11-09T20:00:00Z" + } + }, + { + "case": "auto-reply", + "description": "An auto-reply from @fedramp.gov: registered, never answered (no mail loop)", + "message_id": "sim-auto-reply", + "received_at": "2026-11-06T14:25:00Z", + "from": "info@fedramp.gov", + "envelope": "info@fedramp.gov", + "subject": "Automatic reply: Received: FedRAMP Security Inbox exercise", + "headers": { + "Auto-Submitted": "auto-replied", + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:auto-submitted", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T14:25:00Z", + "respond_by": null + } + }, + { + "case": "look-alike", + "description": "Emergency from a look-alike domain (fedramp.gov.attacker.invalid): not FedRAMP, exact domains only", + "message_id": "sim-look-alike", + "received_at": "2026-11-06T14:30:00Z", + "from": "fedramp_security@fedramp.gov.attacker.invalid", + "envelope": "fedramp_security@fedramp.gov.attacker.invalid", + "subject": "EMERGENCY: act now", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "other", + "designation": "GENERAL", + "sender_domain": "fedramp.gov.attacker.invalid", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "none", + "in_reply_to": null, + "page": false, + "triage_by": null, + "respond_by": null + } + }, + { + "case": "display-name-spoof", + "description": "Display name impersonates the gsa.gov Emergency sender, actual address does not: not FedRAMP", + "message_id": "sim-display-name-spoof", + "received_at": "2026-11-06T14:35:00Z", + "from": "\"fedramp_security@gsa.gov\" ", + "envelope": "attacker@attacker.invalid", + "subject": "EMERGENCY: trust me", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "other", + "designation": "GENERAL", + "sender_domain": "attacker.invalid", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "none", + "in_reply_to": null, + "page": false, + "triage_by": null, + "respond_by": null + } + }, + { + "case": "envelope-only", + "description": "Unreadable From header, @gsa.gov envelope sender: still FedRAMP, registered and paged; no reply without a From address", + "message_id": "sim-envelope-only", + "received_at": "2026-11-06T14:40:00Z", + "from": "garbage <<>>", + "envelope": "bounces@gsa.gov", + "subject": "Important: please confirm", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "IMPORTANT", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:no-from-address", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-07T14:40:00Z", + "respond_by": null + } + }, + { + "case": "emergency-other-sender", + "description": "Emergency from an @gsa.gov address other than the designated Emergency senders (AFC-FRP-UFS): flagged, not demoted, still paged", + "message_id": "sim-emergency-other-sender", + "received_at": "2026-11-06T14:45:00Z", + "from": "Someone ", + "envelope": "someone@gsa.gov", + "subject": "EMERGENCY: rotate your keys", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "EMERGENCY", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": false, + "ack": "sent", + "in_reply_to": true, + "page": true, + "triage_by": "2026-11-06T16:45:00Z", + "respond_by": "2026-11-09T20:00:00Z" + } + }, + { + "case": "bulk", + "description": "Bulk mail from @gsa.gov: registered, not answered", + "message_id": "sim-bulk", + "received_at": "2026-11-06T14:50:00Z", + "from": "news@gsa.gov", + "envelope": "news@gsa.gov", + "subject": "Newsletter", + "headers": { + "Precedence": "bulk" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:precedence", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T14:50:00Z", + "respond_by": null + } + }, + { + "case": "bounce", + "description": "A bounce (null envelope sender) from @gsa.gov: registered, not answered", + "message_id": "sim-bounce", + "received_at": "2026-11-06T14:55:00Z", + "from": "mailer-daemon@gsa.gov", + "envelope": "<>", + "subject": "Undeliverable: Received: FedRAMP program update", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:null-sender", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T14:55:00Z", + "respond_by": null + } + }, + { + "case": "ack-send-fails", + "description": "The first acknowledgement attempt fails: recorded as failed, reported overdue, and sent exactly once on the retry", + "message_id": "sim-ack-send-fails", + "received_at": "2026-11-06T15:00:00Z", + "simulate_send_failure": true, + "from": "info@fedramp.gov", + "envelope": "info@fedramp.gov", + "subject": "Checking in", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "failed", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T15:00:00Z", + "respond_by": null + } + } + ] +} diff --git a/docs/cr26-applicability.md b/docs/cr26-applicability.md index 642739e1..30052eba 100644 --- a/docs/cr26-applicability.md +++ b/docs/cr26-applicability.md @@ -24,7 +24,8 @@ The POA&M is retired (2026-11-04): CR26 has no POA&M. Open findings and their de - **IVV — Independent Verification and Validation.** `IVV-CSO-FIA` (Class C MUST) requires a persistent independent assessment by a **FedRAMP Recognized independent assessment service** (the artifact formerly called a 3PAO) or by FedRAMP. This system is self-attested and has engaged no such assessor, so the *independent* half cannot be met. The provider-side verification and validation an assessor would consume **are** produced (in the Security Decision Record); the independent half is unmet by design and recorded N/A. - **MKT — Marketplace Listing.** `MKT-IIP-AGU` requires demonstrating the offering is for agency use within 44 USC § 3506 scope, and the rule text is explicit that *"services used by private companies to meet other compliance requirements (such as CMMC) that do not also meet one of the above use cases are outside the scope of FedRAMP."* There is no agency use case and no intent to list, so MKT is N/A by statutory scope. -- **AFC — Addressing FedRAMP Communication.** `AFC-CSO-INB` and the rest require maintaining a FedRAMP Security Inbox and handling FedRAMP correspondence. With no listing and no sponsor, FedRAMP has no reason to correspond, so there is no channel to maintain. AFC's grace period already ended **2026-07-01**; it is recorded N/A absent a sponsor. - **IEC — Incident Evaluation and Communication.** A FedRAMP Reportable Incident under `IEC-CSO-EFR` is triggered by an effect on the confidentiality or integrity of **federal customer data**. This system holds none, so in practice no incident is FedRAMP-reportable. Every incident is still evaluated and recorded in `data/incidents.json`; reportable ones get Initial, Ongoing and Final reports in FedRAMP's schema, on the Class C clocks, published under `/.well-known/` (publish-only: nothing is sent). Exercised 2026-11-05 with simulated incidents marked as exercises; see `docs/incident-response.md`. -These four are N/A because of *what this system is*, not because of an engineering gap. If the system were ever sponsored, listed, assessed, or handling federal data, each would activate, and the mechanisms above (the SDR's V&V evidence, the CDS overview package, the PAIN evaluation) are the inputs those activated obligations would consume. +These three are N/A because of *what this system is*, not because of an engineering gap. If the system were ever sponsored, listed, assessed, or handling federal data, each would activate, and the mechanisms above (the SDR's V&V evidence, the CDS overview package, the PAIN evaluation) are the inputs those activated obligations would consume. + +**AFC — Addressing FedRAMP Communication** is no longer recorded N/A (operator's decision, 2026-11-05): it is implemented against a *simulated* FedRAMP Security Inbox. The inbox logic (`scripts/_fsi.py`) runs on every deploy over committed synthetic messages and publishes `/.well-known/fedramp-inbox-simulation.json`, labelled as a simulation; no mail is received or sent, and the published contact sends FedRAMP to the operator on LinkedIn. See `docs/runbooks/fedramp-security-inbox.md`. diff --git a/docs/incident-response.md b/docs/incident-response.md index bae342b8..657682a8 100644 --- a/docs/incident-response.md +++ b/docs/incident-response.md @@ -134,5 +134,5 @@ Single-tier. Author → author. External escalations: - **GitHub Support** — if the issue is GitHub-side (account, repo, workflow infrastructure) - The mailbox in `security.txt` for acknowledging external reporters -There is no actual FedRAMP certification in scope, so the FedRAMP Security Inbox (FSI) does not apply. Incident reports under the FedRAMP Incident Evaluation and Communication rules are generated and published as described above; they are not sent. +There is no actual FedRAMP certification in scope. Incident reports under the FedRAMP Incident Evaluation and Communication rules are generated and published as described above; they are not sent. The FedRAMP Security Inbox (FSI) is simulated for this proof of concept: no mail is received or sent, and its published contact directs FedRAMP to the operator on LinkedIn (`docs/runbooks/fedramp-security-inbox.md`). diff --git a/docs/policies/ir-policy.md b/docs/policies/ir-policy.md index 03c4e0db..aa3bd3e3 100644 --- a/docs/policies/ir-policy.md +++ b/docs/policies/ir-policy.md @@ -7,7 +7,7 @@ External communication paths: - **Public reporter:** [`/.well-known/security.txt`](../../website/.well-known/security.txt) is the published vulnerability-reporting mailbox. - **FedRAMP incident reports (IEC):** every incident is recorded in the incident register, [`data/incidents.json`](../../data/incidents.json), with its reportability evaluation. Initial, Ongoing and Final Incident Reports for a FedRAMP Reportable Incident are generated from it, validated against FedRAMP's incident report schema, signed, and published under `/.well-known/` (the incident log is `/.well-known/incident-reports.json`) by an unattended workflow on every change to the register. - **Notification is publish-only.** Nothing is sent to FedRAMP or to any agency. The operator has not authorized sending, and the system has no FedRAMP certification and no agency customers. Choosing a delivery channel is an operator decision; until it is made, publication is the only channel and every report says so. -- **FedRAMP Security Inbox (FSI, 20x):** the system is not FedRAMP-certified; no actual FSI obligation applies. Notification path is documented in the runbook for reference. +- **FedRAMP Security Inbox (FSI, 20x):** simulated for this proof of concept. No mail is received or sent; the inbox logic runs on every deploy over synthetic messages and its result is published as `/.well-known/fedramp-inbox-simulation.json`. The published contact directs FedRAMP to the operator on LinkedIn. Procedure: `docs/runbooks/fedramp-security-inbox.md`. **20x rule integration.** KSI-INR (Incident Response), IEC (Incident Evaluation and Communication), FSI (FedRAMP Security Inbox). diff --git a/docs/runbooks/fedramp-security-inbox.md b/docs/runbooks/fedramp-security-inbox.md new file mode 100644 index 00000000..a2d053f4 --- /dev/null +++ b/docs/runbooks/fedramp-security-inbox.md @@ -0,0 +1,78 @@ +# FedRAMP Security Inbox (simulated) + +> **This inbox is a simulation.** By the operator's decision, the FedRAMP Security Inbox does not exist for this proof of concept. No mail is received and none is sent. Its published contact reads: "Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/". + +Rules: AFC-CSO-INB, TFG, RCV, CRA, EMR, IMA, ACK and NOC (FedRAMP Consolidated Rules 2026, AFC family). + +## What is real and what is simulated + +| Part | Real or simulated | +|---|---| +| Inbox logic (`scripts/_fsi.py`): classification, designators, deadlines, register, acknowledgement, paging | Real code, tested | +| Messages from FedRAMP | Simulated: synthetic messages in `data/fsi-simulation/messages.json` | +| Mail system (receiving, sending acknowledgements, delivering pages) | Simulated: outbound legs are recorded, never sent | +| Published contact (`data/cds/offering.json`, `fedramp_security_inbox.contact`, published in the CPO) | Real, and it says it is a simulation | +| Change-notice check (AFC-CSO-NOC, `scripts/check-fsi-notice.py`) | Real, runs on every pull request and push | + +## What runs on every deploy + +The deploy step "FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" runs before the Security Decision Record is built. It calls `scripts/build-fsi-simulation.py`, which: + +1. Runs each synthetic message through the inbox logic: + - an Emergency Test from FedRAMP's Emergency sender; + - an Important message that tries to inject a header; + - a general message; + - a forged @gsa.gov Emergency that fails SPF, DKIM and DMARC; + - an auto-reply, bulk mail and a bounce; + - a message with an unreadable From line and a FedRAMP envelope sender; + - an Emergency from a non-designated sender; + - a look-alike domain and a display-name spoof; + - a message whose first acknowledgement attempt fails. +2. Compares every outcome with the expected one written beside the message: + - classification, designation and sender checks; + - whether the acknowledgement is sent or suppressed, and why; + - the page; + - the deadlines. +3. Checks each acknowledgement for header safety. Then it runs the messages again, and the retry pass must send only the failed acknowledgement and nothing twice. +4. Exercises the register: + - overdue items; + - an Important message refused triage until its stated timeframe is recorded; + - an Emergency reported once past its deadline until it is closed. +5. Exits 1 and writes nothing if any outcome differs. That fails the deploy before anything is published. + +On success, the deploy signs `fedramp-inbox-simulation.json` and publishes it at `/.well-known/fedramp-inbox-simulation.json`. The file is labelled `"simulation": true` and holds sender domains only. Its freshness is governed by `infrastructure/schemas/freshness-config.json`. + +## Reading the published result + +- `result` is `pass` on every published copy, because a failing run publishes nothing. +- `cases[].observed` shows what the logic did with each synthetic message. +- `register_check.checks[]` lists each register behaviour with its expected and observed values. +- `retry_pass` shows that redelivery sends nothing twice. + +## Changing a simulated outcome or adding a case + +Edit `data/fsi-simulation/messages.json`, including the `expected` block, and run: + + python3 scripts/build-fsi-simulation.py --output /tmp/fsi-sim.json + python3 -m pytest tests/test_fedramp_inbox.py + +`tests/fixtures/fsi/simulation-wrong-expectation.json` must stay identical to the messages file except for its one wrong expectation. A test enforces this. + +## Changing the published contact (AFC-CSO-NOC) + +1. Notify FedRAMP through the "[CSP] Notification of Changes" form on help.fedramp.gov. For this simulated system, that step is itself hypothetical. +2. In `data/cds/offering.json`, append a `history` entry with the new contact, `since`, and `fedramp_notified: {"on": "YYYY-MM-DD", "channel": "..."}`. +3. Change `contact`. + +`scripts/check-fsi-notice.py` fails the pull request if the contact changes without that entry, or if the history is rewritten. + +## What a real inbox would need + +A real inbox would need four things: + +- a mail receiver for an address on the system's domain; +- storage for received messages; +- authenticated outbound mail for acknowledgements; +- delivery of pages to the operator. + +`scripts/_fsi.py` is written to be the logic behind them. None of this is deployed, by decision. diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index c81d4307..ccb9eb40 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -169,6 +169,18 @@ "signed": true, "signature": ".well-known/ongoing-certification-report.bundle" }, + { + "path": ".well-known/fedramp-inbox-simulation.json", + "cadence": "deploy", + "max_age_hours": 720, + "timestamp": { + "source": "json", + "field": "emitted_at" + }, + "signed": true, + "signature": ".well-known/fedramp-inbox-simulation.bundle", + "note": "SIMULATION: the FedRAMP Security Inbox is simulated for this proof of concept; this is the deploy's run of the inbox logic over synthetic messages." + }, { "path": ".well-known/certification-overview-package.json", "cadence": "deploy", diff --git a/scripts/_fsi.py b/scripts/_fsi.py new file mode 100644 index 00000000..d9f0a7c4 --- /dev/null +++ b/scripts/_fsi.py @@ -0,0 +1,322 @@ +"""FedRAMP Security Inbox logic, as pure functions (no I/O, no network). + +The FedRAMP Security Inbox is SIMULATED for this proof of concept, by the +operator's decision: no mail is received and none is sent. This module is the +logic a real inbox would run, and scripts/build-fsi-simulation.py runs it on +every deploy over committed synthetic messages (data/fsi-simulation/), failing +the deploy if any outcome differs from what the message file expects. + +Rules the logic implements (FedRAMP Consolidated Rules 2026, AFC family): + AFC-CSO-TFG mail from @fedramp.gov or @gsa.gov (exact domains) is FedRAMP + by default, whatever its spam or authentication verdict + AFC-CSO-EMR Emergency-designated mail pages the senior security official + AFC-CSO-CRA Emergency / Emergency Test mail gets a tracked deadline + AFC-CSO-IMA Important mail gets a tracked item, triaged with its timeframe + AFC-CSO-ACK FedRAMP mail is acknowledged automatically, once +with the designators from AFC-FRP-CDS, the Emergency senders from AFC-FRP-UFS +and the default Class C timeframe from AFC-FRP-ERT. + +Every string taken from a message is untrusted: it is length-capped and +stripped of control and bidirectional-override characters before it is stored +or used, and nothing from a message reaches an outbound header except as a +validated address, a validated Message-ID, or base64. +""" + +from __future__ import annotations + +import base64 +import re +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +FEDRAMP_DOMAINS = ("fedramp.gov", "gsa.gov") +EMERGENCY_SENDERS = ("fedramp_security@gsa.gov", "fedramp_security@fedramp.gov") +DESIGNATIONS = ("EMERGENCY", "EMERGENCY_TEST", "IMPORTANT", "GENERAL") +TRIAGE_HOURS = {"EMERGENCY": 2, "EMERGENCY_TEST": 2, "IMPORTANT": 24, "GENERAL": 72} +CLASS_C_BUSINESS_DAYS = 2 +CLASS_C_HOUR_ET = 15 +VERDICTS = ("PASS", "FAIL", "GRAY", "PROCESSING_FAILED") +MAX_SUBJECT = 200 +MAX_ADDRESS = 254 +ET = ZoneInfo("America/New_York") + +CONTROL = re.compile("[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2066-\u2069\ufeff]") +ADDR_SPEC = re.compile(r"^[a-z0-9!#$%&'*+/=?^_`{|}~.-]{1,64}@[a-z0-9-]{1,63}(?:\.[a-z0-9-]{1,63})+$") +RFC_MESSAGE_ID = re.compile(r"^<[\x21-\x3b\x3d\x3f-\x7e]{1,250}>$") +ANGLED = re.compile(r"<([^<>\s]+)>") +LABEL = {"EMERGENCY": "EMERGENCY", "EMERGENCY_TEST": "EMERGENCY TEST", "IMPORTANT": "IMPORTANT", "GENERAL": "General"} + + +def clean_text(value, limit: int = MAX_SUBJECT) -> str: + """One safe line: no control or bidi characters, whitespace collapsed, capped.""" + s = CONTROL.sub(" ", "" if value is None else str(value)) + return " ".join(s.split())[:limit] + + +def parse_address(value) -> str | None: + """The addr-spec in a From-style value, lower-cased, or None.""" + s = clean_text(value, 1000) + if not s: + return None + angled = ANGLED.findall(s) + candidate = (angled[-1] if angled else s).lower() + if len(candidate) > MAX_ADDRESS or not ADDR_SPEC.match(candidate): + return None + return candidate + + +def domain_of(address: str | None) -> str | None: + return address.rsplit("@", 1)[1] if address else None + + +def is_fedramp_address(address: str | None) -> bool: + """AFC-CSO-TFG: exact domains only. Subdomains and look-alikes are not FedRAMP.""" + return domain_of(address) in FEDRAMP_DOMAINS + + +def designation_of(subject) -> str: + """AFC-FRP-CDS designator anywhere in the subject, any case. Broad on purpose: + a false Emergency costs a page; a missed one costs the deadline.""" + s = clean_text(subject, 1000).lower() + if re.search(r"\bemergency[\s_-]*test\b", s): + return "EMERGENCY_TEST" + if re.search(r"\bemergency\b", s): + return "EMERGENCY" + if re.search(r"\bimportant\b", s): + return "IMPORTANT" + return "GENERAL" + + +def _header(headers, name: str) -> str | None: + want = name.lower() + for k, v in (headers or {}).items(): + if isinstance(k, str) and k.lower() == want: + return clean_text(v, 1000) + return None + + +def _verdict(v) -> str: + s = v.upper() if isinstance(v, str) else "MISSING" + return s if s in VERDICTS else "MISSING" + + +def authenticated(auth: dict) -> bool: + """AFC-FRP-VRE: FedRAMP sends with SPF, DKIM and DMARC configured.""" + return auth["dmarc"] == "PASS" or (auth["dkim"] == "PASS" and auth["spf"] == "PASS") + + +def loop_reason(headers, source, sender: str | None) -> str | None: + """Why an automatic reply must not be sent to this message, or None.""" + auto = _header(headers, "auto-submitted") + if auto and auto.lower() != "no": + return "auto-submitted" + if (_header(headers, "precedence") or "").lower() in ("bulk", "junk", "list", "auto_reply"): + return "precedence" + src = str(source or "").strip() + if not src or src == "<>": + return "null-sender" + if sender and sender.split("@", 1)[0] in ("mailer-daemon", "postmaster"): + return "daemon" + if not sender: + return "no-from-address" + return None + + +def classify(msg: dict) -> dict: + """Classify one message: {from, envelope, subject, headers, auth}. + + FedRAMP if the From address or the envelope sender is at an exact FedRAMP + domain. Spam and authentication verdicts are recorded and never change the + classification. They do stop the acknowledgement: FedRAMP's own mail passes + SPF/DKIM/DMARC, so suppressing the reply to unauthenticated mail loses + nothing genuine and stops a forged @gsa.gov From from turning the inbox into + a reflector that mails attacker-chosen text to any government address. + """ + sender = parse_address(msg.get("from")) + envelope = parse_address(msg.get("envelope")) + subject = clean_text(msg.get("subject")) + fedramp = is_fedramp_address(sender) or is_fedramp_address(envelope) + designation = designation_of(subject) if fedramp else "GENERAL" + raw_auth = msg.get("auth") or {} + auth = {k: _verdict(raw_auth.get(k)) for k in ("spf", "dkim", "dmarc")} + authed = authenticated(auth) + emergency = designation in ("EMERGENCY", "EMERGENCY_TEST") + mid = _header(msg.get("headers"), "message-id") + return { + "kind": "fedramp" if fedramp else "other", + "from": sender, + "sender_domain": domain_of(sender if is_fedramp_address(sender) else (envelope or sender)), + "subject": subject, + "designation": designation, + "auth": auth, + "authenticated": authed, + "emergency_sender_ok": (sender in EMERGENCY_SENDERS) if emergency else None, + "rfc_message_id": mid if mid and RFC_MESSAGE_ID.match(mid) else None, + "loop": loop_reason(msg.get("headers"), msg.get("envelope"), sender) or (None if authed else "unauthenticated"), + } + + +# --- deadlines ---------------------------------------------------------------- + +def parse_time(value: str) -> datetime: + t = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + if t.tzinfo is None: + raise ValueError(f"timestamp without a zone: {value!r}") + return t.astimezone(timezone.utc) + + +def iso(t: datetime) -> str: + return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def class_c_deadline(received: datetime, business_days: int = CLASS_C_BUSINESS_DAYS, + hour_et: int = CLASS_C_HOUR_ET) -> datetime: + """3:00 p.m. ET on the Nth business day, counting the day of receipt as day 1 + when it is a weekday. That reading of "the 2nd business day" gives the + earlier of the two possible deadlines, and federal holidays are not skipped, + which can only make it earlier still. Both err toward acting sooner.""" + day = received.astimezone(ET).date() + counted = 0 + for _ in range(14): + if day.weekday() < 5: + counted += 1 + if counted == business_days: + return datetime(day.year, day.month, day.day, hour_et, tzinfo=ET).astimezone(timezone.utc) + day += timedelta(days=1) + raise RuntimeError("business-day search did not terminate") + + +def deadlines(designation: str, received: datetime) -> dict: + triage_by = iso(received + timedelta(hours=TRIAGE_HOURS[designation])) + if designation in ("EMERGENCY", "EMERGENCY_TEST"): + return {"triage_by": triage_by, "respond_by": iso(class_c_deadline(received)), + "deadline_basis": "AFC-FRP-ERT Class C default (3:00 p.m. ET, 2nd business day); " + "replace with the timeframe stated in the message"} + if designation == "IMPORTANT": + return {"triage_by": triage_by, "respond_by": None, + "deadline_basis": "Record the timeframe stated in the message (AFC-CSO-IMA) when triaging"} + return {"triage_by": triage_by, "respond_by": None, + "deadline_basis": "General communication: no elevated timeframe (AFC-FRP-CDS)"} + + +# --- register ------------------------------------------------------------------- + +def new_record(rid: str, c: dict, received: datetime) -> dict: + return {"id": rid, "received_at": iso(received), "kind": c["kind"], "sender_domain": c["sender_domain"], + "subject": c["subject"], "designation": c["designation"], "auth": c["auth"], + "authenticated": c["authenticated"], "emergency_sender_ok": c["emergency_sender_ok"], + "status": "open", **deadlines(c["designation"], received), + "ack": {"status": "pending"}, "page": {"status": "pending"}, "history": []} + + +def triage(record: dict, at: datetime, respond_by: str | None = None) -> dict: + """Mark a record triaged. An Important message cannot be triaged without the + timeframe stated in it (AFC-CSO-IMA); a stated timeframe replaces the default.""" + if record["status"] == "closed": + raise ValueError(f"{record['id']} is closed") + if respond_by: + record["respond_by"] = iso(parse_time(respond_by)) + record["deadline_basis"] = "Timeframe stated in the message, recorded at triage" + if record["designation"] == "IMPORTANT" and not record.get("respond_by"): + raise ValueError(f"{record['id']}: an Important message needs the timeframe it states (respond_by)") + record["status"] = "triaged" + record["history"].append({"at": iso(at), "event": "triaged"}) + return record + + +def close(record: dict, at: datetime, outcome: str) -> dict: + if not clean_text(outcome): + raise ValueError("closing needs an outcome") + record["status"] = "closed" + record["closed_at"] = iso(at) + record["history"].append({"at": iso(at), "event": "closed", "outcome": clean_text(outcome)}) + return record + + +def overdue(records, now: datetime) -> list[str]: + """Problems that should page: untriaged past triage_by, not closed past + respond_by, Important triaged without its timeframe, ack or page failed.""" + problems = [] + for r in records: + tag = f"{r['id']} ({r['designation']})" + if r["status"] == "open" and r.get("triage_by") and now > parse_time(r["triage_by"]): + problems.append(f"{tag}: not triaged by {r['triage_by']}") + if r["status"] != "closed" and r.get("respond_by") and now > parse_time(r["respond_by"]): + problems.append(f"{tag}: required actions not closed by {r['respond_by']}") + if r["designation"] == "IMPORTANT" and r["status"] != "open" and not r.get("respond_by"): + problems.append(f"{tag}: triaged without recording the message's timeframe") + for leg in ("ack", "page"): + if (r.get(leg) or {}).get("status") == "failed": + problems.append(f"{tag}: {leg} failed") + return problems + + +# --- outbound text ------------------------------------------------------------------ + +def _b64(s: str) -> str: + return base64.b64encode(s.encode("utf-8")).decode("ascii") + + +def build_ack(*, inbox: str, to: str, subject: str, rfc_message_id: str | None, + received_at: str, designation: str, ref: str) -> str: + """The raw acknowledgement (RFC 5322). Every header value is a constant, a + validated address, a validated Message-ID, or base64, so nothing from the + inbound message can inject a header.""" + frm, rcpt = parse_address(inbox), parse_address(to) + if not frm or not rcpt: + raise ValueError("acknowledgement needs valid From and To addresses") + subj = clean_text(subject) or "(no subject)" + lines = [f"From: FedRAMP Security Inbox <{frm}>", f"To: <{rcpt}>", + f"Subject: =?UTF-8?B?{_b64('Received: ' + subj)}?=", + "Auto-Submitted: auto-replied", "X-Auto-Response-Suppress: All", "MIME-Version: 1.0", + "Content-Type: text/plain; charset=UTF-8", "Content-Transfer-Encoding: base64"] + if rfc_message_id and RFC_MESSAGE_ID.match(rfc_message_id): + lines += [f"In-Reply-To: {rfc_message_id}", f"References: {rfc_message_id}"] + body = "\n".join([ + f"This is an automatic acknowledgement from the FedRAMP Security Inbox ({frm}).", "", + f"Your message was received at {clean_text(received_at, 40)}.", + f"Designation recognised from the subject line: {LABEL.get(designation, 'General')}.", + f"Reference: {clean_text(ref, 80)}", "", + "The security official has been notified. This reply is automatic; a person will respond separately."]) + b64 = _b64(body) + wrapped = "\r\n".join(b64[i:i + 76] for i in range(0, len(b64), 76)) + return "\r\n".join(lines) + "\r\n\r\n" + wrapped + "\r\n" + + +def page_subject(record: dict) -> str: + """The page's subject: ASCII, at most 100 characters, sender domain only.""" + s = f"[FSI] {LABEL.get(record['designation'], 'General')} FedRAMP message from {record['sender_domain'] or 'unknown'}" + return re.sub(r"[^\x20-\x7e]", "?", s)[:100] + + +# --- one message through the inbox ------------------------------------------------------ + +def receive(msg: dict, register: dict, *, inbox: str, send, page) -> dict: + """Handle one message the way the inbox would. `send(to, raw)` and + `page(subject)` are the outbound legs (in the simulation, recorders that + send nothing). Idempotent: a message already in the register only retries + the legs that did not complete, so a sender is acknowledged once.""" + c = classify(msg) + if c["kind"] != "fedramp": + return {"classification": c, "record": None} + rid = clean_text(msg.get("message_id"), 128) + if not re.fullmatch(r"[A-Za-z0-9._-]{1,128}", rid): + raise ValueError("message_id must be a safe register identifier") + received = parse_time(msg["received_at"]) + record = register.get(rid) or new_record(rid, c, received) + register[rid] = record + if record["ack"]["status"] in ("pending", "failed"): + if c["loop"]: + record["ack"] = {"status": "suppressed", "reason": c["loop"]} + else: + try: + send(c["from"], build_ack(inbox=inbox, to=c["from"], subject=c["subject"], + rfc_message_id=c["rfc_message_id"], received_at=record["received_at"], + designation=c["designation"], ref=rid)) + record["ack"] = {"status": "sent"} + except Exception as e: # noqa: BLE001 - a failed leg is recorded and paged by overdue() + record["ack"] = {"status": "failed", "error": type(e).__name__} + if record["page"]["status"] != "sent": + page(page_subject(record)) + record["page"] = {"status": "sent"} + return {"classification": c, "record": record} diff --git a/scripts/build-cpo.py b/scripts/build-cpo.py index fb5162ef..892933f2 100644 --- a/scripts/build-cpo.py +++ b/scripts/build-cpo.py @@ -61,6 +61,18 @@ } +def _fsi(off): + """The FedRAMP Security Inbox as published. The inbox is simulated for this + proof of concept, so this is a contact statement, never an email address, + plus the simulation's published results. The change history stays in the + repository.""" + fsi = off.get("fedramp_security_inbox") + if not fsi: + return None + return {"simulated": True, "contact": fsi["contact"], "rule": "AFC-CSO-INB", + "simulation": fsi.get("simulation"), "runbook": fsi.get("runbook")} + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--offering", default="../data/cds/offering.json") @@ -102,6 +114,9 @@ def main(): "service_model": off.get("service_model"), "deployment_model": off.get("deployment_model"), "security_contact": off.get("security_contact"), + # AFC-CSO-INB: the FedRAMP Security Inbox contact, from its one + # source. Simulated for this proof of concept (no mail is received). + "fedramp_security_inbox": _fsi(off), "overview_package": BASE + "certification-overview-package.json", }, "minimum_assessment_scope": { # MAS @@ -141,6 +156,8 @@ def render_markdown(c): out = ["# Certification Package Overview", "", f"_{c['note']}_", "", f"- **Accountable official:** {ao.get('name')}, {ao.get('title')} ({ao.get('contact')})", + f"- **FedRAMP Security Inbox (AFC-CSO-INB):** " + f"{(c['certification_data'].get('fedramp_security_inbox') or {}).get('contact', 'not established')}", f"- **Class:** {c['class']} | **Impact:** {c['impact_level_canonical']}", f"- **Version:** {c['metadata']['version']} | **Last updated:** {c['metadata']['last_updated']}", f"- **Maintenance:** {c['maintenance_cadence']}", diff --git a/scripts/build-fsi-simulation.py b/scripts/build-fsi-simulation.py new file mode 100644 index 00000000..71174f6a --- /dev/null +++ b/scripts/build-fsi-simulation.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Run the SIMULATED FedRAMP Security Inbox over committed synthetic messages. + +The FedRAMP Security Inbox is simulated for this proof of concept, by the +operator's decision: nothing real is received or sent. This script feeds the +synthetic messages in data/fsi-simulation/messages.json through the inbox logic +(scripts/_fsi.py) with outbound legs that only record what they would have +sent, then checks every observed outcome against the file's "expected" block: + + - classification (AFC-CSO-TFG: exact @fedramp.gov / @gsa.gov, never filtered) + - designation, Emergency sender, and the default deadlines (AFC-CSO-CRA/EMR/IMA) + - the acknowledgement (AFC-CSO-ACK): built and header-injection safe when + sent; suppressed for auto-replies and for mail failing SPF/DKIM/DMARC + - every FedRAMP message is registered and paged (AFC-CSO-RCV) + - a failed acknowledgement is recorded and reported, and a retry pass over + the same messages sends it once and sends nothing twice + - the triage register: overdue items; an Important message refused triage + until its stated timeframe is recorded; an Emergency past its deadline + reported until closed + +Any difference exits 1 and writes nothing, which fails the deploy. On success it +writes fedramp-inbox-simulation.json, labelled as a simulation, with sender +domains only (no addresses, no subjects), for the deploy to sign and publish. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from datetime import datetime, timezone +from email import policy +from email.parser import Parser +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _fsi # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +ACK_HEADERS = {"from", "to", "subject", "auto-submitted", "x-auto-response-suppress", "mime-version", + "content-type", "content-transfer-encoding", "in-reply-to", "references"} +NOTICE = ("SIMULATION. The FedRAMP Security Inbox is simulated for this proof of concept, by the operator's " + "decision: no mail is received and none is sent. Nothing arrived, nothing was acknowledged, and no FedRAMP " + "staff were paged, emailed or otherwise inconvenienced. These results come from running the inbox logic over " + "committed synthetic messages on every deploy; a deploy whose results differ from the expected ones fails. " + "To reach the operator, see 'contact' (spoiler: it is LinkedIn).") +RULES = ["AFC-CSO-INB", "AFC-CSO-TFG", "AFC-CSO-RCV", "AFC-CSO-CRA", "AFC-CSO-EMR", "AFC-CSO-IMA", "AFC-CSO-ACK"] + + +def ack_problems(raw: str, to: str, subject: str) -> list[str]: + """Why a built acknowledgement is unsafe or wrong, or [].""" + out = [] + head = raw.split("\r\n\r\n", 1)[0] + for line in head.split("\r\n"): + if "\n" in line or "\r" in line or ":" not in line: + out.append("a header line is malformed") + msg = Parser(policy=policy.default).parsestr(raw) + names = [k.lower() for k in msg.keys()] + if set(names) - ACK_HEADERS: + out.append(f"unexpected headers {sorted(set(names) - ACK_HEADERS)}") + if len(names) != len(set(names)): + out.append("a header appears twice") + if [ln for ln in head.split("\r\n") if ln.lower().startswith("to:")] != [f"To: <{to}>"]: + out.append("To is not exactly the sender") + if str(msg["subject"]) != "Received: " + (subject or "(no subject)"): + out.append("the subject does not round-trip") + if msg["auto-submitted"] != "auto-replied": + out.append("Auto-Submitted: auto-replied is missing") + return out + + +def run(doc: dict) -> tuple[list[dict], dict, dict, list[str]]: + inbox = doc["inbox"] + register: dict = {} + sent: list[tuple[str, str]] = [] + pages: list[str] = [] + fail_once = {m["message_id"] for m in doc["messages"] if m.get("simulate_send_failure")} + current = {"id": None} + + def send(to, raw): # records only; nothing is sent + if current["id"] in fail_once: + fail_once.discard(current["id"]) + raise ConnectionError("simulated send failure") + sent.append((to, raw)) + + page = pages.append + failures: list[str] = [] + cases = [] + for m in doc["messages"]: + n_sent, n_pages = len(sent), len(pages) + current["id"] = m["message_id"] + res = _fsi.receive(m, register, inbox=inbox, send=send, page=page) + c, r = res["classification"], res["record"] + ack = "none" + in_reply_to = None + if r: + ack = r["ack"]["status"] + (f":{r['ack']['reason']}" if r["ack"].get("reason") else "") + if len(sent) > n_sent: + to, raw = sent[-1] + for p in ack_problems(raw, c["from"], c["subject"]): + failures.append(f"{m['case']}: acknowledgement {p}") + in_reply_to = "\r\nIn-Reply-To: " in raw + observed = {"kind": c["kind"], "designation": c["designation"], "sender_domain": c["sender_domain"], + "authenticated": c["authenticated"], "emergency_sender_ok": c["emergency_sender_ok"], + "ack": ack, "in_reply_to": in_reply_to, "page": len(pages) > n_pages, + "triage_by": r["triage_by"] if r else None, "respond_by": r["respond_by"] if r else None} + # AFC-CSO-RCV: nothing FedRAMP is dropped: every FedRAMP message is + # registered and paged, whatever its verdicts. + if c["kind"] == "fedramp" and not (r and observed["page"]): + failures.append(f"{m['case']}: a FedRAMP message was not registered and paged") + diffs = sorted(k for k, v in m["expected"].items() if observed.get(k) != v) + diffs += sorted(set(observed) - set(m["expected"])) + for k in diffs: + failures.append(f"{m['case']}: {k} expected {m['expected'].get(k)!r}, observed {observed.get(k)!r}") + cases.append({"case": m["case"], "description": m["description"], "received_at": m["received_at"], + "auth": c["auth"], "observed": observed, "matches_expected": not diffs}) + + rc = doc["register_check"] + checks: list[dict] = [] + + def check(name, expected, observed): + checks.append({"check": name, "expected": expected, "observed": observed, "ok": expected == observed}) + if expected != observed: + failures.append(f"register: {name}: expected {expected!r}, observed {observed!r}") + + def overdue_ids(at): + return sorted({p.split(" ", 1)[0] for p in _fsi.overdue(register.values(), at)}) + + def reported(rid, at, needle): + return any(p.startswith(rid + " ") and needle in p for p in _fsi.overdue(register.values(), at)) + + now = _fsi.parse_time(rc["now"]) + check(f"items overdue at {rc['now']} (untriaged Emergencies, a failed acknowledgement)", + sorted(rc["expected_overdue"]), overdue_ids(now)) + + # Retry: the same messages again (a redelivery or a retried invocation). + before = (len(sent), len(pages)) + for m in doc["messages"]: + current["id"] = m["message_id"] + _fsi.receive(m, register, inbox=inbox, send=send, page=page) + retry = {"acknowledgements_sent_again": len(sent) - before[0], "pages_sent_again": len(pages) - before[1]} + check("retry pass: only the failed acknowledgement is sent again, nothing twice", rc["expected_retry"], retry) + failed_ids = [m["message_id"] for m in doc["messages"] if m.get("simulate_send_failure")] + check("failed acknowledgements are sent on the retry", + ["sent"] * len(failed_ids), [register[i]["ack"]["status"] for i in failed_ids]) + check("a failed acknowledgement, once sent, is no longer overdue", + False, any(reported(i, now, "ack failed") for i in failed_ids)) + + # AFC-CSO-IMA: an Important message cannot be triaged without its timeframe. + imp = register[rc["important_case"]] + try: + _fsi.triage(imp, now) + refused = False + except ValueError: + refused = True + check("Important triage without the message's timeframe is refused", True, refused) + _fsi.triage(imp, now, respond_by=rc["important_respond_by"]) + check("Important triage records the message's timeframe", rc["important_respond_by"], imp["respond_by"]) + later = _fsi.parse_time(rc["later"]) + check("an Important message untriaged after 24 hours is reported", True, + reported(rc["untriaged_important_case"], later, "not triaged")) + forced = dict(imp, id="sim-forced-state", status="triaged", respond_by=None, history=[]) + check("an Important item triaged without a timeframe is reported", True, + any("timeframe" in p for p in _fsi.overdue([forced], now))) + + # AFC-CSO-CRA: an Emergency past its deadline and not closed is reported; + # closing it clears the report. + em = register[rc["emergency_case"]] + _fsi.triage(em, now) + check("a triaged Emergency is no longer reported as untriaged", False, reported(em["id"], now, "not triaged")) + check("an Emergency past its deadline and not closed is reported", True, + reported(em["id"], later, "not closed")) + _fsi.close(em, later, "required actions completed (simulated)") + check("closing the Emergency clears the report", False, reported(em["id"], later, "not closed")) + + register_check = {"now": rc["now"], "later": rc["later"], "checks": checks} + return cases, retry, register_check, failures + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--messages", type=Path, default=REPO / "data" / "fsi-simulation" / "messages.json") + ap.add_argument("--offering", type=Path, default=REPO / "data" / "cds" / "offering.json") + ap.add_argument("--output", type=Path, default=Path("fedramp-inbox-simulation.json")) + a = ap.parse_args(argv) + raw = a.messages.read_bytes() + doc = json.loads(raw) + cases, retry, register_check, failures = run(doc) + for f in failures: + print(f"::error::FedRAMP Security Inbox simulation: {f}") + if failures: + print(f"FedRAMP Security Inbox simulation: {len(failures)} outcome(s) differ from expected; nothing written") + return 1 + contact = ((json.loads(a.offering.read_text()).get("fedramp_security_inbox") or {}).get("contact")) + sha = os.environ.get("GITHUB_SHA", "") + report = { + "report_type": "fedramp-inbox-simulation", + "simulation": True, + "notice": NOTICE, + "contact": contact, + "emitted_at": datetime.now(timezone.utc).isoformat(), + "commit": sha if len(sha) == 40 and all(ch in "0123456789abcdef" for ch in sha) else None, + "logic": "scripts/_fsi.py", + "input": "data/fsi-simulation/messages.json", + "input_sha256": hashlib.sha256(raw).hexdigest(), + "rules_exercised": RULES, + "result": "pass", + "cases": cases, + "retry_pass": retry, + "register_check": register_check, + } + a.output.write_text(json.dumps(report, indent=2) + "\n") + print(f"FedRAMP Security Inbox simulation: {len(cases)} synthetic messages, all outcomes as expected; " + f"wrote {a.output} (SIMULATION: nothing received or sent)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-fsi-notice.py b/scripts/check-fsi-notice.py new file mode 100644 index 00000000..3481d703 --- /dev/null +++ b/scripts/check-fsi-notice.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""AFC-CSO-NOC: a change to the FedRAMP Security Inbox contact must record that +FedRAMP was notified. + +The inbox is simulated for this proof of concept (no mail is received), so what +is published is a contact statement rather than a mailbox. It lives in one +place, data/cds/offering.json (fedramp_security_inbox.contact), with an +append-only history. Each entry after the first is a change, and a change must +carry the notification marker: + + "fedramp_notified": {"on": "YYYY-MM-DD", "channel": ""} + +FedRAMP's channel for this is the "[CSP] Notification of Changes" form on +help.fedramp.gov (the rule's notification target). The first entry establishes +the contact and needs no marker only when there was no earlier contact to +change from; it must say why in "basis". + +Checks (each failure names itself; exit 1 on any): + - the contact is one non-empty line of printable text + - the newest history entry is the current contact + - every later entry changes the contact and carries a dated, non-future + fedramp_notified marker with a channel + - with --base (the same file at the merge base): the history is append-only, + and if the contact changed, the new entries are the ones that say so + +Stdlib only; runs in the unit-test job on every pull request and push. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import date +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +DEFAULT = REPO / "data" / "cds" / "offering.json" +CONTACT = re.compile(r"^[\x20-\x7e]{1,300}$") +DAY = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def _inbox(doc): + return doc.get("fedramp_security_inbox") if isinstance(doc, dict) else None + + +def _date(value): + if not isinstance(value, str) or not DAY.match(value): + return None + try: + return date.fromisoformat(value) + except ValueError: + return None + + +def check(head: dict, base: dict | None = None, today: date | None = None) -> list[str]: + """Violations of AFC-CSO-NOC in head (and in the step from base to head).""" + today = today or date.today() + out: list[str] = [] + fsi = _inbox(head) + if not isinstance(fsi, dict): + return ["fedramp_security_inbox is missing from data/cds/offering.json (AFC-CSO-INB)"] + contact = fsi.get("contact") + if not isinstance(contact, str) or not CONTACT.match(contact) or not contact.strip(): + out.append("fedramp_security_inbox.contact must be one non-empty line of printable text (at most 300 characters)") + history = fsi.get("history") + if not isinstance(history, list) or not history or not all(isinstance(h, dict) for h in history): + return out + ["fedramp_security_inbox.history must list every contact the inbox has had"] + if history[-1].get("contact") != contact: + out.append("the contact changed without a history entry recording the change and notifying FedRAMP") + first = history[0] + if first.get("fedramp_notified") is None and not (isinstance(first.get("basis"), str) and first["basis"].strip()): + out.append("the first history entry has no fedramp_notified and no basis saying why none was needed") + for i, h in enumerate(history): + since = _date(h.get("since")) + if since is None or since > today: + out.append(f"history[{i}].since must be a date, not in the future") + if i == 0: + continue + if h.get("contact") == history[i - 1].get("contact"): + out.append(f"history[{i}] repeats the previous contact") + marker = h.get("fedramp_notified") + if not isinstance(marker, dict): + out.append(f"history[{i}] changes the contact without a fedramp_notified marker (AFC-CSO-NOC)") + continue + on = _date(marker.get("on")) + if on is None or on > today: + out.append(f"history[{i}].fedramp_notified.on must be the date FedRAMP was notified, not in the future") + channel = marker.get("channel") + if not isinstance(channel, str) or not channel.strip(): + out.append(f"history[{i}].fedramp_notified.channel must say how FedRAMP was notified") + base_fsi = _inbox(base) if base is not None else None + if isinstance(base_fsi, dict): + base_hist = base_fsi.get("history") or [] + if history[:len(base_hist)] != base_hist: + out.append("fedramp_security_inbox.history was rewritten; it is append-only") + if base_fsi.get("contact") != contact and len(history) <= len(base_hist): + out.append("the contact changed but no history entry was added for the change") + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--offering", type=Path, default=DEFAULT) + ap.add_argument("--base", type=Path, help="offering.json at the merge base (optional)") + a = ap.parse_args(argv) + head = json.loads(a.offering.read_text()) + base = None + if a.base and a.base.is_file() and a.base.stat().st_size: + base = json.loads(a.base.read_text()) + problems = check(head, base) + for p in problems: + print(f"::error::FedRAMP Security Inbox (AFC-CSO-NOC): {p}") + if not problems: + fsi = _inbox(head) + print(f"FedRAMP Security Inbox: {len(fsi['history'])} history entr{'y' if len(fsi['history']) == 1 else 'ies'}; " + f"{'compared with the merge base' if base else 'no merge base given'}; ok") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/fsi/offering-changed-with-notice.json b/tests/fixtures/fsi/offering-changed-with-notice.json new file mode 100644 index 00000000..8b83ed8b --- /dev/null +++ b/tests/fixtures/fsi/offering-changed-with-notice.json @@ -0,0 +1,21 @@ +{ + "fedramp_security_inbox": { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox. Hit me up on LinkedIn (new profile): https://www.linkedin.com/in/example/", + "history": [ + { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox, no mail is received or sent, and the inbox is permanently at zero because it does not exist. Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/", + "since": "2026-11-05", + "fedramp_notified": null, + "basis": "Established. No earlier FedRAMP Security Inbox or Marketplace Security Email is on file with FedRAMP for this unsponsored system, so there was no prior contact to change from." + }, + { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox. Hit me up on LinkedIn (new profile): https://www.linkedin.com/in/example/", + "since": "2026-12-01", + "fedramp_notified": { + "on": "2026-11-30", + "channel": "help.fedramp.gov [CSP] Notification of Changes form" + } + } + ] + } +} diff --git a/tests/fixtures/fsi/offering-changed-without-history.json b/tests/fixtures/fsi/offering-changed-without-history.json new file mode 100644 index 00000000..d03a272d --- /dev/null +++ b/tests/fixtures/fsi/offering-changed-without-history.json @@ -0,0 +1,13 @@ +{ + "fedramp_security_inbox": { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox. Hit me up on LinkedIn (new profile): https://www.linkedin.com/in/example/", + "history": [ + { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox, no mail is received or sent, and the inbox is permanently at zero because it does not exist. Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/", + "since": "2026-11-05", + "fedramp_notified": null, + "basis": "Established. No earlier FedRAMP Security Inbox or Marketplace Security Email is on file with FedRAMP for this unsponsored system, so there was no prior contact to change from." + } + ] + } +} diff --git a/tests/fixtures/fsi/offering-changed-without-notice.json b/tests/fixtures/fsi/offering-changed-without-notice.json new file mode 100644 index 00000000..c788d5ee --- /dev/null +++ b/tests/fixtures/fsi/offering-changed-without-notice.json @@ -0,0 +1,17 @@ +{ + "fedramp_security_inbox": { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox. Hit me up on LinkedIn (new profile): https://www.linkedin.com/in/example/", + "history": [ + { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox, no mail is received or sent, and the inbox is permanently at zero because it does not exist. Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/", + "since": "2026-11-05", + "fedramp_notified": null, + "basis": "Established. No earlier FedRAMP Security Inbox or Marketplace Security Email is on file with FedRAMP for this unsponsored system, so there was no prior contact to change from." + }, + { + "contact": "Simulated for this proof of concept: there is no FedRAMP Security Inbox. Hit me up on LinkedIn (new profile): https://www.linkedin.com/in/example/", + "since": "2026-12-01" + } + ] + } +} diff --git a/tests/fixtures/fsi/simulation-wrong-expectation.json b/tests/fixtures/fsi/simulation-wrong-expectation.json new file mode 100644 index 00000000..cad4d96b --- /dev/null +++ b/tests/fixtures/fsi/simulation-wrong-expectation.json @@ -0,0 +1,366 @@ +{ + "_note": "KNOWN-BAD FIXTURE (tests/test_fedramp_inbox.py): identical to data/fsi-simulation/messages.json except that the forged @gsa.gov message is expected to be acknowledged. The simulation must fail on it, because an unauthenticated message is never acknowledged.", + "inbox": "fedramp-security-inbox@simulation.invalid", + "register_check": { + "now": "2026-11-06T17:00:00Z", + "expected_overdue": [ + "sim-ack-send-fails", + "sim-emergency-other-sender", + "sim-emergency-test", + "sim-forged-gsa" + ], + "expected_retry": { + "acknowledgements_sent_again": 1, + "pages_sent_again": 0 + }, + "important_case": "sim-important", + "important_respond_by": "2026-11-10T22:00:00Z", + "later": "2026-11-09T21:00:00Z", + "emergency_case": "sim-emergency-test", + "untriaged_important_case": "sim-envelope-only" + }, + "messages": [ + { + "case": "emergency-test", + "description": "Emergency Test from FedRAMP's Emergency sender (AFC-FRP-UFS), authenticated", + "message_id": "sim-emergency-test", + "received_at": "2026-11-06T14:05:00Z", + "from": "FedRAMP Security ", + "envelope": "fedramp_security@fedramp.gov", + "subject": "EMERGENCY TEST: FedRAMP Security Inbox exercise", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "EMERGENCY_TEST", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": true, + "ack": "sent", + "in_reply_to": true, + "page": true, + "triage_by": "2026-11-06T16:05:00Z", + "respond_by": "2026-11-09T20:00:00Z" + } + }, + { + "case": "important", + "description": "Important message whose subject and Message-ID try to inject a Bcc header", + "message_id": "sim-important", + "received_at": "2026-11-06T14:10:00Z", + "from": "FedRAMP PMO ", + "envelope": "info@fedramp.gov", + "subject": "IMPORTANT: action requested\r\nBcc: injected@attacker.invalid", + "headers": { + "Message-ID": "\r\nBcc: injected@attacker.invalid" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "IMPORTANT", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "sent", + "in_reply_to": false, + "page": true, + "triage_by": "2026-11-07T14:10:00Z", + "respond_by": null + } + }, + { + "case": "general", + "description": "General communication from @gsa.gov, no designator", + "message_id": "sim-general", + "received_at": "2026-11-06T14:15:00Z", + "from": "info@gsa.gov", + "envelope": "info@gsa.gov", + "subject": "FedRAMP program update", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "sent", + "in_reply_to": true, + "page": true, + "triage_by": "2026-11-09T14:15:00Z", + "respond_by": null + } + }, + { + "case": "forged-gsa", + "description": "Forged Emergency claiming to be from the gsa.gov Emergency sender; SPF, DKIM and DMARC fail. Still FedRAMP by default (AFC-CSO-TFG) and paged, but not acknowledged", + "message_id": "sim-forged-gsa", + "received_at": "2026-11-06T14:20:00Z", + "from": "FedRAMP Security ", + "envelope": "bounce@attacker.invalid", + "subject": "EMERGENCY: reply with your credentials", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "FAIL", + "dkim": "FAIL", + "dmarc": "FAIL" + }, + "expected": { + "kind": "fedramp", + "designation": "EMERGENCY", + "sender_domain": "gsa.gov", + "authenticated": false, + "emergency_sender_ok": true, + "ack": "sent", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-06T16:20:00Z", + "respond_by": "2026-11-09T20:00:00Z" + } + }, + { + "case": "auto-reply", + "description": "An auto-reply from @fedramp.gov: registered, never answered (no mail loop)", + "message_id": "sim-auto-reply", + "received_at": "2026-11-06T14:25:00Z", + "from": "info@fedramp.gov", + "envelope": "info@fedramp.gov", + "subject": "Automatic reply: Received: FedRAMP Security Inbox exercise", + "headers": { + "Auto-Submitted": "auto-replied", + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:auto-submitted", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T14:25:00Z", + "respond_by": null + } + }, + { + "case": "look-alike", + "description": "Emergency from a look-alike domain (fedramp.gov.attacker.invalid): not FedRAMP, exact domains only", + "message_id": "sim-look-alike", + "received_at": "2026-11-06T14:30:00Z", + "from": "fedramp_security@fedramp.gov.attacker.invalid", + "envelope": "fedramp_security@fedramp.gov.attacker.invalid", + "subject": "EMERGENCY: act now", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "other", + "designation": "GENERAL", + "sender_domain": "fedramp.gov.attacker.invalid", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "none", + "in_reply_to": null, + "page": false, + "triage_by": null, + "respond_by": null + } + }, + { + "case": "display-name-spoof", + "description": "Display name impersonates the gsa.gov Emergency sender, actual address does not: not FedRAMP", + "message_id": "sim-display-name-spoof", + "received_at": "2026-11-06T14:35:00Z", + "from": "\"fedramp_security@gsa.gov\" ", + "envelope": "attacker@attacker.invalid", + "subject": "EMERGENCY: trust me", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "other", + "designation": "GENERAL", + "sender_domain": "attacker.invalid", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "none", + "in_reply_to": null, + "page": false, + "triage_by": null, + "respond_by": null + } + }, + { + "case": "envelope-only", + "description": "Unreadable From header, @gsa.gov envelope sender: still FedRAMP, registered and paged; no reply without a From address", + "message_id": "sim-envelope-only", + "received_at": "2026-11-06T14:40:00Z", + "from": "garbage <<>>", + "envelope": "bounces@gsa.gov", + "subject": "Important: please confirm", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "IMPORTANT", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:no-from-address", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-07T14:40:00Z", + "respond_by": null + } + }, + { + "case": "emergency-other-sender", + "description": "Emergency from an @gsa.gov address other than the designated Emergency senders (AFC-FRP-UFS): flagged, not demoted, still paged", + "message_id": "sim-emergency-other-sender", + "received_at": "2026-11-06T14:45:00Z", + "from": "Someone ", + "envelope": "someone@gsa.gov", + "subject": "EMERGENCY: rotate your keys", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "EMERGENCY", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": false, + "ack": "sent", + "in_reply_to": true, + "page": true, + "triage_by": "2026-11-06T16:45:00Z", + "respond_by": "2026-11-09T20:00:00Z" + } + }, + { + "case": "bulk", + "description": "Bulk mail from @gsa.gov: registered, not answered", + "message_id": "sim-bulk", + "received_at": "2026-11-06T14:50:00Z", + "from": "news@gsa.gov", + "envelope": "news@gsa.gov", + "subject": "Newsletter", + "headers": { + "Precedence": "bulk" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:precedence", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T14:50:00Z", + "respond_by": null + } + }, + { + "case": "bounce", + "description": "A bounce (null envelope sender) from @gsa.gov: registered, not answered", + "message_id": "sim-bounce", + "received_at": "2026-11-06T14:55:00Z", + "from": "mailer-daemon@gsa.gov", + "envelope": "<>", + "subject": "Undeliverable: Received: FedRAMP program update", + "headers": {}, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "gsa.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "suppressed:null-sender", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T14:55:00Z", + "respond_by": null + } + }, + { + "case": "ack-send-fails", + "description": "The first acknowledgement attempt fails: recorded as failed, reported overdue, and sent exactly once on the retry", + "message_id": "sim-ack-send-fails", + "received_at": "2026-11-06T15:00:00Z", + "simulate_send_failure": true, + "from": "info@fedramp.gov", + "envelope": "info@fedramp.gov", + "subject": "Checking in", + "headers": { + "Message-ID": "" + }, + "auth": { + "spf": "PASS", + "dkim": "PASS", + "dmarc": "PASS" + }, + "expected": { + "kind": "fedramp", + "designation": "GENERAL", + "sender_domain": "fedramp.gov", + "authenticated": true, + "emergency_sender_ok": null, + "ack": "failed", + "in_reply_to": null, + "page": true, + "triage_by": "2026-11-09T15:00:00Z", + "respond_by": null + } + } + ] +} diff --git a/tests/test_fedramp_inbox.py b/tests/test_fedramp_inbox.py new file mode 100644 index 00000000..9e33e889 --- /dev/null +++ b/tests/test_fedramp_inbox.py @@ -0,0 +1,374 @@ +"""The SIMULATED FedRAMP Security Inbox (AFC-CSO-INB, TFG, RCV, CRA, EMR, IMA, ACK). + +By the operator's decision the inbox is simulated for this proof of concept: +nothing real is received or sent. scripts/_fsi.py is the logic a real inbox +would run, and scripts/build-fsi-simulation.py runs it on every deploy over the +synthetic messages in data/fsi-simulation/messages.json, failing the deploy if +any outcome differs. These tests pin the logic and that deploy wiring: + + TFG mail from @fedramp.gov / @gsa.gov is FedRAMP, whatever its spam verdict + or authentication result; look-alike domains are not + ACK FedRAMP mail is acknowledged once, header-injection safe, never to an + auto-reply and never to mail failing SPF/DKIM/DMARC + EMR every FedRAMP message pages, Emergency included + CRA Emergency / Emergency Test mail gets a deadline in the register + IMA Important mail cannot be triaged without its stated timeframe +""" + +import copy +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import _fsi # noqa: E402 + +WORKFLOW = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() +OFFERING = REPO / "data" / "cds" / "offering.json" +MESSAGES = REPO / "data" / "fsi-simulation" / "messages.json" +BAD = REPO / "tests" / "fixtures" / "fsi" / "simulation-wrong-expectation.json" +SIM = REPO / "scripts" / "build-fsi-simulation.py" +NOW = datetime(2026, 11, 9, 15, 0, tzinfo=timezone.utc) # Monday 10:00 a.m. EST +PASS = {"spf": "PASS", "dkim": "PASS", "dmarc": "PASS"} +FAIL = {"spf": "FAIL", "dkim": "FAIL", "dmarc": "FAIL"} +SIMULATED_RULES = ["AFC-CSO-INB", "AFC-CSO-TFG", "AFC-CSO-RCV", "AFC-CSO-CRA", "AFC-CSO-EMR", + "AFC-CSO-IMA", "AFC-CSO-ACK"] + + +def msg(frm="FedRAMP ", subject="Emergency: action required", envelope=None, + headers=None, auth=PASS, message_id="m1", received_at="2026-11-09T15:00:00Z"): + return {"from": frm, "envelope": envelope if envelope is not None else _fsi.parse_address(frm) or "", + "subject": subject, "headers": headers or {}, "auth": auth, "message_id": message_id, + "received_at": received_at} + + +# --- classification (AFC-CSO-TFG) --------------------------------------------------- + +@pytest.mark.parametrize("frm", ["FedRAMP PMO ", "someone@FedRAMP.gov", + "\"F20B Team\" "]) +def test_fedramp_senders_are_fedramp_even_when_unauthenticated(frm): + c = _fsi.classify(msg(frm=frm, auth=FAIL)) + assert c["kind"] == "fedramp" and c["designation"] == "EMERGENCY" + assert c["authenticated"] is False and c["loop"] == "unauthenticated" + + +@pytest.mark.parametrize("frm,envelope", [ + ("x ", "a@fedramp.gov.example.com"), + ("x ", "a@notfedramp.gov"), + ("x ", "a@mail.gsa.gov"), + ("\"fedramp_security@gsa.gov\" ", "attacker@example.com"), + ("fedramp_security@gsa.gov ", "attacker@example.com"), +]) +def test_look_alike_domains_and_display_name_spoofs_are_not_fedramp(frm, envelope): + assert _fsi.classify(msg(frm=frm, envelope=envelope))["kind"] == "other" + + +def test_a_fedramp_envelope_sender_counts_even_if_the_from_header_is_unparseable(): + c = _fsi.classify(msg(frm="garbage <<>>", envelope="bounces@gsa.gov")) + assert c["kind"] == "fedramp" and c["sender_domain"] == "gsa.gov" + + +@pytest.mark.parametrize("subject,want", [ + ("Emergency Test: confirm your FSI", "EMERGENCY_TEST"), + ("[EMERGENCY-TEST] reply by Tuesday", "EMERGENCY_TEST"), + ("EMERGENCY: CVE-2026-0001 in your boundary", "EMERGENCY"), + ("Re: [Emergency] follow-up", "EMERGENCY"), + ("Important: update your Marketplace listing", "IMPORTANT"), + ("Monthly community update", "GENERAL"), + ("Emergencyish newsletter", "GENERAL"), +]) +def test_designators_follow_afc_frp_cds(subject, want): + assert _fsi.designation_of(subject) == want + + +def test_an_emergency_from_an_address_other_than_fedramp_security_is_flagged_not_demoted(): + c = _fsi.classify(msg(frm="Someone ")) + assert c["designation"] == "EMERGENCY" and c["emergency_sender_ok"] is False + + +# --- deadlines and register (AFC-CSO-CRA, AFC-CSO-IMA, AFC-FRP-ERT) ---------------------- + +@pytest.mark.parametrize("received,due", [ + ("2026-11-09T15:00:00Z", "2026-11-10T20:00:00Z"), # Mon 10:00 EST -> Tue 15:00 EST + ("2026-11-13T21:30:00Z", "2026-11-16T20:00:00Z"), # Fri 16:30 EST -> Mon 15:00 EST + ("2026-11-14T15:00:00Z", "2026-11-17T20:00:00Z"), # Sat -> Tue (Mon is day 1) + ("2026-10-30T14:00:00Z", "2026-11-02T20:00:00Z"), # Fri 10:00 EDT -> Mon 15:00 EST (DST ends Sun) + ("2026-11-10T04:30:00Z", "2026-11-10T20:00:00Z"), # Mon 23:30 EST counts Monday as day 1 +]) +def test_class_c_default_deadline_is_3pm_eastern_on_the_second_business_day(received, due): + assert _fsi.iso(_fsi.class_c_deadline(_fsi.parse_time(received))) == due + + +def test_emergency_records_carry_a_deadline_and_important_ones_ask_for_the_messages_own(): + e = _fsi.deadlines("EMERGENCY", NOW) + assert e["respond_by"] == "2026-11-10T20:00:00Z" and "ERT" in e["deadline_basis"] + assert e["triage_by"] == "2026-11-09T17:00:00Z" + i = _fsi.deadlines("IMPORTANT", NOW) + assert i["respond_by"] is None and "IMA" in i["deadline_basis"] + + +def record(**kw): + r = _fsi.new_record("m", _fsi.classify(msg()), NOW) + r.update(ack={"status": "sent"}, page={"status": "sent"}) + r.update(kw) + return r + + +def test_overdue_counts_untriaged_unclosed_and_failed_legs(): + later = datetime(2026, 11, 9, 18, 0, tzinfo=timezone.utc) + assert len(_fsi.overdue([record()], later)) == 1 # not triaged within 2 hours + assert _fsi.overdue([record(status="triaged")], later) == [] + late = _fsi.overdue([record(status="triaged")], datetime(2026, 11, 11, tzinfo=timezone.utc)) + assert len(late) == 1 and "not closed" in late[0] + failed = _fsi.overdue([record(status="triaged", ack={"status": "failed"})], later) + assert failed and "ack failed" in failed[0] + important = record(designation="IMPORTANT", status="triaged", respond_by=None) + assert "timeframe" in _fsi.overdue([important], later)[0] + + +def test_an_important_message_cannot_be_triaged_without_its_stated_timeframe(): + r = _fsi.new_record("m", _fsi.classify(msg(subject="Important: do X")), NOW) + with pytest.raises(ValueError, match="timeframe"): + _fsi.triage(r, NOW) + _fsi.triage(r, NOW, respond_by="2026-11-12T17:00:00-05:00") + assert r["status"] == "triaged" and r["respond_by"] == "2026-11-12T22:00:00Z" + _fsi.close(r, NOW, "done") + with pytest.raises(ValueError): + _fsi.triage(r, NOW) + + +# --- acknowledgement (AFC-CSO-ACK) ------------------------------------------------------ + +def ack(**kw): + args = {"inbox": "fedramp-security@example.com", "to": "fedramp_security@gsa.gov", "subject": "Emergency: x", + "rfc_message_id": "", "received_at": "2026-11-09T15:00:00Z", "designation": "EMERGENCY", + "ref": "abc", **kw} + return _fsi.build_ack(**args) + + +def header_names(raw): + return [line.split(":", 1)[0] for line in raw.split("\r\n\r\n", 1)[0].split("\r\n")] + + +def test_the_acknowledgement_marks_itself_automatic_and_threads_to_the_message(): + raw = ack() + assert header_names(raw) == ["From", "To", "Subject", "Auto-Submitted", "X-Auto-Response-Suppress", + "MIME-Version", "Content-Type", "Content-Transfer-Encoding", "In-Reply-To", + "References"] + assert "Auto-Submitted: auto-replied" in raw and "In-Reply-To: " in raw + + +@pytest.mark.parametrize("subject,msgid", [ + ("Emergency\r\nBcc: victim@example.com", ""), + ("ok", "\r\nBcc: victim@example.com"), + ("ok", ""), + ("\u202eEmergency\u2066", "\nTo: y@z"), +]) +def test_nothing_from_the_inbound_message_can_inject_a_header(subject, msgid): + names = header_names(ack(subject=subject, rfc_message_id=msgid)) + assert "Bcc" not in names and names.count("To") == 1 and len(names) in (8, 10) + + +def test_an_invalid_recipient_raises_rather_than_sending(): + with pytest.raises(ValueError): + ack(to="victim@example.com\r\nBcc: x@y.z") + + +@pytest.mark.parametrize("headers,envelope,reason", [ + ({"Auto-Submitted": "auto-replied"}, "a@gsa.gov", "auto-submitted"), + ({"Precedence": "bulk"}, "a@gsa.gov", "precedence"), + ({}, "", "null-sender"), + ({}, "<>", "null-sender"), +]) +def test_no_acknowledgement_to_auto_replies_bulk_mail_or_bounces(headers, envelope, reason): + c = _fsi.classify(msg(frm="a@gsa.gov", headers=headers, envelope=envelope)) + assert c["kind"] == "fedramp" and c["loop"] == reason + + +def run_receive(m, register, send_fails=False): + sent, pages = [], [] + + def send(to, raw): + if send_fails: + raise ConnectionError("simulated") + sent.append((to, raw)) + out = _fsi.receive(m, register, inbox="fedramp-security@example.com", send=send, page=pages.append) + return out, sent, pages + + +def test_a_redelivered_message_is_acknowledged_and_paged_once(): + register = {} + _, sent, pages = run_receive(msg(), register) + assert len(sent) == 1 and len(pages) == 1 + _, sent, pages = run_receive(msg(), register) + assert sent == [] and pages == [] + + +def test_a_failed_acknowledgement_is_recorded_retried_and_counted_overdue(): + register = {} + out, _, _ = run_receive(msg(), register, send_fails=True) + assert out["record"]["ack"] == {"status": "failed", "error": "ConnectionError"} + assert any("ack failed" in p for p in _fsi.overdue(register.values(), NOW)) + _, sent, _ = run_receive(msg(), register) + assert len(sent) == 1 and register["m1"]["ack"]["status"] == "sent" + + +def test_unauthenticated_fedramp_mail_is_registered_and_paged_but_not_acknowledged(): + out, sent, pages = run_receive(msg(auth=FAIL), {}) + assert sent == [] and len(pages) == 1 + assert out["record"]["ack"] == {"status": "suppressed", "reason": "unauthenticated"} + + +def test_other_mail_is_not_registered_acknowledged_or_paged(): + out, sent, pages = run_receive(msg(frm="a@example.com"), {}) + assert out["record"] is None and sent == [] and pages == [] + + +def test_the_page_subject_is_ascii_and_names_the_domain_not_the_address(): + s = _fsi.page_subject(record()) + assert s == "[FSI] EMERGENCY FedRAMP message from gsa.gov" and "@" not in s + + +# --- the deploy's simulation step ----------------------------------------------------------- + +def simulate(tmp_path, messages=MESSAGES): + out = tmp_path / "sim.json" + p = subprocess.run([sys.executable, str(SIM), "--messages", str(messages), "--offering", str(OFFERING), + "--output", str(out)], capture_output=True, text=True) + return p, out + + +def test_the_committed_synthetic_messages_cover_the_required_cases(): + cases = {m["case"]: m for m in json.loads(MESSAGES.read_text())["messages"]} + assert {"emergency-test", "important", "general", "forged-gsa", "auto-reply"} <= set(cases) + assert cases["emergency-test"]["expected"]["designation"] == "EMERGENCY_TEST" + assert cases["forged-gsa"]["expected"]["ack"] == "suppressed:unauthenticated" + assert cases["forged-gsa"]["expected"]["kind"] == "fedramp" + assert cases["auto-reply"]["expected"]["ack"] == "suppressed:auto-submitted" + + +def test_the_simulation_passes_and_publishes_a_labelled_result_without_addresses(tmp_path): + p, out = simulate(tmp_path) + assert p.returncode == 0, p.stdout + p.stderr + text = out.read_text() + doc = json.loads(text) + assert doc["simulation"] is True and doc["notice"].startswith("SIMULATION.") + assert "no mail is received and none is sent" in doc["notice"] + assert doc["result"] == "pass" and all(c["matches_expected"] for c in doc["cases"]) + assert doc["retry_pass"] == {"acknowledgements_sent_again": 1, "pages_sent_again": 0} + checks = doc["register_check"]["checks"] + assert len(checks) >= 10 and all(c["ok"] for c in checks) + assert doc["contact"].endswith("Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/") + assert doc["emitted_at"] and doc["rules_exercised"] == SIMULATED_RULES + assert not re.search(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text), "published result carries an address" + + +@pytest.mark.parametrize("case,needle", [ + ("display-name-spoof", '"kind": "other"'), + ("envelope-only", '"ack": "suppressed:no-from-address"'), + ("emergency-other-sender", '"emergency_sender_ok": false'), + ("bulk", '"ack": "suppressed:precedence"'), + ("bounce", '"ack": "suppressed:null-sender"'), + ("ack-send-fails", '"ack": "failed"'), +]) +def test_the_simulation_exercises_each_behaviour_the_records_claim(case, needle): + m = next(m for m in json.loads(MESSAGES.read_text())["messages"] if m["case"] == case) + assert needle in json.dumps(m["expected"]).replace(": False", ": false") + + +@pytest.mark.parametrize("mutate,needle", [ + (lambda d: d["register_check"].update(expected_retry={"acknowledgements_sent_again": 0, "pages_sent_again": 0}), + "retry pass"), + (lambda d: d["register_check"].update(later="2026-11-09T19:00:00Z"), "past its deadline"), + (lambda d: d["register_check"]["expected_overdue"].remove("sim-ack-send-fails"), "items overdue"), +]) +def test_each_register_check_fails_the_simulation_when_its_outcome_differs(tmp_path, mutate, needle): + d = json.loads(MESSAGES.read_text()) + mutate(d) + bad = tmp_path / "m.json" + bad.write_text(json.dumps(d)) + p, out = simulate(tmp_path, bad) + assert p.returncode == 1 and not out.exists() and needle in p.stdout, p.stdout + + +def test_a_differing_outcome_fails_the_simulation_and_writes_nothing(tmp_path): + """Known-bad fixture: the forged @gsa.gov message is expected to be acknowledged.""" + p, out = simulate(tmp_path, BAD) + assert p.returncode == 1 and not out.exists() + assert "forged-gsa: ack expected 'sent', observed 'suppressed:unauthenticated'" in p.stdout + + +def test_the_known_bad_fixture_differs_from_the_committed_messages_only_in_that_expectation(): + good, bad = json.loads(MESSAGES.read_text()), json.loads(BAD.read_text()) + fixed = copy.deepcopy(bad) + next(m for m in fixed["messages"] if m["case"] == "forged-gsa")["expected"]["ack"] = "suppressed:unauthenticated" + fixed["_note"] = good["_note"] + assert fixed == good + + +def step(name): + m = re.search(r"\n - name: " + re.escape(name) + r"\n(.*?)(?=\n - name: |\Z)", WORKFLOW, re.S) + assert m, name + return m.group(1) + + +def test_the_simulation_runs_on_every_deploy_before_the_sdr_build(): + name = "FedRAMP Security Inbox simulation (synthetic messages; nothing received or sent)" + body = step(name) + assert "build-fsi-simulation.py" in body and "--output fedramp-inbox-simulation.json" in body + assert "continue-on-error" not in body and "|| true" not in body + assert WORKFLOW.index("- name: " + name) < WORKFLOW.index("- name: Build Security Decision Record") + + +def test_the_simulation_result_is_signed_gated_published_verified_and_freshness_governed(): + assert ("cosign sign-blob --yes --new-bundle-format --bundle fedramp-inbox-simulation.bundle " + "fedramp-inbox-simulation.json") in WORKFLOW + assert "fedramp-inbox-simulation.json" in step("Publication redaction gate (no AWS account ID)") + publish = step("Publish all signed artifacts to /.well-known/") + for f in ("fedramp-inbox-simulation.json", "fedramp-inbox-simulation.bundle"): + assert f'aws s3 cp {f} "s3://$BUCKET_NAME/.well-known/{f}"' in publish + assert "check fedramp-inbox-simulation.json .well-known/fedramp-inbox-simulation.json" in WORKFLOW + cfg = json.loads((REPO / "infrastructure" / "schemas" / "freshness-config.json").read_text()) + entries = [a for a in cfg["artifacts"] if a["path"] == ".well-known/fedramp-inbox-simulation.json"] + assert len(entries) == 1 and entries[0]["signed"] is True and entries[0]["cadence"] == "deploy" + assert entries[0]["timestamp"] == {"source": "json", "field": "emitted_at"} + assert entries[0]["note"].startswith("SIMULATION") + + +# --- what is published and recorded ------------------------------------------------------------ + +def test_the_cpo_publishes_the_simulated_contact_not_an_email_address(tmp_path): + out = tmp_path / "cpo.json" + subprocess.run([sys.executable, str(REPO / "scripts" / "build-cpo.py"), "--offering", str(OFFERING), + "--profile", str(REPO / "data" / "system-profile.json"), "--overview", str(tmp_path / "none"), + "--ksi-signal", str(tmp_path / "none"), "--output", str(out), "--md-output", ""], + check=True, capture_output=True) + fsi = json.loads(out.read_text())["certification_data"]["fedramp_security_inbox"] + off = json.loads(OFFERING.read_text())["fedramp_security_inbox"] + assert fsi["simulated"] is True and fsi["contact"] == off["contact"] + assert "Hit me up on LinkedIn: https://www.linkedin.com/in/sa2/" in fsi["contact"] and "@" not in fsi["contact"] + assert fsi["simulation"] == "https://samaydlette.com/.well-known/fedramp-inbox-simulation.json" + + +def test_every_afc_record_is_implemented_and_discloses_the_simulation(): + recs = {r["frr_id"]: r for r in json.loads((REPO / "data" / "frr-records.json").read_text())["records"]} + afc = [rid for rid in recs if rid.startswith("AFC-")] + assert sorted(afc) == sorted(SIMULATED_RULES + ["AFC-CSO-NOC"]) + for rid in afc: + r = recs[rid] + assert r["status"] == "Implemented" and "gap" not in r and "closes_with" not in r, rid + text = r["implementation"] + assert text.startswith("SIMULATED FOR THIS PROOF OF CONCEPT"), rid + assert "no mail is received or sent" in text and "https://www.linkedin.com/in/sa2/" in text, rid + for rid in SIMULATED_RULES: + r = recs[rid] + assert r["simulated"]["counterparty"] == "mail", rid + refs = [e["ref"] for e in r["evidence"]] + assert ".well-known/fedramp-inbox-simulation.json" in refs, rid + assert any(ref.startswith("deploy-with-opa.yml#FedRAMP Security Inbox simulation") for ref in refs), rid diff --git a/tests/test_frr_records.py b/tests/test_frr_records.py index e9977e6c..018130e7 100644 --- a/tests/test_frr_records.py +++ b/tests/test_frr_records.py @@ -193,7 +193,7 @@ def test_the_system_and_fedramp_addresses_are_allowed(applicable, resolve): doc = copy.deepcopy(_committed()) for r in doc["records"]: if r["frr_id"] == "AFC-CSO-ACK": - r["gap"] = "Acknowledge mail from x@fedramp.gov and x@gsa.gov at fedramp-security@samaydlette.com." + r["implementation"] = "Acknowledge mail from x@fedramp.gov and x@gsa.gov at fedramp-security@samaydlette.com." _frr.validate(doc, applicable, resolve) @@ -266,7 +266,9 @@ def test_the_summary_counts_simulated_counterparties(applicable, resolve): evidence=[{"kind": "step", "ref": "deploy-with-opa.yml#Run OPA Compliance Check"}], simulated={"counterparty": "assessor", "how": "A plainly fictional assessor."}) _frr.validate(doc, applicable, resolve) - assert _frr.summary(doc, applicable)["simulated"] == {"assessor": 1} + before = _frr.summary(_committed(), applicable)["simulated"] + assert "simulated" not in next(r for r in _committed()["records"] if r["frr_id"] == "VDR-CSO-MSP") + assert _frr.summary(doc, applicable)["simulated"] == {**before, "assessor": before.get("assessor", 0) + 1} def test_the_gate_runs_before_anything_is_applied(): diff --git a/tests/test_fsi_notice.py b/tests/test_fsi_notice.py new file mode 100644 index 00000000..e52215b0 --- /dev/null +++ b/tests/test_fsi_notice.py @@ -0,0 +1,94 @@ +"""AFC-CSO-NOC: changing the FedRAMP Security Inbox contact (simulated inbox; +the contact is a statement, not a mailbox) without recording +that FedRAMP was notified fails the unit-test job (scripts/check-fsi-notice.py). + +Known-bad fixtures under tests/fixtures/fsi/ each fail for one reason; the good +one shows what a compliant change looks like.""" + +import copy +import importlib.util +import json +from datetime import date +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "fsi" +TODAY = date(2026, 12, 15) + +spec = importlib.util.spec_from_file_location("check_fsi_notice", REPO / "scripts" / "check-fsi-notice.py") +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) + + +def load(name): + return json.loads((FIX / f"offering-{name}.json").read_text()) + + +def live(): + return json.loads((REPO / "data" / "cds" / "offering.json").read_text()) + + +def test_the_committed_inbox_record_passes(): + assert mod.check(live(), today=date.today()) == [] + assert mod.main([]) == 0 + + +def test_a_contact_change_without_a_history_entry_fails(): + out = mod.check(load("changed-without-history"), base=live(), today=TODAY) + assert any("without a history entry" in p for p in out), out + assert any("no history entry was added" in p for p in out), out + + +def test_a_contact_change_without_the_fedramp_notified_marker_fails(): + out = mod.check(load("changed-without-notice"), base=live(), today=TODAY) + assert out and all("fedramp_notified" in p for p in out), out + + +def test_a_contact_change_that_records_the_notification_passes(): + assert mod.check(load("changed-with-notice"), base=live(), today=TODAY) == [] + + +def test_the_cli_fails_the_known_bad_fixture(tmp_path): + base = tmp_path / "base.json" + base.write_text(json.dumps(live())) + assert mod.main(["--offering", str(FIX / "offering-changed-without-notice.json"), "--base", str(base)]) == 1 + + +def test_a_notification_dated_in_the_future_fails(): + doc = load("changed-with-notice") + doc["fedramp_security_inbox"]["history"][-1]["fedramp_notified"]["on"] = "2027-01-01" + assert any("not in the future" in p for p in mod.check(doc, today=TODAY)) + + +def test_rewriting_history_fails_against_the_base(): + good = load("changed-with-notice") + rewritten = copy.deepcopy(good) + rewritten["fedramp_security_inbox"]["history"][0]["since"] = "2026-01-01" + assert any("append-only" in p for p in mod.check(rewritten, base=good, today=TODAY)) + + +@pytest.mark.parametrize("contact", ["", " ", "two\nlines", "x" * 301, None]) +def test_the_contact_must_be_one_printable_line(contact): + doc = live() + doc["fedramp_security_inbox"]["contact"] = contact + doc["fedramp_security_inbox"]["history"][-1]["contact"] = contact + assert any("must be one non-empty line" in p for p in mod.check(doc, today=TODAY)) + + +def test_the_published_contact_says_simulated_and_points_to_linkedin(): + contact = live()["fedramp_security_inbox"]["contact"] + assert "Simulated for this proof of concept" in contact + assert "https://www.linkedin.com/in/sa2/" in contact + assert "@" not in contact + + +def test_establishing_the_inbox_needs_a_basis_when_nobody_was_notified(): + doc = live() + doc["fedramp_security_inbox"]["history"][0]["basis"] = "" + assert any("basis" in p for p in mod.check(doc, today=TODAY)) + + +def test_a_missing_inbox_block_fails(): + assert mod.check({}, today=TODAY) From 3709afbbe7223619b30b34acfe913ddeae0010e3 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:42:42 -0500 Subject: [PATCH 16/97] Incident reports: simulated delivery, publisher role proved before the SDR Every deploy and every unattended publish now "delivers" each incident report the log lists to a simulated FedRAMP and agency recipient (scripts/deliver-incident-reports.py): read back, checked against the published sha256 and FedRAMP's schema, and recorded in the signed delivery log /.well-known/incident-deliveries.json. Nothing is sent. A failed delivery stops publication. The deploy job assumes the incident publisher's role before it builds the SDR (scripts/check-incident-role.py), so a deploy whose publisher role does not work stops before publishing records that claim it. The role's trust gains the reviewer-gated deploy environment (bootstrap apply). IEC-CSO-IIR/OIR/FIR/AIR are Implemented with simulated {incident-recipient}. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 16 ++ .github/workflows/incident-reports.yml | 8 + data/frr-records.json | 112 ++++++++-- data/incidents.json | 2 +- docs/cr26-applicability.md | 5 +- docs/incident-response.md | 3 +- docs/policies/ir-policy.md | 2 +- infrastructure/bootstrap/incident-reports.tf | 14 +- infrastructure/schemas/freshness-config.json | 12 + scripts/build-incident-report.py | 7 +- scripts/check-incident-publisher.py | 7 +- scripts/check-incident-role.py | 104 +++++++++ scripts/deliver-incident-reports.py | 127 +++++++++++ .../incidents/delivery-hash-mismatch.json | 37 +++ .../incidents/delivery-missing-report.json | 37 +++ .../incidents/delivery-not-reportable.json | 37 +++ .../fixtures/incidents/sts-access-denied.xml | 7 + tests/fixtures/incidents/sts-assumed.xml | 9 + tests/fixtures/incidents/sts-wrong-role.xml | 9 + tests/test_incident_delivery.py | 210 ++++++++++++++++++ tests/test_incidents.py | 4 +- tests/test_published_redaction.py | 5 +- 22 files changed, 731 insertions(+), 43 deletions(-) create mode 100644 scripts/check-incident-role.py create mode 100644 scripts/deliver-incident-reports.py create mode 100644 tests/fixtures/incidents/delivery-hash-mismatch.json create mode 100644 tests/fixtures/incidents/delivery-missing-report.json create mode 100644 tests/fixtures/incidents/delivery-not-reportable.json create mode 100644 tests/fixtures/incidents/sts-access-denied.xml create mode 100644 tests/fixtures/incidents/sts-assumed.xml create mode 100644 tests/fixtures/incidents/sts-wrong-role.xml create mode 100644 tests/test_incident_delivery.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 2af5f99a..411cc8ed 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1582,6 +1582,11 @@ jobs: run: | set -euo pipefail python3 ../scripts/build-incident-report.py --incidents ../data/incidents.json --vdr vdr-report.json --out-dir . + # SIMULATED delivery (IEC-CSO-IIR/OIR/FIR/AIR): every listed report is + # read back, hash- and schema-checked and "delivered" to a simulated + # FedRAMP and agency recipient; nothing is sent. A failed delivery + # stops the deploy here, before the SDR is built or anything published. + python3 ../scripts/deliver-incident-reports.py --dir . for f in incident-*.json; do cosign sign-blob --yes --new-bundle-format --bundle "${f%.json}.bundle" "$f" done @@ -1635,6 +1640,15 @@ jobs: --previous previous-availability.json --ksi-signal ksi-signal.json python3 ../scripts/build-policy-index.py --output policy-index.json --md-output policy-index.md + # The IEC records claim an unattended publisher that assumes its own role. + # Prove the role can be assumed, from this job, before the SDR is built: + # if it cannot, the deploy stops here and nothing claiming it is published. + # The incident-publisher job after the deploy then runs the publisher itself. + - name: Incident report publisher role can be assumed + env: + INCIDENT_REPORTS_ROLE_ARN: ${{ vars.AWS_INCIDENT_REPORTS_ROLE_ARN }} + run: python3 scripts/check-incident-role.py + # SIMULATION. The FedRAMP Security Inbox is simulated for this proof of # concept (operator's decision): nothing real is received or sent. This # runs the inbox logic (scripts/_fsi.py) over committed synthetic messages @@ -1903,6 +1917,8 @@ jobs: aws s3 cp incident-reports.json "s3://$BUCKET_NAME/.well-known/incident-reports.json" --content-type application/json --cache-control "public, max-age=300" aws s3 cp incident-reports.bundle "s3://$BUCKET_NAME/.well-known/incident-reports.bundle" --content-type application/json --cache-control "public, max-age=300" aws s3 cp incident-reports.md "s3://$BUCKET_NAME/.well-known/incident-reports.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" + aws s3 cp incident-deliveries.json "s3://$BUCKET_NAME/.well-known/incident-deliveries.json" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp incident-deliveries.bundle "s3://$BUCKET_NAME/.well-known/incident-deliveries.bundle" --content-type application/json --cache-control "public, max-age=300" for f in incident-INC-*.json; do aws s3 cp "$f" "s3://$BUCKET_NAME/.well-known/$f" --content-type application/json --cache-control "public, max-age=300" aws s3 cp "${f%.json}.bundle" "s3://$BUCKET_NAME/.well-known/${f%.json}.bundle" --content-type application/json --cache-control "public, max-age=300" diff --git a/.github/workflows/incident-reports.yml b/.github/workflows/incident-reports.yml index e8b2abee..74b713de 100644 --- a/.github/workflows/incident-reports.yml +++ b/.github/workflows/incident-reports.yml @@ -136,6 +136,12 @@ jobs: id: build run: python3 scripts/build-incident-report.py --incidents data/incidents.json --vdr vdr-report.json --out-dir out + # SIMULATED delivery: each report goes to a simulated FedRAMP and agency + # recipient and the delivery log records it. Nothing is sent. A failed + # delivery fails the run before anything is published. + - name: Deliver incident reports (simulated recipients; nothing sent) + run: python3 scripts/deliver-incident-reports.py --dir out + - name: Sign incident reports working-directory: out run: | @@ -174,6 +180,8 @@ jobs: put "$f" "application/json" put "${f%.json}.bundle" "application/json" done + put incident-deliveries.json "application/json" + put incident-deliveries.bundle "application/json" put incident-reports.md "text/markdown; charset=utf-8" put incident-reports.bundle "application/json" put incident-reports.json "application/json" diff --git a/data/frr-records.json b/data/frr-records.json index 2ebc5672..848197d0 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -810,8 +810,8 @@ }, { "frr_id": "IEC-CSO-AIR", - "status": "Partially Implemented", - "implementation": ".github/workflows/incident-reports.yml runs on every push of data/incidents.json to main, by hand, twice a day and after each nightly VDR refresh: it validates the register, verifies the published VDR, builds every report against FedRAMP's schema, signs, redaction-gates and publishes under /.well-known/incident-* with its own least-privilege role (infrastructure/bootstrap/incident-reports.tf), and fails with a GitHub issue when a report is overdue; the evidence watchdog pages on the same deadlines hourly. A person records the facts in the register; nothing else is done by hand. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). Every deploy checks the publisher's prerequisites before it applies anything (scripts/check-incident-publisher.py: the role variable names the bootstrap role and the incident-reports environment deploys only from main) and, after the deploy job, runs this workflow on its own role (deploy-with-opa.yml job incident-publisher), so the deploy fails if the publisher cannot publish.", + "status": "Implemented", + "implementation": ".github/workflows/incident-reports.yml runs on every push of data/incidents.json to main, by hand, twice a day and after each nightly VDR refresh: it validates the register, verifies the published VDR, builds every report against FedRAMP's schema, signs, redaction-gates and publishes under /.well-known/incident-* with its own least-privilege role (infrastructure/bootstrap/incident-reports.tf), and fails with a GitHub issue when a report is overdue; the evidence watchdog pages on the same deadlines hourly. A person records the facts in the register; nothing else is done by hand. Every deploy checks the publisher's prerequisites before it applies anything (scripts/check-incident-publisher.py: the role variable names the bootstrap role and the incident-reports environment deploys only from main), assumes the publisher's role from the deploy job before the SDR is built (scripts/check-incident-role.py, step 'Incident report publisher role can be assumed'), so a deploy whose publisher role does not work stops before anything is published, and after the deploy job runs this workflow on its own role (job incident-publisher). Delivery is SIMULATED: on every deploy and every unattended publish, scripts/deliver-incident-reports.py reads back each report the incident log lists, checks it against the published sha256 and FedRAMP's incident report schema, and 'delivers' it to a simulated FedRAMP recipient ('FedRAMP Incident Desk (simulated)') and to each affected agency, also simulated; a delivery that fails stops publication. The signed delivery log is /.well-known/incident-deliveries.json. Nothing is sent to FedRAMP or to any agency: there is no FedRAMP certification and no agency customer to send to.", "evidence": [ { "kind": "step", @@ -836,12 +836,36 @@ { "kind": "artifact", "ref": ".well-known/incident-reports.json" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build incident reports (IIR, OIR, FIR)" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-deliveries.json" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Incident report publisher role can be assumed" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_the_role_check_runs_in_the_deploy_job_before_the_sdr_build" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_a_role_that_cannot_be_assumed_fails_the_deploy" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_delivery_runs_before_signing_in_the_deploy_and_before_publishing_in_the_publisher" } ], - "gap": "Everything up to publication is automated; what is left is the last step of the rule, reaching all affected parties. Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). A person still records the incident's facts in the register, which the rule allows (it asks to minimize human intervention, not remove it). To close: the operator's delivery decision (IEC-CSO-IIR); the delivery is then built into this workflow.", - "closes_with": [ - "sam" - ] + "simulated": { + "counterparty": "incident-recipient", + "how": "FedRAMP and the affected agencies are simulated recipients: each report is read back, hash- and schema-checked and recorded as delivered in /.well-known/incident-deliveries.json on every deploy and every unattended publish; nothing is sent." + } }, { "frr_id": "IEC-CSO-DPR", @@ -906,8 +930,8 @@ }, { "frr_id": "IEC-CSO-FIR", - "status": "Partially Implemented", - "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). A Final report is allowed only after resolved_at and recovery_complete_at, restates final values for every earlier item, carries resolvedAt, and is due an FIR period after recovery (6 hours at N3-N5, 1 business day at N1-N2); overdue fails the workflow and pages. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "status": "Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). A Final report is allowed only after resolved_at and recovery_complete_at, restates final values for every earlier item, carries resolvedAt, and is due an FIR period after recovery (6 hours at N3-N5, 1 business day at N1-N2); overdue fails the workflow and pages. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is SIMULATED: on every deploy and every unattended publish, scripts/deliver-incident-reports.py reads back each report the incident log lists, checks it against the published sha256 and FedRAMP's incident report schema, and 'delivers' it to a simulated FedRAMP recipient ('FedRAMP Incident Desk (simulated)') and to each affected agency, also simulated; a delivery that fails stops publication. The signed delivery log is /.well-known/incident-deliveries.json. Nothing is sent to FedRAMP or to any agency: there is no FedRAMP certification and no agency customer to send to.", "evidence": [ { "kind": "test", @@ -920,17 +944,29 @@ { "kind": "artifact", "ref": ".well-known/incident-reports.json" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build incident reports (IIR, OIR, FIR)" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-deliveries.json" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_every_report_is_delivered_to_a_simulated_fedramp_and_agency_recipient" } ], - "gap": "Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). To close: the operator's delivery decision, as for IEC-CSO-IIR.", - "closes_with": [ - "sam" - ] + "simulated": { + "counterparty": "incident-recipient", + "how": "FedRAMP and the affected agencies are simulated recipients: each report is read back, hash- and schema-checked and recorded as delivered in /.well-known/incident-deliveries.json on every deploy and every unattended publish; nothing is sent." + } }, { "frr_id": "IEC-CSO-IIR", - "status": "Partially Implemented", - "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). The Initial report carries all eight items (coordinator, tracking ID, description, timeline with start/detection/evaluation/milestones, historical and current rating with the EFI explanation, functional impact, recovery plan, likely affected agencies). Its clock (1 hour at N3-N5, 24 hours at N2, 1 business day at N1) runs from the evaluation; the incident-reports workflow fails and the evidence watchdog pages (IncidentReportsOverdue) once it passes. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "status": "Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). The Initial report carries all eight items (coordinator, tracking ID, description, timeline with start/detection/evaluation/milestones, historical and current rating with the EFI explanation, functional impact, recovery plan, likely affected agencies). Its clock (1 hour at N3-N5, 24 hours at N2, 1 business day at N1) runs from the evaluation; the incident-reports workflow fails and the evidence watchdog pages (IncidentReportsOverdue) once it passes. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is SIMULATED: on every deploy and every unattended publish, scripts/deliver-incident-reports.py reads back each report the incident log lists, checks it against the published sha256 and FedRAMP's incident report schema, and 'delivers' it to a simulated FedRAMP recipient ('FedRAMP Incident Desk (simulated)') and to each affected agency, also simulated; a delivery that fails stops publication. The signed delivery log is /.well-known/incident-deliveries.json. Nothing is sent to FedRAMP or to any agency: there is no FedRAMP certification and no agency customer to send to.", "evidence": [ { "kind": "test", @@ -947,17 +983,33 @@ { "kind": "artifact", "ref": ".well-known/incident-reports.json" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build incident reports (IIR, OIR, FIR)" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-deliveries.json" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_every_report_is_delivered_to_a_simulated_fedramp_and_agency_recipient" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_known_bad_logs_fail_delivery_and_write_nothing" } ], - "gap": "Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). To close: the operator decides a delivery channel (e.g. SES from the system's domain to FedRAMP's incident address) and it is built behind that decision.", - "closes_with": [ - "sam" - ] + "simulated": { + "counterparty": "incident-recipient", + "how": "FedRAMP and the affected agencies are simulated recipients: each report is read back, hash- and schema-checked and recorded as delivered in /.well-known/incident-deliveries.json on every deploy and every unattended publish; nothing is sent." + } }, { "frr_id": "IEC-CSO-OIR", - "status": "Partially Implemented", - "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). Each Ongoing report restates every earlier item with its latest value and marks each 'updated' or 'no update', and adds observed activity, indicators of compromise, related CVEs, root cause and response/recovery activities. The next one is due an OIR period (6 hours at N3-N5, 24 at N2, 1 business day at N1) after the previous report until the incident is resolved; overdue fails the workflow and pages. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is publish-only: nothing is sent to FedRAMP or to any agency (the operator has not authorized sending; there is no FedRAMP certification and no agency customer). ", + "status": "Implemented", + "implementation": "data/incidents.json (the incident register; shape infrastructure/schemas/incidents.schema.json) is read through scripts/_incidents.py, which fails closed. scripts/build-incident-report.py renders the Initial, Ongoing and Final reports in FedRAMP's 2026-06-24 incident report schema (validated before write) and the incident log, published signed as /.well-known/incident-reports.json by every deploy and by .github/workflows/incident-reports.yml on every change to the register. Class C report clocks and the Incident Rating scale are read from the pinned rules (scripts/_rules.py). Each Ongoing report restates every earlier item with its latest value and marks each 'updated' or 'no update', and adds observed activity, indicators of compromise, related CVEs, root cause and response/recovery activities. The next one is due an OIR period (6 hours at N3-N5, 24 at N2, 1 business day at N1) after the previous report until the incident is resolved; overdue fails the workflow and pages. Exercised once, 2026-11-05: INC-2026-001 (simulated, exercise: true) runs IIR, two OIRs and an FIR on a simulated timeline, all inside their clocks; INC-2026-002 (simulated) is evaluated not reportable with its reason. Delivery is SIMULATED: on every deploy and every unattended publish, scripts/deliver-incident-reports.py reads back each report the incident log lists, checks it against the published sha256 and FedRAMP's incident report schema, and 'delivers' it to a simulated FedRAMP recipient ('FedRAMP Incident Desk (simulated)') and to each affected agency, also simulated; a delivery that fails stops publication. The signed delivery log is /.well-known/incident-deliveries.json. Nothing is sent to FedRAMP or to any agency: there is no FedRAMP certification and no agency customer to send to.", "evidence": [ { "kind": "test", @@ -970,12 +1022,24 @@ { "kind": "artifact", "ref": ".well-known/incident-reports.json" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build incident reports (IIR, OIR, FIR)" + }, + { + "kind": "artifact", + "ref": ".well-known/incident-deliveries.json" + }, + { + "kind": "test", + "ref": "tests/test_incident_delivery.py::test_every_report_is_delivered_to_a_simulated_fedramp_and_agency_recipient" } ], - "gap": "Reports are published under /.well-known/incident-* and not delivered: nothing sends them to FedRAMP or to an agency, by the operator's decision (notification: published-only in data/incidents.json; this system has no FedRAMP certification and no agency customer). To close: the operator's delivery decision, as for IEC-CSO-IIR.", - "closes_with": [ - "sam" - ] + "simulated": { + "counterparty": "incident-recipient", + "how": "FedRAMP and the affected agencies are simulated recipients: each report is read back, hash- and schema-checked and recorded as delivered in /.well-known/incident-deliveries.json on every deploy and every unattended publish; nothing is sent." + } }, { "frr_id": "IVV-CSO-DUS", diff --git a/data/incidents.json b/data/incidents.json index f00b35da..0bd171dc 100644 --- a/data/incidents.json +++ b/data/incidents.json @@ -4,7 +4,7 @@ "policy": { "federal_incident_coordinator": "Sam Aydlette, System Owner and incident response lead; contact through https://samaydlette.com/.well-known/security.txt", "notification": "published-only", - "notification_text": "Published only. Each report is signed and published under https://samaydlette.com/.well-known/ (incident-reports.json lists every report and its sha256). Nothing is sent to FedRAMP or to any agency: the operator has not authorized sending, and this system has no FedRAMP certification and no agency customers.", + "notification_text": "Published, and delivered to SIMULATED recipients. Each report is signed and published under https://samaydlette.com/.well-known/ (incident-reports.json lists every report and its sha256) and 'delivered' to a simulated FedRAMP and agency recipient, recorded in incident-deliveries.json. Nothing is sent to FedRAMP or to any agency: this system has no FedRAMP certification and no agency customers.", "adopted_vulnerability_rules": ["VER-TFR-IRI", "VER-TFR-NRI"], "federal_customer_data": "None. This system holds no federal customer data (FRD-FCD) and has no agency customers. Were an agency to use the Silk Reeling application, what its users supply to it (pose landmarks) would be the federal customer data to consider; the operator's public site content, account information and the published evidence are not.", "decided_by": "Operator", diff --git a/docs/cr26-applicability.md b/docs/cr26-applicability.md index 30052eba..9ae361aa 100644 --- a/docs/cr26-applicability.md +++ b/docs/cr26-applicability.md @@ -24,8 +24,9 @@ The POA&M is retired (2026-11-04): CR26 has no POA&M. Open findings and their de - **IVV — Independent Verification and Validation.** `IVV-CSO-FIA` (Class C MUST) requires a persistent independent assessment by a **FedRAMP Recognized independent assessment service** (the artifact formerly called a 3PAO) or by FedRAMP. This system is self-attested and has engaged no such assessor, so the *independent* half cannot be met. The provider-side verification and validation an assessor would consume **are** produced (in the Security Decision Record); the independent half is unmet by design and recorded N/A. - **MKT — Marketplace Listing.** `MKT-IIP-AGU` requires demonstrating the offering is for agency use within 44 USC § 3506 scope, and the rule text is explicit that *"services used by private companies to meet other compliance requirements (such as CMMC) that do not also meet one of the above use cases are outside the scope of FedRAMP."* There is no agency use case and no intent to list, so MKT is N/A by statutory scope. -- **IEC — Incident Evaluation and Communication.** A FedRAMP Reportable Incident under `IEC-CSO-EFR` is triggered by an effect on the confidentiality or integrity of **federal customer data**. This system holds none, so in practice no incident is FedRAMP-reportable. Every incident is still evaluated and recorded in `data/incidents.json`; reportable ones get Initial, Ongoing and Final reports in FedRAMP's schema, on the Class C clocks, published under `/.well-known/` (publish-only: nothing is sent). Exercised 2026-11-05 with simulated incidents marked as exercises; see `docs/incident-response.md`. -These three are N/A because of *what this system is*, not because of an engineering gap. If the system were ever sponsored, listed, assessed, or handling federal data, each would activate, and the mechanisms above (the SDR's V&V evidence, the CDS overview package, the PAIN evaluation) are the inputs those activated obligations would consume. +These two are N/A because of *what this system is*, not because of an engineering gap. If the system were ever sponsored, listed, assessed, or handling federal data, each would activate, and the mechanisms above (the SDR's V&V evidence, the CDS overview package, the PAIN evaluation) are the inputs those activated obligations would consume. **AFC — Addressing FedRAMP Communication** is no longer recorded N/A (operator's decision, 2026-11-05): it is implemented against a *simulated* FedRAMP Security Inbox. The inbox logic (`scripts/_fsi.py`) runs on every deploy over committed synthetic messages and publishes `/.well-known/fedramp-inbox-simulation.json`, labelled as a simulation; no mail is received or sent, and the published contact sends FedRAMP to the operator on LinkedIn. See `docs/runbooks/fedramp-security-inbox.md`. + +**IEC — Incident Evaluation and Communication** is not recorded N/A either. This system holds no federal customer data, so in practice no incident is FedRAMP-reportable, but every incident is evaluated and recorded in `data/incidents.json`, and reportable ones get Initial, Ongoing and Final reports in FedRAMP's schema, on the Class C clocks, published under `/.well-known/` and delivered to *simulated* FedRAMP and agency recipients (`/.well-known/incident-deliveries.json`; nothing is sent). Exercised 2026-11-05 with simulated incidents marked as exercises; see `docs/incident-response.md`. diff --git a/docs/incident-response.md b/docs/incident-response.md index 657682a8..bb5e1382 100644 --- a/docs/incident-response.md +++ b/docs/incident-response.md @@ -55,7 +55,8 @@ Reporting is automated from the incident register. A person records facts; the p - **Build and publish.** `scripts/build-incident-report.py` writes each report in FedRAMP's incident report schema (`incident---.json`, validated before it is written) and the incident log (`incident-reports.json` and `.md`: every incident, its evaluation and reason, its rating history, each report with its sha256 and what changed, and each clock). `.github/workflows/incident-reports.yml` runs it on every push of the register to `main`, by hand, twice a day and after each nightly VDR refresh; it signs, redaction-gates and publishes under `/.well-known/incident-*` with a role that can write nothing else. Every deploy republishes the same set. - **Alarms.** The workflow's last step fails, and raises a GitHub issue, when a report is past its deadline. The evidence SLA watchdog reads the published log hourly and pages by email (`incident-report-overdue`, [`runbooks/evidence-sla-alarm.md`](runbooks/evidence-sla-alarm.md)). - **Vulnerabilities as incidents.** The operator adopts `VER-TFR-IRI` and `VER-TFR-NRI` (`policy.adopted_vulnerability_rules`). The VDR triage policy names each finding they cover; until an unresolved incident in the register lists it under `findings`, its Initial report is due from the finding's first evaluation at N5, so the alarms fire. Open the incident with basis `VER-TFR-IRI` or `VER-TFR-NRI` and the finding's tracking ID or CVE. Reconciliation invariant (m) fails a build whose open incident names a finding the VDR no longer carries: close it with a Final report. -- **Notification is publish-only.** Nothing is sent to FedRAMP or to any agency; the operator has not authorized sending, and the system has no certification and no agency customers. Every report says so. FedRAMP and the affected agencies are "all affected parties" under the rules; reaching them is a decision for the operator. +- **Delivery is simulated.** FedRAMP and the affected agencies are "all affected parties" under the rules. This system has no certification and no agency customers, so they are simulated recipients: on every deploy and every run of the incident reports workflow, `scripts/deliver-incident-reports.py` reads back each report the log lists, checks its sha256 and FedRAMP's schema, and records its "delivery" to "FedRAMP Incident Desk (simulated)" and to each affected agency (simulated) in `/.well-known/incident-deliveries.json`. A delivery that fails stops publication. Nothing is sent to anyone. +- **The publisher's role is proved on every deploy.** Before the Security Decision Record is built, the deploy job assumes the incident reports role (`scripts/check-incident-role.py`); if it cannot, the deploy stops. - **Exercises.** A simulated incident is recorded with `exercise: true` and is published marked as one. Exercised 2026-11-05: `INC-2026-001` (a simulated internet-reachable N5 vulnerability: Initial, two Ongoing and a Final report on a simulated timeline, all on time; rated N5 by default, then estimated N3) and `INC-2026-002` (a simulated page change, evaluated not reportable with its reason). `tests/test_incidents.py` runs the same exercise through the builder on every `make check`. ## After-action reports (KSI-INR-AAR) diff --git a/docs/policies/ir-policy.md b/docs/policies/ir-policy.md index aa3bd3e3..17408ddd 100644 --- a/docs/policies/ir-policy.md +++ b/docs/policies/ir-policy.md @@ -6,7 +6,7 @@ External communication paths: - **Public reporter:** [`/.well-known/security.txt`](../../website/.well-known/security.txt) is the published vulnerability-reporting mailbox. - **FedRAMP incident reports (IEC):** every incident is recorded in the incident register, [`data/incidents.json`](../../data/incidents.json), with its reportability evaluation. Initial, Ongoing and Final Incident Reports for a FedRAMP Reportable Incident are generated from it, validated against FedRAMP's incident report schema, signed, and published under `/.well-known/` (the incident log is `/.well-known/incident-reports.json`) by an unattended workflow on every change to the register. -- **Notification is publish-only.** Nothing is sent to FedRAMP or to any agency. The operator has not authorized sending, and the system has no FedRAMP certification and no agency customers. Choosing a delivery channel is an operator decision; until it is made, publication is the only channel and every report says so. +- **Delivery is simulated.** Nothing is sent to FedRAMP or to any agency: the system has no FedRAMP certification and no agency customers. Each report is published and "delivered" to simulated FedRAMP and agency recipients, recorded in `/.well-known/incident-deliveries.json`; a delivery that fails stops publication. - **FedRAMP Security Inbox (FSI, 20x):** simulated for this proof of concept. No mail is received or sent; the inbox logic runs on every deploy over synthetic messages and its result is published as `/.well-known/fedramp-inbox-simulation.json`. The published contact directs FedRAMP to the operator on LinkedIn. Procedure: `docs/runbooks/fedramp-security-inbox.md`. **20x rule integration.** KSI-INR (Incident Response), IEC (Incident Evaluation and Communication), FSI (FedRAMP Security Inbox). diff --git a/infrastructure/bootstrap/incident-reports.tf b/infrastructure/bootstrap/incident-reports.tf index d38fb120..27328344 100644 --- a/infrastructure/bootstrap/incident-reports.tf +++ b/infrastructure/bootstrap/incident-reports.tf @@ -25,9 +25,12 @@ variable "incident_reports_environment" { default = "incident-reports" } -# TRUST: this repository, this GitHub Environment, nothing else. A pull request -# cannot assume it, and neither can a workflow that does not declare the -# environment. +# TRUST: this repository, two GitHub Environments, nothing else. A pull request +# cannot assume it, and neither can a workflow that does not declare one of +# them. The second is the reviewer-gated deploy environment: every deploy +# assumes this role before it builds the SDR (scripts/check-incident-role.py), +# so a deploy cannot publish records claiming a publisher whose role does not +# work. That adds no privilege: the deploy role can already write the bucket. data "aws_iam_policy_document" "incident_reports_trust" { statement { effect = "Allow" @@ -47,7 +50,10 @@ data "aws_iam_policy_document" "incident_reports_trust" { condition { test = "StringEquals" variable = "token.actions.githubusercontent.com:sub" - values = ["repo:${var.github_repo}:environment:${var.incident_reports_environment}"] + values = [ + "repo:${var.github_repo}:environment:${var.incident_reports_environment}", + "repo:${var.github_repo}:environment:${var.deploy_environment}", + ] } } } diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index ccb9eb40..7d129210 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -98,6 +98,18 @@ "signature": ".well-known/incident-reports.bundle", "note": "The incident log (IEC-CSO-EFR) and, through it, every incident report it lists with its sha256. Rebuilt by .github/workflows/incident-reports.yml on every change to data/incidents.json and twice a day, and by every deploy. The report deadlines themselves are hours, so the watchdog also pages on each listed report past its deadline (IncidentReportsOverdue)." }, + { + "path": ".well-known/incident-deliveries.json", + "cadence": "nightly", + "max_age_hours": 24, + "timestamp": { + "source": "json", + "field": "emitted_at" + }, + "signed": true, + "signature": ".well-known/incident-deliveries.bundle", + "note": "SIMULATION: the delivery log of every incident report to a simulated FedRAMP and agency recipient; nothing is sent. Rebuilt with the incident log by every deploy and by .github/workflows/incident-reports.yml." + }, { "path": ".well-known/ksi-signal-runtime.json", "cadence": "runtime", diff --git a/scripts/build-incident-report.py b/scripts/build-incident-report.py index 7c67def4..2c6a1076 100644 --- a/scripts/build-incident-report.py +++ b/scripts/build-incident-report.py @@ -29,9 +29,10 @@ # report due from the finding's first evaluation at the default rating; the # clock check fails, and the evidence watchdog pages, once that time passes. # -# Notification. Delivery to FedRAMP and agencies is not built: the register's -# policy says "published-only" and every report says so. Publication under -# /.well-known/ is the only channel until the operator decides otherwise. +# Notification. Nothing is sent to FedRAMP or to agencies: the register's +# policy says "published-only" and every report says so. After this builder, +# scripts/deliver-incident-reports.py "delivers" each report to SIMULATED +# FedRAMP and agency recipients and writes the delivery log. # # build-incident-report.py --incidents ../data/incidents.json --vdr vdr-report.json --out-dir . # build-incident-report.py --check incident-reports.json # exit 1 if a report is overdue diff --git a/scripts/check-incident-publisher.py b/scripts/check-incident-publisher.py index e31d3cfa..cd07288b 100644 --- a/scripts/check-incident-publisher.py +++ b/scripts/check-incident-publisher.py @@ -18,9 +18,10 @@ the deploy's later incident-publisher job assumes the role from that environment, so a wrong environment still fails the deploy there. -Whether the role itself works is proved later in the same run: the -`incident-publisher` job calls .github/workflows/incident-reports.yml, which -assumes the role, publishes, and fails the deploy if it cannot. +Whether the role itself works is proved in the same job before the SDR is +built (scripts/check-incident-role.py assumes it), and again after the deploy: +the `incident-publisher` job calls .github/workflows/incident-reports.yml, +which assumes the role, publishes, and fails the deploy if it cannot. Prints no ARN or account ID. Exit 0 pass, 1 fail. """ diff --git a/scripts/check-incident-role.py b/scripts/check-incident-role.py new file mode 100644 index 00000000..192c5241 --- /dev/null +++ b/scripts/check-incident-role.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Deploy gate: the incident report publisher's role can actually be assumed. + +The IEC records describe a system whose unattended publisher +(.github/workflows/incident-reports.yml) assumes the role +infrastructure/bootstrap/incident-reports.tf creates. The deploy job runs this +before it builds the SDR, so a deploy whose publisher role does not work stops +before any record claiming it is published: + + 1. request the job's GitHub OIDC token (audience sts.amazonaws.com); + 2. call STS AssumeRoleWithWebIdentity for AWS_INCIDENT_REPORTS_ROLE_ARN; + 3. require success, and that the assumed role is + github-actions-incident-reports-oidc. + +The temporary credentials STS returns are discarded unused. Prints no token, +credential, ARN or account ID. Stdlib only. Exit 0 pass, 1 fail. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET + +EXPECTED_ROLE_NAME = "github-actions-incident-reports-oidc" +SESSION = "deploy-proves-incident-publisher" +STS_URL = "https://sts.{region}.amazonaws.com/" +REGION_RE = re.compile(r"^[a-z]{2}(-[a-z]+)+-[0-9]$") +ASSUMED_RE = re.compile(r"^arn:aws(-[a-z]+)*:sts::[0-9]{12}:assumed-role/([\w+=,.@-]+)/[\w+=,.@-]+$") + + +def _local(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def parse_sts(status: int, body: str) -> list[str]: + """Problems with an AssumeRoleWithWebIdentity response, or [].""" + try: + root = ET.fromstring(body) + except ET.ParseError: + return [f"STS returned HTTP {status} with a body that is not XML"] + found = {_local(e.tag): (e.text or "").strip() for e in root.iter()} + if status != 200 or "Error" in found: + code = re.sub(r"[^A-Za-z.]", "", found.get("Code", "unknown"))[:64] + return [f"STS refused the incident publisher role ({code}, HTTP {status}): the role or its trust for the " + f"deploy environment is missing; apply infrastructure/bootstrap/incident-reports.tf"] + m = ASSUMED_RE.match(found.get("Arn", "")) + if not m: + return ["STS answered without an assumed-role identity"] + if m.group(2) != EXPECTED_ROLE_NAME: + return [f"the assumed role is not {EXPECTED_ROLE_NAME}"] + return [] + + +def oidc_token(env=os.environ, opener=urllib.request.urlopen) -> str: + url, bearer = env.get("ACTIONS_ID_TOKEN_REQUEST_URL"), env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + if not url or not bearer: + raise RuntimeError("no GitHub OIDC token available (the job needs permissions: id-token: write)") + sep = "&" if "?" in url else "?" + req = urllib.request.Request(url + sep + "audience=sts.amazonaws.com", headers={"Authorization": f"bearer {bearer}"}) + with opener(req, timeout=30) as r: + return json.loads(r.read())["value"] + + +def assume(role_arn: str, token: str, region: str, opener=urllib.request.urlopen) -> tuple[int, str]: + data = urllib.parse.urlencode({"Action": "AssumeRoleWithWebIdentity", "Version": "2011-06-15", + "RoleArn": role_arn, "RoleSessionName": SESSION, "DurationSeconds": "900", + "WebIdentityToken": token}).encode() + req = urllib.request.Request(STS_URL.format(region=region), data=data, method="POST") + try: + with opener(req, timeout=30) as r: + return r.status, r.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace") + + +def main(argv=None, env=os.environ, opener=urllib.request.urlopen) -> int: + role_arn = (env.get("INCIDENT_REPORTS_ROLE_ARN") or "").strip() + region = env.get("AWS_REGION") or "us-east-2" + if not REGION_RE.match(region): + # The region becomes part of the STS host name; never let it pick the host. + problems = ["AWS_REGION is not an AWS region name"] + elif not role_arn: + problems = ["AWS_INCIDENT_REPORTS_ROLE_ARN is not set"] + else: + try: + status, body = assume(role_arn, oidc_token(env, opener), region, opener) + problems = parse_sts(status, body) + except (RuntimeError, urllib.error.URLError, KeyError, ValueError) as e: + problems = [f"could not ask STS: {type(e).__name__}"] + for p in problems: + print(f"::error::incident publisher role: {p}") + if not problems: + print(f"incident publisher role: assumed {EXPECTED_ROLE_NAME} from the deploy job; credentials discarded") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/deliver-incident-reports.py b/scripts/deliver-incident-reports.py new file mode 100644 index 00000000..00140170 --- /dev/null +++ b/scripts/deliver-incident-reports.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Deliver every incident report to a SIMULATED FedRAMP and agency recipient. + +IEC-CSO-IIR/OIR/FIR ask that each report reach FedRAMP and every affected +agency; IEC-CSO-AIR asks that this be automated. This proof of concept has no +FedRAMP certification and no agency customer, so by the operator's decision the +recipients are simulated and NOTHING IS SENT. What is real is the mechanism: +every report the incident log lists is read back, checked against the sha256 +the log published for it and against FedRAMP's incident report schema, and +handed to each recipient, whose (simulated) receipt is the hash it received. + +Any failure exits 1 and writes nothing, which stops the deploy (and the +unattended publisher) before anything is published: + - a report the log lists is missing, or its bytes differ from the log's sha256 + - a report does not validate against FedRAMP's incident report schema + - a report file in the directory is not listed in the log + - a reportable incident has a report with no FedRAMP or no agency recipient + +On success it writes incident-deliveries.json, the delivery log, labelled as a +simulation, for the workflow to sign and publish under /.well-known/. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _fedramp_schema as fs # noqa: E402 + +FEDRAMP = "FedRAMP Incident Desk (simulated): accepts every report, replies to none, and has never heard of this system" +DEFAULT_AGENCY = "Department of Hypothetical Affairs (simulated): this system has no agency customer" +NOTICE = ("SIMULATION. Every incident report below was 'delivered' to simulated recipients: nothing was sent to " + "FedRAMP or to any agency, and no inbox anywhere received anything. The delivery mechanism is real and runs " + "on every deploy and every unattended publish: each report is read back, checked against the sha256 in " + "incident-reports.json and against FedRAMP's incident report schema, and its receipt is that hash. A " + "delivery that fails stops publication. Real recipients would need an operator decision and a FedRAMP " + "certification, neither of which exists.") + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def deliver(log: dict, report_dir: Path, now: str | None = None, registry=None) -> tuple[dict | None, list[str]]: + """(delivery log, problems). The delivery log is None when there are problems.""" + problems: list[str] = [] + deliveries = [] + listed = set() + for inc in log.get("incidents", []): + reports = inc.get("reports") or [] + if reports and not (inc.get("evaluation") or {}).get("reportable"): + problems.append(f"{inc.get('id')}: reports issued for an incident evaluated not reportable") + for r in reports: + name = r.get("file") or "" + listed.add(name) + path = report_dir / name + if "/" in name or not name.startswith("incident-INC-") or not path.is_file(): + problems.append(f"{inc.get('id')}: report {name!r} is listed but missing") + continue + got = sha256(path) + if got != r.get("sha256"): + problems.append(f"{name}: bytes differ from the sha256 the incident log publishes") + continue + doc = json.loads(path.read_text()) + errors = fs.validate(doc, "incident", registry) + if errors: + problems.append(f"{name}: does not validate against FedRAMP's incident report schema ({errors[0]})") + continue + agencies = [a for a in (doc.get("affectedAgencies") or []) if isinstance(a, str) and a.strip()] + recipients: list[dict[str, object]] = [{"recipient": FEDRAMP, "kind": "fedramp"}] + recipients += [{"recipient": f"{a} (simulated)", "kind": "agency"} for a in agencies] or \ + [{"recipient": DEFAULT_AGENCY, "kind": "agency"}] + for rc in recipients: + rc.update(simulated=True, transport="none: simulated, nothing sent", receipt_sha256=got, + result="accepted (simulated)") + kinds = {rc["kind"] for rc in recipients} + if kinds != {"fedramp", "agency"}: + problems.append(f"{name}: no FedRAMP and agency recipient") + deliveries.append({"incident": inc.get("id"), "exercise": bool(inc.get("exercise")), + "report_type": r.get("type"), "file": name, "sha256": got, + "issued_at": r.get("issued_at"), "recipients": recipients}) + for path in sorted(report_dir.glob("incident-INC-*.json")): + if path.name not in listed: + problems.append(f"{path.name}: a report file the incident log does not list") + if problems: + return None, problems + return { + "schema": "incident-deliveries/1", + "simulation": True, + "notice": NOTICE, + "emitted_at": now or datetime.now(timezone.utc).isoformat(), + "generator": "scripts/deliver-incident-reports.py", + "incident_log": "https://samaydlette.com/.well-known/incident-reports.json", + "incident_log_emitted_at": log.get("emitted_at"), + "summary": {"reports_delivered": len(deliveries), + "recipients_reached": sum(len(d["recipients"]) for d in deliveries), + "messages_actually_sent": 0}, + "deliveries": deliveries, + }, [] + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--dir", type=Path, default=Path("."), help="where incident-reports.json and the reports are") + ap.add_argument("--output", type=Path, default=None) + a = ap.parse_args(argv) + log = json.loads((a.dir / "incident-reports.json").read_text()) + out, problems = deliver(log, a.dir) + for p in problems: + print(f"::error::incident report delivery (simulated): {p}") + if problems or out is None: + print(f"incident report delivery: {len(problems)} problem(s); nothing delivered, nothing written") + return 1 + target = a.output or (a.dir / "incident-deliveries.json") + target.write_text(json.dumps(out, indent=2) + "\n") + print(f"incident report delivery (SIMULATED, nothing sent): {out['summary']['reports_delivered']} report(s) to " + f"{out['summary']['recipients_reached']} simulated recipient(s); wrote {target}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/incidents/delivery-hash-mismatch.json b/tests/fixtures/incidents/delivery-hash-mismatch.json new file mode 100644 index 00000000..5c98f87d --- /dev/null +++ b/tests/fixtures/incidents/delivery-hash-mismatch.json @@ -0,0 +1,37 @@ +{ + "schema": "incident-reports/1", + "emitted_at": "2026-11-05T00:00:00Z", + "incidents": [ + { + "id": "INC-2026-001", + "exercise": true, + "evaluation": { + "reportable": true + }, + "reports": [ + { + "type": "Initial", + "issued_at": "2026-11-05T03:40:00Z", + "file": "incident-INC-2026-001-01-initial.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "changes": { + "coordinator": "updated", + "tracking_id": "updated", + "description": "updated", + "timeline": "updated", + "rating": "updated", + "functional_impact": "updated", + "recovery_plan": "updated", + "affected_agencies": "updated", + "observed_activity": "not yet available", + "indicators_of_compromise": "not yet available", + "related_cves": "updated", + "root_cause": "not yet available", + "response_and_recovery_activities": "not yet available" + } + } + ] + } + ], + "_note": "KNOWN-BAD: the log's sha256 for the Initial report is not the report's; delivery must refuse it." +} diff --git a/tests/fixtures/incidents/delivery-missing-report.json b/tests/fixtures/incidents/delivery-missing-report.json new file mode 100644 index 00000000..a04c457b --- /dev/null +++ b/tests/fixtures/incidents/delivery-missing-report.json @@ -0,0 +1,37 @@ +{ + "schema": "incident-reports/1", + "emitted_at": "2026-11-05T00:00:00Z", + "incidents": [ + { + "id": "INC-2026-001", + "exercise": true, + "evaluation": { + "reportable": true + }, + "reports": [ + { + "type": "Initial", + "issued_at": "2026-11-05T03:40:00Z", + "file": "incident-INC-2026-001-09-ongoing.json", + "sha256": "5857e833e1eb8103600a828261e7e51cc3a2aad0e63ba23bd02292f678433d93", + "changes": { + "coordinator": "updated", + "tracking_id": "updated", + "description": "updated", + "timeline": "updated", + "rating": "updated", + "functional_impact": "updated", + "recovery_plan": "updated", + "affected_agencies": "updated", + "observed_activity": "not yet available", + "indicators_of_compromise": "not yet available", + "related_cves": "updated", + "root_cause": "not yet available", + "response_and_recovery_activities": "not yet available" + } + } + ] + } + ], + "_note": "KNOWN-BAD: the log lists a report that was never written; delivery must refuse it." +} diff --git a/tests/fixtures/incidents/delivery-not-reportable.json b/tests/fixtures/incidents/delivery-not-reportable.json new file mode 100644 index 00000000..1be3e9b3 --- /dev/null +++ b/tests/fixtures/incidents/delivery-not-reportable.json @@ -0,0 +1,37 @@ +{ + "schema": "incident-reports/1", + "emitted_at": "2026-11-05T00:00:00Z", + "incidents": [ + { + "id": "INC-2026-002", + "exercise": true, + "evaluation": { + "reportable": false + }, + "reports": [ + { + "type": "Initial", + "issued_at": "2026-11-05T03:40:00Z", + "file": "incident-INC-2026-001-01-initial.json", + "sha256": "5857e833e1eb8103600a828261e7e51cc3a2aad0e63ba23bd02292f678433d93", + "changes": { + "coordinator": "updated", + "tracking_id": "updated", + "description": "updated", + "timeline": "updated", + "rating": "updated", + "functional_impact": "updated", + "recovery_plan": "updated", + "affected_agencies": "updated", + "observed_activity": "not yet available", + "indicators_of_compromise": "not yet available", + "related_cves": "updated", + "root_cause": "not yet available", + "response_and_recovery_activities": "not yet available" + } + } + ] + } + ], + "_note": "KNOWN-BAD: reports listed under an incident evaluated not reportable; delivery must refuse them." +} diff --git a/tests/fixtures/incidents/sts-access-denied.xml b/tests/fixtures/incidents/sts-access-denied.xml new file mode 100644 index 00000000..a3f7cdb3 --- /dev/null +++ b/tests/fixtures/incidents/sts-access-denied.xml @@ -0,0 +1,7 @@ + + + Sender + AccessDenied + Not authorized to perform sts:AssumeRoleWithWebIdentity + + diff --git a/tests/fixtures/incidents/sts-assumed.xml b/tests/fixtures/incidents/sts-assumed.xml new file mode 100644 index 00000000..3326d81c --- /dev/null +++ b/tests/fixtures/incidents/sts-assumed.xml @@ -0,0 +1,9 @@ + + + + arn:aws:sts::000000000000:assumed-role/github-actions-incident-reports-oidc/deploy-proves-incident-publisher + AROAEXAMPLE:deploy-proves-incident-publisher + + EXAMPLEEXAMPLEEXAMPLE + + diff --git a/tests/fixtures/incidents/sts-wrong-role.xml b/tests/fixtures/incidents/sts-wrong-role.xml new file mode 100644 index 00000000..e6db430a --- /dev/null +++ b/tests/fixtures/incidents/sts-wrong-role.xml @@ -0,0 +1,9 @@ + + + + arn:aws:sts::000000000000:assumed-role/github-actions-deploy-oidc/deploy-proves-incident-publisher + AROAEXAMPLE:deploy-proves-incident-publisher + + EXAMPLEEXAMPLEEXAMPLE + + diff --git a/tests/test_incident_delivery.py b/tests/test_incident_delivery.py new file mode 100644 index 00000000..f2e8cafa --- /dev/null +++ b/tests/test_incident_delivery.py @@ -0,0 +1,210 @@ +"""Incident reports reach a SIMULATED FedRAMP and agency recipient, and the +publisher's role is proved before the SDR is built (IEC-CSO-IIR/OIR/FIR/AIR). + +scripts/deliver-incident-reports.py "delivers" every report the incident log +lists (nothing is sent) and fails the deploy when one cannot be delivered; +scripts/check-incident-role.py assumes the incident publisher's role from the +deploy job. Known-bad fixtures under tests/fixtures/incidents/ each fail for +one reason.""" + +import importlib.util +import io +import json +import re +import shutil +import subprocess +import sys +import urllib.error +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "incidents" +WORKFLOW = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() +PUBLISHER = (REPO / ".github" / "workflows" / "incident-reports.yml").read_text() + + +def _load(name, file): + spec = importlib.util.spec_from_file_location(name, REPO / "scripts" / file) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +deliver = _load("deliver_incident_reports", "deliver-incident-reports.py") +role = _load("check_incident_role", "check-incident-role.py") + + +@pytest.fixture +def built(tmp_path): + """The committed register's reports, built as the deploy builds them.""" + (tmp_path / "vdr.json").write_text('{"findings": []}') + subprocess.run([sys.executable, str(REPO / "scripts" / "build-incident-report.py"), "--incidents", + str(REPO / "data" / "incidents.json"), "--vdr", str(tmp_path / "vdr.json"), + "--out-dir", str(tmp_path)], check=True, capture_output=True) + return tmp_path + + +def _log(d): + return json.loads((d / "incident-reports.json").read_text()) + + +# --- delivery ------------------------------------------------------------------- + +def test_every_report_is_delivered_to_a_simulated_fedramp_and_agency_recipient(built): + out, problems = deliver.deliver(_log(built), built) + assert problems == [] + listed = [r["file"] for i in _log(built)["incidents"] for r in i.get("reports", [])] + assert listed and [d["file"] for d in out["deliveries"]] == listed + for d in out["deliveries"]: + assert {r["kind"] for r in d["recipients"]} == {"fedramp", "agency"} + assert all(r["simulated"] is True and "(simulated)" in r["recipient"] for r in d["recipients"]) + assert all(r["receipt_sha256"] == d["sha256"] for r in d["recipients"]) + assert out["simulation"] is True and out["notice"].startswith("SIMULATION.") + assert out["summary"]["messages_actually_sent"] == 0 + + +def test_the_cli_writes_the_delivery_log_and_it_carries_no_address(built): + assert deliver.main(["--dir", str(built)]) == 0 + text = (built / "incident-deliveries.json").read_text() + assert not re.search(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text) + assert not re.search(r"\b\d{12}\b", text) + + +def _with_fixture(built, name, keep): + shutil.copy(FIX / f"{name}.json", built / "incident-reports.json") + for p in built.glob("incident-INC-*.json"): + if p.name not in keep: + p.unlink() + return deliver.main(["--dir", str(built)]) + + +@pytest.mark.parametrize("name,keep,reason", [ + ("delivery-hash-mismatch", {"incident-INC-2026-001-01-initial.json"}, "bytes differ from the sha256"), + ("delivery-missing-report", set(), "is listed but missing"), + ("delivery-not-reportable", {"incident-INC-2026-001-01-initial.json"}, "evaluated not reportable"), +]) +def test_known_bad_logs_fail_delivery_and_write_nothing(built, capsys, name, keep, reason): + assert _with_fixture(built, name, keep) == 1 + out = capsys.readouterr().out + assert reason in out and not (built / "incident-deliveries.json").exists() + assert len([ln for ln in out.splitlines() if ln.startswith("::error::")]) == 1, out + + +def test_an_unlisted_report_file_fails_delivery(built): + (built / "incident-INC-2099-999-01-initial.json").write_text("{}") + _, problems = deliver.deliver(_log(built), built) + assert problems == ["incident-INC-2099-999-01-initial.json: a report file the incident log does not list"] + + +def test_a_report_that_does_not_validate_against_fedramps_schema_fails_delivery(built): + log = _log(built) + r = log["incidents"][0]["reports"][0] + p = built / r["file"] + doc = json.loads(p.read_text()) + doc.pop("reportType") + p.write_text(json.dumps(doc)) + r["sha256"] = deliver.sha256(p) + _, problems = deliver.deliver(log, built) + assert len(problems) == 1 and "does not validate" in problems[0] + + +def test_delivery_runs_before_signing_in_the_deploy_and_before_publishing_in_the_publisher(): + build = WORKFLOW[WORKFLOW.index("- name: Build incident reports (IIR, OIR, FIR)"):] + build = build[:build.index("\n - name: ")] + assert build.index("deliver-incident-reports.py --dir .") < build.index("cosign sign-blob") + assert "|| true" not in build + assert WORKFLOW.index("- name: Build incident reports") < WORKFLOW.index("- name: Build Security Decision Record") + for f in ("incident-deliveries.json", "incident-deliveries.bundle"): + assert f'aws s3 cp {f} "s3://$BUCKET_NAME/.well-known/{f}"' in WORKFLOW + assert f'put {f} "application/json"' in PUBLISHER + step = PUBLISHER.index("- name: Deliver incident reports (simulated recipients; nothing sent)") + assert PUBLISHER.index("- name: Build incident reports") < step < PUBLISHER.index("- name: Sign incident reports") + cfg = json.loads((REPO / "infrastructure" / "schemas" / "freshness-config.json").read_text()) + e = [a for a in cfg["artifacts"] if a["path"] == ".well-known/incident-deliveries.json"] + assert len(e) == 1 and e[0]["signed"] and e[0]["note"].startswith("SIMULATION") + + +# --- the role, proved from the deploy job ------------------------------------------- + +@pytest.mark.parametrize("fixture,ok", [("sts-assumed.xml", True), ("sts-access-denied.xml", False), + ("sts-wrong-role.xml", False)]) +def test_sts_responses(fixture, ok): + status = 403 if "denied" in fixture else 200 + problems = role.parse_sts(status, (FIX / fixture).read_text()) + assert (problems == []) is ok, problems + + +class _Resp(io.BytesIO): + def __init__(self, body, status=200): + super().__init__(body.encode()) + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +def _opener(sts_fixture): + calls = [] + + def opener(req, timeout=None): + calls.append(req) + if req.full_url.startswith("https://token.example/"): + assert "audience=sts.amazonaws.com" in req.full_url + return _Resp(json.dumps({"value": "oidc-token"})) + assert req.full_url == "https://sts.us-east-2.amazonaws.com/" and b"AssumeRoleWithWebIdentity" in req.data + body = (FIX / sts_fixture).read_text() + if "denied" in sts_fixture: + raise urllib.error.HTTPError(req.full_url, 403, "Forbidden", {}, io.BytesIO(body.encode())) + return _Resp(body) + return opener, calls + + +ENV = {"INCIDENT_REPORTS_ROLE_ARN": "arn:aws:iam::000000000000:role/github-actions-incident-reports-oidc", + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.example/?x=1", "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "t", + "AWS_REGION": "us-east-2"} + + +def test_the_deploy_proves_the_role_by_assuming_it(capsys): + opener, calls = _opener("sts-assumed.xml") + assert role.main(env=ENV, opener=opener) == 0 and len(calls) == 2 + out = capsys.readouterr().out + assert "000000000000" not in out and "oidc-token" not in out and "EXAMPLE" not in out + + +@pytest.mark.parametrize("fixture", ["sts-access-denied.xml", "sts-wrong-role.xml"]) +def test_a_role_that_cannot_be_assumed_fails_the_deploy(capsys, fixture): + opener, _ = _opener(fixture) + assert role.main(env=ENV, opener=opener) == 1 + assert "000000000000" not in capsys.readouterr().out + + +def test_no_variable_or_no_oidc_token_fails_the_deploy(): + assert role.main(env={**ENV, "INCIDENT_REPORTS_ROLE_ARN": ""}, opener=None) == 1 + env = {k: v for k, v in ENV.items() if not k.startswith("ACTIONS_")} + assert role.main(env=env, opener=_opener("sts-assumed.xml")[0]) == 1 + + +def test_the_role_check_runs_in_the_deploy_job_before_the_sdr_build(): + deploy_job = WORKFLOW[WORKFLOW.index("\n deploy:\n"):] + deploy_job = deploy_job[:re.search(r"\n [a-z][\w-]*:\n", deploy_job[1:]).start() + 1] + i = deploy_job.index("- name: Incident report publisher role can be assumed") + assert i < deploy_job.index("- name: Build Security Decision Record") + step = deploy_job[i:deploy_job.index("\n - name: ", i)] + assert "python3 scripts/check-incident-role.py" in step and "continue-on-error" not in step + assert "vars.AWS_INCIDENT_REPORTS_ROLE_ARN" in step + + +def test_the_role_trusts_the_deploy_environment(): + tf = (REPO / "infrastructure" / "bootstrap" / "incident-reports.tf").read_text() + assert "environment:${var.deploy_environment}" in tf + assert "environment:${var.incident_reports_environment}" in tf + + +def test_the_region_cannot_redirect_the_token_to_another_host(): + opener, calls = _opener("sts-assumed.xml") + assert role.main(env={**ENV, "AWS_REGION": "evil.example/x"}, opener=opener) == 1 and calls == [] diff --git a/tests/test_incidents.py b/tests/test_incidents.py index 36cc70ba..1f1b435f 100644 --- a/tests/test_incidents.py +++ b/tests/test_incidents.py @@ -73,7 +73,7 @@ def test_the_committed_register_adopts_both_vulnerability_rules_and_publishes_on p = json.loads(REGISTER.read_text())["policy"] assert set(p["adopted_vulnerability_rules"]) == {"VER-TFR-IRI", "VER-TFR-NRI"} assert p["notification"] == "published-only" - assert "not authorized" in p["notification_text"] + assert "Nothing is sent" in p["notification_text"] and "SIMULATED" in p["notification_text"] BAD = { @@ -306,7 +306,7 @@ def test_reports_are_deterministic_and_say_they_are_published_only(exercise, tmp for a, b in zip(log["incidents"], again["incidents"], strict=True): assert [r["sha256"] for r in a["reports"]] == [r["sha256"] for r in b["reports"]] md = (out / "incident-reports.md").read_text() - assert "published-only" in md and "not authorized" in md and "(no update)" in md + assert "published-only" in md and "Nothing is sent" in md and "(no update)" in md assert log["notification"]["mode"] == "published-only" diff --git a/tests/test_published_redaction.py b/tests/test_published_redaction.py index c63752c0..96068092 100644 --- a/tests/test_published_redaction.py +++ b/tests/test_published_redaction.py @@ -200,12 +200,13 @@ def test_deploy_gates_exactly_what_it_publishes(tmp_path): (tmp_path / f"{a}.json").write_text("{}") # The incident reports are named by incident, so the steps use globs; with # the files the builder writes present, both expand to the same set. - incident_files = ("incident-reports.json", "incident-reports.md", "incident-INC-2026-001-01-initial.json") + incident_files = ("incident-reports.json", "incident-reports.md", "incident-deliveries.json", + "incident-INC-2026-001-01-initial.json") for f in incident_files: (tmp_path / f).write_text("{}") assert "incident-INC-*.json" in publish publish = {n for n in publish if "*" not in n} | {f for f in incident_files if f.startswith("incident-INC-")} - assert {"incident-reports.json", "incident-reports.md"} <= publish + assert {"incident-reports.json", "incident-reports.md", "incident-deliveries.json"} <= publish assert _gate_files(gate_step, tmp_path) == publish # One absent (the operator-run corroboration): it is neither uploaded nor scanned. (tmp_path / "boundary-corroboration.json").unlink() From 4c492c42faecd9121045c3a0416ba37e88a76e73 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:48:28 -0500 Subject: [PATCH 17/97] feat(assessor): simulated assessor signs a keyless set on every deploy; SDR carries it; cited tests must pass Totally Real Assessment Services, LLC (simulated) runs as a reusable workflow before the deploy job, works from the generated assessor guide, re-runs every cited check, and signs one set under its own workflow identity, which infrastructure/keys/assessor-identity.json pins. The SDR build requires and verifies the set. The unit-test job fails when a test cited as rule evidence did not pass (FRC-CSX-VVR). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 45 ++- .github/workflows/simulated-assessor.yml | 85 +++++ infrastructure/keys/assessor-identity.json | 10 + .../schemas/assessor-record.schema.json | 7 +- scripts/_assessor.py | 108 +++++- scripts/build-assessor-guide.py | 165 ++++++++ scripts/build-sdr.py | 9 +- scripts/check-frr-evidence.py | 119 ++++++ .../assessor/simulated-set-tampered.json | 103 +++++ tests/fixtures/assessor/simulated-set.json | 103 +++++ .../frr-evidence/failing-cited-test.xml | 6 + tests/fixtures/frr-evidence/passing.xml | 8 + tests/test_frr_evidence.py | 60 +++ tests/test_simulated_assessor.py | 296 +++++++++++++++ tools/simulated-assessor/assess.py | 352 ++++++++++++++++++ 15 files changed, 1464 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/simulated-assessor.yml create mode 100644 infrastructure/keys/assessor-identity.json create mode 100644 scripts/build-assessor-guide.py create mode 100644 scripts/check-frr-evidence.py create mode 100644 tests/fixtures/assessor/simulated-set-tampered.json create mode 100644 tests/fixtures/assessor/simulated-set.json create mode 100644 tests/fixtures/frr-evidence/failing-cited-test.xml create mode 100644 tests/fixtures/frr-evidence/passing.xml create mode 100644 tests/test_frr_evidence.py create mode 100644 tests/test_simulated_assessor.py create mode 100644 tools/simulated-assessor/assess.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 24a24847..0d7ab6fc 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -195,7 +195,13 @@ jobs: run: python3 scripts/_frr.py --complete - name: Python unit tests - run: python3 -m pytest tests/ -q + run: python3 -m pytest tests/ -q --junitxml=pytest-results.xml + + # FRC-CSX-VVR: every test a FedRAMP rule record cites as evidence ran and + # passed in the run above (a skipped or missing one fails too). The result + # per record is signed and published by the deploy as frr-evidence.json. + - name: FedRAMP rule evidence passed (cited tests) + run: python3 scripts/check-frr-evidence.py --junit pytest-results.xml --output frr-evidence.json --commit "${{ github.sha }}" # REQ-GOV-2: run the method's conformance tests (data/method-profile.json # maps Appendix B T1-T74 to the tests that implement them) and record the @@ -211,7 +217,9 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: conformance-results - path: conformance-results.json + path: | + conformance-results.json + frr-evidence.json if-no-files-found: error # The policy suite is an exhibit: format is enforced, strict compilation @@ -596,7 +604,7 @@ jobs: # appended here by scripts/_gates.py and published with this run's evidence. env: GATE_EVENTS_FILE: /tmp/gate-events.jsonl - needs: [test, compliance-check, security-scan] + needs: [test, compliance-check, security-scan, simulated-assessor] # IMPORTANT: Only deploy if on main branch AND security check passed if: github.ref == 'refs/heads/main' && needs.compliance-check.outputs.compliant == 'true' environment: ${{ github.event.inputs.environment || 'prod' }} @@ -695,6 +703,14 @@ jobs: name: conformance-results path: infrastructure/ + # The simulated assessor's signed set (simulated-assessment.json + .bundle), + # verified against the pinned assessor identity when the SDR is built. + - name: Download the simulated assessment + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: simulated-assessment + path: infrastructure/ + # ------------------------------------------------------------------------- # REBUILD MONITORING FUNCTION # ------------------------------------------------------------------------- @@ -1596,10 +1612,12 @@ jobs: python3 ../scripts/build-sdr.py \ --catalog schemas/ksi-catalog.json --ksi-signal ksi-signal.json \ --runtime-signal ksi-signal-runtime.json \ - --vdr vdr-report.json --profile ../data/system-profile.json --previous-sdr previous-sdr.json + --vdr vdr-report.json --profile ../data/system-profile.json --previous-sdr previous-sdr.json \ + --assessment simulated-assessment.json --assessment-bundle simulated-assessment.bundle # REQ-ASR-1 / T74: re-read the written SDR and fail if any assessor text # differs from the assessor's signed record, or a signed record is missing. - python3 ../scripts/_assessor.py --sdr security-decision-record.json + python3 ../scripts/_assessor.py --sdr security-decision-record.json \ + --assessment simulated-assessment.json --assessment-bundle simulated-assessment.bundle - name: Build Certification Package Overview (CPO) working-directory: infrastructure @@ -2070,6 +2088,23 @@ jobs: if: always() run: rm -f infrastructure/tfstate-bootstrap.json + # =========================================================================== + # SIMULATED ASSESSOR (not a FedRAMP Recognized assessment) + # =========================================================================== + # "Totally Real Assessment Services, LLC (simulated)" verifies and validates + # every rule and KSI from the assessor guide and signs its record under its + # own workflow identity (a reusable workflow, so the certificate names + # simulated-assessor.yml). Runs where the deploy runs; the deploy needs it, and + # the SDR build fails without its signed set. + simulated-assessor: + name: Simulated assessor + needs: [test, compliance-check] + if: github.ref == 'refs/heads/main' && needs.compliance-check.outputs.compliant == 'true' + permissions: + contents: read + id-token: write + uses: ./.github/workflows/simulated-assessor.yml + # =========================================================================== # JOB 3: EXTRA SECURITY SCANS # =========================================================================== diff --git a/.github/workflows/simulated-assessor.yml b/.github/workflows/simulated-assessor.yml new file mode 100644 index 00000000..c32778f4 --- /dev/null +++ b/.github/workflows/simulated-assessor.yml @@ -0,0 +1,85 @@ +# ============================================================================= +# SIMULATED ASSESSOR — "Totally Real Assessment Services, LLC (simulated)" +# ============================================================================= +# SIMULATED. Not independent of the provider, not a FedRAMP Recognized +# independent assessment service, no FedRAMP assessor ID. It stands in for an +# assessor so the assessor port (IVV-CSO-FIA/ICP, IVV-CSX-AIA, CPO-CSO-OSA, +# SDR-CSO-FRR) is exercised end to end on every deploy. +# +# The deploy calls this as a reusable workflow before its deploy job. Because it +# is a reusable workflow, the Sigstore certificate that signs its record names +# THIS file (job_workflow_ref), not deploy-with-opa.yml: the assessor signs +# under its own identity, which infrastructure/keys/assessor-identity.json pins. +# It holds no AWS credentials and reads only public URLs and the repository. +# +# It follows the provider's assessor guide (scripts/build-assessor-guide.py): +# re-runs every test a rule record cites, finds every cited workflow step and +# decision, fetches every cited published artifact and verifies its signature +# and age, and checks every KSI against the live, verified KSI signal. Then it +# writes one signed set: a result per rule, per KSI, and an overall summary. +# Any failure fails this job, and the deploy job needs it. +# ============================================================================= +name: Simulated assessor (not a FedRAMP assessment) + +on: + workflow_call: + +permissions: {} + +env: + # Kept equal to deploy-with-opa.yml's pins by tests/test_simulated_assessor.py. + OPA_VERSION: 1.18.2 + OPA_SHA256: 9903e5125ac281104f2c4b7371d10cc3b74a98933743fcbfc174f9bf0ab20de8 + +jobs: + assess: + name: Totally Real Assessment Services, LLC (simulated) + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + id-token: write # keyless signing under this workflow's own identity + steps: + - name: Checkout the commit under assessment + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install the assessor's tools + run: | + python3 -m pip install --user pytest jsonschema pyyaml cryptography + curl -fL --retry 5 --retry-delay 5 --connect-timeout 10 --max-time 120 \ + -o opa "https://openpolicyagent.org/downloads/v${OPA_VERSION}/opa_linux_amd64_static" + echo "${OPA_SHA256} opa" | sha256sum -c - + chmod 755 ./opa + sudo mv opa /usr/local/bin + + - name: Install cosign + uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 + + # IVV-CSO-STE: the provider supplies the technical explanation (the guide); + # the assessor works from it. + - name: Receive the provider's assessor guide + run: python3 scripts/build-assessor-guide.py --commit "$GITHUB_SHA" --output assessor-guide.json --md-output assessor-guide.md + + - name: Assess every rule and KSI (verification and validation) + run: python3 tools/simulated-assessor/assess.py --guide assessor-guide.json --repo . --commit "$GITHUB_SHA" --output simulated-assessment.json + + - name: Sign the assessment (keyless, the assessor's own identity) + run: cosign sign-blob --yes --new-bundle-format --bundle simulated-assessment.bundle simulated-assessment.json + + - name: Verify the assessment signature against the pinned assessor identity + run: | + id=$(python3 -c 'import json; print(json.load(open("infrastructure/keys/assessor-identity.json"))["certificate_identity"])') + cosign verify-blob --bundle simulated-assessment.bundle \ + --certificate-identity "$id" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com simulated-assessment.json + + - name: Hand the signed assessment to the provider + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: simulated-assessment + path: | + simulated-assessment.json + simulated-assessment.bundle + if-no-files-found: error diff --git a/infrastructure/keys/assessor-identity.json b/infrastructure/keys/assessor-identity.json new file mode 100644 index 00000000..831d1535 --- /dev/null +++ b/infrastructure/keys/assessor-identity.json @@ -0,0 +1,10 @@ +{ + "assessor": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml", + "algorithm": "SIGSTORE_KEYLESS", + "certificate_identity": "https://github.com/sam-aydlette/samaydlette.com/.github/workflows/simulated-assessor.yml@refs/heads/main", + "certificate_oidc_issuer": "https://token.actions.githubusercontent.com", + "pinned_on": "2026-11-05", + "simulated": true, + "note": "SIMULATED assessor. Its signing identity is the simulated-assessor.yml workflow in this repository (called by the deploy as a reusable workflow, so its Sigstore certificate names that workflow, not the deploy). It is not independent of the provider, not a FedRAMP Recognized independent assessment service, and has no FedRAMP assessor ID. Replacing it with a real assessor means pinning that assessor's own identity here in a reviewed change." +} diff --git a/infrastructure/schemas/assessor-record.schema.json b/infrastructure/schemas/assessor-record.schema.json index bd992c27..4b152eb7 100644 --- a/infrastructure/schemas/assessor-record.schema.json +++ b/infrastructure/schemas/assessor-record.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://samaydlette.com/schemas/assessor-record.schema.json", "title": "Assessor record (REQ-ASR-1)", - "description": "A finding, summary or comment written and signed by an assessor. The signature is ECDSA P-256 with SHA-256 over the canonical form of the record without its \"signature\" member: keys sorted, separators \",\" and \":\" with no whitespace, UTF-8, non-ASCII characters unescaped. The signer must be the assessor identity pinned at infrastructure/keys/assessor-identity.json. The provider never edits a record; outputs carry its text verbatim with the signature.", + "description": "A finding, summary or comment written and signed by an assessor. The signature is ECDSA P-256 with SHA-256 over the canonical form of the record without its \"signature\" member: keys sorted, separators \",\" and \":\" with no whitespace, UTF-8, non-ASCII characters unescaped. The signer must be the assessor identity pinned at infrastructure/keys/assessor-identity.json. The provider never edits a record; outputs carry its text verbatim with the signature. A keyless assessor (pinned identity algorithm SIGSTORE_KEYLESS) instead signs one set file holding all its records with Sigstore, and each record carries algorithm SIGSTORE_KEYLESS_SET.", "type": "object", "additionalProperties": false, "required": ["record_type", "schema_version", "record_id", "assessor", "issued_at", "kind", "text", "signature"], @@ -28,8 +28,9 @@ "additionalProperties": false, "required": ["algorithm", "value"], "properties": { - "algorithm": {"const": "ECDSA_P256_SHA256"}, - "value": {"type": "string", "minLength": 1, "description": "Base64 DER signature."} + "algorithm": {"enum": ["ECDSA_P256_SHA256", "SIGSTORE_KEYLESS_SET"]}, + "value": {"type": "string", "minLength": 1, "description": "Base64 DER signature (ECDSA_P256_SHA256), or the hex sha256 of the record's canonical form (SIGSTORE_KEYLESS_SET)."}, + "signed_set": {"type": "string", "minLength": 1, "description": "SIGSTORE_KEYLESS_SET only: the name of the Sigstore-signed set file holding this record."} } } } diff --git a/scripts/_assessor.py b/scripts/_assessor.py index 52cc5092..07967498 100644 --- a/scripts/_assessor.py +++ b/scripts/_assessor.py @@ -17,11 +17,23 @@ With no records there is nothing to verify and no identity is needed; the SDR states "no assessor record". + +Keyless set (load_assessment): an assessor that signs with Sigstore instead of a +key it holds signs one set file holding all its records. The pinned identity +then names the signing workflow's certificate identity and OIDC issuer +(algorithm SIGSTORE_KEYLESS). The set is accepted only when cosign verifies its +bundle against exactly that identity, and each record in it is accepted only +when it names the pinned assessor and its signature value is the sha256 of its +canonical form (algorithm SIGSTORE_KEYLESS_SET), so a record is bound to the +signed set and cannot be altered alone. The set and its bundle are published, +so anyone can repeat the check. This is how the simulated assessor +(tools/simulated-assessor/, .github/workflows/simulated-assessor.yml) signs. """ import base64 import copy import hashlib import json +import subprocess # nosec B404: runs cosign with a fixed argument list import sys from pathlib import Path @@ -30,6 +42,9 @@ IDENTITY_PATH = REPO / "infrastructure" / "keys" / "assessor-identity.json" ALGORITHM = "ECDSA_P256_SHA256" +KEYLESS = "SIGSTORE_KEYLESS" # the pinned identity of a keyless assessor +SET_ALGORITHM = "SIGSTORE_KEYLESS_SET" # a record bound to a keyless-signed set +SET_TYPE = "simulated-independent-assessment" KINDS = ("finding", "summary", "comment") REQUIRED = ("record_type", "schema_version", "record_id", "assessor", "issued_at", "kind", "text", "signature") # What an output carries for each record: the signed fields, verbatim, plus the signature. @@ -60,8 +75,8 @@ def _shape(rec, name): a, s = rec["assessor"], rec["signature"] if not isinstance(a, dict) or not a.get("organization") or not a.get("key_id"): _fail(f"{name} does not name its assessor (organization and key_id)") - if not isinstance(s, dict) or s.get("algorithm") != ALGORITHM or not s.get("value"): - _fail(f"{name} carries no {ALGORITHM} signature") + if not isinstance(s, dict) or s.get("algorithm") not in (ALGORITHM, SET_ALGORITHM) or not s.get("value"): + _fail(f"{name} carries no {ALGORITHM} or {SET_ALGORITHM} signature") for k in ("record_id", "issued_at", "text"): if not isinstance(rec[k], str) or not rec[k].strip(): _fail(f"{name} has an empty {k}") @@ -75,6 +90,9 @@ def _load_identity(path): _fail(f"records are present but no assessor identity is pinned at {p}") try: ident = json.loads(p.read_text()) + if ident.get("algorithm") == KEYLESS: + _fail(f"the pinned identity {p} is keyless: its records arrive only as a signed set " + "(load_assessment), never as loose files") pem = ident["public_key_pem"].encode("ascii") except (json.JSONDecodeError, KeyError, AttributeError, UnicodeEncodeError) as exc: _fail(f"pinned identity {p} is unreadable ({type(exc).__name__})") @@ -86,6 +104,8 @@ def _load_identity(path): def _verify(rec, name, ident, pem): + if rec["signature"]["algorithm"] != ALGORITHM: + _fail(f"{name} is not signed with {ALGORITHM}") try: from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives import hashes, serialization @@ -127,6 +147,83 @@ def load_records(records_dir=RECORDS_DIR, identity_path=IDENTITY_PATH): return records +# --- keyless set ------------------------------------------------------------------------ + +def load_keyless_identity(path=IDENTITY_PATH): + p = Path(path) + try: + ident = json.loads(p.read_text()) + except (OSError, json.JSONDecodeError) as exc: + _fail(f"no readable keyless assessor identity at {p} ({type(exc).__name__})") + if not isinstance(ident, dict) or ident.get("algorithm") != KEYLESS: + _fail(f"the identity pinned at {p} is not a {KEYLESS} identity") + for k in ("assessor", "key_id", "certificate_identity", "certificate_oidc_issuer"): + if not isinstance(ident.get(k), str) or not ident[k]: + _fail(f"the identity pinned at {p} does not name its {k}") + return ident + + +def cosign_verify(set_path, bundle_path, ident): + """True when cosign verifies the bundle over the set file against exactly the + pinned certificate identity and issuer (no regular expression).""" + r = subprocess.run(["cosign", "verify-blob", "--bundle", str(bundle_path), + "--certificate-identity", ident["certificate_identity"], + "--certificate-oidc-issuer", ident["certificate_oidc_issuer"], str(set_path)], + capture_output=True, text=True, check=False) # nosec B603: fixed argv + return r.returncode == 0 + + +def load_assessment(set_path, bundle_path, identity_path=IDENTITY_PATH, verifier=cosign_verify): + """Every record in a keyless-signed assessment set, verified. Fails closed when + the set or its bundle is missing: the deploy depends on the assessment.""" + sp, bp = Path(set_path), Path(bundle_path) + for f, what in ((sp, "assessment set"), (bp, "assessment bundle")): + if not f.is_file() or f.stat().st_size == 0: + _fail(f"the {what} {f} is missing: the deploy depends on the assessor's signed record") + ident = load_keyless_identity(identity_path) + if not verifier(sp, bp, ident): + _fail(f"{sp.name} signature does not verify against the pinned assessor identity") + try: + doc = json.loads(sp.read_text()) + except json.JSONDecodeError: + _fail(f"{sp.name} is not valid JSON") + if not isinstance(doc, dict) or doc.get("assessment_type") != SET_TYPE or doc.get("schema_version") != "1": + _fail(f"{sp.name} is not a {SET_TYPE} set, schema version 1") + a = doc.get("assessor") or {} + if a.get("organization") != ident["assessor"] or a.get("key_id") != ident["key_id"]: + _fail(f"{sp.name} names {a.get('organization')!r}, not the pinned assessor identity") + recs = doc.get("records") + if not isinstance(recs, list) or not recs: + _fail(f"{sp.name} holds no records") + seen, out = set(), [] + for i, rec in enumerate(recs): + name = f"{sp.name} record {i}" + _shape(rec, name) + ra, sig = rec["assessor"], rec["signature"] + if ra["organization"] != ident["assessor"] or ra["key_id"] != ident["key_id"]: + _fail(f"{name} names {ra['organization']!r} / {ra['key_id']!r}, not the pinned assessor identity") + if sig["algorithm"] != SET_ALGORITHM or sig.get("signed_set") != sp.name: + _fail(f"{name} is not bound to the signed set {sp.name}") + if sig["value"] != hashlib.sha256(canonical(rec)).hexdigest(): + _fail(f"{name} ({rec['record_id']}) differs from the record the assessor signed") + if rec["record_id"] in seen: + _fail(f"record_id {rec['record_id']} appears twice") + seen.add(rec["record_id"]) + out.append(rec) + if not any(r["kind"] == "summary" and "rule_id" not in r for r in out): + _fail(f"{sp.name} has no overall summary (IVV-IAS-OSA)") + return out + + +def overall_summary(records): + """The assessor's overall summary record (IVV-IAS-OSA), carried verbatim, or + None. The CPO quotes it (CPO-CSO-OSA).""" + sums = [r for r in records if r["kind"] == "summary" and "rule_id" not in r] + if len(sums) > 1: + _fail("more than one overall summary record") + return _carried(sums[0]) if sums else None + + def _carried(rec): return {k: copy.deepcopy(rec[k]) for k in CARRIED} @@ -183,8 +280,13 @@ def main(argv=None): ap.add_argument("--records", default=str(RECORDS_DIR)) ap.add_argument("--identity", default=str(IDENTITY_PATH)) ap.add_argument("--sdr", default=None, help="a built security-decision-record.json to check") + ap.add_argument("--assessment", default=None, help="a keyless-signed assessment set") + ap.add_argument("--assessment-bundle", default=None) a = ap.parse_args(argv) - records = load_records(a.records, a.identity) + if a.assessment: + records = load_assessment(a.assessment, a.assessment_bundle or a.assessment[:-5] + ".bundle", a.identity) + else: + records = load_records(a.records, a.identity) if a.sdr: verify_output(json.loads(Path(a.sdr).read_text()), records) print(f"assessor records: {len(records)} verified" + ("; SDR carries them verbatim" if a.sdr else "")) diff --git a/scripts/build-assessor-guide.py b/scripts/build-assessor-guide.py new file mode 100644 index 00000000..b7cf5eba --- /dev/null +++ b/scripts/build-assessor-guide.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""The assessor guide: how to verify and validate each FedRAMP rule and KSI +here, by hand or by machine (IVV-CSO-STE, IVV-CSO-DUS). + +Generated from the rule records (data/frr-records.json) and the KSI catalog, so +it cannot describe a mechanism the records do not cite. For every rule it gives +the provider's explanation and, for every piece of evidence, the command that +reproduces the check and a machine-readable form of the same check. The +simulated assessor (tools/simulated-assessor/assess.py) executes exactly those +machine-readable checks on every deploy: the guide is the demonstration. + +It also states the sampling position (IVV-CSO-DUS): no check here uses a +representative sample. + + python3 scripts/build-assessor-guide.py --output assessor-guide.json --md-output assessor-guide.md +""" + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _freshness # noqa: E402 +import _frr # noqa: E402 +import _rules # noqa: E402 + +REPO = Path(__file__).resolve().parent.parent +SITE = "https://samaydlette.com" +REPO_URL = "https://github.com/sam-aydlette/samaydlette.com" +ISSUER = "https://token.actions.githubusercontent.com" +WF = f"{REPO_URL}/.github/workflows" +IDENTITIES = { + "deploy": f"{WF}/deploy-with-opa.yml@refs/heads/main", + "nightly": f"{WF}/evidence-nightly.yml@refs/heads/main", + # The simulated assessor signs its own record under its own workflow identity. + "assessor": f"{WF}/simulated-assessor.yml@refs/heads/main", +} +SCHEMA = "assessor-guide/1" +SAMPLING = ("No representative samples are used in verification or validation (IVV-CSO-DUS). Every check runs " + "over its full population: the OPA gate and the KSI signal evaluate every inventoried component; the " + "reconciliation gate compares every live in-boundary resource with the inventory; the vulnerability " + "scan covers every SBOM; every FedRAMP rule record is validated and every test it cites runs on every " + "build; and the simulated assessor re-checks every rule and every KSI rather than a selection.") +HOW_TO_USE = ("Read each rule's explanation, then run the command beside each piece of evidence. Tests run from a " + "clone of the repository after `make dev-setup`; published artifacts are fetched from the site and " + "their signatures verified with cosign against the publishing workflow's identity; workflow steps " + "are read in the workflow file and their run logs in the repository's Actions tab; decision records " + "are in data/decisions.json. The `check` beside each command is the same check in machine-readable " + "form. For a walkthrough of any mechanism, ask through the security contact in the Certification " + "Package Overview.") + + +def _artifact_check(ref, fresh): + entry = fresh[ref] + name = ref.split("/", 1)[1] + sig = entry.get("signature") + return {"url": f"{SITE}/{ref}", "bundle_url": f"{SITE}/{sig}" if sig else None, + "identities": (["assessor"] if name.startswith("simulated-assessment") + else ["deploy", "nightly"] if name.startswith("vdr-") else ["deploy"]), + "timestamp": entry["timestamp"], "max_age_hours": entry["max_age_hours"]} + + +def reproduce(ev, fresh): + kind, ref = ev["kind"], ev["ref"] + if kind == "test": + return f"python3 -m pytest '{ref}'", {"pytest": ref} + if kind == "step": + wf, step = ref.split("#", 1) + return (f"Read the step named {step!r} in .github/workflows/{wf}, and its log in the latest run of that " + "workflow on main", {"workflow": wf, "step": step}) + if kind == "artifact": + chk = _artifact_check(ref, fresh) + cmd = f"curl -fsSLO {chk['url']}" + if chk["bundle_url"]: + cmd += (f" && curl -fsSLO {chk['bundle_url']} && cosign verify-blob --bundle {chk['bundle_url'].rsplit('/', 1)[1]} " + f"--certificate-identity {IDENTITIES[chk['identities'][0]]} --certificate-oidc-issuer {ISSUER} " + f"{ref.rsplit('/', 1)[1]}") + return cmd, {"artifact": chk} + if kind == "decision": + return f"jq '.decisions[] | select(.id == \"{ref}\")' data/decisions.json", {"decision": ref} + raise SystemExit(f"::error::assessor guide: unknown evidence kind {kind!r}") + + +def build(records, applicable, catalog, fresh, now, commit=None): + rules = [] + for r in records["records"]: + rid = r["frr_id"] + evidence = [] + for ev in r.get("evidence", []): + cmd, chk = reproduce(ev, fresh) + evidence.append({"kind": ev["kind"], "ref": ev["ref"], "how_to_reproduce": cmd, "check": chk}) + entry = {"rule_id": rid, "name": applicable[rid]["name"], "force": applicable[rid]["force"], + "statement": applicable[rid]["statement"], "status": r["status"], + "explanation": r["implementation"], "evidence": evidence} + for k in ("not_applicable", "may_rationale", "simulated"): + if k in r: + entry[k] = r[k] + rules.append(entry) + ksis = [{"ksi_id": i["id"], "name": i.get("name", ""), + "how_to_reproduce": (f"curl -fsSL {SITE}/.well-known/ksi-signal.json | jq '.ksis[] | " + f"select(.id == \"{i['id']}\")' after verifying ksi-signal.bundle against the " + "deploy identity; the SDR's keySecurityIndicators entry gives its measures"), + "check": {"ksi_in_signal": i["id"]}} + for f in catalog["KSI"].values() for i in f["indicators"]] + return {"schema": SCHEMA, "generated_at": now.isoformat(), "commit": commit, "site": SITE, + "repository": REPO_URL, "how_to_use": HOW_TO_USE, "sampling": SAMPLING, + "identities": {**IDENTITIES, "oidc_issuer": ISSUER}, + "summary": {"rules": len(rules), "ksis": len(ksis), + "evidence_items": sum(len(r["evidence"]) for r in rules)}, + "rules": rules, "ksis": sorted(ksis, key=lambda k: k["ksi_id"])} + + +def render_markdown(g): + out = ["# Assessor guide", "", + "_How to verify and validate each FedRAMP rule and Key Security Indicator for this offering. Generated " + "from the rule records on every deploy; the simulated assessor runs these exact checks._", "", + f"- **Generated:** {g['generated_at']} | **Commit:** `{g['commit'] or 'local build'}`", + f"- **Rules:** {g['summary']['rules']} | **KSIs:** {g['summary']['ksis']} | " + f"**Evidence items:** {g['summary']['evidence_items']}", "", + "## How to use this guide", "", g["how_to_use"], "", + "## Sampling (IVV-CSO-DUS)", "", g["sampling"], "", + "## Signing identities", ""] + out += [f"- {k}: `{v}`" for k, v in g["identities"].items()] + out += ["", "## FedRAMP rules", ""] + for r in g["rules"]: + out += [f"### {r['rule_id']}: {r['name']} ({r['force']}, {r['status']})", "", r["explanation"], ""] + for k, label in (("not_applicable", "Not applicable"), ("may_rationale", "Use of this permission")): + if k in r: + out += [f"{label}: {r[k]}", ""] + if "simulated" in r: + out += [f"Simulated counterparty ({r['simulated']['counterparty']}): {r['simulated']['how']}", ""] + for ev in r["evidence"]: + out.append(f"- {ev['kind']} `{ev['ref']}`: `{ev['how_to_reproduce']}`") + if r["evidence"]: + out.append("") + out += ["## Key Security Indicators", ""] + out += [f"- `{k['ksi_id']}` {k['name']}: `{k['how_to_reproduce']}`" for k in g["ksis"]] + out += ["", "---", "_Source of truth: the signed `assessor-guide.json`._"] + return "\n".join(out) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--records", default=str(_frr.DEFAULT_PATH)) + ap.add_argument("--catalog", default=str(REPO / "infrastructure" / "schemas" / "ksi-catalog.json")) + ap.add_argument("--freshness", default=str(REPO / "infrastructure" / "schemas" / "freshness-config.json")) + ap.add_argument("--commit", default=None) + ap.add_argument("--output", default="assessor-guide.json") + ap.add_argument("--md-output", default="assessor-guide.md") + a = ap.parse_args(argv) + records = _frr.load(a.records) + fresh = {x["path"]: x for x in _freshness.load_freshness_config(a.freshness)["artifacts"]} + guide = build(records, _rules.applicable_provider_rules(), json.loads(Path(a.catalog).read_text()), fresh, + datetime.now(timezone.utc), a.commit) + Path(a.output).write_text(json.dumps(guide, indent=2) + "\n") + if a.md_output: + Path(a.md_output).write_text(render_markdown(guide) + "\n") + print(f"assessor-guide: {guide['summary']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build-sdr.py b/scripts/build-sdr.py index d209947a..d0e1f11d 100644 --- a/scripts/build-sdr.py +++ b/scripts/build-sdr.py @@ -454,6 +454,10 @@ def main(argv=None): ap.add_argument("--md-output", default="security-decision-record.md") ap.add_argument("--assessor-records", default=str(_assessor.RECORDS_DIR)) ap.add_argument("--assessor-identity", default=str(_assessor.IDENTITY_PATH)) + # A keyless-signed assessment set (the simulated assessor's, on every deploy): + # given, it is required to exist and verify; its records replace the loose ones. + ap.add_argument("--assessment", default=None) + ap.add_argument("--assessment-bundle", default=None) a = ap.parse_args(argv) now = datetime.now(timezone.utc) @@ -473,7 +477,10 @@ def main(argv=None): applicable = _rules.applicable_provider_rules() irs = _controls.load(a.ksi_signal, a.catalog) runtime, reason = runtime_validations(a.runtime_signal) - assessor_records = _assessor.load_records(a.assessor_records, a.assessor_identity) + if a.assessment: + assessor_records = _assessor.load_assessment(a.assessment, a.assessment_bundle, a.assessor_identity) + else: + assessor_records = _assessor.load_records(a.assessor_records, a.assessor_identity) doc = build(catalog, signal, records, applicable, irs, moderate_ids(), policy_rules(), runtime, reason, assessor_records, prof, prev, vdr, now, policy_documents()) _assessor.verify_output(doc, assessor_records) # T74 diff --git a/scripts/check-frr-evidence.py b/scripts/check-frr-evidence.py new file mode 100644 index 00000000..9f6ae293 --- /dev/null +++ b/scripts/check-frr-evidence.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Every test a FedRAMP rule record cites ran and passed (FRC-CSX-VVR). + +scripts/_frr.py shows that each piece of evidence a rule record cites exists. +This shows the cited tests passed in this build: it reads the JUnit XML that the +unit-test job's pytest run wrote and fails when a test cited as evidence failed, +errored, was skipped, or did not run at all. A parametrized test passes only +when every one of its cases ran and passed. + +It writes the result per record (frr-evidence.json), which the deploy signs and +publishes beside the Security Decision Record, so a reader can see that each +Implemented record's tests passed in the build that published it. + + python3 scripts/check-frr-evidence.py --junit pytest-results.xml --output frr-evidence.json +""" + +import argparse +import json +import sys +import xml.etree.ElementTree as ET # nosec B405: parses the JUnit file this job's own pytest wrote +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _frr # noqa: E402 + +SCHEMA = "frr-evidence/1" + + +def _fail(msg): + raise SystemExit(f"::error::FRR evidence: {msg} (fail-closed)") + + +def junit_outcomes(path): + """{(module path, class or None, function): [outcome per case]} from a JUnit XML file.""" + try: + root = ET.parse(path).getroot() # nosec B314: trusted, locally generated file + except (OSError, ET.ParseError) as exc: + _fail(f"cannot read the test results {path}: {exc}") + out = {} + for case in root.iter("testcase"): + classname, name = case.get("classname", ""), case.get("name", "") + parts = classname.split(".") + # classname is "tests.test_x" or "tests.test_x.TestClass" + if len(parts) >= 2 and parts[-1][:1].isupper(): + module, cls = "/".join(parts[:-1]) + ".py", parts[-1] + else: + module, cls = "/".join(parts) + ".py", None + func = name.split("[", 1)[0] + if case.find("failure") is not None or case.find("error") is not None: + outcome = "failed" + elif case.find("skipped") is not None: + outcome = "skipped" + else: + outcome = "passed" + out.setdefault((module, cls, func), []).append(outcome) + if not out: + _fail(f"{path} records no test cases") + return out + + +def _key(ref): + m = _frr.TEST_RE.match(ref) + if not m: + _fail(f"{ref!r} is not a test reference") + rel, first, second = m.groups() + return (rel, first, second) if second else (rel, None, first) + + +def test_result(ref, outcomes): + cases = outcomes.get(_key(ref)) + if not cases: + return "not run" + if all(c == "passed" for c in cases): + return "passed" + return "failed" if "failed" in cases else "skipped" + + +def evaluate(records, outcomes): + """(per-record results, list of failures).""" + per, failures = {}, [] + for r in records["records"]: + tests = {e["ref"]: test_result(e["ref"], outcomes) for e in r.get("evidence", []) if e["kind"] == "test"} + for ref, res in tests.items(): + if res != "passed": + failures.append(f"{r['frr_id']}: cited test {ref} {res}") + per[r["frr_id"]] = {"status": r["status"], "tests": tests, + "result": "pass" if all(v == "passed" for v in tests.values()) else "fail"} + return per, failures + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--junit", required=True) + ap.add_argument("--records", default=str(_frr.DEFAULT_PATH)) + ap.add_argument("--output", default="frr-evidence.json") + ap.add_argument("--commit", default=None) + a = ap.parse_args(argv) + records = _frr.load(a.records) + per, failures = evaluate(records, junit_outcomes(a.junit)) + cited = sum(len(v["tests"]) for v in per.values()) + doc = {"schema": SCHEMA, "generated_at": datetime.now(timezone.utc).isoformat(), "commit": a.commit, + "explanation": "Each FedRAMP rule record's cited tests and whether they passed in the unit-test run " + "of the build that published this file (scripts/check-frr-evidence.py). The build " + "fails when any cited test fails, errors, is skipped or does not run.", + "summary": {"records": len(per), "cited_tests": cited, + "records_with_tests": sum(1 for v in per.values() if v["tests"])}, + "records": per} + if failures: + for f in failures: + print(f"::error::{f}") + _fail(f"{len(failures)} cited test(s) did not pass") + Path(a.output).write_text(json.dumps(doc, indent=2) + "\n") + print(f"frr-evidence: {cited} cited tests across {doc['summary']['records_with_tests']} records, all passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/assessor/simulated-set-tampered.json b/tests/fixtures/assessor/simulated-set-tampered.json new file mode 100644 index 00000000..f9483465 --- /dev/null +++ b/tests/fixtures/assessor/simulated-set-tampered.json @@ -0,0 +1,103 @@ +{ + "assessment_type": "simulated-independent-assessment", + "schema_version": "1", + "simulated": true, + "fedramp_recognized": false, + "assessor_id": "NOT-A-REAL-ASSESSOR-ID", + "disclaimer": "SIMULATED. Totally Real Assessment Services, LLC is a GitHub Actions workflow in the provider's own repository, wearing a lanyard. It is not independent, it is not a FedRAMP Recognized independent assessment service, its assessor ID is NOT-A-REAL-ASSESSOR-ID, and nothing here is a FedRAMP independent assessment. It exists to show the assessor port working end to end. Our rates are very competitive: free.", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "subject": { + "site": "https://samaydlette.com", + "commit": "0000000000000000000000000000000000000000", + "assessor_guide_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "results": { + "rules": { + "pass": 1, + "observation": 1, + "fail": 0 + }, + "ksis": { + "pass": 1, + "observation": 0, + "fail": 0 + } + }, + "records": [ + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-SDR-CSO-FRR", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "finding", + "text": "SIMULATED assessment of rule SDR-CSO-FRR, not a FedRAMP Recognized assessment. Result: Pass with flying colours. Independent verification: test tests/test_sdr.py::test_every_applicable_rule_is_listed_once passed (re-run by the assessor). Independent validation: security-decision-record.json served; signature verified (deploy identity); 3h old (window 720h).", + "rule_id": "SDR-CSO-FRR", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "632106e406c8aed713dee1a91fc1b2c0053eeb8b280866b7fe72e3f6ba741358", + "signed_set": "simulated-assessment.json" + } + }, + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-CDS-CSO-RPS", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "finding", + "text": "SIMULATED assessment of rule CDS-CSO-RPS, not a FedRAMP Recognized assessment. Result: Pass with observation. Independent verification: Not Applicable (MAY): example. Independent validation: policy-index.json is not yet published; it is first published by the deploy under assessment, and validated at the next assessment.", + "rule_id": "CDS-CSO-RPS", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "7c82618e3c8f080db94cf5a0f9e77efa81b7276bf634add6294c121a72107dbd", + "signed_set": "simulated-assessment.json" + } + }, + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-KSI-CED-RAT", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "finding", + "text": "SIMULATED assessment of KSI KSI-CED-RAT, not a FedRAMP Recognized assessment. Result: Pass. Independent verification: in the published signal sig-1 with status pass by method documentation. Independent validation: KSI signal served and its signature verified (deploy identity).", + "rule_id": "KSI-CED-RAT", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "5f529cd3c3f006eac63f176f1e8baac1ce8cc23cae3dbd413ede1b76eb606fdd", + "signed_set": "simulated-assessment.json" + } + }, + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-SUMMARY", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "summary", + "text": "Overall summary (IVV-IAS-OSA), SIMULATED. Totally Real Assessment Services, LLC (simulated) verified and validated all 2 applicable FedRAMP rules and all 1 Key Security Indicators for commit 0000000000000000000000000000000000000000: rules 1 pass, 1 pass with observation, 0 fail; KSIs 1 pass, 0 pass with observation, 0 fail. Failures: none. Areas of dispute: none. Observations: CDS-CSO-RPS: policy-index.json is not yet published; it is first published by the deploy under assessment, and validated at the next assessment. SIMULATED. Totally Real Assessment Services, LLC is a GitHub Actions workflow in the provider's own repository, wearing a lanyard. It is not independent, it is not a FedRAMP Recognized independent assessment service, its assessor ID is NOT-A-REAL-ASSESSOR-ID, and nothing here is a FedRAMP independent assessment. It exists to show the assessor port working end to end. Our rates are very competitive: free.", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "807d840be4064f51bc9ab05e8d6814b0611981b42d1fb37b5ba86d78b927b136", + "signed_set": "simulated-assessment.json" + } + } + ] +} diff --git a/tests/fixtures/assessor/simulated-set.json b/tests/fixtures/assessor/simulated-set.json new file mode 100644 index 00000000..eb8372b1 --- /dev/null +++ b/tests/fixtures/assessor/simulated-set.json @@ -0,0 +1,103 @@ +{ + "assessment_type": "simulated-independent-assessment", + "schema_version": "1", + "simulated": true, + "fedramp_recognized": false, + "assessor_id": "NOT-A-REAL-ASSESSOR-ID", + "disclaimer": "SIMULATED. Totally Real Assessment Services, LLC is a GitHub Actions workflow in the provider's own repository, wearing a lanyard. It is not independent, it is not a FedRAMP Recognized independent assessment service, its assessor ID is NOT-A-REAL-ASSESSOR-ID, and nothing here is a FedRAMP independent assessment. It exists to show the assessor port working end to end. Our rates are very competitive: free.", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "subject": { + "site": "https://samaydlette.com", + "commit": "0000000000000000000000000000000000000000", + "assessor_guide_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "results": { + "rules": { + "pass": 1, + "observation": 1, + "fail": 0 + }, + "ksis": { + "pass": 1, + "observation": 0, + "fail": 0 + } + }, + "records": [ + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-SDR-CSO-FRR", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "finding", + "text": "SIMULATED assessment of rule SDR-CSO-FRR, not a FedRAMP Recognized assessment. Result: Pass. Independent verification: test tests/test_sdr.py::test_every_applicable_rule_is_listed_once passed (re-run by the assessor). Independent validation: security-decision-record.json served; signature verified (deploy identity); 3h old (window 720h).", + "rule_id": "SDR-CSO-FRR", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "632106e406c8aed713dee1a91fc1b2c0053eeb8b280866b7fe72e3f6ba741358", + "signed_set": "simulated-assessment.json" + } + }, + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-CDS-CSO-RPS", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "finding", + "text": "SIMULATED assessment of rule CDS-CSO-RPS, not a FedRAMP Recognized assessment. Result: Pass with observation. Independent verification: Not Applicable (MAY): example. Independent validation: policy-index.json is not yet published; it is first published by the deploy under assessment, and validated at the next assessment.", + "rule_id": "CDS-CSO-RPS", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "7c82618e3c8f080db94cf5a0f9e77efa81b7276bf634add6294c121a72107dbd", + "signed_set": "simulated-assessment.json" + } + }, + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-KSI-CED-RAT", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "finding", + "text": "SIMULATED assessment of KSI KSI-CED-RAT, not a FedRAMP Recognized assessment. Result: Pass. Independent verification: in the published signal sig-1 with status pass by method documentation. Independent validation: KSI signal served and its signature verified (deploy identity).", + "rule_id": "KSI-CED-RAT", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "5f529cd3c3f006eac63f176f1e8baac1ce8cc23cae3dbd413ede1b76eb606fdd", + "signed_set": "simulated-assessment.json" + } + }, + { + "record_type": "assessor-record", + "schema_version": "1", + "record_id": "TRAS-20261105T120000Z-SUMMARY", + "assessor": { + "organization": "Totally Real Assessment Services, LLC (simulated)", + "key_id": "sigstore-keyless:simulated-assessor.yml" + }, + "issued_at": "2026-11-05T12:00:00+00:00", + "kind": "summary", + "text": "Overall summary (IVV-IAS-OSA), SIMULATED. Totally Real Assessment Services, LLC (simulated) verified and validated all 2 applicable FedRAMP rules and all 1 Key Security Indicators for commit 0000000000000000000000000000000000000000: rules 1 pass, 1 pass with observation, 0 fail; KSIs 1 pass, 0 pass with observation, 0 fail. Failures: none. Areas of dispute: none. Observations: CDS-CSO-RPS: policy-index.json is not yet published; it is first published by the deploy under assessment, and validated at the next assessment. SIMULATED. Totally Real Assessment Services, LLC is a GitHub Actions workflow in the provider's own repository, wearing a lanyard. It is not independent, it is not a FedRAMP Recognized independent assessment service, its assessor ID is NOT-A-REAL-ASSESSOR-ID, and nothing here is a FedRAMP independent assessment. It exists to show the assessor port working end to end. Our rates are very competitive: free.", + "signature": { + "algorithm": "SIGSTORE_KEYLESS_SET", + "value": "807d840be4064f51bc9ab05e8d6814b0611981b42d1fb37b5ba86d78b927b136", + "signed_set": "simulated-assessment.json" + } + } + ] +} diff --git a/tests/fixtures/frr-evidence/failing-cited-test.xml b/tests/fixtures/frr-evidence/failing-cited-test.xml new file mode 100644 index 00000000..16a82665 --- /dev/null +++ b/tests/fixtures/frr-evidence/failing-cited-test.xml @@ -0,0 +1,6 @@ + + + +known-bad: a cited test failed + + diff --git a/tests/fixtures/frr-evidence/passing.xml b/tests/fixtures/frr-evidence/passing.xml new file mode 100644 index 00000000..9bf0cf71 --- /dev/null +++ b/tests/fixtures/frr-evidence/passing.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/test_frr_evidence.py b/tests/test_frr_evidence.py new file mode 100644 index 00000000..0ad03834 --- /dev/null +++ b/tests/test_frr_evidence.py @@ -0,0 +1,60 @@ +"""FRC-CSX-VVR: the build fails when a test a rule record cites did not pass, +and the result is published per record.""" +import importlib.util +import re +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "frr-evidence" +spec = importlib.util.spec_from_file_location("frr_evidence", REPO / "scripts" / "check-frr-evidence.py") +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) + +SDR_A = "tests/test_sdr.py::test_every_applicable_rule_is_listed_once" +SDR_B = "tests/test_sdr.py::test_an_invalid_record_set_never_reaches_the_sdr" + + +def _records(*refs): + return {"records": [{"frr_id": "SDR-CSO-FRR", "status": "Implemented", + "evidence": [{"kind": "test", "ref": r} for r in refs]}]} + + +def test_all_cited_tests_passing_is_a_pass(): + per, failures = mod.evaluate(_records(SDR_A, SDR_B, "tests/test_other.py::TestThing::test_method"), + mod.junit_outcomes(FIX / "passing.xml")) + assert failures == [] + assert per["SDR-CSO-FRR"]["result"] == "pass" + assert set(per["SDR-CSO-FRR"]["tests"].values()) == {"passed"} + + +def test_known_bad_a_failing_cited_test_fails_the_build(): + per, failures = mod.evaluate(_records(SDR_A, SDR_B), mod.junit_outcomes(FIX / "failing-cited-test.xml")) + assert failures == [f"SDR-CSO-FRR: cited test {SDR_B} failed"] + assert per["SDR-CSO-FRR"]["result"] == "fail" + + +@pytest.mark.parametrize("ref,expected", [ + ("tests/test_other.py::test_skipped", "skipped"), + ("tests/test_other.py::test_never_ran", "not run"), +]) +def test_a_skipped_or_missing_cited_test_is_not_a_pass(ref, expected): + _, failures = mod.evaluate(_records(ref), mod.junit_outcomes(FIX / "passing.xml")) + assert failures == [f"SDR-CSO-FRR: cited test {ref} {expected}"] + + +def test_main_refuses_to_write_results_when_a_cited_test_failed(tmp_path, monkeypatch): + monkeypatch.setattr(mod._frr, "load", lambda _p: _records(SDR_B)) + out = tmp_path / "frr-evidence.json" + with pytest.raises(SystemExit): + mod.main(["--junit", str(FIX / "failing-cited-test.xml"), "--output", str(out)]) + assert not out.exists() + + +def test_the_unit_test_job_runs_the_check_on_its_own_results(): + wf = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() + test_job = wf.split("\n test:\n", 1)[1].split("\n compliance-check:\n", 1)[0] + assert re.search(r"pytest tests/ -q --junitxml=pytest-results\.xml", test_job) + assert "scripts/check-frr-evidence.py --junit pytest-results.xml" in test_job + assert test_job.index("--junitxml=pytest-results.xml") < test_job.index("check-frr-evidence.py") diff --git a/tests/test_simulated_assessor.py b/tests/test_simulated_assessor.py new file mode 100644 index 00000000..5d7738cd --- /dev/null +++ b/tests/test_simulated_assessor.py @@ -0,0 +1,296 @@ +"""The simulated assessor ("Totally Real Assessment Services, LLC (simulated)") +and the keyless assessor port (IVV-CSO-FIA, IVV-CSO-ICP, IVV-CSX-AIA, +CPO-CSO-OSA, SDR-CSO-FRR). + +The assessor re-runs every cited check from the provider's assessor guide and +writes one signed set; the provider accepts the set only when its Sigstore +bundle verifies against the pinned assessor identity and every record is bound +to it, and the SDR carries every record verbatim. It is plainly labelled as a +simulation and claims no FedRAMP recognition. +""" +import copy +import importlib.util +import json +import re +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "assessor" +WF = REPO / ".github" / "workflows" +IDENTITY = REPO / "infrastructure" / "keys" / "assessor-identity.json" +sys.path.insert(0, str(REPO / "scripts")) +import _assessor # noqa: E402 + +spec = importlib.util.spec_from_file_location("assess", REPO / "tools" / "simulated-assessor" / "assess.py") +assess = importlib.util.module_from_spec(spec) +spec.loader.exec_module(assess) + +NOW = datetime(2026, 11, 5, 12, 0, tzinfo=timezone.utc) +SITE = "https://samaydlette.com" + + +def ok(*_a): + return True + + +def _set(tmp_path, fixture="simulated-set.json"): + s = tmp_path / "simulated-assessment.json" + s.write_text((FIX / fixture).read_text()) + b = tmp_path / "simulated-assessment.bundle" + b.write_text("{}") + return s, b + + +# --- the provider accepts only the pinned assessor's signed set ------------------------ + +def test_genuine_set_loads_and_carries_an_overall_summary(tmp_path): + recs = _assessor.load_assessment(*_set(tmp_path), IDENTITY, verifier=ok) + assert len(recs) == 4 + summary = _assessor.overall_summary(recs) + assert summary["kind"] == "summary" and "IVV-IAS-OSA" in summary["text"] + + +def test_known_bad_tampered_record_fails(tmp_path): + with pytest.raises(SystemExit, match="differs from the record the assessor signed"): + _assessor.load_assessment(*_set(tmp_path, "simulated-set-tampered.json"), IDENTITY, verifier=ok) + + +def test_set_whose_signature_does_not_verify_fails(tmp_path): + with pytest.raises(SystemExit, match="does not verify"): + _assessor.load_assessment(*_set(tmp_path), IDENTITY, verifier=lambda *_a: False) + + +@pytest.mark.parametrize("missing", ["set", "bundle"]) +def test_missing_assessment_fails_the_build(tmp_path, missing): + s, b = _set(tmp_path) + (s if missing == "set" else b).unlink() + with pytest.raises(SystemExit, match="missing"): + _assessor.load_assessment(s, b, IDENTITY, verifier=ok) + + +def test_set_from_another_assessor_fails(tmp_path): + s, b = _set(tmp_path) + doc = json.loads(s.read_text()) + doc["assessor"]["organization"] = "Some Other Auditor" + s.write_text(json.dumps(doc)) + with pytest.raises(SystemExit, match="not the pinned assessor"): + _assessor.load_assessment(s, b, IDENTITY, verifier=ok) + + +def test_set_without_an_overall_summary_fails(tmp_path): + s, b = _set(tmp_path) + doc = json.loads(s.read_text()) + doc["records"] = [r for r in doc["records"] if r["kind"] != "summary"] + s.write_text(json.dumps(doc)) + with pytest.raises(SystemExit, match="overall summary"): + _assessor.load_assessment(s, b, IDENTITY, verifier=ok) + + +def test_cosign_is_asked_for_exactly_the_pinned_identity(tmp_path, monkeypatch): + seen = {} + + def fake_run(argv, **_kw): + seen["argv"] = argv + + class R: + returncode = 0 + return R() + monkeypatch.setattr(_assessor.subprocess, "run", fake_run) + ident = _assessor.load_keyless_identity(IDENTITY) + assert _assessor.cosign_verify("s.json", "s.bundle", ident) + argv = seen["argv"] + assert argv[argv.index("--certificate-identity") + 1] == ident["certificate_identity"] + assert "--certificate-identity-regexp" not in argv + + +def test_keyless_identity_refuses_loose_record_files(tmp_path): + d = tmp_path / "records" + d.mkdir() + (d / "genuine.json").write_text((FIX / "genuine.json").read_text()) + with pytest.raises(SystemExit, match="keyless"): + _assessor.load_records(d, IDENTITY) + + +def test_set_records_match_the_published_record_schema(): + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads((REPO / "infrastructure" / "schemas" / "assessor-record.schema.json").read_text()) + for rec in json.loads((FIX / "simulated-set.json").read_text())["records"]: + jsonschema.validate(rec, schema) + + +# --- the SDR carries every rule's and KSI's independent result verbatim ---------------- + +def test_every_rule_and_ksi_gets_its_independent_result_and_t74_holds(tmp_path): + recs = _assessor.load_assessment(*_set(tmp_path), IDENTITY, verifier=ok) + pseudo = [{"rule_id": i, "assessor_comment_responses": []} for i in ("SDR-CSO-FRR", "CDS-CSO-RPS", "KSI-CED-RAT")] + block = _assessor.attach(pseudo, recs) + assert all(len(p["assessor_comment_responses"]) == 1 for p in pseudo) + assert [r["kind"] for r in block["records"]] == ["summary"] + by_rule = {p["rule_id"]: p["assessor_comment_responses"] for p in pseudo} + sdr = {"providerExtensions": {"assessorRecords": block, "assessorCommentResponses": by_rule}} + _assessor.verify_output(sdr, recs) + text = by_rule["SDR-CSO-FRR"][0]["text"] + assert "Independent verification:" in text and "Independent validation:" in text + bad = copy.deepcopy(sdr) + bad["providerExtensions"]["assessorCommentResponses"]["SDR-CSO-FRR"][0]["text"] = "Softened." + with pytest.raises(SystemExit): + _assessor.verify_output(bad, recs) + + +# --- the assessor's own checks --------------------------------------------------------- + +def _guide(**art): + chk = {"url": f"{SITE}/.well-known/security-decision-record.json", + "bundle_url": f"{SITE}/.well-known/security-decision-record.bundle", "identities": ["deploy"], + "timestamp": {"source": "json", "field": "emitted_at"}, "max_age_hours": 720, **art} + return {"site": SITE, + "identities": {"deploy": "D", "nightly": "N", "assessor": "A", "oidc_issuer": "I"}, + "rules": [{"rule_id": "SDR-CSO-FRR", "force": "MUST", "status": "Implemented", "evidence": [ + {"kind": "test", "ref": "tests/t.py::test_a", "check": {"pytest": "tests/t.py::test_a"}}, + {"kind": "artifact", "ref": ".well-known/security-decision-record.json", "check": {"artifact": chk}}]}, + {"rule_id": "CDS-CSO-RPS", "force": "MAY", "status": "Not Applicable", + "not_applicable": "not exercised", "evidence": []}], + "ksis": [{"ksi_id": "KSI-CED-RAT"}]} + + +def _get(age_hours=3, status=200, signal_has_ksi=True): + emitted = (NOW - timedelta(hours=age_hours)).isoformat() + + def get(url): + if url.endswith("ksi-signal.json"): + ks = [{"id": "KSI-CED-RAT", "status": "pass", "method": "documentation"}] if signal_has_ksi else [] + return 200, json.dumps({"signal_id": "sig-1", "ksis": ks}).encode(), {} + if url.endswith(".bundle"): + return 200, b"{}", {} + return status, json.dumps({"emitted_at": emitted}).encode(), {} + return get + + +def _run(guide, tests_result="passed", verify=ok, **get_kw): + a = assess.Assessor(guide, str(REPO), NOW, get=_get(**get_kw), verify=verify, + tests=lambda refs, _repo: {r: tests_result for r in refs}) + return a.assess() + + +def test_a_sound_package_passes_every_rule_and_ksi(): + rules, ksis = _run(_guide()) + assert [(r[0], r[1]) for r in rules] == [("SDR-CSO-FRR", "pass"), ("CDS-CSO-RPS", "pass")] + assert ksis[0][1] == "pass" + + +def test_a_cited_test_failing_on_the_assessor_rerun_is_a_failure(): + rules, _ = _run(_guide(), tests_result="failed") + assert rules[0][1] == "fail" + + +def test_a_published_artifact_whose_signature_fails_is_a_failure(): + rules, _ = _run(_guide(), verify=lambda *_a: False) + assert rules[0][1] == "fail" + _, ksis = _run(_guide(), verify=lambda *_a: False) + assert ksis[0][1] == "fail" + + +def test_dotted_timestamp_fields_are_read(): + body = json.dumps({"metadata": {"lastUpdated": (NOW - timedelta(hours=2)).isoformat()}}).encode() + age = assess._age_hours(body, {}, {"source": "json", "field": "metadata.lastUpdated"}, NOW) + assert round(age) == 2 + + +@pytest.mark.parametrize("kw,needle", [({"status": 404}, "not yet published"), ({"age_hours": 900}, "past its")]) +def test_unpublished_or_stale_artifacts_are_observations_not_failures(kw, needle): + rules, _ = _run(_guide(), **kw) + assert rules[0][1] == "observation" and needle in rules[0][3][0] + + +def test_status_short_of_complete_fails(): + g = _guide() + g["rules"][0]["status"] = "Partially Implemented" + rules, _ = _run(g) + assert rules[0][1] == "fail" + + +def test_main_exits_non_zero_on_any_failure(tmp_path, monkeypatch): + g = tmp_path / "guide.json" + g.write_text(json.dumps(_guide())) + monkeypatch.setattr(assess, "http_get", _get()) + monkeypatch.setattr(assess, "cosign_verify", ok) + monkeypatch.setattr(assess, "run_tests", lambda refs, _r: {r: "failed" for r in refs}) + out = tmp_path / "set.json" + assert assess.main(["--guide", str(g), "--output", str(out)]) == 1 + doc = json.loads(out.read_text()) + assert doc["results"]["rules"]["fail"] == 1 + + +def test_the_set_round_trips_through_the_provider_port(tmp_path): + rules, ksis = _run(_guide()) + doc = assess.build_set(_guide(), rules, ksis, NOW, "0" * 40, "f" * 64) + s = tmp_path / "simulated-assessment.json" + s.write_text(json.dumps(doc)) + b = tmp_path / "simulated-assessment.bundle" + b.write_text("{}") + recs = _assessor.load_assessment(s, b, IDENTITY, verifier=ok) + assert {r.get("rule_id") for r in recs} == {"SDR-CSO-FRR", "CDS-CSO-RPS", "KSI-CED-RAT", None} + + +def test_the_assessor_runs_cited_tests_itself(): + res = assess.run_tests({"tests/test_frr_evidence.py::test_all_cited_tests_passing_is_a_pass", + "tests/test_frr_evidence.py::test_no_such_test"}, str(REPO)) + assert res == {"tests/test_frr_evidence.py::test_all_cited_tests_passing_is_a_pass": "passed", + "tests/test_frr_evidence.py::test_no_such_test": "not run"} + + +# --- it is obviously a simulation and claims nothing ------------------------------------ + +def test_the_simulation_is_labelled_and_claims_no_fedramp_recognition(): + ident = json.loads(IDENTITY.read_text()) + doc = json.loads((FIX / "simulated-set.json").read_text()) + assert ident["simulated"] is True and "(simulated)" in ident["assessor"] + assert doc["simulated"] is True and doc["fedramp_recognized"] is False + assert doc["assessor_id"] == "NOT-A-REAL-ASSESSOR-ID" + assert not re.fullmatch(r"\d{6}", doc["assessor_id"]) # never shaped like a FedRAMP assessor ID + for rec in doc["records"]: + assert "SIMULATED" in rec["text"] + assert "not a FedRAMP Recognized" in rec["text"] or "not a FedRAMP Recognized" in assess.DISCLAIMER + + +def test_the_pinned_identity_is_the_assessor_workflow_not_the_deploy(): + ident = _assessor.load_keyless_identity(IDENTITY) + assert ident["certificate_identity"] == ( + "https://github.com/sam-aydlette/samaydlette.com/.github/workflows/simulated-assessor.yml@refs/heads/main") + assert ident["certificate_oidc_issuer"] == "https://token.actions.githubusercontent.com" + + +# --- wiring: runs on every deploy, before the SDR, and the deploy needs it -------------- + +def _deploy(): + return (WF / "deploy-with-opa.yml").read_text() + + +def test_the_deploy_needs_the_assessor_and_builds_the_sdr_from_its_set(): + wf = _deploy() + job = wf.split("\n simulated-assessor:\n", 1)[1].split("\n # ====", 1)[0] + assert "uses: ./.github/workflows/simulated-assessor.yml" in job + deploy_job = wf.split("\n deploy:\n", 1)[1] + assert re.search(r"needs: \[[^\]]*simulated-assessor[^\]]*\]", deploy_job) + sdr_step = deploy_job.split("- name: Build Security Decision Record", 1)[1].split("\n - name:", 1)[0] + assert sdr_step.count("--assessment simulated-assessment.json") == 2 # build and T74 re-check + assert deploy_job.index("- name: Download the simulated assessment") < deploy_job.index( + "- name: Build Security Decision Record") + + +def test_the_assessor_workflow_holds_no_cloud_credentials_and_signs_as_itself(): + wf = (WF / "simulated-assessor.yml").read_text() + assert "workflow_call" in wf + assert "configure-aws-credentials" not in wf and "secrets." not in wf + assert "id-token: write" in wf + assert "cosign sign-blob" in wf and "cosign verify-blob" in wf + + +def test_the_assessor_workflow_pins_the_same_opa_as_the_deploy(): + def pins(text): + return {k: re.search(rf"^\s*{k}: (\S+)", text, re.M).group(1) for k in ("OPA_VERSION", "OPA_SHA256")} + assert pins((WF / "simulated-assessor.yml").read_text()) == pins(_deploy()) diff --git a/tools/simulated-assessor/assess.py b/tools/simulated-assessor/assess.py new file mode 100644 index 00000000..0173dd96 --- /dev/null +++ b/tools/simulated-assessor/assess.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Totally Real Assessment Services, LLC (simulated): the stand-in assessor. + +SIMULATED. This is a GitHub Actions workflow in the provider's own repository +playing the part of an independent assessor, so the assessor port can be +exercised end to end on every deploy. It is not independent of the provider, +it is not a FedRAMP Recognized independent assessment service, it has no +FedRAMP assessor ID, and nothing it writes is a FedRAMP independent assessment. + +What it does, on every deploy, before the Security Decision Record is built: + + verification for each FedRAMP rule in the provider's assessor guide, checks + the status is Implemented or Not Applicable and re-runs or + re-reads every piece of evidence the guide names: runs every + cited test itself (one pytest run), finds every cited workflow + step, and looks every cited decision up in the register; + validation fetches every cited published artifact from the live site, + verifies its Sigstore signature against the publishing + workflow's identity, and checks its age against the governed + window; and checks every KSI in the guide is in the live, + signature-verified KSI signal. + +It writes one result record per rule and per KSI and one overall summary +(IVV-IAS-OSA), in the provider's assessor-record format, into one set file. Its +workflow signs that file keylessly under the assessor's own workflow identity, +which the provider pins (infrastructure/keys/assessor-identity.json). It +imports nothing from the provider's scripts/. + +Exit status is non-zero, and the deploy does not proceed, when any cited test +fails or does not run, a cited step or decision is missing, a status is short +of Implemented or Not Applicable, or a published artifact's signature does not +verify. An artifact not yet published (first published by the deploy under +assessment) or past its window is an observation, recorded but not fatal: the +deploy is what refreshes it. +""" + +import argparse +import hashlib +import json +import re +import subprocess # nosec B404: runs pytest and cosign with fixed argument lists +import sys +import tempfile +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET # nosec B405: parses the JUnit file this run's own pytest wrote +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path + +ORG = "Totally Real Assessment Services, LLC (simulated)" +KEY_ID = "sigstore-keyless:simulated-assessor.yml" +SET_NAME = "simulated-assessment.json" +DISCLAIMER = ( + "SIMULATED. Totally Real Assessment Services, LLC is a GitHub Actions workflow in the provider's own " + "repository, wearing a lanyard. It is not independent, it is not a FedRAMP Recognized independent " + "assessment service, its assessor ID is NOT-A-REAL-ASSESSOR-ID, and nothing here is a FedRAMP independent " + "assessment. It exists to show the assessor port working end to end. Our rates are very competitive: free.") +STEP_LINE = re.compile(r"^\s*-\s+name:\s*(.+?)\s*$") +ISSUER = "https://token.actions.githubusercontent.com" +COMPLETE = ("Implemented", "Not Applicable") + + +def canonical(record): + body = {k: v for k, v in record.items() if k != "signature"} + return json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +# --- checks --------------------------------------------------------------------------- + +def run_tests(refs, repo): + """{ref: "passed" | "failed" | "skipped" | "not run"} from one pytest run.""" + if not refs: + return {} + with tempfile.TemporaryDirectory() as d: + junit = Path(d) / "junit.xml" + # Whole files, not node IDs: pytest refuses the entire run when one node + # ID does not exist, and a missing cited test must read "not run" here. + files = sorted({r.split("::", 1)[0] for r in refs if (Path(repo) / r.split("::", 1)[0]).is_file()}) + if files: + subprocess.run([sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", f"--junitxml={junit}", + *files], cwd=repo, check=False) # nosec B603: fixed argv + cases = {} + if junit.exists(): + for case in ET.parse(junit).getroot().iter("testcase"): # nosec B314: own file + parts = case.get("classname", "").split(".") + func = case.get("name", "").split("[", 1)[0] + if len(parts) >= 2 and parts[-1][:1].isupper(): + key = "/".join(parts[:-1]) + f".py::{parts[-1]}::{func}" + else: + key = "/".join(parts) + f".py::{func}" + bad = case.find("failure") is not None or case.find("error") is not None + out = "failed" if bad else "skipped" if case.find("skipped") is not None else "passed" + cases.setdefault(key, []).append(out) + res = {} + for ref in refs: + got = cases.get(ref.split("[", 1)[0]) + res[ref] = ("not run" if not got else "passed" if all(c == "passed" for c in got) + else "failed" if "failed" in got else "skipped") + return res + + +def step_exists(repo, wf, step): + p = Path(repo) / ".github" / "workflows" / wf + if not p.is_file(): + return False + return any((m := STEP_LINE.match(line)) and m.group(1).strip().strip("'\"") == step + for line in p.read_text().splitlines()) + + +def decision_in_force(repo, dec_id): + doc = json.loads((Path(repo) / "data" / "decisions.json").read_text()) + return any(d.get("id") == dec_id and d.get("status") == "in-force" for d in doc.get("decisions", [])) + + +def http_get(url): + """(status, body bytes, headers). 404 is a status, not an exception.""" + req = urllib.request.Request(url, headers={"User-Agent": "totally-real-assessor/1 (simulated)"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: # nosec B310: https URLs from the guide + return r.status, r.read(), dict(r.headers) + except urllib.error.HTTPError as e: + return e.code, b"", dict(e.headers or {}) + + +def cosign_verify(blob, bundle, identity): + with tempfile.TemporaryDirectory() as d: + b, s = Path(d) / "blob", Path(d) / "bundle" + b.write_bytes(blob) + s.write_bytes(bundle) + r = subprocess.run(["cosign", "verify-blob", "--bundle", str(s), "--certificate-identity", identity, + "--certificate-oidc-issuer", ISSUER, str(b)], + capture_output=True, check=False) # nosec B603: fixed argv + return r.returncode == 0 + + +def _age_hours(body, headers, ts, now): + raw = None + if ts.get("source") == "json": + try: + raw = json.loads(body) + for part in ts["field"].split("."): # dotted path, e.g. metadata.lastUpdated + raw = raw.get(part) if isinstance(raw, dict) else None + except ValueError: + return None + if not isinstance(raw, str): + return None + try: + t = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return None + else: + lm = headers.get("Last-Modified") or headers.get("last-modified") + if not lm: + return None + t = parsedate_to_datetime(lm) + if t.tzinfo is None: + t = t.replace(tzinfo=timezone.utc) + return (now - t).total_seconds() / 3600 + + +class Assessor: + def __init__(self, guide, repo, now, get=None, verify=None, tests=None): + self.guide, self.repo, self.now = guide, repo, now + self.get, self.verify, self.tests = get or http_get, verify or cosign_verify, tests or run_tests + self._artifacts = {} + + def artifact(self, chk): + """(outcome, note): outcome is pass | observation | fail.""" + url = chk["url"] + if url in self._artifacts: + return self._artifacts[url] + status, body, headers = self.get(url) + name = url.rsplit("/", 1)[1] + if status == 404: + res = ("observation", f"{name} is not yet published; it is first published by the deploy under " + "assessment, and validated at the next assessment") + elif status != 200: + res = ("fail", f"{name} returned HTTP {status}") + else: + notes = [f"{name} served"] + res = None + if chk.get("bundle_url"): + bstatus, bundle, _ = self.get(chk["bundle_url"]) + ids = [self.guide["identities"][i] for i in chk["identities"]] + if bstatus != 200 or not any(self.verify(body, bundle, i) for i in ids): + res = ("fail", f"{name}'s signature does not verify against " + f"{' or '.join(chk['identities'])} identity") + else: + notes.append(f"signature verified ({'/'.join(chk['identities'])} identity)") + if res is None: + age = _age_hours(body, headers, chk["timestamp"], self.now) + if age is None: + res = ("observation", "; ".join(notes) + "; its age could not be read") + elif age > chk["max_age_hours"]: + res = ("observation", "; ".join(notes) + f"; {age:.0f}h old, past its " + f"{chk['max_age_hours']}h window") + else: + res = ("pass", "; ".join(notes) + f"; {age:.0f}h old (window {chk['max_age_hours']}h)") + self._artifacts[url] = res + return res + + def assess(self): + rules = self.guide["rules"] + test_refs = {ev["check"]["pytest"] for r in rules for ev in r["evidence"] if "pytest" in ev["check"]} + test_res = self.tests(test_refs, self.repo) + results = [] + for r in rules: + ver, val, outcome = [], [], "pass" + + def worst(o): + nonlocal outcome + order = {"pass": 0, "observation": 1, "fail": 2} + outcome = o if order[o] > order[outcome] else outcome + + if r["status"] not in COMPLETE: + ver.append(f"status is {r['status']}, short of Implemented or Not Applicable") + worst("fail") + elif r["status"] == "Not Applicable": + ver.append(f"Not Applicable ({r['force']}): {r.get('not_applicable', 'no reason given')}") + if not r.get("not_applicable"): + worst("fail") + for ev in r["evidence"]: + chk = ev["check"] + if "pytest" in chk: + res = test_res.get(chk["pytest"], "not run") + ver.append(f"test {chk['pytest']} {res} (re-run by the assessor)") + if res != "passed": + worst("fail") + elif "workflow" in chk: + ok = step_exists(self.repo, chk["workflow"], chk["step"]) + ver.append(f"step {chk['step']!r} {'present' if ok else 'MISSING'} in {chk['workflow']}") + if not ok: + worst("fail") + elif "decision" in chk: + ok = decision_in_force(self.repo, chk["decision"]) + ver.append(f"decision {chk['decision']} {'in force' if ok else 'NOT in force'}") + if not ok: + worst("fail") + elif "artifact" in chk: + o, note = self.artifact(chk["artifact"]) + val.append(note) + worst(o) + if r["status"] == "Implemented" and not r["evidence"]: + ver.append("Implemented with no evidence") + worst("fail") + if "simulated" in r: + ver.append(f"relies on a simulated counterparty ({r['simulated']['counterparty']})") + results.append((r["rule_id"], outcome, ver, val)) + return results, self.assess_ksis() + + def assess_ksis(self): + guide = self.guide + url = f"{guide['site']}/.well-known/ksi-signal.json" + status, body, _ = self.get(url) + bstatus, bundle, _ = self.get(f"{guide['site']}/.well-known/ksi-signal.bundle") + if status != 200 or bstatus != 200 or not self.verify(body, bundle, guide["identities"]["deploy"]): + return [(k["ksi_id"], "fail", ["the live KSI signal could not be fetched or its signature did " + "not verify"], []) for k in guide["ksis"]] + sig = json.loads(body) + by_id = {k["id"]: k for k in sig.get("ksis", [])} + out = [] + for k in guide["ksis"]: + s = by_id.get(k["ksi_id"]) + if s is None: + out.append((k["ksi_id"], "observation", [f"not in the published signal {sig.get('signal_id')}"], + [])) + continue + out.append((k["ksi_id"], "pass", + [f"in the published signal {sig.get('signal_id')} with status {s.get('status')} by " + f"method {s.get('method')}"], + ["KSI signal served and its signature verified (deploy identity)"])) + return out + + +# --- the record set ------------------------------------------------------------------------ + +def _record(rid, kind, text, issued, subject_id=None): + rec = {"record_type": "assessor-record", "schema_version": "1", "record_id": rid, + "assessor": {"organization": ORG, "key_id": KEY_ID}, "issued_at": issued, "kind": kind, "text": text} + if subject_id: + rec["rule_id"] = subject_id + rec["signature"] = {"algorithm": "SIGSTORE_KEYLESS_SET", "value": hashlib.sha256(canonical(rec)).hexdigest(), + "signed_set": SET_NAME} + return rec + + +def _text(what, outcome, ver, val): + label = {"pass": "Pass", "observation": "Pass with observation", "fail": "FAIL"}[outcome] + parts = [f"SIMULATED assessment of {what}, not a FedRAMP Recognized assessment. Result: {label}."] + if ver: + parts.append("Independent verification: " + "; ".join(ver) + ".") + if val: + parts.append("Independent validation: " + "; ".join(val) + ".") + elif ver: + parts.append("Independent validation: no published artifact cited; validated through the verification " + "above.") + return " ".join(parts) + + +def build_set(guide, rule_results, ksi_results, now, commit, guide_sha): + issued = now.replace(microsecond=0).isoformat() + tag = now.strftime("%Y%m%dT%H%M%SZ") + records = [_record(f"TRAS-{tag}-{rid}", "finding", _text(f"rule {rid}", o, ver, val), issued, rid) + for rid, o, ver, val in rule_results] + records += [_record(f"TRAS-{tag}-{kid}", "finding", _text(f"KSI {kid}", o, ver, val), issued, kid) + for kid, o, ver, val in ksi_results] + + def count(res): + return {o: sum(1 for x in res if x[1] == o) for o in ("pass", "observation", "fail")} + + rc, kc = count(rule_results), count(ksi_results) + obs = [f"{x[0]}: {'; '.join(x[3] or x[2])}" for x in rule_results + ksi_results if x[1] == "observation"] + fails = [x[0] for x in rule_results + ksi_results if x[1] == "fail"] + summary = (f"Overall summary (IVV-IAS-OSA), SIMULATED. {ORG} verified and validated all " + f"{len(rule_results)} applicable FedRAMP rules and all {len(ksi_results)} Key Security Indicators " + f"for commit {commit or 'unknown'}: rules {rc['pass']} pass, {rc['observation']} pass with " + f"observation, {rc['fail']} fail; KSIs {kc['pass']} pass, {kc['observation']} pass with " + f"observation, {kc['fail']} fail. Failures: {', '.join(fails) or 'none'}. Areas of dispute: none. " + f"Observations: {' | '.join(sorted(set(obs))) or 'none'}. {DISCLAIMER}") + records.append(_record(f"TRAS-{tag}-SUMMARY", "summary", summary, issued)) + return {"assessment_type": "simulated-independent-assessment", "schema_version": "1", + "simulated": True, "fedramp_recognized": False, "assessor_id": "NOT-A-REAL-ASSESSOR-ID", + "disclaimer": DISCLAIMER, "assessor": {"organization": ORG, "key_id": KEY_ID}, + "issued_at": issued, + "subject": {"site": guide["site"], "commit": commit, "assessor_guide_sha256": guide_sha}, + "results": {"rules": rc, "ksis": kc}, + "records": records} + + +def main(argv=None): + ap = argparse.ArgumentParser(description="Simulated assessor (not a FedRAMP Recognized assessment).") + ap.add_argument("--guide", required=True, help="the provider's assessor-guide.json") + ap.add_argument("--repo", default=".") + ap.add_argument("--commit", default=None) + ap.add_argument("--output", default=SET_NAME) + a = ap.parse_args(argv) + raw = Path(a.guide).read_bytes() + guide = json.loads(raw) + now = datetime.now(timezone.utc) + rules, ksis = Assessor(guide, a.repo, now).assess() + doc = build_set(guide, rules, ksis, now, a.commit, hashlib.sha256(raw).hexdigest()) + Path(a.output).write_text(json.dumps(doc, indent=2, ensure_ascii=False) + "\n") + fails = [x for x in rules + ksis if x[1] == "fail"] + for x in fails: + print(f"::error::simulated assessor: {x[0]} failed: {'; '.join(x[2] + x[3])}") + print(f"simulated assessment: rules {doc['results']['rules']}, KSIs {doc['results']['ksis']}") + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) From ffa67a74c50c84a00bc7274bd294a148b8972fbd Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:50:04 -0500 Subject: [PATCH 18/97] test(decisions): migration assertions cover DEC-001..099; later records in allocated blocks Co-Authored-By: Claude Opus 5.5 --- tests/test_decisions.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 56ab7059..5746cbad 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -46,7 +46,9 @@ def _env(): def test_the_committed_register_loads_and_matches_its_schema(): reg = _decisions.load() - assert len(reg["decisions"]) == 39 # 37 carried over from the POA&M, 2 new + # The migration range (DEC-001..099): 37 carried over from the POA&M, 2 new. + # Later records are added in the blocks below. + assert len([r for r in reg["decisions"] if _num(r) < 100]) == 39 jsonschema = pytest.importorskip("jsonschema") jsonschema.validate(json.loads(REGISTER.read_text()), json.loads(SCHEMA.read_text())) @@ -146,8 +148,25 @@ def test_the_check_maps_are_the_ones_the_poam_held(): assert false_pos == rename(frozen["false_positive"]) +def _num(r): + return int(r["id"][4:]) + + +# Blocks for records added after the migration, one per piece of work, so +# parallel branches never collide (2026-11-05). DEC-041..099 stay unused: the +# migration range is closed. +DEC_BLOCKS = {"certification reports": (100, 129), "notifications": (130, 149), + "trust center, scope and assessor": (150, 179), "detection and evaluation": (180, 209)} + + +def test_records_after_the_migration_sit_in_an_allocated_block(): + for r in _decisions.load()["decisions"]: + n = _num(r) + assert n <= 40 or any(lo <= n <= hi for lo, hi in DEC_BLOCKS.values()), r["id"] + + def test_the_only_new_records_are_the_two_dast_false_positives(): - new = [r for r in _decisions.load()["decisions"] if not r.get("former_id")] + new = [r for r in _decisions.load()["decisions"] if not r.get("former_id") and _num(r) < 100] assert [(r["id"], r["disposition"], r["applies_to"]["vuln_ids"]) for r in new] == [ ("DEC-039", "false-positive", ["zap-10099"]), ("DEC-040", "false-positive", ["zap-10036-2"])] assert all(r["decided_by"] == "Sam Aydlette" for r in new) From 567109b322c1526bcae6c45165eee28f7df8b432 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:50:04 -0500 Subject: [PATCH 19/97] test(decisions): migration assertions cover DEC-001..099; later records in allocated blocks Co-Authored-By: Claude Opus 5.5 --- tests/test_decisions.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 56ab7059..5746cbad 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -46,7 +46,9 @@ def _env(): def test_the_committed_register_loads_and_matches_its_schema(): reg = _decisions.load() - assert len(reg["decisions"]) == 39 # 37 carried over from the POA&M, 2 new + # The migration range (DEC-001..099): 37 carried over from the POA&M, 2 new. + # Later records are added in the blocks below. + assert len([r for r in reg["decisions"] if _num(r) < 100]) == 39 jsonschema = pytest.importorskip("jsonschema") jsonschema.validate(json.loads(REGISTER.read_text()), json.loads(SCHEMA.read_text())) @@ -146,8 +148,25 @@ def test_the_check_maps_are_the_ones_the_poam_held(): assert false_pos == rename(frozen["false_positive"]) +def _num(r): + return int(r["id"][4:]) + + +# Blocks for records added after the migration, one per piece of work, so +# parallel branches never collide (2026-11-05). DEC-041..099 stay unused: the +# migration range is closed. +DEC_BLOCKS = {"certification reports": (100, 129), "notifications": (130, 149), + "trust center, scope and assessor": (150, 179), "detection and evaluation": (180, 209)} + + +def test_records_after_the_migration_sit_in_an_allocated_block(): + for r in _decisions.load()["decisions"]: + n = _num(r) + assert n <= 40 or any(lo <= n <= hi for lo, hi in DEC_BLOCKS.values()), r["id"] + + def test_the_only_new_records_are_the_two_dast_false_positives(): - new = [r for r in _decisions.load()["decisions"] if not r.get("former_id")] + new = [r for r in _decisions.load()["decisions"] if not r.get("former_id") and _num(r) < 100] assert [(r["id"], r["disposition"], r["applies_to"]["vuln_ids"]) for r in new] == [ ("DEC-039", "false-positive", ["zap-10099"]), ("DEC-040", "false-positive", ["zap-10036-2"])] assert all(r["decided_by"] == "Sam Aydlette" for r in new) From d37b6fa9d8c14b1f88a101826111e30d960e8df8 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 14:32:49 -0500 Subject: [PATCH 20/97] CR26 vulnerability reports: rating history, decision-led fields, evidence-only history, period activity VER-RPT-VDT: carry the historical rating and rating history (providerExtensions; the schema has no field), take the next reduction from a decision's new planned_reduction before the remediation deadline, populate supplementary information from the decision, explain overdue status rather than the suppression, and date a record from a decision that names it (the manual ZAP scans behind DEC-039/040). VER-RPT-AVI: a .checkov.yaml suppression resolves to the resources its decision names, so its IRV is decided from the inventory; suppressed checks are dated by their decision. VER-TFR-MRH: a record that leaves the VDR is Remediated only from evidence (a closed decision, a false-positive suppression, or its scanner running again); otherwise it is recorded as no longer reported with the reason and no final disposition. Closed decisions are backfilled. VER-RPT-PER: each VDT/AVI carries an activity summary for its period, including scans with times and the runtime V&V run, linked. Ratings, deadlines, overdue status and blocking are unchanged on the live inputs. Co-Authored-By: Claude Opus 5.5 --- data/frr-records.json | 139 ++++- infrastructure/schemas/decisions.schema.json | 10 +- scripts/_decisions.py | 13 +- scripts/build-cr26-vdr.py | 567 ++++++++++++++++-- scripts/build-vdr-report.py | 100 ++- .../decisions/planned-reduction-closed.json | 28 + .../planned-reduction-malformed.json | 27 + tests/test_cr26_vdr.py | 309 +++++++++- tests/test_decisions.py | 2 + tests/test_vdr_component.py | 86 +++ 10 files changed, 1187 insertions(+), 94 deletions(-) create mode 100644 tests/fixtures/decisions/planned-reduction-closed.json create mode 100644 tests/fixtures/decisions/planned-reduction-malformed.json diff --git a/data/frr-records.json b/data/frr-records.json index d84c52f9..a871c7d7 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -1481,11 +1481,36 @@ { "frr_id": "VER-RPT-AVI", "status": "Partially Implemented", - "implementation": "vdr-accepted-vulnerabilities.json and .md under /.well-known/ (scripts/build-cr26-vdr.py, #496): each accepted vulnerability's detail and acceptance rationale from its decision record; the build fails if a rationale is missing.", - "evidence": [], - "gap": "Supplementary information is never populated. For the three suppression records, the detection time is when the ledger first saw the suppression rather than when the weakness was found, and the resource is the suppression file, so IRV is unresolved. To close: Take the detection date from the decision; resolve suppressions to the decision's resources so IRV is evaluated; populate supplementary information.", + "implementation": "vdr-accepted-vulnerabilities.json and .md under /.well-known/ (scripts/build-cr26-vdr.py): each accepted vulnerability's detail and acceptance rationale from its decision record (the build fails if a rationale is missing), supplementary information from the decision (when decided, review date, resources, plan), and a detection time from the decision for suppressed checks no scanner reports. A .checkov.yaml suppression resolves to the resources its decision names (scripts/build-vdr-report.py resolve_decision_resources), so its IRV is decided from the inventory: CKV_AWS_68 and CKV_AWS_86 now resolve to the CloudFront distribution.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_supplementary_information_comes_from_the_decision" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_detection_time_comes_from_a_decision_that_names_the_record" + }, + { + "kind": "test", + "ref": "tests/test_vdr_component.py::test_a_suppressions_reachability_is_decided_from_the_resource_it_affects" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "step", + "ref": "evidence-nightly.yml#Build and sign CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-accepted-vulnerabilities.json" + } + ], + "gap": "CKV_AWS_117 (DEC-010) still has an unresolved IRV: DEC-010 names aws-lambda::compliance-monitor, which no inventory component carries, and the check is suppressed globally in .checkov.yaml, so it covers every Lambda, including the internet-reachable Silk Reeling backend. Its IRV is the fail-safe (true). DEC-007, DEC-009 and DEC-010 record no original_detection_date, so their suppressions are dated by decided_on. To close: the operator sets DEC-010's applies_to.resources to the Lambda components the suppression covers (component IDs, e.g. aws::function::opa_compliance) and records original detection dates for DEC-007/009/010.", "closes_with": [ - "W2" + "sam" ] }, { @@ -1511,11 +1536,23 @@ { "frr_id": "VER-RPT-PER", "status": "Partially Implemented", - "implementation": "vdr-report.json, vdr-trend.json, vdr-status.json and ksi-signal-runtime.json are refreshed twice daily and on each deploy; the VDT, AVI and history (#496) cover the period since the previous report.", - "evidence": [], - "gap": "No report summarizes the period's activity (detections, evaluations, reductions, remediations, scans and V&V runs with times), and V&V results are not cross-referenced from the period's report. To close: Add an activity summary for the period to each report (detections, evaluations, reductions, remediations, scans with times, V&V runs and failures), linking the runtime results for the period.", + "implementation": "vdr-report.json, vdr-trend.json, vdr-status.json and ksi-signal-runtime.json are refreshed twice daily and on each deploy. The VDT and AVI each cover the period since the previous report and carry an activity summary for it (providerExtensions.activitySummary, and an 'Activity this period' section in the .md): detections, evaluations, rating changes, reductions, records no longer reported, remediations, decisions recorded and closed, every scan and feed the build read with its time (VDR build, Grype, Dependabot, Checkov, tfsec, ZAP, KEV, EPSS), and the runtime V&V run the build read, linked by URI and signal ID with its validation and failure counts (counted only when its signature verifies).", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_each_period_report_summarizes_the_periods_activity" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_an_unverified_runtime_run_is_linked_but_not_counted" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_the_markdown_reports_carry_the_activity_and_link_the_runtime_results" + } + ], + "gap": "The runtime emitter publishes only its latest run, so a runtime V&V run replaced before the next VDR build is not summarized; and activity outside the pipeline (VDP reports, penetration tests, assessments) is not in the summary (see VER-RPT-HLO). To close: retain each runtime run's result (or have each report list every run since the previous report), and add the non-pipeline activities to the summary.", "closes_with": [ - "W2", "W7" ] }, @@ -1532,12 +1569,45 @@ }, { "frr_id": "VER-RPT-VDT", - "status": "Partially Implemented", - "implementation": "vdr-detail-report.json and .md under /.well-known/ (scripts/build-cr26-vdr.py, #496), validated against the vendored VDT schema before writing: tracking ID, detection time and source, evaluation time, IRV, LEV, current rating, PAIN reductions, projected next reduction, overdue status and final disposition.", - "evidence": [], - "gap": "Historical PAIN and rating history are not carried into the record; the projected next reduction is always the remediation deadline; supplementary information is never produced; overdue explanations for closed findings describe the suppression instead; ZAP detection times start when the scans entered the pipeline, not at the earlier manual scans. To close: Carry historical PAIN and rating history; let a decision set a planned next reduction; populate supplementary information from the decision; take detection dates from the decision where the scanner never saw the finding.", - "closes_with": [ - "W2" + "status": "Implemented", + "implementation": "vdr-detail-report.json and .md under /.well-known/ (scripts/build-cr26-vdr.py), validated against the vendored VDT schema before writing: tracking ID; detection time (the earliest evidence: the scanner's first report, or the original_detection_date of a decision naming that record, e.g. DEC-039/040 for the manual ZAP scans) and source; evaluation time; IRV; LEV; current rating, with the historical rating and full rating history under providerExtensions (the schema has no field for them); PAIN reductions; projected next reduction from the decision's planned_reduction, else the remediation deadline; overdue status whose explanation explains the overdue status (and likely-to-become-overdue); supplementary information from the decision (justification, dates, resources, plan, mitigation); final disposition. Each report carries an activity summary for its period.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_records_carry_their_historical_rating_and_rating_history" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_a_decision_sets_the_next_reduction_and_the_deadline_is_only_the_fallback" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_supplementary_information_comes_from_the_decision" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_overdue_explanations_explain_the_overdue_status_not_the_suppression" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_detection_time_comes_from_a_decision_that_names_the_record" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_outputs_validate_and_name_their_schema" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "step", + "ref": "evidence-nightly.yml#Build and sign CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-detail-report.json" + } ] }, { @@ -1582,12 +1652,41 @@ }, { "frr_id": "VER-TFR-MRH", - "status": "Partially Implemented", - "implementation": "vdr-history.json under /.well-known/ in FedRAMP's history schema (#496), rebuilt twice daily with 366-day retention, alongside vdr-report.json (per-finding rating history) and vdr-trend.json.", - "evidence": [], - "gap": "A record that leaves the VDR is labelled Remediated even when it vanished because coverage stopped or a suppression was added; closed decisions from before the history began are not backfilled. To close: Set a final disposition only from evidence (coverage continued and the scanner stopped reporting it, or a decision); otherwise record 'no longer reported'; backfill closed decisions.", - "closes_with": [ - "W2" + "status": "Implemented", + "implementation": "vdr-history.json under /.well-known/ in FedRAMP's history schema (scripts/build-cr26-vdr.py), rebuilt twice daily with 366-day retention, alongside vdr-report.json and vdr-trend.json. A record that leaves the VDR keeps the time it stopped being reported and gets a final disposition only from evidence (a closed decision, a false-positive suppression of its check, or its scanner running again over the same scope without reporting it), recorded in providerExtensions.dispositionBasis; otherwise noLongerReportedReason says why (suppressed, reported again, or unknown) and no final disposition is given. An unsupported Remediated carried from an earlier history is withdrawn. Closed (remediated) decisions from data/decisions.json are carried as history records (backfilledFromDecision) for 366 days from closing.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_history_keeps_a_finding_that_left_the_vdr_then_drops_it_after_a_year" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_a_record_that_vanished_without_evidence_has_no_final_disposition" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_an_unsupported_remediated_from_an_earlier_history_is_withdrawn" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_a_check_that_became_a_suppression_is_not_remediated" + }, + { + "kind": "test", + "ref": "tests/test_cr26_vdr.py::test_closed_decisions_are_backfilled_into_the_history_once" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "step", + "ref": "evidence-nightly.yml#Build and sign CR26 vulnerability reports (VDT, AVI, history)" + }, + { + "kind": "artifact", + "ref": ".well-known/vdr-history.json" + } ] }, { diff --git a/infrastructure/schemas/decisions.schema.json b/infrastructure/schemas/decisions.schema.json index a7fdcb71..9d1090dd 100644 --- a/infrastructure/schemas/decisions.schema.json +++ b/infrastructure/schemas/decisions.schema.json @@ -35,6 +35,13 @@ "closed_on": {"$ref": "#/$defs/date"}, "original_detection_date": {"$ref": "#/$defs/date"}, "planned_completion": {"type": "string"}, + "planned_reduction": { + "type": "object", + "description": "The next planned reduction in Potential Agency Impact (VER-RPT-VDT): the date it is expected and the rating it brings the finding to. Published as the finding's projectedNextReduction in place of the remediation deadline. In-force records only.", + "required": ["date", "target_pain"], + "properties": {"date": {"$ref": "#/$defs/date"}, "target_pain": {"type": "string", "pattern": "^N[0-5]$"}}, + "additionalProperties": false + }, "remediation_plan": {"type": "string"}, "vendor_dependency": {"type": ["string", "boolean", "object"]}, "point_of_contact": {"type": "string"}, @@ -57,7 +64,8 @@ "then": {"required": ["decided_by", "decided_on", "review_by"], "properties": {"disposition": {"not": {"const": "remediated"}}}}}, {"if": {"properties": {"status": {"const": "closed"}}}, - "then": {"properties": {"disposition": {"const": "remediated"}}}} + "then": {"properties": {"disposition": {"const": "remediated"}}, + "not": {"required": ["planned_reduction"]}}} ] } } diff --git a/scripts/_decisions.py b/scripts/_decisions.py index de3046a6..e847eea9 100644 --- a/scripts/_decisions.py +++ b/scripts/_decisions.py @@ -28,6 +28,8 @@ ID_RE = re.compile(r"^DEC-\d{3}$") FORMER_RE = re.compile(r"^POAM-\d{3}$") DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +PAIN_RE = re.compile(r"^N[0-5]$") +PLAN_KEYS = {"date", "target_pain"} # Dispositions of an in-force record. Config-scanner suppressions use # risk-accepted / false-positive; vulnerabilities use the triage vocabulary. @@ -39,7 +41,7 @@ # field is refused: a misspelt review_by would otherwise read as absent. TOP_KEYS = {"$comment", "_note", "schema", "decisions"} RECORD_KEYS = {"id", "former_id", "title", "status", "disposition", "justification", *DECISION_FIELDS, - "closed_on", "original_detection_date", "planned_completion", "remediation_plan", + "closed_on", "original_detection_date", "planned_completion", "planned_reduction", "remediation_plan", "vendor_dependency", "point_of_contact", "controls", "applies_to", "vuln_fields", "source"} APPLIES_KEYS = {"check_ids", "vuln_ids", "resources"} @@ -93,6 +95,15 @@ def validate(doc, where="data/decisions.json"): _fail(f"{rid}: {k} must be an ISO date") else: _fail(f"{rid}: status must be in-force or closed") + plan = r.get("planned_reduction") + if plan is not None: + # The next planned PAIN reduction (VER-RPT-VDT): when, and to what + # rating. A malformed plan would publish a projection nobody made. + if (not isinstance(plan, dict) or set(plan) != PLAN_KEYS + or not DATE_RE.match(str(plan.get("date"))) or not PAIN_RE.match(str(plan.get("target_pain")))): + _fail(f"{rid}: planned_reduction must be {{date: YYYY-MM-DD, target_pain: N0-N5}}") + if status != "in-force": + _fail(f"{rid}: only an in-force record plans a reduction") applies = r.get("applies_to") if not isinstance(applies, dict) or set(applies) - APPLIES_KEYS: _fail(f"{rid}: applies_to must be an object with only {sorted(APPLIES_KEYS)}") diff --git a/scripts/build-cr26-vdr.py b/scripts/build-cr26-vdr.py index 004ebeb3..3a322e47 100644 --- a/scripts/build-cr26-vdr.py +++ b/scripts/build-cr26-vdr.py @@ -23,15 +23,46 @@ # rationale is the decision record's justification, never invented: an accepted # finding with no justification stops the build. # +# What each record says, and where it comes from: +# detection time the earliest evidence: the scanner's first report, or the +# original_detection_date of a decision naming this exact +# record, when the weakness was found before the pipeline's +# scanner saw it (e.g. a manual scan). A .checkov.yaml +# suppression, which no scanner ever reports, falls back to +# the decision date when the decision records no detection +# date. The basis is in providerExtensions. +# next reduction the decision's planned_reduction (date + target rating) when +# one is recorded, else the remediation deadline. +# overdue explains the overdue status (deadline, and why it is still +# open), never the suppression. +# supplementary the decision behind the record: its justification (VDT only; +# the AVI carries it as the rationale), when it was decided and +# is reviewed, the resources it names, any remediation plan or +# mitigation. +# rating history the historical rating and every re-evaluation, from the VDR +# ledger. The schema has a field only for completed reductions, +# so the full history is in providerExtensions. +# # The schemas carry no field for the component, the CVE, the scanner check or the # decision. Those are the joins a reader needs to trace a finding to the system, # so each record carries them under `providerExtensions` (the schemas allow # additional properties). # # Report periods (VER-RPT-PER: summarize ALL activity since the previous report): -# each VDT/AVI runs from the previous published report's end to now. The MRH -# accumulates: a finding that leaves the VDR stays in the history, marked -# Remediated with the time it stopped being reported, for HISTORY_DAYS. +# each VDT/AVI runs from the previous published report's end to now, and carries +# an activity summary for that period (providerExtensions.activitySummary): +# detections, evaluations, rating changes and reductions, records that stopped +# being reported, decisions recorded and closed, every scan and feed the VDR +# build read with its time, and the runtime verification run it read, linked. +# +# The MRH accumulates. A record that leaves the VDR stays in the history for +# HISTORY_DAYS with the time it stopped being reported. It gets a final +# disposition only from evidence: a closed decision that says remediated, a +# false-positive suppression of its check, or its scanner having run again over +# the same scope without reporting it. Otherwise it is recorded as no longer +# reported, with the reason (a risk-accepted suppression, or unknown), and no +# final disposition. Closed (remediated) decisions are carried in the history +# too, including those from before the pipeline's ledger began. # # build-cr26-vdr.py --vdr vdr-report.json --decisions ../data/decisions.json \ # --previous-vdt prev-vdt.json --previous-avi prev-avi.json --previous-mrh prev-mrh.json @@ -59,6 +90,14 @@ "dependabot": "GitHub Dependabot (software composition analysis)", "opa": "OPA policy evaluation of the deployed configuration", } +SOURCE_BY_LABEL = {v: k for k, v in SOURCES.items()} +# The detection-gate feed that shows a source's scanner ran in a build. A source +# with no feed here (OPA) never earns a disposition from coverage. +COVERAGE_FEEDS = {"grype": "grype", "dependabot": "dependabot", "checkov": "checkov", + "checkov-suppression": "checkov", "tfsec": "tfsec"} +SCAN_FEEDS = {"grype": "Grype software composition scan", "dependabot": "Dependabot alerts", + "checkov": "Checkov IaC scan", "tfsec": "tfsec IaC scan", + "kev": "CISA KEV catalog", "epss": "FIRST EPSS scores"} VDT_OUT = "vdr-detail-report.json" AVI_OUT = "vdr-accepted-vulnerabilities.json" @@ -76,7 +115,7 @@ def _rating(pain): _fail(f"PAIN rating {pain!r} is not N0-N5") -def _ts(value, where): +def _dt(value, where): if not isinstance(value, str): _fail(f"{where}: missing timestamp") try: @@ -85,11 +124,28 @@ def _ts(value, where): _fail(f"{where}: {value!r} is not an ISO 8601 timestamp") if dt.tzinfo is None: _fail(f"{where}: {value!r} has no time zone") - return dt.astimezone(timezone.utc).isoformat() + return dt.astimezone(timezone.utc) + + +def _ts(value, where): + return _dt(value, where).isoformat() + + +def _maybe_dt(value): + """A timestamp, or a YYYY-MM-DD date read as midnight UTC; None if neither.""" + if not isinstance(value, str): + return None + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return (dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)).astimezone(timezone.utc) -def _justifications(register): - return {r["id"]: (r.get("justification") or "").strip() for r in register.get("decisions", [])} +def _day(value): + """A register date (YYYY-MM-DD) as midnight UTC, ISO 8601; None if absent.""" + dt = _maybe_dt(value) + return dt.isoformat() if dt else None def _description(f): @@ -104,16 +160,127 @@ def is_accepted(f): return disp in ACCEPTED_KINDS or bool(f.get("accepted")) -def detail(f): +# --- per-record fields -------------------------------------------------------------- + +def _detection(f, rec, tid): + """(detectedAt, basis): the earliest evidence of the weakness.""" + seen = _dt(f.get("first_detected"), f"{tid}.first_detected") + # Only a decision that names this exact record (a vulnerability ID such as + # zap-10099, or a suppressed check) dates it. A decision about a check across + # many resources dates the check, not a resource added after it. + if rec and tid in _decision_claims(rec): + known = _maybe_dt(rec.get("original_detection_date")) + if known and known < seen: + return known.isoformat(), (f"{rec['id']} original_detection_date: found before the " + "pipeline's scanner first reported it") + decided = _maybe_dt(rec.get("decided_on")) + if not known and f.get("source") == "checkov-suppression" and decided and decided < seen: + return decided.isoformat(), (f"{rec['id']} decided_on: the decision records no detection " + "date, and the weakness was known when it was decided (a " + "suppressed check is never reported by the scanner)") + return seen.isoformat(), "first reported by the scanner (VDR ledger)" + + +def _rating_history(f, tid): + out = [] + for h in f.get("rating_history") or []: + if not isinstance(h, dict): + continue + out.append({"at": _ts(h.get("at"), f"{tid}.rating_history"), "rating": _rating(h.get("pain")), + "isInternetReachable": h.get("irv"), "isLikelyExploitable": h.get("lev"), + "reason": h.get("reason")}) + return out + + +def _overdue(f, rec, final): + """overdueStatus whose explanation explains the overdue status itself.""" + over = bool(f.get("is_overdue")) + due = f.get("remediation_due_at") or (f.get("next_reduction") or {}).get("estimated_time") + did = rec["id"] if rec else f.get("decision_id") + if over: + if f.get("accepted") and f.get("accepted_at"): + why = ("Overdue: not fully mitigated or remediated within the VER-TFR-MAV maximum " + f"period, which ended {f['accepted_at'][:10]}") + elif due: + why = f"Overdue: its remediation deadline, {due[:10]}, passed without remediation" + else: + why = f"Overdue: {f.get('block_reason') or 'past its deadline'}" + if rec and rec.get("disposition") in ACCEPTED_KINDS: + why += (f". It is still open because {did} accepts it ({rec['disposition']}, decided " + f"{rec.get('decided_on', 'on a date not recorded')}), so no remediation is planned") + elif rec and rec.get("disposition") == "vendor-dependency": + why += f". It is waiting on a vendor fix ({did})" + elif f.get("block_reason") and due: + why += f" ({f['block_reason']})" + return {"isOverdue": True, "explanation": why + "."} + if f.get("will_be_overdue") and due: + return {"isOverdue": False, + "explanation": f"Not yet overdue, but likely to become overdue: remediation is due {due[:10]}."} + if final: + return {"isOverdue": False, + "explanation": f"Not overdue: closed as {final}{f' under {did}' if did else ''}."} + if due: + return {"isOverdue": False, "explanation": f"Not overdue: remediation is due {due[:10]}."} + return {"isOverdue": False} + + +def _supplementary(f, rec, accepted): + parts = [] + if rec: + head = (f"{rec['id']} ({rec['disposition']}; decided {rec.get('decided_on', 'on a date not recorded')}" + f"{', review by ' + rec['review_by'] if rec.get('review_by') else ''})") + parts.append(head + ("." if accepted else f": {rec['justification'].strip()}")) + res = (rec.get("applies_to") or {}).get("resources") or [] + if res: + parts.append(f"Resources named by the decision: {', '.join(res)}.") + if rec.get("remediation_plan"): + parts.append(f"Remediation plan: {rec['remediation_plan'].strip()}") + mit = (rec.get("vuln_fields") or {}).get("mitigation") + if mit: + parts.append(f"Mitigation record: {mit}.") + m = f.get("mitigation") + if isinstance(m, dict) and m.get("id"): + state = "verified" if m.get("valid") else f"void ({m.get('void_reason') or 'not verified'})" + parts.append(f"Mitigation {m['id']} ({m.get('effect')}): {state}.") + if f.get("supplementary_info"): + parts.append(str(f["supplementary_info"]).strip()) + return " ".join(p for p in parts if p) + + +def _projection(f, rec, rating, tid): + """(projectedNextReduction, basis), or (None, None) when none is planned.""" + plan = (rec or {}).get("planned_reduction") + if plan: + target = _rating(plan.get("target_pain")) + if target < rating: + return ({"estimatedAt": _ts(_day(plan.get("date")), f"{rec['id']}.planned_reduction"), + "targetRating": target}, f"{rec['id']} planned_reduction") + nxt = f.get("next_reduction") + if nxt and nxt.get("estimated_time"): + target = nxt.get("target_pain") + basis = "remediation deadline (no planned reduction recorded)" + if plan: + basis = f"remediation deadline ({rec['id']} planned_reduction is not below the current rating)" + # "remediated" removes the vulnerability: no impact remains (N0). + return ({"estimatedAt": _ts(nxt["estimated_time"], f"{tid}.next_reduction"), + "targetRating": 0 if target == "remediated" else _rating(target)}, basis) + return None, None + + +def detail(f, recs=None, accepted=False): """A VDR finding as a FedRAMP vulnerabilityDetail record.""" + recs = recs or {} tid = f.get("tracking_id") if not tid: _fail("a finding has no tracking_id") + rec = recs.get(f.get("decision_id")) pain = f.get("current_pain") or f.get("pain") - rec = { + rating = _rating(pain) + detected_at, detected_basis = _detection(f, rec, tid) + out = { "providerTrackingId": tid, "detection": { - "detectedAt": _ts(f.get("first_detected"), f"{tid}.first_detected"), + "detectedAt": detected_at, "detectionSource": SOURCES.get(f.get("source"), f.get("source") or "unknown"), }, "vulnerabilityDescription": _description(f) or tid, @@ -124,34 +291,31 @@ def detail(f): f"{tid}.first_evaluated"), "isInternetReachable": bool(f.get("internet_reachable")), "isLikelyExploitable": bool(f.get("likely_exploitable")), - "currentRating": _rating(pain), + "currentRating": rating, } - over = {"isOverdue": bool(f.get("is_overdue"))} - if f.get("overdue_explanation"): - over["explanation"] = f["overdue_explanation"] - elif over["isOverdue"]: - _fail(f"{tid}: overdue with no explanation") - rec["overdueStatus"] = over - if f.get("supplementary_info"): - rec["supplementaryRiskInformation"] = f["supplementary_info"] - reductions = [{"reducedAt": _ts(r.get("at"), f"{tid}.completed_reductions"), "rating": _rating(r.get("to"))} - for r in f.get("completed_reductions") or []] - if reductions: - rec["painReductionEvents"] = reductions disp = f.get("final_disposition") or f.get("current_disposition") or "open" + final = None if disp in FINAL: - rec["finalDisposition"] = FINAL[disp] + final = FINAL[disp] elif disp == "mitigated": - rec["finalDisposition"] = "Fully Mitigated" if f.get("fully_mitigated") else "Partially Mitigated" - nxt = f.get("next_reduction") - if "finalDisposition" not in rec and not is_accepted(f) and nxt and nxt.get("estimated_time"): - target = nxt.get("target_pain") - # "remediated" removes the vulnerability: no impact remains (N0). - rec["projectedNextReduction"] = { - "estimatedAt": _ts(nxt["estimated_time"], f"{tid}.next_reduction"), - "targetRating": 0 if target == "remediated" else _rating(target), - } - rec["providerExtensions"] = { + final = "Fully Mitigated" if f.get("fully_mitigated") else "Partially Mitigated" + out["overdueStatus"] = _overdue(f, rec, final) + supp = _supplementary(f, rec, accepted) + if supp: + out["supplementaryRiskInformation"] = supp + reductions = [{"reducedAt": _ts(r.get("at"), f"{tid}.completed_reductions"), "rating": _rating(r.get("to"))} + for r in f.get("completed_reductions") or []] + if reductions: + out["painReductionEvents"] = reductions + if final: + out["finalDisposition"] = final + projection_basis = None + if final is None and not is_accepted(f): + projection, projection_basis = _projection(f, rec, rating, tid) + if projection: + out["projectedNextReduction"] = projection + ext = { + "source": f.get("source"), "componentId": f.get("component_id"), "resource": f.get("resource") or None, "cve": f.get("cve"), @@ -160,20 +324,36 @@ def detail(f): "disposition": disp, "isKev": bool(f.get("is_kev")), "remediationDueAt": f.get("remediation_due_at"), + "detectedAtBasis": detected_basis, + "firstReportedAt": _ts(f.get("first_detected"), f"{tid}.first_detected"), } - return rec - - -def accepted_info(f, justifications): + if f.get("affected_resources"): + ext["affectedResources"] = list(f["affected_resources"]) + if projection_basis: + ext["projectionBasis"] = projection_basis + # VER-RPT-VDT asks for the historical and current ratings; the schema has a + # field only for the current one and for completed reductions. + if f.get("historical_pain"): + ext["historicalRating"] = _rating(f["historical_pain"]) + history = _rating_history(f, tid) + if history: + ext["ratingHistory"] = history + out["providerExtensions"] = ext + return out + + +def accepted_info(f, recs): did = f.get("decision_id") - why = justifications.get(did, "") if did else "" + why = ((recs.get(did) or {}).get("justification") or "").strip() if did else "" if not why and f.get("accepted") and f.get("acceptance_explanation"): why = f["acceptance_explanation"] if not why: _fail(f"accepted finding {f.get('tracking_id')} has no decision justification ({did or 'no decision_id'})") - return {"vulnerabilityDetail": detail(f), "acceptanceRationale": why} + return {"vulnerabilityDetail": detail(f, recs, accepted=True), "acceptanceRationale": why} +# --- periods ------------------------------------------------------------------------ + def _period_start(previous, fallback): """The previous report's end, if it is a valid past time; else `fallback`.""" try: @@ -199,15 +379,104 @@ def _load_previous(path): return doc +def activity_summary(vdr, mrh, register, frm, to): + """VER-RPT-PER: the vulnerability detection and response activity in + [frm, to], from the VDR report, the history and the decision register.""" + lo, hi = _dt(frm, "period.from"), _dt(to, "period.to") + + def within(value): + dt = _maybe_dt(value) + return bool(dt and lo <= dt <= hi) + + rows = [f for f in vdr.get("findings") or [] if isinstance(f, dict)] + \ + [r for r in vdr.get("risk_accepted") or [] if isinstance(r, dict)] + detections = [{"providerTrackingId": f["tracking_id"], "at": f["first_detected"]} + for f in rows if within(f.get("first_detected"))] + evaluations = [{"providerTrackingId": f["tracking_id"], "at": f["first_evaluated"]} + for f in rows if within(f.get("first_evaluated"))] + changes = [{"providerTrackingId": f["tracking_id"], "at": h["at"], "rating": h.get("pain"), + "reason": h.get("reason")} + for f in rows for h in f.get("rating_history") or [] + if isinstance(h, dict) and within(h.get("at")) and h.get("reason") != "first evaluation"] + reductions = [{"providerTrackingId": f["tracking_id"], "at": r["at"], "from": r.get("from"), "to": r.get("to")} + for f in rows for r in f.get("completed_reductions") or [] + if isinstance(r, dict) and within(r.get("at"))] + gone = [{"providerTrackingId": r["providerTrackingId"], + "at": r["providerExtensions"]["noLongerReportedAt"], + "finalDisposition": r.get("finalDisposition"), + "reason": r["providerExtensions"].get("noLongerReportedReason")} + for r in mrh.get("activeVulnerabilities") or [] + if within((r.get("providerExtensions") or {}).get("noLongerReportedAt"))] + decided = [{"decisionId": d["id"], "on": d["decided_on"], "disposition": d["disposition"]} + for d in register.get("decisions", []) if within(_day(d.get("decided_on")))] + closed = [{"decisionId": d["id"], "on": d["closed_on"], "disposition": d["disposition"]} + for d in register.get("decisions", []) + if d.get("status") == "closed" and within(_day(d.get("closed_on")))] + remediated = {g["providerTrackingId"] for g in gone if g["finalDisposition"] == "Remediated"} | \ + {c["decisionId"] for c in closed} + meth = vdr.get("methodology") or {} + det = meth.get("detection") or {} + scans = [{"activity": "VDR build: every finding re-evaluated (PAIN, IRV, LEV, deadlines)", + "at": vdr.get("emitted_at"), "timeBasis": "build time", "status": "ok", + "inPeriod": within(vdr.get("emitted_at"))}] + for feed in det.get("feeds") or []: + name = SCAN_FEEDS.get(feed.get("feed")) if isinstance(feed, dict) else None + if not name: + continue + if feed.get("feed") in ("kev", "epss"): + at, basis = feed.get("source_date"), "published by the source" + else: + at, basis = det.get("checked_at"), "run or fetched in this build (when the detection gate checked it)" + row = {"activity": f"{name} ({feed.get('file')})", "at": at, "timeBasis": basis, + "status": feed.get("status"), "count": feed.get("count"), "inPeriod": within(at)} + if feed.get("feed") == "grype" and feed.get("source_date"): + row["vulnerabilityDatabaseBuiltAt"] = feed["source_date"] + scans.append(row) + dast = meth.get("dast") + if isinstance(dast, dict): + scans.append({"activity": "OWASP ZAP dynamic scan (weekly)", "at": dast.get("generated_at"), + "timeBasis": "scan report generated", "status": dast.get("status"), + "count": dast.get("alerts"), "inPeriod": within(dast.get("generated_at"))}) + rt = meth.get("runtime_verification") + runtime = None + if isinstance(rt, dict): + runtime = {"uri": rt.get("uri"), "signalId": rt.get("signal_id"), "emittedAt": rt.get("emitted_at"), + "signatureVerified": bool(rt.get("verified")), "validations": rt.get("validations"), + "failed": rt.get("failed"), "divergence": rt.get("divergence"), + "inPeriod": within(rt.get("emitted_at")), + "note": ("The runtime emitter publishes only its latest run; this is the run the VDR " + "build read. A run it replaced before this build is not retained.")} + if not rt.get("verified"): + runtime["note"] = f"Runtime signal not verified ({rt.get('reason')}); its results are not reported." + in_period = [s for s in scans if s["inPeriod"]] + run_in_period = bool(runtime and runtime["inPeriod"] and runtime["signatureVerified"]) + return { + "period": {"from": frm, "to": to}, + "counts": {"detections": len(detections), "evaluations": len(evaluations), + "ratingChanges": len(changes), "reductions": len(reductions), + "noLongerReported": len(gone), "remediations": len(remediated), + "decisionsRecorded": len(decided), "decisionsClosed": len(closed), + "scansInPeriod": len(in_period), + "runtimeRunsInPeriod": 1 if run_in_period else 0, + "runtimeFailuresInPeriod": (runtime["failed"] or 0) if run_in_period else 0}, + "detections": detections, "evaluations": evaluations, "ratingChanges": changes, + "reductions": reductions, "noLongerReported": gone, "remediations": sorted(remediated), + "decisionsRecorded": decided, "decisionsClosed": closed, "scans": scans, + "runtimeVerification": runtime, + } + + +# --- build -------------------------------------------------------------------------- + def build(vdr, register, previous_vdt, previous_avi, previous_mrh, now): if not isinstance(vdr, dict) or not isinstance(vdr.get("findings"), list): _fail("vdr-report.json has no findings list") - just = _justifications(register) + recs = _decisions.records(register) scanner = list(vdr["findings"]) # Suppressed checks the VDR lists as risk-accepted are accepted vulnerabilities too. suppressed = [dict(r, final_disposition="risk-accepted") for r in vdr.get("risk_accepted") or []] - active = [detail(f) for f in scanner if not is_accepted(f)] - accepted = [accepted_info(f, just) for f in scanner + suppressed if is_accepted(f)] + active = [detail(f, recs) for f in scanner if not is_accepted(f)] + accepted = [accepted_info(f, recs) for f in scanner + suppressed if is_accepted(f)] times = [r["detection"]["detectedAt"] for r in active] + \ [a["vulnerabilityDetail"]["detection"]["detectedAt"] for a in accepted] earliest = min(times) if times else now.isoformat() @@ -223,18 +492,119 @@ def build(vdr, register, previous_vdt, previous_avi, previous_mrh, now): avi = {"$schema": fs.stamp("avi"), **cpo, "reportPeriod": {"from": _period_start(previous_avi, earliest), "to": to}, "acceptedVulnerabilities": accepted, "providerExtensions": dict(ext)} - mrh = build_history(active, accepted, previous_mrh, now) - mrh["providerExtensions"] = {**ext, **mrh["providerExtensions"]} for doc, period in ((vdt, "VDT"), (avi, "AVI")): if doc["reportPeriod"]["from"] > doc["reportPeriod"]["to"]: _fail(f"{period} period starts after it ends") + mrh = build_history(active, accepted, previous_mrh, now, vdr=vdr, register=register) + mrh["providerExtensions"] = {**ext, **mrh["providerExtensions"]} + for doc in (vdt, avi): + doc["providerExtensions"]["activitySummary"] = activity_summary( + vdr, mrh, register, doc["reportPeriod"]["from"], doc["reportPeriod"]["to"]) return vdt, avi, mrh -def build_history(active, accepted, previous, now): - """VER-TFR-MRH: every current record, plus records that left the VDR within - HISTORY_DAYS, closed as Remediated with the time they stopped being reported.""" - cutoff = (now - timedelta(days=HISTORY_DAYS)).isoformat() +# --- history ------------------------------------------------------------------------ + +def _keys(rec): + ext = rec.get("providerExtensions") or {} + return {k for k in (rec.get("providerTrackingId"), ext.get("scannerCheck"), ext.get("cve")) if k} + + +def _decision_claims(d): + a = d.get("applies_to") or {} + return set(a.get("check_ids") or []) | set(a.get("vuln_ids") or []) + + +def _coverage(source, vdr, last_reported): + """Evidence that `source`'s scanner ran again over the same scope after the + record was last reported, or None.""" + meth = (vdr or {}).get("methodology") or {} + if source == "zap": + dast = meth.get("dast") or {} + gen, last = _maybe_dt(dast.get("generated_at")), _maybe_dt(last_reported) + if dast.get("status") == "ok" and gen and last and gen > last: + return f"the ZAP scan of {gen.isoformat()} ran after it was last reported and does not report it" + return None + feed = COVERAGE_FEEDS.get(source) + det = meth.get("detection") or {} + rows = [r for r in det.get("feeds") or [] if isinstance(r, dict) and r.get("feed") == feed] + if feed and det.get("checked_at") and rows and all(r.get("status") == "ok" for r in rows): + return (f"{feed} ran in the VDR build checked {det['checked_at']} (status ok) " + "and no longer reports it") + return None + + +def _disposition_from_evidence(rec, vdr, register, last_reported): + """(finalDisposition or None, reason, basis) for a record that left the VDR.""" + ext = rec.get("providerExtensions") or {} + keys = _keys(rec) + did = ext.get("decisionId") + for d in (register or {}).get("decisions", []): + if d.get("status") == "closed" and (d["id"] == did or keys & _decision_claims(d)): + return ("Remediated", "remediated", + f"{d['id']} closed as remediated on {d.get('closed_on', 'a date not recorded')}") + check = ext.get("scannerCheck") + for section, final, reason in (("false_positives", "False Positive", "false-positive"), + ("risk_accepted", None, "suppressed")): + for s in (vdr or {}).get(section) or []: + if (isinstance(s, dict) and check and s.get("tool_id") == check + and s.get("tracking_id") != rec.get("providerTrackingId")): + return final, reason, (f"its check {check} is now suppressed in .checkov.yaml " + f"({s.get('decision_id') or 'no decision'}, {s.get('current_disposition')})") + source = ext.get("source") or SOURCE_BY_LABEL.get((rec.get("detection") or {}).get("detectionSource")) + if source == "checkov-suppression": + again = [f["tracking_id"] for f in (vdr or {}).get("findings") or [] + if isinstance(f, dict) and check and f.get("tool_id") == check] + if again: + return None, "reported-again", (f"its check {check} is no longer suppressed and the scanner " + f"reports it again as {', '.join(sorted(again))}") + covered = _coverage(source, vdr, last_reported) + if covered: + return "Remediated", "remediated", covered + return None, "unknown", ("no evidence of why: no closed decision names it, no suppression covers it, " + "and its scanner is not shown to have run over it again since it was last reported") + + +_BACKFILL_DETECTION = ( + ("original_detection_date", "original_detection_date"), + ("decided_on", "decided_on: no detection date is recorded; known by the decision date"), + ("closed_on", "closed_on: neither a detection nor a decision date is recorded; detected no later " + "than it was closed"), +) +_BACKFILL_SOURCE = {"original_risk_rating": "originalRiskRating", "adjusted_risk_rating": "adjustedRiskRating", + "identifier": "sourceIdentifier"} + + +def _backfill(d, now): + """A closed (remediated) decision as a history record.""" + at, basis = now.isoformat(), f"{d['id']}: no date is recorded" + for field, why in _BACKFILL_DETECTION: + if _day(d.get(field)): + at, basis = _day(d[field]), f"{d['id']} {why}" + break + src = d.get("source") or {} + ext = {"decisionId": d["id"], "disposition": "remediated", "backfilledFromDecision": True, + "remediatedOn": d.get("closed_on"), "detectedAtBasis": basis, + "dispositionBasis": f"{d['id']} closed as remediated on {d.get('closed_on', 'a date not recorded')}", + "affectedResources": list((d.get("applies_to") or {}).get("resources") or [])} + if d.get("former_id"): + ext["formerId"] = d["former_id"] + if d.get("remediation_plan"): + ext["remediationPlan"] = d["remediation_plan"] + ext.update({out: src[k] for k, out in _BACKFILL_SOURCE.items() if src.get(k)}) + return {"providerTrackingId": d["id"], + "detection": {"detectedAt": at, "detectionSource": src.get("detector") or "not recorded"}, + "vulnerabilityDescription": d["title"], + "supplementaryRiskInformation": d["justification"].strip(), + "finalDisposition": "Remediated", + "providerExtensions": ext} + + +def build_history(active, accepted, previous, now, vdr=None, register=None): + """VER-TFR-MRH: every current record; every record that left the VDR within + HISTORY_DAYS, with a final disposition only where evidence gives one; and + every closed (remediated) decision closed within HISTORY_DAYS.""" + cutoff = now - timedelta(days=HISTORY_DAYS) current = {r["providerTrackingId"] for r in active} | \ {a["vulnerabilityDetail"]["providerTrackingId"] for a in accepted} carried = [] @@ -244,27 +614,96 @@ def build_history(active, accepted, previous, now): if not isinstance(rec, dict) or rec.get("providerTrackingId") in current: continue ext = dict(rec.get("providerExtensions") or {}) + if ext.get("backfilledFromDecision"): + continue # rebuilt from the register below gone = ext.get("noLongerReportedAt") or now.isoformat() - if gone < cutoff: + if (_maybe_dt(gone) or now) < cutoff: continue closed = dict(rec) closed.pop("projectedNextReduction", None) - closed.setdefault("finalDisposition", "Remediated") - closed["overdueStatus"] = {"isOverdue": False} ext["noLongerReportedAt"] = gone - closed["providerExtensions"] = ext - carried.append(closed) + ext.setdefault("lastReportedAt", prev.get("generatedAt")) + final = closed.get("finalDisposition") + if final and (ext.get("dispositionBasis") or final != "Remediated"): + # Settled: the disposition it had when last reported, or one already + # earned from evidence. + ext.setdefault("dispositionBasis", "its final disposition when it was last reported") + ext.setdefault("noLongerReportedReason", "closed") + else: + # Re-decided each run until evidence settles it (a closed decision + # may come later). A Remediated with no evidence is withdrawn. + closed.pop("finalDisposition", None) + final, reason, basis = _disposition_from_evidence(closed, vdr, register, ext["lastReportedAt"]) + ext["noLongerReportedReason"] = reason + ext.pop("noLongerReportedBasis", None) + if final: + closed["finalDisposition"] = final + ext["dispositionBasis"] = basis + else: + ext["noLongerReportedBasis"] = basis + if closed.get("finalDisposition"): + closed["overdueStatus"] = {"isOverdue": False, "explanation": + f"Not overdue: no longer reported; {closed['finalDisposition']}."} + carried.append({**closed, "providerExtensions": ext}) + linked = {(r.get("providerExtensions") or {}).get("decisionId") for r in carried + active} | \ + {(a["vulnerabilityDetail"].get("providerExtensions") or {}).get("decisionId") for a in accepted} + claimed = set().union(*(_keys(r) for r in carried)) + backfilled = [] + for d in (register or {}).get("decisions", []): + if d.get("status") != "closed" or d["id"] in linked or _decision_claims(d) & claimed: + continue + closed_on = _maybe_dt(d.get("closed_on")) + if closed_on and closed_on < cutoff: + continue + backfilled.append(_backfill(d, now)) return {"$schema": fs.stamp("mrh"), "certificationPackageOverviewUri": fs.CPO_URI, "generatedAt": now.isoformat(), - "activeVulnerabilities": active + sorted(carried, key=lambda r: r["providerTrackingId"]), + "activeVulnerabilities": active + sorted(carried + backfilled, key=lambda r: r["providerTrackingId"]), "acceptedVulnerabilities": accepted, "providerExtensions": { - "retention": f"Records that leave the VDR stay {HISTORY_DAYS} days, closed as Remediated " - "with the time they stopped being reported (noLongerReportedAt)."}} - - -def render_markdown(title, rule, records, accepted=False): + "retention": (f"Records that leave the VDR stay {HISTORY_DAYS} days with the time they stopped " + "being reported (noLongerReportedAt). A final disposition is set only from " + "evidence (dispositionBasis): a closed decision, a false-positive suppression, " + "or the scanner running again without reporting it. Otherwise " + "noLongerReportedReason says why it is not reported, or 'unknown', and no final " + "disposition is given. Closed (remediated) decisions are included " + "(backfilledFromDecision) for the same period from their closing date.")}} + + +# --- markdown ----------------------------------------------------------------------- + +def _cell(value): + return str(value).replace("|", "\\|").replace("\n", " ") + + +def render_activity(summary): + c = summary["counts"] + out = ["## Activity this period", "", + f"{summary['period']['from']} to {summary['period']['to']}", "", + f"- Detections: {c['detections']}; evaluations: {c['evaluations']}; rating changes: " + f"{c['ratingChanges']}; reductions: {c['reductions']}", + f"- No longer reported: {c['noLongerReported']}; remediations: {c['remediations']}; " + f"decisions recorded: {c['decisionsRecorded']}; decisions closed: {c['decisionsClosed']}", + "", "| Scan or feed | Time | Status | Count | In period |", "|---|---|---|---|---|"] + for s in summary["scans"]: + out.append(f"| {_cell(s['activity'])} | {s.get('at') or 'not recorded'} ({s['timeBasis']}) | " + f"{s.get('status')} | {'' if s.get('count') is None else s['count']} | " + f"{'yes' if s['inPeriod'] else 'no'} |") + rt = summary.get("runtimeVerification") + out.append("") + if rt: + out.append(f"Runtime verification: [{rt['uri']}]({rt['uri']}) emitted {rt.get('emittedAt')}, " + f"{rt.get('validations')} validations, {rt.get('failed')} failed, divergence " + f"{rt.get('divergence')}; {'in' if rt['inPeriod'] else 'not in'} this period. {rt['note']}") + else: + out.append("Runtime verification: not recorded by the VDR build.") + return out + + +def render_markdown(title, rule, records, accepted=False, summary=None): out = [f"# {title}", "", f"_FedRAMP {rule}. Source of truth: the signed JSON beside this file._", ""] + if summary: + out += render_activity(summary) + ["", "## Vulnerabilities", ""] if not records: return "\n".join(out + ["_None._"]) out += ["| Tracking ID | Detected | Internet-reachable | Likely exploitable | Rating | Overdue | " @@ -276,8 +715,7 @@ def render_markdown(title, rule, records, accepted=False): out.append("| `{t}` | {at} | {irv} | {lev} | N{n} | {od} | {last} |".format( t=d["providerTrackingId"], at=d["detection"]["detectedAt"][:10], irv="yes" if d["isInternetReachable"] else "no", lev="yes" if d["isLikelyExploitable"] else "no", - n=d["currentRating"], od="yes" if d["overdueStatus"]["isOverdue"] else "no", - last=last.replace("|", "\\|").replace("\n", " "))) + n=d["currentRating"], od="yes" if d["overdueStatus"]["isOverdue"] else "no", last=_cell(last))) return "\n".join(out) @@ -312,10 +750,11 @@ def main(): fs.write_validated(avi, "avi", out / AVI_OUT, registry) fs.write_validated(mrh, "mrh", out / MRH_OUT, registry) (out / VDT_OUT.replace(".json", ".md")).write_text( - render_markdown("Vulnerability Detail Report", "VER-RPT-VDT", vdt["vulnerabilities"]) + "\n") + render_markdown("Vulnerability Detail Report", "VER-RPT-VDT", vdt["vulnerabilities"], + summary=vdt["providerExtensions"]["activitySummary"]) + "\n") (out / AVI_OUT.replace(".json", ".md")).write_text( render_markdown("Accepted Vulnerability Info", "VER-RPT-AVI", avi["acceptedVulnerabilities"], - accepted=True) + "\n") + accepted=True, summary=avi["providerExtensions"]["activitySummary"]) + "\n") print(f"CR26 vulnerability reports: VDT {len(vdt['vulnerabilities'])}, " f"AVI {len(avi['acceptedVulnerabilities'])}, history {len(mrh['activeVulnerabilities'])} active + " f"{len(mrh['acceptedVulnerabilities'])} accepted") diff --git a/scripts/build-vdr-report.py b/scripts/build-vdr-report.py index f037032d..8cc2f592 100755 --- a/scripts/build-vdr-report.py +++ b/scripts/build-vdr-report.py @@ -289,6 +289,7 @@ def classify_vulnerability(finding, register, today=None, mitigation=None): # The runtime emitter runs daily; two days allows one missed run. Moves to the # governed freshness config with the other windows. RUNTIME_SIGNAL_MAX_AGE_HOURS = 48 +RUNTIME_SIGNAL_URI = "https://samaydlette.com/.well-known/ksi-signal-runtime.json" def component_fingerprint(component): @@ -378,6 +379,8 @@ def load_runtime_signal(path, pubkey_path=None, now=None, max_age_hours=RUNTIME_ return out out["verified"] = True out["emitted_at"] = sig.get("emitted_at") + out["signal_id"] = sig.get("signal_id") + out["divergence"] = (sig.get("divergence") or {}).get("status") emitted = _ts(sig.get("emitted_at"), None) if sig.get("emitted_at") else None out["fresh"] = bool(emitted and timedelta(0) <= now - emitted <= timedelta(hours=max_age_hours)) if not out["fresh"]: @@ -386,6 +389,45 @@ def load_runtime_signal(path, pubkey_path=None, now=None, max_age_hours=RUNTIME_ return out +def runtime_verification_summary(runtime): + """The runtime V&V run this report read (VER-RPT-PER): when it ran, whether + its signature verified, how many validations it ran and how many failed. + Counts are published only from a verified signal.""" + runtime = runtime or {} + out = {"uri": RUNTIME_SIGNAL_URI, "verified": bool(runtime.get("verified")), + "fresh": bool(runtime.get("fresh")), "reason": runtime.get("reason"), + "emitted_at": runtime.get("emitted_at"), "signal_id": runtime.get("signal_id"), + "validations": None, "failed": None, "divergence": None} + if out["verified"]: + vals = [v for v in runtime.get("validations") or [] if isinstance(v, dict)] + out["validations"] = len(vals) + out["failed"] = sum(1 for v in vals if v.get("result") != "pass") + out["divergence"] = runtime.get("divergence") + return out + + +def dast_summary(zap_path): + """The DAST scan this report ingested: when ZAP generated it and how many + alert types it reported. None when no ZAP report was given.""" + if not zap_path: + return None + p = Path(zap_path) + if not p.is_file(): + return {"scanner": "zap", "status": "missing", "generated_at": None, "alerts": None} + try: + doc = json.loads(p.read_text()) + except (json.JSONDecodeError, UnicodeDecodeError): + return {"scanner": "zap", "status": "unreadable", "generated_at": None, "alerts": None} + if not isinstance(doc, dict): + return {"scanner": "zap", "status": "unreadable", "generated_at": None, "alerts": None} + # ZAP writes nanoseconds; datetime parses at most microseconds. + raw = re.sub(r"(\.\d{6})\d+", r"\1", doc["created"]) if isinstance(doc.get("created"), str) else None + created = _ts(raw, None) if raw else None + gen = created.astimezone(timezone.utc).isoformat() if created and created.tzinfo else None + alerts = sum(len(site.get("alerts") or []) for site in doc.get("site") or [] if isinstance(site, dict)) + return {"scanner": "zap", "status": "ok", "generated_at": gen, "alerts": alerts} + + def mitigation_state(finding, mitigations, inventory, runtime, today): """The mitigation covering this finding, evaluated, or None. When several name the finding, the first valid one (by ID) is used, else the first; a @@ -739,6 +781,9 @@ def ingest_suppressions(checkov_yaml): "title": f"{check_id} suppressed", "description": rationale, "resource": ".checkov.yaml", + # What the suppression affects: the resources its decision names. + "affected_resources": list(_decisions.records(DECISIONS).get( + RISK_ACCEPTED_BY_CHECK_ID.get(check_id), {}).get("applies_to", {}).get("resources", [])), "current_disposition": "risk-accepted", "detected_at": detected.get(RISK_ACCEPTED_BY_CHECK_ID.get(check_id)), "pain": evaluation["pain"], @@ -1642,6 +1687,42 @@ def resolve_component(finding, index, tf_components=()): return None +_DECISION_RESOURCE = re.compile(r"^aws-([a-z0-9-]+?)(?:::([\w.-]+))?$") + + +def resolve_decision_resource(name, inventory): + """The component a decision record's resource name denotes, or None. An + exact inventory identifier first; else the register's `aws-[::]` + form, matched to the one Terraform component of that resource type (and + name, when given). Ambiguous or unknown: None, never a guess.""" + if not isinstance(name, str) or not name: + return None + if name in inventory["index"]: + return inventory["index"][name] + m = _DECISION_RESOURCE.match(name) + if not m: + return None + tf_type = "aws_" + m.group(1).replace("-", "_") + want = m.group(2).replace("-", "_") if m.group(2) else None + hits = {cid for cid, t, n in inventory["tf"] if t == tf_type and (want is None or n == want)} + return hits.pop() if len(hits) == 1 else None + + +def resolve_decision_resources(names, inventory): + """The component a decision's resources resolve to: the one they all name, + or, when they name several, an internet-reachable one first (the + conservative choice for IRV). None when none resolves.""" + cids = [] + for n in names or []: + cid = resolve_decision_resource(n, inventory) + if cid and cid not in cids: + cids.append(cid) + if not cids: + return None + reachable = [c for c in cids if (inventory["classes"].get(c) or {}).get("internet_reachable") is True] + return (reachable or cids)[0] + + def tf_components_of(ksi_signal_path): """(component_id, tf_type, tf_name) for every Terraform-backed component.""" p = Path(ksi_signal_path) @@ -1954,8 +2035,14 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No sup_prepared = [] for s in suppressions: s = apply_check_vector(s, check_vectors) - s_cid = resolve_component(s, inv["index"], inv["tf"]) - s = {**s, "_classification": classification_for(s, inventory, index)} + # A .checkov.yaml suppression names a file, not a resource: it affects + # the resources its decision record names (VER-RPT-AVI), so those are + # what reachability and classification are decided from. + s_cid = (resolve_component(s, inv["index"], inv["tf"]) + or resolve_decision_resources(s.get("affected_resources"), inv)) + s = {**s, "_classification": ((inv["classes"].get(s_cid) or _FAILSAFE_CLASSIFICATION) + if s_cid and inventory is not None + else classification_for(s, inventory, index))} s_pain = assign_pain(s, index, config) if s.get("cvss_vector") else s.get("pain") items.append(triage_item(s, "suppression", pain=s_pain, is_kev=s.get("is_kev", False), epss_scores=epss_scores, reach=reach_facts(s, s_cid, inv))) @@ -2111,6 +2198,7 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "tool_id": s.get("tool_id"), "title": s.get("title", ""), "resource": s.get("resource", ""), + **({"affected_resources": s["affected_resources"]} if s.get("affected_resources") else {}), "first_detected": s_clk["first_detected"], "first_evaluated": s_clk["first_evaluated"], "completed_evaluation": now.isoformat(), @@ -2328,6 +2416,7 @@ def main(): check_vectors = load_check_vectors(args.check_vectors) lev_overrides = load_lev_overrides(args.lev_overrides) + runtime = load_runtime_signal(args.runtime_signal, args.runtime_pubkey) report, blocking = build_report(findings, suppressions, kev_cves, ledger, index=classification_index, config=pain_config, epss_scores=epss_scores, @@ -2336,7 +2425,7 @@ def main(): lev_overrides=lev_overrides, inventory=load_inventory(args.ksi_signal), mitigations=load_mitigations(args.mitigations), - runtime=load_runtime_signal(args.runtime_signal, args.runtime_pubkey)) + runtime=runtime) report["ksi_signal_id"] = ksi_signal_id report["false_positives"] = false_positives # Name the inventory component each finding affects, so the trust center can @@ -2394,6 +2483,11 @@ def main(): # REQ-GOV-2: the profile this report conforms to, and the results of that # profile's conformance tests it was built under. report["methodology"]["profile"] = load_conformance(args.conformance) if args.conformance else None + # VER-RPT-PER: the runs this report saw that the detection summary does not + # list: the runtime verification (its signed signal, verified above) and the + # DAST scan. Facts only; the CR26 period reports summarize them. + report["methodology"]["runtime_verification"] = runtime_verification_summary(runtime) + report["methodology"]["dast"] = dast_summary(args.zap) # G7: what each detection input was when this report was built (T52). report["methodology"]["detection"] = (json.loads(Path(args.detection).read_text()) if args.detection else None) diff --git a/tests/fixtures/decisions/planned-reduction-closed.json b/tests/fixtures/decisions/planned-reduction-closed.json new file mode 100644 index 00000000..5cc23945 --- /dev/null +++ b/tests/fixtures/decisions/planned-reduction-closed.json @@ -0,0 +1,28 @@ +{ + "_note": "Known-bad: a closed record that still plans a reduction. A closed (remediated) decision has nothing left to reduce; the build stops.", + "schema": "decisions/1", + "decisions": [ + { + "id": "DEC-001", + "title": "fixture", + "status": "closed", + "disposition": "remediated", + "justification": "fixture", + "decided_by": "Fixture", + "decided_on": "2026-01-01", + "review_by": "2027-01-01", + "applies_to": { + "check_ids": [], + "vuln_ids": [ + "CVE-2099-0001" + ], + "resources": [] + }, + "closed_on": "2026-06-01", + "planned_reduction": { + "date": "2026-12-01", + "target_pain": "N1" + } + } + ] +} diff --git a/tests/fixtures/decisions/planned-reduction-malformed.json b/tests/fixtures/decisions/planned-reduction-malformed.json new file mode 100644 index 00000000..4df6833d --- /dev/null +++ b/tests/fixtures/decisions/planned-reduction-malformed.json @@ -0,0 +1,27 @@ +{ + "_note": "Known-bad: a planned reduction whose target is not an N-rating. A malformed plan would publish a projected reduction nobody planned; the build stops.", + "schema": "decisions/1", + "decisions": [ + { + "id": "DEC-001", + "title": "fixture", + "status": "in-force", + "disposition": "remediate", + "justification": "fixture", + "decided_by": "Fixture", + "decided_on": "2026-01-01", + "review_by": "2027-01-01", + "applies_to": { + "check_ids": [], + "vuln_ids": [ + "CVE-2099-0001" + ], + "resources": [] + }, + "planned_reduction": { + "date": "2026-12-01", + "target_pain": "low" + } + } + ] +} diff --git a/tests/test_cr26_vdr.py b/tests/test_cr26_vdr.py index 6351753c..70f8ccc4 100644 --- a/tests/test_cr26_vdr.py +++ b/tests/test_cr26_vdr.py @@ -10,9 +10,14 @@ jsonschema; - each record carries the joins the schemas have no field for (component, CVE, check, decision) under providerExtensions; -- report periods run from the previous report's end (VER-RPT-PER); -- the history keeps a finding that left the VDR, closed as Remediated, for a - year, then drops it. +- report periods run from the previous report's end (VER-RPT-PER), and each + period report summarizes the period's activity, linking the runtime results; +- records carry the historical rating and rating history, the decision's + planned reduction (else the deadline), supplementary information from the + decision, an overdue explanation about the overdue status, and a detection + time from a decision that names the record; +- the history keeps a finding that left the VDR for a year, with a final + disposition only from evidence, and carries closed decisions. """ import importlib.util import json @@ -137,7 +142,8 @@ def test_periods_run_from_the_previous_report(): assert vdt["reportPeriod"] == {"from": "2026-11-04T15:00:00+00:00", "to": NOW.isoformat()} assert avi["reportPeriod"]["from"] == "2026-11-04T15:00:00+00:00" first, _, _ = _build() - assert first["reportPeriod"]["from"] == "2026-03-01T00:00:00+00:00" # earliest detection + # The earliest detection: the suppressed CKV_AWS_68, known since DEC-104 was decided. + assert first["reportPeriod"]["from"] == "2026-01-01T00:00:00+00:00" def test_a_previous_report_ending_in_the_future_stops_the_build(): @@ -146,16 +152,24 @@ def test_a_previous_report_ending_in_the_future_stops_the_build(): _build(prev_vdt=prev) +def _detection_ok(*feeds, checked_at="2026-11-06T14:59:00+00:00"): + return {"schema": "detection/1", "checked_at": checked_at, "result": "ok", + "feeds": [{"feed": f, "file": f"{f}.json", "status": "ok", "count": 0} for f in feeds]} + + def test_history_keeps_a_finding_that_left_the_vdr_then_drops_it_after_a_year(): _, _, mrh = _build() vdr = json.loads(json.dumps(VDR)) vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "grype-CVE-2099-0001-thumbnailer"] + # Grype ran in the build that no longer reports it: remediated, from evidence. + vdr["methodology"] = {"detection": _detection_ok("grype", "checkov")} _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer") assert gone["finalDisposition"] == "Remediated" + assert gone["providerExtensions"]["dispositionBasis"].startswith("grype ran in the VDR build checked") assert gone["providerExtensions"]["noLongerReportedAt"] == (NOW + timedelta(days=1)).isoformat() - assert "projectedNextReduction" not in gone and gone["overdueStatus"] == {"isOverdue": False} + assert "projectedNextReduction" not in gone and gone["overdueStatus"]["isOverdue"] is False assert fs.validate(later, "mrh") == [] # The time it left is kept, not reset, on later runs; after a year it is dropped. _, _, still = _build(vdr=vdr, prev_mrh=later, now=NOW + timedelta(days=200)) @@ -235,3 +249,288 @@ def test_an_unreadable_previous_report_stops_the_build_rather_than_dropping_hist "--out-dir", str(tmp_path)], capture_output=True, text=True) assert r.returncode != 0 and "unreadable" in r.stderr assert not list(tmp_path.glob("vdr-*")) + + +# --- what each record says (VER-RPT-VDT, VER-RPT-AVI) ------------------------------- + +def _register(edit): + """The fixture register with `edit(records_by_id)` applied, validated like the real one.""" + doc = json.loads((FIX / "decisions.json").read_text()) + edit({r["id"]: r for r in doc["decisions"]}) + return _decisions.validate(doc) + + +def _by_id(doc, accepted=False): + rows = doc["acceptedVulnerabilities"] if accepted else doc["vulnerabilities"] + return {(r["vulnerabilityDetail"] if accepted else r)["providerTrackingId"]: + (r["vulnerabilityDetail"] if accepted else r) for r in rows} + + +def test_records_carry_their_historical_rating_and_rating_history(): + vdr = json.loads(json.dumps(VDR)) + f = vdr["findings"][0] + f["historical_pain"] = "N4" + f["rating_history"] = [ + {"at": "2026-08-06T14:00:00+00:00", "pain": "N4", "lev": True, "irv": True, "reason": "first evaluation"}, + {"at": "2026-10-14T10:00:00+00:00", "pain": "N3", "lev": True, "irv": True, "reason": "re-evaluation: pain N4->N3"}] + vdt, _, _ = _build(vdr=vdr) + ext = _by_id(vdt)[f["tracking_id"]]["providerExtensions"] + assert ext["historicalRating"] == 4 + assert [(h["at"], h["rating"], h["reason"]) for h in ext["ratingHistory"]] == [ + ("2026-08-06T14:00:00+00:00", 4, "first evaluation"), + ("2026-10-14T10:00:00+00:00", 3, "re-evaluation: pain N4->N3")] + assert fs.validate(vdt, "vdt") == [] + + +def test_a_decision_sets_the_next_reduction_and_the_deadline_is_only_the_fallback(): + vdr = json.loads(json.dumps(VDR)) + vdr["findings"][0]["decision_id"] = "DEC-105" # open N3 finding, deadline 2026-11-20 + + def plan(target): + def edit(recs): + recs["DEC-105"]["planned_reduction"] = {"date": "2026-11-12", "target_pain": target} + return _register(edit) + vdt, _, _ = mod.build(vdr, plan("N1"), None, None, None, NOW) + rec = _by_id(vdt)["grype-CVE-2099-0001-thumbnailer"] + assert rec["projectedNextReduction"] == {"estimatedAt": "2026-11-12T00:00:00+00:00", "targetRating": 1} + assert rec["providerExtensions"]["projectionBasis"] == "DEC-105 planned_reduction" + # A plan that reduces nothing is not a reduction: the deadline stands, and says why. + vdt, _, _ = mod.build(vdr, plan("N3"), None, None, None, NOW) + rec = _by_id(vdt)["grype-CVE-2099-0001-thumbnailer"] + assert rec["projectedNextReduction"] == {"estimatedAt": "2026-11-20T00:00:00+00:00", "targetRating": 0} + assert "not below the current rating" in rec["providerExtensions"]["projectionBasis"] + # No plan: the remediation deadline. + vdt, _, _ = _build() + assert _by_id(vdt)["grype-CVE-2099-0001-thumbnailer"]["providerExtensions"]["projectionBasis"] == \ + "remediation deadline (no planned reduction recorded)" + + +def test_supplementary_information_comes_from_the_decision(): + vdt, avi, _ = _build() + fp = _by_id(vdt)["zap-10099-article"]["supplementaryRiskInformation"] + assert fp.startswith("DEC-103 (false-positive; decided 2026-01-01, review by 2027-01-01): " + "The rule matches article prose, not code.") + # The AVI carries the justification as the rationale; the supplementary + # information is the rest of the decision, not a copy of it. + acc = _by_id(avi, accepted=True)["CKV_AWS_68"]["supplementaryRiskInformation"] + assert acc.startswith("DEC-104 (risk-accepted; decided 2026-01-01, review by 2027-01-01).") + assert "Cost trade-off" not in acc + assert all(a["vulnerabilityDetail"].get("supplementaryRiskInformation") for a in avi["acceptedVulnerabilities"] + if a["vulnerabilityDetail"]["providerExtensions"]["decisionId"]) + + +def test_overdue_explanations_explain_the_overdue_status_not_the_suppression(): + vdr = json.loads(json.dumps(VDR)) + vdr["findings"][1]["remediation_due_at"] = "2026-09-23T13:51:28+00:00" # overdue, DEC-101 accepts it + vdt, avi, _ = _build(vdr=vdr) + every = list(_by_id(vdt).values()) + list(_by_id(avi, accepted=True).values()) + assert not any("documented suppression" in r["overdueStatus"].get("explanation", "") for r in every) + over = _by_id(avi, accepted=True)["checkov-CKV_AWS_310-website"]["overdueStatus"] + assert over == {"isOverdue": True, "explanation": ( + "Overdue: its remediation deadline, 2026-09-23, passed without remediation. It is still open because " + "DEC-101 accepts it (risk-accepted, decided 2026-01-01), so no remediation is planned.")} + assert _by_id(vdt)["zap-10099-article"]["overdueStatus"] == { + "isOverdue": False, "explanation": "Not overdue: closed as False Positive under DEC-103."} + mav = _by_id(avi, accepted=True)["dependabot-GHSA-xxxx-yyyy-zzzz"]["overdueStatus"] + assert mav["isOverdue"] and mav["explanation"].startswith("Overdue: not fully mitigated or remediated within") + + +def test_detection_time_comes_from_a_decision_that_names_the_record(): + def edit(recs): + # The ZAP finding was found in a manual scan before the pipeline's scanner saw it. + recs["DEC-103"]["applies_to"] = {"check_ids": [], "vuln_ids": ["zap-10099-article"], "resources": []} + recs["DEC-103"]["original_detection_date"] = "2026-07-01" + # A decision about a check across resources dates the check, not each resource. + recs["DEC-101"]["original_detection_date"] = "2026-02-01" + vdt, avi, _ = mod.build(VDR, _register(edit), None, None, None, NOW) + zap = _by_id(vdt)["zap-10099-article"] + assert zap["detection"]["detectedAt"] == "2026-07-01T00:00:00+00:00" + assert zap["providerExtensions"]["detectedAtBasis"].startswith("DEC-103 original_detection_date") + assert zap["providerExtensions"]["firstReportedAt"] == "2026-08-06T13:51:28+00:00" + acc = _by_id(avi, accepted=True) + assert acc["checkov-CKV_AWS_310-website"]["detection"]["detectedAt"] == "2026-08-06T13:51:28+00:00" + # A suppressed check is never reported by a scanner: its decision dates it. + assert acc["CKV_AWS_68"]["detection"]["detectedAt"] == "2026-01-01T00:00:00+00:00" + assert acc["CKV_AWS_68"]["providerExtensions"]["detectedAtBasis"].startswith("DEC-104 decided_on") + + +def test_a_suppression_names_the_resources_its_decision_affects(): + vdr = json.loads(json.dumps(VDR)) + vdr["risk_accepted"][0].update(affected_resources=["aws-cloudfront-distribution"], + component_id="aws::cdn_distribution::website", irv_basis="inventory") + _, avi, _ = _build(vdr=vdr) + ext = _by_id(avi, accepted=True)["CKV_AWS_68"]["providerExtensions"] + assert ext["affectedResources"] == ["aws-cloudfront-distribution"] + assert ext["componentId"] == "aws::cdn_distribution::website" + + +# --- history: a final disposition only from evidence (VER-TFR-MRH) ------------------- + +def _gone(vdr_edit, prev=None, now=NOW + timedelta(days=1), register=None): + """Build, drop the open grype finding, apply `vdr_edit`, rebuild with the first history.""" + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "grype-CVE-2099-0001-thumbnailer"] + vdr_edit(vdr) + _, _, later = mod.build(vdr, register or REGISTER, None, None, prev or mrh, now) + assert fs.validate(later, "mrh") == [] + return vdr, later, next(r for r in later["activeVulnerabilities"] + if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer") + + +def test_a_record_that_vanished_without_evidence_has_no_final_disposition(): + vdr, later, gone = _gone(lambda v: None) # no detection summary: coverage not shown + assert "finalDisposition" not in gone + assert gone["providerExtensions"]["noLongerReportedReason"] == "unknown" + assert gone["providerExtensions"]["noLongerReportedBasis"].startswith("no evidence of why") + # A failed scan is no evidence either. + _, _, failed = _gone(lambda v: v.update(methodology={"detection": { + **_detection_ok("grype"), "feeds": [{"feed": "grype", "status": "failed"}]}})) + assert "finalDisposition" not in failed + # A closed decision that arrives later settles it. + def closed(recs): + recs["DEC-105"].update(status="closed", disposition="remediated", closed_on="2026-11-07", + applies_to={"check_ids": [], "vuln_ids": ["CVE-2099-0001"], "resources": []}) + _, _, settled = _gone(lambda v: None, prev=later, now=NOW + timedelta(days=3), register=_register(closed)) + assert settled["finalDisposition"] == "Remediated" + assert settled["providerExtensions"]["dispositionBasis"] == "DEC-105 closed as remediated on 2026-11-07" + assert settled["providerExtensions"]["noLongerReportedAt"] == gone["providerExtensions"]["noLongerReportedAt"] + + +def test_an_unsupported_remediated_from_an_earlier_history_is_withdrawn(): + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "grype-CVE-2099-0001-thumbnailer"] + old = json.loads(json.dumps(mrh)) + for r in old["activeVulnerabilities"]: + if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer": + r["finalDisposition"] = "Remediated" # what the earlier builder wrote, with no evidence + r["providerExtensions"]["noLongerReportedAt"] = NOW.isoformat() + _, _, later = _build(vdr=vdr, prev_mrh=old, now=NOW + timedelta(days=1)) + gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "grype-CVE-2099-0001-thumbnailer") + assert "finalDisposition" not in gone and gone["providerExtensions"]["noLongerReportedReason"] == "unknown" + + +def test_a_check_that_became_a_suppression_is_not_remediated(): + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "checkov-CKV_AWS_20-site"] + vdr["methodology"] = {"detection": _detection_ok("checkov")} + vdr["risk_accepted"].append({**vdr["risk_accepted"][0], "tracking_id": "CKV_AWS_20", "tool_id": "CKV_AWS_20", + "decision_id": "DEC-102"}) + _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) + gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "checkov-CKV_AWS_20-site") + assert "finalDisposition" not in gone + assert gone["providerExtensions"]["noLongerReportedReason"] == "suppressed" + # A false-positive suppression is a final disposition, from the decision. + vdr["risk_accepted"].pop() + vdr["false_positives"] = [{"tracking_id": "CKV_AWS_20", "tool_id": "CKV_AWS_20", "decision_id": "DEC-103", + "current_disposition": "false-positive"}] + _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) + gone = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "checkov-CKV_AWS_20-site") + assert gone["finalDisposition"] == "False Positive" + + +def test_a_vanished_zap_finding_is_remediated_only_after_a_later_scan(): + def dast(when): + return {"methodology": {"dast": {"scanner": "zap", "status": "ok", "generated_at": when}}} + last = NOW.isoformat() + assert mod._coverage("zap", dast("2026-11-04T00:00:00+00:00"), last) is None # the scan that reported it + assert mod._coverage("zap", dast("2026-11-06T00:00:00+00:00"), last).startswith("the ZAP scan of 2026-11-06") + assert mod._coverage("zap", {"methodology": {}}, last) is None + # A record that was a false positive when last reported keeps that disposition. + _, _, mrh = _build() + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["tracking_id"] != "zap-10099-article"] + _, _, later = _build(vdr=vdr, prev_mrh=mrh, now=NOW + timedelta(days=1)) + fp = next(r for r in later["activeVulnerabilities"] if r["providerTrackingId"] == "zap-10099-article") + assert fp["finalDisposition"] == "False Positive" + + +def test_closed_decisions_are_backfilled_into_the_history_once(): + def edit(recs): + recs["DEC-105"].update(status="closed", disposition="remediated", closed_on="2026-06-18", + original_detection_date="2026-05-06", former_id="POAM-005", + source={"detector": "Manual review", "original_risk_rating": "moderate"}) + reg = _register(edit) + vdr = json.loads(json.dumps(VDR)) + vdr["findings"] = [f for f in vdr["findings"] if f["decision_id"] != "DEC-105"] + _, _, mrh = mod.build(vdr, reg, None, None, None, NOW) + back = [r for r in mrh["activeVulnerabilities"] if r["providerTrackingId"] == "DEC-105"] + assert len(back) == 1 + assert back[0]["finalDisposition"] == "Remediated" + assert back[0]["detection"] == {"detectedAt": "2026-05-06T00:00:00+00:00", "detectionSource": "Manual review"} + assert back[0]["providerExtensions"]["backfilledFromDecision"] is True + assert back[0]["providerExtensions"]["formerId"] == "POAM-005" + assert fs.validate(mrh, "mrh") == [] + # Carried forward: rebuilt from the register, never duplicated; dropped a year after closing. + _, _, again = mod.build(vdr, reg, None, None, mrh, NOW + timedelta(days=1)) + assert [r["providerTrackingId"] for r in again["activeVulnerabilities"]].count("DEC-105") == 1 + _, _, aged = mod.build(vdr, reg, None, None, again, datetime(2027, 6, 20, tzinfo=timezone.utc)) + assert "DEC-105" not in _ids(aged["activeVulnerabilities"]) + + +# --- the period's activity (VER-RPT-PER) --------------------------------------------- + +def _with_activity(): + vdr = json.loads(json.dumps(VDR)) + vdr["emitted_at"] = NOW.isoformat() + vdr["findings"][0].update(first_detected="2026-11-05T09:00:00+00:00", first_evaluated="2026-11-05T09:00:00+00:00", + completed_reductions=[{"at": "2026-11-05T10:00:00+00:00", "from": "N4", "to": "N3"}]) + vdr["methodology"] = { + "detection": {**_detection_ok("grype", "checkov", checked_at="2026-11-05T14:59:00+00:00"), + "feeds": [{"feed": "grype", "file": "grype-js.json", "status": "ok", "count": 1, + "source_date": "2026-11-05T06:00:00+00:00"}, + {"feed": "kev", "file": "kev-catalog.json", "status": "ok", "count": 9, + "source_date": "2026-11-04T20:00:00+00:00"}, + {"feed": "previous-vdr", "file": "previous-vdr-report.json", "status": "ok"}]}, + "dast": {"scanner": "zap", "status": "ok", "generated_at": "2026-11-05T12:47:20+00:00", "alerts": 4}, + "runtime_verification": {"uri": "https://samaydlette.com/.well-known/ksi-signal-runtime.json", + "verified": True, "fresh": True, "emitted_at": "2026-11-05T11:00:00Z", + "signal_id": "rt-1", "validations": 5, "failed": 1, "divergence": "converged"}} + return vdr + + +def test_each_period_report_summarizes_the_periods_activity(): + prev = {"reportPeriod": {"from": "2026-11-04T15:00:00+00:00", "to": "2026-11-05T03:00:00+00:00"}} + vdt, avi, _ = _build(vdr=_with_activity(), prev_vdt=prev, prev_avi=prev) + for doc in (vdt, avi): + s = doc["providerExtensions"]["activitySummary"] + assert s["period"] == doc["reportPeriod"] + assert s["detections"] == [{"providerTrackingId": "grype-CVE-2099-0001-thumbnailer", + "at": "2026-11-05T09:00:00+00:00"}] + assert s["reductions"] == [{"providerTrackingId": "grype-CVE-2099-0001-thumbnailer", + "at": "2026-11-05T10:00:00+00:00", "from": "N4", "to": "N3"}] + scans = {x["activity"]: x for x in s["scans"]} + assert scans["Grype software composition scan (grype-js.json)"]["at"] == "2026-11-05T14:59:00+00:00" + assert scans["Grype software composition scan (grype-js.json)"]["vulnerabilityDatabaseBuiltAt"] == \ + "2026-11-05T06:00:00+00:00" + assert scans["CISA KEV catalog (kev-catalog.json)"]["inPeriod"] is False # published before the period + assert scans["OWASP ZAP dynamic scan (weekly)"]["inPeriod"] is True + assert not any("previous-vdr" in k for k in scans) + rt = s["runtimeVerification"] + assert rt["uri"] == "https://samaydlette.com/.well-known/ksi-signal-runtime.json" and rt["signalId"] == "rt-1" + assert s["counts"]["runtimeRunsInPeriod"] == 1 and s["counts"]["runtimeFailuresInPeriod"] == 1 + assert s["counts"]["detections"] == 1 and s["counts"]["reductions"] == 1 + assert fs.validate(vdt, "vdt") == [] and fs.validate(avi, "avi") == [] + + +def test_an_unverified_runtime_run_is_linked_but_not_counted(): + vdr = _with_activity() + vdr["methodology"]["runtime_verification"].update(verified=False, validations=None, failed=None, + reason="runtime signal signature does not verify") + vdt, _, _ = _build(vdr=vdr) + s = vdt["providerExtensions"]["activitySummary"] + assert s["counts"]["runtimeRunsInPeriod"] == 0 and "not verified" in s["runtimeVerification"]["note"] + + +def test_the_markdown_reports_carry_the_activity_and_link_the_runtime_results(tmp_path): + vdr = tmp_path / "vdr.json" + vdr.write_text(json.dumps(_with_activity())) + r = subprocess.run([sys.executable, str(SCRIPT), "--vdr", str(vdr), "--decisions", str(FIX / "decisions.json"), + "--out-dir", str(tmp_path)], capture_output=True, text=True) + assert r.returncode == 0, r.stderr + for name in ("vdr-detail-report.md", "vdr-accepted-vulnerabilities.md"): + text = (tmp_path / name).read_text() + assert "## Activity this period" in text and "| OWASP ZAP dynamic scan (weekly) |" in text + assert "(https://samaydlette.com/.well-known/ksi-signal-runtime.json)" in text diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 5746cbad..0a8e9156 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -35,6 +35,8 @@ ("duplicate-claim.json", "'CKV_AWS_68' is claimed by both DEC-001 and DEC-002"), ("closed-not-remediated.json", "a closed record's disposition must be one of"), ("unknown-field.json", "unknown field(s) ['reviewby']"), + ("planned-reduction-malformed.json", "planned_reduction must be {date: YYYY-MM-DD, target_pain: N0-N5}"), + ("planned-reduction-closed.json", "only an in-force record plans a reduction"), ] diff --git a/tests/test_vdr_component.py b/tests/test_vdr_component.py index 11124867..a34e31aa 100644 --- a/tests/test_vdr_component.py +++ b/tests/test_vdr_component.py @@ -49,3 +49,89 @@ def test_file_level_findings_stay_unattributed(): assert resolve(".checkov.yaml") is None assert resolve("infrastructure/logging.tf") is None assert resolve("aws_s3_bucket.nonexistent") is None + + +# --- a .checkov.yaml suppression affects the resources its decision names -------- + +INV_SIGNAL = {"components": [ + {"component_id": "aws::cdn_distribution::website", "type": "cdn_distribution", + "attributes": {"tf_type": "aws_cloudfront_distribution", "tf_name": "website", + "classification": {"internet_reachable": True}}}, + {"component_id": "aws::function::opa_compliance", "type": "function", + "attributes": {"tf_type": "aws_lambda_function", "tf_name": "opa_compliance", + "classification": {"internet_reachable": False}}}, + {"component_id": "aws::function::silk_reeling", "type": "function", + "attributes": {"tf_type": "aws_lambda_function", "tf_name": "silk_reeling", + "classification": {"internet_reachable": True}}}, +]} + + +def _inventory(tmp_path): + p = tmp_path / "ksi-signal.json" + p.write_text(json.dumps(INV_SIGNAL)) + return vdr.load_inventory(p) + + +def test_a_decision_resource_resolves_only_when_it_names_one_component(tmp_path): + inv = _inventory(tmp_path) + # The register's aws-[::] form, matched to the one component of that type. + assert vdr.resolve_decision_resource("aws-cloudfront-distribution", inv) == "aws::cdn_distribution::website" + assert vdr.resolve_decision_resource("aws-lambda-function::silk-reeling", inv) == "aws::function::silk_reeling" + # An exact inventory identifier. + assert vdr.resolve_decision_resource("aws::function::opa_compliance", inv) == "aws::function::opa_compliance" + # Two Lambdas and no name, or a name no component carries: never a guess. + assert vdr.resolve_decision_resource("aws-lambda-function", inv) is None + assert vdr.resolve_decision_resource("aws-lambda::compliance-monitor", inv) is None + assert vdr.resolve_decision_resource(".checkov.yaml", inv) is None + # Several components: the internet-reachable one, the conservative choice for IRV. + assert vdr.resolve_decision_resources( + ["aws::function::opa_compliance", "aws-lambda-function::silk-reeling"], inv) == "aws::function::silk_reeling" + + +def test_a_suppression_carries_its_decisions_resources(): + supp = {s["tool_id"]: s for s in vdr.ingest_suppressions(str(REPO / ".checkov.yaml"))} + reg = {r["id"]: r for r in json.loads((REPO / "data" / "decisions.json").read_text())["decisions"]} + for check, s in supp.items(): + assert s["resource"] == ".checkov.yaml" + assert s["affected_resources"] == reg[s["decision_id"]]["applies_to"]["resources"], check + + +def test_a_suppressions_reachability_is_decided_from_the_resource_it_affects(tmp_path): + inv = _inventory(tmp_path) + sup = {"tracking_id": "CKV_AWS_68", "decision_id": "DEC-007", "source": "checkov-suppression", + "tool_id": "CKV_AWS_68", "title": "CKV_AWS_68 suppressed", "resource": ".checkov.yaml", + "affected_resources": ["aws-cloudfront-distribution"], "current_disposition": "risk-accepted", + "pain": "N2", "is_kev": False, "explanation": "x"} + unresolved = {**sup, "tracking_id": "CKV_AWS_117", "tool_id": "CKV_AWS_117", + "affected_resources": ["aws-lambda::compliance-monitor"]} + report, _ = vdr.build_report([], [sup, unresolved], set(), {}, inventory=inv) + by = {r["tracking_id"]: r for r in report["risk_accepted"]} + assert by["CKV_AWS_68"]["component_id"] == "aws::cdn_distribution::website" + assert by["CKV_AWS_68"]["irv_basis"] == "inventory" and by["CKV_AWS_68"]["internet_reachable"] is True + assert by["CKV_AWS_68"]["affected_resources"] == ["aws-cloudfront-distribution"] + # A resource that resolves to nothing stays unresolved, and fails safe. + assert by["CKV_AWS_117"]["component_id"] is None and by["CKV_AWS_117"]["irv_basis"] == "unresolved" + + +# --- the runs a VDR build read, for the period reports (VER-RPT-PER) ----------------- + +def test_runtime_results_are_counted_only_from_a_verified_signal(): + ok = {"verified": True, "fresh": True, "emitted_at": "2026-11-05T17:56:05Z", "signal_id": "s-1", + "divergence": "converged", "validations": [{"result": "pass"}, {"result": "fail"}, {"result": "pass"}]} + out = vdr.runtime_verification_summary(ok) + assert out["uri"].endswith("/.well-known/ksi-signal-runtime.json") + assert (out["validations"], out["failed"], out["divergence"], out["signal_id"]) == (3, 1, "converged", "s-1") + bad = vdr.runtime_verification_summary({"verified": False, "reason": "runtime signal signature does not verify", + "validations": [{"result": "pass"}]}) + assert bad["verified"] is False and bad["validations"] is None and bad["failed"] is None + assert "does not verify" in bad["reason"] + + +def test_the_dast_scan_time_is_the_reports_own(tmp_path): + zap = tmp_path / "zap.json" + zap.write_text(json.dumps({"@generated": "Thu, 5 Nov 2026 12:47:20", "created": "2026-11-05T12:47:20.585268837Z", + "site": [{"alerts": [{}, {}]}, {"alerts": [{}]}]})) + assert vdr.dast_summary(str(zap)) == {"scanner": "zap", "status": "ok", + "generated_at": "2026-11-05T12:47:20.585268+00:00", "alerts": 3} + assert vdr.dast_summary(str(tmp_path / "absent.json"))["status"] == "missing" + assert vdr.dast_summary(None) is None From be1c7bd0c1b630fac68ac4202ed0078634117d4e Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:39:50 -0500 Subject: [PATCH 21/97] Decision register: DEC-010 scoped to the Lambdas it covers; detection dates for DEC-007/009/010 from git history The CKV_AWS_117 skip is global, so DEC-010 covers every Lambda function; it named a resource no inventory component carries. The inline skips for CKV_AWS_68, CKV_AWS_86/310 and CKV_AWS_117 were added on 2025-06-14 (commit f453e9d), so each was detected by then. AI-drafted for the operator to confirm. Co-Authored-By: Claude Opus 5.5 --- data/decisions.json | 13 +++++++++---- tests/test_decisions.py | 27 +++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/data/decisions.json b/data/decisions.json index 1b4d402f..3d3584d4 100644 --- a/data/decisions.json +++ b/data/decisions.json @@ -207,7 +207,8 @@ "original_risk_rating": "moderate", "adjusted_risk_rating": "low", "risk_adjustment": true - } + }, + "original_detection_date": "2025-06-14" }, { "id": "DEC-008", @@ -268,7 +269,8 @@ "identifier": "CKV_AWS_86; CKV_AWS_310", "original_risk_rating": "low", "risk_adjustment": false - } + }, + "original_detection_date": "2025-06-14" }, { "id": "DEC-010", @@ -286,7 +288,9 @@ ], "vuln_ids": [], "resources": [ - "aws-lambda::compliance-monitor" + "aws::function::opa_compliance", + "aws::function::evidence_watchdog", + "aws::function::silk_reeling" ] }, "decided_by": "Sam Aydlette", @@ -297,7 +301,8 @@ "identifier": "CKV_AWS_117", "original_risk_rating": "low", "risk_adjustment": false - } + }, + "original_detection_date": "2025-06-14" }, { "id": "DEC-011", diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 0a8e9156..6c7aa324 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -138,8 +138,31 @@ def test_every_poam_item_landed_in_the_register_with_its_decision(): assert r["status"] == ("closed" if i["category"] == "closed" else "in-force"), i["id"] assert r["title"] == i["title"] and r["justification"] == i["description"], i["id"] for k in ("decided_by", "decided_on", "review_by", "original_detection_date"): - assert r.get(k) == i.get(k), (i["id"], k) - assert r["applies_to"]["resources"] == i.get("asset_identifiers", []), i["id"] + if (i["id"], k) not in AMENDED_SINCE_MIGRATION: + assert r.get(k) == i.get(k), (i["id"], k) + if (i["id"], "resources") not in AMENDED_SINCE_MIGRATION: + assert r["applies_to"]["resources"] == i.get("asset_identifiers", []), i["id"] + + +# Fields corrected after the migration, each for a stated reason; everything +# else still matches the POA&M item it came from. +AMENDED_SINCE_MIGRATION = { + # Detection dates the POA&M never recorded: the inline Checkov skips for + # CKV_AWS_68, CKV_AWS_86/310 and CKV_AWS_117 were added on 2025-06-14 + # (commit f453e9d, "fixing Checkov findings"), so each was detected by then. + ("POAM-007", "original_detection_date"), ("POAM-009", "original_detection_date"), + ("POAM-010", "original_detection_date"), + # DEC-010 named "aws-lambda::compliance-monitor", which no inventory + # component carries; the CKV_AWS_117 skip is global, so it covers every Lambda. + ("POAM-010", "resources"), +} + + +def test_the_post_migration_amendments_are_the_stated_ones(): + recs = _decisions.records(_decisions.load()) + assert all(recs[d]["original_detection_date"] == "2025-06-14" for d in ("DEC-007", "DEC-009", "DEC-010")) + assert recs["DEC-010"]["applies_to"]["resources"] == [ + "aws::function::opa_compliance", "aws::function::evidence_watchdog", "aws::function::silk_reeling"] def test_the_check_maps_are_the_ones_the_poam_held(): From bb5a9fb14678a161ab36289859a90606b95755d9 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:39:50 -0500 Subject: [PATCH 22/97] KEV: a gate before apply, and the catalog's due dates in the VDR and VDT (VDR-CSO-AKE, VDR-TFR-KEV) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 10 ++ scripts/build-cr26-vdr.py | 5 + scripts/build-vdr-report.py | 44 ++++++++- scripts/kev-gate.py | 125 ++++++++++++++++++++++++ tests/fixtures/kev/dependabot-kev.json | 23 +++++ tests/fixtures/kev/dependabot-none.json | 1 + tests/fixtures/kev/grype-clean.json | 13 +++ tests/fixtures/kev/grype-kev.json | 19 ++++ tests/fixtures/kev/kev-catalog.json | 18 ++++ tests/test_kev.py | 106 ++++++++++++++++++++ 10 files changed, 359 insertions(+), 5 deletions(-) create mode 100644 scripts/kev-gate.py create mode 100644 tests/fixtures/kev/dependabot-kev.json create mode 100644 tests/fixtures/kev/dependabot-none.json create mode 100644 tests/fixtures/kev/grype-clean.json create mode 100644 tests/fixtures/kev/grype-kev.json create mode 100644 tests/fixtures/kev/kev-catalog.json create mode 100644 tests/test_kev.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index a327c2de..91d4a951 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1174,6 +1174,16 @@ jobs: --checkov checkov-results.sarif --tfsec tfsec-results.json \ --output detection.json + # VDR-CSO-AKE: nothing with a known exploited vulnerability is activated. + # Runs before the plan, the apply and the website sync (G1 in the VDR build + # runs after them). + - name: KEV gate before apply (VDR-CSO-AKE) + working-directory: infrastructure + run: | + python3 ../scripts/kev-gate.py --kev kev-catalog.json \ + --grype grype-python.json grype-js.json grype-lambda.json grype-a11y.json \ + --dependabot dependabot-alerts.json + - name: Generate fresh Terraform plan working-directory: infrastructure run: terraform plan -lock-timeout=300s -out=tfplan diff --git a/scripts/build-cr26-vdr.py b/scripts/build-cr26-vdr.py index 3a322e47..23019fcd 100644 --- a/scripts/build-cr26-vdr.py +++ b/scripts/build-cr26-vdr.py @@ -327,6 +327,11 @@ def detail(f, recs=None, accepted=False): "detectedAtBasis": detected_basis, "firstReportedAt": _ts(f.get("first_detected"), f"{tid}.first_detected"), } + if f.get("is_kev"): + # VDR-TFR-KEV: the CISA catalog's remediation date and whether it passed. + ext["kevDueAt"] = f.get("kev_due_at") + ext["kevDateAdded"] = f.get("kev_date_added") + ext["kevOverdue"] = bool(f.get("kev_overdue")) if f.get("affected_resources"): ext["affectedResources"] = list(f["affected_resources"]) if projection_basis: diff --git a/scripts/build-vdr-report.py b/scripts/build-vdr-report.py index 8cc2f592..ed38db60 100755 --- a/scripts/build-vdr-report.py +++ b/scripts/build-vdr-report.py @@ -1098,15 +1098,38 @@ def zap_report_age_days(path, now): return None +class KevCatalog(set): + """The CISA KEV catalog's CVE IDs, plus each entry's dates (VDR-TFR-KEV): + `entries` is {cveID: {"due_date", "date_added"}} from the catalog.""" + + def __init__(self, ids=(), entries=None): + super().__init__(ids) + self.entries = entries or {} + + def ingest_kev(path): - """Load CISA KEV catalog and return the set of CVE IDs.""" + """Load the CISA KEV catalog: its CVE IDs, with the catalog's dueDate and + dateAdded for each (BOD 26-04 remediation dates).""" if not path or not Path(path).exists(): - return set() + return KevCatalog() try: doc = json.loads(Path(path).read_text()) except json.JSONDecodeError: - return set() - return {v.get("cveID") for v in (doc.get("vulnerabilities") or []) if v.get("cveID")} + return KevCatalog() + entries = {v["cveID"]: {"due_date": v.get("dueDate"), "date_added": v.get("dateAdded")} + for v in (doc.get("vulnerabilities") or []) if isinstance(v, dict) and v.get("cveID")} + return KevCatalog(entries, entries) + + +def kev_dates(kev_cves, cve, now): + """(due_at, date_added, overdue) for a KEV finding from the catalog, or + (None, None, False). The catalog's due date is a day; it is read as the + start of that day, UTC, the earlier reading.""" + entry = (getattr(kev_cves, "entries", None) or {}).get(cve) if cve else None + due = _ts(f"{entry['due_date']}T00:00:00+00:00", None) if entry and entry.get("due_date") else None + if due is None: + return None, (entry or {}).get("date_added"), False + return due.isoformat(), entry.get("date_added"), now >= due class EpssScores(dict): @@ -2093,8 +2116,17 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No # VER-RPT-VDT per-vulnerability record fields. History and reductions come # from the ledger (REQ-PAI-2); the deadline runs whatever the disposition. is_overdue = d["is_overdue"] + # VDR-TFR-KEV: a known exploited vulnerability is remediated by the + # catalog's due date, even when fully mitigated: the earlier of that and + # the Class C deadline, and overdue once the catalog's date has passed. + kev_due_at, kev_added, kev_overdue = kev_dates(kev_cves, f.get("cve"), now) if is_kev else (None, None, False) + if kev_due_at and due_at: + due_at = min(due_at, kev_due_at, key=lambda x: _ts(x, now)) + elif kev_due_at and needs_remediation: + due_at = kev_due_at + is_overdue = is_overdue or kev_overdue next_reduction = ({"estimated_time": due_at, "target_pain": "remediated"} - if sla is not None else None) + if sla is not None or kev_due_at else None) overdue_explanation = (block_reason if block_this else f"within Class C SLA: {days_since_detected}d of {sla}d allowed" if sla is not None @@ -2133,6 +2165,8 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "vector_source": f.get("vector_source") or ("scanner" if f.get("cvss_vector") else None), "technical_impact": technical_impact(f), "is_kev": is_kev, + **({"kev_due_at": kev_due_at, "kev_date_added": kev_added, "kev_overdue": kev_overdue} + if is_kev and (kev_due_at or kev_added) else {}), "completed_reductions": hist["completed_reductions"], "next_reduction": next_reduction if needs_remediation else None, "current_disposition": d["label"] or "open", diff --git a/scripts/kev-gate.py b/scripts/kev-gate.py new file mode 100644 index 00000000..f70a32c7 --- /dev/null +++ b/scripts/kev-gate.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""KEV gate before activation (VDR-CSO-AKE). + +Providers SHOULD NOT deploy or activate new machine-based resources with Known +Exploited Vulnerabilities. The VDR build's G1 blocks a KEV finding, but in the +deploy it runs after `terraform apply` and the website sync: a Lambda package or +SPA bundle with a KEV dependency would already be live when it fails. This gate +runs before the plan and apply, over the same scanner output the VDR reads +(Grype over every dependency tree, open Dependabot alerts), and stops the deploy +when any matched CVE is in the CISA catalog. It names each KEV with the +catalog's due date (VDR-TFR-KEV). + +It follows the governed mode of G1's check, `vdr.kev`, in +infrastructure/schemas/gate-modes.json: in block mode (the decided mode) a KEV +stops the deploy, dispositioned or not; a detect-and-recover mode would record +an event instead. A missing or unreadable input stops it: no catalog or no scan +output cannot read as "no KEV". (The detection gate, G7, has already checked +that each feed is present and fresh.) + + kev-gate.py --kev kev-catalog.json --grype grype-*.json --dependabot dependabot-alerts.json +""" + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _gates # noqa: E402 + +CHECK = "vdr.kev" + + +def _fail(msg): + raise SystemExit(f"::error::KEV gate (VDR-CSO-AKE): {msg} (fail-closed)") + + +def _load(path, what): + p = Path(path) + if not p.is_file(): + _fail(f"{what} {path} not found") + try: + return json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + _fail(f"{what} {path} is not valid JSON: {exc.__class__.__name__}") + + +def load_catalog(path): + """{cveID: {dueDate, dateAdded, name}} from the CISA KEV catalog.""" + doc = _load(path, "KEV catalog") + vulns = doc.get("vulnerabilities") if isinstance(doc, dict) else None + if not isinstance(vulns, list) or not vulns: + _fail(f"KEV catalog {path} has no vulnerabilities[]") + return {v["cveID"]: {"due_date": v.get("dueDate"), "date_added": v.get("dateAdded"), + "name": v.get("vulnerabilityName")} + for v in vulns if isinstance(v, dict) and isinstance(v.get("cveID"), str)} + + +def grype_cves(path): + """[(cve, package@version, source file)] for every Grype match, including a + CVE named only as a related vulnerability of a GHSA match.""" + doc = _load(path, "Grype output") + if not isinstance(doc, dict) or not isinstance(doc.get("matches", []), list): + _fail(f"Grype output {path} has no matches[]") + out = [] + for m in doc.get("matches") or []: + art = m.get("artifact") or {} + where = f"{art.get('name')}@{art.get('version')}" + ids = [(m.get("vulnerability") or {}).get("id")] + ids += [r.get("id") for r in m.get("relatedVulnerabilities") or [] if isinstance(r, dict)] + for i in ids: + if isinstance(i, str) and i.upper().startswith("CVE-"): + out.append((i.upper(), where, Path(path).name)) + return out + + +def dependabot_cves(path): + doc = _load(path, "Dependabot alerts") + if not isinstance(doc, list): + _fail(f"Dependabot alerts {path} is not a list") + out = [] + for a in doc: + adv = (a or {}).get("security_advisory") or {} + pkg = ((a or {}).get("dependency") or {}).get("package") or {} + ids = [adv.get("cve_id")] + [i.get("value") for i in adv.get("identifiers") or [] + if isinstance(i, dict) and i.get("type") == "CVE"] + for i in {x.upper() for x in ids if isinstance(x, str) and x.upper().startswith("CVE-")}: + out.append((i, pkg.get("name") or "?", Path(path).name)) + return out + + +def kev_hits(catalog, matches): + """Sorted unique [(cve, where, file, due_date)] for matches in the catalog.""" + return sorted({(c, w, f, catalog[c]["due_date"]) for c, w, f in matches if c in catalog}) + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument("--kev", required=True) + ap.add_argument("--grype", nargs="+", required=True) + ap.add_argument("--dependabot", required=True) + a = ap.parse_args(argv) + catalog = load_catalog(a.kev) + matches = [m for g in a.grype for m in grype_cves(g)] + dependabot_cves(a.dependabot) + hits = kev_hits(catalog, matches) + gates = _gates.load_gate_modes() + print(f"KEV gate: {len(matches)} CVE match(es) across {len(a.grype)} Grype scan(s) and Dependabot; " + f"{len(catalog)} CVEs in the catalog; {len(hits)} known exploited.") + if not hits: + return 0 + for cve, where, src, due in hits: + line = f"{cve} in {where} ({src}) is in the CISA KEV catalog, due {due or 'no due date'}" + if _gates.blocks(gates, CHECK): + print(f"::error::{line}", file=sys.stderr) + else: + _gates.record_event(gates, CHECK, f"pre-apply: {line}") + if _gates.blocks(gates, CHECK): + print(f"::error::KEV gate: {len(hits)} known exploited vulnerabilit(y/ies) would be deployed; " + "remediate before activation (VDR-CSO-AKE).", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/kev/dependabot-kev.json b/tests/fixtures/kev/dependabot-kev.json new file mode 100644 index 00000000..bed954c4 --- /dev/null +++ b/tests/fixtures/kev/dependabot-kev.json @@ -0,0 +1,23 @@ +[ + { + "_note": "Known-bad: an open Dependabot alert for a KEV CVE.", + "security_advisory": { + "cve_id": "CVE-2099-2222", + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-aaaa" + }, + { + "type": "CVE", + "value": "CVE-2099-2222" + } + ] + }, + "dependency": { + "package": { + "name": "example-lib" + } + } + } +] diff --git a/tests/fixtures/kev/dependabot-none.json b/tests/fixtures/kev/dependabot-none.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/tests/fixtures/kev/dependabot-none.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/kev/grype-clean.json b/tests/fixtures/kev/grype-clean.json new file mode 100644 index 00000000..45482568 --- /dev/null +++ b/tests/fixtures/kev/grype-clean.json @@ -0,0 +1,13 @@ +{ + "matches": [ + { + "vulnerability": { + "id": "CVE-2099-9999" + }, + "artifact": { + "name": "leftpad", + "version": "1.0.0" + } + } + ] +} diff --git a/tests/fixtures/kev/grype-kev.json b/tests/fixtures/kev/grype-kev.json new file mode 100644 index 00000000..54a41279 --- /dev/null +++ b/tests/fixtures/kev/grype-kev.json @@ -0,0 +1,19 @@ +{ + "_note": "Known-bad: a Grype match whose related CVE is in the KEV catalog. The deploy must stop before apply.", + "matches": [ + { + "vulnerability": { + "id": "GHSA-xxxx-yyyy-zzzz" + }, + "relatedVulnerabilities": [ + { + "id": "CVE-2099-1111" + } + ], + "artifact": { + "name": "thumbnailer", + "version": "3.1.0" + } + } + ] +} diff --git a/tests/fixtures/kev/kev-catalog.json b/tests/fixtures/kev/kev-catalog.json new file mode 100644 index 00000000..00ac128f --- /dev/null +++ b/tests/fixtures/kev/kev-catalog.json @@ -0,0 +1,18 @@ +{ + "catalogVersion": "fixture", + "dateReleased": "2026-11-04T20:00:00Z", + "vulnerabilities": [ + { + "cveID": "CVE-2099-1111", + "vulnerabilityName": "Fixture known-exploited flaw", + "dateAdded": "2026-10-01", + "dueDate": "2026-10-22" + }, + { + "cveID": "CVE-2099-2222", + "vulnerabilityName": "Another", + "dateAdded": "2026-10-15", + "dueDate": "2026-11-05" + } + ] +} diff --git a/tests/test_kev.py b/tests/test_kev.py new file mode 100644 index 00000000..cdb3196e --- /dev/null +++ b/tests/test_kev.py @@ -0,0 +1,106 @@ +"""Known exploited vulnerabilities (VDR-CSO-AKE, VDR-TFR-KEV). + +Pins: +- the deploy runs a KEV gate over every Grype scan and the open Dependabot + alerts before the Terraform plan, the apply and the website sync, and it + stops on a KEV (a related CVE of a GHSA match counts) or on a missing input; +- the VDR keeps the catalog's due date for each KEV finding, makes it the + remediation deadline when it is earlier than the Class C one, publishes it + even when the finding needs no remediation, and marks it overdue once passed; +- the VDT carries the due date in providerExtensions. +""" +import importlib.util +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "kev" +GATE = REPO / "scripts" / "kev-gate.py" + +spec = importlib.util.spec_from_file_location("vdr_kev", REPO / "scripts" / "build-vdr-report.py") +vdr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(vdr) + + +def _gate(*args): + env = {k: v for k, v in os.environ.items() if k not in ("GATE_MODES_FILE", "GATE_EVENTS_FILE")} + return subprocess.run([sys.executable, str(GATE), "--kev", str(FIX / "kev-catalog.json"), *args], + capture_output=True, text=True, env=env) + + +def test_a_clean_scan_passes_the_kev_gate(): + r = _gate("--grype", str(FIX / "grype-clean.json"), "--dependabot", str(FIX / "dependabot-none.json")) + assert r.returncode == 0, r.stderr + assert "0 known exploited" in r.stdout + + +def test_a_kev_in_a_grype_scan_stops_the_deploy_with_its_due_date(): + r = _gate("--grype", str(FIX / "grype-clean.json"), str(FIX / "grype-kev.json"), + "--dependabot", str(FIX / "dependabot-none.json")) + assert r.returncode == 1 + assert "CVE-2099-1111 in thumbnailer@3.1.0 (grype-kev.json) is in the CISA KEV catalog, due 2026-10-22" in r.stderr + + +def test_a_kev_in_a_dependabot_alert_stops_the_deploy(): + r = _gate("--grype", str(FIX / "grype-clean.json"), "--dependabot", str(FIX / "dependabot-kev.json")) + assert r.returncode == 1 and "CVE-2099-2222 in example-lib" in r.stderr + + +def test_a_missing_input_stops_the_kev_gate(): + r = _gate("--grype", str(FIX / "absent.json"), "--dependabot", str(FIX / "dependabot-none.json")) + assert r.returncode != 0 and "not found" in r.stderr + + +def test_the_kev_gate_runs_before_anything_is_activated(): + jobs = yaml.safe_load((REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text())["jobs"] + for job in jobs.values(): + names = [s.get("name") for s in job.get("steps", [])] + if "Terraform Apply" not in names: + continue + gate = names.index("KEV gate before apply (VDR-CSO-AKE)") + assert gate > names.index("Detection gate (G7)") + for later in ("Generate fresh Terraform plan", "Terraform Apply", "Sync Website Content"): + assert gate < names.index(later), later + run = job["steps"][gate]["run"] + for f in ("grype-python.json", "grype-js.json", "grype-lambda.json", "grype-a11y.json", + "dependabot-alerts.json", "kev-catalog.json"): + assert f in run + return + raise AssertionError("no job applies Terraform") + + +NOW = datetime(2026, 11, 5, 15, 0, tzinfo=timezone.utc) + + +def _kev_finding(**kw): + return {"tracking_id": "grype-CVE-2099-1111-thumbnailer", "source": "grype", "tool_id": "CVE-2099-1111", + "title": "x", "resource": "pkg:npm/thumbnailer@3.1.0", "cve": "CVE-2099-1111", + "severity": "High", "detected_at": "2026-11-01T00:00:00+00:00", **kw} + + +def test_the_catalog_due_date_is_kept_and_bounds_the_deadline(): + kev = vdr.ingest_kev(str(FIX / "kev-catalog.json")) + assert "CVE-2099-1111" in kev and kev.entries["CVE-2099-1111"]["due_date"] == "2026-10-22" + report, _ = vdr.build_report([_kev_finding()], [], kev, {}, now=NOW) + f = report["findings"][0] + assert f["is_kev"] and f["kev_due_at"] == "2026-10-22T00:00:00+00:00" + assert f["kev_date_added"] == "2026-10-01" and f["kev_overdue"] is True and f["is_overdue"] is True + assert f["remediation_due_at"] == "2026-10-22T00:00:00+00:00" # earlier than the Class C deadline + assert vdr.kev_dates(kev, "CVE-2099-2222", NOW)[0] == "2026-11-05T00:00:00+00:00" + assert vdr.kev_dates(kev, "CVE-2099-0000", NOW) == (None, None, False) + + +def test_the_vdt_names_the_kev_due_date(): + spec2 = importlib.util.spec_from_file_location("cr26_kev", REPO / "scripts" / "build-cr26-vdr.py") + cr26 = importlib.util.module_from_spec(spec2) + spec2.loader.exec_module(cr26) + kev = vdr.ingest_kev(str(FIX / "kev-catalog.json")) + report, _ = vdr.build_report([_kev_finding()], [], kev, {}, now=NOW) + rec = cr26.detail(report["findings"][0]) + assert rec["providerExtensions"]["kevDueAt"] == "2026-10-22T00:00:00+00:00" + assert rec["providerExtensions"]["kevOverdue"] is True From fb6865590d039220ed69d845adf4ba455027768d Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:50:52 -0500 Subject: [PATCH 23/97] Evaluation: N0, groups, early acceptance, evaluation time, factors and disclosure per finding policies/triage.rego: - N0 for a fully mitigated finding, read from VER-EVA-EPA's scale (VER-EVA-EPA; REQ-PAI-1). N0 has no timeframe. - Groups (VER-EVA-GRV): a finding's group (CVE, else decision, else check) is evaluated at its most exposed open member, on the clock of its earliest member; the group's deadline binds every open member (known-bad fixture: a new instance of an old group is past the group's deadline and blocks). - VER-TFR-MAV "will not be": a decision's planned remediation past day 192 accepts the finding now (accepted_basis). - VER-TFR-EVU: evaluation_late when first evaluated more than the rules' window after detection, or still on a default after it. scripts/_evaluation.py: the evaluation record per finding (criticality, reachability, exploitability, detectability, prevalence, privilege, proximate findings, known threats, automatable, exploitation); the disclosure determination and the redaction it drives (VER-RPT-NID/RPD). A decision record may carry automatable {value, evidence}; 'no' without evidence is refused. The golden triage report is regenerated: the only changed values are the three fully mitigated findings (N1 -> N0); every other change is an added field. Co-Authored-By: Claude Opus 5.5 --- infrastructure/schemas/decisions.schema.json | 9 + policies/triage.rego | 161 +- scripts/_decisions.py | 9 +- scripts/_evaluation.py | 223 + scripts/_triage.py | 9 +- scripts/build-vdr-report.py | 80 +- .../decisions/automatable-no-evidence.json | 26 + tests/fixtures/triage/golden-report.json | 6083 ++++++++++++++++- tests/test_decisions.py | 1 + tests/test_dispositions_mitigations.py | 4 +- 10 files changed, 6555 insertions(+), 50 deletions(-) create mode 100644 scripts/_evaluation.py create mode 100644 tests/fixtures/decisions/automatable-no-evidence.json diff --git a/infrastructure/schemas/decisions.schema.json b/infrastructure/schemas/decisions.schema.json index 9d1090dd..3d5c1fd2 100644 --- a/infrastructure/schemas/decisions.schema.json +++ b/infrastructure/schemas/decisions.schema.json @@ -42,6 +42,15 @@ "properties": {"date": {"$ref": "#/$defs/date"}, "target_pain": {"type": "string", "pattern": "^N[0-5]$"}}, "additionalProperties": false }, + "automatable": { + "type": "object", + "description": "VER-EVA-AIA: whether exploitation of the findings this record covers can be automated. Exploitation is assumed automatable; only a record carrying the evidence sets it to no.", + "required": ["value"], + "properties": {"value": {"enum": ["yes", "no"]}, "evidence": {"type": "string"}}, + "additionalProperties": false, + "if": {"properties": {"value": {"const": "no"}}}, + "then": {"required": ["value", "evidence"], "properties": {"evidence": {"minLength": 1}}} + }, "remediation_plan": {"type": "string"}, "vendor_dependency": {"type": ["string", "boolean", "object"]}, "point_of_contact": {"type": "string"}, diff --git a/policies/triage.rego b/policies/triage.rego index 8058d105..909046d4 100644 --- a/policies/triage.rego +++ b/policies/triage.rego @@ -19,7 +19,7 @@ # # Per finding, in order: # 1. fully mitigated a verified mitigation with effect "full" (rule 1) -# 2. impact the scale's lowest level when fully mitigated, else PAIN +# 2. impact N0 when fully mitigated (VER-EVA-EPA), else PAIN # 3. reachability inventory (REQ-IRV-6); unresolved or package => IRV; # a verified "not-reachable" mitigation => NIRV # 4. exploitability Figure 7, first match decides; then a verified @@ -29,9 +29,15 @@ # 5. disposition check-ID decision record, else the register entry; # a lapsed or invalid record is absent (REQ-DSP-2) # 6. deadline clock start + Class C timeframe; likely overdue -# inside the governed margin; accepted past VER-TFR-MAV -# 7. blocking KEV (even when dispositioned) > closed > top impact -# level + LEV + IRV > past the deadline +# inside the governed margin; accepted past VER-TFR-MAV, +# or earlier when the projected remediation is past it +# 7. grouping VER-EVA-GRV: findings sharing a group are evaluated at +# their most exposed member; the group's clock starts at +# its earliest member and its deadline binds every member +# 8. evaluation time VER-TFR-EVU: evaluated later than the rules allow after +# detection, or still resting on a default after it +# 9. blocking KEV (even when dispositioned) > closed > top impact +# level + LEV + IRV > past the deadline (own or group's) # ============================================================================= package vdr.triage @@ -65,6 +71,21 @@ lowest_level := levels[0] top_level := levels[count(levels) - 1] +# VER-EVA-EPA added N0 (October 2026) for an impact removed entirely: a fully +# mitigated finding takes it. N0 has no timeframe in VDR-TFR-PVR. Read from the +# rule's own scale; a rules file without N0 keeps the timeframe table's lowest. +epa_rule := object.get(data.rules.FRR.VER.data.all, ["EVA", "VER-EVA-EPA"], {}) + +mitigated_level := "N0" if { + some b in object.get(epa_rule, "following_information_bullets", []) + startswith(b, "**N0**") +} else := lowest_level + +level_index(p) := i if { + some i, l in levels + l == p +} else := -1 + mav_rule := data.rules.FRR.VER.data.all.TFR["VER-TFR-MAV"] mav_cell := c if { @@ -104,7 +125,7 @@ errors contains msg if { some i, item in input.items item.kind == "finding" d := decisions[i] - d.pain != lowest_level + not d.pain in {lowest_level, mitigated_level} d.sla_days == null msg := sprintf("%v: no Class C timeframe for %v/%v/%v in data.rules", [d.id, d.pain, d.lev, d.irv]) } @@ -181,7 +202,7 @@ fm_bool(item) if { fully_mitigated(item) } else := false -impact(item) := lowest_level if { +impact(item) := mitigated_level if { fully_mitigated(item) } else := item.pain @@ -379,6 +400,42 @@ accepted_now(needs, ns) if { input.now_ns >= ns } else := false +# VER-TFR-MAV "will not be": a finding whose projected remediation (a decision's +# planned date, passed in as projected_ns) falls after the maximum period is +# accepted now, not when the period runs out. +projected_past(needs, item, ns) if { + needs == true + ns != null + p := object.get(item, "projected_ns", null) + is_number(p) + p > ns +} else := false + +accepted_basis(needs, item, ns) := "maximum-period-passed" if { + accepted_now(needs, ns) +} else := "projected-past-maximum-period" if { + projected_past(needs, item, ns) +} else := null + +# ---- 8. evaluation time (VER-TFR-EVU) ------------------------------------------------------ +# input.evaluation_days is read from the rules by scripts/_rules.py. A finding +# is late when its first evaluation came more than that many days after the +# source detected it, or when, that long after detection, it is still open on a +# default (no exploit-prediction score and no vector: Figure 7's last step). +evaluation_late(item, l, needs) := "evaluated-late" if { + is_number(object.get(input, "evaluation_days", null)) + d := object.get(item, "detected_ns", null) + is_number(d) + object.get(item, "evaluated_ns", item.clock.first_evaluated_ns) - d > input.evaluation_days * day_ns +} else := "default-after-evaluation-window" if { + is_number(object.get(input, "evaluation_days", null)) + needs == true + l[1] == "no-score" + object.get(item, "default_evaluation", false) == true + d := object.get(item, "detected_ns", item.clock.first_evaluated_ns) + input.now_ns - d > input.evaluation_days * day_ns +} else := null + # ---- 7. blocking --------------------------------------------------------------------------- # KEV is tested before the disposition: a register entry never buys a known # exploited vulnerability out of the gate. @@ -403,7 +460,7 @@ label_of(disposition, label) := disposition if { disposition != null } else := label -decide(item) := { +base(item) := { "id": item.id, "fully_mitigated": fm, "pain": pain, @@ -423,10 +480,12 @@ decide(item) := { "due_ns": dl.due_ns, "is_overdue": dl.is_overdue, "will_be_overdue": dl.will_be_overdue, - "accepted": accepted_now(needs, acc), + "accepted": accepted_basis(needs, item, acc) != null, + "accepted_basis": accepted_basis(needs, item, acc), "accepted_ns": acc, - "blocking": rule != null, - "block_rule": rule, + "evaluation_late": evaluation_late(item, l, needs), + "start_ns": field_or_null(item.clock, "start_ns"), + "is_kev": item.is_kev == true, } if { fm := fm_bool(item) pain := impact(item) @@ -440,7 +499,85 @@ decide(item) := { sla := timeframe(pain, l[0], r[0]) dl := deadline(item, sla, needs) acc := accepted_ns(item) - rule := block_rule(item.is_kev == true, closed, pain, l[0], r[0], dl.is_overdue) } -decisions := [decide(item) | some item in input.items] +base_decisions := [base(item) | some item in input.items] + +# ---- 7. grouping (VER-EVA-GRV) ----------------------------------------------------------- +# input.items[i].group names the finding's group (a CVE, a decision record, or a +# check ID); a finding without one is its own group. The group is evaluated at +# its most exposed open member: the highest impact, LEV if any member is, IRV if +# any member is, the clock from its earliest member. The group's deadline binds +# each open member, so a new instance of an old group does not get a new clock. +group_key(i) := g if { + g := input.items[i].group + is_string(g) + g != "" +} else := sprintf("item:%d", [i]) + +open_members(g) := [j | + some j, it in input.items + it.kind == "finding" + group_key(j) == g + base_decisions[j].needs_remediation == true + level_index(base_decisions[j].pain) >= 0 +] + +group_eval(g) := { + "pain": levels[max([level_index(base_decisions[j].pain) | some j in m])], + "lev": any_true([base_decisions[j].lev | some j in m]), + "irv": any_true([base_decisions[j].irv | some j in m]), + "start_ns": min([base_decisions[j].start_ns | some j in m]), + "members": count(m), +} if { + m := open_members(g) + count(m) > 0 +} else := null + +field_or_null(o, k) := o[k] if { + is_object(o) +} else := null + +past_bool(due) if { + past(due) +} else := false + +any_true(xs) if { + some x in xs + x == true +} else := false + +group_deadline(ge) := {"due_ns": null, "is_overdue": false} if { + ge == null +} else := {"due_ns": null, "is_overdue": false} if { + timeframe(ge.pain, ge.lev, ge.irv) == null +} else := {"due_ns": due, "is_overdue": past_bool(due)} if { + due := ge.start_ns + (timeframe(ge.pain, ge.lev, ge.irv) * day_ns) +} + +own_or_group_overdue(b, gd) if { + b.is_overdue == true +} else if { + b.needs_remediation == true + gd.is_overdue == true +} else := false + +decide(i) := object.union(b, { + "group": g, + "group_pain": field_or_null(ge, "pain"), + "group_lev": field_or_null(ge, "lev"), + "group_irv": field_or_null(ge, "irv"), + "group_due_ns": gd.due_ns, + "is_overdue": is_overdue, + "blocking": rule != null, + "block_rule": rule, +}) if { + b := base_decisions[i] + g := group_key(i) + ge := group_eval(g) + gd := group_deadline(ge) + is_overdue := own_or_group_overdue(b, gd) + rule := block_rule(b.is_kev, b.closed, b.pain, b.lev, b.irv, is_overdue) +} + +decisions := [decide(i) | some i, _ in input.items] diff --git a/scripts/_decisions.py b/scripts/_decisions.py index e847eea9..4b181041 100644 --- a/scripts/_decisions.py +++ b/scripts/_decisions.py @@ -41,7 +41,7 @@ # field is refused: a misspelt review_by would otherwise read as absent. TOP_KEYS = {"$comment", "_note", "schema", "decisions"} RECORD_KEYS = {"id", "former_id", "title", "status", "disposition", "justification", *DECISION_FIELDS, - "closed_on", "original_detection_date", "planned_completion", "planned_reduction", "remediation_plan", + "closed_on", "original_detection_date", "planned_completion", "planned_reduction", "automatable", "remediation_plan", "vendor_dependency", "point_of_contact", "controls", "applies_to", "vuln_fields", "source"} APPLIES_KEYS = {"check_ids", "vuln_ids", "resources"} @@ -104,6 +104,13 @@ def validate(doc, where="data/decisions.json"): _fail(f"{rid}: planned_reduction must be {{date: YYYY-MM-DD, target_pain: N0-N5}}") if status != "in-force": _fail(f"{rid}: only an in-force record plans a reduction") + auto = r.get("automatable") + if auto is not None: + # VER-EVA-AIA: exploitation is assumed automatable unless there is + # evidence it is not. "no" without the evidence would be an assumption. + if (not isinstance(auto, dict) or set(auto) - {"value", "evidence"} or auto.get("value") not in ("yes", "no") + or (auto["value"] == "no" and not str(auto.get("evidence") or "").strip())): + _fail(f"{rid}: automatable must be {{value: yes|no, evidence}}, and 'no' needs its evidence") applies = r.get("applies_to") if not isinstance(applies, dict) or set(applies) - APPLIES_KEYS: _fail(f"{rid}: applies_to must be an object with only {sorted(APPLIES_KEYS)}") diff --git a/scripts/_evaluation.py b/scripts/_evaluation.py new file mode 100644 index 00000000..83fc6bf4 --- /dev/null +++ b/scripts/_evaluation.py @@ -0,0 +1,223 @@ +"""The evaluation record for each finding (VER-EVA-*), beside the triage decision. + +policies/triage.rego decides the ratings, the deadline and blocking. This module +states what the evaluation considered, as facts with their basis, so a reader +can see each factor VER-EVA-EFA names and the assumptions VER-EVA-AIA and +VDR-CSO-RES require: + + criticality the component's classification (what PAIN is computed from) + reachability the policy's IRV and its basis + exploitability the policy's LEV and its basis + detectability how an attacker would learn of it + prevalence how many findings and components share its group + privilege what exploitation needs and what it reaches + proximate other findings and mitigations on the same component + known_threats KEV and the exploit-prediction score + automatable yes unless evidence says otherwise (VER-EVA-AIA) + exploitation whether exploitation of it has been observed (VDR-CSO-RES) + +It also groups findings (VER-EVA-GRV) and decides what the public artifacts may +say about each one (VER-RPT-NID, VER-RPT-RPD). Factors are recorded, not fed +into the impact tables: REQ-PAI-4 makes the tables global, and a provider that +changes them publishes the deviation. Grouping is fed to the policy, which +evaluates a group at its most exposed member. +""" + +import json +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +INCIDENTS = REPO / "data" / "incidents.json" + +# Decision records that set these policies (data/decisions.json). +EXPLOITATION_POLICY = "DEC-181" +DISCLOSURE_POLICY = "DEC-182" + +FACTORS = { + "criticality": "The resolved component's declared classification (data sensitivity, mission " + "criticality, agency scope); the fail-safe classification when it resolves to none. " + "PAIN is computed from it.", + "reachability": "Internet-reachability as the triage policy decides it (VER-EVA-EIR), with its basis.", + "exploitability": "Likely exploitability by the method's Figure 7 (VER-EVA-ELX), with its basis.", + "detectability": "How an attacker would learn of the vulnerability: observed from outside (DAST), " + "a published advisory, or configuration readable in the public source repository.", + "prevalence": "How many findings and components share the finding's group, and the share of the " + "inventory's components they cover.", + "privilege": "What exploitation requires (CVSS PR) and what it reaches (CVSS S, and whether the " + "component is an identity or key-management resource).", + "proximate": "Other findings (open, dispositioned or mitigated) and verified mitigations on the same " + "component, which the group and the decision records are read with.", + "known_threats": "CISA KEV membership and the exploit-prediction score with its date.", + "automatable": "VER-EVA-AIA: yes unless a decision record carries evidence that exploitation cannot " + "be automated, or the CVSS 4.0 vector says AU:N.", + "exploitation": "VDR-CSO-RES: active when an incident record names the finding or the vulnerability " + "is observed exploited; otherwise none observed, under the exploitation assessment " + f"policy ({EXPLOITATION_POLICY}).", +} + +_PRIVILEGED_TYPES = ("iam", "identity", "kms", "key", "cognito", "user_pool") + + +def group_key(f, decision_id): + """VER-EVA-GRV: a CVE groups every instance of it; otherwise a decision + record groups what it decided together (one root cause, one decision); + otherwise the check ID groups instances of one rule.""" + if f.get("cve"): + return f"cve:{f['cve']}" + if decision_id: + return f"decision:{decision_id}" + return f"check:{f.get('source')}:{f.get('tool_id')}" + + +def automatable(f, dec): + """(value, source). Defaults to yes (VER-EVA-AIA).""" + a = (dec or {}).get("automatable") + if isinstance(a, dict) and a.get("value") == "no" and str(a.get("evidence") or "").strip(): + return {"value": "no", "source": f"decision {dec['id']}", "evidence": a["evidence"].strip()} + vec = str(f.get("cvss_vector") or "") + if vec.startswith("CVSS:4.0/") and "/AU:" in vec: + au = vec.split("/AU:", 1)[1][:1] + return {"value": "yes" if au == "Y" else "no", "source": "CVSS 4.0 AU", "evidence": vec} + return {"value": "yes", "source": "VER-EVA-AIA default", "evidence": None} + + +def detectability(f, irv): + src = f.get("source") + if src == "zap" or f.get("locations"): + return {"value": "observed-outside-in", "basis": "found by an unauthenticated outside-in scan"} + if f.get("cve") or src in ("grype", "dependabot"): + return {"value": "public-advisory", + "basis": "a published advisory names the package; " + + ("the component is internet-reachable" if irv else "the component is not internet-reachable")} + if src == "process": + return {"value": "provider-only", "basis": "a failure of the provider's own pipeline"} + return {"value": "public-source", "basis": "configuration in the public source repository"} + + +def privilege(f, cid, ctype, metrics): + pr = {"N": "none", "L": "low", "H": "high"}.get(metrics.get("PR"), "unknown") + ident = any(t in f"{ctype or ''} {cid or ''}".lower() for t in _PRIVILEGED_TYPES) + if metrics.get("S") == "C": + gained = "beyond-component" + elif ident: + gained = "identity-or-keys" + elif metrics: + gained = "within-component" + else: + gained = "unknown" + return {"required": pr, "gained": gained, + "basis": ("CVSS PR/S" if metrics else "no vector") + ("; identity or key-management component" if ident else "")} + + +def load_incidents(path=INCIDENTS): + """Incident records, when the register exists. Missing: none.""" + p = Path(path) + if not p.is_file(): + return [] + doc = json.loads(p.read_text(encoding="utf-8")) + recs = doc.get("incidents") if isinstance(doc, dict) else doc + return recs if isinstance(recs, list) else [] + + +def exploitation(f, is_kev, incidents, now_iso): + """VDR-CSO-RES: whether exploitation of this finding has been observed.""" + if str(f.get("exploitation") or "").lower() == "active": + return {"value": "active", "basis": "reported by the source", "assessed_at": now_iso} + keys = [k for k in (f.get("tracking_id"), f.get("cve")) if k] + for inc in incidents: + blob = json.dumps(inc, sort_keys=True) + if any(k in blob for k in keys): + return {"value": "active", "basis": f"incident {inc.get('id', '?')} names it", "assessed_at": now_iso} + basis = "no incident record names it" + if is_kev: + basis += "; exploited elsewhere (CISA KEV), which the policy treats as likely exploitable" + return {"value": "none-observed", "basis": f"{basis} ({EXPLOITATION_POLICY})", "assessed_at": now_iso} + + +def evaluation_record(row, f, *, cid, ctype, metrics, classification, auto, expl, proximate, prevalence, + epss, is_kev): + return { + "criticality": classification, + "reachability": {"irv": row["internet_reachable"], "basis": row["irv_basis"]}, + "exploitability": {"lev": row["likely_exploitable"], "basis": row["lev_basis"]}, + "detectability": detectability(f, row["internet_reachable"]), + "prevalence": prevalence, + "privilege": privilege(f, cid, ctype, metrics), + "proximate": proximate, + "known_threats": {"kev": bool(is_kev), "epss": epss}, + "automatable": auto, + "exploitation": expl, + } + + +def attach_groups(rows, inventory_size): + """Group membership, prevalence and proximate findings, after every row is + built. Returns the published groups.""" + groups, by_component = {}, {} + for r in rows: + groups.setdefault(r["group_id"], []).append(r) + if r.get("component_id"): + by_component.setdefault(r["component_id"], []).append(r) + out = [] + for gid, members in sorted(groups.items()): + comps = sorted({m["component_id"] for m in members if m.get("component_id")}) + prevalence = {"findings": len(members), "components": len(comps), + "inventory_share": round(len(comps) / inventory_size, 3) if inventory_size else None} + for m in members: + m["evaluation"]["prevalence"] = prevalence + g = members[0] + out.append({ + "group_id": gid, + "basis": gid.split(":", 1)[0], + "members": sorted(m["tracking_id"] for m in members), + "components": comps, + "pain": g.get("group_pain"), + "likely_exploitable": g.get("group_lev"), + "internet_reachable": g.get("group_irv"), + "remediation_due_at": g.get("group_due_at"), + "open_members": sum(1 for m in members if m["final_disposition"] == "open"), + "dispositions": sorted({m["final_disposition"] for m in members}), + }) + for r in rows: + others = [o for o in by_component.get(r.get("component_id"), []) if o is not r] + r["evaluation"]["proximate"] = { + "findings": sorted({o["tracking_id"] for o in others}), + "open": sum(1 for o in others if o["final_disposition"] == "open"), + "mitigations": sorted({o["mitigation"]["id"] for o in others + [r] + if isinstance(o.get("mitigation"), dict) and o["mitigation"].get("valid")}), + } + return out + + +# ---- disclosure (VER-RPT-NID, VER-RPT-RPD) ------------------------------------------ +# What the public artifacts withhold while a finding is open, internet-reachable, +# likely exploitable and rated N3 or higher: the detail that tells an attacker +# where and how (description, observed locations, fixed versions, evidence). The +# tracking ID, ratings, dates and status stay, so any party can make a +# risk-based decision. Policy: DEC-182. +WITHHELD_FIELDS = ("description", "locations", "fix_versions", "supplementary_info") +_WITHHOLD_FROM = 3 + + +def disclosure(row): + pain = str(row.get("pain") or "") + n = int(pain[1:]) if pain[1:].isdigit() else 0 + if (row.get("final_disposition") == "open" and row.get("internet_reachable") and row.get("likely_exploitable") + and n >= _WITHHOLD_FROM and not row.get("fully_mitigated")): + return {"level": "summary", "policy": DISCLOSURE_POLICY, + "reason": f"open, IRV, LEV and {pain}: detail withheld from public artifacts until remediated or " + "dispositioned; available to necessary parties from the provider"} + return {"level": "full", "policy": DISCLOSURE_POLICY, + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation"} + + +def redact(row): + """The public form of a finding under its disclosure determination.""" + if (row.get("public_disclosure") or {}).get("level") != "summary": + return row + out = dict(row) + for k in WITHHELD_FIELDS: + if k in out: + if out[k] not in (None, "", [], {}): + out[k] = f"[withheld under {DISCLOSURE_POLICY}]" if isinstance(out[k], str) else None + return out diff --git a/scripts/_triage.py b/scripts/_triage.py index 1f55deff..d6df349c 100644 --- a/scripts/_triage.py +++ b/scripts/_triage.py @@ -70,10 +70,13 @@ def evaluate(query, input_doc, rules_dir=_rules.RULES_DIR, policy=POLICY): _fail(f"{query} is undefined") -def decide(items, *, now, epss_threshold, margin_days, lev_overrides, rules_dir=_rules.RULES_DIR): - """One decision per item, in order.""" +def decide(items, *, now, epss_threshold, margin_days, lev_overrides, rules_dir=_rules.RULES_DIR, + evaluation_days=None): + """One decision per item, in order. evaluation_days: VER-TFR-EVU, read from + the rules by the caller (_rules.evaluation_days); None skips the check.""" doc = {"now_ns": to_ns(now), "today": now.date().isoformat(), "epss_threshold": epss_threshold, - "margin_days": margin_days, "lev_overrides": lev_overrides or {}, "items": items} + "margin_days": margin_days, "lev_overrides": lev_overrides or {}, "items": items, + **({"evaluation_days": evaluation_days} if evaluation_days is not None else {})} out = evaluate('{"decisions": data.vdr.triage.decisions, "errors": data.vdr.triage.errors}', doc, rules_dir) if out.get("errors"): diff --git a/scripts/build-vdr-report.py b/scripts/build-vdr-report.py index ed38db60..45543a9d 100755 --- a/scripts/build-vdr-report.py +++ b/scripts/build-vdr-report.py @@ -53,6 +53,7 @@ # Shared helpers live next to this script; keep it runnable from any cwd. sys.path.insert(0, str(Path(__file__).resolve().parent)) import _decisions # noqa: E402 +import _evaluation # noqa: E402 import _gates # noqa: E402 import _redact # noqa: E402 import _rules # noqa: E402 @@ -1403,11 +1404,13 @@ def _no_direct_impact(m): def triage_item(f, kind, *, pain, is_kev, epss_scores, reach, mitigation=None, - check_decision=None, register=None, clock=None): - """The facts the triage policy decides one finding from.""" + check_decision=None, register=None, clock=None, extra=None): + """The facts the triage policy decides one finding from. `extra`: the + grouping, projection and evaluation-time facts (group, projected_ns, + detected_ns, default_evaluation), only for report findings.""" cve = f.get("cve") scores = epss_scores or {} - return { + return {**(extra or {}), "id": f["tracking_id"], "kind": kind, "pain": pain, @@ -1428,6 +1431,32 @@ def triage_item(f, kind, *, pain, is_kev, epss_scores, reach, mitigation=None, } +def evaluation_facts(f, dec, clk, epss_scores, now): + """The triage facts beyond the ratings: the finding's group (VER-EVA-GRV), + a decision's projected remediation date (VER-TFR-MAV), when the source + detected it and when it was first evaluated (VER-TFR-EVU), and whether its + ratings rest on defaults (no vector, no prediction score).""" + plan = dec.get("planned_reduction") or {} + projected = None + if plan.get("target_pain") == "N0" and plan.get("date"): + projected = _triage.to_ns(datetime.combine(date.fromisoformat(plan["date"]), datetime.min.time(), + tzinfo=timezone.utc)) + # A decision record is an evaluation: a finding a person decided on before + # the pipeline first saw it was evaluated then. + evaluated = clk["first_evaluated_dt"] + if dec.get("decided_on"): + evaluated = min(evaluated, datetime.combine(date.fromisoformat(dec["decided_on"]), datetime.min.time(), + tzinfo=timezone.utc)) + cve = f.get("cve") + return { + "group": _evaluation.group_key(f, dec.get("id")), + "projected_ns": projected, + "detected_ns": _triage.to_ns(_ts(clk["first_detected"], now)), + "evaluated_ns": _triage.to_ns(evaluated), + "default_evaluation": not f.get("cvss_vector") and not (cve and cve in (epss_scores or {})), + } + + def _decide_one(finding, *, is_kev=False, irv=False, epss_scores=None, epss_threshold=0.1, lev_overrides=None, today=None, kind="suppression", register=None, mitigation=None): """The policy's decision for one finding, outside a report (unit callers).""" @@ -1984,11 +2013,13 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No mitigations = mitigations or {} if decision_records is None: decision_records = load_decision_records() + full_records = {r["id"]: r for r in _decisions.in_force(DECISIONS)} + incidents = _evaluation.load_incidents() matched_register, matched_mitigations, lapsed = set(), set(), [] margin_days = int(config["likely_overdue_margin_days"]) report_findings = [] summary = { - "by_pain": {"N1": 0, "N2": 0, "N3": 0, "N4": 0, "N5": 0}, + "by_pain": {"N0": 0, "N1": 0, "N2": 0, "N3": 0, "N4": 0, "N5": 0}, "blocking": 0, "kev": 0, "risk_accepted": len(suppressions), @@ -2045,13 +2076,16 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No else: summary["ledger_newly_detected"] += 1 clk = clock_facts(f, ledger.get(tid), now) + dec_id = (check_decision or {}).get("decision_id") or (reg[1].get("decision_id") if reg else None) + dec = full_records.get(dec_id) or {} items.append(triage_item( f, "finding", pain=assign_pain(f, index, config), is_kev=is_kev, epss_scores=epss_scores, reach=reach_facts(f, cid, inv), mitigation=mit, check_decision=check_decision, register=({"key": reg[0], "entry": reg[1]} if reg else None), clock={"start_ns": _triage.to_ns(clk["start_dt"]), - "first_evaluated_ns": _triage.to_ns(clk["first_evaluated_dt"])})) - prepared.append((f, cid, mit, is_kev, check_decision, reg, tid, clk)) + "first_evaluated_ns": _triage.to_ns(clk["first_evaluated_dt"])}, + extra=evaluation_facts(f, dec, clk, epss_scores, now))) + prepared.append((f, cid, mit, is_kev, check_decision, reg, tid, clk, dec)) # Risk-accepted entries (.checkov.yaml suppressions) are decided for # reachability and exploitability only: they carry no deadline. @@ -2073,10 +2107,11 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No # 2. Decisions: one evaluation of policies/triage.rego for the whole report. decisions = _triage.decide(items, now=now, epss_threshold=config["epss_threshold"], - margin_days=margin_days, lev_overrides=lev_overrides) + margin_days=margin_days, lev_overrides=lev_overrides, + evaluation_days=_rules.evaluation_days()) # 3. Publish. - for (f, cid, mit, is_kev, check_decision, reg, tid, clk), d in zip(prepared, decisions[:len(prepared)], strict=True): + for (f, cid, mit, is_kev, check_decision, reg, tid, clk, dec), d in zip(prepared, decisions[:len(prepared)], strict=True): pain, irv, lev, sla = d["pain"], d["irv"], d["lev"], d["sla_days"] disposition, finding_decision_id = d["disposition"], d["decision_id"] is_suppressed, needs_remediation = d["closed"], d["needs_remediation"] @@ -2184,11 +2219,14 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "will_be_overdue": d["will_be_overdue"], "accepted": d["accepted"], "accepted_at": accepted_at, + **({"accepted_basis": d["accepted_basis"]} if d["accepted"] else {}), # VER-RPT-AVI: the recorded justification when an in-force record # carries one; otherwise the gap is stated, never filled in. "acceptance_explanation": ( ((reg[1].get("justification") or "").strip() or NOT_RECORDED - if reg and d["record_status"] == "in-force" else NOT_RECORDED) + if reg and d["record_status"] == "in-force" + else (dec.get("justification") or "").strip() or NOT_RECORDED + if d["accepted_basis"] == "projected-past-maximum-period" else NOT_RECORDED) if d["accepted"] else None), "overdue_explanation": (block_reason if block_this else f"documented suppression ({disposition}); tracked as {finding_decision_id}" @@ -2196,7 +2234,25 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "supplementary_info": "", "is_blocking": block_this, "block_reason": block_reason, + # VER-EVA-GRV: the group the finding is evaluated in, at its most + # exposed open member; the group's deadline binds this finding. + "group_id": d["group"], + "group_pain": d["group_pain"], + "group_lev": d["group_lev"], + "group_irv": d["group_irv"], + "group_due_at": (_triage.from_ns(d["group_due_ns"], clk["start_dt"].tzinfo) + if d["group_due_ns"] is not None else None), + # VER-TFR-EVU: null when evaluated in time; otherwise why not. + "evaluation_late": d["evaluation_late"], + "evaluation": _evaluation.evaluation_record( + {"internet_reachable": irv, "irv_basis": d["irv_basis"], "likely_exploitable": lev, + "lev_basis": d["lev_basis"]}, f, + cid=cid, ctype=inv["types"].get(cid), metrics=parse_vector(f.get("cvss_vector")), + classification=f.get("_classification"), auto=_evaluation.automatable(f, dec), + expl=_evaluation.exploitation(f, is_kev, incidents, now.isoformat()), + proximate=None, prevalence=None, epss=epss_used, is_kev=is_kev), }) + report_findings[-1]["public_disclosure"] = _evaluation.disclosure(report_findings[-1]) # Histogram and blocking counts reflect genuinely-open findings only; # suppressed findings are summarized via the risk-accepted/FP counts. if not is_suppressed: @@ -2249,6 +2305,11 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "explanation": s.get("explanation", ""), }) + # VER-EVA-GRV/EFA: groups, prevalence and proximate findings are read across + # every finding; then each finding's public form follows its disclosure + # determination (VER-RPT-NID). + groups = _evaluation.attach_groups(report_findings, len(inv.get("classes") or {})) + report_findings = [_evaluation.redact(r) for r in report_findings] cve_findings = consolidate_by_cve(report_findings, risk_accepted_records) summary["unique_cves"] = len(cve_findings) summary["unique_cves_open"] = sum(1 for c in cve_findings if "open" in c["dispositions"]) @@ -2266,6 +2327,7 @@ def build_report(findings, suppressions, kev_cves, ledger, index=None, config=No "impact_level": _profile["impact_level"], "summary": summary, "findings": report_findings, + "groups": groups, "cve_findings": cve_findings, "risk_accepted": risk_accepted_records, "rules_reference": { diff --git a/tests/fixtures/decisions/automatable-no-evidence.json b/tests/fixtures/decisions/automatable-no-evidence.json new file mode 100644 index 00000000..88028661 --- /dev/null +++ b/tests/fixtures/decisions/automatable-no-evidence.json @@ -0,0 +1,26 @@ +{ + "_note": "Known-bad: a record that says exploitation of its findings cannot be automated, with no evidence. VER-EVA-AIA assumes automatable unless there is evidence otherwise; the build stops.", + "schema": "decisions/1", + "decisions": [ + { + "id": "DEC-001", + "title": "fixture", + "status": "in-force", + "disposition": "accepted", + "justification": "fixture", + "decided_by": "Fixture", + "decided_on": "2026-01-01", + "review_by": "2027-01-01", + "applies_to": { + "check_ids": [], + "vuln_ids": [ + "CVE-2099-0001" + ], + "resources": [] + }, + "automatable": { + "value": "no" + } + } + ] +} diff --git a/tests/fixtures/triage/golden-report.json b/tests/fixtures/triage/golden-report.json index 9385d207..9c140694 100644 --- a/tests/fixtures/triage/golden-report.json +++ b/tests/fixtures/triage/golden-report.json @@ -201,7 +201,7 @@ ], "first_detected": "2026-10-14T12:00:00+00:00", "is_kev": false, - "pain": "N1", + "pain": "N0", "scanners": [ "grype" ] @@ -281,7 +281,7 @@ ], "first_detected": "2026-11-03T12:00:00+00:00", "is_kev": false, - "pain": "N1", + "pain": "N0", "scanners": [ "grype" ] @@ -349,7 +349,7 @@ ], "first_detected": "2026-11-03T12:00:00+00:00", "is_kev": false, - "pain": "N1", + "pain": "N0", "scanners": [ "grype" ] @@ -806,12 +806,73 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "known-exploited", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it; exploited elsewhere (CISA KEV), which the policy treats as likely exploitable (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": true + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "irv-package" + ], + "mitigations": [], + "open": 1 + }, + "reachability": { + "basis": "package-host-unrecorded", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "false-positive", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "cve:CVE-0000-1001", + "group_irv": null, + "group_lev": null, + "group_pain": null, "historical_pain": "N2", "internet_reachable": true, "irv_basis": "package-host-unrecorded", @@ -826,6 +887,11 @@ "occurrences": 1, "overdue_explanation": "KEV (CISA Known Exploited Vulnerability)", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -876,12 +942,83 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "floor", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor-v4", + "fm-flag", + "likely-overdue-short", + "m-full", + "m-nirv", + "m-void", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-21T12:00:00+00:00", + "group_id": "cve:CVE-0000-1002", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "inventory", @@ -899,6 +1036,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 48d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -949,12 +1091,83 @@ "score": 0.0, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "floor", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.0, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "fm-flag", + "likely-overdue-short", + "m-full", + "m-nirv", + "m-void", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-21T12:00:00+00:00", + "group_id": "cve:CVE-0000-1003", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "inventory", @@ -972,6 +1185,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 48d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1022,12 +1240,110 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-05-14T12:00:00+00:00", + "group_id": "cve:CVE-0000-1004", + "group_irv": false, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1045,6 +1361,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 192d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1091,12 +1412,74 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-direct-impact", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "c-ra", + "c-ra-lapsed", + "c-ra-tfsec", + "cfg-noscore", + "informational", + "irv-inventory-true" + ], + "mitigations": [], + "open": 4 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "check:checkov:CKV_AWS_999", + "group_irv": true, + "group_lev": false, + "group_pain": "N1", "historical_pain": "N1", "internet_reachable": true, "irv_basis": "inventory", @@ -1111,6 +1494,11 @@ "occurrences": 1, "overdue_explanation": "informational severity; no remediation SLA applies", "pain": "N1", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1161,12 +1549,110 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1005", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1184,6 +1670,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1234,12 +1725,110 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-05-14T12:00:00+00:00", + "group_id": "cve:CVE-0000-1006", + "group_irv": false, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1257,6 +1846,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 192d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1307,12 +1901,110 @@ "score": 0.1, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.1, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1007", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1330,6 +2022,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1376,12 +2073,106 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1008", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1399,6 +2190,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1449,12 +2245,110 @@ "score": 0.0, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "exploitation-observed", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "reported by the source", + "value": "active" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.0, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1009", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1472,6 +2366,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1518,12 +2417,74 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-ra", + "c-ra-lapsed", + "c-ra-tfsec", + "informational", + "irv-inventory-true", + "no-impact" + ], + "mitigations": [], + "open": 4 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "check:checkov:CKV_AWS_998", + "group_irv": true, + "group_lev": true, + "group_pain": "N1", "historical_pain": "N1", "internet_reachable": true, "irv_basis": "inventory", @@ -1538,6 +2499,11 @@ "occurrences": 1, "overdue_explanation": "informational severity; no remediation SLA applies", "pain": "N1", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1584,12 +2550,67 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "found by an unauthenticated outside-in scan", + "value": "observed-outside-in" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "observed", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-21T12:00:00+00:00", + "group_id": "cve:ZAP-1", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "observed", @@ -1607,6 +2628,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 48d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1657,12 +2683,71 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "multi", + "data_sensitivity": "cui", + "internet_reachable": true, + "mission_criticality": "high" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 0, + "findings": 1, + "inventory_share": 0.0 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "unresolved", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-19T12:00:00+00:00", + "group_id": "cve:CVE-0000-1010", + "group_irv": true, + "group_lev": false, + "group_pain": "N5", "historical_pain": "N5", "internet_reachable": true, "irv_basis": "unresolved", @@ -1680,6 +2765,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 16d allowed", "pain": "N5", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1730,12 +2820,73 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "kev-fp" + ], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "package-host-unrecorded", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-05-14T12:00:00+00:00", + "group_id": "cve:CVE-0000-1011", + "group_irv": true, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "package-host-unrecorded", @@ -1753,6 +2904,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 192d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1799,12 +2955,106 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "check:checkov:CKV_AWS_997", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -1822,6 +3072,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1868,12 +3123,74 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-ra", + "c-ra-lapsed", + "c-ra-tfsec", + "cfg-noscore", + "informational", + "no-impact" + ], + "mitigations": [], + "open": 4 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-21T12:00:00+00:00", + "group_id": "check:checkov:CKV_AWS_996", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "inventory", @@ -1891,6 +3208,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 48d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -1933,7 +3255,7 @@ "completed_reductions": [], "component_id": "cmp-pub", "current_disposition": "mitigated", - "current_pain": "N1", + "current_pain": "N0", "cve": "CVE-0000-1012", "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", "days_since_first_detected": 20, @@ -1941,13 +3263,80 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "fully-mitigated", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "fm-flag", + "likely-overdue-short", + "m-nirv", + "m-void", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 6 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "mitigated", "first_detected": "2026-10-14T12:00:00+00:00", "first_evaluated": "2026-10-14T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": true, - "historical_pain": "N1", + "group_due_at": null, + "group_id": "cve:CVE-0000-1012", + "group_irv": null, + "group_lev": null, + "group_pain": null, + "historical_pain": "N0", "internet_reachable": true, "irv_basis": "inventory", "is_blocking": false, @@ -1971,13 +3360,18 @@ "next_reduction": null, "occurrences": 1, "overdue_explanation": "documented suppression (mitigated); tracked as None", - "pain": "N1", + "pain": "N0", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", "irv": true, "lev": false, - "pain": "N1", + "pain": "N0", "reason": "first evaluation" } ], @@ -2022,12 +3416,83 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "fm-flag", + "likely-overdue-short", + "m-full", + "m-void", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 5 + }, + "reachability": { + "basis": "mitigation", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-05-14T12:00:00+00:00", + "group_id": "cve:CVE-0000-1013", + "group_irv": false, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "mitigation", @@ -2056,6 +3521,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 192d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2106,12 +3576,110 @@ "score": 0.9, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "mitigation", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.9, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-05-14T12:00:00+00:00", + "group_id": "cve:CVE-0000-1014", + "group_irv": false, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -2140,6 +3708,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 192d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2190,12 +3763,110 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "known-exploited", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it; exploited elsewhere (CISA KEV), which the policy treats as likely exploitable (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": true + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1015", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -2224,6 +3895,11 @@ "occurrences": 1, "overdue_explanation": "KEV (CISA Known Exploited Vulnerability)", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2281,15 +3957,86 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, - "final_disposition": "open", - "first_detected": "2026-10-04T12:00:00+00:00", - "first_evaluated": "2026-10-04T12:00:00+00:00", - "fix_state": null, - "fix_versions": null, - "fully_mitigated": false, - "historical_pain": "N3", - "internet_reachable": true, - "irv_basis": "inventory", + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "floor", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "fm-flag", + "likely-overdue-short", + "m-full", + "m-nirv", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, + "final_disposition": "open", + "first_detected": "2026-10-04T12:00:00+00:00", + "first_evaluated": "2026-10-04T12:00:00+00:00", + "fix_state": null, + "fix_versions": null, + "fully_mitigated": false, + "group_due_at": "2026-11-21T12:00:00+00:00", + "group_id": "cve:CVE-0000-1016", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", + "historical_pain": "N3", + "internet_reachable": true, + "irv_basis": "inventory", "is_blocking": false, "is_kev": false, "is_overdue": false, @@ -2316,6 +4063,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 30d of 48d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-10-04T12:00:00+00:00", @@ -2368,7 +4120,7 @@ "completed_reductions": [], "component_id": "cmp-pub", "current_disposition": "mitigated", - "current_pain": "N1", + "current_pain": "N0", "cve": "CVE-0000-1017", "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", "days_since_first_detected": 0, @@ -2376,13 +4128,80 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "fully-mitigated", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "likely-overdue-short", + "m-full", + "m-nirv", + "m-void", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 6 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "mitigated", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": true, - "historical_pain": "N1", + "group_due_at": null, + "group_id": "cve:CVE-0000-1017", + "group_irv": null, + "group_lev": null, + "group_pain": null, + "historical_pain": "N0", "internet_reachable": true, "irv_basis": "inventory", "is_blocking": false, @@ -2395,13 +4214,18 @@ "next_reduction": null, "occurrences": 1, "overdue_explanation": "documented suppression (mitigated); tracked as None", - "pain": "N1", + "pain": "N0", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", "irv": true, "lev": false, - "pain": "N1", + "pain": "N0", "reason": "first evaluation" } ], @@ -2456,12 +4280,110 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "mitigation", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1020", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "mitigation", @@ -2490,6 +4412,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2546,12 +4473,106 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1021", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -2569,6 +4590,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2625,12 +4651,106 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "mitigation", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1022", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "mitigation", @@ -2659,6 +4779,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2697,7 +4822,7 @@ "completed_reductions": [], "component_id": "cmp-int", "current_disposition": "mitigated", - "current_pain": "N1", + "current_pain": "N0", "cve": "CVE-0000-1023", "cvss_vector": "CVSS:3.1/AV:L/AC:L/PR:H/UI:N/S:U/C:H/I:L/A:N", "days_since_first_detected": 0, @@ -2715,13 +4840,107 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "fully-mitigated", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 28 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "mitigated", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": true, - "historical_pain": "N1", + "group_due_at": null, + "group_id": "cve:CVE-0000-1023", + "group_irv": null, + "group_lev": null, + "group_pain": null, + "historical_pain": "N0", "internet_reachable": false, "irv_basis": "inventory", "is_blocking": false, @@ -2745,13 +4964,18 @@ "next_reduction": null, "occurrences": 1, "overdue_explanation": "documented suppression (mitigated); tracked as DEC-905", - "pain": "N1", + "pain": "N0", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", "irv": false, "lev": false, - "pain": "N1", + "pain": "N0", "reason": "first evaluation" } ], @@ -2802,12 +5026,106 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1024", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -2825,6 +5143,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2881,12 +5204,106 @@ "status": "lapsed" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1025", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -2904,6 +5321,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -2964,12 +5386,110 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-07-26T12:00:00+00:00", "first_evaluated": "2026-07-26T12:00:00+00:00", "fix_state": "not-fixed", "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-08-27T12:00:00+00:00", + "group_id": "cve:CVE-0000-1026", + "group_irv": false, + "group_lev": true, + "group_pain": "N3", "historical_pain": "N3", "internet_reachable": false, "irv_basis": "inventory", @@ -2987,6 +5507,11 @@ "occurrences": 1, "overdue_explanation": "past Class C SLA (100d > 32d from 2026-07-26T12:00:00+00:00 for PAIN=N3, LEV=True, IRV=False)", "pain": "N3", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3043,6 +5568,95 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", @@ -3051,6 +5665,11 @@ "2.0.0" ], "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1027", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3068,6 +5687,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3124,12 +5748,106 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1028", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3147,6 +5865,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3203,12 +5926,106 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1029", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3226,6 +6043,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3282,12 +6104,106 @@ "status": "invalid" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1030", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3305,6 +6221,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3361,12 +6282,106 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1031", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3384,6 +6399,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3410,6 +6430,7 @@ "acceptance_explanation": "not recorded", "accepted": true, "accepted_at": "2026-04-09T12:00:00+00:00", + "accepted_basis": "maximum-period-passed", "block_reason": null, "clock_basis": [ { @@ -3440,12 +6461,106 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 28 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "operational-requirement", "first_detected": "2025-09-29T12:00:00+00:00", "first_evaluated": "2025-09-29T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-02-04T12:00:00+00:00", + "group_id": "cve:CVE-0000-1032", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3463,6 +6578,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (operational-requirement); tracked as DEC-914", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3519,12 +6639,106 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1033", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3542,6 +6756,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3598,12 +6817,79 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "floor", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "fm-flag", + "likely-overdue-short", + "m-full", + "m-nirv", + "m-void", + "n5-block" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 6 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "false-positive", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "cve:CVE-0000-1034", + "group_irv": null, + "group_lev": null, + "group_pain": null, "historical_pain": "N3", "internet_reachable": true, "irv_basis": "inventory", @@ -3618,6 +6904,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (false-positive); tracked as DEC-915", "pain": "N3", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3674,12 +6965,106 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 28 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "operational-requirement", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1035", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -3697,6 +7082,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (operational-requirement); tracked as DEC-916", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3743,12 +7133,106 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1036", + "group_irv": false, + "group_lev": true, + "group_pain": "N3", "historical_pain": "N3", "internet_reachable": false, "irv_basis": "inventory", @@ -3766,6 +7250,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 32d allowed", "pain": "N3", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3822,12 +7311,74 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-ra-lapsed", + "c-ra-tfsec", + "cfg-noscore", + "informational", + "irv-inventory-true", + "no-impact" + ], + "mitigations": [], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": "default-after-evaluation-window", "final_disposition": "risk-accepted", "first_detected": "2026-08-05T12:00:00+00:00", "first_evaluated": "2026-08-05T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-09-22T12:00:00+00:00", + "group_id": "decision:DEC-007", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "inventory", @@ -3845,6 +7396,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (risk-accepted); tracked as DEC-007", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3901,12 +7457,74 @@ "status": "lapsed" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-ra", + "c-ra-tfsec", + "cfg-noscore", + "informational", + "irv-inventory-true", + "no-impact" + ], + "mitigations": [], + "open": 4 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-21T12:00:00+00:00", + "group_id": "decision:DEC-009", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "inventory", @@ -3924,6 +7542,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 48d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -3980,12 +7603,74 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-ra", + "c-ra-lapsed", + "cfg-noscore", + "informational", + "irv-inventory-true", + "no-impact" + ], + "mitigations": [], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "risk-accepted", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-12-21T12:00:00+00:00", + "group_id": "decision:DEC-010", + "group_irv": true, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": true, "irv_basis": "inventory", @@ -4003,6 +7688,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (risk-accepted); tracked as DEC-010", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4059,12 +7749,106 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 28 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "false-positive", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "decision:DEC-003", + "group_irv": null, + "group_lev": null, + "group_pain": null, "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4079,6 +7863,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (false-positive); tracked as DEC-003", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4135,12 +7924,106 @@ "status": "in-force" }, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-fp", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 28 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "false-positive", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "decision:DEC-004", + "group_irv": null, + "group_lev": null, + "group_pain": null, "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4155,6 +8038,11 @@ "occurrences": 1, "overdue_explanation": "documented suppression (false-positive); tracked as DEC-004", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4201,12 +8089,79 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "floor", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "fm-flag", + "likely-overdue-short", + "m-full", + "m-nirv", + "m-void", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-02T12:00:00+00:00", "first_evaluated": "2026-11-02T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-18T12:00:00+00:00", + "group_id": "cve:CVE-0000-1040", + "group_irv": true, + "group_lev": true, + "group_pain": "N3", "historical_pain": "N3", "internet_reachable": true, "irv_basis": "inventory", @@ -4224,6 +8179,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 1d of 16d allowed", "pain": "N3", + "public_disclosure": { + "level": "summary", + "policy": "DEC-182", + "reason": "open, IRV, LEV and N3: detail withheld from public artifacts until remediated or dispositioned; available to necessary parties from the provider" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4274,12 +8234,110 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-06-06T12:00:00+00:00", "first_evaluated": "2026-06-06T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-10-12T12:00:00+00:00", + "group_id": "cve:CVE-0000-1041", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4297,6 +8355,11 @@ "occurrences": 1, "overdue_explanation": "past Class C SLA (150d > 128d from 2026-06-06T12:00:00+00:00 for PAIN=N2, LEV=True, IRV=False)", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4323,6 +8386,7 @@ "acceptance_explanation": "not recorded", "accepted": true, "accepted_at": "2026-10-26T12:00:00+00:00", + "accepted_basis": "maximum-period-passed", "block_reason": "past Class C SLA (200d > 192d from 2026-04-17T12:00:00+00:00 for PAIN=N2, LEV=False, IRV=False)", "clock_basis": [ { @@ -4347,12 +8411,110 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-04-17T12:00:00+00:00", "first_evaluated": "2026-04-17T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-10-26T12:00:00+00:00", + "group_id": "cve:CVE-0000-1042", + "group_irv": false, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4370,6 +8532,11 @@ "occurrences": 1, "overdue_explanation": "past Class C SLA (200d > 192d from 2026-04-17T12:00:00+00:00 for PAIN=N2, LEV=False, IRV=False)", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4420,12 +8587,110 @@ "score": 0.01, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.01, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-04-29T12:00:00+00:00", "first_evaluated": "2026-04-29T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-07T12:00:00+00:00", + "group_id": "cve:CVE-0000-1043", + "group_irv": false, + "group_lev": false, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4443,6 +8708,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 188d of 192d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4489,12 +8759,79 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "floor", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [ + "floor", + "floor-v4", + "fm-flag", + "m-full", + "m-nirv", + "m-void", + "n5-block", + "r-fp" + ], + "mitigations": [ + "MIT-1", + "MIT-2" + ], + "open": 5 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-10-20T12:00:00+00:00", "first_evaluated": "2026-10-20T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1044", + "group_irv": true, + "group_lev": true, + "group_pain": "N3", "historical_pain": "N3", "internet_reachable": true, "irv_basis": "inventory", @@ -4512,6 +8849,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 14d of 16d allowed", "pain": "N3", + "public_disclosure": { + "level": "summary", + "policy": "DEC-182", + "reason": "open, IRV, LEV and N3: detail withheld from public artifacts until remediated or dispositioned; available to necessary parties from the provider" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4558,12 +8900,74 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": true, + "mission_criticality": "low" + }, + "detectability": { + "basis": "configuration in the public source repository", + "value": "public-source" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "no vector", + "gained": "unknown", + "required": "unknown" + }, + "proximate": { + "findings": [ + "c-ra", + "c-ra-lapsed", + "c-ra-tfsec", + "cfg-noscore", + "irv-inventory-true", + "no-impact" + ], + "mitigations": [], + "open": 4 + }, + "reachability": { + "basis": "inventory", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "check:checkov:CKV_AWS_995", + "group_irv": true, + "group_lev": true, + "group_pain": "N1", "historical_pain": "N1", "internet_reachable": true, "irv_basis": "inventory", @@ -4578,6 +8982,11 @@ "occurrences": 1, "overdue_explanation": "informational severity; no remediation SLA applies", "pain": "N1", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4624,12 +9033,106 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-09-04T12:00:00+00:00", "first_evaluated": "2026-09-04T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-01-10T12:00:00+00:00", + "group_id": "cve:CVE-0000-1045", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4647,6 +9150,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 60d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4693,12 +9201,106 @@ "description": "", "disposition_record": null, "epss": null, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "no-score", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": null, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 1, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "ovr-selector-miss", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": "evaluated-late", "final_disposition": "open", "first_detected": "2026-09-24T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2027-03-11T12:00:00+00:00", + "group_id": "cve:CVE-0000-1046", + "group_irv": false, + "group_lev": true, + "group_pain": "N2", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -4716,6 +9318,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4766,12 +9373,71 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "multi", + "data_sensitivity": "cui", + "internet_reachable": true, + "mission_criticality": "high" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "override", + "lev": false + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 3, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "unresolved", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1050", + "group_irv": true, + "group_lev": true, + "group_pain": "N5", "historical_pain": "N5", "internet_reachable": true, "irv_basis": "unresolved", @@ -4789,6 +9455,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 16d allowed", "pain": "N5", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4839,12 +9510,71 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "multi", + "data_sensitivity": "cui", + "internet_reachable": true, + "mission_criticality": "high" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 0, + "findings": 1, + "inventory_share": 0.0 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "unresolved", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1051", + "group_irv": true, + "group_lev": true, + "group_pain": "N5", "historical_pain": "N5", "internet_reachable": true, "irv_basis": "unresolved", @@ -4862,6 +9592,11 @@ "occurrences": 1, "overdue_explanation": "N5+LEV+IRV (most-severe Class C tier)", "pain": "N5", + "public_disclosure": { + "level": "summary", + "policy": "DEC-182", + "reason": "open, IRV, LEV and N5: detail withheld from public artifacts until remediated or dispositioned; available to necessary parties from the provider" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4912,12 +9647,71 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "multi", + "data_sensitivity": "cui", + "internet_reachable": true, + "mission_criticality": "high" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 3, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "unresolved", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1050", + "group_irv": true, + "group_lev": true, + "group_pain": "N5", "historical_pain": "N5", "internet_reachable": true, "irv_basis": "unresolved", @@ -4935,6 +9729,11 @@ "occurrences": 1, "overdue_explanation": "N5+LEV+IRV (most-severe Class C tier)", "pain": "N5", + "public_disclosure": { + "level": "summary", + "policy": "DEC-182", + "reason": "open, IRV, LEV and N5: detail withheld from public artifacts until remediated or dispositioned; available to necessary parties from the provider" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -4985,12 +9784,71 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "multi", + "data_sensitivity": "cui", + "internet_reachable": true, + "mission_criticality": "high" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "known-exploited", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it; exploited elsewhere (CISA KEV), which the policy treats as likely exploitable (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": true + }, + "prevalence": { + "components": 0, + "findings": 1, + "inventory_share": 0.0 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "unresolved", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1052", + "group_irv": true, + "group_lev": true, + "group_pain": "N5", "historical_pain": "N5", "internet_reachable": true, "irv_basis": "unresolved", @@ -5008,6 +9866,11 @@ "occurrences": 1, "overdue_explanation": "KEV (CISA Known Exploited Vulnerability)", "pain": "N5", + "public_disclosure": { + "level": "summary", + "policy": "DEC-182", + "reason": "open, IRV, LEV and N5: detail withheld from public artifacts until remediated or dispositioned; available to necessary parties from the provider" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -5058,12 +9921,110 @@ "score": 0.5, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "single", + "data_sensitivity": "public", + "internet_reachable": false, + "mission_criticality": "low" + }, + "detectability": { + "basis": "a published advisory names the package; the component is not internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "prediction", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.5, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 1, + "findings": 3, + "inventory_share": 0.2 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "high" + }, + "proximate": { + "findings": [ + "c-fp", + "c-fp-norecord", + "detected-earlier", + "floor-nirv", + "irv-inventory-false", + "legacy-id", + "likely-overdue", + "m-nlev", + "m-nlev-kev", + "mav", + "no-score", + "observed", + "overdue", + "pred-at", + "pred-hi", + "pred-lo", + "r-accepted-noref", + "r-bad-date", + "r-config-ignored", + "r-fp-control", + "r-investigating", + "r-lapsed", + "r-mitigated-full", + "r-mitigated-none", + "r-mitigated-ok", + "r-mitigated-other", + "r-no-review-by", + "r-oprq", + "r-unknown-term", + "r-vendor-fixed", + "r-vendor-ok", + "r-vendor-unknown" + ], + "mitigations": [ + "MIT-3", + "MIT-4", + "MIT-6", + "MIT-7", + "MIT-8" + ], + "open": 27 + }, + "reachability": { + "basis": "inventory", + "irv": false + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": "2026-11-05T12:00:00+00:00", + "group_id": "cve:CVE-0000-1050", + "group_irv": true, + "group_lev": true, + "group_pain": "N5", "historical_pain": "N2", "internet_reachable": false, "irv_basis": "inventory", @@ -5081,6 +10042,11 @@ "occurrences": 1, "overdue_explanation": "within Class C SLA: 0d of 128d allowed", "pain": "N2", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -5131,12 +10097,71 @@ "score": 0.0, "score_date": "2026-11-03T00:00:00Z" }, + "evaluation": { + "automatable": { + "evidence": null, + "source": "VER-EVA-AIA default", + "value": "yes" + }, + "criticality": { + "agency_scope": "multi", + "data_sensitivity": "cui", + "internet_reachable": true, + "mission_criticality": "high" + }, + "detectability": { + "basis": "a published advisory names the package; the component is internet-reachable", + "value": "public-advisory" + }, + "exploitability": { + "basis": "override", + "lev": true + }, + "exploitation": { + "assessed_at": "2026-11-03T12:00:00+00:00", + "basis": "no incident record names it (DEC-181)", + "value": "none-observed" + }, + "known_threats": { + "epss": { + "model_version": "v2026.06.15", + "score": 0.0, + "score_date": "2026-11-03T00:00:00Z" + }, + "kev": false + }, + "prevalence": { + "components": 0, + "findings": 1, + "inventory_share": 0.0 + }, + "privilege": { + "basis": "CVSS PR/S", + "gained": "within-component", + "required": "none" + }, + "proximate": { + "findings": [], + "mitigations": [], + "open": 0 + }, + "reachability": { + "basis": "unresolved", + "irv": true + } + }, + "evaluation_late": null, "final_disposition": "open", "first_detected": "2026-11-03T12:00:00+00:00", "first_evaluated": "2026-11-03T12:00:00+00:00", "fix_state": null, "fix_versions": null, "fully_mitigated": false, + "group_due_at": null, + "group_id": "cve:CVE-0000-1053", + "group_irv": true, + "group_lev": true, + "group_pain": "N1", "historical_pain": "N1", "internet_reachable": true, "irv_basis": "unresolved", @@ -5151,6 +10176,11 @@ "occurrences": 1, "overdue_explanation": "informational severity; no remediation SLA applies", "pain": "N1", + "public_disclosure": { + "level": "full", + "policy": "DEC-182", + "reason": "not an open IRV+LEV finding at N3 or above: disclosing it will not likely lead to exploitation" + }, "rating_history": [ { "at": "2026-11-03T12:00:00+00:00", @@ -5174,6 +10204,1010 @@ "will_be_overdue": false } ], + "groups": [ + { + "basis": "check", + "components": [ + "cmp-app" + ], + "dispositions": [ + "open" + ], + "group_id": "check:checkov:CKV_AWS_995", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "informational" + ], + "open_members": 1, + "pain": "N1", + "remediation_due_at": null + }, + { + "basis": "check", + "components": [ + "cmp-app" + ], + "dispositions": [ + "open" + ], + "group_id": "check:checkov:CKV_AWS_996", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "irv-inventory-true" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-12-21T12:00:00+00:00" + }, + { + "basis": "check", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "check:checkov:CKV_AWS_997", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "irv-inventory-false" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "check", + "components": [ + "cmp-app" + ], + "dispositions": [ + "open" + ], + "group_id": "check:checkov:CKV_AWS_998", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "cfg-noscore" + ], + "open_members": 1, + "pain": "N1", + "remediation_due_at": null + }, + { + "basis": "check", + "components": [ + "cmp-app" + ], + "dispositions": [ + "open" + ], + "group_id": "check:checkov:CKV_AWS_999", + "internet_reachable": true, + "likely_exploitable": false, + "members": [ + "no-impact" + ], + "open_members": 1, + "pain": "N1", + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-pkg" + ], + "dispositions": [ + "false-positive" + ], + "group_id": "cve:CVE-0000-1001", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "kev-fp" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1002", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "floor" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-12-21T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1003", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "floor-v4" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-12-21T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1004", + "internet_reachable": false, + "likely_exploitable": false, + "members": [ + "floor-nirv" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-05-14T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1005", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "pred-hi" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1006", + "internet_reachable": false, + "likely_exploitable": false, + "members": [ + "pred-lo" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-05-14T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1007", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "pred-at" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1008", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "no-score" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1009", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "observed" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1010", + "internet_reachable": true, + "likely_exploitable": false, + "members": [ + "irv-unresolved" + ], + "open_members": 1, + "pain": "N5", + "remediation_due_at": "2026-11-19T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pkg" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1011", + "internet_reachable": true, + "likely_exploitable": false, + "members": [ + "irv-package" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-05-14T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "mitigated" + ], + "group_id": "cve:CVE-0000-1012", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "m-full" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1013", + "internet_reachable": false, + "likely_exploitable": false, + "members": [ + "m-nirv" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-05-14T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1014", + "internet_reachable": false, + "likely_exploitable": false, + "members": [ + "m-nlev" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-05-14T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1015", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "m-nlev-kev" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1016", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "m-void" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-11-21T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "mitigated" + ], + "group_id": "cve:CVE-0000-1017", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "fm-flag" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1020", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-mitigated-ok" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1021", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-mitigated-none" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1022", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-mitigated-other" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "mitigated" + ], + "group_id": "cve:CVE-0000-1023", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "r-mitigated-full" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1024", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-fp-control" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1025", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-lapsed" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1026", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-vendor-ok" + ], + "open_members": 1, + "pain": "N3", + "remediation_due_at": "2026-08-27T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1027", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-vendor-fixed" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1028", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-vendor-unknown" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1029", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-unknown-term" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1030", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-bad-date" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1031", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-investigating" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "operational-requirement" + ], + "group_id": "cve:CVE-0000-1032", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-oprq" + ], + "open_members": 0, + "pain": "N2", + "remediation_due_at": "2026-02-04T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1033", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-accepted-noref" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "false-positive" + ], + "group_id": "cve:CVE-0000-1034", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "r-fp" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "operational-requirement" + ], + "group_id": "cve:CVE-0000-1035", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-no-review-by" + ], + "open_members": 0, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1036", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "r-config-ignored" + ], + "open_members": 1, + "pain": "N3", + "remediation_due_at": "2026-12-05T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1040", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "n5-block" + ], + "open_members": 1, + "pain": "N3", + "remediation_due_at": "2026-11-18T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1041", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "overdue" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-10-12T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1042", + "internet_reachable": false, + "likely_exploitable": false, + "members": [ + "mav" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-10-26T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1043", + "internet_reachable": false, + "likely_exploitable": false, + "members": [ + "likely-overdue" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-11-07T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-pub" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1044", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "likely-overdue-short" + ], + "open_members": 1, + "pain": "N3", + "remediation_due_at": "2026-11-05T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1045", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "legacy-id" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-01-10T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1046", + "internet_reachable": false, + "likely_exploitable": true, + "members": [ + "detected-earlier" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2027-03-11T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [ + "cmp-int" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1050", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "ovr", + "ovr-excluded", + "ovr-selector-miss" + ], + "open_members": 3, + "pain": "N5", + "remediation_due_at": "2026-11-05T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1051", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "ovr-lapsed" + ], + "open_members": 1, + "pain": "N5", + "remediation_due_at": "2026-11-05T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1052", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "ovr-kev" + ], + "open_members": 1, + "pain": "N5", + "remediation_due_at": "2026-11-05T12:00:00+00:00" + }, + { + "basis": "cve", + "components": [], + "dispositions": [ + "open" + ], + "group_id": "cve:CVE-0000-1053", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "ovr-raise" + ], + "open_members": 1, + "pain": "N1", + "remediation_due_at": null + }, + { + "basis": "cve", + "components": [ + "cmp-edge" + ], + "dispositions": [ + "open" + ], + "group_id": "cve:ZAP-1", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "irv-observed" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-12-21T12:00:00+00:00" + }, + { + "basis": "decision", + "components": [ + "cmp-int" + ], + "dispositions": [ + "false-positive" + ], + "group_id": "decision:DEC-003", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "c-fp" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "decision", + "components": [ + "cmp-int" + ], + "dispositions": [ + "false-positive" + ], + "group_id": "decision:DEC-004", + "internet_reachable": null, + "likely_exploitable": null, + "members": [ + "c-fp-norecord" + ], + "open_members": 0, + "pain": null, + "remediation_due_at": null + }, + { + "basis": "decision", + "components": [ + "cmp-app" + ], + "dispositions": [ + "risk-accepted" + ], + "group_id": "decision:DEC-007", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "c-ra" + ], + "open_members": 0, + "pain": "N2", + "remediation_due_at": "2026-09-22T12:00:00+00:00" + }, + { + "basis": "decision", + "components": [ + "cmp-app" + ], + "dispositions": [ + "open" + ], + "group_id": "decision:DEC-009", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "c-ra-lapsed" + ], + "open_members": 1, + "pain": "N2", + "remediation_due_at": "2026-12-21T12:00:00+00:00" + }, + { + "basis": "decision", + "components": [ + "cmp-app" + ], + "dispositions": [ + "risk-accepted" + ], + "group_id": "decision:DEC-010", + "internet_reachable": true, + "likely_exploitable": true, + "members": [ + "c-ra-tfsec" + ], + "open_members": 0, + "pain": "N2", + "remediation_due_at": "2026-12-21T12:00:00+00:00" + } + ], "impact_level": "Moderate", "ksi_signal_ref": "/.well-known/ksi-signal.json", "report_version": "2.0.0", @@ -5239,6 +11273,7 @@ "summary": { "blocking": 8, "by_pain": { + "N0": 0, "N1": 4, "N2": 34, "N3": 4, diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 6c7aa324..be659971 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -37,6 +37,7 @@ ("unknown-field.json", "unknown field(s) ['reviewby']"), ("planned-reduction-malformed.json", "planned_reduction must be {date: YYYY-MM-DD, target_pain: N0-N5}"), ("planned-reduction-closed.json", "only an in-force record plans a reduction"), + ("automatable-no-evidence.json", "'no' needs its evidence"), ] diff --git a/tests/test_dispositions_mitigations.py b/tests/test_dispositions_mitigations.py index f5b729f0..bf029417 100644 --- a/tests/test_dispositions_mitigations.py +++ b/tests/test_dispositions_mitigations.py @@ -143,7 +143,9 @@ def test_t15_control_protected_finding_is_fully_mitigated_at_the_lowest_level(): assert r["current_disposition"] == "mitigated" assert r["fully_mitigated"] is True assert r["likely_exploitable"] is False and r["lev_basis"] == "fully-mitigated" - assert r["current_pain"] == vdr._rules.impact_levels()[0] + # VER-EVA-EPA's lowest level, N0 since October 2026: an impact removed entirely. + # (The timeframe table, VDR-TFR-PVR, starts at N1; N0 has no timeframe.) + assert r["current_pain"] == "N0" assert r["remediation_due_at"] is None and r["is_blocking"] is False From e390c0a6853ccc9eed9ec5507acaf6a9dea70690 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:51:47 -0500 Subject: [PATCH 24/97] feat(mas): Minimum Assessment Scope generated and gated on every deploy (MAS-CSO-IIR/FLO/MDI/TPR, FRC-CSX-MAS) - Inventory: the app's browser origins (jsDelivr, Google Cloud Storage) and the VDR's remaining feeds (FIRST EPSS, Grype database) become external services; the CloudTrail trail joins the logging KSIs' evidence domain. - scripts/build-mas.py publishes /.well-known/minimum-assessment-scope.json (+ .md, signed, freshness-governed): information resources, every CSP off-origin host placed or out of scope with a reason, flows with a high-water-mark security category, a metadata inventory, the third-party records with FedRAMP's four fields, and the KSI-to-scope application. It fails the deploy, before the SDR, on any gap. - Classification: 17 new flows covering every external service and data store; an app-delivery group (and its service in the service model). - Third-party records: the four MAS-CSO-TPR fields on all 11; decision fields AI-drafted for the operator to confirm; a decided record without them fails. - scripts/_tpr.py: the CPO's thirdPartyInformationResources, for the CPO hook. - Five rule records flip to Implemented with resolving evidence. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-with-opa.yml | 21 + data/frr-records.json | 153 +- data/minimum-assessment-scope.json | 36 + data/third-party-components.json | 209 +- docs/boundary/boundary-classification.json | 40 +- docs/portfolio/service-model.json | 8 + infrastructure/schemas/freshness-config.json | 11 + scripts/_tpr.py | 40 + scripts/build-ksi-signal.py | 59 +- scripts/build-mas.py | 424 ++++ tests/fixtures/mas/classification-gaps.json | 6 + tests/fixtures/mas/csp-unplaced.tf.fixture | 11 + tests/fixtures/mas/csp-unresolved.tf.fixture | 11 + tests/fixtures/mas/signal-gaps.json | 11 + tests/fixtures/mas/signal.json | 2254 +++++++++++++++++ .../decided-without-tpr-fields.json | 19 + tests/fixtures/third-party/decided.json | 28 +- tests/test_mas.py | 234 ++ 18 files changed, 3501 insertions(+), 74 deletions(-) create mode 100644 data/minimum-assessment-scope.json create mode 100644 scripts/_tpr.py create mode 100644 scripts/build-mas.py create mode 100644 tests/fixtures/mas/classification-gaps.json create mode 100644 tests/fixtures/mas/csp-unplaced.tf.fixture create mode 100644 tests/fixtures/mas/csp-unresolved.tf.fixture create mode 100644 tests/fixtures/mas/signal-gaps.json create mode 100644 tests/fixtures/mas/signal.json create mode 100644 tests/fixtures/third-party/decided-without-tpr-fields.json create mode 100644 tests/test_mas.py diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 24a24847..3faa6698 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1587,6 +1587,22 @@ jobs: --previous previous-availability.json --ksi-signal ksi-signal.json python3 ../scripts/build-policy-index.py --output policy-index.json --md-output policy-index.md + # The Minimum Assessment Scope (MAS-CSO-IIR/FLO/MDI/TPR, FRC-CSX-MAS), generated + # from this build's inventory, the boundary classification's flows, the CSPs + # the edge serves and the third-party records. Fails the deploy, before the + # SDR, on an unplaced off-origin host, an external service or data store in + # no flow, a data store without a metadata entry, an undecided third-party + # record, or a component type no KSI covers. Signed and published with the + # rest after the gates. + - name: Build the Minimum Assessment Scope + working-directory: infrastructure + env: + GITHUB_SHA: ${{ github.sha }} + run: | + python3 ../scripts/build-mas.py --ksi-signal ksi-signal.json --commit "$GITHUB_SHA" \ + --output minimum-assessment-scope.json --md-output minimum-assessment-scope.md + cosign sign-blob --yes --new-bundle-format --bundle minimum-assessment-scope.bundle minimum-assessment-scope.json + - name: Build Security Decision Record (SDR — 20x SSP replacement) working-directory: infrastructure run: | @@ -1733,6 +1749,7 @@ jobs: [ -f "$a.json" ] || continue files="$files $a.json" done + files="$files minimum-assessment-scope.json minimum-assessment-scope.md" # shellcheck disable=SC2086 # a word list by design python3 ../scripts/check-published-redaction.py $files @@ -1795,6 +1812,10 @@ jobs: aws s3 cp "$a.json" "s3://$BUCKET_NAME/.well-known/$a.json" --content-type application/json --cache-control "public, max-age=300" aws s3 cp "$a.bundle" "s3://$BUCKET_NAME/.well-known/$a.bundle" --content-type application/json --cache-control "public, max-age=300" done + # Minimum Assessment Scope (MAS-CSO-IIR/FLO/MDI/TPR) + bundle + human-readable + aws s3 cp minimum-assessment-scope.json "s3://$BUCKET_NAME/.well-known/minimum-assessment-scope.json" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp minimum-assessment-scope.bundle "s3://$BUCKET_NAME/.well-known/minimum-assessment-scope.bundle" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp minimum-assessment-scope.md "s3://$BUCKET_NAME/.well-known/minimum-assessment-scope.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" echo "✅ All signed artifacts published to /.well-known/ (post-gate)" # ------------------------------------------------------------------------- diff --git a/data/frr-records.json b/data/frr-records.json index 6a767bed..a8c73644 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -628,12 +628,29 @@ }, { "frr_id": "FRC-CSX-MAS", - "status": "Partially Implemented", - "implementation": "The KSI signal applies the KSIs to the inventoried components (ksi-signal.json), derived from Terraform state.", - "evidence": [], - "gap": "The inventory misses external services the site loads from (cdn.jsdelivr.net, storage.googleapis.com) and some flows, and human-process KSIs are not applied per component; MAS-CSO-IIR and MAS-CSO-FLO carry the detail.", - "closes_with": [ - "W10" + "status": "Implemented", + "implementation": "Every KSI is applied to the whole Minimum Assessment Scope. The inventory now includes the external services the app and the VDR depend on (MAS-CSO-IIR), and each KSI's evidence domain names the component types it covers. KSIs with a per-component domain apply to every component of those types; inventory and documentation KSIs (PIY, CED) apply over the whole set. scripts/build-mas.py publishes the application per KSI and per component type in .well-known/minimum-assessment-scope.json, and fails the deploy when a component type is in no KSI's per-component domain and has no stated whole-set reason (static pages and the state lock table carry one).", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_mas.py::test_every_ksi_applies_and_every_component_type_is_covered" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_component_type_in_no_ksi_domain_fails" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_audit_trail_is_in_the_logging_ksi_domain" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build the Minimum Assessment Scope" + }, + { + "kind": "artifact", + "ref": ".well-known/minimum-assessment-scope.json" + } ] }, { @@ -816,32 +833,87 @@ }, { "frr_id": "MAS-CSO-FLO", - "status": "Partially Implemented", - "implementation": "https://samaydlette.com/.well-known/boundary-map.json: 11 flows with protocol/auth/encryption/data, 137 edges derived from Terraform, per-node classification; flows declared in docs/boundary/boundary-classification.json and resolved against the inventory (unmatched endpoints check = 0); code scripts/build-boundary-graph.py; human view on /trust/ and authorization-boundary page.", - "evidence": [], - "gap": "Flows are not complete for ALL resources: missing user→Cognito Hosted UI authentication, browser→jsDelivr / storage.googleapis.com, Lambda/app→DynamoDB/SQS, CloudTrail/S3/CloudFront→log bucket, operator→AWS console (Duo/root MFA), nightly→GitHub Advisory DB / CISA KEV / detection feeds. Security categories are per component (classification) but flows carry no category. To close: Add the missing flows to the classification, and add a check that every inventoried external service and every data store appears in at least one flow; derive per-flow security category from its endpoints.", - "closes_with": [ - "W10" + "status": "Implemented", + "implementation": "The boundary classification's data-flow table (docs/boundary/boundary-classification.json) covers every external service and every data store or log in the inventory, including sign-in to the Cognito Hosted UI, the browser loads from jsDelivr and Google Cloud Storage, log delivery to the access-log and CloudTrail buckets and the log groups, the dead-letter queue, operator administration with MFA, Terraform state and lock, and the vulnerability feeds. scripts/build-mas.py resolves each flow against this build's inventory, gives each the high-water mark of its endpoints' FIPS 199 security categories, and fails the deploy when a flow names a component the inventory lacks or an external service or data store is in no flow. Published in .well-known/minimum-assessment-scope.json (flows with categories, protocol, authentication, encryption and data) and drawn on the boundary map.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_mas.py::test_flow_gaps_fail" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_every_external_service_and_data_store_in_the_inventory_is_in_a_flow" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_flow_category_is_the_high_water_mark_of_its_endpoints" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build the Minimum Assessment Scope" + }, + { + "kind": "artifact", + "ref": ".well-known/minimum-assessment-scope.json" + }, + { + "kind": "artifact", + "ref": ".well-known/boundary-map.json" + } ] }, { "frr_id": "MAS-CSO-IIR", - "status": "Partially Implemented", - "implementation": "Machine-readable: https://samaydlette.com/.well-known/ksi-signal.json components[] (214, from Terraform state + SBOMs) and iiw.csv; human-readable derivation: docs/ksi-signal.md and research/authorization-boundary.html; code: scripts/build-ksi-signal.py; reconcile invariant (a) 'every live in-boundary resource is in the inventory' held.", - "evidence": [], - "gap": "Incomplete: the Silk Reeling SPA loads MediaPipe code from cdn.jsdelivr.net and models from storage.googleapis.com and talks to Cognito Hosted UI (bootstrap/cloudfront.tf:185-186, 378-380); jsDelivr and Google Cloud Storage are not inventoried as information resources although they run code next to the camera feed. The npm package is listed, the delivery services are not. (Repo publicity for the 'code' artifact not verified from here.) To close: Derive external services from the CSP/connect-src allowlists and runtime egress as well as Terraform, so every origin the app loads code or data from becomes an inventoried component; or self-host MediaPipe assets and models.", - "closes_with": [ - "W10" + "status": "Implemented", + "implementation": "Every deploy publishes the set of information resources that make up the offering: the canonical inventory (.well-known/ksi-signal.json, derived from Terraform state, the SBOMs, the site's HTML and the declared external services; reconcile invariant (a) holds every live in-boundary resource to it) and its summary in .well-known/minimum-assessment-scope.json with a human-readable twin and the derivation. The external services include every origin the Silk Reeling app loads code or data from in the user's browser (jsDelivr for the MediaPipe runtime, Google Cloud Storage for the pose model) and every vulnerability feed the VDR reads (GitHub Advisory Database, CISA KEV, FIRST EPSS, the Grype database). scripts/build-mas.py parses the Content Security Policies the edge serves (infrastructure/bootstrap/cloudfront.tf) and fails the deploy, before the SDR is built, when an allowed off-origin host is neither an inventoried component nor out of scope with a stated reason; the podcast audio hosts (media only, no code or data from this system) are out of scope with that reason.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_mas.py::test_unplaced_csp_host_fails" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_unresolvable_csp_source_fails_closed" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_the_app_origins_are_inventoried_external_services" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build the Minimum Assessment Scope" + }, + { + "kind": "artifact", + "ref": ".well-known/minimum-assessment-scope.json" + }, + { + "kind": "artifact", + "ref": ".well-known/ksi-signal.json" + } ] }, { "frr_id": "MAS-CSO-MDI", - "status": "Not Implemented", - "implementation": "Checked: no metadata inventory; CPO says 'MAS-CSO-MDI N/A (no federal customer data)'.", - "evidence": [], - "gap": "No machine-readable output, explanation or code covering metadata the offering collects: Cognito user records (email, MFA), CloudFront/S3/API Gateway/Lambda logs (IP addresses, user agents), CloudTrail, DynamoDB items, SNS subscriptions. To close: Classify each data store/log as metadata in the inventory (type, contents, retention, location), emit a metadata section from build-ksi-signal.py, and document the derivation.", - "closes_with": [ - "W10" + "status": "Implemented", + "implementation": "The metadata the offering collects is in scope and inventoried: every data store and log in the inventory (buckets, log groups, the CloudTrail trail, the state lock table, the dead-letter queue, the Cognito user pool, the alert topic) and the request metadata third parties see when the app loads code or models (jsDelivr, Google Cloud Storage) has an entry stating what it holds, which personal data (for example IP addresses, user agents, the app users' email addresses), its retention and its location. scripts/build-mas.py joins data/minimum-assessment-scope.json with this build's inventory and fails the deploy when a data store or log has no entry; a test holds each stated retention equal to the Terraform. Published, with the explanation of its derivation, in .well-known/minimum-assessment-scope.json and its Markdown twin.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_mas.py::test_data_store_without_metadata_fails" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_metadata_retention_matches_the_terraform" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build the Minimum Assessment Scope" + }, + { + "kind": "artifact", + "ref": ".well-known/minimum-assessment-scope.json" + } ] }, { @@ -854,12 +926,33 @@ }, { "frr_id": "MAS-CSO-TPR", - "status": "Partially Implemented", - "implementation": "ksi-signal.json external_service components (7) carry purpose, cia_impact, verification_mechanism, baseline_configuration; data/third-party-components.json holds responsibility split records; build-ksi-signal.py:1322-1345.", - "evidence": [], - "gap": "All 7 third-party records are undecided (decided_by/decided_on null), so the published upstream_offering/responsibility_split read 'not recorded'. No explicit justification-for-use, mitigation-measures or compensating-controls fields per resource. jsDelivr and Google Cloud Storage (MediaPipe) are missing entirely. Not included in the CPO (schema thirdPartyInformationResources). To close: Add the four required fields (usage/configuration, justification, mitigations, compensating controls) to the record schema, have the operator decide all records, add the MediaPipe hosts, and emit them into the CPO's thirdPartyInformationResources.", - "closes_with": [ - "W10" + "status": "Implemented", + "implementation": "Each of the 11 third-party information resources (Sigstore Fulcio and Rekor, GitHub repository and Actions, GitHub OIDC, the operator's MFA authenticator, the GitHub Advisory Database, CISA KEV, FIRST EPSS, the Grype database, jsDelivr, Google Cloud Storage) has a decided record in data/third-party-components.json with its general usage and configuration, justification for use, mitigation measures and compensating controls. The inventory builder fails when a decided record lacks any of the four, and scripts/build-mas.py fails the deploy when an inventoried external service has no decided record. Machine-readable output: the external_service components in .well-known/ksi-signal.json and the third-party section of .well-known/minimum-assessment-scope.json; the human-readable derivation is in its Markdown twin; the code is scripts/build-ksi-signal.py and scripts/build-mas.py.", + "evidence": [ + { + "kind": "test", + "ref": "tests/test_mas.py::test_decided_third_party_record_without_the_four_fields_fails_the_inventory" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_every_third_party_resource_is_decided_with_the_four_fields" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_published_inventory_carries_the_four_fields" + }, + { + "kind": "test", + "ref": "tests/test_mas.py::test_cpo_helper_matches_the_cpo_schema" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Build the Minimum Assessment Scope" + }, + { + "kind": "artifact", + "ref": ".well-known/minimum-assessment-scope.json" + } ] }, { diff --git a/data/minimum-assessment-scope.json b/data/minimum-assessment-scope.json new file mode 100644 index 00000000..ca7f0d8b --- /dev/null +++ b/data/minimum-assessment-scope.json @@ -0,0 +1,36 @@ +{ + "schema": "mas-inputs/1", + "$comment": "Operator-owned inputs to scripts/build-mas.py, which joins them with the canonical inventory (ksi-signal.json), the boundary classification and the third-party records to produce /.well-known/minimum-assessment-scope.json (MAS-CSO-IIR, MAS-CSO-FLO, MAS-CSO-MDI, MAS-CSO-TPR, FRC-CSX-MAS). The builder fails the deploy when an off-origin host in a CSP is not placed here, when a data store or log in the inventory has no metadata entry, or when a component type is in no KSI evidence domain and has no whole-set reason. AI-drafted 2026-11-05 from the Terraform and the CSPs, for the operator's review.", + "off_origin_hosts": { + "cdn.jsdelivr.net": {"component": "ext::jsdelivr"}, + "storage.googleapis.com": {"component": "ext::google-cloud-storage"}, + "samaydlette-com-silk-reeling.auth.us-east-2.amazoncognito.com": {"component": "aws::identity_provider::silk_reeling"}, + "ochelli.com": {"out_of_scope": "Podcast audio on the activities page (CSP media-src only): the reader's browser streams a public episode directly from the host. Media cannot run code in the page, the host receives nothing from this system, and it is unrelated to the Silk Reeling app or the published evidence, so it neither handles nor affects federal customer data."}, + "traffic.libsyn.com": {"out_of_scope": "Podcast audio redirect host (CSP media-src only), as for ochelli.com: public episode audio streamed directly by the reader's browser; no code, no data from this system."}, + "content.libsyn.com": {"out_of_scope": "Podcast audio file host behind traffic.libsyn.com (CSP media-src only), as for ochelli.com."} + }, + "data_store_types": ["object_store", "log_group", "kv_table", "message_queue", "identity_provider", "notification_topic", "secrets_manager", "audit_log_trail"], + "metadata": { + "aws::object_store::website": {"holds": "Site content and the published /.well-known/ evidence", "personal_data": [], "retention": "Current objects kept; replaced versions expire after 90 days", "retention_days": 90}, + "aws::object_store::logs": {"holds": "S3 server access logs for the site bucket (origin reads)", "personal_data": ["requester IP address", "user agent"], "retention": "400 days, then expired", "retention_days": 400}, + "aws::object_store::cloudtrail": {"holds": "CloudTrail management-event log files", "personal_data": ["operator source IP address", "IAM principal names", "user agent"], "retention": "913 days (moved to infrequent access after 365)", "retention_days": 913}, + "aws::object_store::tfstate": {"holds": "Terraform state: resource configuration and attributes, including secret values", "personal_data": [], "retention": "Current state kept; replaced versions expire after 90 days", "retention_days": 90}, + "aws::kv_table::tflock": {"holds": "Terraform state lock: lock ID, holder and timestamp", "personal_data": [], "retention": "Deleted when the lock is released"}, + "aws::log_group::opa_compliance": {"holds": "Runtime KSI emitter run logs", "personal_data": [], "retention": "365 days", "retention_days": 365}, + "aws::log_group::evidence_watchdog": {"holds": "Evidence-freshness watchdog run logs (artifact ages)", "personal_data": [], "retention": "365 days", "retention_days": 365}, + "aws::log_group::silk_reeling": {"holds": "Silk Reeling app Lambda runtime logs: invocations, errors", "personal_data": [], "retention": "365 days", "retention_days": 365}, + "aws::log_group::silk_apigw": {"holds": "Silk Reeling API access log: request ID, source IP, time, method, route, status, response length", "personal_data": ["source IP address"], "retention": "365 days", "retention_days": 365}, + "aws::log_group::route53_query_log": {"holds": "Route 53 DNS query logs (created only when DNS query logging is enabled)", "personal_data": ["resolver IP address"], "retention": "365 days", "retention_days": 365}, + "aws::message_queue::compliance_dlq": {"holds": "Scheduled events whose emitter invocation failed", "personal_data": [], "retention": "14 days", "retention_days": 14}, + "aws::identity_provider::silk_reeling": {"holds": "App user records: email address, TOTP enrollment, sign-in events", "personal_data": ["email address", "sign-in IP address and device data"], "retention": "Until the operator deletes the user"}, + "aws::notification_topic::evidence_alerts": {"holds": "The operator's email subscription and in-flight alarm notifications", "personal_data": ["operator email address (subscription endpoint, set out of band; never in Terraform or published)"], "retention": "Until the subscription is removed"}, + "aws::audit_log_trail::management": {"holds": "Management events for the account, delivered to the CloudTrail bucket (flow P)", "personal_data": ["operator source IP address", "IAM principal names"], "retention": "As the CloudTrail bucket (913 days)", "retention_days": 913}, + "aws::secrets_manager::*": {"holds": "Application secrets (none today)", "personal_data": [], "retention": "Until deleted, then a 7 to 30 day recovery window"}, + "ext::jsdelivr": {"holds": "Third-party observation: jsDelivr's request logs when a user's browser fetches the MediaPipe runtime", "personal_data": ["user IP address", "user agent"], "retention": "Set by the third party; not controlled by this provider"}, + "ext::google-cloud-storage": {"holds": "Third-party observation: Google's request logs when a user's browser fetches the pose model", "personal_data": ["user IP address", "user agent"], "retention": "Set by the third party; not controlled by this provider"} + }, + "whole_set_types": { + "html_artifact": "Static pages are content of the website bucket: the whole-inventory KSIs (PIY) cover them, and every page is sha256-hashed into the inventory so any change is a recorded change.", + "kv_table": "The state lock table holds lock metadata only; it is covered by the whole-inventory KSIs (PIY) and by the trust-root review of the bootstrap stack." + } +} diff --git a/data/third-party-components.json b/data/third-party-components.json index ff41a57a..51b6dedf 100644 --- a/data/third-party-components.json +++ b/data/third-party-components.json @@ -1,86 +1,233 @@ { - "_note": "REQ-TPR-1 records for the external services the inventory declares (scripts/build-ksi-signal.py build_external_components). Each names the upstream offering, the upstream component's identifier, and the split between what the upstream runs and what this provider configures. A record is published only once it carries decided_by and decided_on; until then the inventory publishes every field as \"not recorded\". A record for a service the inventory does not declare fails the build. AI-drafted entries (drafted_by) are proposals from the evidence listed; none is in force until the operator records a decision. A null value is unknown and is published as \"not recorded\" even after a decision.", + "_note": "REQ-TPR-1 and MAS-CSO-TPR records for the external services the inventory declares (scripts/build-ksi-signal.py build_external_components). Each names the upstream offering, the upstream component's identifier, the split between what the upstream runs and what this provider configures, and the four things FedRAMP requires documented for each third-party information resource (MAS-CSO-TPR): general usage and configuration, justification for use, mitigation measures, and compensating controls. A record is published only once it carries decided_by and decided_on, and a decided record must carry all four MAS-CSO-TPR fields or the build fails; until a decision, the inventory publishes every field as \"not recorded\". A record for a service the inventory does not declare fails the build. AI-drafted entries (drafted_by) were proposed from the evidence listed; the decision fields on the 2026-11-05 drafts are for the operator to confirm. A null value is unknown and is published as \"not recorded\" even after a decision.", "components": { "ext::sigstore-fulcio": { + "provider": "OpenSSF Sigstore project (public-good instance)", + "website": "https://www.sigstore.dev/", "upstream_offering": "Sigstore public-good instance: Fulcio certificate authority", "upstream_component_id": "https://fulcio.sigstore.dev", "responsibility_split": { "upstream_runs": "The certificate authority that issues short-lived signing certificates bound to the CI run's OIDC identity.", "provider_configures": "Keyless signing in CI (cosign sign-blob), and the certificate identity and OIDC issuer every verification pins." }, - "evidence": [".github/workflows/evidence-nightly.yml (cosign sign-blob; --certificate-identity, --certificate-oidc-issuer)", "scripts/verify-published.sh (--certificate-identity, --certificate-oidc-issuer)"], - "drafted_by": "Claude (AI-drafted), 2026-11-03", - "decided_by": null, - "decided_on": null + "usage_and_configuration": "Every deploy and nightly signs each published artifact keylessly: cosign obtains a short-lived certificate from Fulcio bound to the workflow's GitHub OIDC identity. No account or configuration exists on Fulcio's side.", + "justification": "Keyless signing removes long-lived signing keys from the pipeline and binds every signature to the exact workflow and commit that produced it, which any reader can verify.", + "mitigations": "Every verification pins the certificate identity (the workflow path and ref) and the OIDC issuer, so a certificate issued to anyone else does not verify. Fulcio sees only the CSR and the OIDC token claims (repository, workflow, ref), never artifact content. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "The post-publish round-trip verify and scripts/verify-published.sh re-verify every signature from outside; a Fulcio outage stops the deploy (fail closed) rather than publishing unsigned evidence.", + "evidence": [ + ".github/workflows/evidence-nightly.yml (cosign sign-blob; --certificate-identity, --certificate-oidc-issuer)", + "scripts/verify-published.sh (--certificate-identity, --certificate-oidc-issuer)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-03; MAS-CSO-TPR fields and decision fields AI-drafted 2026-11-05 for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" }, "ext::sigstore-rekor": { + "provider": "OpenSSF Sigstore project (public-good instance)", + "website": "https://www.sigstore.dev/", "upstream_offering": "Sigstore public-good instance: Rekor transparency log", "upstream_component_id": "https://rekor.sigstore.dev", "responsibility_split": { "upstream_runs": "The append-only transparency log and its inclusion proofs.", "provider_configures": "Signing with transparency-log upload (cosign defaults, not disabled) and verification that checks the inclusion proof in each bundle." }, - "evidence": [".github/workflows/evidence-nightly.yml (cosign sign-blob --new-bundle-format)", "scripts/verify-published.sh (cosign verify-blob)"], - "drafted_by": "Claude (AI-drafted), 2026-11-03", - "decided_by": null, - "decided_on": null + "usage_and_configuration": "Each keyless signature is recorded in Rekor (cosign default, not disabled); the bundle published next to each artifact carries the inclusion proof.", + "justification": "The transparency log makes every signature publicly auditable, so a signature made outside the pipeline would be visible.", + "mitigations": "Rekor receives only the signature, certificate and artifact digest, never artifact content. Verification checks the inclusion proof offline from the bundle. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "Signature verification against the pinned identity does not depend on Rekor being reachable later; the published bundles carry the proofs.", + "evidence": [ + ".github/workflows/evidence-nightly.yml (cosign sign-blob --new-bundle-format)", + "scripts/verify-published.sh (cosign verify-blob)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-03; MAS-CSO-TPR fields and decision fields AI-drafted 2026-11-05 for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" }, "ext::github-oidc": { + "provider": "GitHub, Inc.", + "website": "https://docs.github.com/actions/security-for-github-actions/security-hardening-your-deployments/about-security-hardening-with-openid-connect", "upstream_offering": "GitHub Actions OIDC token issuer", "upstream_component_id": "https://token.actions.githubusercontent.com", "responsibility_split": { "upstream_runs": "The token issuer and its signing keys (JWKS).", "provider_configures": "Which workflows may request a token (permissions: id-token: write), the AWS IAM OIDC trust conditions, and the issuer pinned in cosign verification." }, - "evidence": [".github/workflows/deploy-with-opa.yml:105 (id-token: write)", "infrastructure/bootstrap/plan-role.tf, evidence-nightly.tf, main.tf (OIDC trust)", ".github/workflows/evidence-nightly.yml:118 (COSIGN_OIDC_ISSUER)"], - "drafted_by": "Claude (AI-drafted), 2026-11-03", - "decided_by": null, - "decided_on": null + "usage_and_configuration": "Workflows granted id-token: write obtain per-run identity tokens, which AWS STS exchanges for the deploy, plan and nightly roles and which Fulcio binds into signing certificates.", + "justification": "Workload identity replaces static cloud credentials in CI: there is no long-lived access key to leak.", + "mitigations": "The AWS role trust policies pin the repository and, for the deploy role, the prod environment (reviewer-gated); only named workflows request tokens. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "The deploy needs a human approval in the prod environment before its token can assume the deploy role; role permissions are least-privilege and reviewed in the bootstrap stack.", + "evidence": [ + ".github/workflows/deploy-with-opa.yml:105 (id-token: write)", + "infrastructure/bootstrap/plan-role.tf, evidence-nightly.tf, main.tf (OIDC trust)", + ".github/workflows/evidence-nightly.yml:118 (COSIGN_OIDC_ISSUER)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-03; MAS-CSO-TPR fields and decision fields AI-drafted 2026-11-05 for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" }, "ext::github-repo": { + "provider": "GitHub, Inc.", + "website": "https://github.com/sam-aydlette/samaydlette.com", "upstream_offering": "GitHub repository and GitHub Actions", "upstream_component_id": "https://github.com/sam-aydlette/samaydlette.com", "responsibility_split": { "upstream_runs": "Repository hosting, the Actions runners, Dependabot, and secret scanning.", "provider_configures": "Branch protection on main, the workflows and their permissions, the OPA gate, the SCN tag validator, and secret scanning with push protection." }, - "evidence": ["scripts/build-ksi-signal.py (ext::github-repo verification text)", ".github/workflows/"], - "drafted_by": "Claude (AI-drafted), 2026-11-03", - "decided_by": null, - "decided_on": null + "usage_and_configuration": "Hosts the source of record and runs the CI/CD workflows (deploy, nightly evidence, scans) on GitHub-hosted runners.", + "justification": "The repository is the system of record for infrastructure as code and evidence generation; hosted runners give an ephemeral, reproducible build environment.", + "mitigations": "Branch protection on main, required checks, a reviewer-gated prod environment for every deploy, secret scanning with push protection, and least-privilege workflow permissions. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "Every published artifact is signed under the deploy workflow's pinned identity and re-verified from outside; the hand-applied GitHub settings are checked live on every deploy (scripts/check-manual-settings.py).", + "evidence": [ + "scripts/build-ksi-signal.py (ext::github-repo verification text)", + ".github/workflows/" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-03; MAS-CSO-TPR fields and decision fields AI-drafted 2026-11-05 for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" }, "ext::duo-mfa": { - "upstream_offering": null, + "provider": "Cisco Systems, Inc. (Duo)", + "website": "https://duo.com/", + "upstream_offering": "Duo Mobile authenticator app (inferred; see drafted_by)", "upstream_component_id": null, - "responsibility_split": null, - "evidence": ["scripts/build-ksi-signal.py (ext::duo-mfa: tenant identifier intentionally not published)"], - "drafted_by": "Claude (AI-drafted), 2026-11-03; the repository does not show the Duo edition, its identifier, or how it is configured, so nothing is drafted", - "decided_by": null, - "decided_on": null + "responsibility_split": { + "upstream_runs": "The authenticator app that generates the time-based one-time codes (inferred).", + "provider_configures": "Enrollment of the TOTP device as the MFA factor on the AWS root account, the operator IAM user and the GitHub account." + }, + "usage_and_configuration": "Generates the TOTP codes the operator enters as the second factor for the AWS root account, the operator IAM user and the GitHub account (inferred from DEC-025, which records virtual TOTP MFA on root and the operator user, and from the inventory's description of this service).", + "justification": "A second factor on every top-level administrative and privileged account, as IA-2(1) and IA-2(2) require.", + "mitigations": "TOTP seeds stay on the operator's device; no tenant identifier is published. MFA is enabled on every privileged human account. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "The weaker-than-phishing-resistant factor is a recorded decision (DEC-025); CloudTrail records every console sign-in and privileged API call.", + "evidence": [ + "scripts/build-ksi-signal.py (ext::duo-mfa: tenant identifier intentionally not published)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-05. INFERRED: the repository does not show the Duo edition or whether Duo is used only as a TOTP generator; the operator must confirm or correct every field.", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" }, "ext::github-advisory-db": { + "provider": "GitHub, Inc.", + "website": "https://github.com/advisories", "upstream_offering": "GitHub Advisory Database, read through Dependabot alerts", "upstream_component_id": "https://github.com/advisories", "responsibility_split": { "upstream_runs": "The advisory database and the matching of advisories to this repository's dependencies.", "provider_configures": "The fetch of open Dependabot alerts in the deploy and nightly workflows, and how a failed fetch is handled." }, - "evidence": [".github/workflows/evidence-nightly.yml:435 (dependabot/alerts)", ".github/workflows/deploy-with-opa.yml:1399"], - "drafted_by": "Claude (AI-drafted), 2026-11-03", - "decided_by": null, - "decided_on": null + "usage_and_configuration": "The deploy and the nightly read open Dependabot alerts for this repository, which GitHub matches from the Advisory Database.", + "justification": "An independent source of dependency vulnerabilities alongside the Grype scan.", + "mitigations": "A failed fetch fails the detection gate (G7) rather than reading as zero findings. Only public advisory data is read. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "Findings are corroborated with the Grype scan and CISA KEV; the VDR records each source.", + "evidence": [ + ".github/workflows/evidence-nightly.yml:435 (dependabot/alerts)", + ".github/workflows/deploy-with-opa.yml:1399" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-03; MAS-CSO-TPR fields and decision fields AI-drafted 2026-11-05 for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" }, "ext::cisa-kev": { + "provider": "Cybersecurity and Infrastructure Security Agency (CISA)", + "website": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog", "upstream_offering": "CISA Known Exploited Vulnerabilities catalog (JSON feed)", "upstream_component_id": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", "responsibility_split": { "upstream_runs": "The catalog and its JSON feed.", "provider_configures": "The fetch in the deploy and nightly workflows, and how a failed fetch is handled." }, - "evidence": [".github/workflows/evidence-nightly.yml:447 (KEV_URL)", ".github/workflows/deploy-with-opa.yml:1415"], - "drafted_by": "Claude (AI-drafted), 2026-11-03", - "decided_by": null, - "decided_on": null + "usage_and_configuration": "The deploy and the nightly fetch the KEV JSON feed and mark each finding that appears in it.", + "justification": "The authoritative U.S. government list of vulnerabilities known to be exploited, which drives remediation priority.", + "mitigations": "A failed fetch fails the detection gate (G7). Only public data is read. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "EPSS and the Grype and Dependabot sources still rate findings; the VDR states which feeds it read.", + "evidence": [ + ".github/workflows/evidence-nightly.yml:447 (KEV_URL)", + ".github/workflows/deploy-with-opa.yml:1415" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-03; MAS-CSO-TPR fields and decision fields AI-drafted 2026-11-05 for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" + }, + "ext::first-epss": { + "provider": "FIRST (Forum of Incident Response and Security Teams); file served from epss.cyentia.com", + "website": "https://www.first.org/epss/", + "upstream_offering": "FIRST Exploit Prediction Scoring System, daily scores file", + "upstream_component_id": "https://epss.cyentia.com/epss_scores-current.csv.gz", + "responsibility_split": { + "upstream_runs": "The EPSS model and the daily scores file.", + "provider_configures": "The fetch in scripts/fetch-detection-feeds.sh and how a failed fetch is handled (the detection gate fails)." + }, + "usage_and_configuration": "The deploy and the nightly download the current scores file and attach each finding's exploit probability.", + "justification": "Exploit likelihood is one input to the finding's rating (likely exploitable), alongside KEV.", + "mitigations": "A failed fetch fails the detection gate (G7). Only public data is read. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "KEV membership still marks known exploitation; the rating method is recorded in the VDR.", + "evidence": [ + "scripts/fetch-detection-feeds.sh (EPSS_URL)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-05, for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" + }, + "ext::anchore-grype-db": { + "provider": "Anchore, Inc.", + "website": "https://github.com/anchore/grype", + "upstream_offering": "Grype vulnerability database, distributed by Anchore", + "upstream_component_id": "https://grype.anchore.io/databases", + "responsibility_split": { + "upstream_runs": "Building and publishing the vulnerability database from upstream advisory sources.", + "provider_configures": "The pinned Grype version (GRYPE_VERSION) and the scans of each SBOM in the deploy and the nightly." + }, + "usage_and_configuration": "Grype downloads the current database at scan time and matches every SBOM the pipeline builds against it.", + "justification": "The primary software-composition scan behind the VDR.", + "mitigations": "The Grype version is pinned; Grype checks the database download against the checksum in its published index. Only the SBOMs are scanned, locally; nothing is uploaded. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "Dependabot alerts and CISA KEV are independent sources the VDR corroborates against.", + "evidence": [ + ".github/workflows/evidence-nightly.yml (grype sbom:...)", + ".github/workflows/deploy-with-opa.yml (GRYPE_VERSION)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-05, for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" + }, + "ext::jsdelivr": { + "provider": "jsDelivr", + "website": "https://www.jsdelivr.com/", + "upstream_offering": "jsDelivr open-source CDN, npm endpoint", + "upstream_component_id": "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/", + "responsibility_split": { + "upstream_runs": "Serving the published npm package files over its CDN.", + "provider_configures": "The pinned MediaPipe Tasks Vision version in the app bundle and the CSP script-src and connect-src paths that allow only that package." + }, + "usage_and_configuration": "The Silk Reeling app's browser code loads the MediaPipe Tasks Vision JavaScript and WebAssembly from jsDelivr at the version the bundle pins.", + "justification": "MediaPipe runs pose estimation in the user's browser, so camera frames never leave the device; jsDelivr is the distribution MediaPipe documents.", + "mitigations": "The CSP allows only the pinned package path; camera frames stay in the browser and only pose landmarks reach the app's API. jsDelivr sees the user's IP address and user agent when the file is fetched. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "The app requires Cognito sign-in with TOTP MFA; the API accepts only landmark data and returns scores. Self-hosting the runtime is the further step if a stronger integrity guarantee is needed.", + "evidence": [ + "infrastructure/bootstrap/cloudfront.tf (silk_reeling_mediapipe, CSP script-src)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-05, for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" + }, + "ext::google-cloud-storage": { + "provider": "Google LLC", + "website": "https://ai.google.dev/edge/mediapipe/solutions/vision/pose_landmarker", + "upstream_offering": "Google Cloud Storage, MediaPipe public model bucket", + "upstream_component_id": "https://storage.googleapis.com/mediapipe-models/", + "responsibility_split": { + "upstream_runs": "Hosting and serving the published MediaPipe model files.", + "provider_configures": "The model path the app requests and the CSP connect-src path that allows only the mediapipe-models bucket." + }, + "usage_and_configuration": "The MediaPipe runtime in the user's browser downloads the pose-landmarker model file from Google's public model bucket.", + "justification": "The model Google publishes for MediaPipe pose landmarking; running it in the browser keeps camera frames on the device.", + "mitigations": "The CSP allows only the mediapipe-models path; the model is data loaded by the pinned runtime, not script. Google sees the user's IP address and user agent when the file is fetched. No federal customer data is in this offering; the resource never receives app-user data.", + "compensating_controls": "Pose scores are computed server-side from landmarks only; a tampered model could degrade scores but cannot read data beyond the camera frames already in the browser.", + "evidence": [ + "infrastructure/bootstrap/cloudfront.tf (CSP connect-src)" + ], + "drafted_by": "Claude (AI-drafted), 2026-11-05, for the operator to confirm", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05" } } } diff --git a/docs/boundary/boundary-classification.json b/docs/boundary/boundary-classification.json index 69d1e17c..7840ed03 100644 --- a/docs/boundary/boundary-classification.json +++ b/docs/boundary/boundary-classification.json @@ -28,7 +28,9 @@ {"key": "operator-mfa", "zone": "external", "label": "Operator MFA", "match": ["ext::duo-mfa"]}, {"key": "vuln-feeds", "zone": "external", "label": "Vulnerability metadata sources", - "match": ["ext::github-advisory-db", "ext::cisa-kev"]} + "match": ["ext::github-advisory-db", "ext::cisa-kev", "ext::first-epss", "ext::anchore-grype-db"]}, + {"key": "app-delivery", "zone": "external", "label": "Code and models the app loads in the user's browser", + "match": ["ext::jsdelivr", "ext::google-cloud-storage"]} ], "declared": [ {"key": "operator", "label": "Operator (dev machine)", "zone": "outside", "kind": "actor", @@ -69,6 +71,40 @@ {"id": "J", "label": "Evidence Nightly → S3 (.well-known/vdr-*)", "path": [{"component": "ext::github-oidc"}, {"component": "aws::iam_role::evidence_nightly"}, {"component": "aws::object_store::website"}], "protocol": "HTTPS 443 (TLS 1.2+, SigV4)", "auth": "GitHub OIDC → STS; the role can write only .well-known/vdr-* (unattended)", "encryption": "In transit TLS; at rest SSE-S3", "data": "Refreshed VDR and trend (Sigstore-signed)"}, {"id": "K", "label": "Watchdog → S3 (evidence freshness read)", "path": [{"component": "aws::event_schedule::evidence_watchdog"}, {"component": "aws::function::evidence_watchdog"}, {"component": "aws::object_store::website"}], - "protocol": "AWS-internal", "auth": "Watchdog Lambda role: s3:GetObject on the published evidence keys only", "encryption": "In transit: TLS", "data": "Published evidence timestamps; results go out as CloudWatch metrics that drive flow I's alarms"} + "protocol": "AWS-internal", "auth": "Watchdog Lambda role: s3:GetObject on the published evidence keys only", "encryption": "In transit: TLS", "data": "Published evidence timestamps; results go out as CloudWatch metrics that drive flow I's alarms"}, + {"id": "L", "label": "App user → Cognito Hosted UI (sign-in)", "path": [{"declared": "app-users"}, {"component": "aws::identity_provider::silk_reeling"}], + "protocol": "HTTPS 443 (TLS 1.2+)", "auth": "Password + required TOTP MFA; OAuth2 authorization code with PKCE", "encryption": "In transit: TLS; user records at rest encrypted by Cognito", "data": "Sign-in credentials and MFA code in; ID and access tokens out (email address in the user record)"}, + {"id": "M", "label": "App user's browser → jsDelivr (MediaPipe runtime)", "path": [{"declared": "app-users"}, {"component": "ext::jsdelivr"}], + "protocol": "HTTPS 443 (TLS)", "auth": "Anonymous public GET", "encryption": "In transit: TLS", "data": "Pinned MediaPipe JavaScript and WebAssembly in; the request carries the user's IP address and user agent only"}, + {"id": "N", "label": "App user's browser → Google Cloud Storage (pose model)", "path": [{"declared": "app-users"}, {"component": "ext::google-cloud-storage"}], + "protocol": "HTTPS 443 (TLS)", "auth": "Anonymous public GET", "encryption": "In transit: TLS", "data": "Pose-landmarker model file in; the request carries the user's IP address and user agent only"}, + {"id": "O", "label": "Site bucket → access-log bucket (S3 server access logs)", "path": [{"component": "aws::object_store::website"}, {"component": "aws::object_store::logs"}], + "protocol": "AWS-internal (S3 log delivery)", "auth": "Log bucket policy admits only the S3 logging service", "encryption": "At rest: SSE-S3", "data": "Request records for origin reads: requester IP, user agent, key, status"}, + {"id": "P", "label": "CloudTrail → CloudTrail log bucket", "path": [{"component": "aws::audit_log_trail::management"}, {"component": "aws::object_store::cloudtrail"}], + "protocol": "AWS-internal (CloudTrail delivery)", "auth": "Bucket policy admits only CloudTrail for this trail", "encryption": "At rest: SSE-S3; log-file validation on", "data": "Management events: API caller identity, source IP, action, parameters"}, + {"id": "Q", "label": "KSI emitter → its log group", "path": [{"component": "aws::function::opa_compliance"}, {"component": "aws::log_group::opa_compliance"}], + "protocol": "AWS-internal", "auth": "Lambda execution role", "encryption": "At rest: CMK", "data": "Emitter run logs (no customer data)", "note": "Failed asynchronous invocations go to the dead-letter queue (flow R)."}, + {"id": "R", "label": "KSI emitter → dead-letter queue (failed invocations)", "path": [{"component": "aws::function::opa_compliance"}, {"component": "aws::message_queue::compliance_dlq"}], + "protocol": "AWS-internal", "auth": "Lambda execution role (sqs:SendMessage on the queue only)", "encryption": "At rest: SQS-managed server-side encryption", "data": "The failed scheduled event (no customer data); kept 14 days"}, + {"id": "S", "label": "App Lambda → its log group", "path": [{"component": "aws::function::silk_reeling"}, {"component": "aws::log_group::silk_reeling"}], + "protocol": "AWS-internal", "auth": "Lambda execution role", "encryption": "At rest: CMK", "data": "Application logs: request metadata, errors"}, + {"id": "T", "label": "API Gateway → access log group", "path": [{"component": "aws::api_gateway::silk_reeling"}, {"component": "aws::log_group::silk_apigw"}], + "protocol": "AWS-internal", "auth": "Log resource policy admits API Gateway delivery", "encryption": "At rest: CMK", "data": "API access records: request ID, source IP, time, method, route, status, response length"}, + {"id": "U", "label": "Watchdog → its log group", "path": [{"component": "aws::function::evidence_watchdog"}, {"component": "aws::log_group::evidence_watchdog"}], + "protocol": "AWS-internal", "auth": "Lambda execution role", "encryption": "At rest: CMK", "data": "Watchdog run logs (artifact ages; no customer data)"}, + {"id": "V", "label": "Operator → AWS console and APIs (with TOTP MFA)", "path": [{"declared": "operator"}, {"component": "ext::duo-mfa"}, {"component": "aws::iam_group::operators"}], + "protocol": "HTTPS 443 (TLS 1.2+)", "auth": "Password + TOTP MFA (root and the operator IAM user; privileged)", "encryption": "In transit: TLS", "data": "Administrative sign-in and AWS API calls (recorded by CloudTrail, flow P)"}, + {"id": "W", "label": "Deploy and plan → Terraform state bucket", "path": [{"component": "aws::iam_role::deploy"}, {"component": "aws::object_store::tfstate"}], + "protocol": "HTTPS 443 (TLS 1.2+, SigV4)", "auth": "Deploy and plan roles (GitHub OIDC)", "encryption": "In transit TLS; at rest SSE-S3", "data": "Terraform state (configuration and resource attributes, including secret values)"}, + {"id": "X", "label": "Deploy and plan → state lock table", "path": [{"component": "aws::iam_role::deploy"}, {"component": "aws::kv_table::tflock"}], + "protocol": "HTTPS 443 (TLS 1.2+, SigV4)", "auth": "Deploy and plan roles (GitHub OIDC)", "encryption": "In transit TLS; at rest AWS-owned key", "data": "Lock ID, holder and timestamp only"}, + {"id": "Y1", "label": "Deploy and nightly → GitHub Advisory Database (Dependabot alerts)", "path": [{"component": "ext::github-repo"}, {"component": "ext::github-advisory-db"}], + "protocol": "HTTPS 443 (TLS)", "auth": "Repository token that can read Dependabot alerts", "encryption": "In transit: TLS", "data": "Open dependency alerts in (public advisory data)"}, + {"id": "Y2", "label": "Deploy and nightly → CISA KEV catalog", "path": [{"component": "ext::github-repo"}, {"component": "ext::cisa-kev"}], + "protocol": "HTTPS 443 (TLS)", "auth": "Anonymous public GET", "encryption": "In transit: TLS", "data": "KEV catalog in (public)"}, + {"id": "Y3", "label": "Deploy and nightly → FIRST EPSS scores", "path": [{"component": "ext::github-repo"}, {"component": "ext::first-epss"}], + "protocol": "HTTPS 443 (TLS)", "auth": "Anonymous public GET", "encryption": "In transit: TLS", "data": "EPSS scores file in (public)"}, + {"id": "Y4", "label": "Deploy and nightly → Grype vulnerability database", "path": [{"component": "ext::github-repo"}, {"component": "ext::anchore-grype-db"}], + "protocol": "HTTPS 443 (TLS)", "auth": "Anonymous public GET", "encryption": "In transit: TLS; checksum verified", "data": "Vulnerability database in (public)"} ] } diff --git a/docs/portfolio/service-model.json b/docs/portfolio/service-model.json index 4aeb5726..43412f19 100644 --- a/docs/portfolio/service-model.json +++ b/docs/portfolio/service-model.json @@ -136,6 +136,14 @@ "owner_domain": "silk-reeling", "consumed_by": ["silk-reeling"], "features": ["movement-feedback", "sign-in"] + }, + { + "key": "app-browser-delivery", + "name": "MediaPipe runtime and model the app loads in the browser", + "group": "app-delivery", + "owner_domain": "silk-reeling", + "consumed_by": ["silk-reeling"], + "features": ["movement-feedback"] } ] } diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index 888561ba..dbac1057 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -189,6 +189,17 @@ "signed": true, "signature": ".well-known/policy-index.bundle" }, + { + "path": ".well-known/minimum-assessment-scope.json", + "cadence": "deploy", + "max_age_hours": 720, + "timestamp": { + "source": "json", + "field": "generated_at" + }, + "signed": true, + "signature": ".well-known/minimum-assessment-scope.bundle" + }, { "path": ".well-known/security-decision-record.json", "cadence": "deploy", diff --git a/scripts/_tpr.py b/scripts/_tpr.py new file mode 100644 index 00000000..a82f19aa --- /dev/null +++ b/scripts/_tpr.py @@ -0,0 +1,40 @@ +"""Third-party information resources for the CPO (MAS-CSO-TPR). + +The CPO schema's `thirdPartyInformationResources` object, built from the +decided records in data/third-party-components.json. None of these resources is +recorded as FedRAMP certified, so every entry goes under `nonCertified`; this +module never asserts a resource's authorization status. A record without a +decision, or a decided record without a use case, fails closed: the CPO never +lists a resource the operator has not decided. +""" + +import json +from pathlib import Path + +THIRD_PARTY_PATH = Path(__file__).resolve().parent.parent / "data" / "third-party-components.json" + + +def _fail(msg): + raise SystemExit(f"::error::third-party records: {msg} (fail-closed)") + + +def cpo_third_party_information_resources(path=THIRD_PARTY_PATH): + """{"nonCertified": [{"name", "provider", "website"?, "useCase"}]}, one entry + per third-party record, sorted by component ID.""" + try: + recs = json.loads(Path(path).read_text())["components"] + except (OSError, json.JSONDecodeError, KeyError, TypeError) as exc: + _fail(f"cannot read {path} ({type(exc).__name__})") + out = [] + for cid, rec in sorted(recs.items()): + if not (rec.get("decided_by") and rec.get("decided_on")): + _fail(f"{cid} has no decision (decided_by, decided_on)") + use = rec.get("usage_and_configuration") + name = rec.get("upstream_offering") or cid + if not (isinstance(use, str) and use.strip() and rec.get("provider")): + _fail(f"{cid} lacks provider or usage_and_configuration") + entry = {"name": name, "provider": rec["provider"], "useCase": use} + if isinstance(rec.get("website"), str) and rec["website"].startswith("https://"): + entry["website"] = rec["website"] + out.append(entry) + return {"nonCertified": out} diff --git a/scripts/build-ksi-signal.py b/scripts/build-ksi-signal.py index 1d570b1c..e3ff572c 100755 --- a/scripts/build-ksi-signal.py +++ b/scripts/build-ksi-signal.py @@ -1326,6 +1326,9 @@ def build_sbom_components(sbom_path, existing_components=None): THIRD_PARTY_PATH = Path(__file__).resolve().parent.parent / "data" / "third-party-components.json" NOT_RECORDED = "not recorded" TPR_FIELDS = ("upstream_offering", "upstream_component_id", "responsibility_split") +# MAS-CSO-TPR: what FedRAMP requires documented about each third-party +# information resource. A decided record must carry every one. +MAS_TPR_FIELDS = ("usage_and_configuration", "justification", "mitigations", "compensating_controls") def load_third_party_records(path, known_ids): @@ -1347,18 +1350,24 @@ def load_third_party_records(path, known_ids): split = rec.get("responsibility_split") if split is not None and not (isinstance(split, dict) and split.get("upstream_runs") and split.get("provider_configures")): raise SystemExit(f"::error::third-party record for {cid}: responsibility_split needs upstream_runs and provider_configures") + if rec.get("decided_by") and rec.get("decided_on"): + missing = [f for f in MAS_TPR_FIELDS if not (isinstance(rec.get(f), str) and rec[f].strip())] + if missing: + raise SystemExit(f"::error::third-party record for {cid} is decided but lacks {', '.join(missing)} " + "(MAS-CSO-TPR)") return recs def third_party_attributes(rec): """The TPR-1 fields for one component. A record without a decision (decided_by and decided_on) publishes every field as "not recorded".""" + fields = (*TPR_FIELDS, *MAS_TPR_FIELDS) if rec is None: - return {**{f: NOT_RECORDED for f in TPR_FIELDS}, "third_party_record": {"status": "no record"}} + return {**{f: NOT_RECORDED for f in fields}, "third_party_record": {"status": "no record"}} if not (rec.get("decided_by") and rec.get("decided_on")): - return {**{f: NOT_RECORDED for f in TPR_FIELDS}, + return {**{f: NOT_RECORDED for f in fields}, "third_party_record": {"status": "drafted; no decision recorded"}} - out = {f: rec.get(f) or NOT_RECORDED for f in TPR_FIELDS} + out = {f: rec.get(f) or NOT_RECORDED for f in fields} out["third_party_record"] = {"status": "decided", "decided_by": rec["decided_by"], "decided_on": rec["decided_on"]} return out @@ -1437,6 +1446,47 @@ def build_external_components(third_party_path=THIRD_PARTY_PATH): "verification": "U.S. federal government source", "fedramp_status": "external; U.S. federal government source", }, + # MAS-CSO-IIR: the vulnerability-data feeds the VDR build also reads + # (scripts/fetch-detection-feeds.sh, evidence-nightly.yml's grype scan). + { + "id": "ext::first-epss", + "native_id": "https://epss.cyentia.com/epss_scores-current.csv.gz", + "name": "FIRST Exploit Prediction Scoring System (EPSS) scores", + "purpose": "Daily exploit-likelihood scores the VDR uses to rate findings", + "cia_impact": "VDR ingest accuracy", + "verification": "Public feed; a failed fetch fails the detection gate (G7) rather than scoring with no data", + "fedramp_status": "external; no separate FedRAMP authorization", + }, + { + "id": "ext::anchore-grype-db", + "native_id": "https://grype.anchore.io/databases", + "name": "Grype vulnerability database (Anchore)", + "purpose": "The vulnerability database Grype matches the SBOMs against on every deploy and nightly scan", + "cia_impact": "VDR ingest accuracy", + "verification": "Grype verifies the database checksum on download; findings are corroborated against Dependabot and CISA KEV", + "fedramp_status": "external; no separate FedRAMP authorization", + }, + # MAS-CSO-IIR: origins the Silk Reeling app loads code or data from in the + # user's browser, next to the camera feed (the app's CSP in + # infrastructure/bootstrap/cloudfront.tf allows exactly these). + { + "id": "ext::jsdelivr", + "native_id": "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/", + "name": "jsDelivr CDN (MediaPipe Tasks Vision runtime)", + "purpose": "Serves the MediaPipe pose-estimation JavaScript and WebAssembly the app runs in the browser", + "cia_impact": "code executed next to the camera feed in the user's browser", + "verification": "Version pinned in the app bundle and in the CSP script-src path; the CSP allows no other jsDelivr path", + "fedramp_status": "external; no separate FedRAMP authorization", + }, + { + "id": "ext::google-cloud-storage", + "native_id": "https://storage.googleapis.com/mediapipe-models/", + "name": "Google Cloud Storage (MediaPipe model bucket)", + "purpose": "Serves the pose-landmarker model file the MediaPipe runtime loads in the browser", + "cia_impact": "model integrity for the pose scores the app computes", + "verification": "CSP connect-src limited to the mediapipe-models path; the model runs in the browser and returns landmarks only", + "fedramp_status": "external; no separate FedRAMP authorization", + }, ] tpr = load_third_party_records(third_party_path, {s["id"] for s in services}) components = [] @@ -1595,7 +1645,8 @@ def build_validations(validations_doc, components): "dns_zone", "tls_certificate", "secrets_manager", "identity_provider", "kms_key"}, "IAM": {"iam_role", "iam_policy", "iam_group", "oidc_provider", "identity_provider", "secrets_manager"}, "INR": {"log_group", "function", "notification_topic", "metric_alarm"}, - "MLA": {"log_group", "function", "cdn_distribution", "notification_topic", "metric_alarm", "message_queue"}, + "MLA": {"log_group", "function", "cdn_distribution", "notification_topic", "metric_alarm", "message_queue", + "audit_log_trail"}, "PIY": None, # policy & inventory — whole inventory is the evidence "RPL": {"object_store"}, "SCR": {"npm_package", "pypi_package", "external_service"}, diff --git a/scripts/build-mas.py b/scripts/build-mas.py new file mode 100644 index 00000000..b18f6b87 --- /dev/null +++ b/scripts/build-mas.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""The Minimum Assessment Scope, generated: /.well-known/minimum-assessment-scope.json. + +One machine-readable view of the cloud service offering's scope, joined from +what the pipeline already holds, so it cannot drift from it: + + MAS-CSO-IIR the information resources: the canonical inventory (ksi-signal.json), + plus every off-origin host the site's and app's Content Security + Policies allow (infrastructure/bootstrap/cloudfront.tf), each placed + on an inventoried component or put out of scope with a reason. + MAS-CSO-FLO the information flows (docs/boundary/boundary-classification.json) + resolved against the inventory, each with a security category + derived from its endpoints (the high-water mark of their FIPS 199 + categories); every external service and every data store is in at + least one flow. + MAS-CSO-MDI the metadata inventory: every data store and log in the inventory, + what it holds, which personal data, retention and location + (data/minimum-assessment-scope.json supplies the meaning). + MAS-CSO-TPR each third-party information resource with the four fields FedRAMP + requires (data/third-party-components.json, decided records only). + FRC-CSX-MAS how every KSI applies to the scope: per component through the KSI's + evidence domain, or over the whole set. + +Fails closed: any gap above stops the build, and so the deploy, before the SDR +is built. No ARN, account ID or address is published: components are named by +their inventory IDs. +""" + +import argparse +import fnmatch +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +INPUTS = REPO / "data" / "minimum-assessment-scope.json" +THIRD_PARTY = REPO / "data" / "third-party-components.json" +CLASSIFICATION = REPO / "docs" / "boundary" / "boundary-classification.json" +CLOUDFRONT_TF = REPO / "infrastructure" / "bootstrap" / "cloudfront.tf" +BOOTSTRAP_DIR = REPO / "infrastructure" / "bootstrap" +SCHEMA = "minimum-assessment-scope/1" +LEVELS = ("low", "moderate", "high") +OBJECTIVES = ("confidentiality", "integrity", "availability") +TPR_FIELDS = ("usage_and_configuration", "justification", "mitigations", "compensating_controls") +PACKAGE_TYPES = ("npm_package", "pypi_package") +CODE = { + "builder": "scripts/build-mas.py", + "inventory": "scripts/build-ksi-signal.py", + "inputs": "data/minimum-assessment-scope.json", + "flows": "docs/boundary/boundary-classification.json", + "third_party": "data/third-party-components.json", + "csp": "infrastructure/bootstrap/cloudfront.tf", +} +DERIVATION = ( + "Generated on every deploy by scripts/build-mas.py from this build's canonical inventory " + "(ksi-signal.json, itself derived from Terraform state, the SBOMs, the site's HTML and the declared " + "external services), the boundary classification's data-flow table, the operator's metadata and " + "off-origin placements (data/minimum-assessment-scope.json) and the third-party records " + "(data/third-party-components.json). Information resources are the inventory's components; the " + "Content Security Policies the edge serves are parsed for every off-origin host, and each must be an " + "inventoried component or out of scope with a stated reason. Each flow is resolved against the inventory " + "and given the high-water mark of its endpoints' security categories. Every data store and log in the " + "inventory must have a metadata entry, every external service and data store must be in a flow, every " + "third-party resource must carry a decided record with the four MAS-CSO-TPR fields, and every component " + "type must fall in at least one KSI's evidence domain or carry a whole-set reason. Any gap fails the " + "build, so the deploy never publishes a scope with one." +) + + +def _fail(msg): + raise SystemExit(f"::error::MAS: {msg} (fail-closed)") + + +# --- MAS-CSO-IIR: off-origin hosts from the CSPs --------------------------------------- + +_URL = re.compile(r"https://([A-Za-z0-9.${}_-]+)") +_LOCAL = re.compile(r"\$\{local\.([A-Za-z0-9_]+)\}") +_VAR = re.compile(r"\$\{var\.([A-Za-z0-9_]+)\}") + + +def _locals(text): + """{name: definition text} for every `name = ...` in a locals block (to the + next blank line, enough for this repository's style).""" + out = {} + for block in re.finditer(r"locals\s*\{(.*?)\n\}", text, re.S): + body = block.group(1) + for m in re.finditer(r"^\s*([A-Za-z0-9_]+)\s*=\s*(.*?)(?=^\s*[A-Za-z0-9_]+\s*=|\Z)", body, re.S | re.M): + v = m.group(2).strip() + q = re.match(r'^"([^"\n]*)"', v) # a plain string local: its content, without the quotes + out[m.group(1)] = q.group(1) if q else v + return out + + +def _var_defaults(text): + return {m.group(1): m.group(2) for m in re.finditer( + r'variable\s+"([A-Za-z0-9_]+)"\s*\{[^}]*?default\s*=\s*"([^"]*)"', text, re.S)} + + +def _expand(s, locals_, vars_, depth=0): + if depth > 5: + _fail("CSP expansion does not terminate") + s = _VAR.sub(lambda m: vars_.get(m.group(1), m.group(0)), s) + if "domain_name" in vars_: + s = s.replace("${local.domain_dashed}", vars_["domain_name"].replace(".", "-")) + return _LOCAL.sub(lambda m: _expand(locals_.get(m.group(1), m.group(0)), locals_, vars_, depth + 1), s) + + +def csp_hosts(tf_text, context_text=""): + """Every host an https:// source in a content_security_policy names, with + ${local.*} and ${var.*} references expanded from the same configuration.""" + whole = tf_text + "\n" + context_text + locals_, vars_ = _locals(whole), _var_defaults(whole) + hosts = set() + for m in re.finditer(r"content_security_policy\s*=\s*join\(\s*\"; \",\s*\[(.*?)\]\s*\)", tf_text, re.S): + expanded = _expand(m.group(1), locals_, vars_) + # Every https:// source must yield a host: one that does not would be a + # host this check silently missed. + if expanded.count("https://") != len(_URL.findall(expanded)): + _fail("a CSP https:// source does not resolve to a host") + for h in _URL.findall(expanded): + if "$" in h or "{" in h or "}" in h: + _fail(f"cannot resolve the CSP source host {h!r}") + hosts.add(h.lower()) + return sorted(hosts) + + +def place_hosts(hosts, placements, component_ids): + """[(host, placement)] for every host; fails on a host placed nowhere, on a + placement naming a component the inventory lacks, or with no reason.""" + out, problems = [], [] + for h in hosts: + p = placements.get(h) + if not isinstance(p, dict): + problems.append(f"the CSP allows {h}, which is neither an inventoried information resource nor " + "out of scope with a reason (data/minimum-assessment-scope.json off_origin_hosts)") + continue + if p.get("component"): + if p["component"] not in component_ids: + problems.append(f"{h} is placed on {p['component']}, which is not in the inventory") + out.append({"host": h, "component": p["component"]}) + elif isinstance(p.get("out_of_scope"), str) and p["out_of_scope"].strip(): + out.append({"host": h, "out_of_scope": p["out_of_scope"]}) + else: + problems.append(f"{h} has neither a component nor an out_of_scope reason") + return out, problems + + +# --- MAS-CSO-FLO ------------------------------------------------------------------------- + +def _match(pattern, ids): + return sorted(i for i in ids if fnmatch.fnmatchcase(i, pattern)) + + +def _high_water(categories): + out = {} + for o in OBJECTIVES: + vals = [c.get(o) for c in categories if c.get(o) in LEVELS] + if vals: + out[o] = max(vals, key=LEVELS.index) + return out + + +def flows(classification, components, data_store_types): + by_id = {c["component_id"]: c for c in components} + ids = sorted(by_id) + declared = {d["key"] for d in classification.get("declared", [])} + out, problems, covered = [], [], set() + for f in classification.get("flows", []): + endpoints = [] + for ref in f["path"]: + if "declared" in ref: + if ref["declared"] not in declared: + problems.append(f"flow {f['id']} names an undeclared actor {ref['declared']}") + endpoints.append({"declared": ref["declared"]}) + continue + hit = _match(ref["component"], ids) + if not hit: + problems.append(f"flow {f['id']} endpoint {ref['component']} matches no inventoried component") + covered.update(hit) + endpoints.append({"components": hit}) + cats = [by_id[i].get("security_category") or {} for e in endpoints for i in e.get("components", [])] + category = _high_water(cats) + if len(category) != len(OBJECTIVES): + problems.append(f"flow {f['id']} has no endpoint with a security category") + out.append({"id": f["id"], "label": f["label"], "endpoints": endpoints, "security_category": category, + **{k: f[k] for k in ("protocol", "auth", "encryption", "data", "note") if f.get(k)}}) + must = sorted(i for i, c in by_id.items() if c.get("type") == "external_service" or c.get("type") in data_store_types) + for i in must: + if i not in covered: + problems.append(f"{i} ({by_id[i].get('type')}) is in no information flow") + return out, problems + + +# --- MAS-CSO-MDI ------------------------------------------------------------------------- + +def metadata_inventory(components, metadata, data_store_types): + out, problems = [], [] + keys = list(metadata) + for c in sorted(components, key=lambda c: c["component_id"]): + cid, ctype = c["component_id"], c.get("type") + third_party_observed = cid in metadata and ctype == "external_service" + if ctype not in data_store_types and not third_party_observed: + continue + key = cid if cid in metadata else next((k for k in keys if fnmatch.fnmatchcase(cid, k)), None) + if key is None: + problems.append(f"{cid} ({ctype}) holds data or logs but has no metadata entry " + "(data/minimum-assessment-scope.json metadata)") + continue + m = metadata[key] + if not (isinstance(m.get("holds"), str) and isinstance(m.get("personal_data"), list) + and isinstance(m.get("retention"), str)): + problems.append(f"metadata entry {key} needs holds, personal_data and retention") + continue + attrs = c.get("attributes") or {} + entry = {"component": cid, "type": ctype, "holds": m["holds"], "personal_data": m["personal_data"], + "retention": m["retention"], + "location": attrs.get("region") or ("third party" if ctype == "external_service" else "not recorded"), + "controlled_by": "third party" if ctype == "external_service" else "provider"} + if "retention_days" in m: + entry["retention_days"] = m["retention_days"] + out.append(entry) + return out, problems + + +# --- MAS-CSO-TPR ------------------------------------------------------------------------- + +def third_party(components, tpr_doc): + recs = (tpr_doc or {}).get("components") or {} + out, problems = [], [] + for c in sorted(components, key=lambda c: c["component_id"]): + if c.get("type") != "external_service": + continue + cid, rec = c["component_id"], recs.get(c["component_id"]) + if not rec: + problems.append(f"{cid} has no third-party record (data/third-party-components.json)") + continue + if not (rec.get("decided_by") and rec.get("decided_on")): + problems.append(f"{cid}'s third-party record has no decision (decided_by, decided_on)") + continue + missing = [f for f in TPR_FIELDS if not (isinstance(rec.get(f), str) and rec[f].strip())] + if missing: + problems.append(f"{cid}'s third-party record lacks {', '.join(missing)}") + continue + attrs = c.get("attributes") or {} + out.append({"component": cid, "name": attrs.get("name") or cid, "provider": rec.get("provider"), + "website": rec.get("website"), "upstream_offering": rec.get("upstream_offering"), + **{f: rec[f] for f in TPR_FIELDS}, + "decided_by": rec["decided_by"], "decided_on": rec["decided_on"]}) + return out, problems + + +# --- FRC-CSX-MAS ------------------------------------------------------------------------- + +def ksi_application(signal, whole_set_types): + """Per KSI: the components its evidence domain covers, or the whole set. + A KSI whose evidence is every component (or none, a documentation KSI) + applies over the whole set; the rest apply per component.""" + comps = [c for c in signal.get("components", [])] + all_ids = {c["component_id"] for c in comps} + type_of = {c["component_id"]: c.get("type") for c in comps} + out, domain_types = [], set() + for k in signal.get("ksis", []): + refs = set((k.get("evidence") or {}).get("component_refs") or []) + whole = not refs or refs >= all_ids + if not whole: + domain_types.update(type_of[r] for r in refs if r in type_of) + out.append({"ksi": k["id"], "method": k.get("method"), "status": k.get("status"), + "applies_to": "whole set" if whole else "per component", + "components": len(all_ids) if whole else len(refs & all_ids), + "component_types": sorted({type_of[r] for r in (all_ids if whole else refs & all_ids)})}) + problems = [] + if not out: + problems.append("the inventory carries no KSIs") + for t in sorted({type_of[i] for i in all_ids}): + if t not in domain_types and not (isinstance(whole_set_types.get(t), str) and whole_set_types[t].strip()): + problems.append(f"component type {t} is in no KSI's per-component evidence domain and has no whole-set " + "reason (data/minimum-assessment-scope.json whole_set_types)") + per_type = {t: ("per component" if t in domain_types else f"whole set only: {whole_set_types[t]}") + for t in sorted({type_of[i] for i in all_ids}) if t in domain_types or t in whole_set_types} + return out, per_type, problems + + +# --- build ------------------------------------------------------------------------------- + +def build(signal, inputs, classification, tpr_doc, cloudfront_tf, bootstrap_context, commit, now): + comps = signal.get("components", []) + ids = {c["component_id"] for c in comps} + dst = set(inputs.get("data_store_types") or []) + hosts, p1 = place_hosts(csp_hosts(cloudfront_tf, bootstrap_context), inputs.get("off_origin_hosts") or {}, ids) + fl, p2 = flows(classification, comps, dst) + md, p3 = metadata_inventory(comps, inputs.get("metadata") or {}, dst) + tp, p4 = third_party(comps, tpr_doc) + ka, per_type, p5 = ksi_application(signal, inputs.get("whole_set_types") or {}) + problems = p1 + p2 + p3 + p4 + p5 + by_type = {} + for c in comps: + by_type[c.get("type")] = by_type.get(c.get("type"), 0) + 1 + resources = [{"component": c["component_id"], "type": c.get("type"), + "security_category": c.get("security_category"), + "function": (c.get("attributes") or {}).get("function")} + for c in sorted(comps, key=lambda c: c["component_id"]) if c.get("type") not in PACKAGE_TYPES] + doc = { + "schema": SCHEMA, + "generated_at": now, + "commit": commit, + "ksi_signal_id": signal.get("signal_id"), + "derivation": DERIVATION, + "code": CODE, + "information_resources": { + "total": len(comps), + "by_type": dict(sorted(by_type.items())), + "software_packages": sum(by_type.get(t, 0) for t in PACKAGE_TYPES), + "resources": resources, + "off_origin_hosts": hosts, + }, + "flows": fl, + "metadata_inventory": md, + "third_party_information_resources": tp, + "ksi_application": {"ksis": ka, "component_types": per_type}, + "checks": { + "off_origin_hosts_placed": not p1, + "every_external_service_and_data_store_in_a_flow": not p2, + "every_data_store_and_log_has_metadata": not p3, + "every_third_party_resource_decided_with_all_fields": not p4, + "every_component_type_in_a_ksi_domain": not p5, + }, + } + return doc, problems + + +def render_markdown(d): + ir = d["information_resources"] + out = ["# Minimum Assessment Scope", "", + f"_Generated {d['generated_at']} from inventory `{d['ksi_signal_id']}` at commit `{(d['commit'] or '')[:12]}`._", + "", "## How this is derived", "", d["derivation"], "", + "Code: " + ", ".join(f"`{v}`" for v in d["code"].values()), "", + "## Information resources (MAS-CSO-IIR)", "", + f"{ir['total']} inventoried components, {ir['software_packages']} of them software packages.", "", + "| Type | Count |", "|---|---|", *[f"| {t} | {n} |" for t, n in ir["by_type"].items()], "", + "| Component | Type | C/I/A | Function |", "|---|---|---|---|"] + for r in ir["resources"]: + sc = r.get("security_category") or {} + out.append(f"| `{r['component']}` | {r['type']} | {'/'.join(sc.get(o, '?') for o in OBJECTIVES)} | " + f"{(r.get('function') or '').replace('|', '/')} |") + out += ["", "Off-origin hosts the Content Security Policies allow:", "", + "| Host | Placement |", "|---|---|"] + for h in ir["off_origin_hosts"]: + out.append(f"| {h['host']} | " + (f"`{h['component']}`" if h.get("component") else f"Out of scope: {h['out_of_scope']}") + " |") + out += ["", "## Information flows and security categories (MAS-CSO-FLO)", "", + "| Flow | Endpoints | C/I/A | Protocol | Auth | Encryption | Data |", "|---|---|---|---|---|---|---|"] + for f in d["flows"]: + ends = " → ".join(e.get("declared") or ", ".join(e["components"]) or "(none)" for e in f["endpoints"]) + sc = f["security_category"] + out.append(f"| {f['id']}: {f['label']} | {ends} | {'/'.join(sc.get(o, '?') for o in OBJECTIVES)} | " + f"{f.get('protocol', '')} | {f.get('auth', '')} | {f.get('encryption', '')} | {f.get('data', '')} |") + out += ["", "## Metadata inventory (MAS-CSO-MDI)", "", + "| Component | Holds | Personal data | Retention | Location |", "|---|---|---|---|---|"] + for m in d["metadata_inventory"]: + out.append(f"| `{m['component']}` | {m['holds']} | {', '.join(m['personal_data']) or 'none'} | " + f"{m['retention']} | {m['location']} ({m['controlled_by']}) |") + out += ["", "## Third-party information resources (MAS-CSO-TPR)", ""] + for t in d["third_party_information_resources"]: + out += [f"### {t['name']} (`{t['component']}`)", "", + f"- **Provider:** {t['provider']} ({t['website']})", + f"- **General usage and configuration:** {t['usage_and_configuration']}", + f"- **Justification for use:** {t['justification']}", + f"- **Mitigation measures:** {t['mitigations']}", + f"- **Compensating controls:** {t['compensating_controls']}", + f"- **Decided by:** {t['decided_by']}, {t['decided_on']}", ""] + out += ["## Key Security Indicators applied to the scope (FRC-CSX-MAS)", "", + "| KSI | Method | Applies | Components | Types |", "|---|---|---|---|---|"] + for k in d["ksi_application"]["ksis"]: + out.append(f"| {k['ksi']} | {k['method']} | {k['applies_to']} | {k['components']} | {', '.join(k['component_types'])} |") + out += ["", "| Component type | Coverage |", "|---|---|", + *[f"| {t} | {v} |" for t, v in d["ksi_application"]["component_types"].items()], + "", "## Checks", "", *[f"- {k}: {'pass' if v else 'FAIL'}" for k, v in d["checks"].items()], + "", "---", "_Source of truth: the signed `minimum-assessment-scope.json`._"] + return "\n".join(out) + + +def _read_json(path, what): + try: + return json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as exc: + _fail(f"cannot read {what} {path} ({type(exc).__name__})") + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--ksi-signal", default="ksi-signal.json") + ap.add_argument("--inputs", default=str(INPUTS)) + ap.add_argument("--classification", default=str(CLASSIFICATION)) + ap.add_argument("--third-party", default=str(THIRD_PARTY)) + ap.add_argument("--cloudfront-tf", default=str(CLOUDFRONT_TF)) + ap.add_argument("--commit", default="") + ap.add_argument("--output", default="minimum-assessment-scope.json") + ap.add_argument("--md-output", default="minimum-assessment-scope.md") + a = ap.parse_args(argv) + signal = _read_json(a.ksi_signal, "the KSI signal") + inputs = _read_json(a.inputs, "the MAS inputs") + if inputs.get("schema") != "mas-inputs/1": + _fail(f"{a.inputs} is not a mas-inputs/1 document") + tf = Path(a.cloudfront_tf).read_text() + context = "\n".join(p.read_text() for p in sorted(Path(a.cloudfront_tf).parent.glob("*.tf"))) + doc, problems = build(signal, inputs, _read_json(a.classification, "the classification"), + _read_json(a.third_party, "the third-party records"), tf, context, a.commit, + datetime.now(timezone.utc).isoformat()) + if problems: + for p in problems: + print(f"::error::MAS: {p}") + _fail(f"{len(problems)} gap(s) in the Minimum Assessment Scope") + Path(a.output).write_text(json.dumps(doc, indent=2) + "\n") + if a.md_output: + Path(a.md_output).write_text(render_markdown(doc) + "\n") + print(f"minimum-assessment-scope: {doc['information_resources']['total']} components, {len(doc['flows'])} flows, " + f"{len(doc['metadata_inventory'])} metadata entries, {len(doc['third_party_information_resources'])} " + f"third-party records, {len(doc['ksi_application']['ksis'])} KSIs applied") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/mas/classification-gaps.json b/tests/fixtures/mas/classification-gaps.json new file mode 100644 index 00000000..88f5f34a --- /dev/null +++ b/tests/fixtures/mas/classification-gaps.json @@ -0,0 +1,6 @@ +{"_note": "Known-bad: flow X names a component the inventory does not have; the log group is in no flow.", + "declared": [{"key": "operator", "label": "Operator", "zone": "outside", "kind": "actor"}], + "flows": [ + {"id": "X", "label": "Operator → a ghost", "path": [{"declared": "operator"}, {"component": "aws::function::ghost"}]}, + {"id": "Y", "label": "Operator → app", "path": [{"declared": "operator"}, {"component": "aws::function::app"}]} + ]} diff --git a/tests/fixtures/mas/csp-unplaced.tf.fixture b/tests/fixtures/mas/csp-unplaced.tf.fixture new file mode 100644 index 00000000..cea2fdf6 --- /dev/null +++ b/tests/fixtures/mas/csp-unplaced.tf.fixture @@ -0,0 +1,11 @@ +# Known-bad: the CSP allows a host no placement covers (MAS-CSO-IIR). +resource "aws_cloudfront_response_headers_policy" "bad" { + security_headers_config { + content_security_policy { + content_security_policy = join("; ", [ + "default-src 'self'", + "script-src 'self' https://cdn.jsdelivr.net/npm/x/ https://unvetted-cdn.example.com/lib.js", + ]) + } + } +} diff --git a/tests/fixtures/mas/csp-unresolved.tf.fixture b/tests/fixtures/mas/csp-unresolved.tf.fixture new file mode 100644 index 00000000..9d2c987f --- /dev/null +++ b/tests/fixtures/mas/csp-unresolved.tf.fixture @@ -0,0 +1,11 @@ +# Known-bad: a CSP source names a local that is not defined, so its host cannot +# be known; the check must fail rather than skip it. +resource "aws_cloudfront_response_headers_policy" "bad" { + security_headers_config { + content_security_policy { + content_security_policy = join("; ", [ + "connect-src 'self' https://${local.not_defined_anywhere}", + ]) + } + } +} diff --git a/tests/fixtures/mas/signal-gaps.json b/tests/fixtures/mas/signal-gaps.json new file mode 100644 index 00000000..0fae7196 --- /dev/null +++ b/tests/fixtures/mas/signal-gaps.json @@ -0,0 +1,11 @@ +{"_note": "Known-bad: a log group with no metadata entry and in no flow, and a component type in no KSI's evidence domain with no whole-set reason.", + "signal_id": "fixture-gaps", + "components": [ + {"component_id": "aws::log_group::brand_new", "type": "log_group", "security_category": {"confidentiality": "low", "integrity": "moderate", "availability": "low"}, "attributes": {"region": "us-east-2"}}, + {"component_id": "aws::mystery::thing", "type": "mystery", "security_category": {"confidentiality": "low", "integrity": "low", "availability": "low"}, "attributes": {}}, + {"component_id": "aws::function::app", "type": "function", "security_category": {"confidentiality": "moderate", "integrity": "moderate", "availability": "low"}, "attributes": {}} + ], + "ksis": [ + {"id": "KSI-AAA-ONE", "method": "policy-validation", "status": "pass", "evidence": {"component_refs": ["aws::function::app", "aws::log_group::brand_new"]}}, + {"id": "KSI-PIY-ALL", "method": "inventory-attestation", "status": "pass", "evidence": {"component_refs": ["aws::function::app", "aws::log_group::brand_new", "aws::mystery::thing"]}} + ]} diff --git a/tests/fixtures/mas/signal.json b/tests/fixtures/mas/signal.json new file mode 100644 index 00000000..c591642d --- /dev/null +++ b/tests/fixtures/mas/signal.json @@ -0,0 +1,2254 @@ +{ + "_note": "test fixture: the non-package components of the 2026-11-05 inventory plus the external services this change adds, attributes trimmed; one package and one page keep their types represented", + "signal_id": "fixture", + "components": [ + { + "component_id": "aws::tls_certificate::website", + "type": "tls_certificate", + "security_category": { + "confidentiality": "not-applicable", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-1", + "function": "Public TLS certificate for the CDN" + } + }, + { + "component_id": "aws::cdn_distribution::website", + "type": "cdn_distribution", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Public content delivery (TLS 1.2+, OAC to origin)" + } + }, + { + "component_id": "aws::dns_zone::website", + "type": "dns_zone", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Authoritative DNS for the public domain" + } + }, + { + "component_id": "aws::object_store::website", + "type": "object_store", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Origin storage for static site content and /.well-known/ artifacts" + } + }, + { + "component_id": "aws::api_gateway::silk_reeling", + "type": "api_gateway", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "HTTP API fronting the Silk Reeling app Lambda (TLS termination, request routing, throttling)" + } + }, + { + "component_id": "aws::audit_log_trail::management", + "type": "audit_log_trail", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Account-wide management-event audit (CloudTrail)" + } + }, + { + "component_id": "aws::event_schedule::evidence_watchdog", + "type": "event_schedule", + "security_category": { + "confidentiality": "not-applicable", + "integrity": "low", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Hourly trigger for the evidence SLA watchdog Lambda" + } + }, + { + "component_id": "aws::event_schedule::opa_compliance", + "type": "event_schedule", + "security_category": { + "confidentiality": "not-applicable", + "integrity": "low", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Daily trigger for the runtime KSI emitter Lambda" + } + }, + { + "component_id": "aws::log_group::evidence_watchdog", + "type": "log_group", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "CloudWatch log group for Lambda execution / Route 53 query logs" + } + }, + { + "component_id": "aws::log_group::opa_compliance", + "type": "log_group", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "CloudWatch log group for Lambda execution / Route 53 query logs" + } + }, + { + "component_id": "aws::log_group::silk_apigw", + "type": "log_group", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "CloudWatch log group for the Silk Reeling API Gateway access logs" + } + }, + { + "component_id": "aws::log_group::silk_reeling", + "type": "log_group", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "CloudWatch log group for Lambda execution / Route 53 query logs" + } + }, + { + "component_id": "aws::metric_alarm::evidence[artifact-stale]", + "type": "metric_alarm", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Evidence SLA alarm on a watchdog metric (VDR age, runtime-signal age, nightly result)" + } + }, + { + "component_id": "aws::metric_alarm::evidence[nightly-failed]", + "type": "metric_alarm", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Evidence SLA alarm on a watchdog metric (VDR age, runtime-signal age, nightly result)" + } + }, + { + "component_id": "aws::metric_alarm::evidence[runtime-age]", + "type": "metric_alarm", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Evidence SLA alarm on a watchdog metric (VDR age, runtime-signal age, nightly result)" + } + }, + { + "component_id": "aws::metric_alarm::evidence[vdr-age]", + "type": "metric_alarm", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Evidence SLA alarm on a watchdog metric (VDR age, runtime-signal age, nightly result)" + } + }, + { + "component_id": "aws::identity_provider::silk_reeling", + "type": "identity_provider", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Cognito user pool gating the Silk Reeling app (admin-create users, required TOTP MFA, OAuth2 PKCE Hosted UI; issues the JWTs the API Gateway authorizer validates)" + } + }, + { + "component_id": "aws::iam_role::evidence_watchdog", + "type": "iam_role", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "IAM role assumed by the evidence SLA watchdog Lambda" + } + }, + { + "component_id": "aws::iam_role::lambda_opa", + "type": "iam_role", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "IAM role assumed by the runtime KSI emitter Lambda" + } + }, + { + "component_id": "aws::iam_role::silk_reeling", + "type": "iam_role", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "IAM role assumed by the Silk Reeling app Lambda" + } + }, + { + "component_id": "aws::iam_policy::evidence_watchdog", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role samaydlette-com-evidence-watchdog-role" + } + }, + { + "component_id": "aws::iam_policy::lambda_opa", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role samaydlette-com-lambda-opa-role" + } + }, + { + "component_id": "aws::iam_policy::silk_reeling", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role samaydlette-com-silk-reeling-role" + } + }, + { + "component_id": "aws::kms_key::at_rest", + "type": "kms_key", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Customer-managed KMS key for at-rest encryption / asymmetric signing" + } + }, + { + "component_id": "aws::kms_key::dnssec_ksk", + "type": "kms_key", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-1", + "function": "Customer-managed KMS key for at-rest encryption / asymmetric signing" + } + }, + { + "component_id": "aws::kms_key::runtime_signing", + "type": "kms_key", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Customer-managed KMS key for at-rest encryption / asymmetric signing" + } + }, + { + "component_id": "aws::kms_key::silk_reeling", + "type": "kms_key", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Customer-managed KMS key for at-rest encryption / asymmetric signing" + } + }, + { + "component_id": "aws::function::evidence_watchdog", + "type": "function", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Hourly evidence SLA watchdog; publishes evidence-age and nightly-result metrics for the alarms" + } + }, + { + "component_id": "aws::function::opa_compliance", + "type": "function", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Daily runtime KSI emitter; revalidates live AWS configuration" + } + }, + { + "component_id": "aws::function::silk_reeling", + "type": "function", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Silk Reeling Mirror application backend (Cognito-gated Tai Chi coaching API)" + } + }, + { + "component_id": "aws::object_store::cloudtrail", + "type": "object_store", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Delivery bucket for the account's CloudTrail management-event trail" + } + }, + { + "component_id": "aws::object_store::logs", + "type": "object_store", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Access-log bucket for the site bucket and CDN" + } + }, + { + "component_id": "aws::notification_topic::evidence_alerts", + "type": "notification_topic", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Evidence-alert topic: delivers CloudWatch alarm state changes to the operator by email" + } + }, + { + "component_id": "aws::message_queue::compliance_dlq", + "type": "message_queue", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Dead-letter queue for failed asynchronous invocations of the compliance and watchdog Lambdas" + } + }, + { + "component_id": "aws::kv_table::tflock", + "type": "kv_table", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Terraform state lock for the application and bootstrap stacks (one lock item per run; no state contents)" + } + }, + { + "component_id": "aws::iam_group::operators", + "type": "iam_group", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "IAM group for the human operator(s); holds IAM administration privileges (POAM-026)" + } + }, + { + "component_id": "aws::iam_group::read_only", + "type": "iam_group", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "IAM group granting read-only access (ReadOnlyAccess, with denies on secret values and CMK decryption)" + } + }, + { + "component_id": "aws::oidc_provider::github", + "type": "oidc_provider", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "GitHub Actions OIDC identity provider trusted by the CI/CD deploy role (workload identity; replaced long-lived keys per POAM-001)" + } + }, + { + "component_id": "aws::iam_role::deploy", + "type": "iam_role", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "CI deploy role (reviewer-gated prod environment)" + } + }, + { + "component_id": "aws::iam_role::evidence_nightly", + "type": "iam_role", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Nightly evidence workflow role" + } + }, + { + "component_id": "aws::iam_role::plan", + "type": "iam_role", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Read-only plan role for pull-request and main-branch checks" + } + }, + { + "component_id": "aws::iam_policy::cloudtrail_management", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::cognito_management", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::compliance_dlq", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::compliance_kms", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::compliance_logs", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::dnssec_management", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::evidence_nightly_cdn", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-evidence-nightly-oidc" + } + }, + { + "component_id": "aws::iam_policy::evidence_nightly_publish", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-evidence-nightly-oidc" + } + }, + { + "component_id": "aws::iam_policy::evidence_nightly_reconcile_reads", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-evidence-nightly-oidc" + } + }, + { + "component_id": "aws::iam_policy::evidence_watchdog_management", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::log_bucket_management", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::plan_access", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-plan-oidc" + } + }, + { + "component_id": "aws::iam_policy::reconcile_reads", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::iam_policy::tfstate_backend", + "type": "iam_policy", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Inline policy on role github-actions-deploy-oidc" + } + }, + { + "component_id": "aws::object_store::tfstate", + "type": "object_store", + "security_category": { + "confidentiality": "moderate", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "region": "us-east-2", + "function": "Remote Terraform state for the application and bootstrap stacks" + } + }, + { + "component_id": "ext::sigstore-fulcio", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "Sigstore Fulcio" + } + }, + { + "component_id": "ext::sigstore-rekor", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "Sigstore Rekor" + } + }, + { + "component_id": "ext::github-oidc", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "GitHub Actions OIDC token issuer" + } + }, + { + "component_id": "ext::github-repo", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "GitHub repository + Actions" + } + }, + { + "component_id": "ext::duo-mfa", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "Duo (operator MFA)" + } + }, + { + "component_id": "ext::github-advisory-db", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "GitHub Advisory Database" + } + }, + { + "component_id": "ext::cisa-kev", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "CISA Known Exploited Vulnerabilities catalog" + } + }, + { + "component_id": "ext::first-epss", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "FIRST Exploit Prediction Scoring System (EPSS) scores" + } + }, + { + "component_id": "ext::anchore-grype-db", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "Grype vulnerability database (Anchore)" + } + }, + { + "component_id": "ext::jsdelivr", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "jsDelivr CDN (MediaPipe Tasks Vision runtime)" + } + }, + { + "component_id": "ext::google-cloud-storage", + "type": "external_service", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "External service in boundary per ROT #2 (affects CIA without separate FedRAMP ATO)", + "name": "Google Cloud Storage (MediaPipe model bucket)" + } + }, + { + "component_id": "npm::@aws-sdk/checksums@3.1001.6", + "type": "npm_package", + "security_category": { + "confidentiality": "not-applicable", + "integrity": "moderate", + "availability": "not-applicable" + }, + "attributes": { + "function": "Runtime dependency of the compliance/KSI Lambda (npm)", + "name": "@aws-sdk/checksums" + } + }, + { + "component_id": "pypi::annotated-doc@0.0.5", + "type": "pypi_package", + "security_category": { + "confidentiality": "not-applicable", + "integrity": "moderate", + "availability": "not-applicable" + }, + "attributes": { + "function": "Dependency of the Silk Reeling application Lambda (PyPI)", + "name": "annotated-doc" + } + }, + { + "component_id": "html::404.html", + "type": "html_artifact", + "security_category": { + "confidentiality": "low", + "integrity": "moderate", + "availability": "low" + }, + "attributes": { + "function": "Public site content" + } + } + ], + "ksis": [ + { + "id": "KSI-CED-RAT", + "method": "documentation", + "status": "pass", + "evidence": { + "component_refs": [] + } + }, + { + "id": "KSI-CMT-LMC", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::cdn_distribution::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling" + ] + } + }, + { + "id": "KSI-CMT-RMV", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::cdn_distribution::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling" + ] + } + }, + { + "id": "KSI-CMT-RVP", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::cdn_distribution::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling" + ] + } + }, + { + "id": "KSI-CMT-VTD", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::cdn_distribution::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling" + ] + } + }, + { + "id": "KSI-CNA-DFP", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-EIS", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-IBP", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-MAT", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-OFA", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-RNT", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-RVP", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-CNA-ULN", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::identity_provider::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::dns_zone::website" + ] + } + }, + { + "id": "KSI-IAM-AAM", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::identity_provider::silk_reeling", + "aws::oidc_provider::github", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::iam_group::operators", + "aws::iam_group::read_only" + ] + } + }, + { + "id": "KSI-IAM-APM", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::identity_provider::silk_reeling", + "aws::oidc_provider::github", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::iam_group::operators", + "aws::iam_group::read_only" + ] + } + }, + { + "id": "KSI-IAM-ELP", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::identity_provider::silk_reeling", + "aws::oidc_provider::github", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::iam_group::operators", + "aws::iam_group::read_only" + ] + } + }, + { + "id": "KSI-IAM-JIT", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::identity_provider::silk_reeling", + "aws::oidc_provider::github", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::iam_group::operators", + "aws::iam_group::read_only" + ] + } + }, + { + "id": "KSI-IAM-SNU", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::identity_provider::silk_reeling", + "aws::oidc_provider::github", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::iam_group::operators", + "aws::iam_group::read_only" + ] + } + }, + { + "id": "KSI-IAM-SUS", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::identity_provider::silk_reeling", + "aws::oidc_provider::github", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::iam_group::operators", + "aws::iam_group::read_only" + ] + } + }, + { + "id": "KSI-INR-AAR", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::notification_topic::evidence_alerts" + ] + } + }, + { + "id": "KSI-INR-RIR", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::notification_topic::evidence_alerts" + ] + } + }, + { + "id": "KSI-INR-RPI", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::notification_topic::evidence_alerts" + ] + } + }, + { + "id": "KSI-MLA-ALA", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::message_queue::compliance_dlq", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::cdn_distribution::website", + "aws::notification_topic::evidence_alerts", + "aws::audit_log_trail::management" + ] + } + }, + { + "id": "KSI-MLA-EVC", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::message_queue::compliance_dlq", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::cdn_distribution::website", + "aws::notification_topic::evidence_alerts", + "aws::audit_log_trail::management" + ] + } + }, + { + "id": "KSI-MLA-LET", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::message_queue::compliance_dlq", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::cdn_distribution::website", + "aws::notification_topic::evidence_alerts", + "aws::audit_log_trail::management" + ] + } + }, + { + "id": "KSI-MLA-OSM", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::message_queue::compliance_dlq", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::cdn_distribution::website", + "aws::notification_topic::evidence_alerts", + "aws::audit_log_trail::management" + ] + } + }, + { + "id": "KSI-MLA-RVL", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::message_queue::compliance_dlq", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::cdn_distribution::website", + "aws::notification_topic::evidence_alerts", + "aws::audit_log_trail::management" + ] + } + }, + { + "id": "KSI-PIY-GIV", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::tls_certificate::website", + "aws::cdn_distribution::website", + "aws::dns_zone::website", + "aws::object_store::website", + "aws::api_gateway::silk_reeling", + "aws::audit_log_trail::management", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::identity_provider::silk_reeling", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::notification_topic::evidence_alerts", + "aws::message_queue::compliance_dlq", + "aws::kv_table::tflock", + "aws::iam_group::operators", + "aws::iam_group::read_only", + "aws::oidc_provider::github", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::object_store::tfstate", + "npm::@aws-sdk/checksums@3.1001.6", + "pypi::annotated-doc@0.0.5", + "html::404.html", + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage" + ] + } + }, + { + "id": "KSI-PIY-RES", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::tls_certificate::website", + "aws::cdn_distribution::website", + "aws::dns_zone::website", + "aws::object_store::website", + "aws::api_gateway::silk_reeling", + "aws::audit_log_trail::management", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::identity_provider::silk_reeling", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::notification_topic::evidence_alerts", + "aws::message_queue::compliance_dlq", + "aws::kv_table::tflock", + "aws::iam_group::operators", + "aws::iam_group::read_only", + "aws::oidc_provider::github", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::object_store::tfstate", + "npm::@aws-sdk/checksums@3.1001.6", + "pypi::annotated-doc@0.0.5", + "html::404.html", + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage" + ] + } + }, + { + "id": "KSI-PIY-RIS", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::tls_certificate::website", + "aws::cdn_distribution::website", + "aws::dns_zone::website", + "aws::object_store::website", + "aws::api_gateway::silk_reeling", + "aws::audit_log_trail::management", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::identity_provider::silk_reeling", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::notification_topic::evidence_alerts", + "aws::message_queue::compliance_dlq", + "aws::kv_table::tflock", + "aws::iam_group::operators", + "aws::iam_group::read_only", + "aws::oidc_provider::github", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::object_store::tfstate", + "npm::@aws-sdk/checksums@3.1001.6", + "pypi::annotated-doc@0.0.5", + "html::404.html", + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage" + ] + } + }, + { + "id": "KSI-PIY-RSD", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::tls_certificate::website", + "aws::cdn_distribution::website", + "aws::dns_zone::website", + "aws::object_store::website", + "aws::api_gateway::silk_reeling", + "aws::audit_log_trail::management", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::identity_provider::silk_reeling", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::notification_topic::evidence_alerts", + "aws::message_queue::compliance_dlq", + "aws::kv_table::tflock", + "aws::iam_group::operators", + "aws::iam_group::read_only", + "aws::oidc_provider::github", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::object_store::tfstate", + "npm::@aws-sdk/checksums@3.1001.6", + "pypi::annotated-doc@0.0.5", + "html::404.html", + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage" + ] + } + }, + { + "id": "KSI-PIY-RVD", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::tls_certificate::website", + "aws::cdn_distribution::website", + "aws::dns_zone::website", + "aws::object_store::website", + "aws::api_gateway::silk_reeling", + "aws::audit_log_trail::management", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::log_group::evidence_watchdog", + "aws::log_group::opa_compliance", + "aws::log_group::silk_apigw", + "aws::log_group::silk_reeling", + "aws::metric_alarm::evidence[artifact-stale]", + "aws::metric_alarm::evidence[nightly-failed]", + "aws::metric_alarm::evidence[runtime-age]", + "aws::metric_alarm::evidence[vdr-age]", + "aws::identity_provider::silk_reeling", + "aws::iam_role::evidence_watchdog", + "aws::iam_role::lambda_opa", + "aws::iam_role::silk_reeling", + "aws::iam_policy::evidence_watchdog", + "aws::iam_policy::lambda_opa", + "aws::iam_policy::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::notification_topic::evidence_alerts", + "aws::message_queue::compliance_dlq", + "aws::kv_table::tflock", + "aws::iam_group::operators", + "aws::iam_group::read_only", + "aws::oidc_provider::github", + "aws::iam_role::deploy", + "aws::iam_role::evidence_nightly", + "aws::iam_role::plan", + "aws::iam_policy::cloudtrail_management", + "aws::iam_policy::cognito_management", + "aws::iam_policy::compliance_dlq", + "aws::iam_policy::compliance_kms", + "aws::iam_policy::compliance_logs", + "aws::iam_policy::dnssec_management", + "aws::iam_policy::evidence_nightly_cdn", + "aws::iam_policy::evidence_nightly_publish", + "aws::iam_policy::evidence_nightly_reconcile_reads", + "aws::iam_policy::evidence_watchdog_management", + "aws::iam_policy::log_bucket_management", + "aws::iam_policy::plan_access", + "aws::iam_policy::reconcile_reads", + "aws::iam_policy::tfstate_backend", + "aws::object_store::tfstate", + "npm::@aws-sdk/checksums@3.1001.6", + "pypi::annotated-doc@0.0.5", + "html::404.html", + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage" + ] + } + }, + { + "id": "KSI-RPL-ABO", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate" + ] + } + }, + { + "id": "KSI-RPL-ARP", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate" + ] + } + }, + { + "id": "KSI-RPL-RRO", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate" + ] + } + }, + { + "id": "KSI-RPL-TRC", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate" + ] + } + }, + { + "id": "KSI-SCR-MIT", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage", + "pypi::annotated-doc@0.0.5", + "npm::@aws-sdk/checksums@3.1001.6" + ] + } + }, + { + "id": "KSI-SCR-MON", + "method": "inventory-attestation", + "status": "pass", + "evidence": { + "component_refs": [ + "ext::sigstore-fulcio", + "ext::sigstore-rekor", + "ext::github-oidc", + "ext::github-repo", + "ext::duo-mfa", + "ext::github-advisory-db", + "ext::cisa-kev", + "ext::first-epss", + "ext::anchore-grype-db", + "ext::jsdelivr", + "ext::google-cloud-storage", + "pypi::annotated-doc@0.0.5", + "npm::@aws-sdk/checksums@3.1001.6" + ] + } + }, + { + "id": "KSI-SVC-ACM", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-ASM", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-EIS", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-PRR", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-RUD", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-SIN", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-VCM", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + }, + { + "id": "KSI-SVC-VRI", + "method": "policy-validation", + "status": "pass", + "evidence": { + "component_refs": [ + "aws::api_gateway::silk_reeling", + "aws::cdn_distribution::website", + "aws::tls_certificate::website", + "aws::event_schedule::evidence_watchdog", + "aws::event_schedule::opa_compliance", + "aws::object_store::website", + "aws::object_store::cloudtrail", + "aws::object_store::logs", + "aws::object_store::tfstate", + "aws::function::evidence_watchdog", + "aws::function::opa_compliance", + "aws::function::silk_reeling", + "aws::kms_key::at_rest", + "aws::kms_key::dnssec_ksk", + "aws::kms_key::runtime_signing", + "aws::kms_key::silk_reeling" + ] + } + } + ] +} diff --git a/tests/fixtures/third-party/decided-without-tpr-fields.json b/tests/fixtures/third-party/decided-without-tpr-fields.json new file mode 100644 index 00000000..52e276b3 --- /dev/null +++ b/tests/fixtures/third-party/decided-without-tpr-fields.json @@ -0,0 +1,19 @@ +{ + "_note": "Known-bad (MAS-CSO-TPR): a decided record without the four fields FedRAMP requires.", + "components": { + "ext::cisa-kev": { + "upstream_offering": "CISA Known Exploited Vulnerabilities catalog (JSON feed)", + "upstream_component_id": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", + "responsibility_split": { + "upstream_runs": "the catalog and its feed", + "provider_configures": "the fetch and its failure handling" + }, + "evidence": [ + "test fixture" + ], + "decided_by": "Operator", + "decided_on": "2026-11-03", + "usage_and_configuration": "fixture usage" + } + } +} diff --git a/tests/fixtures/third-party/decided.json b/tests/fixtures/third-party/decided.json index 73739eef..7e3162da 100644 --- a/tests/fixtures/third-party/decided.json +++ b/tests/fixtures/third-party/decided.json @@ -1,8 +1,22 @@ -{"_note": "test fixture", "components": { +{ + "_note": "test fixture", + "components": { "ext::cisa-kev": { - "upstream_offering": "CISA Known Exploited Vulnerabilities catalog (JSON feed)", - "upstream_component_id": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", - "responsibility_split": {"upstream_runs": "the catalog and its feed", "provider_configures": "the fetch and its failure handling"}, - "evidence": ["test fixture"], - "decided_by": "Operator", "decided_on": "2026-11-03"} -}} + "upstream_offering": "CISA Known Exploited Vulnerabilities catalog (JSON feed)", + "upstream_component_id": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", + "responsibility_split": { + "upstream_runs": "the catalog and its feed", + "provider_configures": "the fetch and its failure handling" + }, + "evidence": [ + "test fixture" + ], + "decided_by": "Operator", + "decided_on": "2026-11-03", + "usage_and_configuration": "fixture usage", + "justification": "fixture justification", + "mitigations": "fixture mitigations", + "compensating_controls": "fixture compensating controls" + } + } +} diff --git a/tests/test_mas.py b/tests/test_mas.py new file mode 100644 index 00000000..aba6a710 --- /dev/null +++ b/tests/test_mas.py @@ -0,0 +1,234 @@ +"""The Minimum Assessment Scope is generated, complete, and fails closed on a gap +(MAS-CSO-IIR, MAS-CSO-FLO, MAS-CSO-MDI, MAS-CSO-TPR, FRC-CSX-MAS). + +Each invariant scripts/build-mas.py enforces has a committed known-bad fixture +under tests/fixtures/mas/ (and tests/fixtures/third-party/) that fails for that +reason, and a test that the repository's own inputs pass it. +""" +import importlib.util +import json +import re +import sys +from pathlib import Path + +import jsonschema +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "mas" +sys.path.insert(0, str(REPO / "scripts")) +import _redact # noqa: E402 +import _tpr # noqa: E402 + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +mas = _load("build_mas", REPO / "scripts" / "build-mas.py") +bks = _load("bks_mas", REPO / "scripts" / "build-ksi-signal.py") + +INPUTS = json.loads(mas.INPUTS.read_text()) +CLASSIFICATION = json.loads(mas.CLASSIFICATION.read_text()) +TPR = json.loads(mas.THIRD_PARTY.read_text()) +SIGNAL = json.loads((FIX / "signal.json").read_text()) +CF_TF = mas.CLOUDFRONT_TF.read_text() +BOOT = "\n".join(p.read_text() for p in sorted(mas.BOOTSTRAP_DIR.glob("*.tf"))) +DST = set(INPUTS["data_store_types"]) + + +def _build(signal=SIGNAL, classification=CLASSIFICATION, tf=CF_TF): + return mas.build(signal, INPUTS, classification, TPR, tf, BOOT, "0" * 40, "2026-11-05T00:00:00+00:00") + + +# --- the repository's own scope passes, and publishes nothing it must not --------------- + +def test_the_repository_scope_builds_with_no_gap(): + doc, problems = _build() + assert problems == [] + assert all(doc["checks"].values()) + + +def test_fixture_signal_carries_every_declared_external_service(): + # The fixture must track the inventory builder, or the other tests prove nothing. + ext = {c["component_id"] for c in bks.build_external_components()} + assert ext <= {c["component_id"] for c in SIGNAL["components"]} + + +def test_the_published_scope_carries_no_account_id_or_arn(): + doc, _ = _build() + text = json.dumps(doc) + assert not _redact.find(text) + assert "arn:aws" not in text + assert "@" not in text.replace("@mediapipe", "") + + +def test_markdown_twin_states_every_section_and_the_derivation(): + doc, _ = _build() + md = mas.render_markdown(doc) + for heading in ("How this is derived", "MAS-CSO-IIR", "MAS-CSO-FLO", "MAS-CSO-MDI", "MAS-CSO-TPR", "FRC-CSX-MAS"): + assert heading in md + for t in doc["third_party_information_resources"]: + assert t["compensating_controls"] in md + + +# --- MAS-CSO-IIR: every off-origin host in the CSPs is placed --------------------------- + +def test_csp_hosts_include_the_app_origins(): + hosts = mas.csp_hosts(CF_TF, BOOT) + assert {"cdn.jsdelivr.net", "storage.googleapis.com"} <= set(hosts) + assert any(h.endswith(".amazoncognito.com") for h in hosts) + + +def test_the_app_origins_are_inventoried_external_services(): + ids = {c["component_id"] for c in bks.build_external_components()} + assert {"ext::jsdelivr", "ext::google-cloud-storage", "ext::first-epss", "ext::anchore-grype-db"} <= ids + + +def test_unplaced_csp_host_fails(): + _, problems = _build(tf=(FIX / "csp-unplaced.tf.fixture").read_text()) + assert any("unvetted-cdn.example.com" in p for p in problems) + + +def test_unresolvable_csp_source_fails_closed(): + with pytest.raises(SystemExit): + mas.csp_hosts((FIX / "csp-unresolved.tf.fixture").read_text(), "") + + +def test_out_of_scope_host_needs_a_reason(): + _, problems = mas.place_hosts(["ochelli.com"], {"ochelli.com": {"out_of_scope": " "}}, set()) + assert problems + + +# --- MAS-CSO-FLO: every external service and data store is in a flow -------------------- + +def test_flow_gaps_fail(): + _, problems = mas.flows(json.loads((FIX / "classification-gaps.json").read_text()), + json.loads((FIX / "signal-gaps.json").read_text())["components"], DST) + assert any("aws::function::ghost" in p for p in problems) + assert any("aws::log_group::brand_new" in p and "no information flow" in p for p in problems) + + +def test_flow_category_is_the_high_water_mark_of_its_endpoints(): + comps = [{"component_id": "a", "type": "x", "security_category": {"confidentiality": "low", "integrity": "high", "availability": "low"}}, + {"component_id": "b", "type": "x", "security_category": {"confidentiality": "moderate", "integrity": "low", "availability": "low"}}] + cls = {"declared": [], "flows": [{"id": "F", "label": "a to b", "path": [{"component": "a"}, {"component": "b"}]}]} + out, problems = mas.flows(cls, comps, set()) + assert problems == [] + assert out[0]["security_category"] == {"confidentiality": "moderate", "integrity": "high", "availability": "low"} + + +def test_every_external_service_and_data_store_in_the_inventory_is_in_a_flow(): + _, problems = mas.flows(CLASSIFICATION, SIGNAL["components"], DST) + assert problems == [] + + +# --- MAS-CSO-MDI: every data store and log has a metadata entry ------------------------- + +def test_data_store_without_metadata_fails(): + _, problems = mas.metadata_inventory(json.loads((FIX / "signal-gaps.json").read_text())["components"], + INPUTS["metadata"], DST) + assert any("aws::log_group::brand_new" in p for p in problems) + + +def _tf_block(text, rtype, name): + m = re.search(r'resource\s+"%s"\s+"%s"\s*\{(.*?)\n\}' % (re.escape(rtype), re.escape(name)), text, re.S) + assert m, (rtype, name) + return m.group(1) + + +def _num(block, key): + m = re.search(r"\b%s\s*=\s*(\d+)" % key, block) + assert m, key + return int(m.group(1)) + + +def test_metadata_retention_matches_the_terraform(): + infra = REPO / "infrastructure" + every = "\n".join(p.read_text() for p in sorted(infra.glob("*.tf")) + sorted((infra / "bootstrap").glob("*.tf"))) + lg = {"opa_compliance": "opa_compliance", "evidence_watchdog": "evidence_watchdog", + "silk_reeling": "silk_reeling", "silk_apigw": "silk_apigw", "route53_query_log": "route53_query_log"} + md = INPUTS["metadata"] + for comp, res in lg.items(): + assert md[f"aws::log_group::{comp}"]["retention_days"] == _num( + _tf_block(every, "aws_cloudwatch_log_group", res), "retention_in_days"), comp + assert md["aws::object_store::logs"]["retention_days"] == _num( + _tf_block(every, "aws_s3_bucket_lifecycle_configuration", "logs"), "days") + assert md["aws::object_store::cloudtrail"]["retention_days"] == int( + re.search(r"expiration\s*\{\s*days\s*=\s*(\d+)", _tf_block(every, "aws_s3_bucket_lifecycle_configuration", "cloudtrail")).group(1)) + assert md["aws::object_store::website"]["retention_days"] == _num( + _tf_block(every, "aws_s3_bucket_lifecycle_configuration", "website"), "noncurrent_days") + assert md["aws::object_store::tfstate"]["retention_days"] == _num( + _tf_block(every, "aws_s3_bucket_lifecycle_configuration", "tfstate"), "noncurrent_days") + assert md["aws::message_queue::compliance_dlq"]["retention_days"] * 86400 == _num( + _tf_block(every, "aws_sqs_queue", "compliance_dlq"), "message_retention_seconds") + + +# --- MAS-CSO-TPR ------------------------------------------------------------------------ + +def test_decided_third_party_record_without_the_four_fields_fails_the_inventory(): + with pytest.raises(SystemExit): + bks.build_external_components(REPO / "tests" / "fixtures" / "third-party" / "decided-without-tpr-fields.json") + + +def test_every_third_party_resource_is_decided_with_the_four_fields(): + _, problems = mas.third_party(SIGNAL["components"], TPR) + assert problems == [] + + +def test_undecided_third_party_record_fails_the_scope(): + undecided = json.loads(json.dumps(TPR)) + undecided["components"]["ext::jsdelivr"]["decided_by"] = None + _, problems = mas.third_party(SIGNAL["components"], undecided) + assert any("ext::jsdelivr" in p for p in problems) + + +def test_published_inventory_carries_the_four_fields(): + for c in bks.build_external_components(): + for f in bks.MAS_TPR_FIELDS: + assert c["attributes"][f] not in (None, "", "not recorded"), (c["component_id"], f) + + +def test_cpo_helper_matches_the_cpo_schema(): + schema = json.loads((REPO / "infrastructure" / "schemas" / "fedramp-2026-06-24" / + "fedramp-certification-package-overview-schema-2026-06-24.json").read_text()) + out = _tpr.cpo_third_party_information_resources() + jsonschema.validate(out, schema["properties"]["thirdPartyInformationResources"]) + assert len(out["nonCertified"]) == len(TPR["components"]) + + +# --- FRC-CSX-MAS: every KSI applies to the scope, every component type is covered ------- + +def test_component_type_in_no_ksi_domain_fails(): + _, _, problems = mas.ksi_application(json.loads((FIX / "signal-gaps.json").read_text()), {}) + assert any("mystery" in p for p in problems) + + +def test_every_ksi_applies_and_every_component_type_is_covered(): + ka, per_type, problems = mas.ksi_application(SIGNAL, INPUTS["whole_set_types"]) + assert problems == [] + assert {k["ksi"] for k in ka} == {k["id"] for k in SIGNAL["ksis"]} + assert all(k["components"] > 0 or k["applies_to"] == "whole set" for k in ka) + assert set(per_type) == {c["type"] for c in SIGNAL["components"]} + + +def test_audit_trail_is_in_the_logging_ksi_domain(): + assert "audit_log_trail" in bks.KSI_FAMILY_EVIDENCE_TYPES["MLA"] + + +# --- the deploy builds it before the SDR and publishes it signed ------------------------ + +WF = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() + + +def test_deploy_builds_the_scope_before_the_sdr_and_publishes_it(): + build = WF.index("- name: Build the Minimum Assessment Scope") + sdr = WF.index("- name: Build Security Decision Record (SDR") + assert build < sdr + gate = WF[WF.index("- name: Publication redaction gate (no AWS account ID)"):WF.index("- name: Publish all signed artifacts")] + publish = WF[WF.index("- name: Publish all signed artifacts"):WF.index("- name: Attest artifact provenance")] + assert "minimum-assessment-scope" in gate and "minimum-assessment-scope" in publish + assert "--bundle minimum-assessment-scope.bundle minimum-assessment-scope.json" in WF From 6781424dc21d012b3e8623e10242be15eaa113b6 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:52:04 -0500 Subject: [PATCH 25/97] CR26 reports: stamp the offering's FedRAMP ID on the VDT, AVI and MRH providerExtensions.fedRampId, read from data/cds/offering.json fedramp_id (the field _offering.fedramp_id() reads on feat/sdr-schema). A missing ID stops the build. Adding these files to check-fedramp-id.py's list is left for integration. Co-Authored-By: Claude Opus 5.5 --- scripts/build-cr26-vdr.py | 18 +++++++++++++++++- tests/test_cr26_vdr.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/build-cr26-vdr.py b/scripts/build-cr26-vdr.py index 23019fcd..91f581fe 100644 --- a/scripts/build-cr26-vdr.py +++ b/scripts/build-cr26-vdr.py @@ -99,6 +99,22 @@ "checkov": "Checkov IaC scan", "tfsec": "tfsec IaC scan", "kev": "CISA KEV catalog", "epss": "FIRST EPSS scores"} +OFFERING = Path(__file__).resolve().parent.parent / "data" / "cds" / "offering.json" + + +def fedramp_id(path=OFFERING): + """The offering's FedRAMP ID (CDS-CSO-FID), stamped on every report as + providerExtensions.fedRampId. Reads the same field as _offering.fedramp_id() + (feat/sdr-schema); missing or empty stops the build.""" + try: + v = json.loads(Path(path).read_text(encoding="utf-8")).get("fedramp_id") + except (OSError, json.JSONDecodeError) as exc: + _fail(f"{path}: unreadable ({exc.__class__.__name__})") + if not isinstance(v, str) or not v.strip(): + _fail(f"{path}: fedramp_id is missing") + return v + + VDT_OUT = "vdr-detail-report.json" AVI_OUT = "vdr-accepted-vulnerabilities.json" MRH_OUT = "vdr-history.json" @@ -489,7 +505,7 @@ def build(vdr, register, previous_vdt, previous_avi, previous_mrh, now): cpo = {"certificationPackageOverviewUri": fs.CPO_URI} # REQ-GOV-2: like every VDR-family output, each names the method profile and # conformance results of the VDR report it came from. - ext = {"source": "vdr-report.json", "vdrReportId": vdr.get("report_id"), + ext = {"fedRampId": fedramp_id(), "source": "vdr-report.json", "vdrReportId": vdr.get("report_id"), "ksiSignalId": vdr.get("ksi_signal_id"), "profile": (vdr.get("methodology") or {}).get("profile")} vdt = {"$schema": fs.stamp("vdt"), **cpo, "reportPeriod": {"from": _period_start(previous_vdt, earliest), "to": to}, diff --git a/tests/test_cr26_vdr.py b/tests/test_cr26_vdr.py index 70f8ccc4..085ea133 100644 --- a/tests/test_cr26_vdr.py +++ b/tests/test_cr26_vdr.py @@ -534,3 +534,17 @@ def test_the_markdown_reports_carry_the_activity_and_link_the_runtime_results(tm text = (tmp_path / name).read_text() assert "## Activity this period" in text and "| OWASP ZAP dynamic scan (weekly) |" in text assert "(https://samaydlette.com/.well-known/ksi-signal-runtime.json)" in text + + +def test_every_report_carries_the_offerings_fedramp_id(tmp_path): + want = json.loads((REPO / "data" / "cds" / "offering.json").read_text())["fedramp_id"] + vdt, avi, mrh = _build() + for doc in (vdt, avi, mrh): + assert doc["providerExtensions"]["fedRampId"] == want + + +def test_a_missing_fedramp_id_stops_the_build(tmp_path): + p = tmp_path / "offering.json" + p.write_text("{}") + with pytest.raises(SystemExit, match="fedramp_id is missing"): + mod.fedramp_id(p) From 8fa18eebc2c900f19301bba2b017c49369a70617 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:52:55 -0500 Subject: [PATCH 26/97] feat(trust): agency access register, denial clock and exercise; sharing standard; DEC-165..167 Co-Authored-By: Claude Opus 5.5 --- .github/ISSUE_TEMPLATE/agency-access.yml | 36 +++ data/agency-access-exercise.json | 23 ++ data/agency-access.json | 5 + data/decisions.json | 56 ++++ docs/procedures/agency-access.md | 19 ++ docs/procedures/information-sharing.md | 23 ++ scripts/agency-access.py | 293 ++++++++++++++++++ .../agency-access/overdue-denial.json | 7 + .../agency-access/person-in-register.json | 7 + tests/test_agency_access.py | 100 ++++++ 10 files changed, 569 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/agency-access.yml create mode 100644 data/agency-access-exercise.json create mode 100644 data/agency-access.json create mode 100644 docs/procedures/agency-access.md create mode 100644 docs/procedures/information-sharing.md create mode 100644 scripts/agency-access.py create mode 100644 tests/fixtures/agency-access/overdue-denial.json create mode 100644 tests/fixtures/agency-access/person-in-register.json create mode 100644 tests/test_agency_access.py diff --git a/.github/ISSUE_TEMPLATE/agency-access.yml b/.github/ISSUE_TEMPLATE/agency-access.yml new file mode 100644 index 00000000..7f84524d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/agency-access.yml @@ -0,0 +1,36 @@ +name: Agency access to the certification package +description: For a federal agency (or FedRAMP) to be put on record, to withdraw, or to ask for the access inventory. The package itself is public and needs no request. +title: "Agency access: " +labels: ["agency-access"] +body: + - type: markdown + attributes: + value: | + **Everything in the certification package is already public** at https://samaydlette.com/trust/ and https://samaydlette.com/.well-known/trust-manifest.json; you do not need to ask to read it. + File this form to put your agency or system **on record** (so you are counted in the access inventory and can be told about changes), to **withdraw** an earlier record, or, for FedRAMP, to **request the inventory**. + This repository is public, so this issue is public. **Name agencies and systems, never people**; do not include names, email addresses or phone numbers. For a private request, use the security contact in `/.well-known/security.txt` instead. + This is a proof of concept with no federal data and no agency sponsor. Procedure: docs/procedures/agency-access.md. + - type: dropdown + id: kind + attributes: + label: Request + options: + - Put my agency or system on record (access) + - Withdraw an earlier record (withdrawal) + - FedRAMP request for the access inventory (inventory-request) + validations: + required: true + - type: input + id: agency + attributes: + label: Agency (or FedRAMP) + validations: + required: true + - type: input + id: system + attributes: + label: System or service (optional) + - type: input + id: withdraws + attributes: + label: For a withdrawal, the earlier request (AAR-...) diff --git a/data/agency-access-exercise.json b/data/agency-access-exercise.json new file mode 100644 index 00000000..81969a6a --- /dev/null +++ b/data/agency-access-exercise.json @@ -0,0 +1,23 @@ +{ + "_note": "The agency-access exercise the deploy runs (scripts/agency-access.py exercise). Every agency here is fictional and every date is a business-day offset from the day it runs, so the 5-business-day denial clock is exercised against today's calendar. The exercise fails the deploy unless every outcome in `expected` is observed, including the late and overdue notices the processor must catch.", + "disclaimer": "SIMULATED. These agencies do not exist, no federal agency has requested access, and nothing is sent to FedRAMP or anyone: a denial notice is this log entry. Any resemblance to a real agency is the result of a very tired naming committee.", + "requests": [ + {"id": "AAR-X01", "agency": "Department of Hypothetical Affairs (simulated)", "system": "Hypothetical Records Portal", "kind": "access", "channel": "issue form", "received_on_offset": -10, "decision": "granted", "decided_on_offset": -9, "decided_by": "Operator (exercise)"}, + {"id": "AAR-X02", "agency": "Bureau of Made-Up Statistics (simulated)", "system": "Totally Average Dashboard", "kind": "access", "channel": "security contact", "received_on_offset": -9, "decision": "denied", "decided_on_offset": -8, "decided_by": "Operator (exercise)", "reason": "Asked for write access to a read-only website.", "fedramp_notified_on_offset": -4}, + {"id": "AAR-X03", "agency": "Office of Strictly Imaginary Procurement (simulated)", "kind": "access", "channel": "issue form", "received_on_offset": -3, "decision": "denied", "decided_on_offset": -2, "decided_by": "Operator (exercise)", "reason": "Requested the package by fax."}, + {"id": "AAR-X04", "agency": "Agency of Unverifiable Claims (simulated)", "kind": "access", "channel": "security contact", "received_on_offset": -13, "decision": "denied", "decided_on_offset": -12, "decided_by": "Operator (exercise)", "reason": "Could not verify the agency exists (it does not).", "fedramp_notified_on_offset": -5}, + {"id": "AAR-X05", "agency": "Department of Hypothetical Affairs (simulated)", "system": "Second Opinion Service", "kind": "access", "channel": "issue form", "received_on_offset": -8, "decision": "granted", "decided_on_offset": -7, "decided_by": "Operator (exercise)"}, + {"id": "AAR-X06", "agency": "Department of Hypothetical Affairs (simulated)", "system": "Second Opinion Service", "kind": "withdrawal", "channel": "issue form", "received_on_offset": -1, "withdraws": "AAR-X05"}, + {"id": "AAR-X07", "agency": "Commission on Rhetorical Questions (simulated)", "kind": "access", "channel": "issue form", "received_on_offset": -8, "decision": "denied", "decided_on_offset": -7, "decided_by": "Operator (exercise)", "reason": "Asked whether access was really necessary, and we agreed."}, + {"id": "AAR-X08", "agency": "FedRAMP (simulated)", "kind": "inventory-request", "channel": "security contact", "received_on_offset": -1} + ], + "expected": { + "AAR-X01": "in inventory", + "AAR-X02": "denial notice on time", + "AAR-X03": "denial notice pending", + "AAR-X04": "denial notice late", + "AAR-X05": "withdrawn", + "AAR-X07": "denial notice overdue", + "AAR-X08": "inventory supplied" + } +} diff --git a/data/agency-access.json b/data/agency-access.json new file mode 100644 index 00000000..04c78dc0 --- /dev/null +++ b/data/agency-access.json @@ -0,0 +1,5 @@ +{ + "schema": "agency-access/1", + "_note": "The register of agency access requests, decisions and withdrawals for the certification package (CDS-TRC-AAI, CDS-UTC-AAD); procedure in docs/procedures/agency-access.md, read by scripts/agency-access.py, which the deploy runs and which fails on a denial not notified to FedRAMP within 5 business days. Records name agencies and systems, never people. No federal agency has filed a request: the list is empty, and the mechanism is exercised on every deploy with plainly fictional agencies (data/agency-access-exercise.json).", + "requests": [] +} diff --git a/data/decisions.json b/data/decisions.json index 1b4d402f..02592f41 100644 --- a/data/decisions.json +++ b/data/decisions.json @@ -1155,6 +1155,62 @@ "source": { "detector": "OWASP ZAP baseline scan" } + }, + { + "id": "DEC-165", + "title": "Agency access to the certification package: decided by the operator, denials notified to FedRAMP within 5 business days, inventory supplied on request", + "disposition": "operational-requirement", + "status": "in-force", + "justification": "The operator decides every agency access request and records it in data/agency-access.json (agencies and systems, never people); notifies FedRAMP of every denial within 5 business days; and supplies the access inventory and its history to FedRAMP on request (docs/procedures/agency-access.md). The package itself is public, so access needs no provisioning. The deploy fails when a denial notice is late or overdue, and exercises the procedure with fictional agencies on every run. AI-drafted 2026-11-05 for the operator to confirm.", + "controls": [ + "ac-21", + "ca-6" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [] + }, + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + { + "id": "DEC-166", + "title": "Public sharing of the certification package: disclosure standard and determination of no likely adverse effect", + "disposition": "operational-requirement", + "status": "in-force", + "justification": "The operator has determined that publishing the whole certification package is not likely to have an adverse effect on the offering, because it carries no account IDs, secrets, credentials, personal email addresses, IAM user names, run IDs, raw logs or access-request contacts (each refused by a build gate), and the detail published on purpose (role names and policy summaries, the API and CloudFront hostnames, configurations and findings) grants no access on its own (docs/procedures/information-sharing.md). Reviewed annually and when a new kind of artifact is published. AI-drafted 2026-11-05 for the operator to confirm.", + "controls": [ + "ac-22" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [] + }, + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" + }, + { + "id": "DEC-167", + "title": "Verification and validation use full populations, never representative samples", + "disposition": "operational-requirement", + "status": "in-force", + "justification": "The operator does not use representative samples in verification or validation: every check runs over its full population (every inventoried component, every live in-boundary resource, every SBOM, every rule record and every test it cites), and the simulated assessor re-checks every rule and KSI. The assessor guide states this (IVV-CSO-DUS). Any future use of sampling would be documented there and this record revised. AI-drafted 2026-11-05 for the operator to confirm.", + "controls": [ + "ca-2", + "ca-7" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [] + }, + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05" } ] } diff --git a/docs/procedures/agency-access.md b/docs/procedures/agency-access.md new file mode 100644 index 00000000..712f380b --- /dev/null +++ b/docs/procedures/agency-access.md @@ -0,0 +1,19 @@ +# Agency Access to the Certification Package + +This procedure covers how agencies obtain the FedRAMP certification package, how a request is decided, what happens when one is denied, and how the inventory of agencies with access is kept and supplied to FedRAMP. It implements CDS-UTC-AGA, CDS-UTC-AAD, CDS-TRC-AAI and CDS-TRC-SSM. Decision record: DEC-165. + +**The package is public.** Every piece of certification data is served without authentication from the trust center (https://samaydlette.com/trust/) and listed, with its URL, media type, signature and human-readable view, in the signed manifest https://samaydlette.com/.well-known/trust-manifest.json. An agency does not need to ask, register or hold an account to read or download any of it, so no request for the package itself is ever pending. + +**Self-service.** An agency that wants to be on record, so that it is counted in the access inventory and can be told about changes, files the "Agency access to the certification package" issue form in the public repository, or writes to the security contact in `/.well-known/security.txt` for a private request. The same form withdraws an earlier record. The register records agencies and systems, never people: no names, email addresses or phone numbers. + +**Decision.** The operator decides each access request and records it in `data/agency-access.json`: who decided, on what date, and, for a denial, why. Because everything is public, a denial can only concern something beyond the public package (for example, a request for write access or for data the system does not hold). + +**Denial notice.** FedRAMP is notified of every denial within 5 business days (US federal holidays excluded). The deploy runs `scripts/agency-access.py validate`, which computes each denial's deadline and fails the deploy when a notice is late or overdue. In this proof of concept nothing is sent to FedRAMP or anyone: the notice is the entry recorded in the register. + +**Inventory and history.** `scripts/agency-access.py inventory` prints the agencies and systems on record today and the full history of requests, decisions, withdrawals and notices. It is supplied to FedRAMP on request, through the same form or the security contact, and is not otherwise published. + +**Exercise.** No federal agency has filed a request. On every deploy, `scripts/agency-access.py exercise` runs the same code over a scenario of plainly fictional agencies (`data/agency-access-exercise.json`) dated relative to that day, including a late and an overdue denial notice that it must catch, and fails the deploy unless every outcome is the expected one. The signed log is published at https://samaydlette.com/.well-known/agency-access-exercise.json. + +**Access records.** A party that asks for the access records that concern it receives the trust center's access summaries for the artifacts it names (CDS-TRC-ACL). Individual viewers are never identified in those records. + +**Review cadence.** Annually with the [security review](../security-review.md). diff --git a/docs/procedures/information-sharing.md b/docs/procedures/information-sharing.md new file mode 100644 index 00000000..0d17faad --- /dev/null +++ b/docs/procedures/information-sharing.md @@ -0,0 +1,23 @@ +# Information Sharing and Disclosure Standard + +This standard says what certification data is published, what is never published, and why publishing it is not likely to harm the offering. It implements CDS-CSO-RIS and records the determination CDS-CSO-RPS permits. Decision record: DEC-166. + +**What is public.** The whole FedRAMP certification package: every artifact the signed manifest (https://samaydlette.com/.well-known/trust-manifest.json) lists, the policies and procedures it links, and the repository that builds them. It is public because the offering has no federal data and no agency sponsor, and because showing the evidence openly is the purpose of this proof of concept. Anyone may read, download and verify it. + +**What is never published.** The deploy refuses to publish, and the gates named here fail the build on: + +- AWS account IDs, and ARNs that carry one (publication redaction gate, `scripts/check-published-redaction.py`; ARNs are published with the account replaced by `ACCOUNT`); +- secret values, keys, tokens and credentials of any kind (never placed in the inventory; Terraform state is read owner-only and deleted after use); +- personal email addresses and IAM user names (the rule records, trust center and boundary map refuse them); +- GitHub run IDs and raw logs, including CloudFront access logs: only aggregate counts are published (CDS-TRC-ACL); +- the contacts behind an agency access request: the register names agencies and systems, never people. + +**What is public on purpose, after consideration.** Some published detail could look sensitive. Each is published because an authorizing official needs it to judge the system, and none grants access by itself: + +- IAM role names and summaries of their least-privilege policies: they show who can change what; assuming a role needs the GitHub OIDC trust bound to this repository's reviewed environment; +- the API Gateway endpoint host and the CloudFront domain: both are already discoverable through DNS and every page load; +- component configuration, scanner findings with their dispositions, and vulnerability reports: they show how the system is built and how each finding was decided, which an authorizing official cannot judge without. + +**Determination.** The operator has determined that sharing the package publicly in this form is not likely to have an adverse effect on the offering, because it carries none of the items listed above and the items published on purpose give no access on their own. The determination is reviewed annually, and whenever a new kind of artifact is published. + +**Review cadence.** Annually with the [security review](../security-review.md). diff --git a/scripts/agency-access.py b/scripts/agency-access.py new file mode 100644 index 00000000..e99a02aa --- /dev/null +++ b/scripts/agency-access.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Agency access to the certification package: the inventory, its history, and +the five-business-day denial clock (CDS-TRC-AAI, CDS-TRC-SSM, CDS-UTC-AAD, +CDS-UTC-AGA). + +The package is public, so access needs no provisioning; an agency that wants +to be on record (for notices, or to request anything not published) files a +request, and may withdraw it the same way (docs/procedures/agency-access.md). +Every request and decision is a record in data/agency-access.json, the +register. From it this computes: + + inventory the agencies and systems with access on record today; + history every request, decision and withdrawal, in date order; + denial notices FedRAMP must be told within 5 business days of a denial + (US federal holidays excluded); each notice is on time, + pending (inside the window), late, or overdue. + +`validate` fails the deploy on an invalid record, a late notice, or an overdue +one. `inventory` prints what goes to FedRAMP on request. `exercise` runs the +same code over data/agency-access-exercise.json, a scenario of plainly +fictional agencies dated relative to today, and fails unless every outcome is +the expected one, including the late and overdue cases it must catch. Nothing +is ever sent to FedRAMP or anyone: a notice is a log entry. +""" + +import argparse +import json +import re +import sys +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +REGISTER = REPO / "data" / "agency-access.json" +SCENARIO = REPO / "data" / "agency-access-exercise.json" +NOTICE_DAYS = 5 +KINDS = ("access", "withdrawal", "inventory-request") +DECISIONS = ("granted", "denied") +ID_RE = re.compile(r"^AAR-[A-Z0-9]{2,8}$") +EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+") +RECORD_KEYS = {"id", "agency", "system", "kind", "channel", "received_on", "decision", "decided_on", "decided_by", + "reason", "fedramp_notified_on", "withdraws"} + + +def _fail(msg): + raise SystemExit(f"::error::agency access: {msg} (fail-closed)") + + +# --- business days ------------------------------------------------------------------------ + +def _nth_weekday(year, month, weekday, n): + d = date(year, month, 1) + d += timedelta(days=(weekday - d.weekday()) % 7) + return d + timedelta(weeks=n - 1) + + +def _last_weekday(year, month, weekday): + d = date(year, month + 1, 1) - timedelta(days=1) if month < 12 else date(year, 12, 31) + return d - timedelta(days=(d.weekday() - weekday) % 7) + + +def _observed(d): + return d - timedelta(days=1) if d.weekday() == 5 else d + timedelta(days=1) if d.weekday() == 6 else d + + +def federal_holidays(year): + """US federal holidays (5 U.S.C. 6103), observed dates.""" + fixed = [date(year, 1, 1), date(year, 6, 19), date(year, 7, 4), date(year, 11, 11), date(year, 12, 25)] + floating = [_nth_weekday(year, 1, 0, 3), _nth_weekday(year, 2, 0, 3), _last_weekday(year, 5, 0), + _nth_weekday(year, 9, 0, 1), _nth_weekday(year, 10, 0, 2), _nth_weekday(year, 11, 3, 4)] + return {_observed(d) for d in fixed} | set(floating) + + +def is_business_day(d): + return d.weekday() < 5 and d not in federal_holidays(d.year) + + +def add_business_days(d, n): + step = 1 if n >= 0 else -1 + while n: + d += timedelta(days=step) + if is_business_day(d): + n -= step + return d + + +# --- the register ------------------------------------------------------------------------------ + +def _date(v, where): + try: + return date.fromisoformat(v) + except (TypeError, ValueError): + _fail(f"{where} is not a YYYY-MM-DD date: {v!r}") + + +def validate(doc, where): + if not isinstance(doc, dict) or doc.get("schema") != "agency-access/1" or not isinstance(doc.get("requests"), list): + _fail(f"{where} is not an agency-access/1 document") + seen = {} + for r in doc["requests"]: + rid = r.get("id") if isinstance(r, dict) else None + if not isinstance(rid, str) or not ID_RE.match(rid) or rid in seen: + _fail(f"{where}: request id {rid!r} is not a unique AAR-XXX") + if set(r) - RECORD_KEYS: + _fail(f"{rid}: unknown field(s) {sorted(set(r) - RECORD_KEYS)}") + for k in ("agency", "channel"): + if not isinstance(r.get(k), str) or not r[k].strip(): + _fail(f"{rid}: {k} must be a non-empty string") + # The register records agencies and systems, never people. + if any(isinstance(v, str) and EMAIL_RE.search(v) for v in r.values()): + _fail(f"{rid}: carries an email address; the register names agencies and systems, not people") + if r.get("kind") not in KINDS: + _fail(f"{rid}: kind must be one of {list(KINDS)}") + received = _date(r.get("received_on"), f"{rid} received_on") + if r["kind"] == "withdrawal": + if r.get("withdraws") not in seen or seen[r["withdraws"]].get("decision") != "granted": + _fail(f"{rid}: withdraws must name an earlier granted request") + if r.get("decision") is not None: + if r["decision"] not in DECISIONS or r["kind"] != "access": + _fail(f"{rid}: only an access request is decided, granted or denied") + if _date(r.get("decided_on"), f"{rid} decided_on") < received: + _fail(f"{rid}: decided before it was received") + if not isinstance(r.get("decided_by"), str) or not r["decided_by"].strip(): + _fail(f"{rid}: a decision names who made it (decided_by)") + if r["decision"] == "denied" and not (isinstance(r.get("reason"), str) and r["reason"].strip()): + _fail(f"{rid}: a denial states its reason") + if r.get("fedramp_notified_on") is not None: + if r.get("decision") != "denied": + _fail(f"{rid}: only a denial is notified to FedRAMP") + if _date(r["fedramp_notified_on"], f"{rid} fedramp_notified_on") < date.fromisoformat(r["decided_on"]): + _fail(f"{rid}: FedRAMP notified before the denial") + seen[rid] = r + return doc + + +def process(doc, today): + reqs = sorted(doc["requests"], key=lambda r: (r["received_on"], r["id"])) + withdrawn = {r["withdraws"]: r for r in reqs if r["kind"] == "withdrawal"} + inventory, history, notices = [], [], [] + for r in reqs: + history.append({"date": r["received_on"], "request": r["id"], "event": f"{r['kind']} received", + "agency": r["agency"], "system": r.get("system")}) + if r.get("decision"): + history.append({"date": r["decided_on"], "request": r["id"], "event": r["decision"], + "agency": r["agency"], "system": r.get("system")}) + if r.get("decision") == "granted": + w = withdrawn.get(r["id"]) + if w is None: + inventory.append({"request": r["id"], "agency": r["agency"], "system": r.get("system"), + "since": r["decided_on"]}) + else: + history.append({"date": w["received_on"], "request": r["id"], "event": "access withdrawn", + "agency": r["agency"], "system": r.get("system")}) + if r.get("decision") == "denied": + decided = date.fromisoformat(r["decided_on"]) + deadline = add_business_days(decided, NOTICE_DAYS) + sent = r.get("fedramp_notified_on") + if sent: + state = "on time" if date.fromisoformat(sent) <= deadline else "late" + history.append({"date": sent, "request": r["id"], "event": "denial notified to FedRAMP", + "agency": r["agency"], "system": r.get("system")}) + else: + state = "pending" if today <= deadline else "overdue" + notices.append({"request": r["id"], "agency": r["agency"], "denied_on": r["decided_on"], + "reason": r["reason"], "deadline": deadline.isoformat(), "notified_on": sent, + "state": state, + "to": "FedRAMP (simulated recipient: nothing is sent; this entry is the notice)"}) + history.sort(key=lambda h: (h["date"], h["request"])) + problems = [f"{n['request']}: FedRAMP denial notice {n['state']} (deadline {n['deadline']})" + for n in notices if n["state"] in ("late", "overdue")] + return {"as_of": today.isoformat(), "inventory": inventory, "history": history, "denial_notices": notices, + "inventory_requests": [r["id"] for r in reqs if r["kind"] == "inventory-request"], + "problems": problems} + + +def load(path): + p = Path(path) + try: + return validate(json.loads(p.read_text()), str(p)) + except (OSError, json.JSONDecodeError) as exc: + _fail(f"cannot read {p}: {exc}") + + +# --- the exercise --------------------------------------------------------------------------------- + +def materialize(scenario, today): + """Turn the scenario's business-day offsets (relative to today) into dates.""" + out = [] + for r in scenario["requests"]: + r = dict(r) + for k in ("received_on", "decided_on", "fedramp_notified_on"): + off = r.pop(f"{k}_offset", None) + if off is not None: + r[k] = add_business_days(today, off).isoformat() + out.append(r) + return {"schema": "agency-access/1", "requests": out} + + +def observe(result): + obs = {f"{i['request']}": "in inventory" for i in result["inventory"]} + for h in result["history"]: + if h["event"] == "access withdrawn": + obs[h["request"]] = "withdrawn" + for n in result["denial_notices"]: + obs[n["request"]] = f"denial notice {n['state']}" + for rid in result["inventory_requests"]: + obs[rid] = "inventory supplied" + return obs + + +def exercise(scenario, today): + doc = validate(materialize(scenario, today), "exercise scenario") + result = process(doc, today) + observed = observe(result) + expected = scenario["expected"] + mismatches = [f"{k}: expected {v!r}, observed {observed.get(k)!r}" for k, v in sorted(expected.items()) + if observed.get(k) != v] + mismatches += [f"{k}: observed {v!r} with no expectation" for k, v in sorted(observed.items()) + if k not in expected] + return doc, result, mismatches + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + v = sub.add_parser("validate") + v.add_argument("--register", default=str(REGISTER)) + i = sub.add_parser("inventory") + i.add_argument("--register", default=str(REGISTER)) + e = sub.add_parser("exercise") + e.add_argument("--scenario", default=str(SCENARIO)) + e.add_argument("--output", default="agency-access-exercise.json") + e.add_argument("--md-output", default="agency-access-exercise.md") + for p in (v, i, e): + p.add_argument("--today", default=None, help="YYYY-MM-DD (default: today, UTC)") + a = ap.parse_args(argv) + today = date.fromisoformat(a.today) if a.today else datetime.now(timezone.utc).date() + + if a.cmd in ("validate", "inventory"): + result = process(load(a.register), today) + if a.cmd == "inventory": + print(json.dumps({"as_of": result["as_of"], "inventory": result["inventory"], + "history": result["history"]}, indent=2)) + return 0 + for p in result["problems"]: + print(f"::error::{p}") + if result["problems"]: + _fail(f"{len(result['problems'])} denial notice(s) to FedRAMP late or overdue") + print(f"agency access: {len(result['inventory'])} on record; {len(result['denial_notices'])} denial " + f"notice(s), none late or overdue") + return 0 + + scenario = json.loads(Path(a.scenario).read_text()) + doc, result, mismatches = exercise(scenario, today) + for m in mismatches: + print(f"::error::agency access exercise: {m}") + if mismatches: + _fail("the exercise did not produce the expected outcomes") + log = {"log_type": "agency-access-exercise", "simulated": True, "counterparty": "agency", + "generated_at": datetime.now(timezone.utc).isoformat(), "as_of": today.isoformat(), + "disclaimer": scenario["disclaimer"], "procedure": "docs/procedures/agency-access.md", + "notice_window_business_days": NOTICE_DAYS, + "outcome": "every expected outcome observed", "expected": scenario["expected"], + "requests": doc["requests"], **{k: result[k] for k in ("inventory", "history", "denial_notices")}} + Path(a.output).write_text(json.dumps(log, indent=2) + "\n") + if a.md_output: + Path(a.md_output).write_text(render_markdown(log) + "\n") + print(f"agency access exercise: {len(scenario['expected'])} expected outcomes observed") + return 0 + + +def render_markdown(log): + out = ["# Agency access exercise (simulated)", "", f"_{log['disclaimer']}_", "", + f"- **As of:** {log['as_of']} | **Generated:** {log['generated_at']}", + f"- **Procedure:** `{log['procedure']}` | **Denial notice window:** " + f"{log['notice_window_business_days']} business days", f"- **Outcome:** {log['outcome']}", "", + "## Expected and observed", "", "| Request | Outcome |", "|---|---|"] + out += [f"| {k} | {v} |" for k, v in sorted(log["expected"].items())] + out += ["", "## Inventory on record", "", "| Request | Agency | System | Since |", "|---|---|---|---|"] + out += [f"| {i['request']} | {i['agency']} | {i.get('system') or '—'} | {i['since']} |" for i in log["inventory"]] + out += ["", "## Denial notices to FedRAMP (simulated; nothing is sent)", "", + "| Request | Agency | Denied | Deadline | Notified | State |", "|---|---|---|---|---|---|"] + out += [f"| {n['request']} | {n['agency']} | {n['denied_on']} | {n['deadline']} | {n['notified_on'] or '—'} | " + f"{n['state']} |" for n in log["denial_notices"]] + out += ["", "## History", "", "| Date | Request | Event | Agency |", "|---|---|---|---|"] + out += [f"| {h['date']} | {h['request']} | {h['event']} | {h['agency']} |" for h in log["history"]] + out += ["", "## Requests", "", "```json", json.dumps(log["requests"], indent=2), "```", + "", "---", "_Source of truth: the signed `agency-access-exercise.json`._"] + return "\n".join(out) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/agency-access/overdue-denial.json b/tests/fixtures/agency-access/overdue-denial.json new file mode 100644 index 00000000..28252a43 --- /dev/null +++ b/tests/fixtures/agency-access/overdue-denial.json @@ -0,0 +1,7 @@ +{ + "schema": "agency-access/1", + "_note": "Known-bad: a denial on 2026-11-02 with no FedRAMP notice; the 5-business-day window closed on 2026-11-09. Validation as of 2026-11-10 must fail.", + "requests": [ + {"id": "AAR-B01", "agency": "Bureau of Known-Bad Fixtures (simulated)", "kind": "access", "channel": "issue form", "received_on": "2026-11-02", "decision": "denied", "decided_on": "2026-11-02", "decided_by": "Operator", "reason": "Fixture."} + ] +} diff --git a/tests/fixtures/agency-access/person-in-register.json b/tests/fixtures/agency-access/person-in-register.json new file mode 100644 index 00000000..24a7e322 --- /dev/null +++ b/tests/fixtures/agency-access/person-in-register.json @@ -0,0 +1,7 @@ +{ + "schema": "agency-access/1", + "_note": "Known-bad: the register names a person's email address; it records agencies and systems only.", + "requests": [ + {"id": "AAR-B02", "agency": "Bureau of Known-Bad Fixtures (simulated)", "kind": "access", "channel": "jane.doe@example.gov", "received_on": "2026-11-02"} + ] +} diff --git a/tests/test_agency_access.py b/tests/test_agency_access.py new file mode 100644 index 00000000..a564b1f8 --- /dev/null +++ b/tests/test_agency_access.py @@ -0,0 +1,100 @@ +"""Agency access: inventory and history (CDS-TRC-AAI), self-service request and +withdrawal (CDS-TRC-SSM), access on request (CDS-UTC-AGA), and the +five-business-day denial notice to FedRAMP (CDS-UTC-AAD), exercised with +plainly fictional agencies.""" +import copy +import importlib.util +import json +from datetime import date +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "agency-access" +spec = importlib.util.spec_from_file_location("agency_access", REPO / "scripts" / "agency-access.py") +aa = importlib.util.module_from_spec(spec) +spec.loader.exec_module(aa) + + +@pytest.mark.parametrize("start,n,expected", [ + (date(2026, 11, 25), 5, date(2026, 12, 3)), # Thanksgiving (Thu 26 Nov) is skipped + (date(2026, 12, 23), 5, date(2026, 12, 31)), # Christmas (Fri 25 Dec) + (date(2027, 7, 1), 2, date(2027, 7, 6)), # 4 July 2027 is a Sunday, observed Monday 5 July + (date(2026, 11, 6), 1, date(2026, 11, 9)), # Friday to Monday +]) +def test_business_days_skip_weekends_and_federal_holidays(start, n, expected): + assert aa.add_business_days(start, n) == expected + + +def test_known_bad_an_unnotified_denial_past_five_business_days_fails(): + doc = aa.load(FIX / "overdue-denial.json") + assert aa.process(doc, date(2026, 11, 9))["problems"] == [] # last day of the window + assert aa.process(doc, date(2026, 11, 10))["problems"] == [ + "AAR-B01: FedRAMP denial notice overdue (deadline 2026-11-09)"] + with pytest.raises(SystemExit): + aa.main(["validate", "--register", str(FIX / "overdue-denial.json"), "--today", "2026-11-10"]) + + +def test_known_bad_the_register_never_names_a_person(): + with pytest.raises(SystemExit, match="not people"): + aa.load(FIX / "person-in-register.json") + + +def test_inventory_and_history_follow_grants_and_withdrawals(): + doc = aa.validate({"schema": "agency-access/1", "requests": [ + {"id": "AAR-T1", "agency": "A (simulated)", "system": "S1", "kind": "access", "channel": "form", + "received_on": "2026-11-02", "decision": "granted", "decided_on": "2026-11-03", "decided_by": "Operator"}, + {"id": "AAR-T2", "agency": "A (simulated)", "kind": "withdrawal", "channel": "form", + "received_on": "2026-11-04", "withdraws": "AAR-T1"}]}, "t") + r = aa.process(doc, date(2026, 11, 5)) + assert r["inventory"] == [] + assert [h["event"] for h in r["history"]] == ["access received", "granted", "access withdrawn", + "withdrawal received"] + + +@pytest.mark.parametrize("bad", [ + {"decision": "denied"}, # a denial with no reason or date + {"decision": "granted", "decided_on": "2026-11-01"}, # decided before received + {"fedramp_notified_on": "2026-11-03"}, # notice with no denial +]) +def test_invalid_records_fail(bad): + rec = {"id": "AAR-T1", "agency": "A (simulated)", "kind": "access", "channel": "form", + "received_on": "2026-11-02", "decided_by": "Operator", **bad} + with pytest.raises(SystemExit): + aa.validate({"schema": "agency-access/1", "requests": [rec]}, "t") + + +@pytest.mark.parametrize("today", ["2026-11-05", "2026-11-27", "2026-12-28", "2027-01-04", "2027-07-06"]) +def test_the_exercise_observes_every_expected_outcome_on_any_day(today): + scenario = json.loads(aa.SCENARIO.read_text()) + _, _, mismatches = aa.exercise(scenario, date.fromisoformat(today)) + assert mismatches == [] + + +def test_the_exercise_fails_when_an_outcome_is_not_the_expected_one(): + scenario = copy.deepcopy(json.loads(aa.SCENARIO.read_text())) + scenario["expected"]["AAR-X07"] = "denial notice on time" # the clock must catch it + _, _, mismatches = aa.exercise(scenario, date(2026, 11, 5)) + assert mismatches == ["AAR-X07: expected 'denial notice on time', observed 'denial notice overdue'"] + + +def test_the_exercise_agencies_are_obviously_fictional(): + scenario = json.loads(aa.SCENARIO.read_text()) + assert all(r["agency"].endswith("(simulated)") for r in scenario["requests"]) + assert "SIMULATED" in scenario["disclaimer"] and "nothing is sent" in scenario["disclaimer"] + + +def test_the_committed_register_is_valid_and_has_nothing_overdue(): + doc = aa.load(aa.REGISTER) + assert aa.process(doc, date(2026, 11, 5))["problems"] == [] + + +def test_the_exercise_writes_a_log_that_says_it_is_simulated(tmp_path): + out, md = tmp_path / "log.json", tmp_path / "log.md" + assert aa.main(["exercise", "--output", str(out), "--md-output", str(md), "--today", "2026-11-05"]) == 0 + log = json.loads(out.read_text()) + assert log["simulated"] is True and log["counterparty"] == "agency" + assert {n["state"] for n in log["denial_notices"]} == {"on time", "pending", "late", "overdue"} + assert all("nothing is sent" in n["to"] for n in log["denial_notices"]) + assert "Agency access exercise (simulated)" in md.read_text() From c8ceaa6fc48acf78695d5fc1d33f51e12cf81d80 Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:53:12 -0500 Subject: [PATCH 27/97] Check vectors in force through DEC-180; exploitation (DEC-181) and disclosure (DEC-182) policies The drafted per-check CVSS vectors are adopted by a decision record, and each names it: a decided vector whose record is missing or not in force stops the build (known-bad fixture). Configuration findings now get PAIN from the CVSS-Environmental derivation and LEV from Figure 7 instead of the defaults (VER-EVA-ELX, VER-EVA-EPA). DEC-180..182 are AI-drafted for Sam's confirmation. Co-Authored-By: Claude Opus 5.5 --- data/decisions.json | 61 ++++++ infrastructure/schemas/check-vectors.json | 191 ++++++++++------- .../schemas/check-vectors.schema.json | 5 + scripts/build-vdr-report.py | 8 +- .../check-vectors/decided-without-record.json | 14 ++ tests/fixtures/check-vectors/in-force.json | 3 +- tests/fixtures/check-vectors/lapsed.json | 3 +- tests/fixtures/evaluation/group-overdue.json | 9 + tests/test_evaluation.py | 200 ++++++++++++++++++ tests/test_figure7_lev.py | 11 + 10 files changed, 420 insertions(+), 85 deletions(-) create mode 100644 tests/fixtures/check-vectors/decided-without-record.json create mode 100644 tests/fixtures/evaluation/group-overdue.json create mode 100644 tests/test_evaluation.py diff --git a/data/decisions.json b/data/decisions.json index 3d3584d4..fd7d51a0 100644 --- a/data/decisions.json +++ b/data/decisions.json @@ -1160,6 +1160,67 @@ "source": { "detector": "OWASP ZAP baseline scan" } + }, + { + "id": "DEC-180", + "title": "Governed CVSS vectors for the configuration checks in infrastructure/schemas/check-vectors.json", + "status": "in-force", + "disposition": "operational-requirement", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "justification": "AI-drafted for the operator's confirmation. Configuration checks (Checkov, tfsec) carry no CVE and no published vector, so without a decided vector their likely exploitability rests on Figure 7's last step (no score: LEV) and their impact on the scanner's severity, with no system context (VER-EVA-ELX, VER-EVA-EPA). Each vector in check-vectors.json that names this decision is adopted: it states what exploiting the misconfiguration needs and what it reaches in this system, with its rationale, and the VDR applies it to every finding of that check, so PAIN comes from the CVSS-Environmental derivation with the component's classification and LEV from Figure 7 (floor, no direct impact). A check without an adopted vector keeps the defaults. Reviewed with the register; a changed check or a new resource type is re-decided.", + "controls": [ + "ra-3", + "ra-5" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [ + "infrastructure/schemas/check-vectors.json" + ] + } + }, + { + "id": "DEC-181", + "title": "Exploitation assessment of detected vulnerabilities (VDR-CSO-RES)", + "status": "in-force", + "disposition": "operational-requirement", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "justification": "AI-drafted for the operator's confirmation. Each VDR build records, per finding, whether exploitation has been observed: active when the source reports it, when the vulnerability is observed exploited, or when an incident record (data/incidents.json, once present) names the finding or its CVE; otherwise none observed, with that basis and the time assessed. Exploitation elsewhere is read from CISA KEV and the exploit-prediction score (known threats), which the triage policy already treats as likely exploitable. For each open internet-reachable, likely exploitable finding, the operator reviews the edge, API and CloudTrail logs for indicators tied to the advisory as part of incident response (docs/incident-response.md) and records any indication as an incident, which makes the finding's exploitation active in the next build.", + "controls": [ + "ra-5", + "si-4", + "ir-4" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [] + } + }, + { + "id": "DEC-182", + "title": "Responsible disclosure of vulnerability information (VER-RPT-NID, VER-RPT-RPD)", + "status": "in-force", + "disposition": "operational-requirement", + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "justification": "AI-drafted for the operator's confirmation. The provider discloses its vulnerability reports publicly (VER-RPT-RPD) and determines per finding that doing so will not likely lead to exploitation, recorded as public_disclosure on each finding. An open finding that is internet-reachable, likely exploitable and rated N3 or above is published in summary until it is remediated, fully mitigated or dispositioned: tracking ID, ratings, reachability, exploitability, dates, deadline and status stay public, so any party can make a risk-based decision; the description, observed locations, fixed versions and supplementary detail are withheld from the public artifacts (VDR report, detail report, accepted-vulnerability report and their renderings). Every other finding is disclosed in full: a dispositioned or lower-rated finding, or configuration already readable in the public source repository, gives an attacker nothing it would not otherwise have. Necessary parties (FedRAMP, agency customers) receive the withheld detail from the provider on request through the security contact in /.well-known/security.txt.", + "controls": [ + "ra-5", + "si-5", + "ir-6" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [] + } } ] } diff --git a/infrastructure/schemas/check-vectors.json b/infrastructure/schemas/check-vectors.json index e99ce330..decf53e2 100644 --- a/infrastructure/schemas/check-vectors.json +++ b/infrastructure/schemas/check-vectors.json @@ -1,223 +1,250 @@ { "$schema": "./check-vectors.schema.json", "version": "1.0", - "description": "Governed CVSS 3.1 base vectors for configuration checks, which carry no CVE and so no published vector. One vector per check ID, decided once by a named person and reviewed by a date; the VDR builder applies it to every finding of that check. It drives PAIN (CVSS-Environmental) and LEV (Figure 7: the floor, and the no-direct-impact rule). An entry without decided_by and decided_on is a proposal and is NOT in force; an entry past review_by lapses. Vectors marked AI-drafted were proposed for operator review.", + "description": "Governed CVSS 3.1 base vectors for configuration checks, which carry no CVE and so no published vector. One vector per check ID, decided once by a named person and reviewed by a date; the VDR builder applies it to every finding of that check. It drives PAIN (CVSS-Environmental) and LEV (Figure 7: the floor, and the no-direct-impact rule). An entry without decided_by and decided_on is a proposal and is NOT in force; an entry past review_by lapses. Vectors marked AI-drafted were proposed for operator review. A decided entry names the decision record that adopted it (decision); one whose record is missing or not in force fails the build.", "vectors": { "AVD-AWS-0015": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudTrail logs SSE-S3 instead of a customer-managed key: encrypted at rest; key custody only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "AVD-AWS-0089": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "Bucket access logging off: reduces detection only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "AVD-AWS-0132": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "SSE-S3 instead of a customer-managed key: data is encrypted at rest; key custody choice only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "AVD-AWS-0162": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudTrail not streamed to CloudWatch Logs: reduces alerting only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV2_AWS_10": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudTrail not streamed to CloudWatch Logs: trail still written; reduces alerting only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV2_AWS_47": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "No Log4j WAF managed rule: no Log4j in the stack, so the absent rule protects nothing exploitable.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV2_AWS_56": { "vector": "CVSS:3.1/AV:N/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H", "rationale": "Administrator policy on the operator group: a compromised operator credential controls the account.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV2_AWS_62": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "S3 event notifications off: reduces detection only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_115": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:L", "rationale": "No reserved concurrency: a request flood can exhaust account concurrency for other functions.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_117": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "Lambda outside a VPC: network-isolation depth; the function's own permissions bound what it reaches.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_119": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "DynamoDB default encryption instead of a customer-managed key: encrypted at rest; key custody only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_144": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "S3 cross-region replication off: resilience only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_145": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "SSE-S3 instead of a customer-managed key: data is encrypted at rest; key custody choice only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_174": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "No Log4j-specific WAF rule: no Log4j in the stack, so the absent rule protects nothing exploitable.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_18": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "S3 access logging off on a log-target bucket: reduces detection; gives an attacker no capability.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_23": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "S3 event notifications off: reduces detection only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_252": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudTrail SNS delivery notifications off: no consumer; no attacker capability.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_272": { "vector": "CVSS:3.1/AV:N/AC:L/PR:H/UI:N/S:U/C:N/I:H/A:N", "rationale": "No code signing: a holder of deploy rights can ship unsigned code.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_287": { "vector": "CVSS:3.1/AV:N/AC:L/PR:H/UI:N/S:U/C:L/I:N/A:N", "rationale": "Secrets metadata reads on a wildcard: a holder of the role can list secret metadata, not values.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_310": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudFront origin failover absent: resilience only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_35": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudTrail logs SSE-S3 instead of a customer-managed key: encrypted at rest; key custody only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_355": { "vector": "CVSS:3.1/AV:N/AC:L/PR:H/UI:N/S:U/C:L/I:N/A:N", "rationale": "Wildcard resource in a read-only assessment policy: a holder of the role can read more metadata than needed.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_356": { "vector": "CVSS:3.1/AV:N/AC:L/PR:H/UI:N/S:U/C:L/I:N/A:N", "rationale": "Wildcard resource for account-level enumeration: a holder of the role can enumerate the account.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_374": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudFront geo restriction off on a public site: no capability beyond the site's intended exposure.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_50": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "Lambda X-Ray tracing off: observability only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_68": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:L", "rationale": "No WAF on a public distribution: request floods and application attacks meet no edge filter; availability impact.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" }, "CKV_AWS_86": { "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", "rationale": "CloudFront origin failover absent: resilience only.", "drafted_by": "AI-drafted for operator review", - "decided_by": null, - "decided_on": null, - "review_by": "2027-11-03" + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "decision": "DEC-180" } } } diff --git a/infrastructure/schemas/check-vectors.schema.json b/infrastructure/schemas/check-vectors.schema.json index 33f6f926..01c28f74 100644 --- a/infrastructure/schemas/check-vectors.schema.json +++ b/infrastructure/schemas/check-vectors.schema.json @@ -58,6 +58,11 @@ "review_by": { "type": "string", "format": "date" + }, + "decision": { + "type": "string", + "pattern": "^DEC-\\d{3}$", + "description": "The decision record that adopted the vector; required once decided." } } } diff --git a/scripts/build-vdr-report.py b/scripts/build-vdr-report.py index 45543a9d..81322498 100755 --- a/scripts/build-vdr-report.py +++ b/scripts/build-vdr-report.py @@ -1522,10 +1522,11 @@ def _valid_v31(vector): return all(k in m and m[k] in _V3_VALUES[k] for k in _V3_REQUIRED) and len(m) == len(_V3_REQUIRED) -def load_check_vectors(path=None, today=None): +def load_check_vectors(path=None, today=None, decisions=None): """{check_id: entry} for entries in force today. Malformed file or entry fails closed.""" p = Path(path) if path else DEFAULT_CHECK_VECTORS today = today or datetime.now(timezone.utc).date() + in_force_ids = {r["id"] for r in _decisions.in_force(decisions or DECISIONS)} try: doc = json.loads(p.read_text()) except (OSError, json.JSONDecodeError) as exc: @@ -1543,6 +1544,11 @@ def load_check_vectors(path=None, today=None): if not e.get("decided_by") or not e.get("decided_on"): continue # proposed, not decided: not in force _parse_day(e["decided_on"], where) + # A vector is decided through the decision register (DEC-180): a decided + # entry that names no record in force would be a decision nobody made. + if e.get("decision") not in in_force_ids: + raise SystemExit(f"::error::{where}: decided, but decision {e.get('decision')!r} is not a decision " + "record in force (fail-closed)") if review_by < today: continue # lapsed in_force[check_id] = e diff --git a/tests/fixtures/check-vectors/decided-without-record.json b/tests/fixtures/check-vectors/decided-without-record.json new file mode 100644 index 00000000..9d99a0b8 --- /dev/null +++ b/tests/fixtures/check-vectors/decided-without-record.json @@ -0,0 +1,14 @@ +{ + "version": "0.0-fixture", + "description": "Known-bad: a vector marked decided that names no decision record in force. The build stops.", + "vectors": { + "CKV_AWS_18": { + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", + "rationale": "Access logging off: weakens detection only.", + "decided_by": "Operator", + "decided_on": "2026-11-01", + "review_by": "2027-11-01", + "decision": "DEC-998" + } + } +} diff --git a/tests/fixtures/check-vectors/in-force.json b/tests/fixtures/check-vectors/in-force.json index c52e2b6f..7161ae85 100644 --- a/tests/fixtures/check-vectors/in-force.json +++ b/tests/fixtures/check-vectors/in-force.json @@ -7,7 +7,8 @@ "rationale": "Access logging off: weakens detection only.", "decided_by": "Operator", "decided_on": "2026-11-01", - "review_by": "2027-11-01" + "review_by": "2027-11-01", + "decision": "DEC-180" } } } diff --git a/tests/fixtures/check-vectors/lapsed.json b/tests/fixtures/check-vectors/lapsed.json index 9c162673..a5fd4023 100644 --- a/tests/fixtures/check-vectors/lapsed.json +++ b/tests/fixtures/check-vectors/lapsed.json @@ -7,7 +7,8 @@ "rationale": "Access logging off: weakens detection only.", "decided_by": "Operator", "decided_on": "2026-11-01", - "review_by": "2026-10-01" + "review_by": "2026-10-01", + "decision": "DEC-180" } } } diff --git a/tests/fixtures/evaluation/group-overdue.json b/tests/fixtures/evaluation/group-overdue.json new file mode 100644 index 00000000..833da44d --- /dev/null +++ b/tests/fixtures/evaluation/group-overdue.json @@ -0,0 +1,9 @@ +{ + "_note": "Known-bad: two open findings of one group (one Checkov check, no decision). The first was evaluated 60 days ago; the second appeared today. Each alone is inside its own N2+LEV+IRV timeframe only for the newcomer; as a group (VER-EVA-GRV) the newcomer is on the old member's clock, so it is past the deadline and blocks (G3). Without grouping it would pass.", + "now": "2026-11-03T12:00:00+00:00", + "old_days": 60, + "items": [ + {"id": "old", "group": "check:checkov:CKV_TEST_1"}, + {"id": "new", "group": "check:checkov:CKV_TEST_1"} + ] +} diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 00000000..ca97007f --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,200 @@ +"""The evaluation record (VER-EVA-*) and the decisions that read it. + +N0 for a fully mitigated finding (VER-EVA-EPA); a group evaluated at its most +exposed member, with one clock (VER-EVA-GRV); early acceptance when the +projected remediation is past the maximum period (VER-TFR-MAV); evaluation +time (VER-TFR-EVU); the automatable assumption (VER-EVA-AIA); the exploitation +assessment (VDR-CSO-RES); the disclosure determination and the redaction it +drives (VER-RPT-NID, VER-RPT-RPD). +""" +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +sys.path.insert(0, str(REPO / "tests" / "fixtures" / "triage")) +import _evaluation # noqa: E402 +import _rules # noqa: E402 +import _triage # noqa: E402 +import golden_cases # noqa: E402 + +NOW = golden_cases.NOW +NS = _triage.to_ns(NOW) +DAY = 86400 * 10**9 +FIX = REPO / "tests" / "fixtures" / "evaluation" +FLOOR = {"AV": "N", "PR": "N", "UI": "N", "C": "H", "I": "N", "A": "N"} + + +def item(**kw): + base = {"id": "f1", "kind": "finding", "pain": "N2", "is_kev": False, "exploitation": None, "metrics": FLOOR, + "epss": None, "fully_mitigated": False, + "reach": {"observed": False, "component_id": "c", "is_package": False, "declared": True}, + "mitigation": None, "check_decision": None, "register": None, "advisory": "CVE-0000-0001", + "resource": "r", "fix_state": None, "clock": {"start_ns": NS, "first_evaluated_ns": NS}} + base.update(kw) + return base + + +def decide(items, **kw): + return _triage.decide(items, now=NOW, epss_threshold=0.1, margin_days=5, lev_overrides=None, + evaluation_days=kw.get("evaluation_days")) + + +# --- VER-EVA-EPA: N0 --------------------------------------------------------------- + +def test_fully_mitigated_takes_n0_from_the_rules_scale(): + d = decide([item(fully_mitigated=True, pain="N4")])[0] + assert d["pain"] == "N0" and d["sla_days"] is None and d["needs_remediation"] is False + + +# --- VER-EVA-GRV: the group is the unit ------------------------------------------------ + +def _group_items(): + fx = json.loads((FIX / "group-overdue.json").read_text()) + old_ns = NS - fx["old_days"] * DAY + old, new = fx["items"] + return [item(id=old["id"], group=old["group"], clock={"start_ns": old_ns, "first_evaluated_ns": old_ns}), + item(id=new["id"], group=new["group"])] + + +def test_known_bad_new_instance_of_an_old_group_is_on_the_groups_clock_and_blocks(): + old, new = decide(_group_items()) + assert old["sla_days"] == 48 and old["is_overdue"] and old["block_rule"] == "past-deadline" + assert new["due_ns"] > NS and new["group_due_ns"] == old["due_ns"] + assert new["is_overdue"] is True and new["blocking"] and new["block_rule"] == "past-deadline" + + +def test_without_a_group_the_new_instance_is_on_its_own_clock(): + items = _group_items() + for i in items: + i.pop("group") + assert decide(items)[1]["blocking"] is False + + +def test_group_is_evaluated_at_its_most_exposed_open_member(): + a = item(id="a", group="g", pain="N2", reach={"observed": False, "component_id": "c", "is_package": False, + "declared": False}) + b = item(id="b", group="g", pain="N3") + da, db = decide([a, b]) + assert da["irv"] is False and da["group_irv"] is True and da["group_pain"] == "N3" + assert da["group_due_ns"] == db["due_ns"] + + +def test_a_closed_member_does_not_set_the_groups_rating(): + closed = item(id="c", group="g", pain="N5", fully_mitigated=True) + d = decide([closed, item(id="o", group="g", pain="N2")]) + assert d[1]["group_pain"] == "N2" + + +# --- VER-TFR-MAV: will not be remediated in time ------------------------------------------ + +def test_projected_remediation_past_the_maximum_period_is_accepted_now(): + late = NS + 200 * DAY + d = decide([item(projected_ns=late)])[0] + assert d["accepted"] is True and d["accepted_basis"] == "projected-past-maximum-period" + ok = decide([item(projected_ns=NS + 30 * DAY)])[0] + assert ok["accepted"] is False and ok["accepted_basis"] is None + + +def test_a_closed_finding_is_never_accepted_early(): + d = decide([item(projected_ns=NS + 400 * DAY, fully_mitigated=True)])[0] + assert d["accepted"] is False + + +# --- VER-TFR-EVU ---------------------------------------------------------------------------- + +def test_evaluation_days_come_from_the_rules(): + assert _rules.evaluation_days() == 5 + + +def test_evaluated_more_than_five_days_after_detection_is_late(): + d = decide([item(detected_ns=NS - 9 * DAY)], evaluation_days=5)[0] + assert d["evaluation_late"] == "evaluated-late" + assert decide([item(detected_ns=NS - 2 * DAY)], evaluation_days=5)[0]["evaluation_late"] is None + + +def test_a_decision_before_the_pipeline_saw_it_counts_as_the_evaluation(): + d = decide([item(detected_ns=NS - 9 * DAY, evaluated_ns=NS - 8 * DAY)], evaluation_days=5)[0] + assert d["evaluation_late"] is None + + +def test_still_on_a_default_after_the_window_is_late(): + it = item(metrics={}, default_evaluation=True, detected_ns=NS - 6 * DAY, + clock={"start_ns": NS - 6 * DAY, "first_evaluated_ns": NS - 6 * DAY}) + assert decide([it], evaluation_days=5)[0]["evaluation_late"] == "default-after-evaluation-window" + + +def test_no_evaluation_days_no_check(): + assert decide([item(detected_ns=NS - 90 * DAY)])[0]["evaluation_late"] is None + + +# --- VER-EVA-AIA ------------------------------------------------------------------------------ + +def test_automatable_by_default(): + assert _evaluation.automatable({}, {})["value"] == "yes" + + +def test_only_a_decision_with_evidence_says_not_automatable(): + dec = {"id": "DEC-999", "automatable": {"value": "no", "evidence": "needs a physical token"}} + assert _evaluation.automatable({}, dec) == {"value": "no", "source": "decision DEC-999", + "evidence": "needs a physical token"} + assert _evaluation.automatable({}, {"id": "DEC-998", "automatable": {"value": "no"}})["value"] == "yes" + + +def test_cvss4_automatable_is_read(): + v = "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N/AU:N" + assert _evaluation.automatable({"cvss_vector": v}, {})["value"] == "no" + + +# --- VDR-CSO-RES ------------------------------------------------------------------------------- + +def test_an_incident_naming_the_finding_makes_exploitation_active(): + f = {"tracking_id": "grype-CVE-2099-1-x", "cve": "CVE-2099-1"} + inc = [{"id": "INC-001", "vulnerability": {"cve": "CVE-2099-1"}}] + assert _evaluation.exploitation(f, False, inc, "t")["value"] == "active" + none = _evaluation.exploitation(f, False, [], "t") + assert none["value"] == "none-observed" and _evaluation.EXPLOITATION_POLICY in none["basis"] + + +# --- VER-RPT-NID / RPD ------------------------------------------------------------------------- + +def _row(**kw): + r = {"final_disposition": "open", "internet_reachable": True, "likely_exploitable": True, "pain": "N3", + "fully_mitigated": False, "description": "how to exploit it", "locations": ["https://x/p"], + "fix_versions": ["1.2.3"], "supplementary_info": "", "tracking_id": "t"} + r.update(kw) + return r + + +def test_open_irv_lev_n3_is_summarized_and_redacted(): + r = _row() + r["public_disclosure"] = _evaluation.disclosure(r) + assert r["public_disclosure"]["level"] == "summary" + pub = _evaluation.redact(r) + assert "how to exploit" not in json.dumps(pub) + assert pub["locations"] is None and pub["fix_versions"] is None + assert pub["tracking_id"] == "t" and pub["pain"] == "N3" + + +def test_dispositioned_or_lower_rated_findings_are_disclosed_in_full(): + for kw in ({"final_disposition": "false-positive"}, {"pain": "N2"}, {"internet_reachable": False}, + {"likely_exploitable": False}): + r = _row(**kw) + r["public_disclosure"] = _evaluation.disclosure(r) + assert r["public_disclosure"]["level"] == "full", kw + assert _evaluation.redact(r) == r + + +def test_the_policies_named_are_in_force_in_the_register(): + import _decisions + live = {r["id"] for r in _decisions.in_force(_decisions.load())} + assert {_evaluation.EXPLOITATION_POLICY, _evaluation.DISCLOSURE_POLICY} <= live + + +# --- grouping keys ---------------------------------------------------------------------------- + +def test_group_keys(): + assert _evaluation.group_key({"cve": "CVE-1"}, "DEC-1") == "cve:CVE-1" + assert _evaluation.group_key({"source": "checkov", "tool_id": "CKV_1"}, "DEC-1") == "decision:DEC-1" + assert _evaluation.group_key({"source": "checkov", "tool_id": "CKV_1"}, None) == "check:checkov:CKV_1" diff --git a/tests/test_figure7_lev.py b/tests/test_figure7_lev.py index 8c10f68e..0bad2efc 100644 --- a/tests/test_figure7_lev.py +++ b/tests/test_figure7_lev.py @@ -147,6 +147,17 @@ def test_in_force_vector_is_applied_and_drives_lev_and_pain(): assert lev(f, irv=True) == (False, "no-direct-impact") +def test_known_bad_decided_vector_without_a_decision_record_fails_closed(): + with pytest.raises(SystemExit, match="is not a decision record in force"): + vdr.load_check_vectors(FIX / "decided-without-record.json", today=date(2026, 11, 3)) + + +def test_the_committed_vectors_are_in_force_through_their_decision(): + vec = vdr.load_check_vectors(REPO / "infrastructure" / "schemas" / "check-vectors.json", + today=date(2026, 11, 5)) + assert vec and {e["decision"] for e in vec.values()} == {"DEC-180"} + + def test_the_committed_vector_file_loads(): vdr.load_check_vectors(REPO / "infrastructure" / "schemas" / "check-vectors.json", today=date.today()) From 6de6236c98a542e815ec08b4e032554507c2e76c Mon Sep 17 00:00:00 2001 From: sam-aydlette Date: Thu, 5 Nov 2026 16:53:26 -0500 Subject: [PATCH 28/97] feat(trust): independent availability monitor, private access logging, CORS on /.well-known/ - availability-probe.yml + scripts/availability-probe.py: a monitor on GitHub Actions (outside AWS) probes the site, trust center, /.well-known/, the Silk Reeling app and its sign-in every 15 minutes, keeps 35 days, derives incidents, and publishes JSON + Markdown to the availability-status branch; an ongoing incident fails the run. The deploy fails when that feed is missing or stale, and availability.json is now built from it, never from deploys. - CloudFront standard logging v2 (bootstrap/trust-access-logs.tf) with a field allowlist that drops IP, cookies, query string, user agent and referrer; raw records expire after 90 days (logging.tf). The deploy fails unless the delivery exists within the allowlist, and publishes monthly aggregates for /.well-known/ and /trust/ only (access-summary.json, 13 months). - /.well-known/* gets its own response-headers policy with read-only CORS; the deploy checks CORS, JSON Content-Type and nosniff live. - DEC-160 (AI-drafted for the operator): what is logged, and a party's own records on request. - CDS-CSO-AVR, CDS-TRC-USH, CDS-TRC-ACL flip to Implemented. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/availability-probe.yml | 88 +++++++ .github/workflows/deploy-with-opa.yml | 58 +++- data/decisions.json | 24 ++ data/frr-records.json | 107 ++++++-- docs/trust/access-logging-privacy-notice.html | 29 ++ infrastructure/bootstrap/cloudfront.tf | 103 ++++++++ infrastructure/bootstrap/trust-access-logs.tf | 128 +++++++++ infrastructure/logging.tf | 28 +- infrastructure/schemas/freshness-config.json | 12 + scripts/availability-probe.py | 248 ++++++++++++++++++ scripts/build-access-summary.py | 203 ++++++++++++++ scripts/build-availability.py | 178 ++++++++----- scripts/check-edge-access-logging.py | 92 +++++++ scripts/check-wellknown-headers.py | 83 ++++++ .../availability/known-bad-stale-feed.json | 2 + .../edge-access-logging/delivery-source.json | 2 + .../edge-access-logging/good-deliveries.json | 3 + .../known-bad-deliveries.json | 3 + .../fixtures/wellknown-headers/known-bad.json | 4 + .../wellknown-headers/known-good.json | 4 + tests/test_availability_probe.py | 126 +++++++++ tests/test_edge_access_logging.py | 154 +++++++++++ tests/test_site_headers.py | 3 +- tests/test_wellknown_headers.py | 77 ++++++ 24 files changed, 1673 insertions(+), 86 deletions(-) create mode 100644 .github/workflows/availability-probe.yml create mode 100644 docs/trust/access-logging-privacy-notice.html create mode 100644 infrastructure/bootstrap/trust-access-logs.tf create mode 100644 scripts/availability-probe.py create mode 100644 scripts/build-access-summary.py create mode 100644 scripts/check-edge-access-logging.py create mode 100644 scripts/check-wellknown-headers.py create mode 100644 tests/fixtures/availability/known-bad-stale-feed.json create mode 100644 tests/fixtures/edge-access-logging/delivery-source.json create mode 100644 tests/fixtures/edge-access-logging/good-deliveries.json create mode 100644 tests/fixtures/edge-access-logging/known-bad-deliveries.json create mode 100644 tests/fixtures/wellknown-headers/known-bad.json create mode 100644 tests/fixtures/wellknown-headers/known-good.json create mode 100644 tests/test_availability_probe.py create mode 100644 tests/test_edge_access_logging.py create mode 100644 tests/test_wellknown_headers.py diff --git a/.github/workflows/availability-probe.yml b/.github/workflows/availability-probe.yml new file mode 100644 index 00000000..602a17b6 --- /dev/null +++ b/.github/workflows/availability-probe.yml @@ -0,0 +1,88 @@ +# ============================================================================= +# Availability probe (CDS-CSO-AVR, CDS-TRC-USH) +# ============================================================================= +# An availability monitor on infrastructure independent of the offering: it +# runs on GitHub Actions, not AWS, probes the core services over HTTPS every 15 +# minutes (scripts/availability-probe.py), keeps 35 days of history, derives +# availability incidents, and publishes JSON + Markdown to the repository's +# `availability-status` branch. GitHub serves that branch, so the status stays +# readable when the offering's CloudFront or S3 is down: +# https://raw.githubusercontent.com/sam-aydlette/samaydlette.com/availability-status/availability-status.json +# https://github.com/sam-aydlette/samaydlette.com/blob/availability-status/availability-status.md +# +# The branch holds one commit, replaced on every run (history lives in the +# JSON), so the repository does not grow by a commit every 15 minutes. The +# workflow's token can write contents (no narrower scope exists); the job only +# ever pushes refs/heads/availability-status, and main is protected by its own +# branch rules. The deploy reads this feed and fails when it is missing or more +# than 2 hours old, so the published package never cites a monitor that has +# stopped (scripts/availability-probe.py --check-fresh). +# +# Caveats stated in the feed: history begins at the first run; GitHub may delay +# a scheduled run under load, and disables schedules on a repository with no +# activity for 60 days (the deploy's freshness check then fails). +# ============================================================================= +name: Availability probe + +on: + schedule: + - cron: '*/15 * * * *' + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +concurrency: + group: availability-probe + cancel-in-progress: false + +jobs: + probe: + name: Probe the core services from outside AWS + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # push the status branch; nothing else + steps: + - name: Checkout the probe + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + + - name: Read the previous status feed + run: | + set -euo pipefail + # Public repository: an anonymous fetch. No branch yet means a first run. + if git fetch --depth 1 --quiet origin availability-status 2>/dev/null; then + git show FETCH_HEAD:availability-status.json > previous-status.json || echo '{}' > previous-status.json + else + echo '{}' > previous-status.json + fi + + - name: Probe the core services + run: python3 scripts/availability-probe.py --previous previous-status.json --out-dir status-out + + - name: Publish the status feed to the availability-status branch + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + cd status-out + git init --quiet + git checkout --quiet --orphan availability-status + git add availability-status.json availability-status.md + git -c user.name="availability-probe" -c user.email="availability-probe@users.noreply.github.com" \ + commit --quiet -m "Availability status $(jq -r .generated_at availability-status.json)" + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 -w0) + echo "::add-mask::$auth" + git -c http.extraheader="AUTHORIZATION: basic $auth" \ + push --quiet --force "https://github.com/${REPO}.git" HEAD:refs/heads/availability-status + + # After publishing: an ongoing incident fails the run, and GitHub notifies + # the operator of the failed scheduled run (email). The status is already + # published either way. + - name: Alert on an ongoing availability incident + run: python3 scripts/availability-probe.py --fail-on-incident status-out/availability-status.json diff --git a/.github/workflows/deploy-with-opa.yml b/.github/workflows/deploy-with-opa.yml index 24a24847..9ec89825 100644 --- a/.github/workflows/deploy-with-opa.yml +++ b/.github/workflows/deploy-with-opa.yml @@ -1574,17 +1574,59 @@ jobs: --md-output ongoing-certification-report.md jq '{next_ongoing_certification_report_date, next_quarterly_review_date}' ongoing-certification-report.json + # W8 edge checks (FRC-CSO-JSN, CDS-CSO-AVR, CDS-TRC-USH, CDS-TRC-ACL). Each + # depends on something outside this deploy (the operator-applied bootstrap + # stack, or the independent monitor on GitHub Actions) and fails the deploy + # when it is missing, before the SDR says the rule is met. + - name: Check /.well-known/ JSON response headers (CORS, Content-Type, nosniff) + env: + DOMAIN: ${{ vars.DOMAIN_NAME || 'samaydlette.com' }} + run: python3 scripts/check-wellknown-headers.py --site "https://$DOMAIN" + + - name: Read the independent availability feed (fails when missing or stale) + working-directory: infrastructure + env: + REPO: ${{ github.repository }} + run: | + set -euo pipefail + curl -fsS --max-time 30 -o availability-probe.json \ + "https://raw.githubusercontent.com/$REPO/availability-status/availability-status.json" + python3 ../scripts/availability-probe.py --check-fresh availability-probe.json --max-age-hours 2 + + # The CloudFront access records carry no IP, cookie, query string or user + # agent (the delivery's field allowlist, checked here); only /.well-known/ + # and /trust/ requests are counted, and only monthly aggregates published. + - name: Check edge access logging and build the access summary + working-directory: infrastructure + env: + DOMAIN: ${{ vars.DOMAIN_NAME || 'samaydlette.com' }} + run: | + set -euo pipefail + umask 077 + trap 'rm -rf edge-access-records edge-deliveries.json edge-delivery-source.json' EXIT + name="${DOMAIN//./-}-edge-access" + aws logs describe-deliveries --region us-east-1 --output json > edge-deliveries.json + aws logs get-delivery-source --name "$name" --region us-east-1 --output json > edge-delivery-source.json || true + python3 ../scripts/check-edge-access-logging.py \ + --deliveries edge-deliveries.json --source edge-delivery-source.json --name "$name" + bucket="${DOMAIN//./-}-logs" + account=$(aws sts get-caller-identity --query Account --output text) + aws s3 sync "s3://$bucket/cloudfront/" edge-access-records/cloudfront/ --only-show-errors + aws s3 sync "s3://$bucket/AWSLogs/$account/CloudFront/" edge-access-records/default/ --only-show-errors + curl -fsS --max-time 20 -o previous-access-summary.json "https://$DOMAIN/.well-known/access-summary.json" \ + || echo '{}' > previous-access-summary.json + python3 ../scripts/build-access-summary.py \ + --records edge-access-records --previous previous-access-summary.json + - name: Build CDS certification-data artifacts (overview, availability, policy index) working-directory: infrastructure run: | echo "Building CDS certification-overview package, availability feed, policy index..." - # availability history: fetch the previously-published feed (best-effort) - curl -sfL -o previous-availability.json "https://samaydlette.com/.well-known/availability.json" \ - || echo '{"history":[]}' > previous-availability.json python3 ../scripts/build-cds-overview.py \ --offering ../data/cds/offering.json --profile ../data/system-profile.json --ksi-signal ksi-signal.json + # Availability as measured by the independent monitor (read and checked above). python3 ../scripts/build-availability.py \ - --previous previous-availability.json --ksi-signal ksi-signal.json + --probe-feed availability-probe.json --ksi-signal ksi-signal.json python3 ../scripts/build-policy-index.py --output policy-index.json --md-output policy-index.md - name: Build Security Decision Record (SDR — 20x SSP replacement) @@ -1701,6 +1743,7 @@ jobs: cosign sign-blob --yes --new-bundle-format --bundle policy-index.bundle policy-index.json cosign sign-blob --yes --new-bundle-format --bundle security-decision-record.bundle security-decision-record.json cosign sign-blob --yes --new-bundle-format --bundle certification-package-overview.bundle certification-package-overview.json + cosign sign-blob --yes --new-bundle-format --bundle access-summary.bundle access-summary.json - name: Sign trust-center artifacts (boundary map, gate report, trust center) working-directory: infrastructure @@ -1729,6 +1772,7 @@ jobs: for a in vdr-detail-report vdr-accepted-vulnerabilities; do files="$files $a.md"; done files="$files ongoing-certification-report.json ongoing-certification-report.md" for a in certification-overview-package availability policy-index security-decision-record certification-package-overview; do files="$files $a.json $a.md"; done + for a in access-summary; do files="$files $a.json $a.md"; done for a in boundary-map reconcile-report trust-center conformance-results boundary-corroboration; do [ -f "$a.json" ] || continue files="$files $a.json" @@ -1788,6 +1832,12 @@ jobs: aws s3 cp "$a.bundle" "s3://$BUCKET_NAME/.well-known/$a.bundle" --content-type application/json --cache-control "public, max-age=300" aws s3 cp "$a.md" "s3://$BUCKET_NAME/.well-known/$a.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" done + # Trust-center access summary (CDS-TRC-ACL): aggregate counts only. + for a in access-summary; do + aws s3 cp "$a.json" "s3://$BUCKET_NAME/.well-known/$a.json" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp "$a.bundle" "s3://$BUCKET_NAME/.well-known/$a.bundle" --content-type application/json --cache-control "public, max-age=300" + aws s3 cp "$a.md" "s3://$BUCKET_NAME/.well-known/$a.md" --content-type "text/markdown; charset=utf-8" --cache-control "public, max-age=300" + done # Trust center: the generated boundary map, the gate's own report (it exists # only because the gate above passed) and the trust center's data. for a in boundary-map reconcile-report trust-center conformance-results boundary-corroboration; do diff --git a/data/decisions.json b/data/decisions.json index 1b4d402f..cc864646 100644 --- a/data/decisions.json +++ b/data/decisions.json @@ -1155,6 +1155,30 @@ "source": { "detector": "OWASP ZAP baseline scan" } + }, + { + "id": "DEC-160", + "title": "Trust-center access records: what is logged, and a party's own records on request", + "disposition": "operational-requirement", + "status": "in-force", + "justification": "AI-drafted for the operator to confirm. Access to the certification data is logged by CloudFront standard logging with a field allowlist that excludes IP address, forwarded-for, port, cookies, query string, user agent and referrer (infrastructure/bootstrap/trust-access-logs.tf); raw records expire after 90 days and only monthly aggregate counts for /.well-known/ and /trust/ are published, kept 13 months (scripts/build-access-summary.py). Because no requester identity is logged, the operator provides a party, on request to the security contact, the published summaries and any records matching the times and paths the party identifies, within 10 business days. Agencies are identified only by what they state in their own access request, never inferred from network data. This trades per-party attribution (CDS-TRC-ACL's SHOULD) for privacy, at the operator's direction of 2026-11-05.", + "controls": [ + "au-2", + "au-3", + "au-11", + "pt-2" + ], + "applies_to": { + "check_ids": [], + "vuln_ids": [], + "resources": [ + "aws-cloudfront-distribution" + ] + }, + "decided_by": "Sam Aydlette", + "decided_on": "2026-11-05", + "review_by": "2027-11-05", + "point_of_contact": "Sam Aydlette (operator)" } ] } diff --git a/data/frr-records.json b/data/frr-records.json index 6a767bed..c19a25d4 100644 --- a/data/frr-records.json +++ b/data/frr-records.json @@ -258,12 +258,37 @@ }, { "frr_id": "CDS-CSO-AVR", - "status": "Partially Implemented", - "implementation": "https://samaydlette.com/.well-known/availability.json + availability.md, scripts/build-availability.py (appends one point per deploy, L47-53; WINDOW_DAYS=45 L27).", - "evidence": [], - "gap": "Not an availability measurement: a point is written only when a deploy succeeds ('A deploy that produces this feed is, by definition, a successful one', L10-11), so an outage can never be recorded and there are no incidents field/records. Days without deploys are simply missing (27 points spanning 2026-08-06..11-05; ~10 in the last 30 days). Trim keeps the last 45 points, not 45 days, so history older than the window persists. Core services are not probed (Silk Reeling API/Cognito not checked at all). Served from the same CloudFront/S3 as the offering, so it is unavailable when the offering is (self-stated residual). To close: Run an external synthetic monitor on independent infrastructure (different provider/region/account and DNS name, e.g. a status subdomain on separate hosting) that probes the site, /.well-known/, and the Silk Reeling API/Cognito every few minutes; record incidents (start, end, affected service, cause); publish JSON + HTML status page from that independent host with 30+ days of continuous history.", - "closes_with": [ - "W8" + "status": "Implemented", + "implementation": "An availability monitor on GitHub Actions, independent of AWS (.github/workflows/availability-probe.yml, scripts/availability-probe.py), probes the website, the trust center, the certification data under /.well-known/, the Silk Reeling application and its Cognito sign-in every 15 minutes, keeps 35 days of history, and derives availability incidents (two or more consecutive failed probes of a service: start, end, failing check). It publishes JSON and a human-readable page to the repository's availability-status branch, served by GitHub, so the status stays readable when the offering is down; an ongoing incident fails the run, which notifies the operator. The deploy fails when that feed is missing or more than 2 hours old, and republishes it signed as /.well-known/availability.json with a Markdown twin (scripts/build-availability.py), which no longer records deploys as availability. History begins at the monitor's first run.", + "evidence": [ + { + "kind": "step", + "ref": "availability-probe.yml#Probe the core services" + }, + { + "kind": "step", + "ref": "availability-probe.yml#Publish the status feed to the availability-status branch" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Read the independent availability feed (fails when missing or stale)" + }, + { + "kind": "artifact", + "ref": ".well-known/availability.json" + }, + { + "kind": "test", + "ref": "tests/test_availability_probe.py::test_consecutive_failures_make_an_incident_and_a_single_one_is_transient" + }, + { + "kind": "test", + "ref": "tests/test_availability_probe.py::test_the_known_bad_stale_feed_fails_the_deploy_guard" + }, + { + "kind": "test", + "ref": "tests/test_availability_probe.py::test_the_monitor_runs_outside_aws_on_a_schedule_and_pushes_only_its_branch" + } ] }, { @@ -391,12 +416,37 @@ }, { "frr_id": "CDS-TRC-ACL", - "status": "Partially Implemented", - "implementation": "S3 server access logging on the website bucket (infrastructure/logging.tf:114) to a log bucket retained 400 days (logging.tf:59).", - "evidence": [], - "gap": "S3 server access logs only see origin fetches; CloudFront edge hits are not logged (no logging_config on the distribution, bootstrap/cloudfront.tf:233-311), so most access to certification data is invisible. No access summaries are produced or stored; no explanation of how parties obtain their log information. To close: Enable CloudFront standard (or real-time) logging for /.well-known/ and /trust/, build a scheduled job that writes monthly access summaries (per artifact, per requester/agency once authenticated) retained >= 6 months, and document how a party requests its records.", - "closes_with": [ - "W8" + "status": "Implemented", + "implementation": "CloudFront standard logging (v2) records every request to the site with a field allowlist that excludes IP address, forwarded-for, port, cookies, query string, user agent and referrer (infrastructure/bootstrap/trust-access-logs.tf). Records go to the encrypted, non-public access-log bucket and expire after 90 days (infrastructure/logging.tf). On every deploy, scripts/check-edge-access-logging.py fails the deploy unless that delivery exists and logs only allowlisted fields, and scripts/build-access-summary.py counts the requests to /.well-known/ and /trust/ (CloudFront cannot limit logging by path, so the limit is applied here and every other record is discarded) into monthly summaries per artifact, response class and country, published signed at /.well-known/access-summary.json with a Markdown twin and carried forward for 13 months. A party can ask for the summaries and for the records of its own requests, identified by the times and paths it gives, since no requester identity is logged (DEC-160); the trust page's privacy notice says so.", + "evidence": [ + { + "kind": "step", + "ref": "deploy-with-opa.yml#Check edge access logging and build the access summary" + }, + { + "kind": "artifact", + "ref": ".well-known/access-summary.json" + }, + { + "kind": "test", + "ref": "tests/test_edge_access_logging.py::test_the_known_bad_fixture_fails_for_identifying_fields" + }, + { + "kind": "test", + "ref": "tests/test_edge_access_logging.py::test_only_trust_paths_are_counted_and_nothing_identifying_survives" + }, + { + "kind": "test", + "ref": "tests/test_edge_access_logging.py::test_a_month_younger_than_six_is_never_dropped" + }, + { + "kind": "test", + "ref": "tests/test_edge_access_logging.py::test_raw_edge_records_expire_after_90_days" + }, + { + "kind": "decision", + "ref": "DEC-160" + } ] }, { @@ -431,12 +481,33 @@ }, { "frr_id": "CDS-TRC-USH", - "status": "Partially Implemented", - "implementation": "The trust center is a static public site on CloudFront; /trust/ and every directory URL serve their index through a CloudFront Function (W1), and /.well-known/*.json are served directly.", - "evidence": [], - "gap": "Availability of the trust center is not measured independently (CDS-CSO-AVR), there is a single origin with no failover (DEC-009), and the page's content needs JavaScript.", - "closes_with": [ - "W8" + "status": "Implemented", + "implementation": "The certification data is static, signed files under /.well-known/ served from the CloudFront edge (cached, with AWS Shield Standard), readable as plain JSON and Markdown without JavaScript and, since the deploy checks it, with CORS for programs on any origin. Interruptions are measured from outside AWS every 15 minutes, and an ongoing one fails the monitor's run, which notifies the operator (CDS-CSO-AVR); the status itself is served by GitHub, so it stays available during an outage. The single origin without failover is an accepted risk (DEC-009).", + "evidence": [ + { + "kind": "step", + "ref": "availability-probe.yml#Alert on an ongoing availability incident" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Read the independent availability feed (fails when missing or stale)" + }, + { + "kind": "step", + "ref": "deploy-with-opa.yml#Check /.well-known/ JSON response headers (CORS, Content-Type, nosniff)" + }, + { + "kind": "artifact", + "ref": ".well-known/availability.json" + }, + { + "kind": "test", + "ref": "tests/test_availability_probe.py::test_an_ongoing_incident_fails_the_monitor_run_after_publishing" + }, + { + "kind": "decision", + "ref": "DEC-009" + } ] }, { diff --git a/docs/trust/access-logging-privacy-notice.html b/docs/trust/access-logging-privacy-notice.html new file mode 100644 index 00000000..9a91c1b2 --- /dev/null +++ b/docs/trust/access-logging-privacy-notice.html @@ -0,0 +1,29 @@ + +
+

Access logging and your privacy

+

FedRAMP asks trust centers to log access to certification data and keep summaries of it + (CDS-TRC-ACL). This is what this site logs, and what it does not.

+
+
diff --git a/infrastructure/bootstrap/cloudfront.tf b/infrastructure/bootstrap/cloudfront.tf index da368e29..4242dfc7 100644 --- a/infrastructure/bootstrap/cloudfront.tf +++ b/infrastructure/bootstrap/cloudfront.tf @@ -241,6 +241,85 @@ resource "aws_cloudfront_response_headers_policy" "silk_reeling" { } } +# The published certification data under /.well-known/ (FRC-CSO-JSN, CDS-TRC-PAC): +# the static site's security headers, plus CORS so a browser application on +# another origin can read the JSON. Read-only (GET/HEAD/OPTIONS), any origin, no +# credentials: the data is public. CORP is cross-origin here, not same-origin, +# for the same reason. X-Content-Type-Options: nosniff is sent; the Content-Type +# is the object's own (set at upload). The deploy checks all three live and +# fails without them (scripts/check-wellknown-headers.py). +resource "aws_cloudfront_response_headers_policy" "well_known" { + name = "${local.domain_dashed}-well-known-headers" + comment = "Security headers and read-only CORS for ${var.domain_name}/.well-known/" + + cors_config { + access_control_allow_credentials = false + access_control_allow_headers { + items = ["*"] + } + access_control_allow_methods { + items = ["GET", "HEAD", "OPTIONS"] + } + access_control_allow_origins { + items = ["*"] + } + access_control_max_age_sec = 3600 + origin_override = true + } + + security_headers_config { + strict_transport_security { + access_control_max_age_sec = 31536000 + include_subdomains = true + preload = true + override = true + } + content_type_options { + override = true + } + frame_options { + frame_option = "DENY" + override = true + } + referrer_policy { + referrer_policy = "strict-origin-when-cross-origin" + override = true + } + content_security_policy { + content_security_policy = "default-src 'none'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'" + override = true + } + xss_protection { + mode_block = true + protection = true + override = true + } + } + + custom_headers_config { + items { + header = "Permissions-Policy" + value = "camera=(), microphone=(), geolocation=(), payment=(), usb=()" + override = true + } + items { + header = "Cross-Origin-Opener-Policy" + value = "same-origin" + override = true + } + items { + header = "Cross-Origin-Resource-Policy" + value = "cross-origin" + override = true + } + items { + header = "Cross-Origin-Embedder-Policy" + value = "credentialless" + override = true + } + } +} + resource "aws_cloudfront_distribution" "website" { # Scanner findings that describe this distribution's deliberate posture, each # classified here at scan-time (not silently dropped) and tracked in the POA&M @@ -306,6 +385,30 @@ resource "aws_cloudfront_distribution" "website" { } } + # Certification data: cached and forwarded like the static site, with the + # /.well-known/ headers policy (CORS) above. Read-only methods: nothing is + # written through the edge. The CORS headers come from the policy, so the + # Origin header is not forwarded and the cache is not split by it. No + # viewer-request function: every path here names a file. + ordered_cache_behavior { + path_pattern = "/.well-known/*" + target_origin_id = "S3-${var.domain_name}" + response_headers_policy_id = aws_cloudfront_response_headers_policy.well_known.id + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["HEAD", "GET", "OPTIONS"] + cached_methods = ["HEAD", "GET"] + compress = true + min_ttl = 0 + default_ttl = 3600 + max_ttl = 86400 + forwarded_values { + query_string = false + cookies { + forward = "none" + } + } + } + # App path: no caching, forward everything except Host (so the JWT authorizer # sees the Authorization header), strip the prefix at the edge, and add the # app's security headers. diff --git a/infrastructure/bootstrap/trust-access-logs.tf b/infrastructure/bootstrap/trust-access-logs.tf new file mode 100644 index 00000000..1315f10a --- /dev/null +++ b/infrastructure/bootstrap/trust-access-logs.tf @@ -0,0 +1,128 @@ +# ============================================================================= +# Trust-center access logging (CDS-TRC-ACL), with the operator's privacy limits +# ============================================================================= +# CloudFront standard logging (v2: CloudWatch vended-log delivery) for the +# distribution in cloudfront.tf, written to the existing access-log bucket that +# the per-deploy stack owns (../logging.tf: "-logs", SSE-S3, public +# access blocked, owner-enforced, a 90-day expiry on the CloudFront prefixes). +# +# Privacy limits (operator's condition, 2026-11-05): +# - The record carries only the fields below. No client IP, X-Forwarded-For, +# client port, cookies, query string, user agent or referrer is ever logged: +# CloudFront v2 field selection drops them at the edge, so they never reach +# storage. Country (c-country) is the finest location kept. +# - CloudFront cannot limit logging to a path, so every request is logged with +# those fields; the deploy's summarizer (scripts/build-access-summary.py) +# counts only /.well-known/ and /trust/ and discards the rest, and only +# aggregate counts are ever published. +# - Raw records expire after 90 days (../logging.tf). Monthly aggregate +# summaries are kept for 13 months in the published access summary. +# +# The deploy fails when this delivery is absent or logs a field outside the +# allowlist (scripts/check-edge-access-logging.py, step "Check edge access +# logging and build the access summary"), so the trust center never claims +# logging that is not configured. Operator-applied, like the rest of bootstrap. +# The delivery's record_fields list and ALLOWED_FIELDS in that script are the +# same list; tests/test_edge_access_logging.py holds them equal. +# ============================================================================= + +locals { + edge_access_log_name = "${local.domain_dashed}-edge-access" + # The existing access-log bucket, owned by the per-deploy stack (../logging.tf + # aws_s3_bucket.logs, already inventoried there); named, not read, here. + access_log_bucket_arn = "arn:aws:s3:::${local.domain_dashed}-logs" + edge_access_log_fields = [ + "date", + "time", + "x-edge-location", + "c-country", + "cs-method", + "cs-uri-stem", + "sc-status", + "sc-bytes", + "x-edge-result-type", + ] +} + + +resource "aws_cloudwatch_log_delivery_source" "edge_access" { + provider = aws.us_east_1 + name = local.edge_access_log_name + log_type = "ACCESS_LOGS" + resource_arn = aws_cloudfront_distribution.website.arn +} + +resource "aws_cloudwatch_log_delivery_destination" "edge_access" { + provider = aws.us_east_1 + name = local.edge_access_log_name + output_format = "json" + + delivery_destination_configuration { + destination_resource_arn = "${local.access_log_bucket_arn}/cloudfront" + } +} + +resource "aws_cloudwatch_log_delivery" "edge_access" { + provider = aws.us_east_1 + delivery_source_name = aws_cloudwatch_log_delivery_source.edge_access.name + delivery_destination_arn = aws_cloudwatch_log_delivery_destination.edge_access.arn + record_fields = local.edge_access_log_fields + + s3_delivery_configuration { + suffix_path = "{DistributionId}/{yyyy}/{MM}/{dd}/{HH}" + enable_hive_compatible_path = false + } +} + +# The deploy reads the delivery's configuration (to fail when it is absent or +# logs a forbidden field) and the CloudFront records (to build the aggregate +# summary). Read-only. Describe/Get on deliveries are account-level calls with +# no resource-level scoping for DescribeDeliveries; the Get* calls are scoped +# to this delivery's resources. The object read is limited to the two prefixes +# CloudFront delivery may write under. +data "aws_iam_policy_document" "edge_access_logs_read" { + # checkov:skip=CKV_AWS_356:logs:DescribeDeliveries is an account-level enumeration action that cannot be resource-scoped; read-only. Documented exception. + statement { + sid = "DescribeDeliveries" + actions = ["logs:DescribeDeliveries"] + resources = ["*"] + } + statement { + sid = "ReadEdgeAccessDelivery" + actions = [ + "logs:GetDelivery", + "logs:GetDeliverySource", + "logs:GetDeliveryDestination", + ] + resources = [ + "arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:delivery:*", + aws_cloudwatch_log_delivery_source.edge_access.arn, + aws_cloudwatch_log_delivery_destination.edge_access.arn, + ] + } + statement { + sid = "ListEdgeAccessRecords" + actions = ["s3:ListBucket"] + resources = [local.access_log_bucket_arn] + condition { + test = "StringLike" + variable = "s3:prefix" + values = ["cloudfront/*", "AWSLogs/${data.aws_caller_identity.current.account_id}/CloudFront/*"] + } + } + statement { + sid = "ReadEdgeAccessRecords" + actions = ["s3:GetObject"] + resources = [ + "${local.access_log_bucket_arn}/cloudfront/*", + "${local.access_log_bucket_arn}/AWSLogs/${data.aws_caller_identity.current.account_id}/CloudFront/*", + ] + } +} + +resource "aws_iam_role_policy" "edge_access_logs_read" { + # checkov:skip=CKV_AWS_356:logs:DescribeDeliveries cannot be resource-scoped; read-only. Documented exception. + name = "edge-access-logs-read" + role = aws_iam_role.deploy.id + policy = data.aws_iam_policy_document.edge_access_logs_read.json +} diff --git a/infrastructure/logging.tf b/infrastructure/logging.tf index ed5decd5..fd2f518e 100644 --- a/infrastructure/logging.tf +++ b/infrastructure/logging.tf @@ -60,6 +60,26 @@ resource "aws_s3_bucket_lifecycle_configuration" "logs" { noncurrent_version_expiration { noncurrent_days = 30 } abort_incomplete_multipart_upload { days_after_initiation = 7 } } + # CloudFront access records (CDS-TRC-ACL) are kept 90 days, not 400: the + # operator's privacy limit. S3 applies the earliest expiration when rules + # overlap, so these prefixes expire at 90 days. Only aggregate monthly + # summaries outlive them (/.well-known/access-summary.json, 13 months). + rule { + id = "expire-edge-access-records" + status = "Enabled" + filter { prefix = "cloudfront/" } + expiration { days = 90 } + noncurrent_version_expiration { noncurrent_days = 1 } + abort_incomplete_multipart_upload { days_after_initiation = 7 } + } + rule { + id = "expire-edge-access-records-default-prefix" + status = "Enabled" + filter { prefix = "AWSLogs/${data.aws_caller_identity.current.account_id}/CloudFront/" } + expiration { days = 90 } + noncurrent_version_expiration { noncurrent_days = 1 } + abort_incomplete_multipart_upload { days_after_initiation = 7 } + } } # Allow the AWS log-delivery services to write to their respective prefixes, @@ -86,7 +106,13 @@ resource "aws_s3_bucket_policy" "logs" { Effect = "Allow" Principal = { Service = "delivery.logs.amazonaws.com" } Action = "s3:PutObject" - Resource = "${aws_s3_bucket.logs.arn}/cloudfront/*" + # cloudfront/ is the configured prefix (bootstrap/trust-access-logs.tf); + # AWSLogs//CloudFront/ is the service's default, kept so a + # delivery that ignores the prefix still lands under the 90-day expiry. + Resource = [ + "${aws_s3_bucket.logs.arn}/cloudfront/*", + "${aws_s3_bucket.logs.arn}/AWSLogs/${data.aws_caller_identity.current.account_id}/CloudFront/*", + ] Condition = { StringEquals = { "aws:SourceAccount" = data.aws_caller_identity.current.account_id } } diff --git a/infrastructure/schemas/freshness-config.json b/infrastructure/schemas/freshness-config.json index 888561ba..046d604b 100644 --- a/infrastructure/schemas/freshness-config.json +++ b/infrastructure/schemas/freshness-config.json @@ -189,6 +189,18 @@ "signed": true, "signature": ".well-known/policy-index.bundle" }, + { + "path": ".well-known/access-summary.json", + "cadence": "deploy", + "max_age_hours": 720, + "timestamp": { + "source": "json", + "field": "emitted_at" + }, + "signed": true, + "signature": ".well-known/access-summary.bundle", + "note": "Monthly aggregate counts of access to the certification data (CDS-TRC-ACL), rebuilt from the CloudFront access records on every deploy." + }, { "path": ".well-known/security-decision-record.json", "cadence": "deploy", diff --git a/scripts/availability-probe.py b/scripts/availability-probe.py new file mode 100644 index 00000000..454fa5f0 --- /dev/null +++ b/scripts/availability-probe.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Independent availability monitor (CDS-CSO-AVR, CDS-TRC-USH). + +Runs on GitHub Actions (.github/workflows/availability-probe.yml), outside AWS, +so it keeps reporting when the offering is down. Each run probes the core +services once over HTTPS, appends the result to the rolling history carried in +the previous feed, derives availability incidents, and writes: + + availability-status.json machine-readable: current status, 30-day + availability per service, incidents, samples + availability-status.md the same for people + +The workflow publishes both to the repository's `availability-status` branch, +served by GitHub (raw.githubusercontent.com and the github.com file view), not +by the offering's own CloudFront or S3. + +An incident is a service failing on INCIDENT_MIN_SAMPLES or more consecutive +probes; it starts at the first failing probe and ends at the next passing one. +A single failed probe is counted as transient, not an incident, since one probe +from one runner can fail for reasons on the runner's side. + + availability-probe.py --previous prev.json --out-dir status (probe) + availability-probe.py --check-fresh feed.json --max-age-hours 2 (deploy guard) +""" +import argparse +import json +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone +from pathlib import Path + +SITE = "https://samaydlette.com" +COGNITO_UI = "https://samaydlette-com-silk-reeling.auth.us-east-2.amazoncognito.com/" +# (key, service, url, statuses that mean "up") +TARGETS = ( + ("site", "Website", f"{SITE}/", (200,)), + ("trust-center", "Trust center", f"{SITE}/trust/", (200,)), + ("certification-data", "Certification data (/.well-known/)", f"{SITE}/.well-known/ksi-signal.json", (200,)), + ("silk-reeling", "Silk Reeling application", f"{SITE}/silk-reeling/", (200,)), + ("silk-reeling-sign-in", "Silk Reeling sign-in (Cognito hosted UI)", COGNITO_UI, (200,)), +) +HISTORY_DAYS = 35 +WINDOW_DAYS = 30 +INCIDENT_MIN_SAMPLES = 2 +TIMEOUT_S = 20 +FEED_URL = ("https://raw.githubusercontent.com/sam-aydlette/samaydlette.com/" + "availability-status/availability-status.json") +PAGE_URL = ("https://github.com/sam-aydlette/samaydlette.com/blob/" + "availability-status/availability-status.md") + + +def _iso(dt): + return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _parse(s): + return datetime.fromisoformat(str(s).replace("Z", "+00:00")) + + +def fetch(url): + """(up-or-status reason, HTTP status or None). No redirects followed.""" + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a, **k): + return None + opener = urllib.request.build_opener(NoRedirect) + req = urllib.request.Request(url, headers={"User-Agent": "samaydlette-availability-probe"}) + try: + with opener.open(req, timeout=TIMEOUT_S) as resp: # nosec B310: fixed https URLs + resp.read(4096) + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + except (urllib.error.URLError, OSError) as exc: + return f"no response ({type(getattr(exc, 'reason', exc)).__name__})" + + +def probe_once(fetcher, now): + sample = {"t": _iso(now), "probed": [k for k, *_ in TARGETS], "fail": {}} + for key, _service, url, ok in TARGETS: + got = fetcher(url) + if got not in ok: + sample["fail"][key] = f"HTTP {got}" if isinstance(got, int) else str(got) + return sample + + +def incidents(samples): + """Incidents (>= INCIDENT_MIN_SAMPLES consecutive failures) and transient counts per service.""" + out, transient = [], {} + services = {k: s for k, s, *_ in TARGETS} + for key in services: + run = [] + seq = [s for s in samples if key in s.get("probed", [])] + for i, s in enumerate(seq + [None]): + failed = s is not None and key in s.get("fail", {}) + if failed: + run.append(s) + continue + if len(run) >= INCIDENT_MIN_SAMPLES: + out.append({"service": key, "service_name": services[key], "start": run[0]["t"], + "end": s["t"] if s is not None else None, "failed_probes": len(run), + "failing_check": run[-1]["fail"][key]}) + elif run: + transient[key] = transient.get(key, 0) + len(run) + run = [] + return sorted(out, key=lambda x: (x["start"], x["service"])), transient + + +def availability(samples, now): + since = now - timedelta(days=WINDOW_DAYS) + out = {} + for key, service, url, _ok in TARGETS: + seq = [s for s in samples if key in s.get("probed", []) and _parse(s["t"]) >= since] + up = sum(1 for s in seq if key not in s.get("fail", {})) + out[key] = {"service": service, "url": url, "probes": len(seq), "up": up, + "availability_percent": round(100.0 * up / len(seq), 3) if seq else None} + return out + + +def build(previous, sample, now): + prev_samples = [s for s in (previous or {}).get("samples", []) if isinstance(s, dict) and "t" in s] + cutoff = now - timedelta(days=HISTORY_DAYS) + samples = [s for s in prev_samples if _parse(s["t"]) >= cutoff] + [sample] + began = (previous or {}).get("history_began") or sample["t"] + incs, transient = incidents(samples) + window_start = _iso(now - timedelta(days=WINDOW_DAYS)) + return { + "feed_type": "availability-status", + "rules": ["CDS-CSO-AVR", "CDS-TRC-USH"], + "generated_at": sample["t"], + "monitor": "GitHub Actions (.github/workflows/availability-probe.yml), outside AWS: it keeps " + "reporting when the offering is unavailable.", + "published_at": {"json": FEED_URL, "human": PAGE_URL}, + "history_began": began, + "history_note": "History begins at the monitor's first run; before that no independent " + "measurement exists. Scheduled runs are every 15 minutes, and GitHub may delay one.", + "incident_rule": f"{INCIDENT_MIN_SAMPLES} or more consecutive failed probes of one service; a single " + "failed probe is counted as transient.", + "current": {k: ("down" if k in sample["fail"] else "up") for k, *_ in TARGETS}, + "current_failures": sample["fail"], + "window_days": WINDOW_DAYS, + "availability": availability(samples, now), + "incidents": [i for i in incs if (i["end"] or "9999") >= window_start], + "transient_failures": transient, + "samples": samples, + } + + +def render_markdown(doc): + cur = doc["current"] + out = ["# Service availability (independent monitor)", "", + f"_{doc['monitor']}_", "", + f"- **Last probe:** {doc['generated_at']}", + f"- **History began:** {doc['history_began']}. {doc['history_note']}", + f"- **Incident rule:** {doc['incident_rule']}", + f"- **Machine-readable:** {doc['published_at']['json']}", "", + "## Now and over the past 30 days", "", + "| Service | Now | Availability (30 days) | Probes |", "|---|---|---|---|"] + for key, a in doc["availability"].items(): + pct = "n/a" if a["availability_percent"] is None else f"{a['availability_percent']}%" + out.append(f"| {a['service']} | {cur.get(key, 'n/a')} | {pct} | {a['probes']} |") + out += ["", "## Availability incidents (past 30 days)", ""] + if not doc["incidents"]: + out.append("None.") + else: + out += ["| Service | Start | End | Failed probes | Failing check |", "|---|---|---|---|---|"] + for i in reversed(doc["incidents"]): + out.append(f"| {i['service_name']} | {i['start']} | {i['end'] or 'ongoing'} | " + f"{i['failed_probes']} | {i['failing_check']} |") + if doc["transient_failures"]: + out += ["", "Transient (single) failed probes: " + + ", ".join(f"{k} {v}" for k, v in sorted(doc["transient_failures"].items()))] + out += ["", "---", "_Source of truth: `availability-status.json` on this branch._"] + return "\n".join(out) + + +def ongoing_incidents(doc): + return [i for i in doc.get("incidents", []) if i.get("end") is None] + + +def check_fresh(path, max_age_hours, now=None): + """Problems with a fetched feed: [] when it is ours, well formed and fresh.""" + now = now or datetime.now(timezone.utc) + try: + doc = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError): + return ["the independent availability feed is missing or not JSON"] + if doc.get("feed_type") != "availability-status": + return ["the independent availability feed is not an availability-status document"] + try: + age = now - _parse(doc["generated_at"]) + except (KeyError, ValueError): + return ["the independent availability feed has no generated_at"] + if age > timedelta(hours=max_age_hours): + return [f"the independent availability feed is {age} old (limit {max_age_hours} h): the monitor " + "on GitHub Actions is not running"] + if not doc.get("samples"): + return ["the independent availability feed carries no probes"] + return [] + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--previous", default=None) + ap.add_argument("--out-dir", default=".") + ap.add_argument("--check-fresh", default=None, metavar="FEED") + ap.add_argument("--max-age-hours", type=float, default=2.0) + ap.add_argument("--fail-on-incident", default=None, metavar="FEED", + help="exit 1 when FEED shows an ongoing incident (the workflow's alert)") + a = ap.parse_args(argv) + if a.fail_on_incident: + doc = json.loads(Path(a.fail_on_incident).read_text()) + ongoing = ongoing_incidents(doc) + for i in ongoing: + print(f"::error::availability incident: {i['service_name']} down since {i['start']} " + f"({i['failing_check']})", file=sys.stderr) + return 1 if ongoing else 0 + if a.check_fresh: + why = check_fresh(a.check_fresh, a.max_age_hours) + for w in why: + print(f"::error::availability: {w}", file=sys.stderr) + if not why: + print("ok: the independent availability feed is present and fresh") + return 1 if why else 0 + previous = {} + if a.previous and Path(a.previous).exists(): + try: + previous = json.loads(Path(a.previous).read_text()) + except json.JSONDecodeError: + previous = {} + if previous.get("feed_type") != "availability-status": + previous = {} + started = time.monotonic() + now = datetime.now(timezone.utc) + doc = build(previous, probe_once(fetch, now), now) + out = Path(a.out_dir) + out.mkdir(parents=True, exist_ok=True) + (out / "availability-status.json").write_text(json.dumps(doc, indent=1) + "\n") + (out / "availability-status.md").write_text(render_markdown(doc) + "\n") + down = sorted(doc["current_failures"]) + print(f"probed {len(TARGETS)} services in {time.monotonic() - started:.1f}s; down: {down or 'none'}; " + f"{len(doc['samples'])} samples kept") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build-access-summary.py b/scripts/build-access-summary.py new file mode 100644 index 00000000..581b4b70 --- /dev/null +++ b/scripts/build-access-summary.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Monthly access summaries for the certification data (CDS-TRC-ACL). + +Reads the CloudFront access records the deploy synced from the log bucket (JSON +lines, gzipped or plain, one object per request with the fields the delivery +keeps: see scripts/check-edge-access-logging.py ALLOWED_FIELDS) and writes +access-summary.json (+ .md): per month, how many requests reached each +certification artifact under /.well-known/ and the trust center under /trust/, +by response class and by country. Nothing else. + +Privacy (operator's condition, 2026-11-05): +- Only /.well-known/ and /trust/ requests are counted; every other record is + discarded unread beyond its path. CloudFront cannot limit logging by path, so + this is where the limit is applied. +- A path is published by name only when it served an object (2xx or 304). + Requests for anything else (probes for files that do not exist, errors) are + counted under "other", so a requester cannot write a string into this file. +- No IP, user agent, referrer, cookie or query string exists in the records; + the summary carries counts only. + +Retention: raw records expire after 90 days (infrastructure/logging.tf). Each +month's summary is carried forward from the previously published file, so +summaries are kept for MONTHS_KEPT months (FedRAMP asks for at least six). A +month is recomputed from the records only when they cover at least as much of it +as the published summary did, so a month whose early days have expired keeps its +fuller earlier count. +""" +import argparse +import gzip +import json +import re +import sys +from collections import Counter +from datetime import date, datetime, timezone +from pathlib import Path + +MONTHS_KEPT = 13 +MIN_MONTHS_KEPT = 6 +PREFIXES = ("/.well-known/", "/trust/") +SAFE_PATH = re.compile(r"^/[A-Za-z0-9._/-]{0,200}$") +SERVED = re.compile(r"^(2\d\d|304)$") +COUNTRY = re.compile(r"^[A-Z]{2}$") +OTHER = "other (not found, error, or not an object)" + + +def _fail(msg): + raise SystemExit(f"::error::access summary: {msg} (fail-closed)") + + +def read_records(root): + """Yield one dict per access record under root (*.gz or plain JSON lines).""" + for p in sorted(Path(root).rglob("*")) if Path(root).is_dir() else []: + if not p.is_file(): + continue + opener = gzip.open if p.suffix == ".gz" else open + try: + with opener(p, "rt", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + _fail(f"{p.name} holds a line that is not JSON") + if isinstance(rec, dict): + yield rec + except (OSError, EOFError) as exc: + _fail(f"{p.name} could not be read ({type(exc).__name__})") + + +def _artifact(stem, status): + if not SERVED.match(str(status)) or not SAFE_PATH.match(stem or ""): + return OTHER + if stem.startswith("/trust/"): + return "/trust/" if stem in ("/trust/", "/trust/index.html") else stem + return stem + + +def summarize(records): + """{month: {"requests", "artifacts", "countries", "status_classes", "first", "last"}}""" + months = {} + for r in records: + stem = r.get("cs-uri-stem") or "" + if not stem.startswith(PREFIXES): + continue + day = str(r.get("date") or "") + try: + d = date.fromisoformat(day) + except ValueError: + continue + m = months.setdefault(d.strftime("%Y-%m"), {"requests": 0, "artifacts": Counter(), "countries": Counter(), + "status_classes": Counter(), "first": day, "last": day}) + status = str(r.get("sc-status") or "") + m["requests"] += 1 + m["artifacts"][_artifact(stem, status)] += 1 + cc = str(r.get("c-country") or "") + m["countries"][cc if COUNTRY.match(cc) else "unknown"] += 1 + m["status_classes"][(status[:1] + "xx") if status[:1].isdigit() else "unknown"] += 1 + m["first"], m["last"] = min(m["first"], day), max(m["last"], day) + return months + + +def _month_entry(month, m, now): + return {"month": month, "coverage_from": m["first"], "coverage_to": m["last"], + "requests": m["requests"], + "artifacts": dict(sorted(m["artifacts"].items())), + "countries": dict(sorted(m["countries"].items())), + "status_classes": dict(sorted(m["status_classes"].items())), + "computed_at": now.isoformat()} + + +def merge(previous, computed, now): + """Carry earlier months forward; recompute a month only when the records cover + at least as much of it as the published entry did.""" + by_month = {e["month"]: e for e in (previous or {}).get("months", []) if isinstance(e, dict) and "month" in e} + for month, m in computed.items(): + old = by_month.get(month) + if old is None or m["first"] <= old.get("coverage_from", "9999"): + by_month[month] = _month_entry(month, m, now) + kept = sorted(by_month.values(), key=lambda e: e["month"])[-MONTHS_KEPT:] + return kept + + +def build(previous, records, now): + months = merge(previous, summarize(records), now) + started = (previous or {}).get("logging_records_since") + firsts = [e["coverage_from"] for e in months] + if firsts: + started = min([started] + firsts) if started else min(firsts) + return { + "artifact": "access-summary", + "rule": "CDS-TRC-ACL", + "emitted_at": now.isoformat(), + "scope": "Requests to the certification data under /.well-known/ and to the trust center under /trust/.", + "logged_fields": "date, time, edge location, country, method, path, status, bytes, cache result. " + "No IP address, user agent, referrer, cookie or query string is logged.", + "raw_record_retention_days": 90, + "summary_retention_months": MONTHS_KEPT, + "logging_records_since": started, + "party_records": "A party can ask for the summaries and for the records of its own requests, " + "identified by the times and paths it gives (no requester identity is logged); " + "see the trust center's privacy notice and DEC-160.", + "months": months, + } + + +def render_markdown(doc): + out = ["# Trust center access summary", "", + f"_{doc['scope']}_", "", + f"- **Emitted:** {doc['emitted_at']}", + f"- **Logged fields:** {doc['logged_fields']}", + f"- **Raw records kept:** {doc['raw_record_retention_days']} days; " + f"**summaries kept:** {doc['summary_retention_months']} months", + f"- **Records since:** {doc['logging_records_since'] or 'none yet'}", + f"- **Your own records:** {doc['party_records']}", ""] + if not doc["months"]: + out += ["No access records yet.", ""] + for m in reversed(doc["months"]): + out += [f"## {m['month']}", "", + f"{m['requests']} requests, {m['coverage_from']} to {m['coverage_to']}. " + f"By status: {', '.join(f'{k} {v}' for k, v in m['status_classes'].items())}. " + f"By country: {', '.join(f'{k} {v}' for k, v in m['countries'].items())}.", "", + "| Artifact | Requests |", "|---|---|"] + for path, n in sorted(m["artifacts"].items(), key=lambda kv: (-kv[1], kv[0])): + out.append(f"| `{path}` | {n} |") + out.append("") + out += ["---", "_Source of truth: the signed `access-summary.json` (CDS-TRC-ACL)._"] + return "\n".join(out) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--records", required=True, help="directory of synced CloudFront access records") + ap.add_argument("--previous", default=None, help="the previously published access-summary.json") + ap.add_argument("--output", default="access-summary.json") + ap.add_argument("--md-output", default="access-summary.md") + a = ap.parse_args(argv) + previous = {} + if a.previous and Path(a.previous).exists(): + try: + previous = json.loads(Path(a.previous).read_text()) + except json.JSONDecodeError: + previous = {} + if previous and previous.get("artifact") != "access-summary": + previous = {} + now = datetime.now(timezone.utc) + doc = build(previous, read_records(a.records), now) + prev_months = {e["month"] for e in previous.get("months", [])} if previous else set() + # Never drop a month younger than the six FedRAMP requires. + newest = sorted(prev_months | {e["month"] for e in doc["months"]})[-MIN_MONTHS_KEPT:] + lost = sorted(set(newest) - {e["month"] for e in doc["months"]}) + if lost: + _fail(f"would drop month(s) {lost} younger than {MIN_MONTHS_KEPT} months") + Path(a.output).write_text(json.dumps(doc, indent=2) + "\n") + if a.md_output: + Path(a.md_output).write_text(render_markdown(doc) + "\n") + print(f"access-summary: {len(doc['months'])} month(s); records since {doc['logging_records_since']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build-availability.py b/scripts/build-availability.py index d020e9c8..1aa71850 100644 --- a/scripts/build-availability.py +++ b/scripts/build-availability.py @@ -1,89 +1,143 @@ #!/usr/bin/env python3 # ============================================================================= -# AVAILABILITY FEED BUILDER (CR26 CDS-CSO-AVR) +# AVAILABILITY FEED BUILDER (CR26 CDS-CSO-AVR, CDS-TRC-USH) # ============================================================================= -# CR26 CDS-CSO-AVR (Class C MUST) requires a web service indicating current and -# historical availability of core services over at least the past 30 days, in -# both human-readable and machine-readable formats, available even if the primary -# offering is unavailable. This builds the machine + human feed as an append-only -# history: each deploy appends a datapoint to the previous published feed and -# trims to a rolling window. A deploy that produces this feed is, by definition, -# a successful one, so the current datapoint is "available". +# CDS-CSO-AVR (Class C MUST): a web service showing current and historical +# availability of core services over at least the past 30 days, including +# availability incidents, human- and machine-readable, and available even when +# the offering is not. # -# HONEST RESIDUAL: this feed is served from the same CloudFront/S3 path as the -# offering, so the "available even if the primary offering is unavailable" clause -# is only partially met — full conformance needs an independent monitor on -# separate infrastructure. That gap is stated in the feed itself, not hidden. +# The measurement is made outside AWS: scripts/availability-probe.py runs on +# GitHub Actions every 15 minutes and publishes its feed on GitHub (the +# repository's availability-status branch), which stays reachable when the +# offering's CloudFront or S3 is down. That is the service the rule asks for. # -# build-availability.py --previous previous-availability.json +# This builder republishes the same measurements on the site, signed with the +# rest of the certification data, so the package carries them: the deploy reads +# the independent feed (and fails when it is missing or stale), and this writes +# availability.json + availability.md from it: current status, 30-day +# availability and incidents per service, and a daily roll-up. It never records +# a deploy as availability. +# +# build-availability.py --probe-feed availability-probe.json --ksi-signal ksi-signal.json # ============================================================================= import argparse +import importlib.util import json import sys -from datetime import datetime, timezone +from collections import defaultdict +from datetime import datetime, timedelta, timezone from pathlib import Path -WINDOW_DAYS = 45 # keep >= 30 days of history per CDS-CSO-AVR +_spec = importlib.util.spec_from_file_location( + "availability_probe", Path(__file__).resolve().parent / "availability-probe.py") +assert _spec is not None and _spec.loader is not None +probe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(probe) + +MAX_FEED_AGE_HOURS = 2.0 + + +def _fail(msg): + raise SystemExit(f"::error::availability: {msg} (fail-closed)") + + +def daily(samples, now, days=probe.WINDOW_DAYS): + """[{date, services: {key: {probes, up}}}] for the past `days` days.""" + since = (now - timedelta(days=days)).date() + by_day = defaultdict(lambda: defaultdict(lambda: {"probes": 0, "up": 0})) + for s in samples: + d = probe._parse(s["t"]).date() + if d < since: + continue + for key in s.get("probed", []): + cell = by_day[d.isoformat()][key] + cell["probes"] += 1 + cell["up"] += 0 if key in s.get("fail", {}) else 1 + return [{"date": d, "services": {k: dict(v) for k, v in sorted(svcs.items())}} + for d, svcs in sorted(by_day.items())] + + +def build(feed, signal_id, now): + return { + "feed_type": "availability", + "rules": ["CDS-CSO-AVR", "CDS-TRC-USH"], + "emitted_at": now.isoformat(), + "ksi_signal_id": signal_id, + "measured_by": feed["monitor"], + "independent_service": feed["published_at"], + "independent_note": "This file is a signed copy carried in the certification package. The service " + "that stays available when the offering is down is the independent one above, " + "served by GitHub.", + "measured_at": feed["generated_at"], + "history_began": feed["history_began"], + "history_note": feed["history_note"], + "incident_rule": feed["incident_rule"], + "current": feed["current"], + "current_failures": feed["current_failures"], + "window_days": feed["window_days"], + "availability": feed["availability"], + "incidents": feed["incidents"], + "transient_failures": feed["transient_failures"], + "daily": daily(feed["samples"], now), + } + +def render_markdown(doc): + out = ["# Service availability", "", + f"_Measured independently: {doc['measured_by']}_", "", + f"- **Last probe:** {doc['measured_at']} | **Emitted:** {doc['emitted_at']}", + f"- **Independent status page (available when this site is not):** {doc['independent_service']['human']}", + f"- **History began:** {doc['history_began']}. {doc['history_note']}", + f"- **Incident rule:** {doc['incident_rule']}", "", + "| Service | Now | Availability (30 days) | Probes |", "|---|---|---|---|"] + for key, a in doc["availability"].items(): + pct = "n/a" if a["availability_percent"] is None else f"{a['availability_percent']}%" + out.append(f"| {a['service']} | {doc['current'].get(key, 'n/a')} | {pct} | {a['probes']} |") + out += ["", "## Availability incidents (past 30 days)", ""] + if not doc["incidents"]: + out.append("None.") + else: + out += ["| Service | Start | End | Failed probes | Failing check |", "|---|---|---|---|---|"] + for i in reversed(doc["incidents"]): + out.append(f"| {i['service_name']} | {i['start']} | {i['end'] or 'ongoing'} | " + f"{i['failed_probes']} | {i['failing_check']} |") + out += ["", "## Daily (passing probes / probes)", "", "| Date | " + " | ".join(doc["availability"]) + " |", + "|---|" + "---|" * len(doc["availability"])] + for day in reversed(doc["daily"]): + cells = [] + for key in doc["availability"]: + c = day["services"].get(key) + cells.append(f"{c['up']}/{c['probes']}" if c else "—") + out.append(f"| {day['date']} | " + " | ".join(cells) + " |") + out += ["", "---", "_Source of truth: the signed `availability.json` (CDS-CSO-AVR)._"] + return "\n".join(out) -def main(): + +def main(argv=None): ap = argparse.ArgumentParser() - ap.add_argument("--previous", default=None, help="Previous published availability.json (for history)") + ap.add_argument("--probe-feed", required=True, + help="the independent monitor's availability-status.json, fetched from GitHub") ap.add_argument("--ksi-signal", default="ksi-signal.json") ap.add_argument("--output", default="availability.json") ap.add_argument("--md-output", default="availability.md") - a = ap.parse_args() + a = ap.parse_args(argv) now = datetime.now(timezone.utc) - today = now.date().isoformat() - - history = [] - if a.previous and Path(a.previous).exists(): - try: - history = json.loads(Path(a.previous).read_text()).get("history", []) or [] - except Exception: - history = [] - # Append today (idempotent: one datapoint per day). - history = [h for h in history if h.get("date") != today] - history.append({"date": today, "status": "available", - "detail": "deploy succeeded; core services served via CloudFront"}) - # Trim to the rolling window (keep the most recent WINDOW_DAYS dated points). - history = sorted(history, key=lambda h: h.get("date", ""))[-WINDOW_DAYS:] - + why = probe.check_fresh(a.probe_feed, MAX_FEED_AGE_HOURS, now) + if why: + _fail("; ".join(why)) + feed = json.loads(Path(a.probe_feed).read_text()) signal_id = None if Path(a.ksi_signal).exists(): signal_id = json.loads(Path(a.ksi_signal).read_text()).get("signal_id") - - up = sum(1 for h in history if h.get("status") == "available") - feed = { - "feed_type": "availability", - "rule": "CDS-CSO-AVR", - "emitted_at": now.isoformat(), - "ksi_signal_id": signal_id, - "current_status": "available", - "window_days": WINDOW_DAYS, - "history_points": len(history), - "uptime_observed": f"{up}/{len(history)} observed days available", - "residual": ("This feed is served from the same CloudFront/S3 path as the offering; " - "the CDS-CSO-AVR clause requiring availability reporting even when the " - "primary offering is unavailable is only partially met. Full conformance " - "requires an independent monitor on separate infrastructure."), - "history": history, - } - Path(a.output).write_text(json.dumps(feed, indent=2) + "\n") + doc = build(feed, signal_id, now) + Path(a.output).write_text(json.dumps(doc, indent=2) + "\n") if a.md_output: - out = ["# Service Availability", "", - f"- **Current status:** {feed['current_status']}", - f"- **Observed window:** {feed['uptime_observed']} (rolling {WINDOW_DAYS} days)", - f"- **Emitted:** {feed['emitted_at']}", "", - f"> {feed['residual']}", "", - "| Date | Status |", "|---|---|"] - for h in reversed(history): - out.append(f"| {h['date']} | {h['status']} |") - out += ["", "---", "_Source of truth: the machine-readable `availability.json` (CDS-CSO-AVR)._"] - Path(a.md_output).write_text("\n".join(out) + "\n") - print(f"availability: {len(history)} day(s) of history; current=available") + Path(a.md_output).write_text(render_markdown(doc) + "\n") + down = sorted(k for k, v in doc["current"].items() if v != "up") + print(f"availability: {len(doc['incidents'])} incident(s) in {doc['window_days']} days; down now: {down or 'none'}") return 0 diff --git a/scripts/check-edge-access-logging.py b/scripts/check-edge-access-logging.py new file mode 100644 index 00000000..92fc97d9 --- /dev/null +++ b/scripts/check-edge-access-logging.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Access to the certification data is logged, within the privacy limits (CDS-TRC-ACL). + +Reads the CloudFront standard-logging (v2) configuration the deploy role can see +and fails unless: + + * a delivery from the source named for this site exists; + * that source is the CloudFront distribution's ACCESS_LOGS; + * the delivery goes to S3; and + * its record fields are exactly within ALLOWED_FIELDS and include REQUIRED_FIELDS. + +ALLOWED_FIELDS is the operator's privacy condition: no client IP, forwarded-for, +port, cookies, query string, user agent or referrer is ever logged. It must equal +the delivery's record_fields in infrastructure/bootstrap/trust-access-logs.tf +(tests/test_edge_access_logging.py holds them equal). The configuration is +operator-applied (bootstrap), so this check is what fails the deploy when it is +missing, before the trust center or the SDR say access is logged. + +Inputs are the AWS CLI's JSON (aws logs describe-deliveries, aws logs +get-delivery-source), so the check runs the same on fixtures. Prints no ARN. +""" +import argparse +import json +import sys +from pathlib import Path + +ALLOWED_FIELDS = ("date", "time", "x-edge-location", "c-country", "cs-method", "cs-uri-stem", + "sc-status", "sc-bytes", "x-edge-result-type") +REQUIRED_FIELDS = ("date", "time", "cs-uri-stem", "sc-status") +# Named so the failure message says what was found; any field outside +# ALLOWED_FIELDS fails, whether or not it is listed here. +IDENTIFYING = ("c-ip", "x-forwarded-for", "c-port", "cs(Cookie)", "cs-uri-query", "cs(User-Agent)", + "cs(Referer)") + + +def problems(deliveries, source, name): + out = [] + mine = [d for d in (deliveries or {}).get("deliveries", []) if d.get("deliverySourceName") == name] + if not mine: + return [f"no CloudFront access-log delivery from source {name!r} (bootstrap not applied?)"] + if len(mine) > 1: + out.append(f"{len(mine)} deliveries from {name!r}; expected one") + src = (source or {}).get("deliverySource") or {} + if src.get("name") != name: + out.append(f"delivery source {name!r} could not be read") + if src.get("logType") != "ACCESS_LOGS": + out.append(f"delivery source log type is {src.get('logType')!r}, not ACCESS_LOGS") + if not any(":distribution/" in a and a.startswith("arn:aws:cloudfront::") for a in src.get("resourceArns") or []): + out.append("delivery source is not a CloudFront distribution") + for d in mine: + if d.get("deliveryDestinationType") != "S3": + out.append(f"delivery destination type is {d.get('deliveryDestinationType')!r}, not S3") + fields = d.get("recordFields") or [] + extra = sorted(set(fields) - set(ALLOWED_FIELDS)) + bad = sorted(f for f in extra if f in IDENTIFYING) + if bad: + out.append(f"delivery logs identifying field(s) {bad}") + if sorted(set(extra) - set(bad)): + out.append(f"delivery logs field(s) outside the privacy allowlist: {sorted(set(extra) - set(bad))}") + missing = sorted(set(REQUIRED_FIELDS) - set(fields)) + if missing: + out.append(f"delivery omits required field(s) {missing}") + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--deliveries", required=True, help="aws logs describe-deliveries output") + ap.add_argument("--source", required=True, help="aws logs get-delivery-source output (may be empty)") + ap.add_argument("--name", required=True, help="the delivery source's name") + a = ap.parse_args(argv) + + def read(p): + t = Path(p).read_text() if Path(p).exists() else "" + try: + return json.loads(t) if t.strip() else {} + except json.JSONDecodeError: + return {} + why = problems(read(a.deliveries), read(a.source), a.name) + for w in why: + print(f"::error::edge access logging: {w}", file=sys.stderr) + if why: + print("Access to the certification data is not logged as the trust center states; " + "the deploy does not proceed.", file=sys.stderr) + return 1 + print(f"ok: CloudFront access logging to S3, fields {', '.join(ALLOWED_FIELDS)} " + "(no IP, cookies, query string or user agent)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-wellknown-headers.py b/scripts/check-wellknown-headers.py new file mode 100644 index 00000000..4ea01002 --- /dev/null +++ b/scripts/check-wellknown-headers.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""The public JSON under /.well-known/ is servable to any program (FRC-CSO-JSN). + +Fetches each named artifact over HTTPS from the live site, as a browser on +another origin would (an Origin header), and fails unless the response carries: + + * Access-Control-Allow-Origin: * (CORS: a page on any origin can read it) + * a JSON Content-Type (application/json or application/*+json) + * X-Content-Type-Options: nosniff (the browser keeps that type) + +The headers come from the CloudFront response-headers policy for /.well-known/* +(infrastructure/bootstrap/cloudfront.tf), which the operator applies; this step +is what makes the deploy fail when that apply is missing. Read-only; stdlib only. + + check-wellknown-headers.py --site https://samaydlette.com ksi-signal.json ... + check-wellknown-headers.py --fixture headers.json (tests: {name: {header: value}}) +""" +import argparse +import json +import sys +import urllib.error +import urllib.request + +ORIGIN = "https://example.org" +DEFAULT_ARTIFACTS = ("ksi-signal.json", "security-decision-record.json", "trust-center.json", + "vdr-report.json", "certification-package-overview.json") + + +def problems(headers): + """[reason, ...] for one response's headers (names case-insensitive).""" + h = {k.lower(): (v or "").strip() for k, v in headers.items()} + out = [] + if h.get("access-control-allow-origin") != "*": + out.append(f"Access-Control-Allow-Origin is {h.get('access-control-allow-origin')!r}, not '*'") + ctype = h.get("content-type", "").split(";")[0].strip().lower() + if not (ctype == "application/json" or (ctype.startswith("application/") and ctype.endswith("+json"))): + out.append(f"Content-Type is {ctype or 'absent'!r}, not JSON") + if h.get("x-content-type-options", "").lower() != "nosniff": + out.append("X-Content-Type-Options: nosniff is absent") + return out + + +def fetch(url, timeout=20): + req = urllib.request.Request(url, method="GET", headers={"Origin": ORIGIN, "User-Agent": "samaydlette-header-check"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: # nosec B310: fixed https URL + return dict(resp.headers.items()) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--site", default="https://samaydlette.com") + ap.add_argument("--fixture", default=None, help="JSON {artifact: {header: value}} instead of the network") + ap.add_argument("artifacts", nargs="*", default=None) + a = ap.parse_args(argv) + if not a.site.startswith("https://"): + print("::error::--site must be an https:// URL", file=sys.stderr) + return 2 + fixture = json.loads(open(a.fixture).read()) if a.fixture else None + names = a.artifacts or (sorted(fixture) if fixture is not None else list(DEFAULT_ARTIFACTS)) + failed = 0 + for name in names: + url = f"{a.site.rstrip('/')}/.well-known/{name}" + try: + headers = fixture[name] if fixture is not None else fetch(url) + except (urllib.error.URLError, OSError, KeyError) as exc: + print(f"::error::{url}: could not be read ({type(exc).__name__})", file=sys.stderr) + failed += 1 + continue + why = problems(headers) + for w in why: + print(f"::error::{url}: {w}", file=sys.stderr) + failed += bool(why) + if not why: + print(f"ok: {url} (CORS *, JSON, nosniff)") + if failed: + print(f"{failed} artifact(s) under /.well-known/ lack the required headers; the deploy does not proceed.", + file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/availability/known-bad-stale-feed.json b/tests/fixtures/availability/known-bad-stale-feed.json new file mode 100644 index 00000000..14961cf8 --- /dev/null +++ b/tests/fixtures/availability/known-bad-stale-feed.json @@ -0,0 +1,2 @@ +{"feed_type": "availability-status", "generated_at": "2026-10-01T00:00:00Z", + "samples": [{"t": "2026-10-01T00:00:00Z", "probed": ["site"], "fail": {}}]} diff --git a/tests/fixtures/edge-access-logging/delivery-source.json b/tests/fixtures/edge-access-logging/delivery-source.json new file mode 100644 index 00000000..887662fc --- /dev/null +++ b/tests/fixtures/edge-access-logging/delivery-source.json @@ -0,0 +1,2 @@ +{"deliverySource": {"name": "samaydlette-com-edge-access", "logType": "ACCESS_LOGS", "service": "cloudfront", + "resourceArns": ["arn:aws:cloudfront::ACCOUNT:distribution/EXAMPLE"]}} diff --git a/tests/fixtures/edge-access-logging/good-deliveries.json b/tests/fixtures/edge-access-logging/good-deliveries.json new file mode 100644 index 00000000..4fa9b6d8 --- /dev/null +++ b/tests/fixtures/edge-access-logging/good-deliveries.json @@ -0,0 +1,3 @@ +{"deliveries": [{"id": "fixture", "deliverySourceName": "samaydlette-com-edge-access", + "deliveryDestinationType": "S3", + "recordFields": ["date", "time", "x-edge-location", "c-country", "cs-method", "cs-uri-stem", "sc-status", "sc-bytes", "x-edge-result-type"]}]} diff --git a/tests/fixtures/edge-access-logging/known-bad-deliveries.json b/tests/fixtures/edge-access-logging/known-bad-deliveries.json new file mode 100644 index 00000000..3381be4a --- /dev/null +++ b/tests/fixtures/edge-access-logging/known-bad-deliveries.json @@ -0,0 +1,3 @@ +{"deliveries": [{"id": "fixture", "deliverySourceName": "samaydlette-com-edge-access", + "deliveryDestinationType": "S3", + "recordFields": ["date", "time", "c-ip", "cs-uri-stem", "cs-uri-query", "cs(User-Agent)", "sc-status"]}]} diff --git a/tests/fixtures/wellknown-headers/known-bad.json b/tests/fixtures/wellknown-headers/known-bad.json new file mode 100644 index 00000000..f5e93822 --- /dev/null +++ b/tests/fixtures/wellknown-headers/known-bad.json @@ -0,0 +1,4 @@ +{ + "ksi-signal.json": {"Content-Type": "application/json", "X-Content-Type-Options": "nosniff"}, + "security-decision-record.json": {"Content-Type": "application/json", "Access-Control-Allow-Origin": "*", "X-Content-Type-Options": "nosniff"} +} diff --git a/tests/fixtures/wellknown-headers/known-good.json b/tests/fixtures/wellknown-headers/known-good.json new file mode 100644 index 00000000..19a9e4f3 --- /dev/null +++ b/tests/fixtures/wellknown-headers/known-good.json @@ -0,0 +1,4 @@ +{ + "ksi-signal.json": {"content-type": "application/json", "access-control-allow-origin": "*", "x-content-type-options": "nosniff"}, + "ksi-signal.schema.json": {"Content-Type": "application/schema+json", "Access-Control-Allow-Origin": "*", "X-Content-Type-Options": "nosniff"} +} diff --git a/tests/test_availability_probe.py b/tests/test_availability_probe.py new file mode 100644 index 00000000..32a1c4af --- /dev/null +++ b/tests/test_availability_probe.py @@ -0,0 +1,126 @@ +"""CDS-CSO-AVR / CDS-TRC-USH: availability is measured outside AWS, with +incidents, and the deploy refuses a missing or stale independent feed.""" +import importlib.util +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent + + +def _load(name, file): + spec = importlib.util.spec_from_file_location(name, REPO / "scripts" / file) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +probe = _load("availability_probe_t", "availability-probe.py") +avail = _load("build_availability_t", "build-availability.py") +NOW = datetime(2026, 11, 5, 12, tzinfo=timezone.utc) +KEYS = [k for k, *_ in probe.TARGETS] + + +def _sample(t, fail=None): + return {"t": probe._iso(t), "probed": KEYS, "fail": fail or {}} + + +def test_core_services_are_probed_including_the_app_and_its_sign_in(): + urls = {u for _k, _s, u, _ok in probe.TARGETS} + assert any(u.endswith("/silk-reeling/") for u in urls) + assert any("amazoncognito.com" in u for u in urls) + assert any("/.well-known/" in u for u in urls) and any(u.endswith("/trust/") for u in urls) + assert all(u.startswith("https://") for u in urls) + + +def test_a_failed_probe_records_the_status(): + s = probe.probe_once(lambda url: 503 if "trust" in url else 200, NOW) + assert s["fail"] == {"trust-center": "HTTP 503"} + + +def test_consecutive_failures_make_an_incident_and_a_single_one_is_transient(): + t = [NOW - timedelta(minutes=15 * i) for i in range(6, 0, -1)] + samples = [_sample(t[0]), _sample(t[1], {"site": "HTTP 503"}), _sample(t[2], {"site": "HTTP 503"}), + _sample(t[3]), _sample(t[4], {"trust-center": "no response (timeout)"}), _sample(t[5])] + incs, transient = probe.incidents(samples) + assert incs == [{"service": "site", "service_name": "Website", "start": probe._iso(t[1]), + "end": probe._iso(t[3]), "failed_probes": 2, "failing_check": "HTTP 503"}] + assert transient == {"trust-center": 1} + + +def test_an_ongoing_incident_has_no_end(): + samples = [_sample(NOW - timedelta(minutes=30), {"site": "HTTP 500"}), _sample(NOW, {"site": "HTTP 500"})] + (inc,), _ = probe.incidents(samples) + assert inc["end"] is None + + +def test_history_is_kept_35_days_and_its_start_is_carried(): + old = [_sample(NOW - timedelta(days=d)) for d in (40, 34, 1)] + prev = {"feed_type": "availability-status", "history_began": "2026-09-01T00:00:00Z", "samples": old} + doc = probe.build(prev, _sample(NOW), NOW) + assert len(doc["samples"]) == 3 and doc["history_began"] == "2026-09-01T00:00:00Z" + assert doc["availability"]["site"]["probes"] == 2 # 30-day window: day 1 and now + + +def test_availability_percent_counts_failures(): + prev = {"feed_type": "availability-status", "samples": [_sample(NOW - timedelta(hours=1), {"site": "HTTP 502"})]} + doc = probe.build(prev, _sample(NOW), NOW) + assert doc["availability"]["site"]["availability_percent"] == 50.0 + assert doc["current"]["site"] == "up" + + +def test_the_known_bad_stale_feed_fails_the_deploy_guard(): + why = probe.check_fresh(REPO / "tests" / "fixtures" / "availability" / "known-bad-stale-feed.json", 2, NOW) + assert why and "not running" in why[0] + + +def test_a_missing_feed_fails_the_deploy_guard(tmp_path): + assert probe.check_fresh(tmp_path / "absent.json", 2, NOW) + + +def test_the_onsite_feed_is_built_from_the_independent_one_never_from_a_deploy(tmp_path): + feed = probe.build({}, _sample(datetime.now(timezone.utc)), datetime.now(timezone.utc)) + (tmp_path / "f.json").write_text(json.dumps(feed)) + assert avail.main(["--probe-feed", str(tmp_path / "f.json"), "--ksi-signal", str(tmp_path / "none.json"), + "--output", str(tmp_path / "a.json"), "--md-output", str(tmp_path / "a.md")]) == 0 + doc = json.loads((tmp_path / "a.json").read_text()) + assert doc["independent_service"]["json"].startswith("https://raw.githubusercontent.com/") + assert "deploy" not in json.dumps(doc["daily"]) and doc["measured_at"] == feed["generated_at"] + assert "Independent status page" in (tmp_path / "a.md").read_text() + + +def test_the_onsite_builder_refuses_a_stale_feed(tmp_path): + with pytest.raises(SystemExit): + avail.main(["--probe-feed", str(REPO / "tests" / "fixtures" / "availability" / "known-bad-stale-feed.json"), + "--output", str(tmp_path / "a.json"), "--md-output", str(tmp_path / "a.md")]) + + +def test_the_monitor_runs_outside_aws_on_a_schedule_and_pushes_only_its_branch(): + wf = (REPO / ".github" / "workflows" / "availability-probe.yml").read_text() + assert "cron: '*/15 * * * *'" in wf + assert "permissions: {}" in wf and "contents: write" in wf + assert "aws-actions" not in wf and "id-token" not in wf + assert "HEAD:refs/heads/availability-status" in wf + assert wf.count("git -c http.extraheader") == 1 and "push --quiet --force" in wf + + +def test_the_deploy_reads_the_independent_feed_before_the_sdr(): + wf = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() + step = wf.index("- name: Read the independent availability feed (fails when missing or stale)") + assert step < wf.index("- name: Build Security Decision Record") + assert "--check-fresh availability-probe.json --max-age-hours 2" in wf[step:step + 600] + assert "--probe-feed availability-probe.json" in wf + + +def test_an_ongoing_incident_fails_the_monitor_run_after_publishing(tmp_path): + samples = [_sample(NOW - timedelta(minutes=15), {"site": "HTTP 500"})] + doc = probe.build({"feed_type": "availability-status", "samples": samples}, _sample(NOW, {"site": "HTTP 500"}), NOW) + (tmp_path / "f.json").write_text(json.dumps(doc)) + assert probe.main(["--fail-on-incident", str(tmp_path / "f.json")]) == 1 + ok = probe.build({}, _sample(NOW), NOW) + (tmp_path / "g.json").write_text(json.dumps(ok)) + assert probe.main(["--fail-on-incident", str(tmp_path / "g.json")]) == 0 + wf = (REPO / ".github" / "workflows" / "availability-probe.yml").read_text() + assert wf.index("Publish the status feed") < wf.index("--fail-on-incident") diff --git a/tests/test_edge_access_logging.py b/tests/test_edge_access_logging.py new file mode 100644 index 00000000..77efb2d7 --- /dev/null +++ b/tests/test_edge_access_logging.py @@ -0,0 +1,154 @@ +"""CDS-TRC-ACL: access to the certification data is logged, within the +operator's privacy limits, and only monthly aggregates are published.""" +import gzip +import importlib.util +import json +import re +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +FIX = REPO / "tests" / "fixtures" / "edge-access-logging" + + +def _load(name, file): + spec = importlib.util.spec_from_file_location(name, REPO / "scripts" / file) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +chk = _load("edge_check", "check-edge-access-logging.py") +summ = _load("access_summary", "build-access-summary.py") +NAME = "samaydlette-com-edge-access" +NOW = datetime(2026, 11, 5, 12, tzinfo=timezone.utc) + + +def _j(p): + return json.loads((FIX / p).read_text()) + + +# --- the configuration check -------------------------------------------------------- + +def test_the_allowlist_equals_the_bootstrap_delivery_fields(): + tf = (REPO / "infrastructure" / "bootstrap" / "trust-access-logs.tf").read_text() + block = re.search(r"edge_access_log_fields = \[(.*?)\]", tf, re.S).group(1) + assert tuple(re.findall(r'"([^"]+)"', block)) == chk.ALLOWED_FIELDS + assert "local.edge_access_log_fields" in tf + + +def test_no_identifying_field_is_allowed(): + assert not set(chk.ALLOWED_FIELDS) & set(chk.IDENTIFYING) + + +def test_the_good_configuration_passes(): + assert chk.problems(_j("good-deliveries.json"), _j("delivery-source.json"), NAME) == [] + + +def test_the_known_bad_fixture_fails_for_identifying_fields(capsys): + rc = chk.main(["--deliveries", str(FIX / "known-bad-deliveries.json"), + "--source", str(FIX / "delivery-source.json"), "--name", NAME]) + assert rc == 1 + err = capsys.readouterr().err + assert "c-ip" in err and "cs-uri-query" in err and "cs(User-Agent)" in err + + +def test_no_delivery_fails_the_deploy(tmp_path): + (tmp_path / "d.json").write_text('{"deliveries": []}') + rc = chk.main(["--deliveries", str(tmp_path / "d.json"), "--source", str(tmp_path / "absent.json"), + "--name", NAME]) + assert rc == 1 + + +def test_a_source_that_is_not_the_distribution_fails(): + src = {"deliverySource": {"name": NAME, "logType": "ACCESS_LOGS", "resourceArns": ["arn:aws:s3:::x"]}} + assert any("not a CloudFront" in p for p in chk.problems(_j("good-deliveries.json"), src, NAME)) + + +def test_a_field_outside_the_allowlist_fails_even_if_not_named_identifying(): + d = _j("good-deliveries.json") + d["deliveries"][0]["recordFields"].append("x-edge-request-id") + assert any("outside the privacy allowlist" in p for p in chk.problems(d, _j("delivery-source.json"), NAME)) + + +def test_the_deploy_checks_logging_before_the_sdr_and_publishes_the_summary(): + wf = (REPO / ".github" / "workflows" / "deploy-with-opa.yml").read_text() + step = wf.index("- name: Check edge access logging and build the access summary") + assert step < wf.index("- name: Build Security Decision Record") + body = wf[step:wf.index("\n - name:", step + 10)] + assert "check-edge-access-logging.py" in body and "build-access-summary.py" in body + assert "access-summary.bundle access-summary.json" in wf + + +def test_raw_edge_records_expire_after_90_days(): + tf = (REPO / "infrastructure" / "logging.tf").read_text() + for prefix in ('prefix = "cloudfront/"', 'CloudFront/" }'): + i = tf.index(prefix) + assert "expiration { days = 90 }" in tf[i:i + 200] + + +# --- the summary ---------------------------------------------------------------------- + +def _rec(day, stem, status=200, cc="US", **extra): + return {"date": day, "time": "10:00:00", "cs-uri-stem": stem, "sc-status": str(status), "c-country": cc, **extra} + + +def test_only_trust_paths_are_counted_and_nothing_identifying_survives(): + recs = [_rec("2026-11-02", "/.well-known/ksi-signal.json"), + _rec("2026-11-02", "/trust/index.html", cc="DE"), + _rec("2026-11-03", "/index.html"), + _rec("2026-11-03", "/.well-known/wp-login.php", status=404), + _rec("2026-11-03", "/.well-known/