diff --git a/.github/workflows/2026.yml b/.github/workflows/2026.yml deleted file mode 100644 index 0f41e98..0000000 --- a/.github/workflows/2026.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: CR26 Deployment -on: - push: - branches: - - main - workflow_dispatch: -permissions: - contents: read - pages: write - id-token: write -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true -jobs: - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - # actions/checkout@v5.0.1 - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd - with: - submodules: recursive - - # actions/configure-pages@v6.0.0 - - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d - - # actions/setup-python@v6.2.0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: 3.x - - # oven-sh/setup-bun@v2.2.0 - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 - - # astral-sh/setup-uv@v8.3.0 - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 - - - name: Install zensical - working-directory: tools - run: uv venv && uv pip install "zensical>=0.0.70" - - - name: Install tool dependencies - working-directory: tools - run: bun install --frozen-lockfile - - - name: Sync rules submodule - working-directory: tools - run: bun sync - - - name: Build - working-directory: tools - run: | - sed -i 's|^site_url = .*|site_url = "https://preview.fedramp.gov/2026/"|' ../zensical.toml - bun run build - # actions/upload-pages-artifact@v5.0.0 - - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 - with: - path: html/ - - - id: deployment - # actions/deploy-pages@v5.0.0 - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 diff --git a/.github/workflows/build-fr-zensical.yml b/.github/workflows/build-fr-zensical.yml new file mode 100644 index 0000000..2f9a9bf --- /dev/null +++ b/.github/workflows/build-fr-zensical.yml @@ -0,0 +1,24 @@ +name: Deploy CR26 to GitHub Pages + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +jobs: + site: + uses: FedRAMP/shared-workflows/.github/workflows/build-fr-zensical.yml@v1 + with: + # Requires the shared workflow's deploy input before this caller is merged. + deploy: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} + use-bun: true + submodules: true + venv-path: tools/.venv + build-script: tools/scripts/build-ci.sh diff --git a/AGENTS.md b/AGENTS.md index 3e4908e..87ace6c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,6 +73,7 @@ bunx tsc -p tsconfig.json --noEmit - `bun run check`: run tests, TypeScript checks, and non-failing content style warnings. - `bun run build`: regenerate `src/` and build `html/`. - `bun run sync`: update the `tools/rules` submodule. +- CI intentionally runs `bun run sync` to build the latest `FedRAMP/rules` `main`, rather than the submodule revision recorded in this repository. Preserve this behavior; repeat builds of a site commit may consume newer rules. - `bunx tsc -p tsconfig.json --noEmit`: type-check the Bun and Node scripts. Generic read-only commands such as `git`, `rg`, and `sed` may run from the repository root. @@ -104,3 +105,7 @@ Generated definitions must remain one alphabetical list of `##` term headings. D When the user asks for a changelog, branch changelog, public change summary, or summary of all changes on the current branch, use `$public-branch-changelog` from `.agents/skills/public-branch-changelog/SKILL.md`. This workflow is mandatory for those requests. Agents without skill support should read that file and follow it directly. + +## Source-derived tests + +Tests comparing generated output with rules or manual content must read the current local source files and derive expected content from them. Do not freeze production wording, identifiers, counts, dates, or destinations in assertions. Keep source expectations independent of generator view models; use small synthetic fixtures with explicit expected results for behavior and edge cases. NIST expectations come from the vendored OSCAL catalog, not copied control facts. diff --git a/tools/README.md b/tools/README.md index a7ef244..03bc1de 100644 --- a/tools/README.md +++ b/tools/README.md @@ -531,3 +531,27 @@ Other generated FRR, KSI, CTL, and full control reference pages receive Definitions, ruleset indexes, summaries, deadlines, and manual pages are unaffected. Navigation changes update this selection on the next generation, including during `bun run dev`. Omitting `generated.search` disables this policy. + +## Testing source fidelity + +`test-support/sources.ts` (under `scripts/`) takes one in-memory snapshot per test +process of the local rules JSON, configuration, navigation, vendored OSCAL catalog, +and manual content. Fixture loaders return clones; tests never sync or modify the +source files. Restart the test process after changing sources. + +Source-contract checks derive record membership and text from those inputs and +mapping filters, independently of `collectArtifacts` view models. They compare +Markdown and rendered HTML, including definitions, rules, indicators, controls, +baselines, deadlines, summary tables, and trigrams. The expected output inventory +comes from source and configuration. Manual copy checks enumerate `content/` first +so missing copies fail, and compare assets byte-for-byte. HTML checks account for +rendering syntax while retaining source wording and punctuation. + +Synthetic fixtures remain appropriate for formatting, filtering, escaping and +failure behavior. Do not add fixed production examples or snapshots to these +checks. `source-contract.test.ts` deliberately corrupts outputs to test that the +comparison helpers detect drift. The build integration test explicitly invokes +the full build before inspecting output; fixture builds use temporary directories. +Run `bun run check` for the full gate. The configured Zensical build may need +network access for fonts; a font-download failure is a build-environment failure, +not a reason to weaken content assertions. diff --git a/tools/package.json b/tools/package.json index e0453b6..5f3d604 100644 --- a/tools/package.json +++ b/tools/package.json @@ -11,6 +11,7 @@ "sync": "./scripts/sync-rules-submodule.sh", "check": "bun run ./scripts/check.ts", "check:style": "bun run ./scripts/style-warnings.ts", + "check:site": "bun run ./scripts/verify-site.ts", "test": "bun test" }, "devDependencies": { diff --git a/tools/rules b/tools/rules index a89d784..c856867 160000 --- a/tools/rules +++ b/tools/rules @@ -1 +1 @@ -Subproject commit a89d78477ddea98dc57eca24f5007794c2b4eeb6 +Subproject commit c85686799c695e267a3fb7ff0debc66cc5c62bcb diff --git a/tools/scripts/build-ci.sh b/tools/scripts/build-ci.sh new file mode 100644 index 0000000..54125cb --- /dev/null +++ b/tools/scripts/build-ci.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Resolve tools/ regardless of the caller's working directory. +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +echo "::group::Install Bun dependencies" +bun install --frozen-lockfile +echo "::endgroup::" + +echo "::group::Sync rules" +# CI deliberately builds the latest upstream main, not the recorded gitlink. +bun run sync +echo "::endgroup::" + +echo "::group::Clear previous build output" + +# Start without old HTML that could mask missing build output. +rm -rf ../html + +echo "::endgroup::" + +echo "::group::Build site and run tests and checks" +# The integration test in bun run check runs the full build and verifies its +# output. Keep that tested artifact for upload instead of rebuilding afterward. +bun run check +echo "::endgroup::" + +echo "::group::Verify generated output" +bun run check:site +echo "::endgroup::" diff --git a/tools/scripts/build-markdown.test.ts b/tools/scripts/build-markdown.test.ts index 1750ae6..ff0bd19 100644 --- a/tools/scripts/build-markdown.test.ts +++ b/tools/scripts/build-markdown.test.ts @@ -1,3 +1,15 @@ +import { + sources, + loadRules, + loadToolConfig, + readTree, +} from "./test-support/sources"; +import { + assertArtifactSource, + expectedOutputPaths, + assertManualCopy, + assertManualHtml, +} from "./test-support/source-contract"; import Handlebars from "handlebars"; import { afterAll, describe, expect, test } from "bun:test"; import { execFile, spawn } from "node:child_process"; @@ -16,19 +28,8 @@ import { promisify } from "node:util"; import Ajv2020 from "ajv/dist/2020"; import addFormats from "ajv-formats"; import type { AnySchema } from "ajv"; -import { - buildMarkdown, - collectArtifacts, - loadRules, - OUTPUT_DIR, - RULES_FILE, -} from "./build-markdown"; -import { - loadToolConfig, - REPO_ROOT, - resolveToolPath, - type ToolConfig, -} from "./config"; +import { buildMarkdown, collectArtifacts, RULES_FILE } from "./build-markdown"; +import { REPO_ROOT, resolveToolPath, type ToolConfig } from "./config"; import { deploy } from "./deploy"; import { findZensicalTagWarnings, @@ -42,11 +43,8 @@ const DEFAULT_RULES_REMOTE_BRANCH = "main"; const RULES_SCHEMA_FILE = resolveToolPath( "rules/schemas/fedramp-consolidated-rules.schema.json", ); -const MACHINE_PICTOGRAPH = - ':lucide-computer:{ .machine title="This content is machine-generated from FedRAMP Machine-Readable Rules." }'; const PERSON_PICTOGRAPH = ':lucide-person-standing:{ .person title="This content was written by a human just for this page." }'; -const MACHINE_PICTO_SPAN = `${MACHINE_PICTOGRAPH}`; const PERSON_PICTO_SPAN = `${PERSON_PICTOGRAPH}`; const WARNING_ORANGE = "\x1b[38;5;208m"; const WARNING_RESET = "\x1b[0m"; @@ -118,7 +116,8 @@ function printContentPictographWarnings(): void { `${WARNING_ORANGE}${WARNING_MARK} Content markdown files should declare picto.source in frontmatter:${WARNING_RESET}`, "", ...contentPictographWarnings.map( - (warning) => ` ${WARNING_ORANGE}${WARNING_MARK} ${warning}${WARNING_RESET}`, + (warning) => + ` ${WARNING_ORANGE}${WARNING_MARK} ${warning}${WARNING_RESET}`, ), "", ].join("\n"), @@ -136,7 +135,8 @@ function printRulesSubmoduleSyncWarnings(): void { `${WARNING_ORANGE}Rules submodule sync warnings:${WARNING_RESET}`, "", ...rulesSubmoduleSyncWarnings.map( - (warning) => ` ${WARNING_ORANGE}${WARNING_MARK} ${warning}${WARNING_RESET}`, + (warning) => + ` ${WARNING_ORANGE}${WARNING_MARK} ${warning}${WARNING_RESET}`, ), "", ].join("\n"), @@ -176,84 +176,17 @@ function expectWithFailureSummary( } } -function expectTextOrder( - contents: string, - expectedTexts: string[], - description: string, -): void { - expectWithFailureSummary(description, () => { - let previousIndex = -1; - for (const expectedText of expectedTexts) { - const index = contents.indexOf(expectedText, previousIndex + 1); - expect( - index, - `${description}: missing or out-of-order text: ${expectedText}`, - ).toBeGreaterThan(previousIndex); - previousIndex = index; - } - }); -} - type RulesForTest = Awaited>; type RequirementDocumentForTest = RulesForTest["FRR"][string]; type ArtifactForTest = ReturnType[number]; -function slugifyTerm(term: string): string { - return term - .toLowerCase() - .replace(/\s+/g, "-") - .replace(/[^a-z0-9-]/g, ""); -} - -function relatedTermsGroupAnchorId(tag: string): string { - return `related-terms-group-${slugifyTerm(tag)}`; -} - -function referenceIndexRowMarkdown(artifact: ArtifactForTest): string[] { - return artifact.context.referenceIndexRows.map((row) => { - const rulesetCell = row.multipleLinks - ? `${row.name}
${row.links - .map((link) => `[${link.label}](${link.href})`) - .join("
")}` - : `[${row.name}](${row.href})`; - - return `| ${row.acronym} | ${rulesetCell} | ${row.status} | ${row.counts} | ${row.updated} |`; - }); -} - -function expectedImportantRelatedTermRows(rules: RulesForTest): string[] { - const taggedTerms = new Map(); - - for (const entry of Object.values(rules.FRD.data.all ?? {})) { - const tag = entry.tag?.trim(); - if (!tag) { - continue; - } - - const terms = taggedTerms.get(tag) ?? []; - terms.push(entry.term); - taggedTerms.set(tag, terms); - } - - return Array.from(taggedTerms.entries()) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([tag, terms]) => { - const linkedTerms = terms - .sort((left, right) => left.localeCompare(right)) - .map((term) => `[${term}](#${slugifyTerm(term)}){ data-preview }`) - .join("
"); - - return `| ${tag} | ${linkedTerms} |`; - }); -} - function findArtifact( artifacts: ArtifactForTest[], relativePath: string, ): ArtifactForTest { - const artifact = artifacts.find((entry) => entry.relativePath === relativePath); + const artifact = artifacts.find( + (entry) => entry.relativePath === relativePath, + ); if (!artifact) { throw new Error(`Expected generated artifact: ${relativePath}`); } @@ -261,53 +194,12 @@ function findArtifact( return artifact; } -function firstArtifactMatching( - artifacts: ArtifactForTest[], - predicate: (artifact: ArtifactForTest) => boolean, - description: string, -): ArtifactForTest { - const artifact = artifacts.find(predicate); - if (!artifact) { - throw new Error(`Expected generated artifact matching: ${description}`); - } - - return artifact; -} - -function artifactWithMappingId( - artifacts: ArtifactForTest[], - mappingId: string, -): ArtifactForTest { - return firstArtifactMatching( - artifacts, - (artifact) => artifact.mappingId === mappingId, - `mapping id ${mappingId}`, +function expectedPictograph(config: ToolConfig) { + const icon = config.pictographs.source.machine.replace( + /\s*}$/, + ` title="${config.pictographs.tooltips.machine}" }`, ); -} - -function artifactsWithMappingId( - artifacts: ArtifactForTest[], - mappingId: string, -): ArtifactForTest[] { - return artifacts.filter((artifact) => artifact.mappingId === mappingId); -} - -function artifactOfType( - artifacts: ArtifactForTest[], - documentType: ArtifactForTest["documentType"], -): ArtifactForTest { - return firstArtifactMatching( - artifacts, - (artifact) => artifact.documentType === documentType, - `document type ${documentType}`, - ); -} - -function artifactsOfType( - artifacts: ArtifactForTest[], - documentType: ArtifactForTest["documentType"], -): ArtifactForTest[] { - return artifacts.filter((artifact) => artifact.documentType === documentType); + return `${icon}`; } function setRulesSourceStatuses( @@ -325,92 +217,6 @@ function setRulesSourceStatuses( } } -function firstRequirementInArtifact(artifact: ArtifactForTest): { - id: string; - title: string; -} { - for (const section of artifact.context.sections) { - const requirement = section.requirements[0]; - if (requirement) { - return requirement; - } - } - - throw new Error(`Expected ${artifact.relativePath} to include a requirement.`); -} - -async function readGeneratedArtifact(artifact: ArtifactForTest): Promise { - return readFile(artifact.outputPath, "utf8"); -} - -function deadlineRowMarkdown(row: { - shortName: string; - displayName: string; - href: string; - optionalAdoption: string; - obtain: string; - maintain: string; - graceEnds: string; -}): string { - return `| [${row.displayName}](${row.href}) | ${row.optionalAdoption} | ${row.obtain} | ${row.maintain} | ${row.graceEnds} |`; -} - -function taggedDocumentSummaryRowMarkdown(row: { - label: string; - href: string; - summary: string; - applicableRuleCount: number; -}): string { - return `| [**${row.label}**](${row.href}) | ${row.summary} |`; -} - -function taggedDocumentSummaryStatsMarkdown(artifact: ArtifactForTest): string { - const stats = artifact.context.taggedDocumentSummaryStats; - if (!stats) { - throw new Error( - `Expected ${artifact.relativePath} to include tagged summary stats.`, - ); - } - - return `!!! tip "There are ${stats.rulesetCount} applicable rulesets with ${stats.ruleCount} total applicable rules."`; -} - -function expectDeadlineRowsFromArtifact( - contents: string, - artifact: ArtifactForTest, - description: string, -): void { - const expectedRows = artifact.context.deadlineTables.flatMap((table) => - table.rows.map(deadlineRowMarkdown), - ); - expect(expectedRows.length).toBeGreaterThan(0); - - expectTextOrder(contents, expectedRows, description); -} - -function expectTaggedDocumentSummaryRowsFromArtifact( - contents: string, - artifact: ArtifactForTest, - description: string, -): void { - const expectedRows = artifact.context.taggedDocumentSummaryRows.map( - taggedDocumentSummaryRowMarkdown, - ); - expect(expectedRows.length).toBeGreaterThan(0); - - expectTextOrder(contents, expectedRows, description); -} - -function expectTaggedDocumentSummaryStatsFromArtifact( - contents: string, - artifact: ArtifactForTest, - description: string, -): void { - expectWithFailureSummary(description, () => { - expect(contents).toContain(taggedDocumentSummaryStatsMarkdown(artifact)); - }); -} - function testRequirementDocument(options: { name: string; shortName: string; @@ -589,9 +395,7 @@ async function listRelativeFiles(root: string): Promise { if (entry.isDirectory()) { const childFiles = await listRelativeFiles(entryPath); - return childFiles.map((childFile) => - path.join(entry.name, childFile), - ); + return childFiles.map((childFile) => path.join(entry.name, childFile)); } if (entry.isFile()) { @@ -630,7 +434,6 @@ function stripGeneratedManualPageAdornments(contents: string): string { continue; } - outputLines.push(line); } @@ -649,7 +452,7 @@ async function findManualSrcContentDrift( srcRoot: string, contentRoot: string, ): Promise { - const srcMarkdownPaths = (await listRelativeFiles(srcRoot)) + const srcMarkdownPaths = (await listRelativeFiles(contentRoot)) .filter((relativePath) => relativePath.endsWith(".md")) .sort(); const drift: ManualSrcContentDrift[] = []; @@ -657,7 +460,8 @@ async function findManualSrcContentDrift( for (const relativePath of srcMarkdownPaths) { const srcPath = path.join(srcRoot, relativePath); const contentPath = path.join(contentRoot, relativePath); - if (!(await fileExists(contentPath))) { + if (!(await fileExists(srcPath))) { + drift.push({ relativePath }); continue; } @@ -702,48 +506,9 @@ interface ZensicalNavLocation { sectionLabel: string; } -function navLocationsInZensicalConfig( - source: string, -): Map { - const locationByPath = new Map(); - const sectionHrefByLabel = new Map(); - let currentSectionLabel: string | null = null; - const topLevelSectionPattern = - /^ \{\s*(?:"([^"]+)"|([A-Za-z][A-Za-z0-9 _'-]*))\s*=/; - - for (const line of source.split(/\r?\n/)) { - const sectionMatch = line.match(topLevelSectionPattern); - if (sectionMatch) { - currentSectionLabel = (sectionMatch[1] ?? sectionMatch[2] ?? "").trim(); - } - - if (!currentSectionLabel) { - continue; - } - - for (const pathMatch of line.matchAll(/"([^"]+\.md)"/g)) { - const relativePath = pathMatch[1]; - if (!relativePath) { - continue; - } - - if (!sectionHrefByLabel.has(currentSectionLabel)) { - sectionHrefByLabel.set(currentSectionLabel, relativePath); - } - - if (!locationByPath.has(relativePath)) { - locationByPath.set(relativePath, { - sectionHref: sectionHrefByLabel.get(currentSectionLabel), - sectionLabel: currentSectionLabel, - }); - } - } - } - - return locationByPath; -} - -async function findBoldMarkdownHeadingWarnings(root: string): Promise { +async function findBoldMarkdownHeadingWarnings( + root: string, +): Promise { const markdownPaths = (await listRelativeFiles(root)) .filter((relativePath) => relativePath.endsWith(".md")) .sort(); @@ -788,9 +553,7 @@ function pictoFrontmatterValue( return null; } - const pictoIndex = frontmatter.findIndex( - (line) => line.trim() === "picto:", - ); + const pictoIndex = frontmatter.findIndex((line) => line.trim() === "picto:"); if (pictoIndex === -1) { return null; } @@ -884,9 +647,18 @@ function generatedMappingPictoStatusFailures(config: ToolConfig): string[] { asRecords(config.generated.fullControlReferenceDocuments), ], ["deadlineDocuments", asRecords(config.generated.deadlineDocuments)], - ["taggedDocumentSummaries", asRecords(config.generated.taggedDocumentSummaries)], - ["referenceIndexDocuments", asRecords(config.generated.referenceIndexDocuments)], - ["frrCollectionDocuments", asRecords(config.generated.frrCollectionDocuments)], + [ + "taggedDocumentSummaries", + asRecords(config.generated.taggedDocumentSummaries), + ], + [ + "referenceIndexDocuments", + asRecords(config.generated.referenceIndexDocuments), + ], + [ + "frrCollectionDocuments", + asRecords(config.generated.frrCollectionDocuments), + ], ["ruleDocuments", asRecords(config.generated.ruleDocuments)], ]; const failures: string[] = []; @@ -922,6 +694,19 @@ function pictographTooltipFailures(config: ToolConfig): string[] { return failures; } +let markdownBuildPromise: ReturnType | undefined; +function markdownBuild() { + return (markdownBuildPromise ??= (async () => { + await sources(); + await deploy(); + const summary = await buildMarkdown(); + expect(summary.artifacts.map((a) => a.relativePath).sort()).toEqual( + await expectedOutputPaths(), + ); + return summary; + })()); +} + describe("build-markdown", () => { test("the consolidated rules source exists", async () => { await access(RULES_FILE); @@ -1006,7 +791,10 @@ describe("build-markdown", () => { }); } - const expectedHead = await expectedRulesHead(rulesPath, expectedRulesBranch); + const expectedHead = await expectedRulesHead( + rulesPath, + expectedRulesBranch, + ); const syncFailureSummary = [ `tools/rules is not synced to ${expectedHead.source}.`, @@ -1034,705 +822,29 @@ describe("build-markdown", () => { }); }); - test("builds configured markdown files from the JSON source", async () => { - const config = await loadToolConfig(); - const rules = await loadRules(config); - const expectedArtifacts = collectArtifacts(rules, config); - - await deploy(); - const summary = await buildMarkdown(); - expect(summary.artifactCount).toBe(expectedArtifacts.length); - - const searchableReferences = new Set([ - "complete-ruleset-reference", "complete-ksi-reference", - "complete-rev5-controls-reference", "full-rev5-control-reference", - ]); - const searchContentTypes = new Set(["FRR", "KSI", "CTL", "CTL_REFERENCE"]); - for (const artifact of summary.artifacts) { - const excluded = searchContentTypes.has(artifact.documentType) && - !searchableReferences.has(artifact.mappingId); - expect(artifact.context.searchExclude, artifact.relativePath).toBe(excluded); - const markdown = await readFile(artifact.outputPath, "utf8"); - const frontmatter = markdown.split("---")[1] ?? ""; - expect(/search:\s*\n exclude: true/.test(frontmatter), artifact.relativePath).toBe(excluded); - } - - - const relativePaths = summary.artifacts - .map((artifact) => artifact.relativePath) - .sort(); - expect(relativePaths).toEqual( - expectedArtifacts.map((artifact) => artifact.relativePath).sort(), - ); - for (const mapping of config.generated.definitionDocuments ?? []) { - expect(relativePaths).toContain(mapping.output); - } - for (const mapping of config.generated.referenceIndexDocuments ?? []) { - expect(relativePaths).toContain(mapping.output); - } - for (const mapping of config.generated.frrCollectionDocuments ?? []) { - expect(relativePaths).toContain(mapping.output); - } - for (const mapping of config.generated.taggedDocumentSummaries ?? []) { - expect(relativePaths).toContain(mapping.output); - } - - for (const mapping of config.generated.deadlineDocuments ?? []) { - for (const artifact of artifactsWithMappingId(expectedArtifacts, mapping.id)) { - expect(relativePaths).toContain(artifact.relativePath); - } - } - - for (const mapping of config.generated.ksiDocuments ?? []) { - const artifacts = artifactsWithMappingId(expectedArtifacts, mapping.id); - expect(artifacts.length).toBeGreaterThan(0); - for (const artifact of artifacts) { - expect(relativePaths).toContain(artifact.relativePath); - } - } - for (const mapping of config.generated.controlDocuments ?? []) { - const artifacts = artifactsWithMappingId(expectedArtifacts, mapping.id); - expect(artifacts.length).toBeGreaterThan(0); - for (const artifact of artifacts) { - expect(relativePaths).toContain(artifact.relativePath); - } - } - for (const mapping of config.generated.fullControlReferenceDocuments ?? - []) { - const artifacts = artifactsWithMappingId(expectedArtifacts, mapping.id); - expect(artifacts.length).toBe(21); - for (const artifact of artifacts) { - expect(relativePaths).toContain(artifact.relativePath); - } - } - - for (const mapping of config.generated.ruleDocuments) { - const artifacts = artifactsWithMappingId(expectedArtifacts, mapping.id); - expect(artifacts.length).toBeGreaterThan(0); - for (const artifact of artifacts) { - expect(relativePaths).toContain(artifact.relativePath); - } - } - - const referenceArtifactPaths = relativePaths.filter((relativePath) => - relativePath.startsWith("reference/"), - ); - expect(referenceArtifactPaths).toHaveLength( - expectedArtifacts.filter((artifact) => - artifact.relativePath.startsWith("reference/"), - ).length, - ); - - const contentTypeTagByDocumentType = new Map([ - ["CTL", "Controls"], - ["CTL_REFERENCE", "Controls"], - ["DEADLINES", "Deadlines"], - ["FRD", "Definitions"], - ["FRR", "Rules"], - ["FRR_REFERENCE_INDEX", "Rules"], - ["TRIGRAMS", "Rules"], - ["FRR_TAGGED_SUMMARY", "Rules"], - ["KSI", "Key Security Indicators"], - ]); - const audienceTagByPathPrefix = new Map([ - ["advisors/", "Advisors"], - ["agencies/", "Federal Agencies"], - ["assessors/", "Independent Assessors"], - ["providers/", "Cloud Service Providers"], - ]); - - for (const artifact of expectedArtifacts) { - await access(artifact.outputPath); - const contents = await readFile(artifact.outputPath, "utf8"); - const expectedContentTypeTag = contentTypeTagByDocumentType.get( - artifact.documentType, - ); - if (!expectedContentTypeTag) { - throw new Error( - `Missing expected content-type tag for ${artifact.documentType}`, + for (const documentType of [ + "FRR", + "FRD", + "KSI", + "CTL", + "CTL_REFERENCE", + "DEADLINES", + "FRR_TAGGED_SUMMARY", + "FRR_REFERENCE_INDEX", + "TRIGRAMS", + ] as const) { + test(`${documentType} Markdown matches current authoritative source records`, async () => { + const summary = await markdownBuild(); + for (const artifact of summary.artifacts.filter( + (a) => a.documentType === documentType, + )) { + await assertArtifactSource( + artifact, + await readFile(artifact.outputPath, "utf8"), ); } - expect(artifact.context.tags).toContain(expectedContentTypeTag); - - for (const [pathPrefix, audienceTag] of audienceTagByPathPrefix) { - if (artifact.relativePath.startsWith(pathPrefix)) { - expect(artifact.context.tags).toContain(audienceTag); - } - } - - expect(contents).toContain(`# ${artifact.title}`); - expect(contents.trim().length).toBeGreaterThan(0); - } - - const referenceIndexArtifact = artifactOfType( - expectedArtifacts, - "FRR_REFERENCE_INDEX", - ); - const referenceIndexContents = await readGeneratedArtifact( - referenceIndexArtifact, - ); - expect(referenceIndexContents).toStartWith( - [ - "---", - `title: ${JSON.stringify(referenceIndexArtifact.title)}`, - `description: ${JSON.stringify(referenceIndexArtifact.context.description ?? "")}`, - `purpose: ${JSON.stringify(referenceIndexArtifact.context.purpose ?? "")}`, - 'google_doc: ""', - "tags:", - ...referenceIndexArtifact.context.tags.map((tag) => ` - ${tag}`), - "picto:", - " source: machine", - "---", - "", - referenceIndexArtifact.context.pictoSpan ?? "", - "", - `# ${referenceIndexArtifact.title}`, - ].join("\n"), - ); - for (const paragraph of referenceIndexArtifact.context.purposeParagraphs) { - expect(referenceIndexContents).toContain(paragraph); - } - expectTextOrder( - referenceIndexContents, - [ - `# ${referenceIndexArtifact.title}`, - referenceIndexArtifact.context.introduction ?? "", - "| Acronym | Ruleset | Status | Counts | Most Recently Updated |", - ], - "Generated reference index should place configured introduction before the table", - ); - expect(referenceIndexContents).toContain( - "| Acronym | Ruleset | Status | Counts | Most Recently Updated |", - ); - expectTextOrder( - referenceIndexContents, - referenceIndexRowMarkdown(referenceIndexArtifact), - "Generated reference index should render source-derived rows in acronym order", - ); - - for (const [relativePath, expectedTag] of [ - ["reference/20x/index.md", "20x"], - ["reference/rev5/index.md", "Rev5"], - ] as const) { - const pathTaggedArtifact = findArtifact(expectedArtifacts, relativePath); - const pathTaggedContents = - await readGeneratedArtifact(pathTaggedArtifact); - - expect(pathTaggedArtifact.context.tags).toContain(expectedTag); - expect(pathTaggedContents).toContain("tags:"); - expect(pathTaggedContents).toContain(` - ${expectedTag}`); - } - - const effectiveDateArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => artifact.context.effectiveEntries.length > 0, - "an artifact with effective-date entries", - ); - const effectiveDateContents = await readGeneratedArtifact(effectiveDateArtifact); - const effectiveDateTexts = effectiveDateArtifact.context.effectiveEntries.flatMap( - (entry) => [ - `!!! info "Effective Date(s) & Overall Applicability for ${entry.audienceLabel}"`, - `- **${entry.statusLabel}**`, - ...entry.dateLines.map((line) => `- **${line.label}:** ${line.value}`), - ...entry.classLines.map((line) => `- **${line.label}:** ${line.value}`), - ], - ); - expectTextOrder( - effectiveDateContents, - effectiveDateTexts, - "Generated effective-date metadata should render source-derived dates", - ); - expect(effectiveDateContents).not.toContain("[object Object]"); - - const definitionsArtifact = artifactOfType(expectedArtifacts, "FRD"); - const definitionsContents = await readGeneratedArtifact(definitionsArtifact); - const definitionsPurpose = rules.FRD.info.purpose; - const definitionHeaders = Array.from( - definitionsContents.matchAll(/^## (.+)$/gm), - (match) => match[1], - ).filter((heading) => heading !== "Important Related Terms"); - const definitionTerms = Object.values(rules.FRD.data.all ?? {}) - .map((entry) => entry.term) - .sort((left, right) => left.localeCompare(right)); - const relatedTermRows = expectedImportantRelatedTermRows(rules); - const definitionIntroOrder = [ - "# FedRAMP Definitions", - definitionsPurpose ?? "", - ]; - if (relatedTermRows.length) { - definitionIntroOrder.push( - "## Important Related Terms", - "| Related Terms Group | Terms |", - ); - } - if (definitionTerms[0]) { - definitionIntroOrder.push(`## ${definitionTerms[0]}`); - } - - expect(definitionsPurpose).toBeTruthy(); - expect(definitionsContents).toStartWith( - `---\ntags:\n${definitionsArtifact.context.tags.map((tag) => ` - ${tag}`).join("\n")}\n---\n\n${definitionsArtifact.context.pictoSpan}\n\n# ${definitionsArtifact.title}`, - ); - expectTextOrder( - definitionsContents, - definitionIntroOrder, - "Generated FRD markdown should place info.purpose and related terms table before the definitions", - ); - expect(definitionsContents).not.toContain("**Subsets**"); - expect(definitionsContents).not.toContain( - '??? abstract "Background & Authority"', - ); - expect(definitionsContents).not.toContain("Effective Date(s)"); - expect(definitionsContents).not.toContain("Overall Applicability"); - expect(definitionsContents).toContain('!!! quote ""'); - expect(definitionsContents).not.toContain("## General Terms"); - expect(definitionsContents).not.toContain("## Related Terms:"); - for (const row of relatedTermRows) { - expect(definitionsContents).toContain(row); - } - const firstTaggedDefinition = Object.values(rules.FRD.data.all ?? {}).find( - (entry) => entry.tag?.trim(), - ); - if (firstTaggedDefinition?.tag) { - expect(definitionsContents).toContain( - `**Related Terms Group:** [${firstTaggedDefinition.tag}](#${relatedTermsGroupAnchorId( - firstTaggedDefinition.tag, - )})`, - ); - } - expect(definitionHeaders).toEqual(definitionTerms); - - const ksiThemeArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.documentType === "KSI" && artifact.context.indicators.length > 0, - "a theme-based KSI artifact", - ); - const ksiThemeContents = await readGeneratedArtifact(ksiThemeArtifact); - const ksiThemeIndicator = ksiThemeArtifact.context.indicators[0]; - if (!ksiThemeIndicator) { - throw new Error( - `Expected ${ksiThemeArtifact.relativePath} to include a KSI indicator.`, - ); - } - expect(ksiThemeContents).toStartWith( - `---\nsearch:\n exclude: true\ntags:\n${ksiThemeArtifact.context.tags.map((tag) => ` - ${tag}`).join("\n")}\n---\n\n${ksiThemeArtifact.context.pictoSpan}\n\n# ${ksiThemeArtifact.title}`, - ); - expect(ksiThemeContents).not.toContain("**Subsets**"); - expect(ksiThemeContents).toContain(ksiThemeIndicator.id); - expect(ksiThemeContents).toContain(`### ${ksiThemeIndicator.title}`); - if (ksiThemeIndicator.controlLinks[0]) { - const control = ksiThemeIndicator.controlLinks[0]; - expect(ksiThemeContents).toContain("**Related SP 800-53 Controls:**"); - expect(ksiThemeContents).toContain( - `[${control.label}](${control.url}){ data-preview }`, - ); - expect(control.url).toContain("reference/controls/"); - } - - const ksiReferenceArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.documentType === "KSI" && artifact.context.sections.length > 0, - "a grouped KSI reference artifact", - ); - const ksiReferenceContents = await readGeneratedArtifact(ksiReferenceArtifact); - const firstKsiSection = ksiReferenceArtifact.context.sections[0]; - const firstKsiSectionRequirement = firstKsiSection?.requirements[0]; - if (!firstKsiSection || !firstKsiSectionRequirement) { - throw new Error( - `Expected ${ksiReferenceArtifact.relativePath} to include grouped KSI indicators.`, - ); - } - expect(ksiReferenceContents).toStartWith( - `---\ntags:\n${ksiReferenceArtifact.context.tags.map((tag) => ` - ${tag}`).join("\n")}\n---\n\n${ksiReferenceArtifact.context.pictoSpan}\n\n# ${ksiReferenceArtifact.title}`, - ); - expect(ksiReferenceContents).not.toContain("**Subsets**"); - expectTextOrder( - ksiReferenceContents, - [ - `# ${ksiReferenceArtifact.title}`, - `## ${firstKsiSection.title}`, - `### ${firstKsiSectionRequirement.title}`, - firstKsiSectionRequirement.id, - ], - "Generated KSI reference markdown should group indicators by source theme", - ); - - for (const deadlineArtifact of artifactsOfType(expectedArtifacts, "DEADLINES")) { - const deadlineContents = await readGeneratedArtifact(deadlineArtifact); - expect(deadlineContents).toContain(`# ${deadlineArtifact.title}`); - expect(deadlineContents).toContain( - "| Ruleset | Optional Adoption Allowed | Obtaining Initial Certification | Maintaining Ongoing Certification | Grace Period Ends |", - ); - expectDeadlineRowsFromArtifact( - deadlineContents, - deadlineArtifact, - `Generated deadline markdown should render source-derived rows for ${deadlineArtifact.relativePath}`, - ); - } - - for (const summaryArtifact of artifactsOfType( - expectedArtifacts, - "FRR_TAGGED_SUMMARY", - )) { - const summaryContents = await readGeneratedArtifact(summaryArtifact); - expect(summaryContents).toContain(`# ${summaryArtifact.title}`); - expectTaggedDocumentSummaryStatsFromArtifact( - summaryContents, - summaryArtifact, - `Generated tagged summary should render aggregate stats for ${summaryArtifact.relativePath}`, - ); - - if (summaryArtifact.context.taggedDocumentSummaryRows.length) { - expect(summaryContents).toContain("| Ruleset | Summary |"); - expect(summaryContents).toContain("

**Applicable Rules:**"); - expectTaggedDocumentSummaryRowsFromArtifact( - summaryContents, - summaryArtifact, - `Generated tagged summary should render source-derived rows for ${summaryArtifact.relativePath}`, - ); - } else { - expect(summaryContents).toContain("No matching rules are currently available."); - } - } - - for (const artifact of artifactsOfType(expectedArtifacts, "FRR")) { - const contents = await readGeneratedArtifact(artifact); - for (const section of artifact.context.sections) { - expect(contents).toContain(`## ${section.title}`); - for (const group of section.applicabilityGroups) { - const expectedGroup = [ - ``, - `${group.label}:`, - '', - ...group.values.map( - (value) => - `${value}`, - ), - ]; - expectTextOrder( - contents, - expectedGroup, - `Generated FRR markdown should render ${group.label.toLowerCase()} applicability labels for ${section.title}`, - ); - } - - const requirement = section.requirements[0]; - if (requirement) { - expect(contents).toContain(requirement.id); - expect(contents).toContain(`### ${requirement.title}`); - } - - if (section.applicabilityGroups.length && requirement) { - expectTextOrder( - contents, - [ - `## ${section.title}`, - section.descriptionParagraphs.at(-1) ?? "", - '
', - `### ${requirement.title}`, - ], - `Generated FRR markdown should place applicability labels after the subset description and before its requirements for ${section.title}`, - ); - } - } - } - - const controlGuidanceArtifact = findArtifact( - expectedArtifacts, - "reference/rev5-control-guidance.md", - ); - const controlGuidanceContents = await readGeneratedArtifact( - controlGuidanceArtifact, - ); - const accessControlFamily = - controlGuidanceArtifact.context.controlFamilies.find( - (family) => family.id === "AC", - ); - const externalSystemsControl = accessControlFamily?.controls.find( - (control) => control.id === "AC-20", - ); - expect(controlGuidanceArtifact.title).toBe("Rev5 Control Guidance"); - expect(relativePaths).not.toContain("reference/rev5-controls.md"); - expect(controlGuidanceContents).toContain("# Rev5 Control Guidance"); - expect(externalSystemsControl?.commonGuidance?.guidance).toContain( - "The interrelated controls of AC-20, CA-3, and SA-9 should be differentiated as follows:", - ); - - const fullControlIndexArtifact = findArtifact( - expectedArtifacts, - "reference/controls/index.md", - ); - const fullControlIndexContents = await readGeneratedArtifact( - fullControlIndexArtifact, - ); - expect(fullControlIndexArtifact.context.controlCount).toBe(1014); - expect(fullControlIndexArtifact.context.controlFamilies).toHaveLength(20); - expect(fullControlIndexContents).toContain( - "| [Access Control](access-control.md){ data-preview } | `AC` | 131 |", - ); - expect(fullControlIndexContents).toContain( - "Withdrawn controls are not included.", - ); - - const fullAccessControlArtifact = findArtifact( - expectedArtifacts, - "reference/controls/access-control.md", - ); - const fullAccessControlContents = await readGeneratedArtifact( - fullAccessControlArtifact, - ); - const fullAccessControls = - fullAccessControlArtifact.context.controlFamilies[0]?.controls ?? []; - const ac01 = fullAccessControls.find((control) => control.id === "AC-01"); - const ac0201 = fullAccessControls.find( - (control) => control.id === "AC-02-01", - ); - const ac20 = fullAccessControls.find((control) => control.id === "AC-20"); - expect(fullAccessControls).toHaveLength(131); - expect(ac01?.baselineClasses).toEqual(["Class B", "Class C", "Class D"]); - expect(ac0201?.baselineClasses).toEqual(["Class C", "Class D"]); - expect(ac0201?.externalUrl).toBe( - "https://myctrl.tools/frameworks/nist-800-53-r5/ac-2-1", - ); - expect(ac20?.hasFedrampContent).toBe(true); - expectTextOrder( - fullAccessControlContents, - [ - "## AC-01 (Policy and Procedures) { #ac-01 }", - "FedRAMP Rev5 Baselines:", - "!!! quote", - "_This control does not have additional FedRAMP guidance or FedRAMP-assigned parameter values._", - "**External Link for Additional Information:**", - "## AC-02 (01) (Automated System Account Management) { #ac-02-01 }", - "https://myctrl.tools/frameworks/nist-800-53-r5/ac-2-1", - ], - "Full Rev5 control family pages should render active catalog statements, baseline tags, external links, and FedRAMP information", - ); - expect(fullAccessControlContents).not.toContain( - "## AC-02 (10) (Shared and Group Account Credential Change)", - ); - expect(fullAccessControlContents).not.toContain( - '??? abstract "Control information"', - ); - const ac01Section = - fullAccessControlContents.split("## AC-02 (Account Management)")[0] ?? ""; - expect(ac01Section).not.toContain('!!! info ""'); - - const certificationArtifact = findArtifact( - expectedArtifacts, - "providers/rev5/rules/fedramp-certification.md", - ); - const certificationContents = await readGeneratedArtifact( - certificationArtifact, - ); - const baselineRequirement = certificationArtifact.context.sections - .flatMap((section) => section.requirements) - .find((requirement) => requirement.id === "FRC-CSF-BSL"); - if (!baselineRequirement) { - throw new Error( - "Expected the Rev5 FedRAMP Certification page to include FRC-CSF-BSL.", - ); - } - const baselineCounts = Object.fromEntries( - baselineRequirement.variantSections.map((variant) => [ - variant.title, - variant.rev5ControlFamilies.reduce( - (count, family) => count + family.controls.length, - 0, - ), - ]), - ); - expect(baselineCounts).toEqual({ - "Class B": 155, - "Class C": 322, - "Class D": 409, }); - expectTextOrder( - certificationContents, - [ - '???+ info "Rev5 Control List"', - "- **Access Control (AC)**", - "- `AC-01` ([Policy and Procedures](../../../reference/controls/access-control.md#ac-01){ data-preview })", - "- `AT-02 (02)` ([Insider Threat](../../../reference/controls/awareness-and-training.md#at-02-02){ data-preview })", - ], - "Generated Rev5 baseline rules should render grouped, OSCAL-enriched control lists", - ); - - const assessmentArtifact = findArtifact( - expectedArtifacts, - "providers/rev5/rules/independent-verification-and-validation.md", - ); - const assessmentContents = await readGeneratedArtifact(assessmentArtifact); - expect(assessmentContents).toContain( - "- `CA-08` ([Penetration Testing](../../../reference/controls/assessment-authorization-and-monitoring.md#ca-08){ data-preview })", - ); - expect(assessmentContents).toContain( - "- `CA-08 (01)` ([Independent Penetration Testing Agent or Team](../../../reference/controls/assessment-authorization-and-monitoring.md#ca-08-01){ data-preview })", - ); - - const workflowArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => artifact.context.flows.length > 0, - "an FRR artifact with an activity workflow", - ); - const workflowContents = await readGeneratedArtifact(workflowArtifact); - const workflow = workflowArtifact.context.flows[0]; - if (!workflow) { - throw new Error(`Expected ${workflowArtifact.relativePath} to include a workflow.`); - } - expect(workflowContents).toContain(`## Activity Workflow: ${workflow.title}`); - expect(workflowContents).toContain("``` mermaid"); - for (const line of workflow.mermaidLines) { - expect(workflowContents).toContain(line); - } - - const schemaArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.context.sections.some((section) => - section.requirements.some((requirement) => requirement.schema), - ), - "an FRR artifact with related JSON schema metadata", - ); - const schemaContents = await readGeneratedArtifact(schemaArtifact); - const schemaRequirement = schemaArtifact.context.sections - .flatMap((section) => section.requirements) - .find((requirement) => requirement.schema); - if (!schemaRequirement?.schema) { - throw new Error(`Expected ${schemaArtifact.relativePath} to include schema metadata.`); - } - expectTextOrder( - schemaContents, - [ - `### ${schemaRequirement.title}`, - `!!! schema "Related JSON Schema: [${schemaRequirement.schema.name}](${schemaRequirement.schema.url})"`, - '!!! quote ""', - ], - "Generated requirement markdown should place related JSON schema metadata before the statement", - ); - - const notificationArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.context.sections.some((section) => - section.requirements.some( - (requirement) => requirement.notifications.length > 0, - ), - ), - "an FRR artifact with notification metadata", - ); - const notificationContents = await readGeneratedArtifact(notificationArtifact); - const notificationRequirement = notificationArtifact.context.sections - .flatMap((section) => section.requirements) - .find((requirement) => requirement.notifications.length > 0); - const notification = notificationRequirement?.notifications[0]; - if (!notification) { - throw new Error( - `Expected ${notificationArtifact.relativePath} to include notification metadata.`, - ); - } - const notificationDestination = notification.href - ? `[${notification.linkLabel}](${notification.href})` - : `${notification.name}${notification.targetDetail ?? ""}`; - expect(notificationContents).toContain( - `Notify ${notification.party} via ${notification.methodLabel}: ${notificationDestination}.`, - ); - - const bracketedNotificationArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.context.sections.some((section) => - section.requirements.some((requirement) => - requirement.notifications.some( - (entry) => entry.href && entry.linkLabel.includes("\\["), - ), - ), - ), - "an FRR artifact with bracketed notification link text", - ); - const bracketedNotificationContents = await readGeneratedArtifact( - bracketedNotificationArtifact, - ); - const bracketedNotification = bracketedNotificationArtifact.context.sections - .flatMap((section) => section.requirements) - .flatMap((requirement) => requirement.notifications) - .find((entry) => entry.href && entry.linkLabel.includes("\\[")); - if (!bracketedNotification?.href) { - throw new Error( - `Expected ${bracketedNotificationArtifact.relativePath} to include bracketed notification link text.`, - ); - } - expect(bracketedNotificationContents).toContain( - `[${bracketedNotification.linkLabel}](${bracketedNotification.href})`, - ); - - const classARelatedArtifact = findArtifact( - expectedArtifacts, - "reference/20x/a/related.md", - ); - const classARelatedSectionTitles = classARelatedArtifact.context.sections.map( - (section) => section.title, - ); - expect(classARelatedSectionTitles).toContain( - "Mandatory Class A Rules: Certification Data Sharing (CDS)", - ); - expect(classARelatedSectionTitles).toContain( - "Recommended Class A Rules: Certification Package Overview (CPO)", - ); - expect(classARelatedSectionTitles).toContain( - "Optional Class A Rules: Collaborative Continuous Monitoring (CCM)", - ); - expect( - classARelatedArtifact.context.sections - .flatMap((section) => section.requirements) - .filter((requirement) => requirement.id === "CDS-CSO-AVR"), - ).toHaveLength(1); - - const referenceArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.context.sections.some((section) => - section.requirements.some((requirement) => requirement.reference), - ), - "an FRR artifact with a requirement reference link", - ); - const referenceContents = await readGeneratedArtifact(referenceArtifact); - const referenceRequirement = referenceArtifact.context.sections - .flatMap((section) => section.requirements) - .find((requirement) => requirement.reference); - if (!referenceRequirement?.reference) { - throw new Error( - `Expected ${referenceArtifact.relativePath} to include reference metadata.`, - ); - } - expect(referenceContents).toContain( - `**Reference:** [${referenceRequirement.reference.label}](${referenceRequirement.reference.url})`, - ); - - const frrCollectionMapping = config.generated.frrCollectionDocuments?.[0]; - if (frrCollectionMapping) { - const frrCollectionArtifact = artifactWithMappingId( - expectedArtifacts, - frrCollectionMapping.id, - ); - const frrCollectionContents = - await readGeneratedArtifact(frrCollectionArtifact); - const frrCollectionRequirement = - firstRequirementInArtifact(frrCollectionArtifact); - expect(frrCollectionContents).not.toContain("Effective Date(s)"); - expect(frrCollectionContents).not.toContain("Activity Workflow"); - expect(frrCollectionContents).not.toContain("``` mermaid"); - expect(frrCollectionContents).toContain(frrCollectionRequirement.id); - } - - const contentDefinitionsPath = path.join( - resolveToolPath(config.paths.content), - "definitions.md", - ); - await expect(access(contentDefinitionsPath)).rejects.toThrow(); - }); + } test("renders source-only pictographs for generated pages", async () => { const config = await loadToolConfig(); @@ -1746,7 +858,7 @@ describe("build-markdown", () => { expect( artifact.context.pictoSpan, `${artifact.relativePath} should render only the machine source pictograph`, - ).toBe(MACHINE_PICTO_SPAN); + ).toBe(expectedPictograph(config)); } }); @@ -1756,6 +868,18 @@ describe("build-markdown", () => { const statement = "Providers MUST comply, MUST NOT disclose, SHOULD review, SHOULD NOT skip, and MAY proceed. Providers MUST document decisions."; const terms = ["MUST", "MUST NOT", "SHOULD", "SHOULD NOT", "MAY"]; + rules.FRD.data = { + all: Object.fromEntries( + [...terms, "Certification Package"].map((term, i) => [ + `FRD-T${i}`, + { + term, + definition: `Synthetic definition of ${term}.`, + do_not_link: terms.includes(term), + }, + ]), + ), + }; const expectedStatement = (href: string) => `Providers [MUST](${href}#must){ data-preview } comply, [MUST NOT](${href}#must-not){ data-preview } disclose, [SHOULD](${href}#should){ data-preview } review, [SHOULD NOT](${href}#should-not){ data-preview } skip, and [MAY](${href}#may){ data-preview } proceed. Providers [MUST](${href}#must){ data-preview } document decisions.`; const syntheticDocument = testRequirementDocument({ @@ -1873,10 +997,9 @@ describe("build-markdown", () => { .find((entry) => entry.id === "SYN-GEN-ONE")!; expect(requirement.statementParagraphs[0]).toBe(expectedStatement(href)); - expect(requirement.variantSections.map((variant) => variant.title)).toEqual([ - "Class B", - "Low", - ]); + expect( + requirement.variantSections.map((variant) => variant.title), + ).toEqual(["Class B", "Low"]); for (const variant of requirement.variantSections) { expect(variant.statementParagraphs).toEqual([expectedStatement(href)]); } @@ -1893,7 +1016,10 @@ describe("build-markdown", () => { "Providers MUST NOT omit this.", ]); expect(requirement.terms).toEqual([ - { label: "Certification Package", href: `${href}#certification-package` }, + { + label: "Certification Package", + href: `${href}#certification-package`, + }, ]); } @@ -1902,7 +1028,8 @@ describe("build-markdown", () => { "reference/synthetic-ruleset.md", ); expect( - documentArtifact.context.sections[0]?.requirements[0]?.statementParagraphs[1], + documentArtifact.context.sections[0]?.requirements[0] + ?.statementParagraphs[1], ).toBe( "See [REL-GEN-ONE (Related Requirement)](related.md#related-requirement){ data-preview }.", ); @@ -1975,7 +1102,7 @@ describe("build-markdown", () => { ).toBe(false); }); - test("expands source.types \"all\" to common, 20x, and Rev5 rule content", async () => { + test('expands source.types "all" to common, 20x, and Rev5 rule content', async () => { const config = await loadToolConfig(); const rules = structuredClone(await loadRules(config)); const syntheticDocument = testRequirementDocument({ @@ -2159,7 +1286,8 @@ describe("build-markdown", () => { BON: { "OMT-BON-ONE": { name: "Omitted Class B Requirement", - statement: "This document is not included in the linked rule mapping.", + statement: + "This document is not included in the linked rule mapping.", affects: ["Providers"], }, }, @@ -2477,10 +1605,12 @@ describe("build-markdown", () => { indicators: { "KSI-SYN-VAR": { name: "Class Variant Indicator", - statement: "Indicators MUST remain unchanged and MAY use force terms.", + statement: + "Indicators MUST remain unchanged and MAY use force terms.", varies_by_class: { b: { - statement: "Class B indicator SHOULD render and MUST NOT change.", + statement: + "Class B indicator SHOULD render and MUST NOT change.", }, c: { statement: "Class C indicator should not render.", @@ -2531,9 +1661,9 @@ describe("build-markdown", () => { "Indicators MUST remain unchanged and MAY use force terms.", ]); expect(indicator?.variantSections[0]?.title).toBe("Class B"); - expect( - indicator?.variantSections[0]?.statementParagraphs.join("\n"), - ).toBe("Class B indicator SHOULD render and MUST NOT change."); + expect(indicator?.variantSections[0]?.statementParagraphs.join("\n")).toBe( + "Class B indicator SHOULD render and MUST NOT change.", + ); expect( indicator?.variantSections[0]?.statementParagraphs.join("\n"), ).not.toContain("Class C indicator should not render."); @@ -2666,13 +1796,18 @@ describe("build-markdown", () => { test("renders optional introductions across generated mappings", async () => { const config = await loadToolConfig(); const rules = await loadRules(config); - const introduction = "Introduction **Markdown** & [link](https://example.com)\n- first\n- second\n\nAnother paragraph."; + const introduction = + "Introduction **Markdown** & [link](https://example.com)\n- first\n- second\n\nAnother paragraph."; const engine = Handlebars.create(); for (const file of await readdir(resolveToolPath(config.paths.partials))) { if (file.endsWith(".hbs")) { - engine.registerPartial(path.basename(file, ".hbs"), await readFile( - path.join(resolveToolPath(config.paths.partials), file), "utf8", - )); + engine.registerPartial( + path.basename(file, ".hbs"), + await readFile( + path.join(resolveToolPath(config.paths.partials), file), + "utf8", + ), + ); } } const templates = new Map(); @@ -2683,7 +1818,8 @@ describe("build-markdown", () => { for (const mapping of mappings) mapping.introduction = value; } } - if (copy.generated.definitions) copy.generated.definitions.introduction = value; + if (copy.generated.definitions) + copy.generated.definitions.introduction = value; return copy; }; const baseline = collectArtifacts(rules, configure()); @@ -2693,14 +1829,20 @@ describe("build-markdown", () => { expect(artifact.context.introduction).toBe(introduction); let render = templates.get(artifact.templatePath); if (!render) { - render = engine.compile(await readFile(artifact.templatePath, "utf8"), { noEscape: true }); + render = engine.compile(await readFile(artifact.templatePath, "utf8"), { + noEscape: true, + }); templates.set(artifact.templatePath, render); } const output = render(artifact.context); expect(output).toContain(`# ${artifact.title}\n\n${introduction}\n\n`); expect(output.split(introduction).length).toBe(2); - expect(output.replace(`${introduction}\n\n`, "")).toBe(render(baseline[index]!.context)); - expect(render({ ...artifact.context, introduction: "" })).toBe(render(baseline[index]!.context)); + expect(output.replace(`${introduction}\n\n`, "")).toBe( + render(baseline[index]!.context), + ); + expect(render({ ...artifact.context, introduction: "" })).toBe( + render(baseline[index]!.context), + ); } for (const artifact of collectArtifacts(rules, configure(" \n\t "))) { expect(artifact.context.introduction).toBeUndefined(); @@ -2708,8 +1850,16 @@ describe("build-markdown", () => { const legacy = configure(introduction); legacy.generated.trigramDocuments = []; legacy.generated.definitionDocuments = []; - legacy.generated.definitions = { enabled: true, output: "legacy-definitions.md", introduction }; - expect(collectArtifacts(rules, legacy).find((artifact) => artifact.relativePath === "legacy-definitions.md")?.context.introduction).toBe(introduction); + legacy.generated.definitions = { + enabled: true, + output: "legacy-definitions.md", + introduction, + }; + expect( + collectArtifacts(rules, legacy).find( + (artifact) => artifact.relativePath === "legacy-definitions.md", + )?.context.introduction, + ).toBe(introduction); }); test("builds configured FRD definition document mappings", async () => { @@ -2784,12 +1934,16 @@ describe("build-markdown", () => { " - Definitions", "---", "", - MACHINE_PICTO_SPAN, + expectedPictograph(config), "", "# Custom FedRAMP Definitions", ].join("\n"), ); - expect(contents).toContain("## Important Related Terms"); + expect(contents.includes("## Important Related Terms")).toBe( + Object.values((await loadRules()).FRD.data) + .flatMap((bucket) => Object.values(bucket)) + .some((entry) => Boolean(entry.tag?.trim())), + ); expect(contents).not.toContain("## General Terms"); expect(contents).not.toContain("Effective Date(s)"); } finally { @@ -2927,7 +2081,10 @@ describe("content quality", () => { continue; } - const contents = await readFile(path.join(contentPath, relativePath), "utf8"); + const contents = await readFile( + path.join(contentPath, relativePath), + "utf8", + ); const tags = zensicalTagsFromFrontmatter(contents); expect( tags.some((tag) => contentTypeTags.has(tag)), @@ -3091,6 +2248,7 @@ describe("manual content source drift", () => { const config = await loadToolConfig(); const srcPath = resolveToolPath(config.paths.src); const contentPath = resolveToolPath(config.paths.content); + await markdownBuild(); const drift = await findManualSrcContentDrift(srcPath, contentPath); expectWithFailureSummary( @@ -3116,9 +2274,7 @@ describe("build pipeline", () => { const config = await loadToolConfig(); const rules = await loadRules(config); const expectedArtifacts = collectArtifacts(rules, config); - const expectedGeneratedFiles = expectedArtifacts - .map((artifact) => artifact.relativePath) - .sort(); + const expectedGeneratedFiles = await expectedOutputPaths(); const srcPath = resolveToolPath(config.paths.src); const contentPath = resolveToolPath(config.paths.content); const htmlPath = resolveToolPath(config.paths.html); @@ -3139,41 +2295,22 @@ describe("build pipeline", () => { ); expect(manifest.files).toEqual(expectedGeneratedFiles); - const trigramHtml = await readFile(path.join(htmlPath, "trigrams/index.html"), "utf8"); - const tables = trigramHtml.match(/][\s\S]*?<\/table>/g) ?? []; - expect(tables).toHaveLength(1); - expect(tables[0]).toStartWith(""); - expect(trigramHtml).toContain("javascripts/tablesort.js"); - expect(trigramHtml).toContain('name="description"'); - const lookup = expectedArtifacts.find((artifact) => artifact.documentType === "TRIGRAMS")!; - const renderedLinks = [...tables[0]!.matchAll(/href="([^"]+)"/g)].map((match) => match[1]!); - expect(renderedLinks).toHaveLength(lookup.context.trigramRows.filter((row) => row.href).length); - expect(tables[0]).toContain(""); - for (const href of renderedLinks) { - const url = new URL(href, "https://example.test/trigrams/"); - const targetHtml = await readFile(path.join(htmlPath, url.pathname, "index.html"), "utf8"); - if (url.hash) expect(targetHtml, href).toContain(`id="${decodeURIComponent(url.hash.slice(1))}"`); - } - - - const contentFiles = await listRelativeFiles(contentPath); - for (const relativePath of contentFiles) { - await access(path.join(srcPath, relativePath)); + const { content } = await sources(); + const copies = await readTree(srcPath); + for (const [filename, source] of content) { + assertManualCopy(source, copies.get(filename), filename); + if (filename.endsWith(".md")) { + assertManualHtml( + source.toString(), + await readFile(markdownToHtmlPath(htmlPath, filename), "utf8"), + filename, + ); + } else { + const output = await readFile(path.join(htmlPath, filename)); + assertManualCopy(source, output, filename); + } } - const copiedIndexMarkdown = await readFile( - path.join(srcPath, "index.md"), - "utf8", - ); - const contentIndexMarkdown = await readFile( - path.join(contentPath, "index.md"), - "utf8", - ); - expect(copiedIndexMarkdown).toContain(''); - expect(normalizeManualMarkdownForComparison(copiedIndexMarkdown)).toBe( - normalizeManualMarkdownForComparison(contentIndexMarkdown), - ); - const zensicalConfig = await readFile( resolveToolPath(config.paths.zensicalConfig), "utf8", @@ -3208,156 +2345,19 @@ describe("build pipeline", () => { expect(generatedMarkdown).not.toContain("[object Object]"); } - - - for (const relativePath of [ - "index.html", - "search.json", - "sitemap.xml", - "assets/fr-only-logo-black.png", - "stylesheets/extra.css", - "authority/m-24-15/m-24-15-official.png", - ]) { - await access(path.join(htmlPath, relativePath)); - } - - const renderedDefinitionArtifact = artifactOfType(expectedArtifacts, "FRD"); - const renderedKsiThemeArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.documentType === "KSI" && artifact.context.indicators.length > 0, - "a rendered KSI theme artifact", - ); - const renderedKsiThemeIndicator = - renderedKsiThemeArtifact.context.indicators[0]; - const renderedKsiReferenceArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.documentType === "KSI" && artifact.context.sections.length > 0, - "a rendered KSI reference artifact", - ); - const renderedDeadlineArtifact = artifactOfType(expectedArtifacts, "DEADLINES"); - const renderedResponsibilitiesArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - (config.generated.frrCollectionDocuments ?? []).some( - (mapping) => mapping.id === artifact.mappingId, - ), - "a rendered FRR collection artifact", - ); - const renderedAgencyArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.documentType === "FRR" && - artifact.relativePath.startsWith("agencies/rules/"), - "a rendered agency rules artifact", - ); - const renderedApplicabilityArtifact = firstArtifactMatching( - expectedArtifacts, - (artifact) => - artifact.documentType === "FRR" && - artifact.context.sections.some( - (section) => section.applicabilityGroups.length > 0, - ), - "a rendered FRR artifact with subset applicability", - ); - if (!renderedKsiThemeIndicator) { - throw new Error( - `Expected ${renderedKsiThemeArtifact.relativePath} to include an indicator.`, + for (const artifact of expectedArtifacts) { + await assertArtifactSource( + artifact, + await readFile(artifact.outputPath, "utf8"), ); - } - const renderedDefinitionTerm = - Object.values(rules.FRD.data.all ?? {}) - .map((entry) => entry.term) - .sort((left, right) => left.localeCompare(right))[0] ?? - rules.FRD.info.name; - const renderedKsiThemeHtmlPath = path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedKsiThemeArtifact.relativePath), - ); - const renderedKsiReferenceHtmlPath = path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedKsiReferenceArtifact.relativePath), - ); - const renderedDeadlineHtmlPath = path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedDeadlineArtifact.relativePath), - ); - const renderedResponsibilitiesHtmlPath = path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedResponsibilitiesArtifact.relativePath), - ); - const renderedAgencyHtmlPath = path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedAgencyArtifact.relativePath), - ); - const renderedApplicabilityHtmlPath = path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedApplicabilityArtifact.relativePath), - ); - - const renderedPages = [ - { - path: path.relative( - htmlPath, - markdownToHtmlPath(htmlPath, renderedDefinitionArtifact.relativePath), + await assertArtifactSource( + artifact, + await readFile( + markdownToHtmlPath(htmlPath, artifact.relativePath), + "utf8", ), - expectedText: [renderedDefinitionArtifact.title, renderedDefinitionTerm], - }, - { - path: renderedKsiThemeHtmlPath, - expectedText: [ - renderedKsiThemeArtifact.title, - renderedKsiThemeIndicator.id, - ], - }, - { - path: renderedKsiReferenceHtmlPath, - expectedText: [ - renderedKsiReferenceArtifact.title, - renderedKsiReferenceArtifact.context.sections[0]?.title ?? "", - renderedKsiReferenceArtifact.context.sections[0]?.requirements[0]?.id ?? - "", - ], - }, - { - path: renderedDeadlineHtmlPath, - expectedText: [renderedDeadlineArtifact.title], - }, - { - path: renderedResponsibilitiesHtmlPath, - expectedText: [renderedResponsibilitiesArtifact.title], - }, - { - path: renderedAgencyHtmlPath, - expectedText: [renderedAgencyArtifact.title], - }, - ]; - - for (const page of renderedPages) { - const contents = await readFile(path.join(htmlPath, page.path), "utf8"); - - for (const expectedText of page.expectedText) { - expect(contents).toContain(expectedText); - } - expect(contents).not.toContain("{{"); - expect(contents).not.toContain("[object Object]"); - } - - const renderedApplicabilityHtml = await readFile( - path.join(htmlPath, renderedApplicabilityHtmlPath), - "utf8", - ); - const firstApplicabilityValue = renderedApplicabilityArtifact.context.sections - .flatMap((section) => section.applicabilityGroups) - .flatMap((group) => group.values)[0]; - expect(renderedApplicabilityHtml).toContain( - '
', - ); - if (firstApplicabilityValue) { - expect(renderedApplicabilityHtml).toContain( - `${firstApplicabilityValue}`, + true, ); } - }, 20_000); + }, 120_000); }); diff --git a/tools/scripts/build.ts b/tools/scripts/build.ts index 84d074c..94b3fff 100644 --- a/tools/scripts/build.ts +++ b/tools/scripts/build.ts @@ -49,6 +49,8 @@ async function main(): Promise { await rm(path.join(REPO_ROOT, ".cache"), { recursive: true, force: true }); await runCommand(zensicalBinary(), [ "build", + "--clean", + "--strict", "-f", resolveToolPath(config.paths.zensicalConfig), ]); diff --git a/tools/scripts/source-contract.test.ts b/tools/scripts/source-contract.test.ts new file mode 100644 index 0000000..d38d0a7 --- /dev/null +++ b/tools/scripts/source-contract.test.ts @@ -0,0 +1,244 @@ +import { expect, test } from "bun:test"; +import { + assertManualCopy, + assertRecords, + assertText, + markdownText, + selectedRules, + type RuleEntry, +} from "./test-support/source-contract"; +import { loadRules, type Rules } from "./test-support/sources"; + +const record = (id: string, text: string) => + `### Example\n\n!!! abstract "${id}"\n\n!!! quote ""\n ${text}\n`; +const fixture = (): Rules => ({ + FRD: { info: { name: "Definitions", web_name: "definitions" }, data: {} }, + FRR: { + SYN: { + info: { + name: "Synthetic", + web_name: "synthetic", + subsets: { + ONE: { + name: "Examples", + applicability: { types: ["20x"], classes: ["B"] }, + }, + }, + }, + data: { + all: { + ONE: { + "SYN-ONE-AAA": { + statement: "Original exact text.", + affects: ["Providers"], + }, + }, + }, + }, + }, + }, + KSI: {}, + CTL: {}, +}); + +test("source expectations follow changed, added, and removed records", () => { + const rules = fixture(); + const source = { types: ["20x"], affects: ["Providers"] }; + const entries = rules.FRR.SYN!.data.all!.ONE!; + assertRecords( + record("SYN-ONE-AAA", entries["SYN-ONE-AAA"]!.statement!), + selectedRules(rules, source), + "original", + ); + entries["SYN-ONE-AAA"]!.statement = + "Changed exact text, including punctuation!"; + expect(() => + assertRecords( + record("SYN-ONE-AAA", "Original exact text."), + selectedRules(rules, source), + "stale", + ), + ).toThrow(); + entries["SYN-ONE-BBB"] = { statement: "New text.", affects: ["Providers"] }; + const current = Object.entries(entries) + .map(([id, entry]) => record(id, entry.statement!)) + .join("\n"); + assertRecords(current, selectedRules(rules, source), "updated"); + delete entries["SYN-ONE-AAA"]; + expect(() => + assertRecords(current, selectedRules(rules, source), "removed"), + ).toThrow(); + assertRecords( + record("SYN-ONE-BBB", "New text."), + selectedRules(rules, source), + "removed and rebuilt", + ); +}); + +test("record verification rejects missing, altered, duplicate and unexpected output", () => { + const expected: Array<[string, RuleEntry]> = [ + ["SYN-ONE-AAA", { statement: "Do this, exactly." }], + ]; + const correct = record("SYN-ONE-AAA", "Do this, exactly."); + for (const broken of [ + "", + correct.replace("exactly", "differently"), + correct + correct, + correct + record("SYN-ONE-BBB", "Extra"), + ]) { + expect(() => assertRecords(broken, expected, "corrupted output")).toThrow(); + } +}); + +test("selection follows current class, audience, bucket and subset applicability", () => { + const rules = fixture(); + expect(selectedRules(rules, { types: ["rev5"] })).toEqual([]); + expect(selectedRules(rules, { types: ["20x"], classes: ["A"] })).toEqual([]); + expect( + selectedRules(rules, { types: ["20x"], affects: ["Assessors"] }), + ).toEqual([]); + expect(selectedRules(rules, { types: ["20x"], includeAll: false })).toEqual( + [], + ); + rules.FRR.SYN!.info.subsets!.ONE!.applicability!.classes = ["A"]; + expect(selectedRules(rules, { types: ["20x"], classes: ["A"] })).toHaveLength( + 1, + ); +}); + +test("render normalization preserves wording, punctuation, entities and inline boundaries", () => { + assertText( + '

Use this & that.

', + "Use **this** & that.", + "formatting", + true, + ); + assertText( + "

prefix, then act!

", + "prefix, then act!", + "inline", + true, + ); + expect(markdownText("1. First item\n2. Second item")).toContain("First item"); + for (const changed of [ + "Use this & that!", + "Use that & this.", + "Use this and that.", + ]) + expect(() => + assertText("

Use this & that.

", changed, "changed", true), + ).toThrow(); +}); + +test("manual copy verification detects missing pages and changed assets", () => { + const source = Buffer.from("# A source page\n\nExact text.\n"); + assertManualCopy( + source, + Buffer.from('generated\n\n' + source), + "page.md", + ); + expect(() => assertManualCopy(source, undefined, "page.md")).toThrow(); + expect(() => + assertManualCopy( + source, + Buffer.from("# A source page\n\nWrong text."), + "page.md", + ), + ).toThrow(); + expect(() => + assertManualCopy( + Buffer.from([0, 1, 2]), + Buffer.from([0, 1, 3]), + "asset.bin", + ), + ).toThrow(); +}); + +test("source fixture clones cannot mutate the cached rules", async () => { + const first = await loadRules(); + const original = first.FRD.info.name; + first.FRD.info.name = "Fixture-only name"; + expect((await loadRules()).FRD.info.name).toBe(original); +}); + +test("real-source clones rebuild after definition, date and baseline changes without editing assertions", async () => { + const { mkdtemp, mkdir, writeFile, readFile, rm } = + await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const path = await import("node:path"); + const { sources } = await import("./test-support/sources"); + const { buildMarkdown } = await import("./build-markdown"); + const { assertArtifactSource } = + await import("./test-support/source-contract"); + const snapshot = structuredClone(await sources()); + const temp = await mkdtemp(path.join(tmpdir(), "source-evolution-")); + try { + Object.assign(snapshot.config.paths, { + src: path.join(temp, "src"), + content: path.join(temp, "content"), + rulesFile: path.join(temp, "rules.json"), + }); + await mkdir(snapshot.config.paths.content); + const definitions = snapshot.rules.FRD.data.all!; + definitions["FRD-SYNTHETIC"] = { + term: "Synthetic original term", + definition: "Original fixture wording.", + }; + async function buildAndVerify() { + await writeFile( + snapshot.config.paths.rulesFile, + JSON.stringify(snapshot.rules), + ); + const summary = await buildMarkdown(snapshot.config); + for (const artifact of summary.artifacts.filter((a) => + ["FRD", "DEADLINES", "CTL_REFERENCE"].includes(a.documentType), + )) { + await assertArtifactSource( + artifact, + await readFile(artifact.outputPath, "utf8"), + false, + snapshot, + ); + } + return summary; + } + await buildAndVerify(); + definitions["FRD-SYNTHETIC"] = { + term: "Synthetic renamed term", + definition: "Changed fixture wording, precisely!", + }; + for (const doc of Object.values(snapshot.rules.FRR)) { + for (const effective of [ + doc.info.effective, + doc.info["20x"]?.effective, + doc.info.rev5?.effective, + ]) + if (effective?.date) effective.date.optional_adoption = "2099-01-02"; + } + const baseline = + snapshot.rules.FRR.FRC!.data.rev5!.CSF!["FRC-CSF-BSL"]!.varies_by_class!; + const variant = Object.values(baseline).find((v) => + Object.values(v.rev5_controls_list ?? {}).some((ids) => ids.length), + )!; + const list = Object.values(variant.rev5_controls_list!).find( + (ids) => ids.length, + )!; + list.pop(); + await buildAndVerify(); + delete definitions["FRD-SYNTHETIC"]; + await buildAndVerify(); + } finally { + await rm(temp, { recursive: true, force: true }); + } +}, 30_000); + +test("manual diagram verification preserves braces and catches changed diagram text", async () => { + const { assertManualHtml } = await import("./test-support/source-contract"); + const code = 'flowchart LR\n node@{ img: "/asset.jpg", h: 72 }\n'; + const source = `\`\`\`mermaid\n${code}\`\`\`\n`; + const html = `
${code}
`; + assertManualHtml(source, html, "diagram.md"); + expect(() => + assertManualHtml(source, html.replace("72", "73"), "diagram.md"), + ).toThrow(); +}); diff --git a/tools/scripts/test-support/source-contract.ts b/tools/scripts/test-support/source-contract.ts new file mode 100644 index 0000000..f14960b --- /dev/null +++ b/tools/scripts/test-support/source-contract.ts @@ -0,0 +1,1064 @@ +import { expect } from "bun:test"; +import path from "node:path"; +import type { BuildArtifact } from "../build-markdown"; +import type { RuleDocumentSourceConfig } from "../config"; +import { normalizeOscalControlId } from "../oscal-catalog"; +import { sources, type Rules } from "./sources"; + +export const slug = (text: string) => + text + .toLowerCase() + .replace(/\s+/g, "-") + .replace(/[^a-z0-9-]/g, ""); +export function htmlText(html: string): string { + // Add boundaries only at block elements; inline markup must not split words. + const blocks = html + .replace(/]*>[\s\S]*?<\/sup>/g, "") + .replace( + /<\/?(?:p|div|li|h[1-6]|tr|td|th|br|pre|section|summary)\b[^>]*>/gi, + " ", + ); + let text = ""; + new HTMLRewriter() + .on("*", { + text(chunk) { + text += chunk.text; + }, + }) + .transform(`
${blocks}
`); + return Bun.markdown.render(text).replace(/\s+/g, " ").trim(); +} +export function markdownText(markdown: string): string { + return htmlText( + Bun.markdown.render( + markdown + .replace(/:(?:material|lucide|octicons|fontawesome)-[a-z0-9-]+:/g, "") + .replace(/\[\^[^\]]+\]/g, "") + .replace(/\{[^{}\n]*(?:data-preview|#[a-z]|\.[a-z])[^{}\n]*\}/g, "") + .replace(/^ +/gm, ""), + ), + ); +} +export function mainHtml(html: string): string { + const body = html.match(/]*>([\s\S]*?)<\/article>/i)?.[1]; + if (body === undefined) throw new Error("Missing rendered article"); + return body.replace(/]*class="headerlink"[^>]*>[\s\S]*?<\/a>/g, ""); +} +export function assertText( + actual: string, + expected: string | undefined, + where: string, + html = false, +) { + if (!expected?.trim()) return; + expect(html ? htmlText(actual) : markdownText(actual), where).toContain( + markdownText(expected), + ); +} +export function buckets(source: { + types?: string[]; + includeAll?: boolean; + allPosition?: string; +}): string[] { + const types = + source.types?.includes("all") || !source.types + ? ["20x", "rev5"] + : source.types; + return source.includeAll === false + ? types + : source.allPosition === "first" + ? ["all", ...types] + : [...types, "all"]; +} +export function documentKeys( + rules: Rules, + source: { + documents?: string[] | "ALL"; + document?: string; + ignoreDocuments?: string[]; + }, +): string[] { + const selected = + source.documents === "ALL" + ? Object.keys(rules.FRR) + : (source.documents ?? + (source.document ? [source.document] : Object.keys(rules.FRR))); + return selected.filter((key) => !source.ignoreDocuments?.includes(key)); +} +export type RuleEntry = NonNullable< + Rules["FRR"][string]["data"]["all"] +>[string][string]; +export function selectedRules( + rules: Rules, + source: Omit & { + types?: string[]; + }, + key?: string, +): Array<[string, RuleEntry]> { + const result: Array<[string, RuleEntry]> = []; + const classes = source.classes?.map((x) => x.toLowerCase()); + for (const documentKey of key ? [key] : documentKeys(rules, source)) { + const doc = rules.FRR[documentKey]!; + const types = buckets(source).filter((x) => x !== "all"); + const subsets = Object.assign( + {}, + doc.info.subsets, + ...types.map((t) => doc.info[t as "20x" | "rev5"]?.subsets), + ); + for (const bucket of buckets(source)) { + for (const [section, entries] of Object.entries( + doc.data[bucket as "all"] ?? {}, + )) { + if (source.sections && !source.sections.includes(section)) continue; + const applicability = subsets[section]?.applicability; + if ( + applicability?.types?.length && + !applicability.types.some((t: string) => + types.includes(t.toLowerCase()), + ) + ) + continue; + if ( + classes?.length && + applicability?.classes?.length && + !applicability.classes.some((c: string) => + classes.includes(c.toLowerCase()), + ) + ) + continue; + if ( + source.affects?.length && + applicability?.affects?.length && + !applicability.affects.some((a: string) => + source.affects!.some((b) => a.toLowerCase() === b.toLowerCase()), + ) + ) + continue; + for (const [id, entry] of Object.entries(entries)) { + if ( + source.affects?.length && + !entry.affects?.some((a) => + source.affects!.some((b) => a.toLowerCase() === b.toLowerCase()), + ) + ) + continue; + if ( + classes?.length && + entry.varies_by_class && + !Object.keys(entry.varies_by_class).some((c) => + classes.includes(c.toLowerCase()), + ) + ) + continue; + result.push([id, entry]); + } + } + } + } + return result; +} +export function recordBlocks(text: string, html = false): Map { + const chunks = html + ? text.split(/(?=(); + for (const chunk of chunks) { + const id = html + ? (chunk.match( + /class="admonition-title"[^>]*>\s*((?:FRD|KSI|[A-Z]{3})-[A-Z0-9-]+)\s*]*>\s*((?:FRD|KSI|[A-Z]{3})-[A-Z0-9-]+)\s* n.name.replace(/[.!?]+$/, "")), + ...Object.values(entry.rev5_controls_list ?? {}).flat(), + ...(entry.notes ?? []), + ...(entry.following_information ?? []), + ...(entry.following_information_bullets ?? []), + ...(entry.corrective_actions ?? []), + ]; + for (const [key, variant] of Object.entries(entry.varies_by_class ?? {})) { + if ( + !classes?.length || + classes.some((c) => c.toLowerCase() === key.toLowerCase()) + ) + values.push(...entryText(variant)); + } + for (const variant of Object.values(entry.varies_by_level ?? {})) + values.push(...entryText(variant)); + for (const example of entry.examples ?? []) + values.push(...(example.key_tests ?? []), ...(example.examples ?? [])); + for (const change of entry.updated ?? []) + values.push(change.comment, change.date); + return values.filter((x): x is string => Boolean(x)); +} +export function assertStatement( + block: string, + source: string | undefined, + where: string, + html = false, +) { + if (!source?.trim()) return; + const expected = source + .split(/\n\s*\n/) + .map(markdownText) + .filter(Boolean); + const quote = html + ? block.split(/
/)[1] + : block.split('!!! quote ""')[1]; + expect(quote, `${where}: missing statement quote`).toBeDefined(); + const actual = html + ? [ + ...quote!.matchAll( + /]*class="admonition-title")[^>]*>([\s\S]*?)<\/p>/g, + ), + ] + .slice(0, expected.length) + .map((m) => htmlText(m[1]!)) + : quote! + .trim() + .split(/\n\s*\n/) + .slice(0, expected.length) + .map(markdownText); + expect(actual, `${where}: exact statement paragraphs`).toEqual(expected); +} +export function assertRecords( + text: string, + entries: Array<[string, RuleEntry]>, + where: string, + html = false, + classes?: string[], +) { + const blocks = recordBlocks(text, html); + expect( + [...blocks.keys()].sort(), + `${where}: exact record membership`, + ).toEqual(entries.map(([id]) => id).sort()); + for (const [id, entry] of entries) { + assertStatement(blocks.get(id)!, entry.statement, `${where}: ${id}`, html); + for (const value of entryText(entry, classes)) + assertText(blocks.get(id)!, value, `${where}: ${id}: ${value}`, html); + } +} +export async function assertArtifactSource( + artifact: BuildArtifact, + text: string, + html = false, + snapshot?: Awaited>, +) { + const { rules, config, catalog, navigation } = snapshot ?? (await sources()); + const where = artifact.relativePath; + if (html) text = mainHtml(text); + else { + const policy = config.generated.search; + if ( + policy && + ["FRR", "KSI", "CTL", "CTL_REFERENCE"].includes(artifact.documentType) + ) { + const included = new Set(); + const project = navigation.project as { nav: unknown[] }; + const pages = (node: unknown): void => { + if (typeof node === "string") included.add(node); + else if (Array.isArray(node)) node.forEach(pages); + else if (node && typeof node === "object") + Object.values(node).forEach(pages); + }; + for (const section of policy.includedNavigationSections) { + let node: unknown = project.nav; + for (const label of section) { + const entries: unknown[] = Array.isArray(node) ? node : [node]; + const matches = entries.filter( + (entry): entry is Record => + Boolean(entry && typeof entry === "object" && label in entry), + ); + expect( + matches.length, + `navigation section ${section.join(" / ")}`, + ).toBe(1); + node = matches[0]![label]; + } + pages(node); + } + const excluded = !included.has(where); + expect( + /search:\s*\n exclude: true/.test(text.split("---")[1] ?? ""), + `${where}: search policy`, + ).toBe(excluded); + } + } + await assertTables(artifact, text, html, snapshot); + if (artifact.documentType === "FRR") { + const mapping = [ + ...config.generated.ruleDocuments, + ...(config.generated.frrCollectionDocuments ?? []), + ].find((m) => m.id === artifact.mappingId)!; + if ( + "relatedRulesOutput" in mapping && + mapping.relatedRulesOutput === artifact.relativePath + ) { + const selected = selectedRules(rules, mapping.source); + const all = selectedRules(rules, { types: ["all"] }); + const existing = new Set(selected.map(([id]) => id)); + const related = new Set( + selected + .flatMap(([, e]) => e.related ?? []) + .filter((id) => !existing.has(id)), + ); + assertRecords( + text, + all.filter(([id]) => related.has(id)), + where, + html, + mapping.source.classes, + ); + } else { + const entries = selectedRules( + rules, + mapping.source, + artifact.sourceDocument, + ); + assertRecords(text, entries, where, html, mapping.source.classes); + if (artifact.sourceDocument) { + const doc = rules.FRR[artifact.sourceDocument]!; + assertText(text, doc.info.name, `${where}: ruleset title`, html); + assertText(text, doc.info.purpose, `${where}: ruleset purpose`, html); + } + } + } else if (artifact.documentType === "FRD") { + const mapping = config.generated.definitionDocuments?.find( + (m) => m.id === artifact.mappingId, + )!; + const entries = buckets(mapping.source).flatMap((bucket) => + Object.entries(rules.FRD.data[bucket as "all"] ?? {}), + ); + const blocks = recordBlocks(text, html); + expect([...blocks.keys()].sort(), where).toEqual( + entries.map(([id]) => id).sort(), + ); + for (const [id, entry] of entries) + for (const value of [ + entry.term, + entry.definition, + entry.note, + ...(entry.notes ?? []), + ...(entry.alts ?? []), + ]) + assertText(blocks.get(id)!, value, `${where}: ${id}`, html); + expect([...blocks.keys()], `${where}: alphabetical definitions`).toEqual( + [...entries] + .sort(([, a], [, b]) => a.term.localeCompare(b.term)) + .map(([id]) => id), + ); + if (!html) { + const groups = new Map(); + for (const [, entry] of entries) + if (entry.tag?.trim()) + groups.set(entry.tag.trim(), [ + ...(groups.get(entry.tag.trim()) ?? []), + entry.term, + ]); + for (const [tag, terms] of groups) { + const anchor = `related-terms-group-${slug(tag)}`; + expect(text, `${where}: related terms ${tag}`).toContain( + `${tag}`, + ); + for (const term of terms) + expect(text, `${where}: related term ${term}`).toContain( + `[${term}](#${slug(term)}){ data-preview }`, + ); + } + } + for (const [id, entry] of entries) + assertStatement( + blocks.get(id)!, + entry.definition, + `${where}: ${id}`, + html, + ); + assertText(text, rules.FRD.info.purpose, where, html); + } else if (artifact.documentType === "KSI") { + const mapping = config.generated.ksiDocuments?.find( + (m) => m.id === artifact.mappingId, + )!; + const keys = artifact.sourceDocument + ? [artifact.sourceDocument] + : mapping.source.themes === "ALL" + ? Object.keys(rules.KSI) + : (mapping.source.themes ?? [mapping.source.theme!]); + const relatedMapping = config.generated.ruleDocuments.find( + (m) => m.id === mapping.relatedIndicatorsFromRuleDocumentMappingId, + ); + const related = relatedMapping + ? new Set( + selectedRules(rules, relatedMapping.source).flatMap( + ([, entry]) => entry.related ?? [], + ), + ) + : undefined; + const entries = keys + .flatMap((key) => Object.entries(rules.KSI[key]!.indicators)) + .filter( + ([id, entry]) => + (!related || related.has(id)) && + (!mapping.source.classes?.length || + !entry.varies_by_class || + Object.keys(entry.varies_by_class).some((c) => + mapping.source.classes!.some( + (v) => v.toLowerCase() === c.toLowerCase(), + ), + )), + ); + assertRecords(text, entries, where, html, mapping.source.classes); + for (const key of keys) + if ( + entries.some(([id]) => Object.hasOwn(rules.KSI[key]!.indicators, id)) + ) { + assertText(text, rules.KSI[key]!.name, `${where}: theme name`, html); + assertText( + text, + rules.KSI[key]!.theme, + `${where}: theme description`, + html, + ); + } + } else if ( + artifact.documentType === "CTL" || + artifact.documentType === "CTL_REFERENCE" + ) { + const full = artifact.documentType === "CTL_REFERENCE"; + const mapping = [ + ...(config.generated.controlDocuments ?? []), + ...(config.generated.fullControlReferenceDocuments ?? []), + ].find((m) => m.id === artifact.mappingId)!; + const familyKeys = artifact.sourceDocument + ? [artifact.sourceDocument] + : mapping.source.families === "ALL" + ? [...catalog.families.keys()] + : mapping.source.families; + const isIndex = full && !artifact.sourceDocument; + const baseline = + rules.FRR.FRC?.data.rev5?.CSF?.["FRC-CSF-BSL"]?.varies_by_class ?? {}; + const controlSections = new Map(); + if (!isIndex) + for (const chunk of text.split(html ? /(?=]*id="([^"]+)"/)?.[1] + : chunk.match(/^## .*\{ #([^ }]+)/)?.[1] + : html + ? chunk.match(/]*>\s*([A-Z]{2}-[0-9() -]+)\s* + full ? c.status !== "withdrawn" : Boolean(rules.CTL[key]?.[c.id]), + ); + if (!controls.length) continue; + assertText(text, family.title, where, html); + if (isIndex) { + const actual = artifact.context.controlFamilies.find( + (f) => f.id === key, + )!; + expect(actual.controlCount, `${where}: ${key}`).toBe(controls.length); + continue; + } + for (const control of controls) { + expectedControlIds.push(control.id); + const section = controlSections.get(control.id); + expect( + section, + `${where}: missing control ${control.id}`, + ).toBeDefined(); + assertText(section!, control.title, `${where}: ${control.id}`, html); + for (const line of control.statementLines) + assertText( + section!, + line, + `${where}: ${control.id}: statement`, + html, + ); + const source = rules.CTL[key]?.[control.id]; + for (const data of [ + source, + ...Object.values(source?.varies_by_class ?? {}), + ]) { + for (const line of data?.guidance ?? []) + assertText( + section!, + line, + `${where}: ${control.id}: guidance`, + html, + ); + for (const param of data?.parameters ?? []) + assertText( + section!, + param.value, + `${where}: ${control.id}: ${param.parameterId}`, + html, + ); + } + if (full) { + const expected = Object.entries(baseline) + .filter(([, v]) => + Object.values(v.rev5_controls_list ?? {}) + .flat() + .some((id) => normalizeOscalControlId(id) === control.id), + ) + .map(([c]) => `Class ${c.toUpperCase()}`); + const actual = artifact.context.controlFamilies + .flatMap((f) => f.controls) + .find((c) => c.id === control.id)!; + expect( + actual.baselineClasses, + `${where}: ${control.id}: baseline`, + ).toEqual(expected); + const renderedClasses = [ + ...section!.matchAll(/class="subset-applicability__tag">([^<]+) m[1]!); + expect( + renderedClasses, + `${where}: ${control.id}: rendered baseline`, + ).toEqual(expected); + } + } + } + if (!isIndex) + expect( + [...controlSections.keys()].sort(), + `${where}: exact control membership`, + ).toEqual(expectedControlIds.sort()); + } +} +export function normalizeManual(text: string): string { + return text + .replace(/\r\n?/g, "\n") + .replace(/^.+<\/span>\n*/gm, "") + .replace(/^(---\n[\s\S]*?\n---)\n*/, "$1\n\n") + .trimEnd(); +} +export function assertManualCopy( + source: Buffer, + output: Buffer | undefined, + filename: string, +) { + expect(output, `${filename}: missing copied source`).toBeDefined(); + if (filename.endsWith(".md")) + expect(normalizeManual(output!.toString()), filename).toBe( + normalizeManual(source.toString()), + ); + else + expect(output!.equals(source), `${filename}: changed asset bytes`).toBe( + true, + ); +} + +export function expectedTrigrams( + rules: Rules, + config: Awaited>["config"], + mapping = config.generated.trigramDocuments![0]!, +) { + const relative = (target: string) => + path.posix.relative(path.posix.dirname(mapping.output), target); + const definitions = config.generated.definitionDocuments!.find( + (m) => m.id === mapping.definitionDocumentMappingId, + )!; + const ruleMapping = config.generated.ruleDocuments.find( + (m) => m.id === mapping.ruleDocumentMappingId, + )!; + const ksiMapping = config.generated.ksiDocuments!.find( + (m) => m.id === mapping.ksiDocumentMappingId, + )!; + const rulePath = (key: string) => + ruleMapping.output.replace( + /\{FRR\}|\{document\}/g, + rules.FRR[key]!.info.web_name.toLowerCase(), + ); + const rows: Array<{ + trigram: string; + name: string; + type: "Collection" | "Ruleset" | "Ruleset subset" | "KSI theme"; + href?: string; + }> = [ + { + trigram: rules.FRD.info.short_name!, + name: rules.FRD.info.name, + type: "Collection", + href: relative(definitions.output), + }, + ]; + const mappings = Object.values(config.generated) + .filter(Array.isArray) + .flat() as Array<{ id: string; output: string }>; + for (const collection of mapping.collections) + rows.push({ + trigram: collection.trigram, + name: collection.name, + type: "Collection", + href: relative( + mappings.find((m) => m.id === collection.mappingId)!.output, + ), + }); + const subsets = new Map>(); + for (const [key, doc] of Object.entries(rules.FRR).sort(([a], [b]) => + a.localeCompare(b), + )) { + rows.push({ + trigram: doc.info.short_name ?? key, + name: doc.info.name, + type: "Ruleset", + href: relative(rulePath(key)), + }); + for (const [id, subset] of Object.entries(doc.info.subsets ?? {})) + subsets.set(id, [ + ...(subsets.get(id) ?? []), + { key, name: subset.name ?? id }, + ]); + } + for (const [id, occurrences] of subsets) + rows.push({ + trigram: id, + name: occurrences[0]!.name, + type: "Ruleset subset", + href: + occurrences.length === 1 + ? `${relative(rulePath(occurrences[0]!.key))}#${slug(occurrences[0]!.name)}` + : undefined, + }); + for (const [key, theme] of Object.entries(rules.KSI)) + rows.push({ + trigram: theme.short_name ?? key, + name: theme.name, + type: "KSI theme", + href: `${relative(ksiMapping.output)}#${slug(theme.name)}`, + }); + return rows.sort((a, b) => a.trigram.localeCompare(b.trigram, "en")); +} + +export function tableRows(text: string, html = false): string[][] { + if (html) + return [...text.matchAll(/]*>([\s\S]*?)<\/tr>/g)] + .filter((m) => m[1]!.includes(" + [...m[1]!.matchAll(/]*>([\s\S]*?)<\/td>/g)].map((c) => + htmlText(c[1]!), + ), + ); + return text + .split("\n") + .filter((line) => /^\|/.test(line) && !/^\|\s*:?-/.test(line)) + .map((line) => + line + .slice(1, line.lastIndexOf("|")) + .split(/(? !["Ruleset", "Acronym", "Trigram"].includes(cells[0]!)); +} +export async function assertTables( + artifact: BuildArtifact, + text: string, + html: boolean, + snapshot?: Awaited>, +) { + const { config, rules } = snapshot ?? (await sources()); + const rows = tableRows(text, html); + const where = artifact.relativePath; + if (artifact.documentType === "TRIGRAMS") { + const mapping = config.generated.trigramDocuments!.find( + (m) => m.id === artifact.mappingId, + )!; + const expected = expectedTrigrams(rules, config, mapping); + expect(rows, where).toEqual( + expected.map((row) => [row.trigram, markdownText(row.name), row.type]), + ); + for (const row of expected.filter((r) => r.href)) { + const href = html ? renderedHref(where, row.href!) : row.href!; + expect(text, `${where}: ${row.trigram} destination`).toContain( + html ? `href="${href}"` : `](${href})`, + ); + } + } + if (artifact.documentType === "DEADLINES") { + const mapping = config.generated.deadlineDocuments!.find( + (m) => m.id === artifact.mappingId, + )!; + const types = buckets({ types: mapping.source.types, includeAll: false }); + const type = + types.find( + (t) => where === mapping.output.replace(/\{type\}|\{version\}/g, t), + ) ?? types[0]!; + const expected = documentKeys(rules, mapping.source) + .flatMap((key) => { + const doc = rules.FRR[key]!; + if ( + mapping.source.affects?.length && + !selectedRules( + rules, + { types: mapping.source.types, affects: mapping.source.affects }, + key, + ).length + ) + return []; + const effective = + doc.info.effective ?? doc.info[type as "20x" | "rev5"]?.effective; + if (!effective) return []; + const date = effective.date; + const label = doc.info.short_name + ? `${doc.info.name} (${doc.info.short_name})` + : doc.info.name; + const grace = + date?.grace?.default === undefined + ? "" + : date.grace.until_next_assessment + ? `On the first FedRAMP independent assessment started after ${date.grace.default}` + : String(date.grace.default); + return [ + [ + label, + String(date?.optional_adoption ?? ""), + String(date?.obtain ?? ""), + String(date?.maintain ?? ""), + grace, + ], + ]; + }) + .sort((a, b) => + !a[3] ? (!b[3] ? 0 : 1) : !b[3] ? -1 : a[3].localeCompare(b[3]), + ); + expect(rows, where).toEqual(expected.map((row) => row.map(markdownText))); + } + if (artifact.documentType === "FRR_TAGGED_SUMMARY") { + const mapping = config.generated.taggedDocumentSummaries!.find( + (m) => m.id === artifact.mappingId, + )!; + const tags = mapping.source.tags ?? [mapping.source.tag!]; + const expected = documentKeys(rules, mapping.source) + .filter((key) => tags.includes(rules.FRR[key]!.info.tag!)) + .map((key) => ({ + doc: rules.FRR[key]!, + entries: selectedRules(rules, mapping.source, key), + })) + .filter((item) => item.entries.length); + expect(rows.length, where).toBe(expected.length); + for (const { doc, entries } of expected) { + const row = rows.find((row) => row[0]?.includes(doc.info.name)); + expect(row, `${where}: ${doc.info.name}`).toBeDefined(); + expect(row![1], where).toContain(markdownText(doc.info.purpose ?? "")); + expect(row![1], where).toContain(`Applicable Rules: ${entries.length}`); + } + assertText( + text, + `There are ${expected.length} applicable rulesets with ${expected.reduce((n, item) => n + item.entries.length, 0)} total applicable rules.`, + where, + html, + ); + } + if (artifact.documentType === "FRR_REFERENCE_INDEX") { + const mapping = config.generated.referenceIndexDocuments!.find( + (m) => m.id === artifact.mappingId, + )!; + const expected = documentKeys(rules, mapping.source) + .map((key) => ({ + doc: rules.FRR[key]!, + entries: selectedRules(rules, mapping.source, key), + })) + .filter((item) => item.entries.length); + expect(rows.length, where).toBe(expected.length); + for (const { doc, entries } of expected) { + const row = rows.find((row) => row[0] === doc.info.short_name); + expect(row, `${where}: ${doc.info.short_name}`).toBeDefined(); + expect(row![1], where).toContain(doc.info.name); + expect(row![2], where).toBe( + doc.info.status + ? doc.info.status[0]!.toUpperCase() + doc.info.status.slice(1) + : "Unknown", + ); + expect(row![3], where).toContain(`Rules: ${entries.length}`); + const dates = entries + .flatMap(([, e]) => (e.updated ?? []).map((c) => c.date ?? "")) + .sort(); + expect(row![4], where).toBe(dates.at(-1) ?? ""); + } + } +} + +export async function expectedOutputPaths() { + const { rules, config, catalog } = await sources(); + const paths: string[] = []; + const add = (output: string, count: number, behavior?: string) => { + if (count || behavior !== "skip") paths.push(output); + }; + for (const mapping of config.generated.definitionDocuments ?? []) + add( + mapping.output, + buckets(mapping.source).flatMap((b) => + Object.keys(rules.FRD.data[b as "all"] ?? {}), + ).length, + mapping.emptyBehavior, + ); + for (const mapping of config.generated.ruleDocuments) { + const entries = selectedRules(rules, mapping.source); + if (mapping.outputMode === "documents") { + for (const key of documentKeys(rules, mapping.source)) { + const name = rules.FRR[key]!.info.web_name.toLowerCase(); + const output = /\{FRR\}|\{document\}/.test(mapping.output) + ? mapping.output.replace(/\{FRR\}|\{document\}/g, name) + : `${mapping.output.replace(/\/$/, "")}/${name}.md`; + add( + output, + selectedRules(rules, mapping.source, key).length, + mapping.emptyBehavior, + ); + } + } else add(mapping.output, entries.length, mapping.emptyBehavior); + if (mapping.relatedRulesOutput) { + const included = new Set(entries.map(([id]) => id)); + const all = new Set( + selectedRules(rules, { types: ["all"] }).map(([id]) => id), + ); + const missing = entries + .flatMap(([, e]) => e.related ?? []) + .filter((id) => all.has(id) && !included.has(id)); + add(mapping.relatedRulesOutput, missing.length, "skip"); + } + } + for (const mapping of config.generated.frrCollectionDocuments ?? []) + add( + mapping.output, + selectedRules(rules, mapping.source).length, + mapping.emptyBehavior, + ); + for (const mapping of config.generated.ksiDocuments ?? []) { + const keys = + mapping.source.themes === "ALL" + ? Object.keys(rules.KSI) + : (mapping.source.themes ?? [mapping.source.theme!]); + const ruleMapping = config.generated.ruleDocuments.find( + (m) => m.id === mapping.relatedIndicatorsFromRuleDocumentMappingId, + ); + const related = ruleMapping + ? new Set( + selectedRules(rules, ruleMapping.source).flatMap( + ([, e]) => e.related ?? [], + ), + ) + : undefined; + const count = (key: string) => + Object.entries(rules.KSI[key]!.indicators).filter( + ([id, e]) => + (!related || related.has(id)) && + (!mapping.source.classes?.length || + !e.varies_by_class || + Object.keys(e.varies_by_class).some((c) => + mapping.source.classes!.some( + (x) => x.toLowerCase() === c.toLowerCase(), + ), + )), + ).length; + if (mapping.outputMode === "single") + add( + mapping.output, + keys.reduce((n, k) => n + count(k), 0), + related ? "skip" : mapping.emptyBehavior, + ); + else + for (const key of keys) { + const name = rules.KSI[key]!.web_name.toLowerCase(); + const output = /\{KSI\}|\{theme\}/.test(mapping.output) + ? mapping.output.replace(/\{KSI\}|\{theme\}/g, name) + : `${mapping.output.replace(/\/$/, "")}/${name}.md`; + add(output, count(key), related ? "skip" : mapping.emptyBehavior); + } + } + for (const mapping of config.generated.controlDocuments ?? []) { + const keys = + mapping.source.families === "ALL" + ? Object.keys(rules.CTL) + : mapping.source.families; + if (mapping.outputMode !== "families") + add( + mapping.output, + keys.reduce((n, k) => n + Object.keys(rules.CTL[k] ?? {}).length, 0), + mapping.emptyBehavior, + ); + else + for (const key of keys) { + const name = slug(catalog.families.get(key)!.title); + const output = /\{CTL\}|\{family\}/.test(mapping.output) + ? mapping.output.replace(/\{CTL\}|\{family\}/g, name) + : `${mapping.output.replace(/\/$/, "")}/${name}.md`; + add( + output, + Object.keys(rules.CTL[key] ?? {}).length, + mapping.emptyBehavior, + ); + } + } + for (const mapping of config.generated.fullControlReferenceDocuments ?? []) { + paths.push(path.posix.join(mapping.output, "index.md")); + for (const key of mapping.source.families === "ALL" + ? [...catalog.families.keys()] + : mapping.source.families) + paths.push( + path.posix.join( + mapping.output, + `${slug(catalog.families.get(key)!.title)}.md`, + ), + ); + } + for (const mapping of config.generated.deadlineDocuments ?? []) { + const types = buckets({ types: mapping.source.types, includeAll: false }); + for (const type of types) { + const count = documentKeys(rules, mapping.source).filter((key) => { + const doc = rules.FRR[key]!; + const audienceMatches = + !mapping.source.affects?.length || + buckets({ types: mapping.source.types }).some((b) => + Object.values(doc.data[b as "all"] ?? {}).some((section) => + Object.values(section).some((e) => + e.affects?.some((a) => mapping.source.affects!.includes(a)), + ), + ), + ); + return ( + audienceMatches && + (doc.info.effective || doc.info[type as "20x" | "rev5"]?.effective) + ); + }).length; + const output = /\{type\}|\{version\}/.test(mapping.output) + ? mapping.output.replace(/\{type\}|\{version\}/g, type) + : types.length === 1 + ? mapping.output + : `${mapping.output.replace(/\/$/, "")}/${type}.md`; + add(output, count, "skip"); + } + } + for (const mapping of config.generated.referenceIndexDocuments ?? []) + add( + mapping.output, + selectedRules(rules, mapping.source).length, + mapping.emptyBehavior, + ); + for (const mapping of config.generated.taggedDocumentSummaries ?? []) { + const tags = mapping.source.tags ?? [mapping.source.tag!]; + const count = documentKeys(rules, mapping.source).filter( + (k) => + tags.includes(rules.FRR[k]!.info.tag!) && + selectedRules(rules, mapping.source, k).length, + ).length; + add(mapping.output, count, mapping.emptyBehavior); + } + for (const mapping of config.generated.trigramDocuments ?? []) + paths.push(mapping.output); + return paths.sort(); +} + +export function renderedHref(from: string, href: string): string { + if (/^(?:[a-z]+:|\/|#)/i.test(href)) return href; + const [filename, hash] = href.split("#"); + if (!filename) return href; + const target = path.posix + .normalize(path.posix.join(path.posix.dirname(from), filename)) + .replace(/(?:index)?\.md$/, ""); + const origin = from.endsWith("index.md") + ? path.posix.dirname(from) + : from.replace(/\.md$/, ""); + let relative = path.posix.relative(origin, target) || "."; + if ( + (filename.endsWith("/") || filename.endsWith(".md")) && + !relative.endsWith("/") + ) + relative += "/"; + return relative + (hash ? `#${hash}` : ""); +} +export function assertManualHtml( + source: string, + html: string, + filename: string, +) { + const rendered = mainHtml(html); + const body = source + .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "") + .replace(//g, ""); + const prose = body.replace(/```[\s\S]*?```/g, ""); + for (const block of prose.split(/\n\s*\n/)) { + const paragraph = block.replace(/^\s*(?:!!!|\?\?\?|===).*$/gm, "").trim(); + if (/^[A-Za-z]/.test(paragraph) && !paragraph.includes("|")) + assertText( + rendered, + paragraph, + `content/${filename}: rendered source text`, + true, + ); + // Headings and individual list items are checked separately from paragraphs. + for (const line of paragraph.split(/(?=^\s*(?:#{1,6} |[-*] |\d+\. ))/m)) + if (/^\s*(?:#{1,6} |[-*] |\d+\. )/.test(line)) + assertText( + rendered, + line.replace(/^\s*(?:#{1,6} |[-*] |\d+\. )/, ""), + `content/${filename}: heading/list`, + true, + ); + } + for (const line of prose + .split("\n") + .filter((line) => /^\|/.test(line) && !/^\|\s*:?-/.test(line))) { + for (const cell of line.slice(1, line.lastIndexOf("|")).split(/(?", ">"); + expect(htmlText(rendered), `content/${filename}: fenced content`).toContain( + htmlText(escaped), + ); + } + const links: string[] = []; + new HTMLRewriter() + .on("a", { + element(e) { + const href = e.getAttribute("href"); + if (href) links.push(href); + }, + }) + .transform( + Bun.markdown.html( + body.replace(/^\[\^[^\]]+\]:.*$/gm, "").replace(/\[\^[^\]]+\]/g, ""), + ), + ); + for (const href of links) { + const expected = renderedHref(filename, href); + expect(rendered, `content/${filename}: link ${href}`).toContain( + `href="${expected}"`, + ); + } +} diff --git a/tools/scripts/test-support/sources.ts b/tools/scripts/test-support/sources.ts new file mode 100644 index 0000000..a8ab7fa --- /dev/null +++ b/tools/scripts/test-support/sources.ts @@ -0,0 +1,48 @@ +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { parse } from "smol-toml"; +import { loadToolConfig as readConfig, resolveToolPath } from "../config"; +import type { loadRules as readRules } from "../build-markdown"; +import { parseOscalCatalog } from "../oscal-catalog"; + +export type Rules = Awaited>; +export async function readTree(root: string): Promise> { + const files = new Map(); + async function visit(directory: string) { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const filename = path.join(directory, entry.name); + if (entry.isDirectory()) await visit(filename); + else if (entry.isFile()) + files.set( + path.relative(root, filename).split(path.sep).join("/"), + await readFile(filename), + ); + } + } + await visit(root); + return files; +} + +// One immutable-in-practice snapshot per test process. Never fetch or sync inputs. +const snapshot = (async () => { + const config = await readConfig(); + const [json, xml, toml, content] = await Promise.all([ + readFile(resolveToolPath(config.paths.rulesFile), "utf8"), + readFile(resolveToolPath(config.paths.oscalCatalogFile), "utf8"), + readFile(resolveToolPath(config.paths.zensicalConfig), "utf8"), + readTree(resolveToolPath(config.paths.content)), + ]); + return { + config, + rules: JSON.parse(json) as Rules, + catalog: parseOscalCatalog(xml), + navigation: parse(toml), + content, + }; +})(); +export const sources = () => snapshot; +// Callers may modify fixtures without contaminating subsequent tests. +export const loadToolConfig = async () => + structuredClone((await snapshot).config); +export const loadRules = async (_config?: unknown) => + structuredClone((await snapshot).rules); diff --git a/tools/scripts/trigrams.test.ts b/tools/scripts/trigrams.test.ts index 62d86a9..115ed1b 100644 --- a/tools/scripts/trigrams.test.ts +++ b/tools/scripts/trigrams.test.ts @@ -2,8 +2,9 @@ import { expect, test } from "bun:test"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { buildMarkdown, collectArtifacts, loadRules } from "./build-markdown"; -import { loadToolConfig } from "./config"; +import { buildMarkdown, collectArtifacts } from "./build-markdown"; +import { loadToolConfig, loadRules, type Rules } from "./test-support/sources"; +import { expectedTrigrams } from "./test-support/source-contract"; test("trigram lookup covers collections, rulesets, subsets, and themes with valid targets", async () => { const config = await loadToolConfig(); @@ -13,42 +14,11 @@ test("trigram lookup covers collections, rulesets, subsets, and themes with vali (artifact) => artifact.documentType === "TRIGRAMS", )!; const rows = lookup.context.trigramRows; - const expected = [rules.FRD.info.short_name!, "FRR", "KSI", "CTL"]; - for (const [key, document] of Object.entries(rules.FRR)) { - const trigram = document.info.short_name ?? key; - expected.push( - trigram, - ...Object.keys(document.info.subsets ?? {}), - ); - } - expected.push( - ...Object.entries(rules.KSI).map( - ([key, theme]) => theme.short_name ?? key, - ), - ); - expect(rows.map((row) => row.trigram)).toEqual( - [...new Set(expected)].sort((a, b) => a.localeCompare(b, "en")), - ); - expect(new Set(rows.map((row) => row.trigram)).size).toBe(rows.length); - expect(new Set(rows.map((row) => row.type))).toEqual( - new Set(["Collection", "Ruleset", "Ruleset subset", "KSI theme"]), - ); - for (const excluded of ["FRD-SNT", "AFC-CSO", "CDS-CSO", "AFC-CSO-ABC", "KSI-CED", "KSI-CED-RAT"]) { - expect(rows.some((row) => row.trigram === excluded)).toBe(false); - } - const href = (trigram: string) => - rows.find((row) => row.trigram === trigram)?.href; - expect(href("FRD")).toBe("definitions.md"); - expect(href("MKT")).toBe("reference/marketplace-listing.md"); - expect(href("CSO")).toBeUndefined(); - expect(href("TRC")).toBe("reference/certification-data-sharing.md#fedramp-compatible-trust-centers"); - expect(href("CED")).toBe( - "reference/key-security-indicators.md#cybersecurity-education", - ); + expect(rows).toEqual(expectedTrigrams(rules, config)); for (const row of rows) { if (row.type === "Ruleset subset") { - const occurrences = Object.values(rules.FRR).filter( - (document) => Object.hasOwn(document.info.subsets ?? {}, row.trigram), + const occurrences = Object.values(rules.FRR).filter((document) => + Object.hasOwn(document.info.subsets ?? {}, row.trigram), ).length; expect(Boolean(row.href), row.trigram).toBe(occurrences === 1); } @@ -71,20 +41,12 @@ test("shared subset trigrams use a stable representative regardless of source or const rows = collectArtifacts(rules, config).find( (artifact) => artifact.documentType === "TRIGRAMS", )!.context.trigramRows; - expect(rows.filter((row) => row.trigram === "CSO")).toEqual([{ - trigram: "CSO", - name: "General Provider Responsibilities", - type: "Ruleset subset", - href: undefined, - }]); - expect(rows.filter((row) => row.trigram === "FRP")).toHaveLength(1); - expect(rows.filter((row) => row.trigram === "IAS")).toHaveLength(1); + expect(rows).toEqual(expectedTrigrams(rules, config)); }); test("trigrams follow moved mappings and fall back to the KSI theme key", async () => { - const config = structuredClone(await loadToolConfig()); - const rules = structuredClone(await loadRules(config)); - delete rules.KSI.CED!.short_name; + const { config, rules } = await trigramFixture(); + delete rules.KSI.XYZ!.short_name; config.generated.trigramDocuments![0]!.output = "lookup/trigrams.md"; config.generated.definitionDocuments![0]!.output = "glossary.md"; const rows = collectArtifacts(rules, config).find( @@ -93,14 +55,13 @@ test("trigrams follow moved mappings and fall back to the KSI theme key", async expect(rows.find((row) => row.trigram === "FRD")!.href).toBe( "../glossary.md", ); - expect(rows.find((row) => row.trigram === "CED")!.href).toBe( - "../reference/key-security-indicators.md#cybersecurity-education", + expect(rows.find((row) => row.trigram === "XYZ")!.href).toBe( + "../reference/key-security-indicators.md#sample-theme", ); }); test("trigrams reject missing targets and duplicate identifiers", async () => { - const config = structuredClone(await loadToolConfig()); - const rules = await loadRules(config); + const { config, rules } = await trigramFixture(); config.generated.trigramDocuments![0]!.collections.push({ trigram: "FRD", name: "Duplicate", @@ -118,8 +79,7 @@ test("trigrams reject missing targets and duplicate identifiers", async () => { }); test("trigram rendering escapes table cells, tracks its manifest, and protects manual content", async () => { - const config = structuredClone(await loadToolConfig()); - const rules = structuredClone(await loadRules(config)); + const { config, rules } = await trigramFixture(); const temp = await mkdtemp(path.join(tmpdir(), "fedramp-trigrams-")); try { config.paths.src = path.join(temp, "src"); @@ -137,8 +97,8 @@ test("trigram rendering escapes table cells, tracks its manifest, and protects m "| [FRD](definitions.md) | Definitions \\| Terms Lookup | Collection |", ); expect(markdown.match(/\| Trigram \| Name \| Type \|/g)?.length).toBe(1); - expect(markdown).toContain("| CSO | General Provider Responsibilities | Ruleset subset |"); - expect(markdown).not.toContain("[CSO]"); + expect(markdown).toContain("| SHR | Shared subset | Ruleset subset |"); + expect(markdown).not.toContain("[SHR]"); expect(markdown).toContain('title: "FedRAMP Trigrams"'); expect(markdown).toContain("source: machine"); const manifest = JSON.parse( @@ -162,3 +122,88 @@ test("trigram rendering escapes table cells, tracks its manifest, and protects m await rm(temp, { recursive: true, force: true }); } }); + +async function trigramFixture() { + const config = await loadToolConfig(); + for (const key of Object.keys(config.generated)) { + if (Array.isArray(config.generated[key as keyof typeof config.generated])) + (config.generated as unknown as Record)[key] = []; + } + delete config.generated.search; + const doc = (key: string) => ({ + info: { + name: `${key} rules`, + short_name: key, + web_name: key.toLowerCase(), + subsets: { SHR: { name: "Shared subset" } }, + }, + data: { + all: { + SHR: { + [`${key}-SHR-ONE`]: { + name: "Sample rule", + statement: "Follow this synthetic rule.", + }, + }, + }, + }, + }); + const rules: Rules = { + FRD: { + info: { name: "Definitions", short_name: "FRD", web_name: "definitions" }, + data: { + all: { + "FRD-ONE": { term: "Example", definition: "Synthetic definition." }, + }, + }, + }, + FRR: { AAA: doc("AAA"), BBB: doc("BBB") }, + KSI: { + XYZ: { + name: "Sample Theme", + web_name: "sample-theme", + short_name: "XYZ", + indicators: { + "KSI-XYZ-ONE": { name: "Indicator", statement: "Measure this." }, + }, + }, + }, + CTL: {}, + }; + config.generated.definitionDocuments = [ + { + id: "fedramp-definitions", + output: "definitions.md", + source: { collection: "FRD", types: ["all"] }, + }, + ]; + config.generated.ruleDocuments = [ + { + id: "rules", + output: "reference/{FRR}.md", + outputMode: "documents", + source: { collection: "FRR", documents: "ALL", types: ["all"] }, + }, + ]; + config.generated.ksiDocuments = [ + { + id: "ksi", + output: "reference/key-security-indicators.md", + outputMode: "single", + source: { collection: "KSI", themes: "ALL" }, + }, + ]; + config.generated.trigramDocuments = [ + { + id: "trigrams", + title: "FedRAMP Trigrams", + description: "Fixture", + output: "trigrams.md", + definitionDocumentMappingId: "fedramp-definitions", + ruleDocumentMappingId: "rules", + ksiDocumentMappingId: "ksi", + collections: [], + }, + ]; + return { config, rules }; +} diff --git a/tools/scripts/verify-site.ts b/tools/scripts/verify-site.ts new file mode 100644 index 0000000..db123db --- /dev/null +++ b/tools/scripts/verify-site.ts @@ -0,0 +1,80 @@ +import { stat } from "node:fs/promises"; +import path from "node:path"; +import { loadToolConfig, resolveToolPath } from "./config"; + +async function isNonemptyFile(file: string): Promise { + try { + const info = await stat(file); + return info.isFile() && info.size > 0; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function main(): Promise { + const config = await loadToolConfig(); + const src = resolveToolPath(config.paths.src); + const html = resolveToolPath(config.paths.html); + const manifestPath = path.join(src, config.generated.manifest); + + const manifest: unknown = await Bun.file(manifestPath).json(); + + if ( + typeof manifest !== "object" || + manifest === null || + !("files" in manifest) || + !Array.isArray(manifest.files) || + !manifest.files.every((file): file is string => typeof file === "string") + ) { + throw new Error(`Invalid generated manifest: ${manifestPath}`); + } + + const files = manifest.files as string[]; + + if (files.length === 0) { + throw new Error("Generated Markdown manifest is empty"); + } + + const missing: string[] = []; + + if (!(await isNonemptyFile(path.join(html, "index.html")))) { + missing.push("Site homepage: index.html"); + } + + for (const name of files) { + if (!(await isNonemptyFile(path.join(src, name)))) { + missing.push(`Generated source: ${name}`); + continue; + } + + if (path.extname(name).toLowerCase() !== ".md") { + continue; + } + + const stem = name.slice(0, -3); + const candidates = [ + path.join(html, `${stem}.html`), + path.join(html, stem, "index.html"), + ]; + + const exists = await Promise.all(candidates.map(isNonemptyFile)); + + if (!exists.some(Boolean)) { + missing.push(`HTML for ${name}`); + } + } + + if (missing.length > 0) { + throw new Error(`Site output is incomplete:\n- ${missing.join("\n- ")}`); + } + + console.log(`Verified output for ${files.length} generated artifacts.`); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +});
CSO