diff --git a/test/_helpers/first-use.ts b/test/_helpers/first-use.ts new file mode 100644 index 0000000..cc131bd --- /dev/null +++ b/test/_helpers/first-use.ts @@ -0,0 +1,177 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join, relative } from "node:path"; + +import ts from "typescript"; + +/** + * Readers for the package's first-use path: the first fenced block of + * `docs-content/quickstart.md`, the first fenced block under `## Usage` in `README.md`, and the + * install command both documents print. Every reader works on the text of the document as it is + * on disk at test time, never on a copy held here. + */ + +/** + * The compiler options of a new project, as `tsc --init` of the TypeScript this repository pins + * writes them: `strict` plus `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes` and + * `verbatimModuleSyntax`, NodeNext modules. The emit-only settings it also writes (`sourceMap`, + * `declaration`, `declarationMap`, `jsx`) are left out because nothing is emitted, and Node's + * types are loaded where `tsc --init` writes `types: []`, because every first-use example is a + * Node program. + */ +const READER_OPTIONS: ts.CompilerOptions = { + module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, + target: ts.ScriptTarget.ESNext, + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes: true, + verbatimModuleSyntax: true, + isolatedModules: true, + noUncheckedSideEffectImports: true, + moduleDetection: ts.ModuleDetectionKind.Force, + skipLibCheck: true, + noEmit: true, + types: ["node"], +}; + +/** + * Every TypeScript error a reader's new project reports for one example, as `line N: TSxxxx + * message`; empty when the example compiles. The example is compiled as an ES module sitting in + * the repository root, with each specifier in `paths` resolved to the single source file the + * bundler compiles into that published entry point and its types. Only the example's own errors + * are returned. + */ +export function compileErrors( + root: string, + paths: Readonly>, + code: string, +): string[] { + const file = join(root, "first-use-example.mts"); + const options: ts.CompilerOptions = { + ...READER_OPTIONS, + typeRoots: [join(root, "node_modules", "@types")], + paths: Object.fromEntries(Object.entries(paths).map(([pkg, entry]) => [pkg, [entry]])), + }; + const host = ts.createCompilerHost(options, true); + const getSourceFile = host.getSourceFile.bind(host); + host.getSourceFile = (name, language, onError, create) => + name === file + ? ts.createSourceFile(name, code, language, true) + : getSourceFile(name, language, onError, create); + const fileExists = host.fileExists.bind(host); + host.fileExists = (name) => name === file || fileExists(name); + const readFile = host.readFile.bind(host); + host.readFile = (name) => (name === file ? code : readFile(name)); + const program = ts.createProgram([file], options, host); + return ts + .getPreEmitDiagnostics(program, program.getSourceFile(file)) + .filter((d) => d.file?.fileName === file) + .map((d) => { + const line = d.file?.getLineAndCharacterOfPosition(d.start ?? 0).line ?? 0; + const text = ts.flattenDiagnosticMessageText(d.messageText, " "); + return `line ${String(line + 1)}: TS${String(d.code)} ${text}`; + }); +} + +/** One fenced code block: its info string split into language and tags, and its body. */ +export interface Fence { + readonly lang: string; + readonly tags: readonly string[]; + readonly body: string; +} + +/** Every fenced block in a chunk of markdown, in document order. */ +export function fences(markdown: string): Fence[] { + const out: Fence[] = []; + let open: { marker: string; lang: string; tags: string[] } | undefined; + let buf: string[] = []; + for (const line of markdown.split("\n")) { + if (open === undefined) { + const opened = /^(`{3,}|~{3,})\s*(.*)$/.exec(line); + if (opened !== null) { + const tokens = (opened[2] ?? "").trim().split(/\s+/).filter(Boolean); + open = { marker: opened[1] ?? "```", lang: tokens[0] ?? "", tags: tokens.slice(1) }; + buf = []; + } + continue; + } + if (line.trimEnd() === open.marker) { + out.push({ lang: open.lang, tags: open.tags, body: buf.join("\n") }); + open = undefined; + continue; + } + buf.push(line); + } + if (open !== undefined) throw new Error("unterminated code fence"); + return out; +} + +/** The lines of one `##` section, from its heading to the next `##` heading. */ +export function section(markdown: string, heading: string): string { + const lines = markdown.split("\n"); + const start = lines.indexOf(heading); + if (start === -1) throw new Error(`no ${heading} section`); + let end = lines.length; + for (let i = start + 1; i < lines.length; i += 1) { + if (lines[i]?.startsWith("## ") === true) { + end = i; + break; + } + } + return lines.slice(start + 1, end).join("\n"); +} + +/** + * The record stream a first-use example builds: the string literals assigned to the first `const` + * that holds one, concatenated. Each literal is decoded by the JSON string grammar, which is the + * JavaScript one for every escape these examples use (`\\`, `\r`). `undefined` when there is none. + * + * The literals are read one at a time with a sticky pattern rather than by one pattern with a + * repeated group, which would backtrack exponentially on a long run of empty literals. + */ +export function recordStreamLiteral(code: string): string | undefined { + const literal = /\s*("(?:[^"\\\n]|\\.)*")[ \t]*(\+)?/y; + for (const declared of code.matchAll(/const \w+ =/g)) { + literal.lastIndex = declared.index + declared[0].length; + const parts: string[] = []; + for (let match = literal.exec(code); match !== null; match = literal.exec(code)) { + parts.push(JSON.parse(match[1] ?? '""') as string); + if (match[2] === undefined) break; + } + if (parts.length > 0) return parts.join(""); + } + return undefined; +} + +/** Every file under `dir`, keyed by its exact contents, mapped to its path from `root`. */ +export function fixturesByContent(root: string, dir: string): Map { + const byText = new Map(); + const walk = (current: string): void => { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const full = join(current, entry.name); + if (entry.isDirectory()) walk(full); + else byText.set(readFileSync(full, "latin1"), relative(root, full)); + } + }; + walk(dir); + return byText; +} + +/** + * The package names a document tells a reader to install: the first non-flag argument of every + * `npm install`, `npm i`, `npm add`, `pnpm add`, `pnpm install`, `pnpm i`, `yarn add`, `bun add` + * or `deno add npm:`, read as a package name, so any `@version` suffix, closing quote or backtick, + * or sentence-ending full stop is left off. A local path or a protocol specifier (`file:../pkg`, + * `link:`, `git+https:`) installs whatever that path or URL holds rather than resolving a package + * name, so it is not a registry specifier and is not read. + */ +export function installSpecifiers(markdown: string): string[] { + const out: string[] = []; + const command = + /\b(?:npm (?:install|add|i)|pnpm (?:add|install|i)|yarn add|bun add|deno add)((?:[ \t]+-{1,2}[\w-]+)*)[ \t]+(?:npm:)?(@?\w[\w.-]*(?:\/[\w.-]+)?)(?![\w.:/+-])/g; + for (const match of markdown.matchAll(command)) { + const spec = match[2]; + if (spec !== undefined) out.push(spec.replace(/\.+$/, "")); + } + return out; +} diff --git a/test/docs-content.test.ts b/test/docs-content.test.ts index 9f07e91..5373aae 100644 --- a/test/docs-content.test.ts +++ b/test/docs-content.test.ts @@ -1,9 +1,21 @@ import { execFileSync } from "node:child_process"; +import { rmSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { beforeAll } from "vitest"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { docSnippetSuite } from "@cosyte/vitest-config/snippets"; +import { + docSnippetSuite, + extractRunnableSnippets, + runSnippet, +} from "@cosyte/vitest-config/snippets"; + +import { + compileErrors, + fences, + fixturesByContent, + recordStreamLiteral, +} from "./_helpers/first-use.js"; /** * Doc/code-agreement gate. Every ```` ```ts runnable ```` block in `docs-content/` is extracted, @@ -18,6 +30,8 @@ import { docSnippetSuite } from "@cosyte/vitest-config/snippets"; */ const root = join(import.meta.dirname, ".."); const distEntry = join(root, "dist", "index.mjs"); +const resolve = (specifier: string): string | undefined => + specifier === "@cosyte/astm" ? distEntry : undefined; beforeAll(() => { execFileSync("pnpm", ["build"], { cwd: root, stdio: "inherit" }); @@ -25,5 +39,88 @@ beforeAll(() => { docSnippetSuite({ docsDir: join(root, "docs-content"), - resolve: (specifier) => (specifier === "@cosyte/astm" ? distEntry : undefined), + resolve, +}); + +/** + * The quickstart's FIRST example is the first thing a reader runs from the docs site, so it is held + * to more than the sweep above: it must be the block the sweep executes, its record stream must be a + * committed synthetic fixture (the page is public, and `test/fixtures` is the corpus `pnpm phi-scan` + * reads), and a changed value in it must turn this suite red. Its temp modules live inside the root, + * as the harness requires, and are removed when the file is done, so nothing is left on disk. + */ +const quickstart = readFileSync(join(root, "docs-content", "quickstart.md"), "utf8"); +const firstBlock = fences(quickstart)[0]; +const firstRunnable = extractRunnableSnippets(quickstart)[0]; +const firstUseTmp = join(root, ".cosyte-first-use-snippets"); +/** + * The snippet harness strips types without checking them, so compiling is checked separately, the + * way a reader's new TypeScript project compiles the block, against the source entry point the + * bundler compiles into the published types. A program over the source takes seconds to check, so + * these cases state their own budget. + */ +const sourcePaths = { "@cosyte/astm": join(root, "src", "index.ts") }; +const COMPILE_TIMEOUT = 60_000; + +afterAll(() => { + rmSync(firstUseTmp, { recursive: true, force: true }); +}); + +describe("the quickstart's first example", () => { + it("AC-AS1: is a runnable TypeScript block, so the sweep above executes it", () => { + expect(firstBlock?.lang).toBe("ts"); + expect(firstBlock?.tags).toContain("runnable"); + expect(firstBlock?.tags).not.toContain("throws"); + expect(firstRunnable?.code).toBe(firstBlock?.body); + }); + + it( + "AC-AS1: compiles in a new TypeScript project against the package's types", + () => { + expect(compileErrors(root, sourcePaths, firstBlock?.body ?? "")).toEqual([]); + }, + COMPILE_TIMEOUT, + ); + + it( + "AC-AS1: a block that does not compile is reported, so it turns this suite red", + () => { + const code = firstBlock?.body ?? ""; + expect(code.split("first?.value;").length - 1).toBe(1); + const mutated = code.replace("first?.value;", "first.value;"); + expect(compileErrors(root, sourcePaths, mutated)).toEqual([ + expect.stringContaining("TS18048"), + ]); + }, + COMPILE_TIMEOUT, + ); + + it("AC-AS1: runs against the built package and every claimed value holds", async () => { + expect(firstRunnable).toBeDefined(); + if (firstRunnable === undefined) return; + await runSnippet(firstRunnable, { resolve, tmpDir: firstUseTmp }); + }); + + it("AC-AS4: its record stream is a byte-for-byte copy of a fixture under test/fixtures", () => { + const stream = recordStreamLiteral(firstRunnable?.code ?? ""); + expect(stream).toBeDefined(); + const fixtures = fixturesByContent(root, join(root, "test", "fixtures")); + expect(fixtures.get(stream ?? "")).toBeDefined(); + }); + + it("AC-AS3: a changed claimed value turns the run red", async () => { + const code = firstRunnable?.code ?? ""; + expect(code.split('// => "28.6"').length - 1).toBe(1); + const mutated = code.replace('// => "28.6"', '// => "31.4"'); + await expect(runSnippet(mutated, { resolve, tmpDir: firstUseTmp })).rejects.toThrow(); + }); + + it("AC-AS3: a changed input value turns the run red and leaves the fixture corpus", async () => { + const code = firstRunnable?.code ?? ""; + expect(code.split("|28.6|").length - 1).toBe(1); + const mutated = code.replace("|28.6|", "|31.4|"); + const fixtures = fixturesByContent(root, join(root, "test", "fixtures")); + expect(fixtures.get(recordStreamLiteral(mutated) ?? "")).toBeUndefined(); + await expect(runSnippet(mutated, { resolve, tmpDir: firstUseTmp })).rejects.toThrow(); + }); }); diff --git a/test/fixtures/tier1-quickstart-result.astm b/test/fixtures/tier1-quickstart-result.astm new file mode 100644 index 0000000..34d7d67 --- /dev/null +++ b/test/fixtures/tier1-quickstart-result.astm @@ -0,0 +1 @@ +H|\^& P|1|PRAC|LAB O|1|ACC R|1|^^^687|28.6|U/L||N||F L|1|N \ No newline at end of file diff --git a/test/fixtures/tier1-readme-result.astm b/test/fixtures/tier1-readme-result.astm new file mode 100644 index 0000000..ca62954 --- /dev/null +++ b/test/fixtures/tier1-readme-result.astm @@ -0,0 +1 @@ +H|\^&|||analyzer^1|||||||P|LIS02-A2|20240115103000 P|1|PRACTICE-0001|LAB-0002|||||F O|1|SPEC-7|ACC-42|^^^687|||20240115102500 R|1|^^^687|28.6|U/L|10-40|N||F L|1|N \ No newline at end of file diff --git a/test/install-specifier.test.ts b/test/install-specifier.test.ts new file mode 100644 index 0000000..202c23d --- /dev/null +++ b/test/install-specifier.test.ts @@ -0,0 +1,48 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { installSpecifiers } from "./_helpers/first-use.js"; + +/** + * The install command a reader copies has to fetch THIS package. The subject is package identity: + * each specifier the two first-use documents print is compared with `package.json` `name`, and a + * mismatch names both strings. + */ +const root = join(import.meta.dirname, ".."); +const { name } = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { name: string }; + +describe("the documented install specifier", () => { + for (const doc of ["docs-content/installation.md", "README.md"]) { + it(`AC-AS6: every install command in ${doc} names package.json name`, () => { + const specs = installSpecifiers(readFileSync(join(root, doc), "utf8")); + expect(specs.length, `${doc} prints no install command`).toBeGreaterThan(0); + for (const spec of specs) { + expect(spec, `${doc} installs "${spec}", package.json name is "${name}"`).toBe(name); + } + }); + } + + it("AC-AS6: a specifier that is not the package name is read as the name it prints", () => { + expect(installSpecifiers("npm install @cosyte/astn\n`pnpm add -D @cosyte/astm@0.0.1`")).toEqual( + ["@cosyte/astn", "@cosyte/astm"], + ); + }); + + it("AC-AS6: every install command form a reader may copy is read, inline code included", () => { + const forms = [ + "pnpm i @cosyte/astn", + "pnpm install @cosyte/astn", + "npm add @cosyte/astn", + "deno add npm:@cosyte/astn", + "run `npm install @cosyte/astn` first", + "then run npm install @cosyte/astn.", + ]; + for (const form of forms) expect(installSpecifiers(form), form).toEqual(["@cosyte/astn"]); + expect(installSpecifiers("pnpm install\npnpm install --frozen-lockfile")).toEqual([]); + expect( + installSpecifiers("pnpm add file:../astm\nnpm install git+https://x.test/astm.git"), + ).toEqual([]); + }); +}); diff --git a/test/readme-usage.test.ts b/test/readme-usage.test.ts index d159824..aee4b28 100644 --- a/test/readme-usage.test.ts +++ b/test/readme-usage.test.ts @@ -44,6 +44,14 @@ import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + compileErrors, + fences as helperFences, + fixturesByContent, + recordStreamLiteral, + section as helperSection, +} from "./_helpers/first-use.js"; + const REPO_ROOT = process.cwd(); const README_PATH = join(REPO_ROOT, "README.md"); const ENTRY_POINT = join(REPO_ROOT, "src", "index.ts"); @@ -189,4 +197,41 @@ describe("the README Usage example", () => { }, CASE_TIMEOUT, ); + + it("AC-AS2: the block executed above is the first fenced block under ## Usage", () => { + const first = helperFences(helperSection(readFileSync(README_PATH, "utf8"), "## Usage"))[0]; + expect(first?.lang).toBe("ts"); + expect(usageFences[0]?.body).toBe(first?.body); + }); + + it( + "AC-AS2: compiles in a new TypeScript project against the package's types", + () => { + const paths = { "@cosyte/astm": ENTRY_POINT }; + expect(compileErrors(REPO_ROOT, paths, usageFences[0]?.body ?? "")).toEqual([]); + }, + CASE_TIMEOUT, + ); + + it( + "AC-AS2: a block that does not compile is reported, so it turns this suite red", + () => { + const code = usageFences[0]?.body ?? ""; + expect(code.split("first?.value, ").length - 1).toBe(1); + const mutated = code.replace("first?.value, ", "first.value, "); + expect(compileErrors(REPO_ROOT, { "@cosyte/astm": ENTRY_POINT }, mutated)).toEqual([ + expect.stringContaining("TS18048"), + ]); + }, + CASE_TIMEOUT, + ); + + it("AC-AS4: its record stream is a byte-for-byte copy of a fixture under test/fixtures", () => { + const stream = recordStreamLiteral(usageFences[0]?.body ?? ""); + expect(stream).toBeDefined(); + const fixtures = fixturesByContent(REPO_ROOT, join(REPO_ROOT, "test", "fixtures")); + expect(fixtures.get(stream ?? "")).toBeDefined(); + const mutated = (usageFences[0]?.body ?? "").replace("|28.6|", "|31.4|"); + expect(fixtures.get(recordStreamLiteral(mutated) ?? "")).toBeUndefined(); + }); });