From f610ffb3b256c1af410db68b09b2b53f2caa2011 Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Wed, 23 Sep 2026 16:31:29 +0000 Subject: [PATCH 1/3] S0329-parser-first-use-path: hold both first examples to a committed fixture and execute them The record streams printed in the quickstart's first example and the README Usage example are committed byte for byte as test/fixtures/tier1-quickstart-result.astm and test/fixtures/tier1-readme-result.astm, inside the corpus pnpm phi-scan reads and the Tier-3 round-trip sweeps; the suites assert each example's stream is a copy. The quickstart's first block is checked to be the runnable block the sweep executes, run explicitly, with a changed-claim and a changed-input negative control. The README gate is pinned to the first fenced block under the Usage heading. Install commands in README.md and installation.md are compared with package.json name. No document text changed, so no changeset. --- test/_helpers/first-use.ts | 100 +++++++++++++++++++++ test/docs-content.test.ts | 69 +++++++++++++- test/fixtures/tier1-quickstart-result.astm | 1 + test/fixtures/tier1-readme-result.astm | 1 + test/install-specifier.test.ts | 32 +++++++ test/readme-usage.test.ts | 22 +++++ 6 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 test/_helpers/first-use.ts create mode 100644 test/fixtures/tier1-quickstart-result.astm create mode 100644 test/fixtures/tier1-readme-result.astm create mode 100644 test/install-specifier.test.ts diff --git a/test/_helpers/first-use.ts b/test/_helpers/first-use.ts new file mode 100644 index 0000000..711393d --- /dev/null +++ b/test/_helpers/first-use.ts @@ -0,0 +1,100 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join, relative } from "node:path"; + +/** + * Readers for the package's first-use path: the first fenced block of + * `docs-content/quickstart.md`, the first fenced block under `## Usage` in `README.md`, and the + * install command both documents print. Every reader works on the text of the document as it is + * on disk at test time, never on a copy held here. + */ + +/** One fenced code block: its info string split into language and tags, and its body. */ +export interface Fence { + readonly lang: string; + readonly tags: readonly string[]; + readonly body: string; +} + +/** Every fenced block in a chunk of markdown, in document order. */ +export function fences(markdown: string): Fence[] { + const out: Fence[] = []; + let open: { marker: string; lang: string; tags: string[] } | undefined; + let buf: string[] = []; + for (const line of markdown.split("\n")) { + if (open === undefined) { + const opened = /^(`{3,}|~{3,})\s*(.*)$/.exec(line); + if (opened !== null) { + const tokens = (opened[2] ?? "").trim().split(/\s+/).filter(Boolean); + open = { marker: opened[1] ?? "```", lang: tokens[0] ?? "", tags: tokens.slice(1) }; + buf = []; + } + continue; + } + if (line.trimEnd() === open.marker) { + out.push({ lang: open.lang, tags: open.tags, body: buf.join("\n") }); + open = undefined; + continue; + } + buf.push(line); + } + if (open !== undefined) throw new Error("unterminated code fence"); + return out; +} + +/** The lines of one `##` section, from its heading to the next `##` heading. */ +export function section(markdown: string, heading: string): string { + const lines = markdown.split("\n"); + const start = lines.indexOf(heading); + if (start === -1) throw new Error(`no ${heading} section`); + let end = lines.length; + for (let i = start + 1; i < lines.length; i += 1) { + if (lines[i]?.startsWith("## ") === true) { + end = i; + break; + } + } + return lines.slice(start + 1, end).join("\n"); +} + +/** + * The record stream a first-use example builds: the string literals assigned to the first `const` + * that holds one, concatenated. Each literal is decoded by the JSON string grammar, which is the + * JavaScript one for every escape these examples use (`\\`, `\r`). `undefined` when there is none. + */ +export function recordStreamLiteral(code: string): string | undefined { + const assigned = /const \w+ =((?:\s*"(?:[^"\\\n]|\\.)*"\s*\+?)+)\s*;/.exec(code); + if (assigned === null) return undefined; + const parts = (assigned[1] ?? "").match(/"(?:[^"\\\n]|\\.)*"/g) ?? []; + return parts.map((part) => JSON.parse(part) as string).join(""); +} + +/** Every file under `dir`, keyed by its exact contents, mapped to its path from `root`. */ +export function fixturesByContent(root: string, dir: string): Map { + const byText = new Map(); + const walk = (current: string): void => { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const full = join(current, entry.name); + if (entry.isDirectory()) walk(full); + else byText.set(readFileSync(full, "latin1"), relative(root, full)); + } + }; + walk(dir); + return byText; +} + +/** + * The package names a document tells a reader to install: the first non-flag argument of every + * `npm install`, `npm i`, `pnpm add`, `yarn add` or `bun add`, with any `@version` suffix removed. + */ +export function installSpecifiers(markdown: string): string[] { + const out: string[] = []; + const command = + /\b(?:npm (?:install|i)|pnpm add|yarn add|bun add)((?:[ \t]+-{1,2}[\w-]+)*)[ \t]+([^\s`'"]+)/g; + for (const match of markdown.matchAll(command)) { + const spec = match[2]; + if (spec === undefined) continue; + const versionAt = spec.indexOf("@", 1); + out.push(versionAt === -1 ? spec : spec.slice(0, versionAt)); + } + return out; +} diff --git a/test/docs-content.test.ts b/test/docs-content.test.ts index 9f07e91..8d0e444 100644 --- a/test/docs-content.test.ts +++ b/test/docs-content.test.ts @@ -1,9 +1,16 @@ import { execFileSync } from "node:child_process"; +import { rmSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { beforeAll } from "vitest"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { docSnippetSuite } from "@cosyte/vitest-config/snippets"; +import { + docSnippetSuite, + extractRunnableSnippets, + runSnippet, +} from "@cosyte/vitest-config/snippets"; + +import { fences, fixturesByContent, recordStreamLiteral } from "./_helpers/first-use.js"; /** * Doc/code-agreement gate. Every ```` ```ts runnable ```` block in `docs-content/` is extracted, @@ -18,6 +25,8 @@ import { docSnippetSuite } from "@cosyte/vitest-config/snippets"; */ const root = join(import.meta.dirname, ".."); const distEntry = join(root, "dist", "index.mjs"); +const resolve = (specifier: string): string | undefined => + specifier === "@cosyte/astm" ? distEntry : undefined; beforeAll(() => { execFileSync("pnpm", ["build"], { cwd: root, stdio: "inherit" }); @@ -25,5 +34,59 @@ beforeAll(() => { docSnippetSuite({ docsDir: join(root, "docs-content"), - resolve: (specifier) => (specifier === "@cosyte/astm" ? distEntry : undefined), + resolve, +}); + +/** + * The quickstart's FIRST example is the first thing a reader runs from the docs site, so it is held + * to more than the sweep above: it must be the block the sweep executes, its record stream must be a + * committed synthetic fixture (the page is public, and `test/fixtures` is the corpus `pnpm phi-scan` + * reads), and a changed value in it must turn this suite red. Its temp modules live inside the root, + * as the harness requires, and are removed when the file is done, so nothing is left on disk. + */ +const quickstart = readFileSync(join(root, "docs-content", "quickstart.md"), "utf8"); +const firstBlock = fences(quickstart)[0]; +const firstRunnable = extractRunnableSnippets(quickstart)[0]; +const firstUseTmp = join(root, ".cosyte-first-use-snippets"); + +afterAll(() => { + rmSync(firstUseTmp, { recursive: true, force: true }); +}); + +describe("the quickstart's first example", () => { + it("AC-AS1: is a runnable TypeScript block, so the sweep above executes it", () => { + expect(firstBlock?.lang).toBe("ts"); + expect(firstBlock?.tags).toContain("runnable"); + expect(firstBlock?.tags).not.toContain("throws"); + expect(firstRunnable?.code).toBe(firstBlock?.body); + }); + + it("AC-AS1: runs against the built package and every claimed value holds", async () => { + expect(firstRunnable).toBeDefined(); + if (firstRunnable === undefined) return; + await runSnippet(firstRunnable, { resolve, tmpDir: firstUseTmp }); + }); + + it("AC-AS4: its record stream is a byte-for-byte copy of a fixture under test/fixtures", () => { + const stream = recordStreamLiteral(firstRunnable?.code ?? ""); + expect(stream).toBeDefined(); + const fixtures = fixturesByContent(root, join(root, "test", "fixtures")); + expect(fixtures.get(stream ?? "")).toBeDefined(); + }); + + it("AC-AS3: a changed claimed value turns the run red", async () => { + const code = firstRunnable?.code ?? ""; + expect(code.split('// => "28.6"').length - 1).toBe(1); + const mutated = code.replace('// => "28.6"', '// => "31.4"'); + await expect(runSnippet(mutated, { resolve, tmpDir: firstUseTmp })).rejects.toThrow(); + }); + + it("AC-AS3: a changed input value turns the run red and leaves the fixture corpus", async () => { + const code = firstRunnable?.code ?? ""; + expect(code.split("|28.6|").length - 1).toBe(1); + const mutated = code.replace("|28.6|", "|31.4|"); + const fixtures = fixturesByContent(root, join(root, "test", "fixtures")); + expect(fixtures.get(recordStreamLiteral(mutated) ?? "")).toBeUndefined(); + await expect(runSnippet(mutated, { resolve, tmpDir: firstUseTmp })).rejects.toThrow(); + }); }); diff --git a/test/fixtures/tier1-quickstart-result.astm b/test/fixtures/tier1-quickstart-result.astm new file mode 100644 index 0000000..34d7d67 --- /dev/null +++ b/test/fixtures/tier1-quickstart-result.astm @@ -0,0 +1 @@ +H|\^& P|1|PRAC|LAB O|1|ACC R|1|^^^687|28.6|U/L||N||F L|1|N \ No newline at end of file diff --git a/test/fixtures/tier1-readme-result.astm b/test/fixtures/tier1-readme-result.astm new file mode 100644 index 0000000..ca62954 --- /dev/null +++ b/test/fixtures/tier1-readme-result.astm @@ -0,0 +1 @@ +H|\^&|||analyzer^1|||||||P|LIS02-A2|20240115103000 P|1|PRACTICE-0001|LAB-0002|||||F O|1|SPEC-7|ACC-42|^^^687|||20240115102500 R|1|^^^687|28.6|U/L|10-40|N||F L|1|N \ No newline at end of file diff --git a/test/install-specifier.test.ts b/test/install-specifier.test.ts new file mode 100644 index 0000000..652e989 --- /dev/null +++ b/test/install-specifier.test.ts @@ -0,0 +1,32 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { installSpecifiers } from "./_helpers/first-use.js"; + +/** + * The install command a reader copies has to fetch THIS package. The subject is package identity: + * each specifier the two first-use documents print is compared with `package.json` `name`, and a + * mismatch names both strings. + */ +const root = join(import.meta.dirname, ".."); +const { name } = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { name: string }; + +describe("the documented install specifier", () => { + for (const doc of ["docs-content/installation.md", "README.md"]) { + it(`AC-AS6: every install command in ${doc} names package.json name`, () => { + const specs = installSpecifiers(readFileSync(join(root, doc), "utf8")); + expect(specs.length, `${doc} prints no install command`).toBeGreaterThan(0); + for (const spec of specs) { + expect(spec, `${doc} installs "${spec}", package.json name is "${name}"`).toBe(name); + } + }); + } + + it("AC-AS6: a specifier that is not the package name is read as the name it prints", () => { + expect(installSpecifiers("npm install @cosyte/astn\n`pnpm add -D @cosyte/astm@0.0.1`")).toEqual( + ["@cosyte/astn", "@cosyte/astm"], + ); + }); +}); diff --git a/test/readme-usage.test.ts b/test/readme-usage.test.ts index d159824..9f1be49 100644 --- a/test/readme-usage.test.ts +++ b/test/readme-usage.test.ts @@ -44,6 +44,13 @@ import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + fences as helperFences, + fixturesByContent, + recordStreamLiteral, + section as helperSection, +} from "./_helpers/first-use.js"; + const REPO_ROOT = process.cwd(); const README_PATH = join(REPO_ROOT, "README.md"); const ENTRY_POINT = join(REPO_ROOT, "src", "index.ts"); @@ -189,4 +196,19 @@ describe("the README Usage example", () => { }, CASE_TIMEOUT, ); + + it("AC-AS2: the block executed above is the first fenced block under ## Usage", () => { + const first = helperFences(helperSection(readFileSync(README_PATH, "utf8"), "## Usage"))[0]; + expect(first?.lang).toBe("ts"); + expect(usageFences[0]?.body).toBe(first?.body); + }); + + it("AC-AS4: its record stream is a byte-for-byte copy of a fixture under test/fixtures", () => { + const stream = recordStreamLiteral(usageFences[0]?.body ?? ""); + expect(stream).toBeDefined(); + const fixtures = fixturesByContent(REPO_ROOT, join(REPO_ROOT, "test", "fixtures")); + expect(fixtures.get(stream ?? "")).toBeDefined(); + const mutated = (usageFences[0]?.body ?? "").replace("|28.6|", "|31.4|"); + expect(fixtures.get(recordStreamLiteral(mutated) ?? "")).toBeUndefined(); + }); }); From 7cb05e2c182f1f4312b0b98742064f4600599c34 Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Wed, 23 Sep 2026 17:14:30 +0000 Subject: [PATCH 2/3] S0329-parser-first-use-path: read the example's record stream without a backtracking regex CodeQL flagged the single pattern with a repeated group (exponential backtracking on a long run of empty literals). The literals are now read one at a time with a sticky pattern. --- test/_helpers/first-use.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/test/_helpers/first-use.ts b/test/_helpers/first-use.ts index 711393d..7ff359e 100644 --- a/test/_helpers/first-use.ts +++ b/test/_helpers/first-use.ts @@ -60,12 +60,22 @@ export function section(markdown: string, heading: string): string { * The record stream a first-use example builds: the string literals assigned to the first `const` * that holds one, concatenated. Each literal is decoded by the JSON string grammar, which is the * JavaScript one for every escape these examples use (`\\`, `\r`). `undefined` when there is none. + * + * The literals are read one at a time with a sticky pattern rather than by one pattern with a + * repeated group, which would backtrack exponentially on a long run of empty literals. */ export function recordStreamLiteral(code: string): string | undefined { - const assigned = /const \w+ =((?:\s*"(?:[^"\\\n]|\\.)*"\s*\+?)+)\s*;/.exec(code); - if (assigned === null) return undefined; - const parts = (assigned[1] ?? "").match(/"(?:[^"\\\n]|\\.)*"/g) ?? []; - return parts.map((part) => JSON.parse(part) as string).join(""); + const literal = /\s*("(?:[^"\\\n]|\\.)*")[ \t]*(\+)?/y; + for (const declared of code.matchAll(/const \w+ =/g)) { + literal.lastIndex = declared.index + declared[0].length; + const parts: string[] = []; + for (let match = literal.exec(code); match !== null; match = literal.exec(code)) { + parts.push(JSON.parse(match[1] ?? '""') as string); + if (match[2] === undefined) break; + } + if (parts.length > 0) return parts.join(""); + } + return undefined; } /** Every file under `dir`, keyed by its exact contents, mapped to its path from `root`. */ From f2bab46ae45d97b6a44dc4df5af6916c8f1d8e4b Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Thu, 24 Sep 2026 04:42:17 +0000 Subject: [PATCH 3/3] S0329-parser-first-use-path: compile both first examples as a reader's new TypeScript project does The snippet harness and tsx strip types without checking them, so pnpm test would stay green over a first-use example that does not compile. Both first examples now compile with the options tsc --init writes for a new project (strict, noUncheckedIndexedAccess, exactOptionalPropertyTypes, verbatimModuleSyntax, NodeNext), against src/index.ts, and the suite fails on any error the example itself carries. Both compile today; each carries a control that drops the undefined guard on the first result and must be reported. The install-specifier reader also reads pnpm i, pnpm install, npm add and deno add npm:, stops a specifier at a full stop and skips a protocol specifier, so no printed form passes vacuously. --- test/_helpers/first-use.ts | 77 +++++++++++++++++++++++++++++++--- test/docs-content.test.ts | 36 +++++++++++++++- test/install-specifier.test.ts | 16 +++++++ test/readme-usage.test.ts | 23 ++++++++++ 4 files changed, 146 insertions(+), 6 deletions(-) diff --git a/test/_helpers/first-use.ts b/test/_helpers/first-use.ts index 7ff359e..cc131bd 100644 --- a/test/_helpers/first-use.ts +++ b/test/_helpers/first-use.ts @@ -1,6 +1,8 @@ import { readFileSync, readdirSync } from "node:fs"; import { join, relative } from "node:path"; +import ts from "typescript"; + /** * Readers for the package's first-use path: the first fenced block of * `docs-content/quickstart.md`, the first fenced block under `## Usage` in `README.md`, and the @@ -8,6 +10,69 @@ import { join, relative } from "node:path"; * on disk at test time, never on a copy held here. */ +/** + * The compiler options of a new project, as `tsc --init` of the TypeScript this repository pins + * writes them: `strict` plus `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes` and + * `verbatimModuleSyntax`, NodeNext modules. The emit-only settings it also writes (`sourceMap`, + * `declaration`, `declarationMap`, `jsx`) are left out because nothing is emitted, and Node's + * types are loaded where `tsc --init` writes `types: []`, because every first-use example is a + * Node program. + */ +const READER_OPTIONS: ts.CompilerOptions = { + module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, + target: ts.ScriptTarget.ESNext, + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes: true, + verbatimModuleSyntax: true, + isolatedModules: true, + noUncheckedSideEffectImports: true, + moduleDetection: ts.ModuleDetectionKind.Force, + skipLibCheck: true, + noEmit: true, + types: ["node"], +}; + +/** + * Every TypeScript error a reader's new project reports for one example, as `line N: TSxxxx + * message`; empty when the example compiles. The example is compiled as an ES module sitting in + * the repository root, with each specifier in `paths` resolved to the single source file the + * bundler compiles into that published entry point and its types. Only the example's own errors + * are returned. + */ +export function compileErrors( + root: string, + paths: Readonly>, + code: string, +): string[] { + const file = join(root, "first-use-example.mts"); + const options: ts.CompilerOptions = { + ...READER_OPTIONS, + typeRoots: [join(root, "node_modules", "@types")], + paths: Object.fromEntries(Object.entries(paths).map(([pkg, entry]) => [pkg, [entry]])), + }; + const host = ts.createCompilerHost(options, true); + const getSourceFile = host.getSourceFile.bind(host); + host.getSourceFile = (name, language, onError, create) => + name === file + ? ts.createSourceFile(name, code, language, true) + : getSourceFile(name, language, onError, create); + const fileExists = host.fileExists.bind(host); + host.fileExists = (name) => name === file || fileExists(name); + const readFile = host.readFile.bind(host); + host.readFile = (name) => (name === file ? code : readFile(name)); + const program = ts.createProgram([file], options, host); + return ts + .getPreEmitDiagnostics(program, program.getSourceFile(file)) + .filter((d) => d.file?.fileName === file) + .map((d) => { + const line = d.file?.getLineAndCharacterOfPosition(d.start ?? 0).line ?? 0; + const text = ts.flattenDiagnosticMessageText(d.messageText, " "); + return `line ${String(line + 1)}: TS${String(d.code)} ${text}`; + }); +} + /** One fenced code block: its info string split into language and tags, and its body. */ export interface Fence { readonly lang: string; @@ -94,17 +159,19 @@ export function fixturesByContent(root: string, dir: string): Map { rmSync(firstUseTmp, { recursive: true, force: true }); @@ -61,6 +74,27 @@ describe("the quickstart's first example", () => { expect(firstRunnable?.code).toBe(firstBlock?.body); }); + it( + "AC-AS1: compiles in a new TypeScript project against the package's types", + () => { + expect(compileErrors(root, sourcePaths, firstBlock?.body ?? "")).toEqual([]); + }, + COMPILE_TIMEOUT, + ); + + it( + "AC-AS1: a block that does not compile is reported, so it turns this suite red", + () => { + const code = firstBlock?.body ?? ""; + expect(code.split("first?.value;").length - 1).toBe(1); + const mutated = code.replace("first?.value;", "first.value;"); + expect(compileErrors(root, sourcePaths, mutated)).toEqual([ + expect.stringContaining("TS18048"), + ]); + }, + COMPILE_TIMEOUT, + ); + it("AC-AS1: runs against the built package and every claimed value holds", async () => { expect(firstRunnable).toBeDefined(); if (firstRunnable === undefined) return; diff --git a/test/install-specifier.test.ts b/test/install-specifier.test.ts index 652e989..202c23d 100644 --- a/test/install-specifier.test.ts +++ b/test/install-specifier.test.ts @@ -29,4 +29,20 @@ describe("the documented install specifier", () => { ["@cosyte/astn", "@cosyte/astm"], ); }); + + it("AC-AS6: every install command form a reader may copy is read, inline code included", () => { + const forms = [ + "pnpm i @cosyte/astn", + "pnpm install @cosyte/astn", + "npm add @cosyte/astn", + "deno add npm:@cosyte/astn", + "run `npm install @cosyte/astn` first", + "then run npm install @cosyte/astn.", + ]; + for (const form of forms) expect(installSpecifiers(form), form).toEqual(["@cosyte/astn"]); + expect(installSpecifiers("pnpm install\npnpm install --frozen-lockfile")).toEqual([]); + expect( + installSpecifiers("pnpm add file:../astm\nnpm install git+https://x.test/astm.git"), + ).toEqual([]); + }); }); diff --git a/test/readme-usage.test.ts b/test/readme-usage.test.ts index 9f1be49..aee4b28 100644 --- a/test/readme-usage.test.ts +++ b/test/readme-usage.test.ts @@ -45,6 +45,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { + compileErrors, fences as helperFences, fixturesByContent, recordStreamLiteral, @@ -203,6 +204,28 @@ describe("the README Usage example", () => { expect(usageFences[0]?.body).toBe(first?.body); }); + it( + "AC-AS2: compiles in a new TypeScript project against the package's types", + () => { + const paths = { "@cosyte/astm": ENTRY_POINT }; + expect(compileErrors(REPO_ROOT, paths, usageFences[0]?.body ?? "")).toEqual([]); + }, + CASE_TIMEOUT, + ); + + it( + "AC-AS2: a block that does not compile is reported, so it turns this suite red", + () => { + const code = usageFences[0]?.body ?? ""; + expect(code.split("first?.value, ").length - 1).toBe(1); + const mutated = code.replace("first?.value, ", "first.value, "); + expect(compileErrors(REPO_ROOT, { "@cosyte/astm": ENTRY_POINT }, mutated)).toEqual([ + expect.stringContaining("TS18048"), + ]); + }, + CASE_TIMEOUT, + ); + it("AC-AS4: its record stream is a byte-for-byte copy of a fixture under test/fixtures", () => { const stream = recordStreamLiteral(usageFences[0]?.body ?? ""); expect(stream).toBeDefined();