S0329-parser-first-use-path: hold both first examples to a committed fixture and execute them

test/_helpers/first-use.ts

@@ -0,0 +1,177 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { join, relative } from "node:path";
+
+import ts from "typescript";
+
+/**
+ * Readers for the package's first-use path: the first fenced block of
+ * `docs-content/quickstart.md`, the first fenced block under `## Usage` in `README.md`, and the
+ * install command both documents print. Every reader works on the text of the document as it is
+ * on disk at test time, never on a copy held here.
+ */
+
+/**
+ * The compiler options of a new project, as `tsc --init` of the TypeScript this repository pins
+ * writes them: `strict` plus `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes` and
+ * `verbatimModuleSyntax`, NodeNext modules. The emit-only settings it also writes (`sourceMap`,
+ * `declaration`, `declarationMap`, `jsx`) are left out because nothing is emitted, and Node's
+ * types are loaded where `tsc --init` writes `types: []`, because every first-use example is a
+ * Node program.
+ */
+const READER_OPTIONS: ts.CompilerOptions = {
+  module: ts.ModuleKind.NodeNext,
+  moduleResolution: ts.ModuleResolutionKind.NodeNext,
+  target: ts.ScriptTarget.ESNext,
+  strict: true,
+  noUncheckedIndexedAccess: true,
+  exactOptionalPropertyTypes: true,
+  verbatimModuleSyntax: true,
+  isolatedModules: true,
+  noUncheckedSideEffectImports: true,
+  moduleDetection: ts.ModuleDetectionKind.Force,
+  skipLibCheck: true,
+  noEmit: true,
+  types: ["node"],
+};
+
+/**
+ * Every TypeScript error a reader's new project reports for one example, as `line N: TSxxxx
+ * message`; empty when the example compiles. The example is compiled as an ES module sitting in
+ * the repository root, with each specifier in `paths` resolved to the single source file the
+ * bundler compiles into that published entry point and its types. Only the example's own errors
+ * are returned.
+ */
+export function compileErrors(
+  root: string,
+  paths: Readonly<Record<string, string>>,
+  code: string,
+): string[] {
+  const file = join(root, "first-use-example.mts");
+  const options: ts.CompilerOptions = {
+    ...READER_OPTIONS,
+    typeRoots: [join(root, "node_modules", "@types")],
+    paths: Object.fromEntries(Object.entries(paths).map(([pkg, entry]) => [pkg, [entry]])),
+  };
+  const host = ts.createCompilerHost(options, true);
+  const getSourceFile = host.getSourceFile.bind(host);
+  host.getSourceFile = (name, language, onError, create) =>
+    name === file
+      ? ts.createSourceFile(name, code, language, true)
+      : getSourceFile(name, language, onError, create);
+  const fileExists = host.fileExists.bind(host);
+  host.fileExists = (name) => name === file || fileExists(name);
+  const readFile = host.readFile.bind(host);
+  host.readFile = (name) => (name === file ? code : readFile(name));
+  const program = ts.createProgram([file], options, host);
+  return ts
+    .getPreEmitDiagnostics(program, program.getSourceFile(file))
+    .filter((d) => d.file?.fileName === file)
+    .map((d) => {
+      const line = d.file?.getLineAndCharacterOfPosition(d.start ?? 0).line ?? 0;
+      const text = ts.flattenDiagnosticMessageText(d.messageText, " ");
+      return `line ${String(line + 1)}: TS${String(d.code)} ${text}`;
+    });
+}
+
+/** One fenced code block: its info string split into language and tags, and its body. */
+export interface Fence {
+  readonly lang: string;
+  readonly tags: readonly string[];
+  readonly body: string;
+}
+
+/** Every fenced block in a chunk of markdown, in document order. */
+export function fences(markdown: string): Fence[] {
+  const out: Fence[] = [];
+  let open: { marker: string; lang: string; tags: string[] } | undefined;
+  let buf: string[] = [];
+  for (const line of markdown.split("\n")) {
+    if (open === undefined) {
+      const opened = /^(`{3,}|~{3,})\s*(.*)$/.exec(line);
+      if (opened !== null) {
+        const tokens = (opened[2] ?? "").trim().split(/\s+/).filter(Boolean);
+        open = { marker: opened[1] ?? "```", lang: tokens[0] ?? "", tags: tokens.slice(1) };
+        buf = [];
+      }
+      continue;
+    }
+    if (line.trimEnd() === open.marker) {
+      out.push({ lang: open.lang, tags: open.tags, body: buf.join("\n") });
+      open = undefined;
+      continue;
+    }
+    buf.push(line);
+  }
+  if (open !== undefined) throw new Error("unterminated code fence");
+  return out;
+}
+
+/** The lines of one `##` section, from its heading to the next `##` heading. */
+export function section(markdown: string, heading: string): string {
+  const lines = markdown.split("\n");
+  const start = lines.indexOf(heading);
+  if (start === -1) throw new Error(`no ${heading} section`);
+  let end = lines.length;
+  for (let i = start + 1; i < lines.length; i += 1) {
+    if (lines[i]?.startsWith("## ") === true) {
+      end = i;
+      break;
+    }
+  }
+  return lines.slice(start + 1, end).join("\n");
+}
+
+/**
+ * The record stream a first-use example builds: the string literals assigned to the first `const`
+ * that holds one, concatenated. Each literal is decoded by the JSON string grammar, which is the
+ * JavaScript one for every escape these examples use (`\\`, `\r`). `undefined` when there is none.
+ *
+ * The literals are read one at a time with a sticky pattern rather than by one pattern with a
+ * repeated group, which would backtrack exponentially on a long run of empty literals.
+ */
+export function recordStreamLiteral(code: string): string | undefined {
+  const literal = /\s*("(?:[^"\\\n]|\\.)*")[ \t]*(\+)?/y;
+  for (const declared of code.matchAll(/const \w+ =/g)) {
+    literal.lastIndex = declared.index + declared[0].length;
+    const parts: string[] = [];
+    for (let match = literal.exec(code); match !== null; match = literal.exec(code)) {
+      parts.push(JSON.parse(match[1] ?? '""') as string);
+      if (match[2] === undefined) break;
+    }
+    if (parts.length > 0) return parts.join("");
+  }
+  return undefined;
+}
+
+/** Every file under `dir`, keyed by its exact contents, mapped to its path from `root`. */
+export function fixturesByContent(root: string, dir: string): Map<string, string> {
+  const byText = new Map<string, string>();
+  const walk = (current: string): void => {
+    for (const entry of readdirSync(current, { withFileTypes: true })) {
+      const full = join(current, entry.name);
+      if (entry.isDirectory()) walk(full);
+      else byText.set(readFileSync(full, "latin1"), relative(root, full));
+    }
+  };
+  walk(dir);
+  return byText;
+}
+
+/**
+ * The package names a document tells a reader to install: the first non-flag argument of every
+ * `npm install`, `npm i`, `npm add`, `pnpm add`, `pnpm install`, `pnpm i`, `yarn add`, `bun add`
+ * or `deno add npm:`, read as a package name, so any `@version` suffix, closing quote or backtick,
+ * or sentence-ending full stop is left off. A local path or a protocol specifier (`file:../pkg`,
+ * `link:`, `git+https:`) installs whatever that path or URL holds rather than resolving a package
+ * name, so it is not a registry specifier and is not read.
+ */
+export function installSpecifiers(markdown: string): string[] {
+  const out: string[] = [];
+  const command =
+    /\b(?:npm (?:install|add|i)|pnpm (?:add|install|i)|yarn add|bun add|deno add)((?:[ \t]+-{1,2}[\w-]+)*)[ \t]+(?:npm:)?(@?\w[\w.-]*(?:\/[\w.-]+)?)(?![\w.:/+-])/g;
+  for (const match of markdown.matchAll(command)) {
+    const spec = match[2];
+    if (spec !== undefined) out.push(spec.replace(/\.+$/, ""));
+  }
+  return out;
+}

test/docs-content.test.ts

@@ -1,9 +1,21 @@
 import { execFileSync } from "node:child_process";
+import { rmSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 
-import { beforeAll } from "vitest";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
 
-import { docSnippetSuite } from "@cosyte/vitest-config/snippets";
+import {
+  docSnippetSuite,
+  extractRunnableSnippets,
+  runSnippet,
+} from "@cosyte/vitest-config/snippets";
+
+import {
+  compileErrors,
+  fences,
+  fixturesByContent,
+  recordStreamLiteral,
+} from "./_helpers/first-use.js";
 
 /**
  * Doc/code-agreement gate. Every ```` ```ts runnable ```` block in `docs-content/` is extracted,
@@ -18,12 +30,97 @@ import { docSnippetSuite } from "@cosyte/vitest-config/snippets";
  */
 const root = join(import.meta.dirname, "..");
 const distEntry = join(root, "dist", "index.mjs");
+const resolve = (specifier: string): string | undefined =>
+  specifier === "@cosyte/astm" ? distEntry : undefined;
 
 beforeAll(() => {
   execFileSync("pnpm", ["build"], { cwd: root, stdio: "inherit" });
 }, 120_000);
 
 docSnippetSuite({
   docsDir: join(root, "docs-content"),
-  resolve: (specifier) => (specifier === "@cosyte/astm" ? distEntry : undefined),
+  resolve,
+});
+
+/**
+ * The quickstart's FIRST example is the first thing a reader runs from the docs site, so it is held
+ * to more than the sweep above: it must be the block the sweep executes, its record stream must be a
+ * committed synthetic fixture (the page is public, and `test/fixtures` is the corpus `pnpm phi-scan`
+ * reads), and a changed value in it must turn this suite red. Its temp modules live inside the root,
+ * as the harness requires, and are removed when the file is done, so nothing is left on disk.
+ */
+const quickstart = readFileSync(join(root, "docs-content", "quickstart.md"), "utf8");
+const firstBlock = fences(quickstart)[0];
+const firstRunnable = extractRunnableSnippets(quickstart)[0];
+const firstUseTmp = join(root, ".cosyte-first-use-snippets");
+/**
+ * The snippet harness strips types without checking them, so compiling is checked separately, the
+ * way a reader's new TypeScript project compiles the block, against the source entry point the
+ * bundler compiles into the published types. A program over the source takes seconds to check, so
+ * these cases state their own budget.
+ */
+const sourcePaths = { "@cosyte/astm": join(root, "src", "index.ts") };
+const COMPILE_TIMEOUT = 60_000;
+
+afterAll(() => {
+  rmSync(firstUseTmp, { recursive: true, force: true });
+});
+
+describe("the quickstart's first example", () => {
+  it("AC-AS1: is a runnable TypeScript block, so the sweep above executes it", () => {
+    expect(firstBlock?.lang).toBe("ts");
+    expect(firstBlock?.tags).toContain("runnable");
+    expect(firstBlock?.tags).not.toContain("throws");
+    expect(firstRunnable?.code).toBe(firstBlock?.body);
+  });
+
+  it(
+    "AC-AS1: compiles in a new TypeScript project against the package's types",
+    () => {
+      expect(compileErrors(root, sourcePaths, firstBlock?.body ?? "")).toEqual([]);
+    },
+    COMPILE_TIMEOUT,
+  );
+
+  it(
+    "AC-AS1: a block that does not compile is reported, so it turns this suite red",
+    () => {
+      const code = firstBlock?.body ?? "";
+      expect(code.split("first?.value;").length - 1).toBe(1);
+      const mutated = code.replace("first?.value;", "first.value;");
+      expect(compileErrors(root, sourcePaths, mutated)).toEqual([
+        expect.stringContaining("TS18048"),
+      ]);
+    },
+    COMPILE_TIMEOUT,
+  );
+
+  it("AC-AS1: runs against the built package and every claimed value holds", async () => {
+    expect(firstRunnable).toBeDefined();
+    if (firstRunnable === undefined) return;
+    await runSnippet(firstRunnable, { resolve, tmpDir: firstUseTmp });
+  });
+
+  it("AC-AS4: its record stream is a byte-for-byte copy of a fixture under test/fixtures", () => {
+    const stream = recordStreamLiteral(firstRunnable?.code ?? "");
+    expect(stream).toBeDefined();
+    const fixtures = fixturesByContent(root, join(root, "test", "fixtures"));
+    expect(fixtures.get(stream ?? "")).toBeDefined();
+  });
+
+  it("AC-AS3: a changed claimed value turns the run red", async () => {
+    const code = firstRunnable?.code ?? "";
+    expect(code.split('// => "28.6"').length - 1).toBe(1);
+    const mutated = code.replace('// => "28.6"', '// => "31.4"');
+    await expect(runSnippet(mutated, { resolve, tmpDir: firstUseTmp })).rejects.toThrow();
+  });
+
+  it("AC-AS3: a changed input value turns the run red and leaves the fixture corpus", async () => {
+    const code = firstRunnable?.code ?? "";
+    expect(code.split("|28.6|").length - 1).toBe(1);
+    const mutated = code.replace("|28.6|", "|31.4|");
+    const fixtures = fixturesByContent(root, join(root, "test", "fixtures"));
+    expect(fixtures.get(recordStreamLiteral(mutated) ?? "")).toBeUndefined();
+    await expect(runSnippet(mutated, { resolve, tmpDir: firstUseTmp })).rejects.toThrow();
+  });
 });

test/fixtures/tier1-quickstart-result.astm

@@ -0,0 +1 @@
+H|\^&P|1|PRAC|LABO|1|ACCR|1|^^^687|28.6|U/L||N||FL|1|N

test/fixtures/tier1-readme-result.astm

@@ -0,0 +1 @@
+H|\^&|||analyzer^1|||||||P|LIS02-A2|20240115103000P|1|PRACTICE-0001|LAB-0002|||||FO|1|SPEC-7|ACC-42|^^^687|||20240115102500R|1|^^^687|28.6|U/L|10-40|N||FL|1|N

test/install-specifier.test.ts

@@ -0,0 +1,48 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { installSpecifiers } from "./_helpers/first-use.js";
+
+/**
+ * The install command a reader copies has to fetch THIS package. The subject is package identity:
+ * each specifier the two first-use documents print is compared with `package.json` `name`, and a
+ * mismatch names both strings.
+ */
+const root = join(import.meta.dirname, "..");
+const { name } = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { name: string };
+
+describe("the documented install specifier", () => {
+  for (const doc of ["docs-content/installation.md", "README.md"]) {
+    it(`AC-AS6: every install command in ${doc} names package.json name`, () => {
+      const specs = installSpecifiers(readFileSync(join(root, doc), "utf8"));
+      expect(specs.length, `${doc} prints no install command`).toBeGreaterThan(0);
+      for (const spec of specs) {
+        expect(spec, `${doc} installs "${spec}", package.json name is "${name}"`).toBe(name);
+      }
+    });
+  }
+
+  it("AC-AS6: a specifier that is not the package name is read as the name it prints", () => {
+    expect(installSpecifiers("npm install @cosyte/astn\n`pnpm add -D @cosyte/astm@0.0.1`")).toEqual(
+      ["@cosyte/astn", "@cosyte/astm"],
+    );
+  });
+
+  it("AC-AS6: every install command form a reader may copy is read, inline code included", () => {
+    const forms = [
+      "pnpm i @cosyte/astn",
+      "pnpm install @cosyte/astn",
+      "npm add @cosyte/astn",
+      "deno add npm:@cosyte/astn",
+      "run `npm install @cosyte/astn` first",
+      "then run npm install @cosyte/astn.",
+    ];
+    for (const form of forms) expect(installSpecifiers(form), form).toEqual(["@cosyte/astn"]);
+    expect(installSpecifiers("pnpm install\npnpm install --frozen-lockfile")).toEqual([]);
+    expect(
+      installSpecifiers("pnpm add file:../astm\nnpm install git+https://x.test/astm.git"),
+    ).toEqual([]);
+  });
+});