test: cover fail-loud guard error paths in build-disciplines

scripts/build-disciplines.mjs

@@ -46,7 +46,7 @@ const OUT = {
  * two escapes the vendored prompt actually uses: \` and \\). Returns the cooked
  * text and the index just past the closing backtick.
  */
-function readTemplate(src, openIdx) {
+export function readTemplate(src, openIdx) {
   let raw = "";
   let i = openIdx + 1;
   for (; i < src.length; i++) {
@@ -64,7 +64,7 @@ function readTemplate(src, openIdx) {
 }
 
 /** The single template literal returned by `functionName` in prompt.ts. */
-function extractSingleTemplate(src, functionName) {
+export function extractSingleTemplate(src, functionName) {
   const at = src.indexOf(`export function ${functionName}`);
   if (at < 0) throw new Error(`${functionName} not found in prompt.ts`);
   const ret = src.indexOf("return `", at);
@@ -73,7 +73,7 @@ function extractSingleTemplate(src, functionName) {
 }
 
 /** Every template literal returned inside `functionName`, in source order. */
-function extractAllTemplates(src, functionName) {
+export function extractAllTemplates(src, functionName) {
   const at = src.indexOf(`export function ${functionName}`);
   if (at < 0) throw new Error(`${functionName} not found in prompt.ts`);
   // Bound the search to this function body (up to the next top-level export).
@@ -93,7 +93,7 @@ function extractAllTemplates(src, functionName) {
 
 // --- Constant substitution -------------------------------------------------
 
-function readConstants(src) {
+export function readConstants(src) {
   const dir = src.match(/OPEN_WIKI_DIR\s*=\s*"([^"]+)"/);
   if (!dir) throw new Error("OPEN_WIKI_DIR not found in constants.ts");
   const OPEN_WIKI_DIR = dir[1];
@@ -108,7 +108,7 @@ function readConstants(src) {
  * interpolation left over is a loud failure: OpenWiki added a dynamic piece this
  * generator does not understand, so a human must extend it.
  */
-function substitute(text, consts) {
+export function substitute(text, consts) {
   let out = text
     .replaceAll("${OPEN_WIKI_DIR}", consts.OPEN_WIKI_DIR)
     .replaceAll("${UPDATE_METADATA_PATH}", consts.UPDATE_METADATA_PATH)
@@ -232,7 +232,7 @@ const RESIDUAL_VOCAB = [
   "/Users/",
 ];
 
-function assertNoResidualVocab(text, where) {
+export function assertNoResidualVocab(text, where) {
   for (const token of RESIDUAL_VOCAB) {
     if (text.includes(token)) {
       throw new Error(
@@ -297,7 +297,7 @@ const ADAPTED_SECTIONS = {
  * assert the header set is exactly SECTIONS (order included) and that no text is
  * lost — join(all parts) must reproduce the input.
  */
-function splitSections(systemPrompt) {
+export function splitSections(systemPrompt) {
   const lines = systemPrompt.split("\n");
   const headerSet = new Set(SECTIONS.map((s) => s.header));
   const parts = []; // { header|null, lines[] }
@@ -531,10 +531,14 @@ function buildWikiFormat(sections, sha) {
 
 // --- Entry -----------------------------------------------------------------
 
-export function generate() {
-  const promptSrc = readFileSync(PROMPT_TS, "utf8");
-  const constantsSrc = readFileSync(CONSTANTS_TS, "utf8");
-  const provenance = readFileSync(PROVENANCE, "utf8");
+// `sources` lets tests inject crafted prompt/constants/provenance strings to
+// exercise the fail-loud guards (mode-template count, section drift, residual
+// vocab) without a fixture repo. Called with no args in production, it reads the
+// real vendored files, so the drift-lock behavior is unchanged.
+export function generate(sources = {}) {
+  const promptSrc = sources.promptSrc ?? readFileSync(PROMPT_TS, "utf8");
+  const constantsSrc = sources.constantsSrc ?? readFileSync(CONSTANTS_TS, "utf8");
+  const provenance = sources.provenance ?? readFileSync(PROVENANCE, "utf8");
   const sha = provenance.match(/\b([0-9a-f]{40})\b/)?.[1];
   if (!sha) throw new Error("no pinned SHA in vendor/openwiki/PROVENANCE.md");
 

tests/build-disciplines.test.ts

@@ -3,7 +3,12 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { describe, expect, test } from "vitest";
 
-import { generate } from "../scripts/build-disciplines.mjs";
+import {
+  assertNoResidualVocab,
+  generate,
+  splitSections,
+  substitute,
+} from "../scripts/build-disciplines.mjs";
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
 const REPO_ROOT = path.resolve(HERE, "..");
@@ -123,3 +128,164 @@ describe("state-schema.md field set is locked to vendored UpdateMetadata", () =>
     expect([...new Set(documented)].sort()).toEqual(typeFields.sort());
   });
 });
+
+// The drift-lock above only proves the guards do not fire on TODAY's prompt.ts.
+// These tests prove each guard actually THROWS when upstream drifts in the way it
+// is meant to catch — otherwise a silently-broken guard would let a real OpenWiki
+// change slip through with a byte-identical regenerate (see the silent-drop case).
+// They drive the guards through the generator's own exported parsing functions
+// with crafted source strings, so no fixture repo is needed.
+describe("generator guards fail loudly on prompt.ts drift (error paths)", () => {
+  const CONSTS = {
+    OPEN_WIKI_DIR: "openwiki",
+    UPDATE_METADATA_PATH: "openwiki/.last-update.json",
+  };
+
+  // The vendored system-prompt section headers, in order. Kept here (not imported)
+  // so a test that reorders/renames/drops one is expressing the drift explicitly.
+  const SECTION_HEADERS = [
+    "Run discipline:",
+    "Subagent discipline:",
+    "Planning discipline:",
+    "Git discipline:",
+    "Existing documentation discipline:",
+    "Root agent instruction files:",
+    "OpenWiki CLI reference:",
+    "Security and privacy rules:",
+    "Documentation goals:",
+    "Section quality rules:",
+    "Required documentation structure:",
+    "Mode-specific behavior:",
+  ];
+
+  // Build a minimal-but-valid system prompt: an intro then every header in order.
+  // Each header (except the empty "Mode-specific behavior:" placeholder) gets one
+  // benign body line that is deliberately NOT header-shaped (no trailing colon),
+  // so it is never mistaken for a section. `overrides` swaps a header's body,
+  // letting a negative test inject a stray header-shaped line into one bucket.
+  function buildSystemPrompt(
+    headers: string[] = SECTION_HEADERS,
+    overrides: Record<string, string[]> = {},
+  ): string {
+    const lines = ["Intro paragraph one.", "Intro paragraph two.", ""];
+    for (const h of headers) {
+      lines.push(h);
+      const fallback = h === "Mode-specific behavior:" ? [] : ["Guidance for this section."];
+      for (const b of overrides[h] ?? fallback) lines.push(b);
+    }
+    return lines.join("\n");
+  }
+
+  // --- substitute: unhandled-interpolation guard ---
+
+  test("substitute resolves the known ${...} interpolations without throwing", () => {
+    const out = substitute(
+      "dir=${OPEN_WIKI_DIR} meta=${UPDATE_METADATA_PATH} mode=${createModeInstructions(command)}",
+      CONSTS,
+    );
+    expect(out).toBe("dir=openwiki meta=openwiki/.last-update.json mode=");
+  });
+
+  test("substitute throws on an unknown ${...} interpolation (new upstream dynamic piece)", () => {
+    expect(() => substitute("start ${mysteryDynamicPiece} end", CONSTS)).toThrow(
+      /unhandled interpolation in prompt\.ts: \$\{mysteryDynamicPiece\}/,
+    );
+  });
+
+  // --- splitSections: baseline proving the fixture is well-formed ---
+
+  test("splitSections accepts the well-formed fixture (baseline for the negatives)", () => {
+    const { intro, sections } = splitSections(buildSystemPrompt());
+    expect(intro).toBe("Intro paragraph one.\nIntro paragraph two.");
+    expect(sections["Run discipline:"]).toBe("Guidance for this section.");
+    // The mode placeholder must parse as empty — its body is composed elsewhere.
+    expect(sections["Mode-specific behavior:"]).toBe("");
+  });
+
+  // --- splitSections: silent-drop guard (the most important one) ---
+
+  test("splitSections throws when a new section is absorbed into the dropped CLI bucket", () => {
+    // A brand-new upstream header lands inside "OpenWiki CLI reference:" (a bucket
+    // wijzer drops). The known-header set can't see it, so WITHOUT this guard the
+    // new doctrine silently vanishes and a fresh regenerate stays byte-identical —
+    // the drift-lock would stay green while a discipline was lost.
+    const prompt = buildSystemPrompt(SECTION_HEADERS, {
+      "OpenWiki CLI reference:": ["Guidance for this section.", "Telemetry discipline:"],
+    });
+    expect(() => splitSections(prompt)).toThrow(
+      /possible new upstream section "Telemetry discipline:" absorbed into the unrendered "OpenWiki CLI reference:" bucket/,
+    );
+  });
+
+  test("splitSections throws when content follows the Mode-specific behavior placeholder", () => {
+    // Same silent-drop family: new upstream text after the mode interpolation
+    // would be swallowed by the placeholder that wijzer composes itself.
+    const prompt = buildSystemPrompt(SECTION_HEADERS, {
+      "Mode-specific behavior:": ["An extra upstream paragraph."],
+    });
+    expect(() => splitSections(prompt)).toThrow(
+      /content follows 'Mode-specific behavior:' beyond the mode interpolation/,
+    );
+  });
+
+  // --- splitSections: section-header drift guard (renamed / reordered / missing) ---
+
+  test("splitSections throws when a header is renamed", () => {
+    const renamed = SECTION_HEADERS.map((h) =>
+      h === "Git discipline:" ? "Version-control discipline:" : h,
+    );
+    expect(() => splitSections(buildSystemPrompt(renamed))).toThrow(/section headers drifted/);
+  });
+
+  test("splitSections throws when headers are reordered", () => {
+    const reordered = [...SECTION_HEADERS];
+    // Swap "Planning discipline:" and "Git discipline:".
+    [reordered[2], reordered[3]] = [reordered[3], reordered[2]];
+    expect(() => splitSections(buildSystemPrompt(reordered))).toThrow(/section headers drifted/);
+  });
+
+  test("splitSections throws when a header is dropped entirely", () => {
+    const missing = SECTION_HEADERS.filter((h) => h !== "Security and privacy rules:");
+    expect(() => splitSections(buildSystemPrompt(missing))).toThrow(/section headers drifted/);
+  });
+
+  // --- assertNoResidualVocab: residual DeepAgents vocabulary guard ---
+
+  test("assertNoResidualVocab throws when DeepAgents vocabulary survives translation", () => {
+    expect(() =>
+      assertNoResidualVocab("Use read_file to open the page.", "disciplines.md"),
+    ).toThrow(/untranslated DeepAgents vocabulary "read_file" survived in disciplines\.md/);
+  });
+
+  test("assertNoResidualVocab passes when only real Claude Code tools remain", () => {
+    expect(() =>
+      assertNoResidualVocab("Use `Read` and `Write` on repository-relative paths.", "disciplines.md"),
+    ).not.toThrow();
+  });
+
+  // --- generate: mode-template count guard ---
+
+  test("generate throws when createModeInstructions no longer returns exactly 3 templates", () => {
+    // Inject a prompt.ts whose createModeInstructions returns 2 templates (an
+    // upstream mode was added or removed). Everything before the count check must
+    // succeed, so the crafted source also carries a createSystemPrompt template
+    // and the two constants.
+    const promptSrc = [
+      "export function createSystemPrompt(command) {",
+      "  return `A system prompt body without sections.`;",
+      "}",
+      "export function createModeInstructions(command) {",
+      "  return `chat mode text`;",
+      "  return `init mode text`;", // only 2 — the update mode is gone
+      "}",
+    ].join("\n");
+    const constantsSrc = [
+      'export const OPEN_WIKI_DIR = "openwiki";',
+      "export const UPDATE_METADATA_PATH = `${OPEN_WIKI_DIR}/.last-update.json`;",
+    ].join("\n");
+    const provenance = `Pinned at ${"a".repeat(40)}.`;
+    expect(() => generate({ promptSrc, constantsSrc, provenance })).toThrow(
+      /expected 3 mode templates \(chat, init, update\), found 2/,
+    );
+  });
+});