diff --git a/scripts/build-disciplines.mjs b/scripts/build-disciplines.mjs index 2f13afc..884efc1 100644 --- a/scripts/build-disciplines.mjs +++ b/scripts/build-disciplines.mjs @@ -46,7 +46,7 @@ const OUT = { * two escapes the vendored prompt actually uses: \` and \\). Returns the cooked * text and the index just past the closing backtick. */ -function readTemplate(src, openIdx) { +export function readTemplate(src, openIdx) { let raw = ""; let i = openIdx + 1; for (; i < src.length; i++) { @@ -64,7 +64,7 @@ function readTemplate(src, openIdx) { } /** The single template literal returned by `functionName` in prompt.ts. */ -function extractSingleTemplate(src, functionName) { +export function extractSingleTemplate(src, functionName) { const at = src.indexOf(`export function ${functionName}`); if (at < 0) throw new Error(`${functionName} not found in prompt.ts`); const ret = src.indexOf("return `", at); @@ -73,7 +73,7 @@ function extractSingleTemplate(src, functionName) { } /** Every template literal returned inside `functionName`, in source order. */ -function extractAllTemplates(src, functionName) { +export function extractAllTemplates(src, functionName) { const at = src.indexOf(`export function ${functionName}`); if (at < 0) throw new Error(`${functionName} not found in prompt.ts`); // Bound the search to this function body (up to the next top-level export). @@ -93,7 +93,7 @@ function extractAllTemplates(src, functionName) { // --- Constant substitution ------------------------------------------------- -function readConstants(src) { +export function readConstants(src) { const dir = src.match(/OPEN_WIKI_DIR\s*=\s*"([^"]+)"/); if (!dir) throw new Error("OPEN_WIKI_DIR not found in constants.ts"); const OPEN_WIKI_DIR = dir[1]; @@ -108,7 +108,7 @@ function readConstants(src) { * interpolation left over is a loud failure: OpenWiki added a dynamic piece this * generator does not understand, so a human must extend it. */ -function substitute(text, consts) { +export function substitute(text, consts) { let out = text .replaceAll("${OPEN_WIKI_DIR}", consts.OPEN_WIKI_DIR) .replaceAll("${UPDATE_METADATA_PATH}", consts.UPDATE_METADATA_PATH) @@ -232,7 +232,7 @@ const RESIDUAL_VOCAB = [ "/Users/", ]; -function assertNoResidualVocab(text, where) { +export function assertNoResidualVocab(text, where) { for (const token of RESIDUAL_VOCAB) { if (text.includes(token)) { throw new Error( @@ -297,7 +297,7 @@ const ADAPTED_SECTIONS = { * assert the header set is exactly SECTIONS (order included) and that no text is * lost — join(all parts) must reproduce the input. */ -function splitSections(systemPrompt) { +export function splitSections(systemPrompt) { const lines = systemPrompt.split("\n"); const headerSet = new Set(SECTIONS.map((s) => s.header)); const parts = []; // { header|null, lines[] } @@ -531,10 +531,14 @@ function buildWikiFormat(sections, sha) { // --- Entry ----------------------------------------------------------------- -export function generate() { - const promptSrc = readFileSync(PROMPT_TS, "utf8"); - const constantsSrc = readFileSync(CONSTANTS_TS, "utf8"); - const provenance = readFileSync(PROVENANCE, "utf8"); +// `sources` lets tests inject crafted prompt/constants/provenance strings to +// exercise the fail-loud guards (mode-template count, section drift, residual +// vocab) without a fixture repo. Called with no args in production, it reads the +// real vendored files, so the drift-lock behavior is unchanged. +export function generate(sources = {}) { + const promptSrc = sources.promptSrc ?? readFileSync(PROMPT_TS, "utf8"); + const constantsSrc = sources.constantsSrc ?? readFileSync(CONSTANTS_TS, "utf8"); + const provenance = sources.provenance ?? readFileSync(PROVENANCE, "utf8"); const sha = provenance.match(/\b([0-9a-f]{40})\b/)?.[1]; if (!sha) throw new Error("no pinned SHA in vendor/openwiki/PROVENANCE.md"); diff --git a/tests/build-disciplines.test.ts b/tests/build-disciplines.test.ts index a4d7542..d0ecf13 100644 --- a/tests/build-disciplines.test.ts +++ b/tests/build-disciplines.test.ts @@ -3,7 +3,12 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, test } from "vitest"; -import { generate } from "../scripts/build-disciplines.mjs"; +import { + assertNoResidualVocab, + generate, + splitSections, + substitute, +} from "../scripts/build-disciplines.mjs"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(HERE, ".."); @@ -123,3 +128,164 @@ describe("state-schema.md field set is locked to vendored UpdateMetadata", () => expect([...new Set(documented)].sort()).toEqual(typeFields.sort()); }); }); + +// The drift-lock above only proves the guards do not fire on TODAY's prompt.ts. +// These tests prove each guard actually THROWS when upstream drifts in the way it +// is meant to catch — otherwise a silently-broken guard would let a real OpenWiki +// change slip through with a byte-identical regenerate (see the silent-drop case). +// They drive the guards through the generator's own exported parsing functions +// with crafted source strings, so no fixture repo is needed. +describe("generator guards fail loudly on prompt.ts drift (error paths)", () => { + const CONSTS = { + OPEN_WIKI_DIR: "openwiki", + UPDATE_METADATA_PATH: "openwiki/.last-update.json", + }; + + // The vendored system-prompt section headers, in order. Kept here (not imported) + // so a test that reorders/renames/drops one is expressing the drift explicitly. + const SECTION_HEADERS = [ + "Run discipline:", + "Subagent discipline:", + "Planning discipline:", + "Git discipline:", + "Existing documentation discipline:", + "Root agent instruction files:", + "OpenWiki CLI reference:", + "Security and privacy rules:", + "Documentation goals:", + "Section quality rules:", + "Required documentation structure:", + "Mode-specific behavior:", + ]; + + // Build a minimal-but-valid system prompt: an intro then every header in order. + // Each header (except the empty "Mode-specific behavior:" placeholder) gets one + // benign body line that is deliberately NOT header-shaped (no trailing colon), + // so it is never mistaken for a section. `overrides` swaps a header's body, + // letting a negative test inject a stray header-shaped line into one bucket. + function buildSystemPrompt( + headers: string[] = SECTION_HEADERS, + overrides: Record = {}, + ): string { + const lines = ["Intro paragraph one.", "Intro paragraph two.", ""]; + for (const h of headers) { + lines.push(h); + const fallback = h === "Mode-specific behavior:" ? [] : ["Guidance for this section."]; + for (const b of overrides[h] ?? fallback) lines.push(b); + } + return lines.join("\n"); + } + + // --- substitute: unhandled-interpolation guard --- + + test("substitute resolves the known ${...} interpolations without throwing", () => { + const out = substitute( + "dir=${OPEN_WIKI_DIR} meta=${UPDATE_METADATA_PATH} mode=${createModeInstructions(command)}", + CONSTS, + ); + expect(out).toBe("dir=openwiki meta=openwiki/.last-update.json mode="); + }); + + test("substitute throws on an unknown ${...} interpolation (new upstream dynamic piece)", () => { + expect(() => substitute("start ${mysteryDynamicPiece} end", CONSTS)).toThrow( + /unhandled interpolation in prompt\.ts: \$\{mysteryDynamicPiece\}/, + ); + }); + + // --- splitSections: baseline proving the fixture is well-formed --- + + test("splitSections accepts the well-formed fixture (baseline for the negatives)", () => { + const { intro, sections } = splitSections(buildSystemPrompt()); + expect(intro).toBe("Intro paragraph one.\nIntro paragraph two."); + expect(sections["Run discipline:"]).toBe("Guidance for this section."); + // The mode placeholder must parse as empty — its body is composed elsewhere. + expect(sections["Mode-specific behavior:"]).toBe(""); + }); + + // --- splitSections: silent-drop guard (the most important one) --- + + test("splitSections throws when a new section is absorbed into the dropped CLI bucket", () => { + // A brand-new upstream header lands inside "OpenWiki CLI reference:" (a bucket + // wijzer drops). The known-header set can't see it, so WITHOUT this guard the + // new doctrine silently vanishes and a fresh regenerate stays byte-identical — + // the drift-lock would stay green while a discipline was lost. + const prompt = buildSystemPrompt(SECTION_HEADERS, { + "OpenWiki CLI reference:": ["Guidance for this section.", "Telemetry discipline:"], + }); + expect(() => splitSections(prompt)).toThrow( + /possible new upstream section "Telemetry discipline:" absorbed into the unrendered "OpenWiki CLI reference:" bucket/, + ); + }); + + test("splitSections throws when content follows the Mode-specific behavior placeholder", () => { + // Same silent-drop family: new upstream text after the mode interpolation + // would be swallowed by the placeholder that wijzer composes itself. + const prompt = buildSystemPrompt(SECTION_HEADERS, { + "Mode-specific behavior:": ["An extra upstream paragraph."], + }); + expect(() => splitSections(prompt)).toThrow( + /content follows 'Mode-specific behavior:' beyond the mode interpolation/, + ); + }); + + // --- splitSections: section-header drift guard (renamed / reordered / missing) --- + + test("splitSections throws when a header is renamed", () => { + const renamed = SECTION_HEADERS.map((h) => + h === "Git discipline:" ? "Version-control discipline:" : h, + ); + expect(() => splitSections(buildSystemPrompt(renamed))).toThrow(/section headers drifted/); + }); + + test("splitSections throws when headers are reordered", () => { + const reordered = [...SECTION_HEADERS]; + // Swap "Planning discipline:" and "Git discipline:". + [reordered[2], reordered[3]] = [reordered[3], reordered[2]]; + expect(() => splitSections(buildSystemPrompt(reordered))).toThrow(/section headers drifted/); + }); + + test("splitSections throws when a header is dropped entirely", () => { + const missing = SECTION_HEADERS.filter((h) => h !== "Security and privacy rules:"); + expect(() => splitSections(buildSystemPrompt(missing))).toThrow(/section headers drifted/); + }); + + // --- assertNoResidualVocab: residual DeepAgents vocabulary guard --- + + test("assertNoResidualVocab throws when DeepAgents vocabulary survives translation", () => { + expect(() => + assertNoResidualVocab("Use read_file to open the page.", "disciplines.md"), + ).toThrow(/untranslated DeepAgents vocabulary "read_file" survived in disciplines\.md/); + }); + + test("assertNoResidualVocab passes when only real Claude Code tools remain", () => { + expect(() => + assertNoResidualVocab("Use `Read` and `Write` on repository-relative paths.", "disciplines.md"), + ).not.toThrow(); + }); + + // --- generate: mode-template count guard --- + + test("generate throws when createModeInstructions no longer returns exactly 3 templates", () => { + // Inject a prompt.ts whose createModeInstructions returns 2 templates (an + // upstream mode was added or removed). Everything before the count check must + // succeed, so the crafted source also carries a createSystemPrompt template + // and the two constants. + const promptSrc = [ + "export function createSystemPrompt(command) {", + " return `A system prompt body without sections.`;", + "}", + "export function createModeInstructions(command) {", + " return `chat mode text`;", + " return `init mode text`;", // only 2 — the update mode is gone + "}", + ].join("\n"); + const constantsSrc = [ + 'export const OPEN_WIKI_DIR = "openwiki";', + "export const UPDATE_METADATA_PATH = `${OPEN_WIKI_DIR}/.last-update.json`;", + ].join("\n"); + const provenance = `Pinned at ${"a".repeat(40)}.`; + expect(() => generate({ promptSrc, constantsSrc, provenance })).toThrow( + /expected 3 mode templates \(chat, init, update\), found 2/, + ); + }); +});