test(notes): add LLM-notes eval harness and prompt snapshot tests

biome.jsonc

@@ -13,7 +13,10 @@
       "!**/dist",
       "!**/out",
       "!.turbo",
-      "!**/node_modules"
+      "!**/node_modules",
+      // Recorded eval fixtures — opaque provider-response blobs in the cache's native (compact) format.
+      // Formatting them would churn on every re-record and diverge from what the harness writes.
+      "!packages/notes/test/eval/fixtures/cache"
     ]
   },
   "formatter": {

packages/notes/src/llm/cache.ts

@@ -33,11 +33,16 @@ export interface CacheIdentity {
  * temperature and maxTokens, and any structured-output schema/tool — so a changed input misses
  * cleanly. Lets a dry-run → `--apply` backfill (or a retried run) reuse prior generations instead of
  * paying the LLM again. Best-effort: any read/parse/write failure falls back to a live call.
+ *
+ * `refresh` skips the read and always calls the provider, writing the result over any existing entry.
+ * Regenerating on purpose needs that: the cache is otherwise read-first, so a stale entry would be
+ * served back forever and the refresh would never happen.
  */
 export function withContentHashCache(
   provider: LLMProvider,
   identity: CacheIdentity,
   dir: string = defaultCacheDir(),
+  { refresh = false }: { refresh?: boolean } = {},
 ): LLMProvider {
   return {
     name: provider.name,
@@ -63,10 +68,12 @@ export function withContentHashCache(
         .digest('hex');
       const file = path.join(dir, `${key}.json`);
 
-      try {
-        return JSON.parse(await fs.readFile(file, 'utf-8')) as CompleteResult;
-      } catch {
-        // Cache miss or unreadable entry — fall through to a live call.
+      if (!refresh) {
+        try {
+          return JSON.parse(await fs.readFile(file, 'utf-8')) as CompleteResult;
+        } catch {
+          // Cache miss or unreadable entry — fall through to a live call.
+        }
       }
 
       const result = await provider.complete(messages, options);

packages/notes/src/llm/tasks/categorize.ts

@@ -18,7 +18,7 @@ import {
   type TaskValidator,
 } from './shared.js';
 
-function buildSystemPrompt(categories: LLMCategory[] | undefined): string {
+export function buildSystemPrompt(categories: LLMCategory[] | undefined): string {
   const categorySection = buildCategorySection(
     categories,
     `Categories: Group into meaningful categories (e.g., "Core", "UI", "API", "Performance", "Bug Fixes", "Documentation").`,

packages/notes/src/llm/tasks/enhance-and-categorize.ts

@@ -25,7 +25,7 @@ export interface CombinedResult {
   categories: CategorizedEntries[];
 }
 
-function buildSystemPrompt(categories: LLMCategory[] | undefined, style: string | undefined): string {
+export function buildSystemPrompt(categories: LLMCategory[] | undefined, style: string | undefined): string {
   const styleText = style || 'Use past tense ("Added feature" not "Add feature"). Be concise.';
 
   const categorySection = buildCategorySection(

packages/notes/src/llm/tasks/enhance.ts

@@ -6,7 +6,7 @@ import type { LLMMessage } from '../messages.js';
 import { resolveSystemPrompt } from '../prompts.js';
 import { INSTRUCTION_HIERARCHY, renderEntry } from './shared.js';
 
-function buildSystemPrompt(style: string | undefined): string {
+export function buildSystemPrompt(style: string | undefined): string {
   const styleText = style ? `- ${style}` : '- Use past tense ("Added feature" not "Add feature")';
   return `You are improving changelog entries for a software project.
 Given a technical commit message, rewrite it as a clear, user-friendly changelog entry.

packages/notes/src/llm/tasks/release-notes.ts

@@ -5,7 +5,7 @@ import type { LLMMessage } from '../messages.js';
 import { resolveSystemPrompt } from '../prompts.js';
 import { INSTRUCTION_HIERARCHY, renderEntries } from './shared.js';
 
-const DEFAULT_SYSTEM_PROMPT = `You are writing release notes for a software project.
+export const DEFAULT_SYSTEM_PROMPT = `You are writing release notes for a software project.
 
 ${INSTRUCTION_HIERARCHY}
 

packages/notes/src/llm/tasks/summarize.ts

@@ -4,7 +4,7 @@ import type { LLMMessage } from '../messages.js';
 import { resolveSystemPrompt } from '../prompts.js';
 import { INSTRUCTION_HIERARCHY, renderEntries } from './shared.js';
 
-const DEFAULT_SYSTEM_PROMPT = `You are creating a summary of changes for a software release.
+export const DEFAULT_SYSTEM_PROMPT = `You are creating a summary of changes for a software release.
 Create a brief summary (2-3 sentences) that captures the main themes of this release.
 
 ${INSTRUCTION_HIERARCHY}

packages/notes/test/eval/assertions.ts

@@ -0,0 +1,52 @@
+/**
+ * Deterministic quality checks for generated release notes. Each returns the list of violations it
+ * found (empty = clean), so the spec can assert `toEqual([])` and surface the offending lines on
+ * failure. Kept pure and content-based so they hold for both replayed fixtures and live-model output.
+ */
+
+/** Internal HTML markers (`<!-- releasekit-… -->`) must never leak into user-facing notes. */
+export function findMarkerLeaks(text: string): string[] {
+  return text.match(/<!--\s*releasekit-[^>]*-->/g) ?? [];
+}
+
+/** A model that restates "Updated dependencies" per bump produces noisy, duplicated churn lines. */
+export function findDuplicateDependencyChurn(text: string): string[] {
+  const lines = text.split('\n').filter((line) => /updated dependencies/i.test(line));
+  return lines.length > 1 ? lines : [];
+}
+
+/** Guard against a truncated (empty/near-empty) or runaway generation. */
+export function checkLengthBounds(text: string, min: number, max: number): string[] {
+  const n = text.trim().length;
+  if (n < min) return [`output too short: ${n} < ${min} chars`];
+  if (n > max) return [`output too long: ${n} > ${max} chars`];
+  return [];
+}
+
+const IRREGULAR_PAST = new Set(['made', 'built', 'brought', 'rewrote', 'drove', 'took', 'gave', 'kept', 'shipped']);
+
+function looksPastTense(word: string): boolean {
+  const w = word.toLowerCase().replace(/[^a-z]/g, '');
+  return w.endsWith('ed') || IRREGULAR_PAST.has(w);
+}
+
+/** Markdown list-item lines — bullets (`- …`, `* …`) and ordered items (`1. …`). */
+export function listItemLines(text: string): string[] {
+  return text.split('\n').filter((line) => /^\s*(?:[-*]|\d+\.)\s+\S/.test(line));
+}
+
+/**
+ * The style guides all mandate past tense ("Added feature", not "Add feature"). Check that a majority
+ * of list items lead with a past-tense verb — a ratio, not all-or-nothing, since an item may
+ * legitimately open with a proper noun or an API name. Both bullet and numbered lists are inspected,
+ * so a model that switches to a numbered list can't bypass the check. Pure prose (no list items) has
+ * no reliable per-change verb to inspect, so it is not tense-checked here.
+ */
+export function checkPastTenseLeaning(text: string, minRatio = 0.6): string[] {
+  const items = listItemLines(text);
+  if (items.length === 0) return [];
+  const firstWords = items.map((line) => line.replace(/^\s*(?:[-*]|\d+\.)\s+/, '').split(/\s+/)[0] ?? '');
+  const pastCount = firstWords.filter(looksPastTense).length;
+  const ratio = pastCount / items.length;
+  return ratio >= minRatio ? [] : [`only ${pastCount}/${items.length} items lead past-tense (< ${minRatio})`];
+}

packages/notes/test/eval/fixtures/cache/203073b4cd17409091bd9e90fdfdb46b681cac7015dfa98ae70ea09cc081c3fc.json

@@ -0,0 +1 @@
+{"content":"## What's new in 2.0.0\n\nThis release brought real-time capabilities to the widget and smoothed over a few rough edges since 1.4.2.\n\n### Added\n- Added a streaming API for real-time updates.\n- Added deeplink support for the mobile client.\n\n### Fixed\n- Fixed a crash that occurred when the config file was missing.\n\n### Changed\n- Improved error messages for invalid configuration.\n- Updated dependencies.\n\nThanks to everyone who contributed to this release!"}

packages/notes/test/eval/fixtures/release-notes-basic.json

@@ -0,0 +1,30 @@
+{
+  "context": {
+    "packageName": "@acme/widget",
+    "version": "2.0.0",
+    "previousVersion": "1.4.2",
+    "date": "2026-01-01"
+  },
+  "entries": [
+    {
+      "type": "added",
+      "description": "Add a streaming API for real-time updates"
+    },
+    {
+      "type": "added",
+      "description": "Add deeplink support for the mobile client"
+    },
+    {
+      "type": "fixed",
+      "description": "Fix a crash when the config file is missing"
+    },
+    {
+      "type": "changed",
+      "description": "Improve error messages for invalid configuration"
+    },
+    {
+      "type": "changed",
+      "description": "Update dependencies"
+    }
+  ]
+}

packages/notes/test/eval/fixtures/release-notes-basic.recorded.md

@@ -0,0 +1,18 @@
+<!-- recorded: hand-authored -->
+
+## What's new in 2.0.0
+
+This release brought real-time capabilities to the widget and smoothed over a few rough edges since 1.4.2.
+
+### Added
+- Added a streaming API for real-time updates.
+- Added deeplink support for the mobile client.
+
+### Fixed
+- Fixed a crash that occurred when the config file was missing.
+
+### Changed
+- Improved error messages for invalid configuration.
+- Updated dependencies.
+
+Thanks to everyone who contributed to this release!

packages/notes/test/eval/harness.ts

@@ -0,0 +1,134 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import type { ChangelogEntry } from '../../src/core/types.js';
+import { type CacheIdentity, withContentHashCache } from '../../src/llm/cache.js';
+import type { ReleaseNotesContext } from '../../src/llm/index.js';
+import type { CompleteResult, LLMMessage, LLMProvider, ProviderCapabilities } from '../../src/llm/provider.js';
+
+/**
+ * Eval harness for the LLM-notes pipeline. Golden commit sets run through the real pipeline against
+ * recorded provider responses, replayed deterministically in CI via the existing on-disk cache format
+ * ({@link withContentHashCache}) — no API keys needed. The value is in the deterministic assertions:
+ * a prompt or post-processing regression changes the output, and a changed prompt busts the cache key,
+ * so a missing fixture is itself the signal to re-record.
+ *
+ * Modes (env):
+ *   default              strict replay from the committed cache; a cache miss fails loudly.
+ *   RELEASEKIT_EVAL_RECORD=1   seed the cache from the human-readable `*.recorded.md` fixtures (no model).
+ *   RELEASEKIT_EVAL=1          run a real provider (Ollama), recording fresh responses into the cache
+ *                              and back into `*.recorded.md`.
+ *
+ * The recording modes pass `refresh` so the cache writes through instead of serving the entry it is
+ * meant to replace. They rewrite the entries the cases they run produce, and nothing else — changing
+ * a prompt therefore leaves the old entry behind under its now-unreachable key. To re-record from a
+ * clean slate, delete the fixture cache first:
+ *
+ *   rm -rf packages/notes/test/eval/fixtures/cache
+ *   RELEASEKIT_EVAL_RECORD=1 pnpm --filter @releasekit/notes test
+ */
+
+export const EVAL_DIR = fileURLToPath(new URL('.', import.meta.url));
+export const CACHE_DIR = fileURLToPath(new URL('./fixtures/cache', import.meta.url));
+
+// Fixed across record and replay: the cache key folds in provider name + identity, so both must be
+// stable regardless of which underlying model (canned, offline, or a real Ollama) produced a response.
+const EVAL_PROVIDER_NAME = 'eval';
+export const EVAL_IDENTITY: CacheIdentity = { model: 'eval-fixture' };
+
+/**
+ * Every mode reports these, the real provider included — capabilities are not free-floating metadata.
+ * A task consults them to decide whether to send a structured-output `schema`/`toolName`, and both are
+ * part of the cache key, so a provider that advertises different capabilities keys the same golden
+ * input differently. Ollama advertises `structuredOutputs: true`; letting that through would mean a
+ * live recording of a structured case could never be found by the replay that has to read it back.
+ */
+export const CAPABILITIES: ProviderCapabilities = { systemRole: true, structuredOutputs: false, toolUse: false };
+
+export const isLiveMode = process.env.RELEASEKIT_EVAL === '1' || process.env.RELEASEKIT_EVAL === 'true';
+export const isRecordMode = process.env.RELEASEKIT_EVAL_RECORD === '1' || process.env.RELEASEKIT_EVAL_RECORD === 'true';
+
+const strictOfflineProvider: LLMProvider = {
+  name: EVAL_PROVIDER_NAME,
+  capabilities: CAPABILITIES,
+  async complete(): Promise<CompleteResult> {
+    throw new Error(
+      'eval replay: no recorded fixture for this request. The prompt or golden input changed — ' +
+        're-record with RELEASEKIT_EVAL_RECORD=1 (from *.recorded.md) or RELEASEKIT_EVAL=1 (live provider).',
+    );
+  },
+};
+
+/** Base provider that returns a fixed canned response, used to seed fixtures from `*.recorded.md`. */
+function cannedProvider(content: string): LLMProvider {
+  return {
+    name: EVAL_PROVIDER_NAME,
+    capabilities: CAPABILITIES,
+    async complete(): Promise<CompleteResult> {
+      return { content };
+    },
+  };
+}
+
+/**
+ * Re-brand a real provider under the fixed eval name and capabilities so its recorded responses key
+ * like the rest, and tee each response back into `*.recorded.md` so the human-readable fixture and
+ * the cache entry stay the same generation — with the model that produced it recorded alongside.
+ */
+export function asEvalProvider(base: LLMProvider, caseName: string, provenance: string): LLMProvider {
+  return {
+    name: EVAL_PROVIDER_NAME,
+    capabilities: CAPABILITIES,
+    async complete(messages: LLMMessage[], options): Promise<CompleteResult> {
+      const result = await base.complete(messages, options);
+      writeRecorded(caseName, result.content, provenance);
+      return result;
+    },
+  };
+}
+
+export interface GoldenCase {
+  entries: ChangelogEntry[];
+  context: ReleaseNotesContext;
+}
+
+/** Load a golden input fixture (a real-shaped commit set with a fixed date, so cache keys are stable). */
+export function loadGoldenCase(name: string): GoldenCase {
+  const path = fileURLToPath(new URL(`./fixtures/${name}.json`, import.meta.url));
+  return JSON.parse(readFileSync(path, 'utf-8')) as GoldenCase;
+}
+
+function recordedPath(name: string): string {
+  return fileURLToPath(new URL(`./fixtures/${name}.recorded.md`, import.meta.url));
+}
+
+// Provenance header stamped on a recording. The assertions' calibration (length bounds, tense ratio)
+// is implicitly tuned to whatever produced the fixture, and the cache key deliberately pins the model
+// to EVAL_IDENTITY so replay works — which leaves nothing else recording what actually generated it.
+const PROVENANCE_HEADER = /^<!--\s*recorded:.*?-->\n+/;
+
+function loadRecorded(name: string): string {
+  return readFileSync(recordedPath(name), 'utf-8').replace(PROVENANCE_HEADER, '').trimEnd();
+}
+
+function writeRecorded(name: string, content: string, provenance: string): void {
+  writeFileSync(recordedPath(name), `<!-- recorded: ${provenance} -->\n\n${content.trimEnd()}\n`);
+}
+
+/**
+ * The provider each eval case runs through. Live mode wraps a real Ollama provider (recording as it
+ * goes); record mode seeds from the committed markdown; default mode replays strictly from the cache.
+ * Every mode wraps {@link withContentHashCache} so the on-disk format is identical across them.
+ */
+export async function evalProvider(caseName: string): Promise<LLMProvider> {
+  if (isLiveMode) {
+    const { OllamaProvider } = await import('../../src/llm/ollama.js');
+    const model = process.env.RELEASEKIT_EVAL_MODEL ?? 'llama3.2';
+    const baseURL = process.env.OLLAMA_BASE_URL;
+    const live = asEvalProvider(new OllamaProvider({ model, baseURL }), caseName, `ollama/${model}`);
+    return withContentHashCache(live, EVAL_IDENTITY, CACHE_DIR, { refresh: true });
+  }
+  if (isRecordMode) {
+    return withContentHashCache(cannedProvider(loadRecorded(caseName)), EVAL_IDENTITY, CACHE_DIR, { refresh: true });
+  }
+  return withContentHashCache(strictOfflineProvider, EVAL_IDENTITY, CACHE_DIR);
+}

packages/notes/test/eval/notes-eval.spec.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest';
+import type { CompleteResult, LLMProvider } from '../../src/llm/provider.js';
+import { generateReleaseNotes } from '../../src/llm/tasks/release-notes.js';
+import {
+  checkLengthBounds,
+  checkPastTenseLeaning,
+  findDuplicateDependencyChurn,
+  findMarkerLeaks,
+} from './assertions.js';
+import { asEvalProvider, CAPABILITIES, evalProvider, isLiveMode, loadGoldenCase } from './harness.js';
+
+/**
+ * Golden-fixture eval for the LLM-notes pipeline. Runs a real commit set through the real pipeline and
+ * checks the output against deterministic quality assertions. By default the provider response is
+ * replayed from a committed cache fixture (no keys, deterministic); with RELEASEKIT_EVAL=1 it runs a
+ * live model, and with RELEASEKIT_EVAL_RECORD=1 it re-seeds the fixture from the recorded markdown.
+ * The assertions are the regression net — they hold whether the response is replayed or freshly
+ * generated, so a prompt or post-processing change that degrades output is caught.
+ */
+describe('notes eval: release notes', () => {
+  it(
+    'should generate clean release notes for the basic golden case',
+    async () => {
+      const golden = loadGoldenCase('release-notes-basic');
+      const provider = await evalProvider('release-notes-basic');
+
+      const notes = await generateReleaseNotes(provider, golden.entries, golden.context);
+
+      expect(findMarkerLeaks(notes)).toEqual([]);
+      expect(findDuplicateDependencyChurn(notes)).toEqual([]);
+      expect(checkLengthBounds(notes, 80, 4000)).toEqual([]);
+      expect(checkPastTenseLeaning(notes)).toEqual([]);
+    },
+    isLiveMode ? 180_000 : 10_000,
+  );
+
+  // Guards the one thing that makes record→replay work at all. Capabilities decide whether a task
+  // sends a structured-output schema/toolName, and those are part of the cache key — so a live
+  // recording made under the real provider's capabilities would key differently from the replay that
+  // has to read it back. Ollama advertises structuredOutputs: true, so this fails the moment the
+  // wrapper passes the base provider's capabilities through instead of the harness's fixed set.
+  it('should report the fixed eval capabilities regardless of what the live provider advertises', () => {
+    const ollamaLike: LLMProvider = {
+      name: 'ollama',
+      capabilities: { systemRole: true, structuredOutputs: true, toolUse: false, honorsTemperature: true },
+      complete: async (): Promise<CompleteResult> => ({ content: '' }),
+    };
+
+    expect(asEvalProvider(ollamaLike, 'unused', 'test').capabilities).toEqual(CAPABILITIES);
+  });
+});