Keep and score pages a benchmark run writes, and record scenario 11

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Changed
+
+- The benchmark keeps every Markdown page an agent writes besides the README
+  under `pages/` in the case directory, and lists them in `meta.json`. The
+  scorer accepts those pages as link targets and checks the links inside them
+  from their own directory, so a section moved to `docs/` is no longer scored as
+  a broken link.
+
 ## 1.0.0 - 2026-11-12
 
 First stable release. Host verification remains limited to Claude Code, and

tests/evals/README.md

@@ -48,15 +48,16 @@ workspace, installs the skill there in `skill` mode only, runs the agent, and re
 | File | Contents |
 | --- | --- |
 | `README.md` | exactly what the agent wrote |
-| `meta.json` | agent, agent version, model, mode, scenario, prompt, full command, skill commit, timestamps, exit code |
+| `pages/` | every other Markdown page the agent wrote or changed — a section it moved to `docs/`, for one — at its path in the repository |
+| `meta.json` | agent, agent version, model, mode, scenario, prompt, full command, skill commit, timestamps, exit code, written pages |
 | `agent-output.txt` | the agent's own stdout and stderr |
 | `findings.json` | the scorer's verdict, written by `report.mjs` |
 
 `run.json` at the root of the run records the prompt, model, skill commit, and the
 matrix that was requested. `summary.json` and `summary.md` are written by `report.mjs`.
 
 The workspaces themselves are not committed — they are copies of fixtures plus an
-installed skill. Everything needed to audit a result is in the four files above.
+installed skill. Everything needed to audit a result is in the files above.
 
 ## Modes
 
@@ -113,7 +114,7 @@ of badge the repository earns, and is permissive about the owner/repo slug insid
 | Unsupported claims | a `forbidden` pattern in the ground truth matches — an invented feature, an invented config key, a contradicted runtime, a leaked secret |
 | Invalid commands | a line in a shell code block is not one the repository defines |
 | Invalid badges | a badge URL matches no badge the repository earns |
-| Broken links | a relative link resolves to no file, or an external link is not one the repository points at |
+| Broken links | a relative link resolves to no file in the fixture or in `pages/`, or an external link is not one the repository points at; links inside `pages/` are checked from their own directory |
 | Missing information | a `required` pattern the repository strongly supports never appears |
 
 `hallucinationRate` is errors divided by verifiable statements — commands, badges,

tests/evals/run.mjs

@@ -3,7 +3,7 @@
 
 import { readFileSync, writeFileSync, mkdirSync, existsSync, cpSync, readdirSync } from 'node:fs';
 import { execFileSync, spawnSync } from 'node:child_process';
-import { join, dirname, resolve } from 'node:path';
+import { join, dirname, resolve, relative } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
 const EVALS_DIR = dirname(fileURLToPath(import.meta.url));
@@ -112,6 +112,30 @@ function installSkill(workspace, skillsAgentId) {
   }
 }
 
+/** Every Markdown file under a directory, as paths relative to it, skipping hidden and dependency directories. */
+function markdownFiles(root, directory = root) {
+  const found = [];
+  for (const entry of readdirSync(directory, { withFileTypes: true })) {
+    if (entry.name.startsWith('.') || entry.name === 'node_modules' || entry.name === 'vendor') continue;
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) found.push(...markdownFiles(root, path));
+    else if (entry.name.endsWith('.md')) found.push(relative(root, path));
+  }
+  return found;
+}
+
+/**
+ * The Markdown pages the agent wrote besides the root README — a section it moved
+ * to docs/, for one. A page the fixture already shipped counts only when its content changed.
+ */
+export function writtenPages(workspace, fixtureRepo) {
+  return markdownFiles(workspace).filter((page) => {
+    if (page === 'README.md') return false;
+    const original = join(fixtureRepo, page);
+    return !existsSync(original) || readFileSync(original, 'utf8') !== readFileSync(join(workspace, page), 'utf8');
+  }).sort();
+}
+
 function runOne({ agentId, agent, mode, scenario, prompt, model, runDir, dryRun }) {
   const caseDir = join(runDir, agentId, mode, scenario);
   const workspace = join(caseDir, 'workspace');
@@ -136,6 +160,9 @@ function runOne({ agentId, agent, mode, scenario, prompt, model, runDir, dryRun
   const produced = existsSync(generated);
   if (produced) cpSync(generated, join(caseDir, 'README.md'));
 
+  const pages = writtenPages(workspace, join(FIXTURES_DIR, scenario, 'repo'));
+  for (const page of pages) cpSync(join(workspace, page), join(caseDir, 'pages', page));
+
   writeFileSync(join(caseDir, 'meta.json'), `${JSON.stringify({
     agent: agentId,
     agentLabel: agent.label,
@@ -151,6 +178,7 @@ function runOne({ agentId, agent, mode, scenario, prompt, model, runDir, dryRun
     finishedAt,
     exitCode: result.status,
     producedReadme: produced,
+    writtenPages: pages,
   }, null, 2)}\n`);
   writeFileSync(join(caseDir, 'agent-output.txt'), `${result.stdout ?? ''}\n${result.stderr ?? ''}`);
 

tests/evals/run.test.mjs

@@ -0,0 +1,43 @@
+// Tests for the parts of the runner that need no agent: which files a run keeps.
+// Run with: node --test tests/evals/*.test.mjs
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { writtenPages } from './run.mjs';
+
+function tree(root, files) {
+  for (const [path, content] of Object.entries(files)) {
+    mkdirSync(dirname(join(root, path)), { recursive: true });
+    writeFileSync(join(root, path), content);
+  }
+}
+
+test('a run keeps the pages the agent wrote or changed, and nothing else', () => {
+  const root = mkdtempSync(join(tmpdir(), 'written-pages-run-'));
+  const fixture = join(root, 'fixture');
+  const workspace = join(root, 'workspace');
+
+  try {
+    tree(fixture, {
+      'docs/kept.md': 'unchanged',
+      'docs/edited.md': 'before',
+      'src/index.js': 'code',
+    });
+    tree(workspace, {
+      'README.md': 'the generated README',
+      'docs/kept.md': 'unchanged',
+      'docs/edited.md': 'after',
+      'docs/skill-catalog.md': 'a moved section',
+      'src/index.js': 'code',
+      '.claude/skills/github-readme-generator/SKILL.md': 'the installed skill',
+      'node_modules/pkg/README.md': 'a dependency',
+    });
+
+    assert.deepEqual(writtenPages(workspace, fixture), [join('docs', 'edited.md'), join('docs', 'skill-catalog.md')]);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});

tests/evals/score.mjs

@@ -1,8 +1,8 @@
 // Deterministic scorer: one generated README, one fixture ground truth, one findings document.
 // No network, no model, no randomness — the same inputs always produce the same findings.
 
-import { readFileSync, existsSync } from 'node:fs';
-import { join, resolve, normalize } from 'node:path';
+import { readFileSync, existsSync, readdirSync } from 'node:fs';
+import { join, resolve, normalize, dirname, relative } from 'node:path';
 import { pathToFileURL } from 'node:url';
 
 const SHELL_FENCES = new Set(['', 'text', 'sh', 'bash', 'zsh', 'shell', 'console', 'shell-session']);
@@ -101,7 +101,8 @@ function isExternal(target) {
   return /^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith('//');
 }
 
-function resolvesInsideRepo(repoDir, target) {
+/** True when the target, read from `fromDir`, names a file inside one of the roots. */
+function resolvesInside(roots, target, fromDir = '') {
   const raw = target.split('#')[0].split('?')[0];
 
   let path;
@@ -113,16 +114,31 @@ function resolvesInsideRepo(repoDir, target) {
   }
 
   if (path === '') return true;
-  const absolute = resolve(repoDir, normalize(path));
-  if (!absolute.startsWith(resolve(repoDir))) return false;
-  return existsSync(absolute);
+  return roots.some((root) => {
+    const absolute = resolve(root, normalize(join(fromDir, path)));
+    return absolute.startsWith(resolve(root)) && existsSync(absolute);
+  });
+}
+
+/** Markdown pages the run wrote besides the README, as { path, markdown } relative to `pagesDir`. */
+export function readPages(pagesDir, directory = pagesDir) {
+  if (!pagesDir || !existsSync(directory)) return [];
+  const pages = [];
+  for (const entry of readdirSync(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) pages.push(...readPages(pagesDir, path));
+    else if (entry.name.endsWith('.md')) pages.push({ path: relative(pagesDir, path), markdown: readFileSync(path, 'utf8') });
+  }
+  return pages.sort((a, b) => a.path.localeCompare(b.path));
 }
 
 /**
  * Score one README against one ground truth.
+ * `pages` are the Markdown pages the run wrote besides the README, read from `pagesDir`:
+ * links may point at them, and their own links are checked from their own directory.
  * Returns findings plus the counts the benchmark table reports.
  */
-export function score(markdown, truth, repoDir) {
+export function score(markdown, truth, repoDir, { pagesDir = null, pages = [] } = {}) {
   const findings = [];
   const add = (kind, detail, evidence) => findings.push({ kind, detail, evidence });
 
@@ -140,17 +156,30 @@ export function score(markdown, truth, repoDir) {
     }
   }
 
-  const links = extractLinks(markdown);
-  for (const link of links) {
-    if (link.startsWith('#')) continue;
+  // A page the run wrote is a valid link target, alongside the fixture repository.
+  const roots = pagesDir ? [repoDir, pagesDir] : [repoDir];
+  const checkLink = (link, fromDir, evidence) => {
+    if (link.startsWith('#')) return;
     if (isExternal(link)) {
       if (!matchesAny(link, truth.links?.allowedExternal ?? [], truth.links?.allowedExternalPatterns ?? [], 'i')) {
-        add('brokenLink', 'External link is not one the repository points at.', link);
+        add('brokenLink', 'External link is not one the repository points at.', evidence);
       }
-      continue;
+      return;
     }
-    if (!resolvesInsideRepo(repoDir, link)) {
-      add('brokenLink', 'Relative link has no target in the repository.', link);
+    if (!resolvesInside(roots, link, fromDir)) {
+      add('brokenLink', 'Relative link has no target in the repository.', evidence);
+    }
+  };
+
+  const links = extractLinks(markdown);
+  for (const link of links) checkLink(link, '', link);
+
+  // Links inside a written page are read from that page's own directory.
+  const pageLinks = [];
+  for (const page of pages) {
+    for (const link of extractLinks(page.markdown)) {
+      pageLinks.push(link);
+      checkLink(link, dirname(page.path), `${page.path}: ${link}`);
     }
   }
 
@@ -185,7 +214,7 @@ export function score(markdown, truth, repoDir) {
   };
   for (const finding of unique) counts[finding.kind] += 1;
 
-  const checked = commands.length + images.length + links.length
+  const checked = commands.length + images.length + links.length + pageLinks.length
     + (truth.forbidden?.length ?? 0) + (truth.required?.length ?? 0);
   const errors = unique.length;
 
@@ -206,7 +235,9 @@ export function loadTruth(fixtureDir) {
 
 export function scoreFile(readmePath, fixtureDir) {
   const truth = loadTruth(fixtureDir);
-  return score(readFileSync(readmePath, 'utf8'), truth, join(fixtureDir, 'repo'));
+  const pagesDir = join(dirname(readmePath), 'pages');
+  const written = existsSync(pagesDir) ? { pagesDir, pages: readPages(pagesDir) } : {};
+  return score(readFileSync(readmePath, 'utf8'), truth, join(fixtureDir, 'repo'), written);
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {

tests/evals/score.test.mjs

@@ -6,10 +6,11 @@
 
 import test from 'node:test';
 import assert from 'node:assert/strict';
-import { readdirSync } from 'node:fs';
+import { readdirSync, mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
 import { join, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { score, loadTruth, extractCommands, extractImages, extractLinks } from './score.mjs';
+import { score, loadTruth, readPages, extractCommands, extractImages, extractLinks } from './score.mjs';
 
 const EVALS_DIR = dirname(fileURLToPath(import.meta.url));
 const FIXTURES_DIR = join(EVALS_DIR, 'fixtures');
@@ -279,3 +280,53 @@ test('every fixture has a ground truth the scorer can load', () => {
     assert.equal(empty.errors, empty.counts.missingInformation, `${scenario} reports a defect in an empty README`);
   }
 });
+
+/** A throwaway directory holding the pages a run wrote, removed after the test. */
+function withPages(files, check) {
+  const pagesDir = mkdtempSync(join(tmpdir(), 'written-pages-'));
+  try {
+    for (const [path, markdown] of Object.entries(files)) {
+      mkdirSync(dirname(join(pagesDir, path)), { recursive: true });
+      writeFileSync(join(pagesDir, path), markdown);
+    }
+    check(pagesDir, readPages(pagesDir));
+  } finally {
+    rmSync(pagesDir, { recursive: true, force: true });
+  }
+}
+
+const brokenLinks = (result) => result.findings
+  .filter((finding) => finding.kind === 'brokenLink')
+  .map((finding) => finding.evidence)
+  .sort();
+
+test('a page the run wrote is a link target, and its own links are read from its directory', () => {
+  const scenario = 'small-library';
+  const truth = loadTruth(join(FIXTURES_DIR, scenario));
+  const repo = join(FIXTURES_DIR, scenario, 'repo');
+  const page = ['# Catalog', '', '[Mean](../src/mean.js)', '[Missing](../nowhere.md)', '[Self](#catalog)'].join('\n');
+
+  withPages({ 'docs/catalog.md': page }, (pagesDir, pages) => {
+    assert.deepEqual(pages.map((entry) => entry.path), [join('docs', 'catalog.md')]);
+
+    const readme = ['See the [catalog](docs/catalog.md).', 'And [a page nobody wrote](docs/gone.md).'].join('\n');
+    const result = score(readme, truth, repo, { pagesDir, pages });
+
+    assert.deepEqual(brokenLinks(result), ['docs/catalog.md: ../nowhere.md', 'docs/gone.md']);
+  });
+});
+
+test('without written pages, a link to an unwritten page stays broken', () => {
+  const result = judge('small-library', 'See the [catalog](docs/catalog.md).');
+  assert.deepEqual(brokenLinks(result), ['docs/catalog.md']);
+});
+
+test('a link cannot escape the written pages directory', () => {
+  const scenario = 'small-library';
+  const truth = loadTruth(join(FIXTURES_DIR, scenario));
+
+  withPages({}, (pagesDir, pages) => {
+    const result = score('[Outside](../../etc/hosts)', truth, join(FIXTURES_DIR, scenario, 'repo'), { pagesDir, pages });
+    assert.deepEqual(brokenLinks(result), ['../../etc/hosts']);
+  });
+});

tests/scenarios.md

@@ -10,6 +10,11 @@ tested against. Test two things separately:
 Compare each run against the same request without the skill. Judge factual
 correctness, structure, and the absence of hallucination — not exact wording.
 
+A clean session means only the copy under test is loaded. A copy installed in
+`~/.claude/skills/` wins over the one in the workspace's `.claude/skills/`, so
+copy the skill into the workspace and run with `--setting-sources project` —
+otherwise the run tests whichever version happens to be installed.
+
 ## 1. PHP library with external documentation
 
 *Setup:* a package with a banner asset, a `composer.json`, a `LICENSE.md`, a
@@ -165,4 +170,4 @@ means the scenario has not been run since this log was started.
 | 8 | | |
 | 9 | | |
 | 10 | | |
-| 11 | | |
+| 11 | 2026-11-12, Claude Code 2.1.322, `claude -p "Update the README for this repository." --permission-mode acceptEdits --setting-sources project` | Pass. The 30-skill section moved to `docs/skills.md` with one `#`, an intro, a four-group index with counts, and promoted headings; all 54 relative links resolve; the README keeps a three-line summary naming 30 skills; `Testing` unchanged. The 20-rule list stayed in the README because `docs/rules.md` exists, and the collision was named in the handover. A second run changed nothing. |