From d06cd56c7f59607262fddc14bd930df53538dacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Thu, 12 Nov 2026 13:57:23 +0100 Subject: [PATCH 1/2] Keep and score the pages a benchmark run writes besides the README --- CHANGELOG.md | 8 +++++ tests/evals/README.md | 7 +++-- tests/evals/run.mjs | 30 +++++++++++++++++- tests/evals/run.test.mjs | 43 ++++++++++++++++++++++++++ tests/evals/score.mjs | 63 ++++++++++++++++++++++++++++---------- tests/evals/score.test.mjs | 55 +++++++++++++++++++++++++++++++-- 6 files changed, 184 insertions(+), 22 deletions(-) create mode 100644 tests/evals/run.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2518cee..ec6dd9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed + +- The benchmark keeps every Markdown page an agent writes besides the README + under `pages/` in the case directory, and lists them in `meta.json`. The + scorer accepts those pages as link targets and checks the links inside them + from their own directory, so a section moved to `docs/` is no longer scored as + a broken link. + ## 1.0.0 - 2026-11-12 First stable release. Host verification remains limited to Claude Code, and diff --git a/tests/evals/README.md b/tests/evals/README.md index a68a16d..38c9092 100644 --- a/tests/evals/README.md +++ b/tests/evals/README.md @@ -48,7 +48,8 @@ workspace, installs the skill there in `skill` mode only, runs the agent, and re | File | Contents | | --- | --- | | `README.md` | exactly what the agent wrote | -| `meta.json` | agent, agent version, model, mode, scenario, prompt, full command, skill commit, timestamps, exit code | +| `pages/` | every other Markdown page the agent wrote or changed — a section it moved to `docs/`, for one — at its path in the repository | +| `meta.json` | agent, agent version, model, mode, scenario, prompt, full command, skill commit, timestamps, exit code, written pages | | `agent-output.txt` | the agent's own stdout and stderr | | `findings.json` | the scorer's verdict, written by `report.mjs` | @@ -56,7 +57,7 @@ workspace, installs the skill there in `skill` mode only, runs the agent, and re matrix that was requested. `summary.json` and `summary.md` are written by `report.mjs`. The workspaces themselves are not committed — they are copies of fixtures plus an -installed skill. Everything needed to audit a result is in the four files above. +installed skill. Everything needed to audit a result is in the files above. ## Modes @@ -113,7 +114,7 @@ of badge the repository earns, and is permissive about the owner/repo slug insid | Unsupported claims | a `forbidden` pattern in the ground truth matches — an invented feature, an invented config key, a contradicted runtime, a leaked secret | | Invalid commands | a line in a shell code block is not one the repository defines | | Invalid badges | a badge URL matches no badge the repository earns | -| Broken links | a relative link resolves to no file, or an external link is not one the repository points at | +| Broken links | a relative link resolves to no file in the fixture or in `pages/`, or an external link is not one the repository points at; links inside `pages/` are checked from their own directory | | Missing information | a `required` pattern the repository strongly supports never appears | `hallucinationRate` is errors divided by verifiable statements — commands, badges, diff --git a/tests/evals/run.mjs b/tests/evals/run.mjs index 89d7b52..864c6fe 100644 --- a/tests/evals/run.mjs +++ b/tests/evals/run.mjs @@ -3,7 +3,7 @@ import { readFileSync, writeFileSync, mkdirSync, existsSync, cpSync, readdirSync } from 'node:fs'; import { execFileSync, spawnSync } from 'node:child_process'; -import { join, dirname, resolve } from 'node:path'; +import { join, dirname, resolve, relative } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; const EVALS_DIR = dirname(fileURLToPath(import.meta.url)); @@ -112,6 +112,30 @@ function installSkill(workspace, skillsAgentId) { } } +/** Every Markdown file under a directory, as paths relative to it, skipping hidden and dependency directories. */ +function markdownFiles(root, directory = root) { + const found = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.name.startsWith('.') || entry.name === 'node_modules' || entry.name === 'vendor') continue; + const path = join(directory, entry.name); + if (entry.isDirectory()) found.push(...markdownFiles(root, path)); + else if (entry.name.endsWith('.md')) found.push(relative(root, path)); + } + return found; +} + +/** + * The Markdown pages the agent wrote besides the root README — a section it moved + * to docs/, for one. A page the fixture already shipped counts only when its content changed. + */ +export function writtenPages(workspace, fixtureRepo) { + return markdownFiles(workspace).filter((page) => { + if (page === 'README.md') return false; + const original = join(fixtureRepo, page); + return !existsSync(original) || readFileSync(original, 'utf8') !== readFileSync(join(workspace, page), 'utf8'); + }).sort(); +} + function runOne({ agentId, agent, mode, scenario, prompt, model, runDir, dryRun }) { const caseDir = join(runDir, agentId, mode, scenario); const workspace = join(caseDir, 'workspace'); @@ -136,6 +160,9 @@ function runOne({ agentId, agent, mode, scenario, prompt, model, runDir, dryRun const produced = existsSync(generated); if (produced) cpSync(generated, join(caseDir, 'README.md')); + const pages = writtenPages(workspace, join(FIXTURES_DIR, scenario, 'repo')); + for (const page of pages) cpSync(join(workspace, page), join(caseDir, 'pages', page)); + writeFileSync(join(caseDir, 'meta.json'), `${JSON.stringify({ agent: agentId, agentLabel: agent.label, @@ -151,6 +178,7 @@ function runOne({ agentId, agent, mode, scenario, prompt, model, runDir, dryRun finishedAt, exitCode: result.status, producedReadme: produced, + writtenPages: pages, }, null, 2)}\n`); writeFileSync(join(caseDir, 'agent-output.txt'), `${result.stdout ?? ''}\n${result.stderr ?? ''}`); diff --git a/tests/evals/run.test.mjs b/tests/evals/run.test.mjs new file mode 100644 index 0000000..ba7ecf6 --- /dev/null +++ b/tests/evals/run.test.mjs @@ -0,0 +1,43 @@ +// Tests for the parts of the runner that need no agent: which files a run keeps. +// Run with: node --test tests/evals/*.test.mjs + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { writtenPages } from './run.mjs'; + +function tree(root, files) { + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), content); + } +} + +test('a run keeps the pages the agent wrote or changed, and nothing else', () => { + const root = mkdtempSync(join(tmpdir(), 'written-pages-run-')); + const fixture = join(root, 'fixture'); + const workspace = join(root, 'workspace'); + + try { + tree(fixture, { + 'docs/kept.md': 'unchanged', + 'docs/edited.md': 'before', + 'src/index.js': 'code', + }); + tree(workspace, { + 'README.md': 'the generated README', + 'docs/kept.md': 'unchanged', + 'docs/edited.md': 'after', + 'docs/skill-catalog.md': 'a moved section', + 'src/index.js': 'code', + '.claude/skills/github-readme-generator/SKILL.md': 'the installed skill', + 'node_modules/pkg/README.md': 'a dependency', + }); + + assert.deepEqual(writtenPages(workspace, fixture), [join('docs', 'edited.md'), join('docs', 'skill-catalog.md')]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/evals/score.mjs b/tests/evals/score.mjs index eecf3ba..71d838b 100644 --- a/tests/evals/score.mjs +++ b/tests/evals/score.mjs @@ -1,8 +1,8 @@ // Deterministic scorer: one generated README, one fixture ground truth, one findings document. // No network, no model, no randomness — the same inputs always produce the same findings. -import { readFileSync, existsSync } from 'node:fs'; -import { join, resolve, normalize } from 'node:path'; +import { readFileSync, existsSync, readdirSync } from 'node:fs'; +import { join, resolve, normalize, dirname, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; const SHELL_FENCES = new Set(['', 'text', 'sh', 'bash', 'zsh', 'shell', 'console', 'shell-session']); @@ -101,7 +101,8 @@ function isExternal(target) { return /^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith('//'); } -function resolvesInsideRepo(repoDir, target) { +/** True when the target, read from `fromDir`, names a file inside one of the roots. */ +function resolvesInside(roots, target, fromDir = '') { const raw = target.split('#')[0].split('?')[0]; let path; @@ -113,16 +114,31 @@ function resolvesInsideRepo(repoDir, target) { } if (path === '') return true; - const absolute = resolve(repoDir, normalize(path)); - if (!absolute.startsWith(resolve(repoDir))) return false; - return existsSync(absolute); + return roots.some((root) => { + const absolute = resolve(root, normalize(join(fromDir, path))); + return absolute.startsWith(resolve(root)) && existsSync(absolute); + }); +} + +/** Markdown pages the run wrote besides the README, as { path, markdown } relative to `pagesDir`. */ +export function readPages(pagesDir, directory = pagesDir) { + if (!pagesDir || !existsSync(directory)) return []; + const pages = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) pages.push(...readPages(pagesDir, path)); + else if (entry.name.endsWith('.md')) pages.push({ path: relative(pagesDir, path), markdown: readFileSync(path, 'utf8') }); + } + return pages.sort((a, b) => a.path.localeCompare(b.path)); } /** * Score one README against one ground truth. + * `pages` are the Markdown pages the run wrote besides the README, read from `pagesDir`: + * links may point at them, and their own links are checked from their own directory. * Returns findings plus the counts the benchmark table reports. */ -export function score(markdown, truth, repoDir) { +export function score(markdown, truth, repoDir, { pagesDir = null, pages = [] } = {}) { const findings = []; const add = (kind, detail, evidence) => findings.push({ kind, detail, evidence }); @@ -140,17 +156,30 @@ export function score(markdown, truth, repoDir) { } } - const links = extractLinks(markdown); - for (const link of links) { - if (link.startsWith('#')) continue; + // A page the run wrote is a valid link target, alongside the fixture repository. + const roots = pagesDir ? [repoDir, pagesDir] : [repoDir]; + const checkLink = (link, fromDir, evidence) => { + if (link.startsWith('#')) return; if (isExternal(link)) { if (!matchesAny(link, truth.links?.allowedExternal ?? [], truth.links?.allowedExternalPatterns ?? [], 'i')) { - add('brokenLink', 'External link is not one the repository points at.', link); + add('brokenLink', 'External link is not one the repository points at.', evidence); } - continue; + return; } - if (!resolvesInsideRepo(repoDir, link)) { - add('brokenLink', 'Relative link has no target in the repository.', link); + if (!resolvesInside(roots, link, fromDir)) { + add('brokenLink', 'Relative link has no target in the repository.', evidence); + } + }; + + const links = extractLinks(markdown); + for (const link of links) checkLink(link, '', link); + + // Links inside a written page are read from that page's own directory. + const pageLinks = []; + for (const page of pages) { + for (const link of extractLinks(page.markdown)) { + pageLinks.push(link); + checkLink(link, dirname(page.path), `${page.path}: ${link}`); } } @@ -185,7 +214,7 @@ export function score(markdown, truth, repoDir) { }; for (const finding of unique) counts[finding.kind] += 1; - const checked = commands.length + images.length + links.length + const checked = commands.length + images.length + links.length + pageLinks.length + (truth.forbidden?.length ?? 0) + (truth.required?.length ?? 0); const errors = unique.length; @@ -206,7 +235,9 @@ export function loadTruth(fixtureDir) { export function scoreFile(readmePath, fixtureDir) { const truth = loadTruth(fixtureDir); - return score(readFileSync(readmePath, 'utf8'), truth, join(fixtureDir, 'repo')); + const pagesDir = join(dirname(readmePath), 'pages'); + const written = existsSync(pagesDir) ? { pagesDir, pages: readPages(pagesDir) } : {}; + return score(readFileSync(readmePath, 'utf8'), truth, join(fixtureDir, 'repo'), written); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/tests/evals/score.test.mjs b/tests/evals/score.test.mjs index bc056c6..1902db5 100644 --- a/tests/evals/score.test.mjs +++ b/tests/evals/score.test.mjs @@ -6,10 +6,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { readdirSync } from 'node:fs'; +import { readdirSync, mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { score, loadTruth, extractCommands, extractImages, extractLinks } from './score.mjs'; +import { score, loadTruth, readPages, extractCommands, extractImages, extractLinks } from './score.mjs'; const EVALS_DIR = dirname(fileURLToPath(import.meta.url)); const FIXTURES_DIR = join(EVALS_DIR, 'fixtures'); @@ -279,3 +280,53 @@ test('every fixture has a ground truth the scorer can load', () => { assert.equal(empty.errors, empty.counts.missingInformation, `${scenario} reports a defect in an empty README`); } }); + +/** A throwaway directory holding the pages a run wrote, removed after the test. */ +function withPages(files, check) { + const pagesDir = mkdtempSync(join(tmpdir(), 'written-pages-')); + try { + for (const [path, markdown] of Object.entries(files)) { + mkdirSync(dirname(join(pagesDir, path)), { recursive: true }); + writeFileSync(join(pagesDir, path), markdown); + } + check(pagesDir, readPages(pagesDir)); + } finally { + rmSync(pagesDir, { recursive: true, force: true }); + } +} + +const brokenLinks = (result) => result.findings + .filter((finding) => finding.kind === 'brokenLink') + .map((finding) => finding.evidence) + .sort(); + +test('a page the run wrote is a link target, and its own links are read from its directory', () => { + const scenario = 'small-library'; + const truth = loadTruth(join(FIXTURES_DIR, scenario)); + const repo = join(FIXTURES_DIR, scenario, 'repo'); + const page = ['# Catalog', '', '[Mean](../src/mean.js)', '[Missing](../nowhere.md)', '[Self](#catalog)'].join('\n'); + + withPages({ 'docs/catalog.md': page }, (pagesDir, pages) => { + assert.deepEqual(pages.map((entry) => entry.path), [join('docs', 'catalog.md')]); + + const readme = ['See the [catalog](docs/catalog.md).', 'And [a page nobody wrote](docs/gone.md).'].join('\n'); + const result = score(readme, truth, repo, { pagesDir, pages }); + + assert.deepEqual(brokenLinks(result), ['docs/catalog.md: ../nowhere.md', 'docs/gone.md']); + }); +}); + +test('without written pages, a link to an unwritten page stays broken', () => { + const result = judge('small-library', 'See the [catalog](docs/catalog.md).'); + assert.deepEqual(brokenLinks(result), ['docs/catalog.md']); +}); + +test('a link cannot escape the written pages directory', () => { + const scenario = 'small-library'; + const truth = loadTruth(join(FIXTURES_DIR, scenario)); + + withPages({}, (pagesDir, pages) => { + const result = score('[Outside](../../etc/hosts)', truth, join(FIXTURES_DIR, scenario, 'repo'), { pagesDir, pages }); + assert.deepEqual(brokenLinks(result), ['../../etc/hosts']); + }); +}); From 51ea64d125d53fd16366184c5ccf3a22bbb04066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Thu, 12 Nov 2026 14:02:26 +0100 Subject: [PATCH 2/2] Record the first run of scenario 11 under Claude Code --- tests/scenarios.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/scenarios.md b/tests/scenarios.md index e221361..8011e96 100644 --- a/tests/scenarios.md +++ b/tests/scenarios.md @@ -10,6 +10,11 @@ tested against. Test two things separately: Compare each run against the same request without the skill. Judge factual correctness, structure, and the absence of hallucination — not exact wording. +A clean session means only the copy under test is loaded. A copy installed in +`~/.claude/skills/` wins over the one in the workspace's `.claude/skills/`, so +copy the skill into the workspace and run with `--setting-sources project` — +otherwise the run tests whichever version happens to be installed. + ## 1. PHP library with external documentation *Setup:* a package with a banner asset, a `composer.json`, a `LICENSE.md`, a @@ -165,4 +170,4 @@ means the scenario has not been run since this log was started. | 8 | | | | 9 | | | | 10 | | | -| 11 | | | +| 11 | 2026-11-12, Claude Code 2.1.322, `claude -p "Update the README for this repository." --permission-mode acceptEdits --setting-sources project` | Pass. The 30-skill section moved to `docs/skills.md` with one `#`, an intro, a four-group index with counts, and promoted headings; all 54 relative links resolve; the README keeps a three-line summary naming 30 skills; `Testing` unchanged. The 20-rule list stayed in the README because `docs/rules.md` exists, and the collision was named in the handover. A second run changed nothing. |