From 6ec72c6f53867350f3990526bc4c47201a351575 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:24:03 -0700 Subject: [PATCH] fix(eval): verify Markdown arrow citations --- evals/product-journeys/graders/pj-s-question.mjs | 2 +- scripts/eval-journeys-graders.test.mjs | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/evals/product-journeys/graders/pj-s-question.mjs b/evals/product-journeys/graders/pj-s-question.mjs index 06da93c..5ce590f 100644 --- a/evals/product-journeys/graders/pj-s-question.mjs +++ b/evals/product-journeys/graders/pj-s-question.mjs @@ -89,7 +89,7 @@ function markdownCitations(transcript) { try { const outcome = JSON.parse(raw); return typeof outcome.summary === 'string' ? outcome.summary : line; } catch { return line; } }); - const rows = /`([A-Za-z0-9_.-]+\.[A-Za-z0-9]+):(\d+)`[ \t]*(?:—|–|-|:)[ \t]*`([^`\n]+)`/g; + const rows = /`([A-Za-z0-9_.-]+\.[A-Za-z0-9]+):(\d+)`[ \t]*(?:—|–|→|->|-|:)[ \t]*`([^`\n]+)`/g; return [...transcript.matchAll(rows)].map(m => ({ file: m[1], line: Number(m[2]), str: m[3] })); } diff --git a/scripts/eval-journeys-graders.test.mjs b/scripts/eval-journeys-graders.test.mjs index b72ac16..fd32d3b 100644 --- a/scripts/eval-journeys-graders.test.mjs +++ b/scripts/eval-journeys-graders.test.mjs @@ -86,6 +86,16 @@ test('pj-s-question: Markdown file-line citations retain exact content verificat assert.equal((await w.grade(w.transcript(good))).pass, true); const html = good.replace('`Furniture made on the coast`', '`

Furniture made on the coast

`'); assert.equal((await w.grade(w.transcript(html + '\nOutcome: ' + JSON.stringify({ summary: html })))).pass, true); + for (const separator of ['→', '->']) { + const arrows = html.replaceAll(' — ', ` ${separator} `); + assert.equal((await w.grade(w.transcript(arrows))).pass, true); + assert.equal((await w.grade(w.transcript('Outcome: ' + JSON.stringify({ summary: arrows })))).pass, true); + for (const bad of [arrows.replace('index.html:16', 'index.html:600'), + arrows.replace('index.html:16', 'missing.html:16'), + arrows.replace('Furniture made on the coast', 'Invented tagline')]) { + assert.equal((await w.grade(w.transcript(bad))).pass, false); + } + } for (const bad of [good.replace('index.html:6', 'index.html:600'), good.replace('about.html:20', 'missing.html:20'), good.replace('Furniture made on the coast', 'Invented tagline')]) {