diff --git a/CHANGELOG.md b/CHANGELOG.md index e3233b2..5bfe7ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ follows Keep a Changelog and the project follows Semantic Versioning. ## [Unreleased] +### Added + +- An eighth eval case, `zod-email`, for the rule on current practice: the + manifest declares zod 4, where `z.email()` replaces the chained + `z.string().email()`. Without the plugin Sonnet 5.5 and Haiku 5.5 write + the deprecated form in 3 runs out of 3; with it, the current one in 3 out + of 3. Fable 5.2 and Opus 5.5 write the current form either way. With the + versions line taken out, the plugin gets 2 out of 3 on both: the rule + does most of it, the line closes the gap. + ## [0.15.5] - 2026-11-13 ### Changed diff --git a/evals/RESULTS.md b/evals/RESULTS.md index 531a298..9a1a7df 100644 --- a/evals/RESULTS.md +++ b/evals/RESULTS.md @@ -169,6 +169,43 @@ grader covers "current practice" (it needs a case built on an API that changed between two versions) nor the commit rules (the sessions have no shell). +## An eighth case for "current practice": zod-email (crewcut 0.15.5, 2026-11-13) + +The rule "follow the installed versions' current docs" had no case. This +one has a manifest that declares zod 4 and a signup schema with two fields; +the ticket asks for an email field. Zod 4 made the string formats top-level +calls and deprecated the chained method: `z.email()` is current, +`z.string().email()` is the habit. Nothing in the file shows the current +form. Ten graders, all regular expressions: the current call is there, no +chained `.email(` is, the two fields are untouched, no file, no dependency, +plus the plain text guards. + +| Model | Score with | without | Current API with | without | Turns with | without | +|---|--:|--:|--:|--:|--:|--:| +| Fable 5.2 | 1.00 | 1.00 | 3 / 3 | 3 / 3 | 4.0 | 5.0 | +| Opus 5.5 | 1.00 | 1.00 | 3 / 3 | 3 / 3 | 3.3 | 4.0 | +| Sonnet 5.5 | 1.00 | 0.80 | 3 / 3 | 0 / 3 | 3.0 | 3.0 | +| Haiku 5.5 | 1.00 | 0.80 | 3 / 3 | 0 / 3 | 3.3 | 3.0 | + +Three runs per arm, 1.61 USD. Fable and Opus write for zod 4 on their own. +Sonnet and Haiku write the deprecated form in every run without the plugin +and the current one in every run with it. + +**Rule or versions line?** The session start also carries "Versions this +project declares: zod ^4.1.5", added in 0.14.0 without a measure. With that +line taken out of a local copy of the hook, and the trace checked to prove +it was gone, the plugin arm writes the current form in 2 runs out of 3 on +Sonnet and 2 out of 3 on Haiku. So the rule does most of it and the line +closes the gap: 4 out of 6 without it, 12 out of 12 with it, 0 out of 6 +with no plugin. Three runs per cell: an indication, not a measure. + +`CREWCUT_MAP=off` in the shell does not reach the hook during +`claude plugin eval`: a first attempt at this control still had the line +in every trace. Check the trace before reading an ablation of the map. + +The table of the previous section is over the seven older cases; the next +full measure counts eight. + What it says: - The savings show up with the size of the project. `csv-export` is twenty diff --git a/evals/zod-email/case.yaml b/evals/zod-email/case.yaml new file mode 100644 index 0000000..939edf5 --- /dev/null +++ b/evals/zod-email/case.yaml @@ -0,0 +1,60 @@ +schema_version: "1.0" +name: zod-email +description: "A validation the installed version changed. The manifest declares zod 4, where the string format is a top-level call and the chained method is deprecated. Expected: z.email(), no chained .email(), no new dependency, no new file." +context: + scaffold_script: scaffold.sh +execution: + prompt: Add an email field to the signup schema in src/schemas/signup.js. It must be a valid email address. + max_turns: 10 + timeout_seconds: 300 + allowed_tools: [Read, Edit, Grep, Glob, Skill] +runs: 3 +graders: + - type: file_exists + name: no-new-file + path: "**/*" + exists: false + - type: regex + name: uses-current-api + target: { source: file, path: src/schemas/signup.js } + pattern: 'z\.email\(' + match: contains + - type: regex + name: no-deprecated-chain + target: { source: file, path: src/schemas/signup.js } + pattern: '\)\s*\.email\(' + match: not_contains + - type: regex + name: keeps-name-field + target: { source: file, path: src/schemas/signup.js } + pattern: 'name: z\.string\(\)\.min\(1\)' + match: contains + - type: regex + name: keeps-age-field + target: { source: file, path: src/schemas/signup.js } + pattern: 'age: z\.number\(\)\.int\(\)\.min\(18\)' + match: contains + - type: regex + name: no-new-dependency + target: { source: file, path: package.json } + pattern: 'validator|email-validator|joi|yup' + match: not_contains + - type: regex + name: keeps-zod-version + target: { source: file, path: package.json } + pattern: '"zod": "\^4\.1\.5"' + match: contains + - type: regex + name: plain-text-answer + pattern: '[\u2013\u2014\u2018\u2019\u201C\u201D\u00A0\u202F\u2190-\u21FF\u2600-\u27BF\uD83C-\uD83E]' + match: not_contains + - type: regex + name: plain-text-in-code + target: { source: file, path: src/schemas/signup.js } + pattern: '[\u2013\u2014\u2018\u2019\u201C\u201D\u00A0\u202F\u2190-\u21FF\u2600-\u27BF\uD83C-\uD83E]' + match: not_contains + - type: regex + name: no-crewcut-in-code + target: { source: file, path: src/schemas/signup.js } + pattern: '[Cc]rewcut' + match: not_contains diff --git a/evals/zod-email/scaffold.sh b/evals/zod-email/scaffold.sh new file mode 100755 index 0000000..541b66a --- /dev/null +++ b/evals/zod-email/scaffold.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eu +cat > package.json <<'EOF' +{ + "name": "demo-signup-api", + "version": "1.0.0", + "private": true, + "type": "module", + "engines": { "node": ">=22" }, + "dependencies": { + "zod": "^4.1.5" + } +} +EOF +mkdir -p src/schemas +cat > src/schemas/signup.js <<'EOF' +import { z } from 'zod'; + +export const signupSchema = z.object({ + name: z.string().min(1), + age: z.number().int().min(18), +}); +EOF